http: correct progress when downloading gzipped content appcode/163.2135 clion/163.2133 dbe/163.2134 idea/163.2132 idea/163.2136 phpstorm/163.2137 pycharm/163.2139 pycharm/163.2143 rubymine/163.2142 webstorm/163.2141 webstorm/163.2144
authorSergey Simonchik <sergey.simonchik@jetbrains.com>
Mon, 1 Aug 2016 23:49:26 +0000 (02:49 +0300)
committerSergey Simonchik <sergey.simonchik@jetbrains.com>
Mon, 1 Aug 2016 23:49:26 +0000 (02:49 +0300)
platform/platform-api/src/com/intellij/util/io/CompressedBytesReadAwareGZIPInputStream.java [new file with mode: 0644]
platform/platform-api/src/com/intellij/util/io/HttpRequests.java
platform/platform-api/src/com/intellij/util/net/NetUtils.java

diff --git a/platform/platform-api/src/com/intellij/util/io/CompressedBytesReadAwareGZIPInputStream.java b/platform/platform-api/src/com/intellij/util/io/CompressedBytesReadAwareGZIPInputStream.java
new file mode 100644 (file)
index 0000000..6c1a487
--- /dev/null
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.io;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.zip.GZIPInputStream;
+
+public class CompressedBytesReadAwareGZIPInputStream extends GZIPInputStream {
+  private final BytesReadAwareInputStream myInputStream;
+
+  private CompressedBytesReadAwareGZIPInputStream(@NotNull BytesReadAwareInputStream inputStream) throws IOException {
+    super(inputStream);
+    myInputStream = inputStream;
+  }
+
+  public long getCompressedBytesRead() {
+    return myInputStream.myBytesRead.get();
+  }
+
+  @NotNull
+  public static CompressedBytesReadAwareGZIPInputStream create(@NotNull InputStream inputStream) throws IOException {
+    return new CompressedBytesReadAwareGZIPInputStream(new BytesReadAwareInputStream(inputStream));
+  }
+
+  private static class BytesReadAwareInputStream extends InputStream {
+    private final InputStream myInputStream;
+    private final AtomicLong myBytesRead = new AtomicLong(0);
+
+    public BytesReadAwareInputStream(@NotNull InputStream inputStream) {
+      myInputStream = inputStream;
+    }
+
+    public int read() throws IOException {
+      long bytesReadBefore = myBytesRead.get();
+      int data = myInputStream.read();
+      myBytesRead.compareAndSet(bytesReadBefore, bytesReadBefore + 1);
+      return data;
+    }
+
+    @Override
+    public int read(@NotNull byte[] b) throws IOException {
+      long bytesReadBefore = myBytesRead.get();
+      int bytesRead = myInputStream.read(b);
+      myBytesRead.compareAndSet(bytesReadBefore, bytesReadBefore + bytesRead);
+      return bytesRead;
+    }
+
+    @Override
+    public int read(@NotNull byte[] b, int off, int len) throws IOException {
+      long bytesReadBefore = myBytesRead.get();
+      int bytesRead = myInputStream.read(b, off, len);
+      myBytesRead.compareAndSet(bytesReadBefore, bytesReadBefore + bytesRead);
+      return bytesRead;
+    }
+
+    public long skip(long n) throws IOException {
+      long bytesReadBefore = myBytesRead.get();
+      long bytesSkipped = myInputStream.skip(n);
+      myBytesRead.compareAndSet(bytesReadBefore, bytesReadBefore + bytesSkipped);
+      return bytesSkipped;
+    }
+
+    public int available() throws IOException {
+      return myInputStream.available();
+    }
+
+    public void close() throws IOException {
+      myInputStream.close();
+    }
+
+    @Override
+    public synchronized void mark(int readlimit) {
+      myInputStream.mark(readlimit);
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+      myInputStream.reset();
+    }
+
+    @Override
+    public boolean markSupported() {
+      return myInputStream.markSupported();
+    }
+  }
+}
index acbdc552ab7cdf69e7778c59bb5a8a8aa5b14a0c..3db597dcec5ddad25581e9d7a524fe2f88e42dc1 100644 (file)
@@ -42,7 +42,6 @@ import java.net.*;
 import java.nio.charset.Charset;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.zip.GZIPInputStream;
 
 /**
  * Handy class for reading data from HTTP connections with built-in support for HTTP redirects and gzipped content and automatic cleanup.
@@ -280,7 +279,7 @@ public final class HttpRequests {
         myInputStream = getConnection().getInputStream();
         if (myBuilder.myGzip && "gzip".equalsIgnoreCase(getConnection().getContentEncoding())) {
           //noinspection IOResourceOpenedButNotSafelyClosed
-          myInputStream = new GZIPInputStream(myInputStream);
+          myInputStream = CompressedBytesReadAwareGZIPInputStream.create(myInputStream);
         }
       }
       return myInputStream;
index f62c0f750d91ec02e52fbf175900d481e819b55e..1c02684465ac6041406e2ba139552fd56aa66bcc 100644 (file)
@@ -19,7 +19,9 @@ import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.progress.ProcessCanceledException;
 import com.intellij.openapi.progress.ProgressIndicator;
 import com.intellij.openapi.util.SystemInfo;
+import com.intellij.util.ObjectUtils;
 import com.intellij.util.SystemProperties;
+import com.intellij.util.io.CompressedBytesReadAwareGZIPInputStream;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -166,37 +168,40 @@ public class NetUtils {
   }
 
   /**
-   * @param indicator           Progress indicator.
-   * @param inputStream         source stream
-   * @param outputStream        destination stream
-   * @param expectedContentSize expected content size, used in progress indicator (negative means unknown length)
-   * @return bytes copied
+   * @param indicator             progress indicator
+   * @param inputStream           source stream
+   * @param outputStream          destination stream
+   * @param expectedContentLength expected content length in bytes, used in progress indicator (negative means unknown length).
+   *                              For gzipped content, it's an expected length of gzipped/compressed content.
+   *                              E.g. for HTTP, it means how many bytes should be sent over the network.
+   * @return the total number of bytes written to the destination stream (may exceed expectedContentLength for gzipped content)
    * @throws IOException              if IO error occur
    * @throws ProcessCanceledException if process was canceled.
    */
   public static int copyStreamContent(@Nullable ProgressIndicator indicator,
                                       @NotNull InputStream inputStream,
                                       @NotNull OutputStream outputStream,
-                                      int expectedContentSize) throws IOException, ProcessCanceledException {
+                                      int expectedContentLength) throws IOException, ProcessCanceledException {
     if (indicator != null) {
       indicator.checkCanceled();
-      if (expectedContentSize < 0) {
+      if (expectedContentLength < 0) {
         indicator.setIndeterminate(true);
       }
     }
-
+    CompressedBytesReadAwareGZIPInputStream gzipStream = ObjectUtils.tryCast(inputStream, CompressedBytesReadAwareGZIPInputStream.class);
     final byte[] buffer = new byte[8 * 1024];
     int count;
-    int total = 0;
+    int bytesWritten = 0;
+    long bytesRead = 0;
     while ((count = inputStream.read(buffer)) > 0) {
       outputStream.write(buffer, 0, count);
-      total += count;
+      bytesWritten += count;
+      bytesRead = gzipStream != null ? gzipStream.getCompressedBytesRead() : bytesWritten;
 
       if (indicator != null) {
         indicator.checkCanceled();
-
-        if (expectedContentSize > 0) {
-          indicator.setFraction((double)total / expectedContentSize);
+        if (expectedContentLength > 0) {
+          indicator.setFraction((double)bytesRead / expectedContentLength);
         }
       }
     }
@@ -205,11 +210,11 @@ public class NetUtils {
       indicator.checkCanceled();
     }
 
-    if (total < expectedContentSize) {
-      throw new IOException(String.format("Connection closed at byte %d. Expected %d bytes.", total, expectedContentSize));
+    if (bytesRead < expectedContentLength) {
+      throw new IOException(String.format("Connection closed at byte %d. Expected %d bytes.", bytesRead, expectedContentLength));
     }
 
-    return total;
+    return bytesWritten;
   }
 
   public static boolean isSniEnabled() {