Merge pull request facebook#7 from jasta/gzip-handling

Add Content-Encoding support to stetho-urlconnection and stetho-okhttp

README.md

@@ -97,8 +97,9 @@ client.networkInterceptors().add(new StethoInterceptor());
 
 If you are using `HttpURLConnection`, you can use `StethoURLConnectionManager`
 to assist with integration though you should be aware that there are some
-caveats with this approach.  In particular, compressed payload sizes may not be
-visualized even though compression is indeed in effect.
+caveats with this approach.  In particular, you must explicitly add
+`Accept-Encoding: gzip` to the request headers and manually handle compressed
+responses in order for Stetho to report compressed payload sizes.
 
 See the `stetho-sample` project for more details.
 

stetho-okhttp/build.gradle

@@ -48,4 +48,6 @@ dependencies {
     // Needed for Robolectric and PowerMock to be combined in a single test.
     androidTestCompile 'org.powermock:powermock-module-junit4-rule:1.6.1'
     androidTestCompile 'org.powermock:powermock-classloading-xstream:1.6.1'
+
+    androidTestCompile 'com.squareup.okhttp:mockwebserver:2.2.0'
 }

stetho-okhttp/src/main/java/com/facebook/stetho/okhttp/StethoInterceptor.java

@@ -1,27 +1,19 @@
 package com.facebook.stetho.okhttp;
 
-import javax.annotation.Nullable;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.concurrent.atomic.AtomicInteger;
-
 import com.facebook.stetho.inspector.network.DefaultResponseHandler;
 import com.facebook.stetho.inspector.network.NetworkEventReporter;
 import com.facebook.stetho.inspector.network.NetworkEventReporterImpl;
-
-import com.squareup.okhttp.Connection;
-import com.squareup.okhttp.Interceptor;
-import com.squareup.okhttp.MediaType;
-import com.squareup.okhttp.Request;
-import com.squareup.okhttp.RequestBody;
-import com.squareup.okhttp.Response;
-import com.squareup.okhttp.ResponseBody;
+import com.squareup.okhttp.*;
 import okio.BufferedSink;
 import okio.BufferedSource;
 import okio.Okio;
 
+import javax.annotation.Nullable;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.atomic.AtomicInteger;
+
 /**
  * Provides easy integration with <a href="http://square.github.io/okhttp/">OkHttp</a> 2.2.0+
  * by way of the new <a href="https://github.com/square/okhttp/wiki/Interceptors">Interceptor</a>
@@ -86,6 +78,7 @@ public Response intercept(Chain chain) throws IOException {
       responseStream = mEventReporter.interpretResponseStream(
           requestId,
           contentType != null ? contentType.toString() : null,
+          response.header("Content-Encoding"),
           responseStream,
           new DefaultResponseHandler(mEventReporter, requestId));
       if (responseStream != null) {

stetho-okhttp/src/test/java/com/facebook/stetho/okhttp/StethoInterceptorTest.java

@@ -2,8 +2,13 @@
 
 import android.net.Uri;
 import android.os.Build;
-import com.facebook.stetho.inspector.network.*;
+import com.facebook.stetho.inspector.network.DecompressionHelper;
+import com.facebook.stetho.inspector.network.NetworkEventReporter;
+import com.facebook.stetho.inspector.network.NetworkEventReporterImpl;
+import com.facebook.stetho.inspector.network.ResponseHandler;
 import com.squareup.okhttp.*;
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import com.squareup.okhttp.mockwebserver.MockWebServer;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -14,7 +19,6 @@
 import org.powermock.api.mockito.PowerMockito;
 import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
 import org.powermock.modules.junit4.rule.PowerMockRule;
 import org.robolectric.RobolectricTestRunner;
 import org.robolectric.annotation.Config;
@@ -23,26 +27,28 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.zip.GZIPOutputStream;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.mockito.Matchers.*;
 
 @Config(emulateSdk = Build.VERSION_CODES.JELLY_BEAN)
 @RunWith(RobolectricTestRunner.class)
-@PowerMockIgnore({ "org.mockito.*", "org.robolectric.*", "android.*" })
+@PowerMockIgnore({ "org.mockito.*", "org.robolectric.*", "android.*", "javax.net.ssl.*" })
 @PrepareForTest(NetworkEventReporterImpl.class)
 public class StethoInterceptorTest {
   @Rule
   public PowerMockRule rule = new PowerMockRule();
 
   @Test
-  public void testManualChainHappyPath() throws IOException {
+  public void testHappyPath() throws IOException {
     PowerMockito.mockStatic(NetworkEventReporterImpl.class);
 
     final NetworkEventReporter mockEventReporter = Mockito.mock(NetworkEventReporter.class);
     InOrder inOrder = Mockito.inOrder(mockEventReporter);
     Mockito.when(mockEventReporter.isEnabled()).thenReturn(true);
-    ByteArrayOutputStream capturedOutput = fakeInterpretResponseStream(mockEventReporter);
+    ByteArrayOutputStream capturedOutput = hookAlmostRealInterpretResponseStream(mockEventReporter);
     PowerMockito.when(NetworkEventReporterImpl.get()).thenReturn(mockEventReporter);
 
     StethoInterceptor interceptor = new StethoInterceptor();
@@ -84,16 +90,73 @@ public void testManualChainHappyPath() throws IOException {
     inOrder.verifyNoMoreInteractions();
   }
 
+  @Test
+  public void testWithCompression() throws IOException {
+    PowerMockito.mockStatic(NetworkEventReporterImpl.class);
+
+    final NetworkEventReporter mockEventReporter = Mockito.mock(NetworkEventReporter.class);
+    Mockito.when(mockEventReporter.isEnabled()).thenReturn(true);
+    ByteArrayOutputStream capturedOutput = hookAlmostRealInterpretResponseStream(mockEventReporter);
+    PowerMockito.when(NetworkEventReporterImpl.get()).thenReturn(mockEventReporter);
+
+    byte[] uncompressedData = repeat(".", 1024).getBytes();
+    byte[] compressedData = compress(uncompressedData);
+
+    MockWebServer server = new MockWebServer();
+    server.play();
+    server.enqueue(new MockResponse()
+        .setBody(compressedData)
+        .addHeader("Content-Encoding: gzip"));
+
+    OkHttpClient client = new OkHttpClient();
+    client.networkInterceptors().add(new StethoInterceptor());
+
+    Request request = new Request.Builder()
+        .url(server.getUrl("/"))
+        .build();
+    Response response = client.newCall(request).execute();
+
+    // Verify that the final output and the caller both saw the uncompressed stream.
+    assertArrayEquals(uncompressedData, response.body().bytes());
+    assertArrayEquals(uncompressedData, capturedOutput.toByteArray());
+
+    // And verify that the StethoInterceptor was able to see both.
+    Mockito.verify(mockEventReporter)
+        .dataReceived(
+            anyString(),
+            eq(compressedData.length),
+            eq(uncompressedData.length));
+
+    server.shutdown();
+  }
+
+  private static String repeat(String s, int reps) {
+    StringBuilder b = new StringBuilder(s.length() * reps);
+    while (reps-- > 0) {
+      b.append(s);
+    }
+    return b.toString();
+  }
+
+  private static byte[] compress(byte[] data) throws IOException {
+    ByteArrayOutputStream buf = new ByteArrayOutputStream();
+    GZIPOutputStream out = new GZIPOutputStream(buf);
+    out.write(data);
+    out.close();
+    return buf.toByteArray();
+  }
+
   /**
    * Provide a suitably "real" implementation of
    * {@link NetworkEventReporter#interpretResponseStream} for our mock to test that
    * events are properly delegated.
    */
-  private static ByteArrayOutputStream fakeInterpretResponseStream(
+  private static ByteArrayOutputStream hookAlmostRealInterpretResponseStream(
       final NetworkEventReporter mockEventReporter) {
     final ByteArrayOutputStream capturedOutput = new ByteArrayOutputStream();
     Mockito.when(
         mockEventReporter.interpretResponseStream(
+            anyString(),
             anyString(),
             anyString(),
             any(InputStream.class),
@@ -104,13 +167,16 @@ private static ByteArrayOutputStream fakeInterpretResponseStream(
               public InputStream answer(InvocationOnMock invocationOnMock) throws Throwable {
                 Object[] args = invocationOnMock.getArguments();
                 String requestId = (String)args[0];
-                InputStream responseStream = (InputStream)args[2];
-                return new ResponseHandlingInputStream(
-                    responseStream,
+                String contentEncoding = (String)args[2];
+                InputStream responseStream = (InputStream)args[3];
+                ResponseHandler responseHandler = (ResponseHandler)args[4];
+                return DecompressionHelper.teeInputWithDecompression(
+                    null /* networkPeerManager */,
                     requestId,
+                    responseStream,
                     capturedOutput,
-                    null /* networkPeerManager */,
-                    new DefaultResponseHandler(mockEventReporter, requestId));
+                    contentEncoding,
+                    responseHandler);
               }
             });
     return capturedOutput;

stetho-sample/src/main/java/com/facebook/stetho/sample/Networker.java

@@ -6,6 +6,7 @@
 import com.facebook.stetho.urlconnection.SimpleRequestEntity;
 import com.facebook.stetho.urlconnection.StethoURLConnectionManager;
 
+import javax.annotation.Nullable;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -14,6 +15,7 @@
 import java.net.URL;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Executors;
+import java.util.zip.GZIPInputStream;
 
 /**
  * Very simple centralized network middleware for illustration purposes.
@@ -26,6 +28,9 @@ public class Networker {
   private static final int READ_TIMEOUT_MS = 10000;
   private static final int CONNECT_TIMEOUT_MS = 15000;
 
+  private static final String HEADER_ACCEPT_ENCODING = "Accept-Encoding";
+  private static final String GZIP_ENCODING = "gzip";
+
   public static synchronized Networker get() {
     if (sInstance == null) {
       sInstance = new Networker();
@@ -65,18 +70,17 @@ private HttpResponse doFetch() throws IOException {
       HttpURLConnection conn = configureAndConnectRequest();
       try {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
-        InputStream responseStream = conn.getInputStream();
+        InputStream rawStream = conn.getInputStream();
         try {
-          // Let Stetho see.
-          responseStream = stethoManager.interpretResponseStream(
-              responseStream,
-              null /* customResponseHandler */);
-          if (responseStream != null) {
-            copy(responseStream, out, new byte[1024]);
+          // Let Stetho see the raw, possibly compressed stream.
+          rawStream = stethoManager.interpretResponseStream(rawStream);
+          InputStream decompressedStream = applyDecompressionIfApplicable(conn, rawStream);
+          if (decompressedStream != null) {
+            copy(decompressedStream, out, new byte[1024]);
           }
         } finally {
-          if (responseStream != null) {
-            responseStream.close();
+          if (rawStream != null) {
+            rawStream.close();
           }
         }
         return new HttpResponse(conn.getResponseCode(), out.toByteArray());
@@ -98,6 +102,10 @@ private HttpURLConnection configureAndConnectRequest() throws IOException {
         conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
         conn.setRequestMethod(request.method.toString());
 
+        // Adding this disables transparent gzip compression so that we can intercept
+        // the raw stream and display the correct response body size.
+        requestDecompression(conn);
+
         SimpleRequestEntity requestEntity = null;
         if (request.body != null) {
           requestEntity = new ByteArrayRequestEntity(request.body);
@@ -134,6 +142,19 @@ private HttpURLConnection configureAndConnectRequest() throws IOException {
     }
   }
 
+  private static void requestDecompression(HttpURLConnection conn) {
+    conn.setRequestProperty(HEADER_ACCEPT_ENCODING, GZIP_ENCODING);
+  }
+
+  @Nullable
+  private static InputStream applyDecompressionIfApplicable(
+      HttpURLConnection conn, @Nullable InputStream in) throws IOException {
+    if (in != null && GZIP_ENCODING.equals(conn.getContentEncoding())) {
+      return new GZIPInputStream(in);
+    }
+    return in;
+  }
+
   private static void copy(InputStream in, OutputStream out, byte[] buf) throws IOException {
     if (in == null) {
       return;

stetho-urlconnection/src/main/java/com/facebook/stetho/urlconnection/StethoURLConnectionManager.java

@@ -3,7 +3,6 @@
 import com.facebook.stetho.inspector.network.DefaultResponseHandler;
 import com.facebook.stetho.inspector.network.NetworkEventReporter;
 import com.facebook.stetho.inspector.network.NetworkEventReporterImpl;
-import com.facebook.stetho.inspector.network.ResponseHandler;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -17,6 +16,15 @@
  * Individual connection flow manager that aids in communicating network events to Stetho
  * via the {@link NetworkEventReporter} API.  This class is stateful and should be instantiated
  * for each individual HTTP request.
+ * <p>
+ * Be aware that there are caveats with inspection using {@link HttpURLConnection} on Android:
+ * <ul>
+ * <li>Compressed payload sizes are typically not available, even when compression was in use over
+ * the wire.
+ * <li>Redirects are by default handled internally, making it impossible to visualize them.
+ * To visualize them, redirects must be handled manually by invoking
+ * {@link HttpURLConnection#setFollowRedirects(boolean)}.
+ * </ul>
  */
 @NotThreadSafe
 public class StethoURLConnectionManager {
@@ -104,34 +112,33 @@ public void httpExchangeFailed(IOException ex) {
 
   /**
    * Deliver the response stream from {@link HttpURLConnection#getInputStream()} to
-   * Stetho so that it can be intercepted.  Note that this should be
-   * the uncompressed stream if gzip encoded was used, so manually wrapping it in a
-   * {@link java.util.zip.GZIPInputStream} would be required.  If you do this, the
-   * raw encoded sizes will be incorrect by default which can be fixed by supplying
-   * your own custom {@link ResponseHandler}.
+   * Stetho so that it can be intercepted.  Note that compression is transparently
+   * supported on modern Android systems and no special awareness is necessary for
+   * gzip compression on the wire.  Unfortunately this means that it is sometimes impossible
+   * to determine whether compression actually occurred and so Stetho may report inflated
+   * byte counts.
+   * <p>
+   * If the {@code Content-Length} header is provided by the server, this will be assumed to be
+   * the raw byte count on the wire.
    *
-   * @param responseStream Stream as furnished by {@link HttpURLConnection#getInputStream()} or a
-   *     decompressing one if compression was used.
-   * @param customResponseHandler Custom response handler hook to allow callers to report
-   *     the compressed and uncompressed sizes if desired.  This is not required and can
-   *     be null.
+   * @param responseStream Stream as furnished by {@link HttpURLConnection#getInputStream()}.
    *
    * @return The filtering stream which is to be read after this method is called.
    */
-  public InputStream interpretResponseStream(
-      @Nullable InputStream responseStream,
-      @Nullable ResponseHandler customResponseHandler) {
+  public InputStream interpretResponseStream(@Nullable InputStream responseStream) {
     throwIfNoConnection();
     if (isStethoEnabled()) {
-      ResponseHandler responseHandler = customResponseHandler;
-      if (responseHandler == null) {
-        responseHandler = new DefaultResponseHandler(mStethoHook, getStethoRequestId());
-      }
+      // Note that Content-Encoding is stripped out by HttpURLConnection on modern versions of
+      // Android (fun fact, it's powered by okhttp) when decompression is handled transparently.
+      // When this occurs, we will not be able to report the compressed size properly.  Callers,
+      // however, can disable this behaviour which will once again give us access to the raw
+      // Content-Encoding so that we can handle it properly.
       responseStream = mStethoHook.interpretResponseStream(
           getStethoRequestId(),
           mConnection.getHeaderField("Content-Type"),
+          mConnection.getHeaderField("Content-Encoding"),
           responseStream,
-          responseHandler);
+          new DefaultResponseHandler(mStethoHook, getStethoRequestId()));
     }
     return responseStream;
   }