From 96f30cdc560c263f69b2e42b0f76b8aa17735d75 Mon Sep 17 00:00:00 2001 From: Eric Lawrence Date: Fri, 10 Apr 2020 14:47:55 -0500 Subject: [PATCH] Promote traffic_annotation value Improves #3 --- FiddlerImportNetlog/FiddlerInterface.cs | 8 ++-- FiddlerImportNetlog/Importer.cs | 37 +++++++++++++------ .../Properties/AssemblyInfo.cs | 8 +++- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/FiddlerImportNetlog/FiddlerInterface.cs b/FiddlerImportNetlog/FiddlerInterface.cs index 38baadb..0ba7ca3 100644 --- a/FiddlerImportNetlog/FiddlerInterface.cs +++ b/FiddlerImportNetlog/FiddlerInterface.cs @@ -8,7 +8,7 @@ namespace FiddlerImportNetlog { - [ProfferFormat("NetLog JSON", "Chromium's JSON-based event log format (v1.2.1). See https://dev.chromium.org/for-testers/providing-network-details for more details.")] + [ProfferFormat("NetLog JSON", "Chromium's JSON-based event log format (v1.2.3). See https://dev.chromium.org/for-testers/providing-network-details for more details.")] public class HTTPArchiveFormatImport : ISessionImporter { public Session[] ImportSessions(string sFormat, Dictionary dictOptions, EventHandler evtProgressNotifications) @@ -46,7 +46,7 @@ public Session[] ImportSessions(string sFormat, Dictionary dictO oSR = new StreamReader(strmContent); } else - { + { Stream oFS = File.OpenRead(sFilename); // Check to see if this file data was GZIP'd or PKZIP'd. @@ -108,7 +108,7 @@ private MemoryStream GetUnzippedBytes(Stream oFS) long fileLength = oFS.Length; if (fileLength > Int32.MaxValue) throw new IOException("file over 2gb"); - + int index = 0; int count = (int)fileLength; byte[] bytes = new byte[count]; @@ -125,4 +125,4 @@ private MemoryStream GetUnzippedBytes(Stream oFS) public void Dispose() { } } -} \ No newline at end of file +} diff --git a/FiddlerImportNetlog/Importer.cs b/FiddlerImportNetlog/Importer.cs index 586b56a..7266547 100644 --- a/FiddlerImportNetlog/Importer.cs +++ b/FiddlerImportNetlog/Importer.cs @@ -24,6 +24,7 @@ struct Magics public int SOCKET; // Events + public int REQUEST_ALIVE; public int URL_REQUEST_START_JOB; public int SEND_HEADERS; public int SEND_QUIC_HEADERS; @@ -72,7 +73,7 @@ private static ArrayList FilterExtensions(ArrayList al) } return alOut; } - + private static DateTime GetTimeStamp(object o, long baseTime) { long t = baseTime; @@ -115,7 +116,7 @@ internal NetlogImporter(StreamReader oSR, List listSessions, EventHandl FiddlerApplication.DoNotifyUser("This JSON file does not seem to contain NetLog data.", "Unexpected Data"); Session sessFile = Session.BuildFromData(false, new HTTPRequestHeaders( - String.Format("/file.json"), + String.Format("/file.json"), new[] { "Host: IMPORTED", "Date: " + DateTime.UtcNow.ToString() }), Utilities.emptyByteArray, new HTTPResponseHeaders(200, "File Data", new[] { "Content-Type: application/json; charset=utf-8" }), @@ -156,6 +157,7 @@ public bool ExtractSessionsFromJSON(Hashtable htFile) #region GetEventTypes // HTTP-level Events + NetLogMagics.REQUEST_ALIVE = getIntValue(htEventTypes["REQUEST_ALIVE"], -987); NetLogMagics.URL_REQUEST_START_JOB = getIntValue(htEventTypes["URL_REQUEST_START_JOB"], -997); NetLogMagics.SEND_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST_HEADERS"], -996); NetLogMagics.SEND_QUIC_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_QUIC_SEND_REQUEST_HEADERS"], -995); @@ -231,7 +233,7 @@ public bool ExtractSessionsFromJSON(Hashtable htFile) _listSessions.Add(Session.BuildFromData(false, new HTTPRequestHeaders( - String.Format("/Enabled_Extensions"), // TODO: Add Machine name? + String.Format("/ENABLED_EXTENSIONS"), // TODO: Add Machine name? new[] { "Host: NETLOG" }), Utilities.emptyByteArray, new HTTPResponseHeaders(200, "Analyzed Data", new[] { "Content-Type: application/json; charset=utf-8" }), @@ -610,7 +612,7 @@ private int GenerateSessionsFromURLRequests(Dictionary> dic int iRequest = 0; iLastPct = 75; - // Iterate over each URLRequest's events bucket and parse one or more Sessions out of it. + // Iterate over each URLRequest's events bucket and parse one or more Sessions out of it. foreach (KeyValuePair> kvpUR in dictURLRequests) { ++iRequest; @@ -640,6 +642,7 @@ private void ParseSessionsFromBucket(KeyValuePair> kvpUR) string sURL = String.Empty; string sMethod = "GET"; + string sTrafficAnnotation = String.Empty; SessionTimers oTimers = new SessionTimers(); int cbDroppedResponseBody = 0; @@ -657,6 +660,11 @@ private void ParseSessionsFromBucket(KeyValuePair> kvpUR) #region ParseImportantEvents // C# cannot |switch()| on non-constant case values. Hrmph. + if (iType == NetLogMagics.REQUEST_ALIVE) + { + sTrafficAnnotation = getIntValue(htParams["traffic_annotation"], 0).ToString(); + continue; + } if (iType == NetLogMagics.URL_REQUEST_START_JOB) { if (bHasStartJob) @@ -679,6 +687,7 @@ private void ParseSessionsFromBucket(KeyValuePair> kvpUR) sMethod = (string)htParams["method"]; dictSessionFlags["X-Netlog-URLRequest-ID"] = kvpUR.Key.ToString(); dictSessionFlags["X-ProcessInfo"] = String.Format("{0}:0", sClient); + dictSessionFlags["X-Netlog-Traffic_Annotation"] = sTrafficAnnotation; // In case we don't get these later. oTimers.ClientBeginRequest = oTimers.FiddlerGotRequestHeaders = oTimers.FiddlerBeginRequest = GetTimeStamp(htEvent["time"], baseTime); @@ -726,13 +735,19 @@ private void ParseSessionsFromBucket(KeyValuePair> kvpUR) if (iType == NetLogMagics.COOKIE_INCLUSION_STATUS) { - string sCookieName = (htParams["name"] as string) ?? "(name-unavailable)"; string sOperation = (htParams["operation"] as string) ?? String.Empty; - string sExclusionReasons = (htParams["exclusion_reason"] as string) ?? String.Empty; - // {"params": - // { "exclusion_reason":"EXCLUDE_SAMESITE_LAX, DO_NOT_WARN", "name"="foo", "operation"="store"}, - // "source":{"id":3431,"type":1} - // "COOKIE_INCLUSION_STATUS":411 + string sCookieName = (htParams["name"] as string) ?? "(name-unavailable)"; + // TODO: As of Chrome 81, CookieInclusionStatusNetLogParams also adds |domain| and |path| attributes available if "sensitive" data is included. + + // In Chrome 81.3993, the |exclusion_reason| field was renamed to |status| because the |cookie_inclusion_status| entries are + // now also emitted for included cookies. + string sExclusionReasons = (htParams["exclusion_reason"] as string); + if (String.IsNullOrEmpty(sExclusionReasons)) sExclusionReasons = (htParams["status"] as string) ?? String.Empty; + + // If the log indicates that the cookie was included, just skip it for now. + // TODO: Offer a richer cookie-debugging story that exposes the domain/path/and inclusion status. + // https://source.chromium.org/chromium/chromium/src/+/master:net/cookies/canonical_cookie.cc;l=899?q=GetDebugString%20cookie&ss=chromium&originalUrl=https:%2F%2Fcs.chromium.org%2F + if (sExclusionReasons.OICContains("include")) continue; // See |ExclusionReason| list in https://cs.chromium.org/chromium/src/net/cookies/canonical_cookie.h?type=cs&q=EXCLUDE_SAMESITE_LAX&sq=package:chromium&g=0&l=304 // EXCLUDE_HTTP_ONLY, EXCLUDE_SECURE_ONLY,EXCLUDE_DOMAIN_MISMATCH,EXCLUDE_NOT_ON_PATH,EXCLUDE_INVALID_PREFIX @@ -831,7 +846,7 @@ private static void AnnotateHeadersWithUnstoredCookies(HTTPResponseHeaders oRPH, listExclusions.Clear(); } - private void BuildAndAddSession(ref SessionFlags oSF, ref HTTPRequestHeaders oRQH, HTTPResponseHeaders oRPH, MemoryStream msResponseBody, + private void BuildAndAddSession(ref SessionFlags oSF, ref HTTPRequestHeaders oRQH, HTTPResponseHeaders oRPH, MemoryStream msResponseBody, Dictionary dictSessionFlags, string sURL, string sMethod, SessionTimers oTimers, int cbDroppedResponseBody) { // TODO: Sanity-check missing headers. diff --git a/FiddlerImportNetlog/Properties/AssemblyInfo.cs b/FiddlerImportNetlog/Properties/AssemblyInfo.cs index 4d21d81..fede15b 100644 --- a/FiddlerImportNetlog/Properties/AssemblyInfo.cs +++ b/FiddlerImportNetlog/Properties/AssemblyInfo.cs @@ -6,9 +6,15 @@ [assembly: AssemblyCopyright("Copyright ©2019 Eric Lawrence")] [assembly: System.Resources.NeutralResourcesLanguage("en-US")] [assembly: ComVisible(false)] -[assembly: AssemblyVersion("1.2.1.0")] // ALWAYS UPDATE THE VERSION in the [ProfferFormat] attribute to match! +[assembly: AssemblyVersion("1.2.3.0")] // ALWAYS UPDATE THE VERSION in the [ProfferFormat] attribute to match! [assembly: Fiddler.RequiredVersion("4.6.0.0")] +// v1.2.3 +// Add |traffic_annotation| to session properties + +// v1.2.2 +// Update Cookie Inclusion reasons to match latest CL 81.0.3993 https://chromium-review.googlesource.com/c/chromium/src/+/1960865 + // v1.2.1 // Add Cookie Exclusion warnings