Skip to content

Commit

Permalink
v1.3.1
Browse files Browse the repository at this point in the history 
Add HSTS upgrades
Add socket addresses to list
  • Loading branch information
Eric Lawrence committed Aug 30, 2021
1 parent 6d428bd commit 4a9b8c8
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 28 deletions.
4 changes: 2 additions & 2 deletions FiddlerImportNetlog/FiddlerInterface.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

namespace FiddlerImportNetlog
{
[ProfferFormat("NetLog JSON", "Chromium's JSON-based event log format (v1.3). See https://dev.chromium.org/for-testers/providing-network-details for more details.")]
[ProfferFormat("NetLog JSON", "Chromium's JSON-based event log format (v1.3.1). See https://dev.chromium.org/for-testers/providing-network-details for more details.")]
public class HTTPArchiveFormatImport : ISessionImporter
{
public Session[] ImportSessions(string sFormat, Dictionary<string, object> dictOptions, EventHandler<Fiddler.ProgressCallbackEventArgs> evtProgressNotifications)
Expand Down Expand Up @@ -48,7 +48,7 @@ public Session[] ImportSessions(string sFormat, Dictionary<string, object> dictO
else
{
Stream oFS = File.OpenRead(sFilename);

FiddlerApplication.Log.LogFormat("!NetLog Importer is loading {0}", sFilename);
// Check to see if this file data was GZIP'd or PKZIP'd.
bool bWasGZIP = false;
bool bWasPKZIP = false;
Expand Down
76 changes: 51 additions & 25 deletions FiddlerImportNetlog/Importer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,14 @@ struct Magics
public int SEND_QUIC_HEADERS;
public int SEND_HTTP2_HEADERS;
public int READ_HEADERS;
public int FAKE_RESPONSE_HEADERS_CREATED;
public int COOKIE_INCLUSION_STATUS;
public int FILTERED_BYTES_READ;
public int SEND_BODY;
public int SEND_REQUEST;
public int SSL_CERTIFICATES_RECEIVED;
public int SSL_HANDSHAKE_MESSAGE_RECEIVED;
public int TCP_CONNECT;
}

internal static string DescribeExceptionWithStack(Exception eX)
Expand Down Expand Up @@ -151,12 +153,14 @@ private void ExtractSessionsFromTraceJSON(ArrayList alTraceEvents)
NetLogMagics.SEND_QUIC_HEADERS = 6;
NetLogMagics.SEND_HTTP2_HEADERS = 7;
NetLogMagics.READ_HEADERS = 8;
NetLogMagics.COOKIE_INCLUSION_STATUS = 9;
NetLogMagics.FILTERED_BYTES_READ = 10;
NetLogMagics.SEND_BODY = 11;
NetLogMagics.SEND_REQUEST = 12;
NetLogMagics.SSL_CERTIFICATES_RECEIVED = 13;
NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = 14;
NetLogMagics.FAKE_RESPONSE_HEADERS_CREATED = 9;
NetLogMagics.COOKIE_INCLUSION_STATUS = 10;
NetLogMagics.FILTERED_BYTES_READ = 11;
NetLogMagics.SEND_BODY = 12;
NetLogMagics.SEND_REQUEST = 13;
NetLogMagics.SSL_CERTIFICATES_RECEIVED = 14;
NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = 15;
NetLogMagics.TCP_CONNECT = 16;

List<Hashtable> listEvents = new List<Hashtable>();
foreach (Hashtable htItem in alTraceEvents)
Expand Down Expand Up @@ -196,14 +200,14 @@ private void ExtractSessionsFromTraceJSON(ArrayList alTraceEvents)
iType != NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED) continue;
// Get (or create) the List of entries for this SOCKET.
if (!dictSecureSockets.ContainsKey(iSocketID))
if (!dictSockets.ContainsKey(iSocketID))
{
events = new List<Hashtable>();
dictSecureSockets.Add(iSocketID, events);
dictSockets.Add(iSocketID, events);
}
else
{
events = dictSecureSockets[iSocketID];
events = dictSockets[iSocketID];
}
// Add this event to the SOCKET's list.
events.Add(htEvent);
Expand Down Expand Up @@ -270,7 +274,7 @@ private void ExtractSessionsFromTraceJSON(ArrayList alTraceEvents)
sessSummary.utilSetResponseBody(sbClientInfo.ToString());*/

//GenerateDebugTreeSession(dictURLRequests);
//GenerateSocketListSession(dictSecureSockets);
//GenerateSocketListSession(dictSockets);

NotifyProgress(1, "Import Completed.");
}
Expand Down Expand Up @@ -328,14 +332,16 @@ public bool ExtractSessionsFromJSON(Hashtable htFile)
NetLogMagics.SEND_QUIC_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_QUIC_SEND_REQUEST_HEADERS"], -995);
NetLogMagics.SEND_HTTP2_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_HTTP2_SEND_REQUEST_HEADERS"], -994);
NetLogMagics.READ_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_READ_RESPONSE_HEADERS"], -993);
NetLogMagics.FILTERED_BYTES_READ = getIntValue(htEventTypes["URL_REQUEST_JOB_FILTERED_BYTES_READ"], -992);
NetLogMagics.COOKIE_INCLUSION_STATUS = getIntValue(htEventTypes["COOKIE_INCLUSION_STATUS"], -991);
NetLogMagics.SEND_BODY = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST_BODY"], -990);
NetLogMagics.SEND_REQUEST = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST"], -989);
NetLogMagics.FAKE_RESPONSE_HEADERS_CREATED = getIntValue(htEventTypes["URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED"], -992);
NetLogMagics.FILTERED_BYTES_READ = getIntValue(htEventTypes["URL_REQUEST_JOB_FILTERED_BYTES_READ"], -991);
NetLogMagics.COOKIE_INCLUSION_STATUS = getIntValue(htEventTypes["COOKIE_INCLUSION_STATUS"], -990);
NetLogMagics.SEND_BODY = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST_BODY"], -989);
NetLogMagics.SEND_REQUEST = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST"], -988);

// Socket-level Events
NetLogMagics.SSL_CERTIFICATES_RECEIVED = getIntValue(htEventTypes["SSL_CERTIFICATES_RECEIVED"], -988);
NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = getIntValue(htEventTypes["SSL_HANDSHAKE_MESSAGE_RECEIVED"], -987);
NetLogMagics.SSL_CERTIFICATES_RECEIVED = getIntValue(htEventTypes["SSL_CERTIFICATES_RECEIVED"], -987);
NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = getIntValue(htEventTypes["SSL_HANDSHAKE_MESSAGE_RECEIVED"], -986);
NetLogMagics.TCP_CONNECT = getIntValue(htEventTypes["TCP_CONNECT"], -987);

// Get ALL event type names as strings for pretty print view
dictEventTypes = new Dictionary<int, string>();
Expand Down Expand Up @@ -410,7 +416,7 @@ public bool ExtractSessionsFromJSON(Hashtable htFile)
int iEvent = -1;
int iLastPct = 25;
var dictURLRequests = new Dictionary<int, List<Hashtable>>();
var dictSecureSockets = new Dictionary<int, List<Hashtable>>();
var dictSockets = new Dictionary<int, List<Hashtable>>();

// Loop over events; bucket those associated to URLRequests by the source request's ID.
ArrayList alEvents = htFile["events"] as ArrayList;
Expand All @@ -435,17 +441,18 @@ public bool ExtractSessionsFromJSON(Hashtable htFile)
int iSocketID = getIntValue(htSource["id"], -1);

if (iType != NetLogMagics.SSL_CERTIFICATES_RECEIVED &&
iType != NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED) continue;
iType != NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED &&
iType != NetLogMagics.TCP_CONNECT) continue;

// Get (or create) the List of entries for this SOCKET.
if (!dictSecureSockets.ContainsKey(iSocketID))
if (!dictSockets.ContainsKey(iSocketID))
{
events = new List<Hashtable>();
dictSecureSockets.Add(iSocketID, events);
dictSockets.Add(iSocketID, events);
}
else
{
events = dictSecureSockets[iSocketID];
events = dictSockets[iSocketID];
}
// Add this event to the SOCKET's list.
events.Add(htEvent);
Expand Down Expand Up @@ -506,7 +513,7 @@ public bool ExtractSessionsFromJSON(Hashtable htFile)
sessSummary.utilSetResponseBody(sbClientInfo.ToString());

GenerateDebugTreeSession(dictURLRequests);
GenerateSocketListSession(dictSecureSockets);
GenerateSocketListSession(dictSockets);

NotifyProgress(1, "Import Completed.");
return true;
Expand Down Expand Up @@ -600,6 +607,24 @@ private void GenerateSocketListSession(Dictionary<int, List<Hashtable>> dictSock
int iType = getIntValue(htEvent["type"], -1);
var htParams = (Hashtable) htEvent["params"];

if (iType == NetLogMagics.TCP_CONNECT)
{
if (htParams.ContainsKey("local_address"))
{
htThisSocket.Add("local_address", htParams["local_address"]);
}
//"remote_address", "local_address", "address_list"
if (htParams.ContainsKey("remote_address"))
{
htThisSocket.Add("remote_address", htParams["remote_address"]);
}
if (htParams.ContainsKey("address_list"))
{
htThisSocket.Add("address_list", htParams["address_list"]);
}
continue;
}

if (iType == NetLogMagics.SSL_CERTIFICATES_RECEIVED)
{
StringBuilder sbCertsReceived = new StringBuilder();
Expand Down Expand Up @@ -730,7 +755,7 @@ private void GenerateSocketListSession(Dictionary<int, List<Hashtable>> dictSock
{
_listSessions.Add(Session.BuildFromData(false,
new HTTPRequestHeaders(
String.Format("/SECURE_SOCKETS"), // TODO: Add Machine name?
String.Format("/SOCKETS"), // TODO: Add Machine name?
new[] { "Host: NETLOG" }),
Utilities.emptyByteArray,
new HTTPResponseHeaders(200, "Analyzed Data", new[] { "Content-Type: application/json; charset=utf-8" }),
Expand Down Expand Up @@ -857,7 +882,7 @@ private void ParseSessionsFromBucket(KeyValuePair<int, List<Hashtable>> kvpUR)
// Most events we care about should have parameters. LANDMINE_MEME HERE
if (iType != NetLogMagics.SEND_REQUEST && null == htParams) continue;

FiddlerApplication.Log.LogFormat("URLRequest#{0} - Event type: {1} - {2}", kvpUR.Key, iType, sURL);
// FiddlerApplication.Log.LogFormat("URLRequest#{0} - Event type: {1} - {2}", kvpUR.Key, iType, sURL);

#region ParseImportantEvents
// C# cannot |switch()| on non-constant case values. Hrmph.
Expand Down Expand Up @@ -1032,7 +1057,8 @@ private void ParseSessionsFromBucket(KeyValuePair<int, List<Hashtable>> kvpUR)
continue;
}

if (iType == NetLogMagics.READ_HEADERS)
if ((iType == NetLogMagics.READ_HEADERS) ||
(iType == NetLogMagics.FAKE_RESPONSE_HEADERS_CREATED))
{
ArrayList alHeaderLines = htParams["headers"] as ArrayList;
oTimers.ServerBeginResponse = oTimers.FiddlerGotResponseHeaders = GetTimeStamp(htEvent["time"], baseTime);
Expand Down
21 changes: 20 additions & 1 deletion FiddlerImportNetlog/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,28 @@
[assembly: AssemblyCopyright("Copyright ©2021 Eric Lawrence")]
[assembly: System.Resources.NeutralResourcesLanguage("en-US")]
[assembly: ComVisible(false)]
[assembly: AssemblyVersion("1.3.0.0")] // ALWAYS UPDATE THE VERSION in the [ProfferFormat] attribute in FiddlerInterface.cs to match!
[assembly: AssemblyVersion("1.3.1.0")] // ALWAYS UPDATE THE VERSION in the [ProfferFormat] attribute in FiddlerInterface.cs to match!
[assembly: Fiddler.RequiredVersion("4.6.0.0")]


/*
TODO:
HTTP_STREAM_JOB has a binding between the request and the socket. Hook them up so we can propagate the connection info to the URL_REQUEST-generated Sessions.
t=3262 [st=0] SOCKET_POOL_BOUND_TO_SOCKET
--> source_dependency = 1250 (SOCKET)
t=3262 [st=0] HTTP_STREAM_JOB_BOUND_TO_REQUEST
--> source_dependency = 1701 (URL_REQUEST)
*/

// v1.3.1.0
// Support for FAKE_RESPONSE_HEADERS_CREATED for HSTS and Automatic HTTPS upgrades
// Add socket address info to generated SOCKETS list's session

// v1.3.0.1
// Less Log spam
// Write imported filename to log

// v1.3
// Support importing NetLog events from a Chromium trace json file

Expand Down

0 comments on commit 4a9b8c8

Please sign in to comment.