[move] Added source file to debug info and used it to finesse trace debugger experience (MystenLabs#21495)

## Description 

This PR adds source file information to debug info for the file that
this info was generated for/from. Previously, dependency source files
used by the trace debugger are those in the `build` directory, as they
are easy to find and readily available. This however had two unfortunate
consequences for the trace debugger user:
- if during debugging, the user enters a function in a dependent source
file, they will not have access to Move IDE's code inspection
capabilities for this file (the reason for it is that source files in
the `build` directory are not "buildable" by the compiler due to lack of
the manifest file so the symbols required for on-hover, go-to-def etc.
cannot be produced)
- if the user uses go-to-def to go to a dependent source file to set a
breakpoint there, the breakpoint would not trigger (the reason for it is
that the dependent file reached this way is not the one in the `build`
directory that is known to the trace debugger but rather the actual
original dependent source file)

Having the original dependent file path in debug info allows us to
remedy these two shortcomings. Of course it is possible that this
original dependent file path no longer exists, in which case trace
debugger will default to using a file from the `build` directory.

## Test plan 

A new test for debug info v2 was added. All tests must pass. I also
tested manually that both deficiencies were remedies when using trace
debugger.

Author: awelc

Cargo.lock

@@ -1,6 +1,7 @@
 // Copyright (c) The Move Contributors
 // SPDX-License-Identifier: Apache-2.0
 
+import * as crypto from 'crypto';
 import * as fs from 'fs';
 import * as path from 'path';
 import { ModuleInfo } from './utils';
@@ -39,7 +40,8 @@ interface JSONSrcFunctionMapEntry {
 }
 
 interface JSONSrcRootObject {
-    version?: number;
+    version?: number; // introduced in debug info v1
+    from_file_path?: string; // introduced in debug info v2
     definition_location: JSONSrcDefinitionLocation;
     module_name: string[];
     struct_map: Record<string, JSONSrcStructSourceMapEntry>;
@@ -207,15 +209,10 @@ function readDebugInfo(
     debugInfoLinesMap: Map<string, Set<number>>,
     failOnNoSourceFile: boolean,
 ): IDebugInfo | undefined {
-    const sourceMapJSON: JSONSrcRootObject = JSON.parse(fs.readFileSync(debugInfoPath, 'utf8'));
+    const debugInfoJSON: JSONSrcRootObject = JSON.parse(fs.readFileSync(debugInfoPath, 'utf8'));
 
-    const fileHash = Buffer.from(sourceMapJSON.definition_location.file_hash).toString('base64');
-    const modInfo: ModuleInfo = {
-        addr: sourceMapJSON.module_name[0],
-        name: sourceMapJSON.module_name[1]
-    };
-    const functions = new Map<string, IDebugInfoFunction>();
-    const fileInfo = filesMap.get(fileHash);
+    let fileHash = Buffer.from(debugInfoJSON.definition_location.file_hash).toString('base64');
+    let fileInfo = filesMap.get(fileHash);
     if (!fileInfo) {
         if (failOnNoSourceFile) {
             throw new Error('Could not find file with hash: '
@@ -226,10 +223,26 @@ function readDebugInfo(
             return undefined;
         }
     }
+
+    /// If the actual file for which debug information was generated
+    /// still exists, use it as it will likely be "buildable" (after all
+    /// debug info was genrated at build time) and thus will work better
+    /// in the IDE setting (e.g., with IDE's code inspection features).
+    if (debugInfoJSON.from_file_path !== undefined &&
+        fs.existsSync(debugInfoJSON.from_file_path)) {
+        [fileHash, fileInfo] = createFileInfo(debugInfoJSON.from_file_path);
+        filesMap.set(fileHash, fileInfo);
+    }
+
+    const modInfo: ModuleInfo = {
+        addr: debugInfoJSON.module_name[0],
+        name: debugInfoJSON.module_name[1]
+    };
+    const functions = new Map<string, IDebugInfoFunction>();
     const debugInfoLines = debugInfoLinesMap.get(fileHash) ?? new Set<number>;
-    prePopulateDebugInfoLines(sourceMapJSON, fileInfo, debugInfoLines);
+    prePopulateDebugInfoLines(debugInfoJSON, fileInfo, debugInfoLines);
     debugInfoLinesMap.set(fileHash, debugInfoLines);
-    const functionMap = sourceMapJSON.function_map;
+    const functionMap = debugInfoJSON.function_map;
     for (const funEntry of Object.values(functionMap)) {
         let nameStart = funEntry.definition_location.start;
         let nameEnd = funEntry.definition_location.end;
@@ -298,6 +311,33 @@ function readDebugInfo(
     return { filePath: fileInfo.path, fileHash, modInfo, functions, optimizedLines: [] };
 }
 
+
+/**
+ * Creates IFileInfo for a file on a given path and returns it along with
+ * the file hash.
+ *
+ * @param filePath path to the file.
+ * @returns a tuple with the file hash and the file info.
+ */
+export function createFileInfo(filePath: string): [string, IFileInfo] {
+    const content = fs.readFileSync(filePath, 'utf8');
+    const numFileHash = computeFileHash(content);
+    const lines = content.split('\n');
+    const fileInfo = { path: filePath, content, lines };
+    const fileHash = Buffer.from(numFileHash).toString('base64');
+    return [fileHash, fileInfo];
+}
+
+/**
+ * Computes the SHA-256 hash of a file's contents.
+ *
+ * @param fileContents contents of the file.
+ */
+function computeFileHash(fileContents: string): Uint8Array {
+    const hash = crypto.createHash('sha256').update(fileContents).digest();
+    return new Uint8Array(hash);
+}
+
 /**
  * Pre-populates the set of source file lines that are present in the debug info
  * with lines corresponding to the definitions of module, structs, enums, and functions

external-crates/move/Cargo.lock

@@ -2,11 +2,11 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { EventEmitter } from 'events';
-import * as crypto from 'crypto';
 import * as fs from 'fs';
 import * as path from 'path';
 import toml from 'toml';
 import {
+    createFileInfo,
     IFileInfo,
     ILocalInfo,
     IDebugInfo,
@@ -130,23 +130,23 @@ interface IRuntimeStackFrame {
     /**
     *  Path to the disassembled bytecode file containing currently executing instruction.
     */
-    bcodeFilePath: undefined | string;
+    bcodeFilePath?: string;
     /**
      *  File hash of the source file containing currently executing instruction.
      */
     srcFileHash: string;
     /**
      *  File hash of the disassembled bytecode file containing currently executing instruction.
      */
-    bcodeFileHash: undefined | string;
+    bcodeFileHash?: string;
     /**
      * Current line in the source file corresponding to currently viewed instruction.
      */
     srcLine: number; // 1-based
     /**
      * Current line in the disassembled bytecode file corresponding to currently viewed instruction.
      */
-    bcodeLine: undefined | number; // 1-based
+    bcodeLine?: number; // 1-based
     /**
      *  Local variable types by variable frame index.
      */
@@ -165,20 +165,20 @@ interface IRuntimeStackFrame {
      * Line in the source file of the last call instruction that was processed in this frame.
      * It's needed to make sure that step/next into/over call works correctly.
      */
-    lastCallInstructionSrcLine: number | undefined;
+    lastCallInstructionSrcLine?: number;
     /**
      * Line in the disassembled bytecode file of the last call instruction that was processed in this frame.
      * It's needed to make sure that step/next into/over call works correctly.
      */
-    lastCallInstructionBcodeLine: number | undefined;
+    lastCallInstructionBcodeLine?: number;
     /**
      * Lines that are not present in the debug info.
      */
     optimizedSrcLines: number[];
     /**
      * Lines that are not present in the bytecode map.
      */
-    optimizedBcodeLines: undefined | number[];
+    optimizedBcodeLines?: number[];
     /**
      * Disassembly mode has been triggered (we have both
      * source and disassembly mode available for this frame).
@@ -378,8 +378,8 @@ export class Runtime extends EventEmitter {
 
         // create a mapping from source file hash to its corresponding debug info
         const srcDebugInfosHashMap = new Map<string, IDebugInfo>;
-        for (const [_, sourceMap] of srcDebugInfo) {
-            srcDebugInfosHashMap.set(sourceMap.fileHash, sourceMap);
+        for (const [_, info] of srcDebugInfo) {
+            srcDebugInfosHashMap.set(info.fileHash, info);
         }
 
         // if we are missing source debug infos (and thus source files), but have bytecode debug infos
@@ -1268,11 +1268,7 @@ function hashToFileMap(
             if (stats.isDirectory()) {
                 processDirectory(filePath);
             } else if (path.extname(f) === extension) {
-                const content = fs.readFileSync(filePath, 'utf8');
-                const numFileHash = computeFileHash(content);
-                const lines = content.split('\n');
-                const fileInfo = { path: filePath, content, lines };
-                const fileHash = Buffer.from(numFileHash).toString('base64');
+                const [fileHash, fileInfo] = createFileInfo(filePath);
                 filesMap.set(fileHash, fileInfo);
             }
         }
@@ -1391,13 +1387,3 @@ function getPkgNameFromManifest(pkgRoot: string): string | undefined {
     const packageName = parsedManifest.package.name;
     return packageName;
 }
-
-/**
- * Computes the SHA-256 hash of a file's contents.
- *
- * @param fileContents contents of the file.
- */
-function computeFileHash(fileContents: string): Uint8Array {
-    const hash = crypto.createHash('sha256').update(fileContents).digest();
-    return new Uint8Array(hash);
-}

external-crates/move/crates/move-analyzer/trace-adapter/src/debug_info_utils.ts

@@ -244,20 +244,20 @@ export type TraceEvent =
         id: number,
         name: string,
         srcFileHash: string
-        bcodeFileHash: undefined | string,
+        bcodeFileHash?: string,
         isNative: boolean,
         localsTypes: string[],
         localsNames: ILocalInfo[],
         paramValues: RuntimeValueType[]
         optimizedSrcLines: number[]
-        optimizedBcodeLines: undefined | number[]
+        optimizedBcodeLines?: number[]
     }
     | { type: TraceEventKind.CloseFrame, id: number }
     | {
         type: TraceEventKind.Instruction,
         pc: number,
         srcLoc: ILoc,
-        bcodeLoc: undefined | ILoc,
+        bcodeLoc?: ILoc,
         kind: TraceInstructionKind
     }
     | { type: TraceEventKind.Effect, effect: EventEffect };
@@ -320,23 +320,23 @@ interface ITraceGenFrameInfo {
     /**
      * Path to a disassembled bytecode file containing function represented by the frame.
      */
-    bcodeFilePath: undefined | string;
+    bcodeFilePath?: string;
     /**
      * Hash of a source file containing function represented by the frame.
      */
     srcFileHash: string;
     /**
      * Hash of a disassembled bytecode file containing function represented by the frame.
      */
-    bcodeFileHash: undefined | string;
+    bcodeFileHash?: string;
     /**
      * Code lines in a given source file that have been optimized away.
      */
     optimizedSrcLines: number[];
     /**
      * Code lines in a given disassembled bytecode file that have been optimized away.
      */
-    optimizedBcodeLines: undefined | number[];
+    optimizedBcodeLines?: number[];
     /**
      * Name of the function represented by the frame.
      */
@@ -348,7 +348,7 @@ interface ITraceGenFrameInfo {
     /**
     * Information for a given function in a disassembled byc file.
     */
-    bcodeFunEntry: undefined | IDebugInfoFunction;
+    bcodeFunEntry?: IDebugInfoFunction;
 }
 
 /**

external-crates/move/crates/move-analyzer/trace-adapter/src/runtime.ts

@@ -0,0 +1,10 @@
+[package]
+name = "stepping_dbg_info_2"
+edition = "2024.beta"
+
+[dependencies]
+MoveStdlib = { local = "../../../../move-stdlib" }
+
+[addresses]
+stepping_dbg_info_2 = "0x0"
+std = "0x1"

external-crates/move/crates/move-analyzer/trace-adapter/src/trace_utils.ts

@@ -0,0 +1 @@
+{"version":2,"from_file_path": "some/nonexistent/path","definition_location":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":349,"end":350},"module_name":["0000000000000000000000000000000000000000000000000000000000000000","m"],"struct_map":{},"enum_map":{},"function_map":{"0":{"location":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":353,"end":387},"definition_location":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":357,"end":360},"type_parameters":[],"parameters":[["p#0#0",{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":361,"end":362}]],"returns":[{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":370,"end":373}],"locals":[],"nops":{},"code_map":{"0":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":380,"end":381},"1":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":384,"end":385},"2":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":382,"end":383},"3":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":380,"end":385}},"is_native":false},"1":{"location":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":397,"end":556},"definition_location":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":401,"end":405},"type_parameters":[],"parameters":[],"returns":[],"locals":[["_res#1#0",{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":422,"end":426}]],"nops":{},"code_map":{"0":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":433,"end":435},"1":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":429,"end":436},"2":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":418,"end":426},"3":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":449,"end":453},"4":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":460,"end":464},"5":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":456,"end":465},"6":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":454,"end":455},"7":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":442,"end":446},"8":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":478,"end":482},"9":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":489,"end":493},"10":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":485,"end":494},"11":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":483,"end":484},"12":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":471,"end":475},"13":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":494,"end":495}},"is_native":false},"2":{"location":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":321,"end":556},"definition_location":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":321,"end":556},"type_parameters":[],"parameters":[],"returns":[],"locals":[],"nops":{},"code_map":{"0":{"file_hash":[143,173,198,181,146,141,90,55,178,38,100,160,9,39,15,6,83,178,250,132,117,135,106,3,48,249,152,61,147,142,60,210],"start":321,"end":556}},"is_native":false}},"constant_map":{}}