create turbo-static for compile time graph analysis

Cargo.toml

@@ -3,7 +3,8 @@ resolver = "2"
 
 members = [
   "crates/node-file-trace",
-  "crates/tower-uds",
+  "crates/tower-uds", 
+  "crates/turbo-static",
   "crates/turbo-tasks*",
   "crates/turbopack*",
   "crates/turborepo*",

crates/turbo-static/.gitignore

@@ -0,0 +1,2 @@
+file
+graph.cypherl

crates/turbo-static/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "turbo-static"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+bincode = "1.3.3"
+clap = { workspace = true, features = ["derive"] }
+crossbeam-channel = "0.5.12"
+fjall = { version = "0.6.3", features = ["bloom"] }
+ignore = "0.4.22"
+itertools.workspace = true
+lsp-server = "0.7.6"
+lsp-types = "0.95.1"
+proc-macro2 = { workspace = true, features = ["span-locations"] }
+serde = { workspace = true, features = ["derive"] }
+serde_json.workspace = true
+serde_path_to_error = "0.1.16"
+syn = { version = "2", features = ["parsing", "full", "visit", "extra-traits"] }
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+tracing.workspace = true
+walkdir = "2.5.0"
+
+[lints]
+workspace = true

crates/turbo-static/readme.md

@@ -0,0 +1,29 @@
+# Turbo Static
+
+Leverages rust-analyzer to build a complete view into the static dependency graph for
+your turbo tasks project.
+
+## How it works
+
+- find all occurences of #[turbo_tasks::function] across all the packages you want to query
+- for each of the tasks we find, query rust analyzer to see which tasks call them
+- apply some very basis control flow analysis to determine whether the call is make 1 time, 0/1 times, or 0+ times,
+  corresponding to direct calls, conditionals, or for loops. nested conditionals collapse
+
+## Usage
+
+This uses an in memory persisted database to cache rust-analyzer queries.
+To reset the cache, pass the `--reindex` flag. Running will produce a
+`graph.cypherl` file which can be loaded into any cypher-compatible database.
+
+```bash
+# run neoj4
+docker run \
+    --publish=7474:7474 --publish=7687:7687 \
+    --volume=$HOME/neo4j/data:/data \
+    neo4j
+# run it passing in the root folders you want to analyze.
+# the system will recursively parse all rust code looking
+# for turbo tasks functions
+cargo run --release -- ../../../turbo ../../../next.js
+```

crates/turbo-static/src/call_resolver.rs

@@ -0,0 +1,125 @@
+use fjall::PartitionCreateOptions;
+
+use crate::{lsp_client::RAClient, Identifier, IdentifierReference};
+
+pub struct CallResolver<'a> {
+    client: &'a mut RAClient,
+    fjall: &'a fjall::Keyspace,
+    handle: fjall::PartitionHandle,
+}
+
+impl<'a> CallResolver<'a> {
+    pub fn new(client: &'a mut RAClient, fjall: &'a fjall::Keyspace) -> Self {
+        let handle = fjall
+            .open_partition("links", PartitionCreateOptions::default())
+            .unwrap();
+        Self {
+            client,
+            fjall,
+            handle,
+        }
+    }
+
+    pub fn cached(&self) -> usize {
+        self.handle.len().unwrap()
+    }
+
+    pub fn cleared(mut self) -> Self {
+        self.fjall.delete_partition(self.handle).unwrap();
+        self.handle = self
+            .fjall
+            .open_partition("links", PartitionCreateOptions::default())
+            .unwrap();
+        self
+    }
+
+    pub fn resolve(&mut self, ident: &Identifier) -> Vec<IdentifierReference> {
+        if let Some(data) = self.handle.get(ident.to_string()).unwrap() {
+            tracing::info!("skipping {}", ident);
+            return bincode::deserialize(&data).unwrap();
+        };
+
+        tracing::info!("checking {}", ident);
+
+        let mut count = 0;
+        let _response = loop {
+            let response = self.client.request(lsp_server::Request {
+                id: 1.into(),
+                method: "textDocument/prepareCallHierarchy".to_string(),
+                params: serde_json::to_value(&lsp_types::CallHierarchyPrepareParams {
+                    text_document_position_params: lsp_types::TextDocumentPositionParams {
+                        position: ident.range.start,
+                        text_document: lsp_types::TextDocumentIdentifier {
+                            uri: lsp_types::Url::from_file_path(&ident.path).unwrap(),
+                        },
+                    },
+                    work_done_progress_params: lsp_types::WorkDoneProgressParams {
+                        work_done_token: Some(lsp_types::ProgressToken::String(
+                            "prepare".to_string(),
+                        )),
+                    },
+                })
+                .unwrap(),
+            });
+            if let Some(Some(value)) = response.result.as_ref().map(|r| r.as_array()) {
+                if !value.is_empty() {
+                    break value.to_owned();
+                }
+                count += 1;
+            }
+
+            // textDocument/prepareCallHierarchy will sometimes return an empty array so try
+            // at most 5 times
+            if count > 5 {
+                tracing::warn!("discovered isolated task {}", ident);
+                break vec![];
+            }
+
+            std::thread::sleep(std::time::Duration::from_secs(1));
+        };
+
+        // callHierarchy/incomingCalls
+        let response = self.client.request(lsp_server::Request {
+            id: 1.into(),
+            method: "callHierarchy/incomingCalls".to_string(),
+            params: serde_json::to_value(lsp_types::CallHierarchyIncomingCallsParams {
+                partial_result_params: lsp_types::PartialResultParams::default(),
+                item: lsp_types::CallHierarchyItem {
+                    name: ident.name.to_owned(),
+                    kind: lsp_types::SymbolKind::FUNCTION,
+                    data: None,
+                    tags: None,
+                    detail: None,
+                    uri: lsp_types::Url::from_file_path(&ident.path).unwrap(),
+                    range: ident.range,
+                    selection_range: ident.range,
+                },
+                work_done_progress_params: lsp_types::WorkDoneProgressParams {
+                    work_done_token: Some(lsp_types::ProgressToken::String("prepare".to_string())),
+                },
+            })
+            .unwrap(),
+        });
+
+        let links = if let Some(e) = response.error {
+            tracing::warn!("unable to resolve {}: {:?}", ident, e);
+            vec![]
+        } else {
+            let response: Result<Vec<lsp_types::CallHierarchyIncomingCall>, _> =
+                serde_path_to_error::deserialize(response.result.unwrap());
+
+            response
+                .unwrap()
+                .into_iter()
+                .map(|i| i.into())
+                .collect::<Vec<IdentifierReference>>()
+        };
+
+        let data = bincode::serialize(&links).unwrap();
+
+        tracing::debug!("links: {:?}", links);
+
+        self.handle.insert(ident.to_string(), data).unwrap();
+        links
+    }
+}

crates/turbo-static/src/identifier.rs

@@ -0,0 +1,99 @@
+use std::{fs, path::PathBuf};
+
+use lsp_types::{CallHierarchyIncomingCall, CallHierarchyItem, Range};
+
+/// A task that references another, with the range of the reference
+#[derive(Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize, Clone, Debug)]
+pub struct IdentifierReference {
+    pub identifier: Identifier,
+    pub references: Vec<Range>, // the places where this identifier is used
+}
+
+/// identifies a task by its file, and range in the file
+#[derive(Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize, Clone)]
+pub struct Identifier {
+    pub path: String,
+    // technically you can derive this from the name and range but it's easier to just store it
+    pub name: String,
+    // post_transform_name: Option<String>,
+    pub range: lsp_types::Range,
+}
+
+impl Identifier {
+    /// check the span matches and the text matches
+    ///
+    /// `same_location` is used to check if the location of the identifier is
+    /// the same as the other
+    pub fn equals_ident(&self, other: &syn::Ident, match_location: bool) -> bool {
+        *other == self.name
+            && (!match_location
+                || (self.range.start.line == other.span().start().line as u32
+                    && self.range.start.character == other.span().start().column as u32))
+    }
+
+    fn get_name(item: &CallHierarchyItem) -> String {
+        // open file, find range inside, extract text
+        let file = fs::read_to_string(item.uri.path()).unwrap();
+        let start = item.selection_range.start;
+        let end = item.selection_range.end;
+        file.lines()
+            .nth(start.line as usize)
+            .unwrap()
+            .chars()
+            .skip(start.character as usize)
+            .take(end.character as usize - start.character as usize)
+            .collect()
+    }
+}
+
+impl From<(PathBuf, syn::Ident)> for Identifier {
+    fn from((path, ident): (PathBuf, syn::Ident)) -> Self {
+        Self {
+            path: path.display().to_string(),
+            name: ident.to_string(),
+            // post_transform_name: None,
+            range: Range {
+                start: lsp_types::Position {
+                    line: ident.span().start().line as u32 - 1,
+                    character: ident.span().start().column as u32,
+                },
+                end: lsp_types::Position {
+                    line: ident.span().end().line as u32 - 1,
+                    character: ident.span().end().column as u32,
+                },
+            },
+        }
+    }
+}
+
+impl From<CallHierarchyIncomingCall> for IdentifierReference {
+    fn from(item: CallHierarchyIncomingCall) -> Self {
+        Self {
+            identifier: Identifier {
+                name: Identifier::get_name(&item.from),
+                // post_transform_name: Some(item.from.name),
+                path: item.from.uri.path().to_owned(),
+                range: item.from.selection_range,
+            },
+            references: item.from_ranges,
+        }
+    }
+}
+
+impl std::fmt::Debug for Identifier {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(self, f)
+    }
+}
+
+impl std::fmt::Display for Identifier {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}:{}#{}",
+            self.path,
+            self.range.start.line,
+            self.name.to_string(),
+        )
+    }
+}