create turbo-static for compile time graph analysis

Cargo.toml

@@ -3,7 +3,8 @@ resolver = "2"
 
 members = [
   "crates/node-file-trace",
-  "crates/tower-uds",
+  "crates/tower-uds", 
+  "crates/turbo-static",
   "crates/turbo-tasks*",
   "crates/turbopack*",
   "crates/turborepo*",

crates/turbo-static/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "turbo-static"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+bincode = "1.3.3"
+clap = { workspace = true, features = ["derive"] }
+crossbeam-channel = "0.5.12"
+fjall = { version = "0.6.3", features = ["bloom"] }
+ignore = "0.4.22"
+itertools.workspace = true
+lsp-server = "0.7.6"
+lsp-types = "0.95.1"
+proc-macro2 = { workspace = true, features = ["span-locations"] }
+serde_json.workspace = true
+serde_path_to_error = "0.1.16"
+syn = { version = "2", features = ["parsing", "full", "visit", "extra-traits"] }
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+tracing.workspace = true
+walkdir = "2.5.0"
+
+[lints]
+workspace = true

crates/turbo-static/docker-compose.yml

@@ -0,0 +1,23 @@
+version: "3"
+
+services:
+  memgraph:
+    image: memgraph/memgraph-mage:latest
+    container_name: memgraph-1
+    pull_policy: always
+    ports:
+      - "7687:7687"
+      - "7444:7444"
+    command: ["--log-level=TRACE"]
+
+  lab:
+    image: memgraph/lab:latest
+    container_name: memgraph-2
+    pull_policy: always
+    ports:
+      - "3000:3000"
+    depends_on:
+      - memgraph
+    environment:
+      - QUICK_CONNECT_MG_HOST=memgraph
+      - QUICK_CONNECT_MG_PORT=7687

crates/turbo-static/file/partitions/links/config.json

@@ -0,0 +1,8 @@
+{
+  "path": "/Users/arlyon/Programming/nextpack/turbo/crates/turbo-static/file/partitions/links",
+  "block_size": 4096,
+  "level_count": 7,
+  "level_ratio": 8,
+  "type": "Standard",
+  "compression": "Lz4"
+}

crates/turbo-static/file/partitions/links/levels.json

@@ -0,0 +1,9 @@
+[
+  [],
+  [],
+  [],
+  [],
+  [],
+  [],
+  []
+]

crates/turbo-static/readme.md

@@ -0,0 +1,18 @@
+# Turbo Static
+
+Leverages rust-analyzer to build a complete view into the static dependency graph for
+your turbo tasks project.
+
+## How it works
+
+- find all occurences of #[turbo_tasks::function] across all the packages you want to query
+- for each of the tasks we find, query rust analyzer to see which tasks call them
+- apply some very basis control flow analysis to determine whether the call is make 1 time, 0/1 times, or 0+ times,
+  corresponding to direct calls, conditionals, or for loops. nested conditionals collapse
+
+## Stretch goals
+
+- evaluate where Vcs end up to track data flow through the app also
+- a few different visualization formats
+  - dot
+  - neoj4 / graph dbs

crates/turbo-static/src/lsp_client.rs

@@ -0,0 +1,165 @@
+use std::{path::PathBuf, process};
+
+use crossbeam_channel::bounded;
+use lsp_server::{IoThreads, Message};
+
+/// An LSP client for Rust Analyzer (RA) that launches it as a subprocess.
+pub struct RAClient {
+    /// Handle to the client
+    handle: process::Child,
+    sender: Option<crossbeam_channel::Sender<Message>>,
+    receiver: Option<crossbeam_channel::Receiver<Message>>,
+}
+
+impl RAClient {
+    /// Create a new LSP client for Rust Analyzer.
+    pub fn new() -> Self {
+        let stdin = process::Stdio::piped();
+        let stdout = process::Stdio::piped();
+        let stderr = process::Stdio::inherit();
+
+        let child = process::Command::new("rust-analyzer")
+            .stdin(stdin)
+            .stdout(stdout)
+            .stderr(stderr)
+            // .env("RA_LOG", "info")
+            .spawn()
+            .expect("Failed to start RA LSP server");
+        Self {
+            handle: child,
+            sender: None,
+            receiver: None,
+        }
+    }
+
+    pub fn start(&mut self, folders: &[PathBuf]) {
+        let mut stdout = self.handle.stdout.take().unwrap();
+        let mut stdin = self.handle.stdin.take().unwrap();
+
+        let (writer_sender, writer_receiver) = bounded::<Message>(0);
+        let writer = std::thread::spawn(move || {
+            writer_receiver
+                .into_iter()
+                .try_for_each(|it| it.write(&mut stdin))
+        });
+
+        let (reader_sender, reader_receiver) = bounded::<Message>(0);
+        let reader = std::thread::spawn(move || {
+            let mut reader = std::io::BufReader::new(stdout);
+            while let Ok(Some(msg)) = Message::read(&mut reader) {
+                reader_sender
+                    .send(msg)
+                    .expect("receiver was dropped, failed to send a message");
+            }
+        });
+
+        self.sender = Some(writer_sender);
+        self.receiver = Some(reader_receiver);
+
+        let root_path = std::fs::canonicalize(folders.first().unwrap())
+            .unwrap()
+            .to_string_lossy()
+            .to_string();
+        let workspace_paths = folders
+            .iter()
+            .map(|p| std::fs::canonicalize(p).unwrap())
+            .map(|p| lsp_types::WorkspaceFolder {
+                name: p.file_name().unwrap().to_string_lossy().to_string(),
+                uri: lsp_types::Url::from_file_path(p).unwrap(),
+            })
+            .collect::<Vec<_>>();
+
+        let resp = self.request(lsp_server::Request {
+            id: 1.into(),
+            method: "initialize".to_string(),
+            params: serde_json::to_value(&lsp_types::InitializeParams {
+                root_uri: Some(lsp_types::Url::from_file_path(&root_path).unwrap()),
+                root_path: Some(root_path),
+                workspace_folders: Some(workspace_paths),
+                process_id: Some(std::process::id()),
+                capabilities: lsp_types::ClientCapabilities {
+                    workspace: Some(lsp_types::WorkspaceClientCapabilities {
+                        workspace_folders: Some(true),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                initialization_options: None,
+                work_done_progress_params: lsp_types::WorkDoneProgressParams {
+                    work_done_token: Some(lsp_types::ProgressToken::String("prepare".to_string())),
+                },
+                trace: None,
+                client_info: None,
+                locale: None,
+            })
+            .unwrap(),
+        });
+
+        let resp = self.notify(lsp_server::Notification {
+            method: "initialized".to_string(),
+            params: serde_json::to_value(&lsp_types::InitializedParams {}).unwrap(),
+        });
+    }
+
+    pub fn request(&mut self, message: lsp_server::Request) -> lsp_server::Response {
+        tracing::debug!("sending {:?}", message);
+        self.sender
+            .as_mut()
+            .unwrap()
+            .send(Message::Request(message))
+            .expect("failed to send message");
+
+        loop {
+            match self
+                .receiver
+                .as_mut()
+                .unwrap()
+                .recv()
+                .expect("failed to receive message")
+            {
+                lsp_server::Message::Response(response) => {
+                    tracing::debug!("received {:?}", response);
+                    return response;
+                }
+                m => tracing::trace!("unexpected message: {:?}", m),
+            }
+        }
+    }
+
+    pub fn notify(&mut self, message: lsp_server::Notification) {
+        self.sender
+            .as_mut()
+            .unwrap()
+            .send(Message::Notification(message))
+            .expect("failed to send message");
+    }
+}
+
+impl Drop for RAClient {
+    fn drop(&mut self) {
+        if self.sender.is_some() {
+            let resp = self.request(lsp_server::Request {
+                id: 1.into(),
+                method: "shutdown".to_string(),
+                params: serde_json::to_value(&()).unwrap(),
+            });
+
+            if resp.error.is_none() {
+                tracing::info!("shutting down RA LSP server");
+                self.notify(lsp_server::Notification {
+                    method: "exit".to_string(),
+                    params: serde_json::to_value(&()).unwrap(),
+                });
+                self.handle
+                    .wait()
+                    .expect("failed to wait for RA LSP server");
+                tracing::info!("shut down RA LSP server");
+            } else {
+                tracing::error!("failed to shutdown RA LSP server: {:#?}", resp);
+            }
+        }
+
+        self.sender = None;
+        self.receiver = None;
+    }
+}