ruff server - A new built-in LSP for Ruff, written in Rust (astral-…

…sh#10158)

<!--
Thank you for contributing to Ruff! To help us out with reviewing,
please consider the following:

- Does this pull request include a summary of the change? (See below.)
- Does this pull request include a descriptive title?
- Does this pull request include references to any relevant issues?
-->

## Summary

This PR introduces the `ruff_server` crate and a new `ruff server`
command. `ruff_server` is a re-implementation of
[`ruff-lsp`](https://github.com/astral-sh/ruff-lsp), written entirely in
Rust. It brings significant performance improvements, much tighter
integration with Ruff, a foundation for supporting entirely new language
server features, and more!

This PR is an early version of `ruff_lsp` that we're calling the
**pre-release** version. Anyone is more than welcome to use it and
submit bug reports for any issues they encounter - we'll have some
documentation on how to set it up with a few common editors, and we'll
also provide a pre-release VSCode extension for those interested.

This pre-release version supports:
- **Diagnostics for `.py` files**
- **Quick fixes**
- **Full-file formatting**
- **Range formatting**
- **Multiple workspace folders**
- **Automatic linter/formatter configuration** - taken from any
`pyproject.toml` files in the workspace.

Many thanks to @MichaReiser for his [proof-of-concept
work](astral-sh#7262), which was important
groundwork for making this PR possible.

## Architectural Decisions

I've made an executive choice to go with `lsp-server` as a base
framework for the LSP, in favor of `tower-lsp`. There were several
reasons for this:

1. I would like to avoid `async` in our implementation. LSPs are mostly
computationally bound rather than I/O bound, and `async` adds a lot of
complexity to the API, while also making harder to reason about
execution order. This leads into the second reason, which is...
2. Any handlers that mutate state should be blocking and run in the
event loop, and the state should be lock-free. This is the approach that
`rust-analyzer` uses (also with the `lsp-server`/`lsp-types` crates as a
framework), and it gives us assurances about data mutation and execution
order. `tower-lsp` doesn't support this, which has caused some
[issues](ebkalderon/tower-lsp#284) around data
races and out-of-order handler execution.
3. In general, I think it makes sense to have tight control over
scheduling and the specifics of our implementation, in exchange for a
slightly higher up-front cost of writing it ourselves. We'll be able to
fine-tune it to our needs and support future LSP features without
depending on an upstream maintainer.

## Test Plan

The pre-release of `ruff_server` will have snapshot tests for common
document editing scenarios. An expanded test suite is on the roadmap for
future version of `ruff_server`.

Cargo.toml

@@ -32,6 +32,7 @@ console_error_panic_hook = { version = "0.1.7" }
 console_log = { version = "1.0.0" }
 countme = { version = "3.0.1" }
 criterion = { version = "0.5.1", default-features = false }
+crossbeam = { version = "0.8.4" }
 dirs = { version = "5.0.0" }
 drop_bomb = { version = "0.1.5" }
 env_logger = { version = "0.10.1" }
@@ -52,10 +53,14 @@ is-macro = { version = "0.3.5" }
 is-wsl = { version = "0.4.0" }
 itertools = { version = "0.12.1" }
 js-sys = { version = "0.3.67" }
+jod-thread = { version = "0.1.2" }
 lalrpop-util = { version = "0.20.0", default-features = false }
 lexical-parse-float = { version = "0.8.0", features = ["format"] }
+libc = { version = "0.2.153" }
 libcst = { version = "1.1.0", default-features = false }
 log = { version = "0.4.17" }
+lsp-server = { version = "0.7.6" }
+lsp-types = { version = "0.95.0", features = ["proposed"] }
 memchr = { version = "2.7.1" }
 mimalloc = { version = "0.1.39" }
 natord = { version = "1.0.9" }
@@ -97,6 +102,7 @@ toml = { version = "0.8.9" }
 tracing = { version = "0.1.40" }
 tracing-indicatif = { version = "0.3.6" }
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+tracing-tree = { version = "0.2.4" }
 typed-arena = { version = "2.0.2" }
 unic-ucd-category = { version = "0.9" }
 unicode-ident = { version = "1.0.12" }

crates/ruff/Cargo.toml

@@ -20,6 +20,7 @@ ruff_macros = { path = "../ruff_macros" }
 ruff_notebook = { path = "../ruff_notebook" }
 ruff_python_ast = { path = "../ruff_python_ast" }
 ruff_python_formatter = { path = "../ruff_python_formatter" }
+ruff_server = { path = "../ruff_server" }
 ruff_source_file = { path = "../ruff_source_file" }
 ruff_text_size = { path = "../ruff_text_size" }
 ruff_workspace = { path = "../ruff_workspace" }
@@ -52,6 +53,8 @@ tempfile = { workspace = true }
 thiserror = { workspace = true }
 toml = { workspace = true }
 tracing = { workspace = true, features = ["log"] }
+tracing-subscriber = { workspace = true, features = ["registry"]}
+tracing-tree = { workspace = true }
 walkdir = { workspace = true }
 wild = { workspace = true }
 

crates/ruff/src/args.rs

@@ -126,6 +126,8 @@ pub enum Command {
     GenerateShellCompletion { shell: clap_complete_command::Shell },
     /// Run the Ruff formatter on the given files or directories.
     Format(FormatCommand),
+    /// Run the language server.
+    Server(ServerCommand),
     /// Display Ruff's version
     Version {
         #[arg(long, value_enum, default_value = "text")]
@@ -494,6 +496,9 @@ pub struct FormatCommand {
     pub range: Option<FormatRange>,
 }
 
+#[derive(Clone, Debug, clap::Parser)]
+pub struct ServerCommand;
+
 #[derive(Debug, Clone, Copy, clap::ValueEnum)]
 pub enum HelpFormat {
     Text,

crates/ruff/src/commands/mod.rs

@@ -7,6 +7,7 @@ pub(crate) mod format;
 pub(crate) mod format_stdin;
 pub(crate) mod linter;
 pub(crate) mod rule;
+pub(crate) mod server;
 pub(crate) mod show_files;
 pub(crate) mod show_settings;
 pub(crate) mod version;

crates/ruff/src/commands/server.rs

@@ -0,0 +1,69 @@
+use crate::ExitStatus;
+use anyhow::Result;
+use ruff_linter::logging::LogLevel;
+use ruff_server::Server;
+use tracing::{level_filters::LevelFilter, metadata::Level, subscriber::Interest, Metadata};
+use tracing_subscriber::{
+    layer::{Context, Filter, SubscriberExt},
+    Layer, Registry,
+};
+use tracing_tree::time::Uptime;
+
+pub(crate) fn run_server(log_level: LogLevel) -> Result<ExitStatus> {
+    let trace_level = if log_level == LogLevel::Verbose {
+        Level::TRACE
+    } else {
+        Level::DEBUG
+    };
+
+    let subscriber = Registry::default().with(
+        tracing_tree::HierarchicalLayer::default()
+            .with_indent_lines(true)
+            .with_indent_amount(2)
+            .with_bracketed_fields(true)
+            .with_targets(true)
+            .with_writer(|| Box::new(std::io::stderr()))
+            .with_timer(Uptime::default())
+            .with_filter(LoggingFilter { trace_level }),
+    );
+
+    tracing::subscriber::set_global_default(subscriber)?;
+
+    let server = Server::new()?;
+
+    server.run().map(|()| ExitStatus::Success)
+}
+
+struct LoggingFilter {
+    trace_level: Level,
+}
+
+impl LoggingFilter {
+    fn is_enabled(&self, meta: &Metadata<'_>) -> bool {
+        let filter = if meta.target().starts_with("ruff") {
+            self.trace_level
+        } else {
+            Level::INFO
+        };
+
+        meta.level() <= &filter
+    }
+}
+
+impl<S> Filter<S> for LoggingFilter {
+    fn enabled(&self, meta: &Metadata<'_>, _cx: &Context<'_, S>) -> bool {
+        self.is_enabled(meta)
+    }
+
+    fn callsite_enabled(&self, meta: &'static Metadata<'static>) -> Interest {
+        if self.is_enabled(meta) {
+            Interest::always()
+        } else {
+            Interest::never()
+        }
+    }
+
+    fn max_level_hint(&self) -> Option<LevelFilter> {
+        Some(LevelFilter::from_level(self.trace_level))
+    }
+}

crates/ruff/src/lib.rs

@@ -7,7 +7,7 @@ use std::process::ExitCode;
 use std::sync::mpsc::channel;
 
 use anyhow::Result;
-use args::GlobalConfigArgs;
+use args::{GlobalConfigArgs, ServerCommand};
 use clap::CommandFactory;
 use colored::Colorize;
 use log::warn;
@@ -190,6 +190,7 @@ pub fn run(
         }
         Command::Check(args) => check(args, global_options),
         Command::Format(args) => format(args, global_options),
+        Command::Server(args) => server(args, global_options.log_level()),
     }
 }
 
@@ -203,6 +204,12 @@ fn format(args: FormatCommand, global_options: GlobalConfigArgs) -> Result<ExitS
     }
 }
 
+#[allow(clippy::needless_pass_by_value)] // TODO: remove once we start taking arguments from here
+fn server(args: ServerCommand, log_level: LogLevel) -> Result<ExitStatus> {
+    let ServerCommand {} = args;
+    commands::server::run_server(log_level)
+}
+
 pub fn check(args: CheckCommand, global_options: GlobalConfigArgs) -> Result<ExitStatus> {
     let (cli, config_arguments) = args.partition(global_options)?;
 

crates/ruff_formatter/src/lib.rs

@@ -545,6 +545,10 @@ impl PrintedRange {
         &self.code
     }
 
+    pub fn into_code(self) -> String {
+        self.code
+    }
+
     /// The range the formatted code corresponds to in the source document.
     pub fn source_range(&self) -> TextRange {
         self.source_range