Add support for sparse index by serving registry over HTTP as static file

Cargo.toml

@@ -20,6 +20,7 @@ mirroring-dummy = []
 db-sled = ["sled"]
 db-redis = ["redis"]
 db-mongo = ["mongodb", "bson"]
+sparse-index = []
 
 [dependencies]
 tokio = { version = "1.1", features = ["macros", "rt-multi-thread", "fs", "io-util"] }

src/config.rs

@@ -158,6 +158,18 @@ impl ServerConfig {
     }
 }
 
+#[allow(dead_code)]
+#[derive(Debug, Clone, Deserialize)]
+pub struct SparseIndexConfig {
+    pub(crate) path: String,
+}
+impl Default for SparseIndexConfig {
+    fn default() -> Self {
+        Self {
+            path: String::from("api/v1/crates"),
+        }
+    }
+}
 #[derive(Debug, Clone, Deserialize)]
 pub struct Config {
     #[serde(default)]
@@ -168,6 +180,8 @@ pub struct Config {
     pub index_config: IndexConfig,
     #[serde(default)]
     pub server_config: ServerConfig,
+    #[serde(default)]
+    pub sparse_index_config: SparseIndexConfig,
 }
 
 impl Default for Config {
@@ -177,6 +191,7 @@ impl Default for Config {
             db_config: Default::default(),
             index_config: Config::index_config_default(),
             server_config: Default::default(),
+            sparse_index_config: Default::default(),
         }
     }
 }

src/get.rs

@@ -53,7 +53,7 @@ pub fn apis(
 }
 
 #[tracing::instrument(skip(path))]
-fn into_boxed_filters(path: Vec<String>) -> BoxedFilter<()> {
+pub(crate) fn into_boxed_filters(path: Vec<String>) -> BoxedFilter<()> {
     let (h, t) = path.split_at(1);
     t.iter().fold(warp::path(h[0].clone()).boxed(), |accm, s| {
         accm.and(warp::path(s.clone())).boxed()

src/main.rs

@@ -9,6 +9,7 @@ mod index_manager;
 mod models;
 mod post;
 mod put;
+mod sparse;
 mod utils;
 
 use crate::config::Config;
@@ -130,6 +131,8 @@ async fn run_server(config: Config) -> anyhow::Result<()> {
         not(all(feature = "db-sled", feature = "db-redis"))
     ))]
     let db_manager = MongoDbManager::new(&config.db_config).await?;
+    #[cfg(feature = "sparse-index")]
+    let local_index_path = config.index_config.local_path.clone();
     let index_manager = IndexManager::new(config.index_config).await?;
     index_manager.pull().await?;
 
@@ -145,9 +148,14 @@ async fn run_server(config: Config) -> anyhow::Result<()> {
         #[cfg(feature = "crates-io-mirroring")]
         Arc::new(cache_dir_path),
         dl_path,
-    )
-    .with(warp::trace::request())
-    .recover(handle_rejection);
+    );
+
+    #[cfg(feature = "sparse-index")]
+    let routes = routes.or(sparse::apis(config.sparse_index_config, local_index_path));
+
+    let routes = routes
+        .with(warp::trace::request())
+        .recover(handle_rejection);
 
     warp::serve(routes)
         .run(server_config.to_socket_addr())
@@ -192,6 +200,7 @@ fn matches() -> ArgMatches<'static> {
         (@arg GIT_EMAIL: --("git-email") +takes_value "Sets an author and committer email address")
         (@arg ADDRESS: --("address") +takes_value "Sets an address HTTP server runs on")
         (@arg PORT: --("port") +takes_value "Sets a port number HTTP server listens")
+        (@arg SPARSE_INDEX_PATH: --("sparse-index-path") +takes_value "Sets the path of sparse index service. Default to /api/v1/crates")
     )
     .get_matches()
 }
@@ -299,5 +308,10 @@ async fn main() -> anyhow::Result<()> {
         config.server_config.port = port;
     }
 
+    #[cfg(feature = "sparse-index")]
+    if let Some(p) = matches.value_of("SPARSE_INDEX_PATH") {
+        config.sparse_index_config.path = p.to_string();
+    }
+
     run_server(config).await
 }

src/sparse.rs

@@ -0,0 +1,45 @@
+#![cfg(feature = "sparse-index")]
+
+use std::convert::Infallible;
+use std::path::PathBuf;
+
+use warp::path::Tail;
+use warp::{reject, Filter, Rejection, Reply};
+
+use crate::config::SparseIndexConfig;
+use crate::get::into_boxed_filters;
+
+#[tracing::instrument(skip(sparse_index_config, local_index_path))]
+pub fn apis(
+    sparse_index_config: SparseIndexConfig,
+    local_index_path: PathBuf,
+) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    into_boxed_filters(
+        sparse_index_config
+            .path
+            .split('/')
+            .map(ToString::to_string)
+            .filter(|s| !s.is_empty())
+            .collect::<Vec<_>>(),
+    )
+    .and(warp::path::tail())
+    .and(with_local_index_path(local_index_path))
+    .and_then(read_crate_index)
+}
+
+#[tracing::instrument(skip(path))]
+fn with_local_index_path(
+    path: PathBuf,
+) -> impl Filter<Extract = (PathBuf,), Error = Infallible> + Clone {
+    warp::any().map(move || path.clone())
+}
+
+#[tracing::instrument(skip(tail, local_index_path))]
+async fn read_crate_index(tail: Tail, local_index_path: PathBuf) -> Result<String, Rejection> {
+    if tail.as_str().starts_with(".") {
+        Err(reject::not_found())
+    } else {
+        std::fs::read_to_string(local_index_path.join(tail.as_str()))
+            .map_err(|_| reject::not_found())
+    }
+}

src/utils.rs

@@ -12,6 +12,7 @@ use std::sync::Arc;
 use tokio::sync::RwLock;
 use warp::{Filter, Rejection, Reply};
 
+#[allow(dead_code)]
 #[inline]
 pub fn always_true<T>(_: T) -> bool {
     true