diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5ce1c80a..230c9035f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,8 @@ jobs: for f in ./test-binaries/*; do echo "running $f" chmod +x $f # GH action artifacts don't handle permissions - $f --ignored || exit 1 + # run build-tests. Limited to one thread since we don't support parallel builds. + $f --ignored --test-threads=1 || exit 1 done - name: Clean up the database diff --git a/.gitignore b/.gitignore index e4d65bc4e..99643b163 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ target .vagrant .rustwide .rustwide-docker +.archive_cache .workspace diff --git a/Cargo.lock b/Cargo.lock index 43ea8757a..51fa96703 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -242,9 +242,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byteorder" -version = "1.3.4" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" @@ -264,6 +264,37 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "716960a18f978640f25101b5cbf1c6f6b0d3192fab36a2d98ca96f0ecbe41010" +[[package]] +name = "bzip2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf8012c8a15d5df745fcf258d93e6149dcf102882c8d8702d9cff778eab43a8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.10+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17fa3d1ac1ca21c5c4e36a97f3c3eb25084576f6fc47bf0139c1123434216c6c" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cast" version = "0.2.3" @@ -729,6 +760,7 @@ dependencies = [ "arc-swap", "backtrace", "base64 0.13.0", + "bzip2 0.4.2", "chrono", "comrak", "crates-index", @@ -771,6 +803,7 @@ dependencies = [ "schemamama_postgres", "semver", "serde", + "serde_cbor", "serde_json", "slug", "string_cache", @@ -780,6 +813,7 @@ dependencies = [ "systemstat", "tempfile", "tera", + "test-case", "thiserror", "thread_local", "time 0.1.43", @@ -787,6 +821,7 @@ dependencies = [ "toml", "url 2.2.1", "walkdir", + "zip", "zstd", ] @@ -2536,9 +2571,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a21852a652ad6f610c9510194f398ff6f8692e334fd1145fed931f7fbe44ea" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ "proc-macro2", ] @@ -3696,6 +3731,19 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "test-case" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b114ece25254e97bf48dd4bfc2a12bad0647adacfe4cae1247a9ca6ad302cec" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2", + "quote", + "syn", + "version_check 0.9.2", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -4373,6 +4421,20 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" +[[package]] +name = "zip" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8264fcea9b7a036a4a5103d7153e988dbc2ebbafb34f68a3c2d404b6b82d74b6" +dependencies = [ + "byteorder", + "bzip2 0.3.3", + "crc32fast", + "flate2", + "thiserror", + "time 0.1.43", +] + [[package]] name = "zstd" version = "0.5.2+zstd.1.4.5" diff --git a/Cargo.toml b/Cargo.toml index 7f4764c7f..79369c353 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,9 @@ font-awesome-as-a-crate = { path = "crates/font-awesome-as-a-crate" } dashmap = "3.11.10" string_cache = "0.8.0" postgres-types = { version = "0.2", features = ["derive"] } +zip = "0.5.11" +bzip2 = "0.4.2" +serde_cbor = "0.11.1" getrandom = "0.2.1" # Async @@ -104,6 +107,7 @@ criterion = "0.3" kuchiki = "0.8" rand = "0.8" mockito = "0.29" +test-case = "1.2.0" [build-dependencies] time = "0.1" diff --git a/benches/compression.rs b/benches/compression.rs index 1fed4da13..d95801b8a 100644 --- a/benches/compression.rs +++ b/benches/compression.rs @@ -1,8 +1,6 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; use docs_rs::storage::{compress, decompress, CompressionAlgorithm}; -const ALGORITHM: CompressionAlgorithm = CompressionAlgorithm::Zstd; - pub fn regex_capture_matches(c: &mut Criterion) { // this isn't a great benchmark because it only tests on one file // ideally we would build a whole crate and compress each file, taking the average @@ -11,11 +9,29 @@ pub fn regex_capture_matches(c: &mut Criterion) { c.benchmark_group("regex html") .throughput(Throughput::Bytes(html_slice.len() as u64)) - .bench_function("compress", |b| { - b.iter(|| compress(black_box(html_slice), ALGORITHM)); + .bench_function("compress zstd", |b| { + b.iter(|| compress(black_box(html_slice), CompressionAlgorithm::Zstd)); + }) + .bench_function("decompress zstd", |b| { + b.iter(|| { + decompress( + black_box(html_slice), + CompressionAlgorithm::Zstd, + 5 * 1024 * 1024, + ) + }); + }) + .bench_function("compress bzip2", |b| { + b.iter(|| compress(black_box(html_slice), CompressionAlgorithm::Bzip2)); }) - .bench_function("decompress", |b| { - b.iter(|| decompress(black_box(html_slice), ALGORITHM, 5 * 1024 * 1024)); + .bench_function("decompress bzip2", |b| { + b.iter(|| { + decompress( + black_box(html_slice), + CompressionAlgorithm::Bzip2, + 5 * 1024 * 1024, + ) + }); }); } diff --git a/docker-compose.yml b/docker-compose.yml index 15e8f2e23..b5124bc0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,15 +53,16 @@ services: entrypoint: > /bin/sh -c " mkdir -p /data/rust-docs-rs; - minio server /data; + minio server /data --console-address ":9001"; " ports: - "9000:9000" + - "9001:9001" volumes: - minio-data:/data environment: - MINIO_ACCESS_KEY: cratesfyi - MINIO_SECRET_KEY: secret_key + MINIO_ROOT_USER: cratesfyi + MINIO_ROOT_PASSWORD: secret_key healthcheck: test: [ diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index a40ffe7fb..6bc5c0d3b 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -574,7 +574,7 @@ impl Context for BinContext { fn storage(self) -> Storage = Storage::new( self.pool()?, self.metrics()?, - &*self.config()?, + self.config()?, )?; fn config(self) -> Config = Config::from_env()?; fn metrics(self) -> Metrics = Metrics::new()?; diff --git a/src/config.rs b/src/config.rs index 72b19fdf6..e48e51690 100644 --- a/src/config.rs +++ b/src/config.rs @@ -51,6 +51,10 @@ pub struct Config { // For unit-tests the number has to be higher. pub(crate) random_crate_search_view_size: u32, + // where do we want to store the locally cached index files + // for the remote archives? + pub(crate) local_archive_cache_path: PathBuf, + // Content Security Policy pub(crate) csp_report_only: bool, @@ -127,6 +131,11 @@ impl Config { csp_report_only: env("DOCSRS_CSP_REPORT_ONLY", false)?, + local_archive_cache_path: env( + "DOCSRS_ARCHIVE_INDEX_CACHE_PATH", + PathBuf::from(".archive_cache"), + )?, + rustwide_workspace: env("DOCSRS_RUSTWIDE_WORKSPACE", PathBuf::from(".workspace"))?, inside_docker: env("DOCSRS_DOCKER", false)?, docker_image: maybe_env("DOCSRS_LOCAL_DOCKER_IMAGE")? diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 5e39a9c7c..0af5fbb44 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -38,6 +38,7 @@ pub(crate) fn add_package_into_database( has_examples: bool, compression_algorithms: std::collections::HashSet, repository_id: Option, + archive_storage: bool, ) -> Result { debug!("Adding package into database"); let crate_id = initialize_package_in_database(conn, metadata_pkg)?; @@ -56,12 +57,12 @@ pub(crate) fn add_package_into_database( keywords, have_examples, downloads, files, doc_targets, is_library, doc_rustc_version, documentation_url, default_target, features, - repository_id + repository_id, archive_storage ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, - $19, $20, $21, $22, $23, $24, $25, $26 + $19, $20, $21, $22, $23, $24, $25, $26, $27 ) ON CONFLICT (crate_id, version) DO UPDATE SET release_time = $3, @@ -87,7 +88,8 @@ pub(crate) fn add_package_into_database( documentation_url = $23, default_target = $24, features = $25, - repository_id = $26 + repository_id = $26, + archive_storage = $27 RETURNING id", &[ &crate_id, @@ -116,6 +118,7 @@ pub(crate) fn add_package_into_database( &default_target, &features, &repository_id, + &archive_storage, ], )?; diff --git a/src/db/file.rs b/src/db/file.rs index 9eea778f5..e124078e2 100644 --- a/src/db/file.rs +++ b/src/db/file.rs @@ -8,7 +8,7 @@ //! However, postgres is still available for testing and backwards compatibility. use crate::error::Result; -use crate::storage::{CompressionAlgorithms, Storage}; +use crate::storage::{CompressionAlgorithm, CompressionAlgorithms, Storage}; use serde_json::Value; use std::path::{Path, PathBuf}; @@ -34,6 +34,18 @@ pub fn add_path_into_database>( )) } +pub fn add_path_into_remote_archive>( + storage: &Storage, + archive_path: &str, + path: P, +) -> Result<(Value, CompressionAlgorithm)> { + let (file_list, algorithm) = storage.store_all_in_archive(archive_path, path.as_ref())?; + Ok(( + file_list_to_json(file_list.into_iter().collect()), + algorithm, + )) +} + fn file_list_to_json(file_list: Vec<(PathBuf, String)>) -> Value { Value::Array( file_list diff --git a/src/db/migrate.rs b/src/db/migrate.rs index 737da174f..b93dc18a4 100644 --- a/src/db/migrate.rs +++ b/src/db/migrate.rs @@ -749,6 +749,11 @@ pub fn migrate(version: Option, conn: &mut Client) -> crate::error::Res "ALTER TABLE builds RENAME COLUMN cratesfyi_version TO docsrs_version", "ALTER TABLE builds RENAME COLUMN docsrs_version TO cratesfyi_version", ), + migration!( + context, 30, "add archive-storage marker for releases", + "ALTER TABLE releases ADD COLUMN archive_storage BOOL NOT NULL DEFAULT FALSE;", + "ALTER TABLE releases DROP COLUMN archive_storage;", + ), ]; for migration in migrations { diff --git a/src/db/mod.rs b/src/db/mod.rs index 6545591ac..27645d1d0 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -5,7 +5,7 @@ pub(crate) use self::add_package::{ add_build_into_database, add_doc_coverage, add_package_into_database, }; pub use self::delete::{delete_crate, delete_version}; -pub use self::file::add_path_into_database; +pub use self::file::{add_path_into_database, add_path_into_remote_archive}; pub use self::migrate::migrate; pub use self::pool::{Pool, PoolClient, PoolError}; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index c4a3e5051..d1cb5a5aa 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1,13 +1,13 @@ use crate::db::file::add_path_into_database; use crate::db::{ add_build_into_database, add_doc_coverage, add_package_into_database, - update_crate_data_in_database, Pool, + add_path_into_remote_archive, update_crate_data_in_database, Pool, }; use crate::docbuilder::{crates::crates_from_path, Limits}; use crate::error::Result; use crate::index::api::ReleaseData; use crate::repositories::RepositoryStatsUpdater; -use crate::storage::CompressionAlgorithms; +use crate::storage::{rustdoc_archive_path, source_archive_path}; use crate::utils::{copy_dir_all, parse_rustc_version, CargoMetadata}; use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage}; use crate::{Config, Context, Index, Metrics, Storage}; @@ -359,16 +359,25 @@ impl RustwideBuilder { &metadata, )?; } - let new_algs = self.upload_docs(name, version, local_storage.path())?; - algs.extend(new_algs); + let (_, new_alg) = add_path_into_remote_archive( + &self.storage, + &rustdoc_archive_path(name, version), + local_storage.path(), + )?; + algs.insert(new_alg); }; // Store the sources even if the build fails debug!("adding sources into database"); - let prefix = format!("sources/{}/{}", name, version); - let (files_list, new_algs) = - add_path_into_database(&self.storage, &prefix, build.host_source_dir())?; - algs.extend(new_algs); + let files_list = { + let (files_list, new_alg) = add_path_into_remote_archive( + &self.storage, + &source_archive_path(name, version), + build.host_source_dir(), + )?; + algs.insert(new_alg); + files_list + }; let has_examples = build.host_source_dir().join("examples").is_dir(); if res.result.successful { @@ -403,6 +412,7 @@ impl RustwideBuilder { has_examples, algs, repository, + true, )?; if let Some(doc_coverage) = res.doc_coverage { @@ -670,21 +680,6 @@ impl RustwideBuilder { copy_dir_all(source, dest).map_err(Into::into) } - fn upload_docs( - &self, - name: &str, - version: &str, - local_storage: &Path, - ) -> Result { - debug!("Adding documentation into database"); - add_path_into_database( - &self.storage, - &format!("rustdoc/{}/{}", name, version), - local_storage, - ) - .map(|t| t.1) - } - fn should_build(&self, conn: &mut Client, name: &str, version: &str) -> Result { if self.skip_build_if_exists { // Check whether no successful builds are present in the database. @@ -751,8 +746,6 @@ mod tests { let version = DUMMY_CRATE_VERSION; let default_target = "x86_64-unknown-linux-gnu"; - assert!(env.config().include_default_targets); - let mut builder = RustwideBuilder::init(env).unwrap(); builder .build_package(crate_, version, PackageKind::CratesIo) @@ -766,6 +759,7 @@ mod tests { r.rustdoc_status, r.default_target, r.doc_targets, + r.archive_storage, cov.total_items FROM crates as c @@ -782,6 +776,7 @@ mod tests { assert!(row.get::<_, bool>("rustdoc_status")); assert_eq!(row.get::<_, String>("default_target"), default_target); assert!(row.get::<_, Option>("total_items").is_some()); + assert!(row.get::<_, bool>("archive_storage")); let mut targets: Vec = row .get::<_, Value>("doc_targets") @@ -805,16 +800,31 @@ mod tests { let storage = env.storage(); let web = env.frontend(); - let base = format!("rustdoc/{}/{}", crate_, version); + // doc archive exists + let doc_archive = rustdoc_archive_path(crate_, version); + assert!(storage.exists(&doc_archive)?); + + // source archive exists + let source_archive = source_archive_path(crate_, version); + assert!(storage.exists(&source_archive)?); // default target was built and is accessible - assert!(storage.exists(&format!("{}/{}/index.html", base, crate_path))?); + assert!(storage.exists_in_archive(&doc_archive, &format!("{}/index.html", crate_path))?); assert_success(&format!("/{}/{}/{}", crate_, version, crate_path), web)?; + // source is also packaged + assert!(storage.exists_in_archive(&source_archive, "src/lib.rs")?); + assert_success( + &format!("/crate/{}/{}/source/src/lib.rs", crate_, version), + web, + )?; + // other targets too for target in DEFAULT_TARGETS { - let target_docs_present = - storage.exists(&format!("{}/{}/{}/index.html", base, target, crate_path))?; + let target_docs_present = storage.exists_in_archive( + &doc_archive, + &format!("{}/{}/index.html", target, crate_path), + )?; let target_url = format!( "/{}/{}/{}/{}/index.html", diff --git a/src/storage/archive_index.rs b/src/storage/archive_index.rs new file mode 100644 index 000000000..091b5319d --- /dev/null +++ b/src/storage/archive_index.rs @@ -0,0 +1,110 @@ +use crate::error::Result; +use crate::storage::{compression::CompressionAlgorithm, FileRange}; +use anyhow::{bail, Context as _}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::io; +use std::path::{Path, PathBuf}; + +#[derive(Deserialize, Serialize)] +pub(crate) struct FileInfo { + range: FileRange, + compression: CompressionAlgorithm, +} + +impl FileInfo { + pub(crate) fn range(&self) -> FileRange { + self.range.clone() + } + pub(crate) fn compression(&self) -> CompressionAlgorithm { + self.compression + } +} + +#[derive(Deserialize, Serialize)] +pub(crate) struct Index { + files: HashMap, +} + +impl Index { + pub(crate) fn load(reader: impl io::Read) -> Result { + serde_cbor::from_reader(reader).context("deserialization error") + } + + pub(crate) fn save(&self, writer: impl io::Write) -> Result<()> { + serde_cbor::to_writer(writer, self).context("serialization error") + } + + pub(crate) fn new_from_zip(zipfile: &mut R) -> Result { + let mut archive = zip::ZipArchive::new(zipfile)?; + + // get file locations + let mut files: HashMap = HashMap::with_capacity(archive.len()); + for i in 0..archive.len() { + let zf = archive.by_index(i)?; + + files.insert( + PathBuf::from(zf.name()), + FileInfo { + range: FileRange::new( + zf.data_start(), + zf.data_start() + zf.compressed_size() - 1, + ), + compression: match zf.compression() { + zip::CompressionMethod::Bzip2 => CompressionAlgorithm::Bzip2, + c => bail!("unsupported compression algorithm {} in zip-file", c), + }, + }, + ); + } + + Ok(Index { files }) + } + + pub(crate) fn find_file>(&self, path: P) -> Result<&FileInfo> { + self.files + .get(path.as_ref()) + .ok_or_else(|| super::PathNotFoundError.into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use zip::write::FileOptions; + + fn validate_index(index: &Index) { + assert_eq!(index.files.len(), 1); + + let fi = index.files.get(&PathBuf::from("testfile1")).unwrap(); + assert_eq!(fi.range, FileRange::new(39, 459)); + assert_eq!(fi.compression, CompressionAlgorithm::Bzip2); + } + + #[test] + fn index_create_save_load() { + let mut tf = tempfile::tempfile().unwrap(); + + let objectcontent: Vec = (0..255).collect(); + + let mut archive = zip::ZipWriter::new(tf); + archive + .start_file( + "testfile1", + FileOptions::default().compression_method(zip::CompressionMethod::Bzip2), + ) + .unwrap(); + archive.write_all(&objectcontent).unwrap(); + tf = archive.finish().unwrap(); + + let index = Index::new_from_zip(&mut tf).unwrap(); + validate_index(&index); + + let mut buf = Vec::new(); + index.save(&mut buf).unwrap(); + + let new_index = Index::load(io::Cursor::new(&buf)).unwrap(); + validate_index(&new_index); + } +} diff --git a/src/storage/compression.rs b/src/storage/compression.rs index 0e50bfe5e..649542023 100644 --- a/src/storage/compression.rs +++ b/src/storage/compression.rs @@ -1,11 +1,18 @@ use anyhow::Error; -use std::{collections::HashSet, fmt, io::Read}; +use bzip2::read::{BzDecoder, BzEncoder}; +use bzip2::Compression; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashSet, + fmt, + io::{self, Read}, +}; pub type CompressionAlgorithms = HashSet; macro_rules! enum_id { ($vis:vis enum $name:ident { $($variant:ident = $discriminant:expr,)* }) => { - #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] + #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] $vis enum $name { $($variant = $discriminant,)* } @@ -48,6 +55,7 @@ macro_rules! enum_id { enum_id! { pub enum CompressionAlgorithm { Zstd = 0, + Bzip2 = 1, } } @@ -61,6 +69,13 @@ impl Default for CompressionAlgorithm { pub fn compress(content: impl Read, algorithm: CompressionAlgorithm) -> Result, Error> { match algorithm { CompressionAlgorithm::Zstd => Ok(zstd::encode_all(content, 9)?), + CompressionAlgorithm::Bzip2 => { + let mut compressor = BzEncoder::new(content, Compression::best()); + + let mut data = vec![]; + compressor.read_to_end(&mut data)?; + Ok(data) + } } } @@ -74,6 +89,9 @@ pub fn decompress( match algorithm { CompressionAlgorithm::Zstd => zstd::stream::copy_decode(content, &mut buffer)?, + CompressionAlgorithm::Bzip2 => { + io::copy(&mut BzDecoder::new(content), &mut buffer)?; + } } Ok(buffer.into_inner()) diff --git a/src/storage/database.rs b/src/storage/database.rs index 7a427cadb..268f15aa4 100644 --- a/src/storage/database.rs +++ b/src/storage/database.rs @@ -1,9 +1,9 @@ -use super::{Blob, StorageTransaction}; +use super::{Blob, FileRange, StorageTransaction}; use crate::db::Pool; use crate::error::Result; use crate::Metrics; use postgres::Transaction; -use std::sync::Arc; +use std::{convert::TryFrom, sync::Arc}; pub(crate) struct DatabaseBackend { pool: Pool, @@ -21,24 +21,54 @@ impl DatabaseBackend { Ok(conn.query(query, &[&path])?[0].get(0)) } - pub(super) fn get(&self, path: &str, max_size: usize) -> Result { + pub(super) fn get( + &self, + path: &str, + max_size: usize, + range: Option, + ) -> Result { use std::convert::TryInto; - // The maximum size for a BYTEA (the type used for `content`) is 1GB, so this cast is safe: // https://www.postgresql.org/message-id/162867790712200946i7ba8eb92v908ac595c0c35aee%40mail.gmail.com let max_size = max_size.min(std::i32::MAX as usize) as i32; - // The size limit is checked at the database level, to avoid receiving data altogether if - // the limit is exceeded. - let rows = self.pool.get()?.query( - "SELECT - path, mime, date_updated, compression, - (CASE WHEN LENGTH(content) <= $2 THEN content ELSE NULL END) AS content, - (LENGTH(content) > $2) AS is_too_big - FROM files - WHERE path = $1;", - &[&path, &(max_size)], - )?; + let rows = if let Some(r) = range { + // when we only want to get a range we can validate already if the range is small enough + if (r.end() - r.start() + 1) > max_size as u64 { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + crate::error::SizeLimitReached, + ) + .into()); + } + let range_start = i32::try_from(*r.start())?; + + self.pool.get()?.query( + "SELECT + path, mime, date_updated, compression, + substring(content from $2 for $3) as content, + FALSE as is_too_big + FROM files + WHERE path = $1;", + &[ + &path, + &(range_start + 1), // postgres substring is 1-indexed + &((r.end() - r.start() + 1) as i32), + ], + )? + } else { + // The size limit is checked at the database level, to avoid receiving data altogether if + // the limit is exceeded. + self.pool.get()?.query( + "SELECT + path, mime, date_updated, compression, + (CASE WHEN LENGTH(content) <= $2 THEN content ELSE NULL END) AS content, + (LENGTH(content) > $2) AS is_too_big + FROM files + WHERE path = $1;", + &[&path, &(max_size)], + )? + }; if rows.is_empty() { Err(super::PathNotFoundError.into()) diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 436e4f51f..dcca1bf36 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,3 +1,4 @@ +mod archive_index; mod compression; mod database; mod s3; @@ -7,19 +8,23 @@ use self::database::DatabaseBackend; use self::s3::S3Backend; use crate::error::Result; use crate::{db::Pool, Config, Metrics}; -use anyhow::ensure; +use anyhow::{anyhow, ensure}; use chrono::{DateTime, Utc}; use path_slash::PathExt; use std::{ collections::{HashMap, HashSet}, ffi::OsStr, fmt, fs, + io::{self, Write}, + ops::RangeInclusive, path::{Path, PathBuf}, sync::Arc, }; const MAX_CONCURRENT_UPLOADS: usize = 1000; +type FileRange = RangeInclusive; + #[derive(Debug, thiserror::Error)] #[error("path not found")] pub(crate) struct PathNotFoundError; @@ -33,6 +38,12 @@ pub(crate) struct Blob { pub(crate) compression: Option, } +impl Blob { + pub(crate) fn is_empty(&self) -> bool { + self.mime == "application/x-empty" + } +} + fn get_file_list_from_dir>(path: P, files: &mut Vec) -> Result<()> { let path = path.as_ref(); @@ -97,16 +108,18 @@ enum StorageBackend { pub struct Storage { backend: StorageBackend, + config: Arc, } impl Storage { - pub fn new(pool: Pool, metrics: Arc, config: &Config) -> Result { + pub fn new(pool: Pool, metrics: Arc, config: Arc) -> Result { Ok(Storage { + config: config.clone(), backend: match config.storage_backend { StorageKind::Database => { StorageBackend::Database(DatabaseBackend::new(pool, metrics)) } - StorageKind::S3 => StorageBackend::S3(Box::new(S3Backend::new(metrics, config)?)), + StorageKind::S3 => StorageBackend::S3(Box::new(S3Backend::new(metrics, &config)?)), }, }) } @@ -118,10 +131,78 @@ impl Storage { } } + fn max_file_size_for(&self, path: &str) -> usize { + if path.ends_with(".html") { + self.config.max_file_size_html + } else { + self.config.max_file_size + } + } + + pub(crate) fn fetch_rustdoc_file( + &self, + name: &str, + version: &str, + path: &str, + archive_storage: bool, + ) -> Result { + Ok(if archive_storage { + self.get_from_archive( + &rustdoc_archive_path(name, version), + path, + self.max_file_size_for(path), + )? + } else { + // Add rustdoc prefix, name and version to the path for accessing the file stored in the database + let remote_path = format!("rustdoc/{}/{}/{}", name, version, path); + self.get(&remote_path, self.max_file_size_for(path))? + }) + } + + pub(crate) fn fetch_source_file( + &self, + name: &str, + version: &str, + path: &str, + archive_storage: bool, + ) -> Result { + Ok(if archive_storage { + self.get_from_archive( + &source_archive_path(name, version), + path, + self.max_file_size_for(path), + )? + } else { + let remote_path = format!("sources/{}/{}/{}", name, version, path); + self.get(&remote_path, self.max_file_size_for(path))? + }) + } + + pub(crate) fn rustdoc_file_exists( + &self, + name: &str, + version: &str, + path: &str, + archive_storage: bool, + ) -> Result { + Ok(if archive_storage { + self.exists_in_archive(&rustdoc_archive_path(name, version), path)? + } else { + // Add rustdoc prefix, name and version to the path for accessing the file stored in the database + let remote_path = format!("rustdoc/{}/{}/{}", name, version, path); + self.exists(&remote_path)? + }) + } + + pub(crate) fn exists_in_archive(&self, archive_path: &str, path: &str) -> Result { + let index = self.get_index_for(archive_path)?; + Ok(index.find_file(path).is_ok()) + } + pub(crate) fn get(&self, path: &str, max_size: usize) -> Result { let mut blob = match &self.backend { - StorageBackend::Database(db) => db.get(path, max_size), - StorageBackend::S3(s3) => s3.get(path, max_size), + StorageBackend::Database(db) => db.get(path, max_size, None), + StorageBackend::S3(s3) => s3.get(path, max_size, None), }?; if let Some(alg) = blob.compression { blob.content = decompress(blob.content.as_slice(), alg, max_size)?; @@ -130,6 +211,157 @@ impl Storage { Ok(blob) } + pub(super) fn get_range( + &self, + path: &str, + max_size: usize, + range: FileRange, + compression: Option, + ) -> Result { + let mut blob = match &self.backend { + StorageBackend::Database(db) => db.get(path, max_size, Some(range)), + StorageBackend::S3(s3) => s3.get(path, max_size, Some(range)), + }?; + // `compression` represents the compression of the file-stream inside the archive. + // We don't compress the whole archive, so the encoding of the archive's blob is irrelevant + // here. + if let Some(alg) = compression { + blob.content = decompress(blob.content.as_slice(), alg, max_size)?; + blob.compression = None; + } + Ok(blob) + } + + fn get_index_for(&self, archive_path: &str) -> Result { + // remote/folder/and/x.zip.index + let remote_index_path = format!("{}.index", archive_path); + let local_index_path = self + .config + .local_archive_cache_path + .join(&remote_index_path); + + if local_index_path.exists() { + let mut file = fs::File::open(local_index_path)?; + archive_index::Index::load(&mut file) + } else { + let index_content = self.get(&remote_index_path, std::usize::MAX)?.content; + + fs::create_dir_all( + local_index_path + .parent() + .ok_or_else(|| anyhow!("index path without parent"))?, + )?; + let mut file = fs::File::create(&local_index_path)?; + file.write_all(&index_content)?; + + archive_index::Index::load(&mut &index_content[..]) + } + } + + pub(crate) fn get_from_archive( + &self, + archive_path: &str, + path: &str, + max_size: usize, + ) -> Result { + let index = self.get_index_for(archive_path)?; + let info = index.find_file(path)?; + + let blob = self.get_range( + archive_path, + max_size, + info.range(), + Some(info.compression()), + )?; + assert_eq!(blob.compression, None); + + Ok(Blob { + path: format!("{}/{}", archive_path, path), + mime: detect_mime(&path).into(), + date_updated: blob.date_updated, + content: blob.content, + compression: None, + }) + } + + pub(crate) fn store_all_in_archive( + &self, + archive_path: &str, + root_dir: &Path, + ) -> Result<(HashMap, CompressionAlgorithm)> { + let mut file_paths = HashMap::new(); + + // We are only using the `zip` library to create the archives and the matching + // index-file. The ZIP format allows more compression formats, and these can even be mixed + // in a single archive. + // + // Decompression happens by fetching only the part of the remote archive that contains + // the compressed stream of the object we put into the archive. + // For decompression we are sharing the compression algorithms defined in + // `storage::compression`. So every new algorithm to be used inside ZIP archives + // also has to be added as supported algorithm for storage compression, together + // with a mapping in `storage::archive_index::Index::new_from_zip`. + + let options = + zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Bzip2); + + let mut zip = zip::ZipWriter::new(io::Cursor::new(Vec::new())); + for file_path in get_file_list(root_dir)? { + let mut file = fs::File::open(root_dir.join(&file_path))?; + + zip.start_file(file_path.to_str().unwrap(), options)?; + io::copy(&mut file, &mut zip)?; + + let mime = detect_mime(&file_path); + file_paths.insert(file_path, mime.to_string()); + } + + let mut zip_content = zip.finish()?.into_inner(); + let index = archive_index::Index::new_from_zip(&mut io::Cursor::new(&mut zip_content))?; + let mut index_content = vec![]; + index.save(&mut index_content)?; + let alg = CompressionAlgorithm::default(); + let compressed_index_content = compress(&index_content[..], alg)?; + + let remote_index_path = format!("{}.index", &archive_path); + + // additionally store the index in the local cache, so it's directly available + let local_index_path = self + .config + .local_archive_cache_path + .join(&remote_index_path); + if local_index_path.exists() { + fs::remove_file(&local_index_path)?; + } + fs::create_dir_all(local_index_path.parent().unwrap())?; + let mut local_index_file = fs::File::create(&local_index_path)?; + local_index_file.write_all(&index_content)?; + + self.store_inner( + vec![ + Blob { + path: archive_path.to_string(), + mime: "application/zip".to_owned(), + content: zip_content, + compression: None, + date_updated: Utc::now(), + }, + Blob { + path: remote_index_path, + mime: "application/octet-stream".to_owned(), + content: compressed_index_content, + compression: Some(alg), + date_updated: Utc::now(), + }, + ] + .into_iter() + .map(Ok), + )?; + + let file_alg = CompressionAlgorithm::Bzip2; + Ok((file_paths, file_alg)) + } + fn transaction(&self, f: F) -> Result where F: FnOnce(&mut dyn StorageTransaction) -> Result, @@ -292,6 +524,14 @@ fn detect_mime(file_path: impl AsRef) -> &'static str { } } +pub(crate) fn rustdoc_archive_path(name: &str, version: &str) -> String { + format!("rustdoc/{0}/{1}.zip", name, version) +} + +pub(crate) fn source_archive_path(name: &str, version: &str) -> String { + format!("sources/{0}/{1}.zip", name, version) +} + #[cfg(test)] mod test { use super::*; @@ -382,6 +622,41 @@ mod backend_tests { Ok(()) } + fn test_get_range(storage: &Storage) -> Result<()> { + let blob = Blob { + path: "foo/bar.txt".into(), + mime: "text/plain".into(), + date_updated: Utc::now(), + compression: None, + content: b"test content\n".to_vec(), + }; + + storage.store_blobs(vec![blob.clone()])?; + + assert_eq!( + blob.content[0..=4], + storage + .get_range("foo/bar.txt", std::usize::MAX, 0..=4, None)? + .content + ); + assert_eq!( + blob.content[5..=12], + storage + .get_range("foo/bar.txt", std::usize::MAX, 5..=12, None)? + .content + ); + + for path in &["bar.txt", "baz.txt", "foo/baz.txt"] { + assert!(storage + .get_range(path, std::usize::MAX, 0..=4, None) + .unwrap_err() + .downcast_ref::() + .is_some()); + } + + Ok(()) + } + fn test_get_too_big(storage: &Storage) -> Result<()> { const MAX_SIZE: usize = 1024; @@ -449,6 +724,73 @@ mod backend_tests { Ok(()) } + fn test_store_all_in_archive(storage: &Storage, metrics: &Metrics) -> Result<()> { + let dir = tempfile::Builder::new() + .prefix("docs.rs-upload-archive-test") + .tempdir()?; + let files = ["Cargo.toml", "src/main.rs"]; + for &file in &files { + let path = dir.path().join(file); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, "data")?; + } + + let local_index_location = storage + .config + .local_archive_cache_path + .join("folder/test.zip.index"); + + assert!(!local_index_location.exists()); + + let (stored_files, compression_alg) = + storage.store_all_in_archive("folder/test.zip", dir.path())?; + + assert!(local_index_location.exists()); + assert!(storage.exists("folder/test.zip.index")?); + + assert_eq!(compression_alg, CompressionAlgorithm::Bzip2); + assert_eq!(stored_files.len(), files.len()); + for name in &files { + let name = Path::new(name); + assert!(stored_files.contains_key(name)); + } + assert_eq!( + stored_files.get(Path::new("Cargo.toml")).unwrap(), + "text/toml" + ); + assert_eq!( + stored_files.get(Path::new("src/main.rs")).unwrap(), + "text/rust" + ); + + // delete the existing index to test the download of it + fs::remove_file(&local_index_location)?; + + // the first exists-query will download and store the index + assert!(!local_index_location.exists()); + assert!(storage.exists_in_archive("folder/test.zip", "Cargo.toml")?); + + // the second one will use the local index + assert!(local_index_location.exists()); + assert!(storage.exists_in_archive("folder/test.zip", "src/main.rs")?); + + let file = storage.get_from_archive("folder/test.zip", "Cargo.toml", std::usize::MAX)?; + assert_eq!(file.content, b"data"); + assert_eq!(file.mime, "text/toml"); + assert_eq!(file.path, "folder/test.zip/Cargo.toml"); + + let file = storage.get_from_archive("folder/test.zip", "src/main.rs", std::usize::MAX)?; + assert_eq!(file.content, b"data"); + assert_eq!(file.mime, "text/rust"); + assert_eq!(file.path, "folder/test.zip/src/main.rs"); + + assert_eq!(2, metrics.uploaded_files_total.get()); + + Ok(()) + } + fn test_store_all(storage: &Storage, metrics: &Metrics) -> Result<()> { let dir = tempfile::Builder::new() .prefix("docs.rs-upload-test") @@ -643,6 +985,7 @@ mod backend_tests { test_batched_uploads, test_exists, test_get_object, + test_get_range, test_get_too_big, test_delete_prefix, test_delete_percent, @@ -651,6 +994,7 @@ mod backend_tests { tests_with_metrics { test_store_blobs, test_store_all, + test_store_all_in_archive, } } } diff --git a/src/storage/s3.rs b/src/storage/s3.rs index 14b418cc6..8cb7d426e 100644 --- a/src/storage/s3.rs +++ b/src/storage/s3.rs @@ -1,4 +1,4 @@ -use super::{Blob, StorageTransaction}; +use super::{Blob, FileRange, StorageTransaction}; use crate::{Config, Metrics}; use anyhow::{anyhow, Context, Error}; use chrono::{DateTime, NaiveDateTime, Utc}; @@ -84,13 +84,19 @@ impl S3Backend { }) } - pub(super) fn get(&self, path: &str, max_size: usize) -> Result { + pub(super) fn get( + &self, + path: &str, + max_size: usize, + range: Option, + ) -> Result { self.runtime.block_on(async { let res = self .client .get_object(GetObjectRequest { bucket: self.bucket.to_string(), key: path.into(), + range: range.map(|r| format!("bytes={}-{}", r.start(), r.end())), ..Default::default() }) .await diff --git a/src/test/fakes.rs b/src/test/fakes.rs index e61757712..91f2f5edb 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -1,12 +1,14 @@ use super::TestDatabase; + use crate::docbuilder::{BuildResult, DocCoverage}; +use crate::error::Result; use crate::index::api::{CrateData, CrateOwner, ReleaseData}; -use crate::storage::Storage; +use crate::storage::{rustdoc_archive_path, source_archive_path, Storage}; use crate::utils::{Dependency, MetadataPackage, Target}; -use anyhow::{Context, Error}; +use anyhow::Context; use chrono::{DateTime, Utc}; use postgres::Client; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; #[must_use = "FakeRelease does nothing until you call .create()"] @@ -25,6 +27,7 @@ pub(crate) struct FakeRelease<'a> { registry_release_data: ReleaseData, has_docs: bool, has_examples: bool, + archive_storage: bool, /// This stores the content, while `package.readme` stores the filename readme: Option<&'a str>, github_stats: Option, @@ -90,6 +93,7 @@ impl<'a> FakeRelease<'a> { readme: None, github_stats: None, doc_coverage: None, + archive_storage: false, } } @@ -150,6 +154,11 @@ impl<'a> FakeRelease<'a> { self } + pub(crate) fn archive_storage(mut self, new: bool) -> Self { + self.archive_storage = new; + self + } + /// Since we switched to LOL HTML, all data must have a valid and . /// To avoid duplicating them in every test, this just makes up some content. pub(crate) fn rustdoc_file(mut self, path: &'a str) -> Self { @@ -242,15 +251,15 @@ impl<'a> FakeRelease<'a> { } /// Returns the release_id - pub(crate) fn create(mut self) -> Result { + pub(crate) fn create(mut self) -> Result { use std::fs; use std::path::Path; - let tempdir = tempfile::Builder::new().prefix("docs.rs-fake").tempdir()?; let package = self.package; let db = self.db; let mut rustdoc_files = self.rustdoc_files; let storage = self.storage; + let archive_storage = self.archive_storage; // Upload all source files as rustdoc files // In real life, these would be highlighted HTML, but for testing we just use the files themselves. @@ -269,40 +278,71 @@ impl<'a> FakeRelease<'a> { } } - let upload_files = |prefix: &str, files: &[(&str, &[u8])], target: Option<&str>| { - let mut path_prefix = tempdir.path().join(prefix); - if let Some(target) = target { - path_prefix.push(target); - } - fs::create_dir(&path_prefix)?; + #[derive(Debug)] + enum FileKind { + Rustdoc, + Sources, + } + let create_temp_dir = || { + tempfile::Builder::new() + .prefix("docs.rs-fake") + .tempdir() + .unwrap() + }; + + let store_files_into = |files: &[(&str, &[u8])], base_path: &Path| { for (path, data) in files { if path.starts_with('/') { anyhow::bail!("absolute paths not supported"); } // allow `src/main.rs` if let Some(parent) = Path::new(path).parent() { - let path = path_prefix.join(parent); + let path = base_path.join(parent); fs::create_dir_all(&path) .with_context(|| format!("failed to create {}", path.display()))?; } - let file = path_prefix.join(&path); + let file = base_path.join(&path); log::debug!("writing file {}", file.display()); fs::write(file, data)?; } + Ok(()) + }; - let prefix = format!( - "{}/{}/{}/{}", - prefix, - package.name, - package.version, - target.unwrap_or("") + let upload_files = |kind: FileKind, source_directory: &Path| { + log::debug!( + "adding directory {:?} from {}", + kind, + source_directory.display() ); - log::debug!("adding directory {} from {}", prefix, path_prefix.display()); - crate::db::add_path_into_database(&storage, &prefix, path_prefix) + if archive_storage { + let archive = match kind { + FileKind::Rustdoc => rustdoc_archive_path(&package.name, &package.version), + FileKind::Sources => source_archive_path(&package.name, &package.version), + }; + log::debug!("store in archive: {:?}", archive); + let (files_list, new_alg) = + crate::db::add_path_into_remote_archive(&storage, &archive, source_directory)?; + let mut hm = HashSet::new(); + hm.insert(new_alg); + Ok((files_list, hm)) + } else { + let prefix = match kind { + FileKind::Rustdoc => "rustdoc", + FileKind::Sources => "sources", + }; + crate::db::add_path_into_database( + &storage, + &format!("{}/{}/{}/", prefix, package.name, package.version), + source_directory, + ) + } }; - let (source_meta, mut algs) = upload_files("source", &self.source_files, None)?; + log::debug!("before upload source"); + let source_tmp = create_temp_dir(); + store_files_into(&self.source_files, source_tmp.path())?; + let (source_meta, algs) = upload_files(FileKind::Sources, source_tmp.path())?; log::debug!("added source files {}", source_meta); // If the test didn't add custom builds, inject a default one @@ -317,15 +357,24 @@ impl<'a> FakeRelease<'a> { rustdoc_files.push((&index, DEFAULT_CONTENT)); } - let (rustdoc_meta, new_algs) = upload_files("rustdoc", &rustdoc_files, None)?; - algs.extend(new_algs); - log::debug!("added rustdoc files {}", rustdoc_meta); + let rustdoc_tmp = create_temp_dir(); + let rustdoc_path = rustdoc_tmp.path(); + + // store default target files + store_files_into(&rustdoc_files, rustdoc_path)?; + log::debug!("added rustdoc files"); for target in &package.targets[1..] { let platform = target.src_path.as_ref().unwrap(); - upload_files("rustdoc", &rustdoc_files, Some(platform))?; + let platform_dir = rustdoc_path.join(platform); + fs::create_dir(&platform_dir)?; + + store_files_into(&rustdoc_files, &platform_dir)?; log::debug!("added platform files for {}", platform); } + + let (rustdoc_meta, _) = upload_files(FileKind::Rustdoc, rustdoc_path)?; + log::debug!("uploaded rustdoc files: {}", rustdoc_meta); } let repository = match self.github_stats { @@ -333,7 +382,8 @@ impl<'a> FakeRelease<'a> { None => None, }; - let crate_dir = tempdir.path(); + let crate_tmp = create_temp_dir(); + let crate_dir = crate_tmp.path(); if let Some(markdown) = self.readme { fs::write(crate_dir.join("README.md"), markdown)?; } @@ -355,6 +405,7 @@ impl<'a> FakeRelease<'a> { self.has_examples, algs, repository, + archive_storage, )?; crate::db::update_crate_data_in_database( &mut db.conn(), @@ -380,7 +431,7 @@ struct FakeGithubStats { } impl FakeGithubStats { - fn create(&self, conn: &mut Client) -> Result { + fn create(&self, conn: &mut Client) -> Result { let existing_count: i64 = conn .query_one("SELECT COUNT(*) FROM repositories;", &[])? .get(0); @@ -455,7 +506,7 @@ impl FakeBuild { storage: &Storage, release_id: i32, default_target: &str, - ) -> Result<(), Error> { + ) -> Result<()> { let build_id = crate::db::add_build_into_database(conn, release_id, &self.result)?; if let Some(db_build_log) = self.db_build_log.as_deref() { diff --git a/src/test/mod.rs b/src/test/mod.rs index 15b41a29a..9c2ed8470 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -141,6 +141,12 @@ impl TestEnvironment { .cleanup_after_test() .expect("failed to cleanup after tests"); } + + if let Some(config) = self.config.get() { + if config.local_archive_cache_path.exists() { + fs::remove_dir_all(&config.local_archive_cache_path).unwrap(); + } + } } pub(crate) fn base_config(&self) -> Config { @@ -160,6 +166,9 @@ impl TestEnvironment { config.s3_bucket = format!("docsrs-test-bucket-{}", rand::random::()); config.s3_bucket_is_temporary = true; + config.local_archive_cache_path = + std::env::temp_dir().join(format!("docsrs-test-index-{}", rand::random::())); + config } @@ -194,7 +203,7 @@ impl TestEnvironment { self.storage .get_or_init(|| { Arc::new( - Storage::new(self.db().pool(), self.metrics(), &*self.config()) + Storage::new(self.db().pool(), self.metrics(), self.config()) .expect("failed to initialize the storage"), ) }) diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index b247a3ce0..5f07746df 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -25,6 +25,7 @@ pub struct CrateDetails { build_status: bool, last_successful_build: Option, rustdoc_status: bool, + pub archive_storage: bool, repository_url: Option, homepage_url: Option, keywords: Option, @@ -94,6 +95,7 @@ impl CrateDetails { releases.release_time, releases.build_status, releases.rustdoc_status, + releases.archive_storage, releases.repository_url, releases.homepage_url, releases.keywords, @@ -173,6 +175,7 @@ impl CrateDetails { build_status: krate.get("build_status"), last_successful_build: None, rustdoc_status: krate.get("rustdoc_status"), + archive_storage: krate.get("archive_storage"), repository_url: krate.get("repository_url"), homepage_url: krate.get("homepage_url"), keywords: krate.get("keywords"), diff --git a/src/web/file.rs b/src/web/file.rs index 5dc6f4158..6c5179a73 100644 --- a/src/web/file.rs +++ b/src/web/file.rs @@ -42,11 +42,6 @@ impl File { ))); response } - - /// Checks if mime type of file is "application/x-empty" - pub(super) fn is_empty(&self) -> bool { - self.0.mime == "application/x-empty" - } } #[cfg(test)] diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index b29cd2659..41d849fca 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -338,11 +338,6 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { rendering_time.step("fetch from storage"); - // Add rustdoc prefix, name and version to the path for accessing the file stored in the database - req_path.insert(0, "rustdoc"); - req_path.insert(1, &name); - req_path.insert(2, &version); - // Create the path to access the file from let mut path = req_path.join("/"); if path.ends_with('/') { @@ -353,7 +348,7 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { let mut path = ctry!(req, percent_decode(path.as_bytes()).decode_utf8()); // Attempt to load the file from the database - let file = match File::from_path(storage, &path, config) { + let blob = match storage.fetch_rustdoc_file(&name, &version, &path, krate.archive_storage) { Ok(file) => file, Err(err) => { log::debug!("got error serving {}: {}", path, err); @@ -361,15 +356,18 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { path.to_mut().push_str("/index.html"); req_path.push("index.html"); - return if ctry!(req, storage.exists(&path)) { - redirect(&name, &version, &req_path[3..]) - } else if req_path.get(3).map_or(false, |p| p.contains('-')) { + return if ctry!( + req, + storage.rustdoc_file_exists(&name, &version, &path, krate.archive_storage) + ) { + redirect(&name, &version, &req_path) + } else if req_path.get(0).map_or(false, |p| p.contains('-')) { // This is a target, not a module; it may not have been built. // Redirect to the default target and show a search page instead of a hard 404. redirect( &format!("/crate/{}", name), &format!("{}/target-redirect", version), - &req_path[3..], + &req_path, ) } else { Err(Nope::ResourceNotFound.into()) @@ -381,7 +379,7 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { if !path.ends_with(".html") { rendering_time.step("serve asset"); - return Ok(file.serve()); + return Ok(File(blob).serve()); } rendering_time.step("find latest path"); @@ -409,9 +407,6 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { let (target, inner_path) = { let mut inner_path = req_path.clone(); - // Drop the `rustdoc/:crate/:version[/:platform]` prefix - inner_path.drain(..3).for_each(drop); - let target = if inner_path.len() > 1 && krate .metadata @@ -470,7 +465,7 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { metadata: krate.metadata.clone(), krate, } - .into_response(&file.0.content, config.max_parse_memory, req, &path) + .into_response(&blob.content, config.max_parse_memory, req, &path) } /// Checks whether the given path exists. @@ -483,36 +478,32 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { /// `rustdoc/crate/version[/platform]/module/[kind.name.html|index.html]` /// /// Returns a path that can be appended to `/crate/version/` to create a complete URL. -fn path_for_version( - req_path: &[&str], - known_platforms: &[String], - storage: &Storage, - config: &Config, -) -> String { - // Simple case: page exists in the latest version, so just change the version number - if File::from_path(storage, &req_path.join("/"), config).is_ok() { - // NOTE: this adds 'index.html' if it wasn't there before - return req_path[3..].join("/"); - } +fn path_for_version(file_path: &[&str], crate_details: &CrateDetails) -> String { // check if req_path[3] is the platform choice or the name of the crate // Note we don't require the platform to have a trailing slash. - let platform = if known_platforms.iter().any(|s| s == req_path[3]) && req_path.len() >= 4 { - req_path[3] + let platform = if crate_details + .metadata + .doc_targets + .iter() + .any(|s| s == file_path[0]) + && !file_path.is_empty() + { + file_path[0] } else { "" }; let is_source_view = if platform.is_empty() { // /{name}/{version}/src/{crate}/index.html - req_path.get(3).copied() == Some("src") + file_path.get(0).copied() == Some("src") } else { // /{name}/{version}/{platform}/src/{crate}/index.html - req_path.get(4).copied() == Some("src") + file_path.get(1).copied() == Some("src") }; // this page doesn't exist in the latest version - let last_component = *req_path.last().unwrap(); + let last_component = *file_path.last().unwrap(); let search_item = if last_component == "index.html" { // this is a module - req_path.get(req_path.len() - 2).copied() + file_path.get(file_path.len() - 2).copied() // no trailing slash; no one should be redirected here but we handle it gracefully anyway } else if last_component == platform { // nothing to search for @@ -540,7 +531,6 @@ pub fn target_redirect_handler(req: &mut Request) -> IronResult { let pool = extension!(req, Pool); let mut conn = pool.get()?; let storage = extension!(req, Storage); - let config = extension!(req, Config); let base = redirect_base(req); let updater = extension!(req, RepositoryStatsUpdater); @@ -551,27 +541,37 @@ pub fn target_redirect_handler(req: &mut Request) -> IronResult { // [crate, :name, :version, target-redirect, :target, *path] // is transformed to - // [rustdoc, :name, :version, :target?, *path] + // [:target?, *path] // path might be empty, but target is guaranteed to be there because of the route used let file_path = { - let mut file_path = req.url.path(); - file_path[0] = "rustdoc"; - file_path.remove(3); - if file_path[3] == crate_details.metadata.default_target { - file_path.remove(3); + let mut path = req.url.path(); + path.drain(0..4); // crate, name, version, target-redirect + + if path[0] == crate_details.metadata.default_target { + path.remove(0); } - if let Some(last @ &mut "") = file_path.last_mut() { + // if it ends with a `/`, we add `index.html`. + if let Some(last @ &mut "") = path.last_mut() { *last = "index.html"; } - file_path + path + }; + + let path = if ctry!( + req, + storage.rustdoc_file_exists( + name, + version, + &file_path.join("/"), + crate_details.archive_storage + ) + ) { + // Simple case: page exists in the other target & version, so just change these + file_path.join("/") + } else { + path_for_version(&file_path, &crate_details) }; - let path = path_for_version( - &file_path, - &crate_details.metadata.doc_targets, - storage, - config, - ); let url = format!( "{base}/{name}/{version}/{path}", base = base, @@ -640,6 +640,7 @@ mod test { use kuchiki::traits::TendrilSink; use reqwest::StatusCode; use std::collections::BTreeMap; + use test_case::test_case; fn try_latest_version_redirect( path: &str, @@ -668,14 +669,16 @@ mod test { .with_context(|| anyhow::anyhow!("no redirect found for {}", path)) } - #[test] + #[test_case(true)] + #[test_case(false)] // regression test for https://github.com/rust-lang/docs.rs/issues/552 - fn settings_html() { + fn settings_html(archive_storage: bool) { wrapper(|env| { // first release works, second fails env.fake_release() .name("buggy") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("settings.html") .rustdoc_file("directory_1/index.html") .rustdoc_file("directory_2.html/index.html") @@ -686,6 +689,7 @@ mod test { env.fake_release() .name("buggy") .version("0.2.0") + .archive_storage(archive_storage) .build_result_failed() .create()?; let web = env.frontend(); @@ -701,12 +705,14 @@ mod test { }); } - #[test] - fn default_target_redirects_to_base() { + #[test_case(true)] + #[test_case(false)] + fn default_target_redirects_to_base(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("dummy") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .create()?; @@ -721,6 +727,7 @@ mod test { env.fake_release() .name("dummy") .version("0.2.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .default_target(target) .create()?; @@ -734,6 +741,7 @@ mod test { env.fake_release() .name("dummy") .version("0.3.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .rustdoc_file("all.html") .default_target(target) @@ -752,12 +760,14 @@ mod test { }); } - #[test] - fn go_to_latest_version() { + #[test_case(true)] + #[test_case(false)] + fn go_to_latest_version(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("dummy") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/blah/index.html") .rustdoc_file("dummy/blah/blah.html") .rustdoc_file("dummy/struct.will-be-deleted.html") @@ -765,6 +775,7 @@ mod test { env.fake_release() .name("dummy") .version("0.2.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/blah/index.html") .rustdoc_file("dummy/blah/blah.html") .create()?; @@ -799,18 +810,21 @@ mod test { }) } - #[test] - fn go_to_latest_version_keeps_platform() { + #[test_case(true)] + #[test_case(false)] + fn go_to_latest_version_keeps_platform(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("dummy") .version("0.1.0") + .archive_storage(archive_storage) .add_platform("x86_64-pc-windows-msvc") .rustdoc_file("dummy/struct.Blah.html") .create()?; env.fake_release() .name("dummy") .version("0.2.0") + .archive_storage(archive_storage) .add_platform("x86_64-pc-windows-msvc") .create()?; @@ -843,17 +857,20 @@ mod test { }) } - #[test] - fn redirect_latest_goes_to_crate_if_build_failed() { + #[test_case(true)] + #[test_case(false)] + fn redirect_latest_goes_to_crate_if_build_failed(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("dummy") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .create()?; env.fake_release() .name("dummy") .version("0.2.0") + .archive_storage(archive_storage) .build_result_failed() .create()?; @@ -865,22 +882,26 @@ mod test { }) } - #[test] - fn redirect_latest_does_not_go_to_yanked_versions() { + #[test_case(true)] + #[test_case(false)] + fn redirect_latest_does_not_go_to_yanked_versions(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("dummy") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .create()?; env.fake_release() .name("dummy") .version("0.2.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .create()?; env.fake_release() .name("dummy") .version("0.2.1") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .yanked(true) .create()?; @@ -902,24 +923,28 @@ mod test { }) } - #[test] - fn redirect_latest_with_all_yanked() { + #[test_case(true)] + #[test_case(false)] + fn redirect_latest_with_all_yanked(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("dummy") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .yanked(true) .create()?; env.fake_release() .name("dummy") .version("0.2.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .yanked(true) .create()?; env.fake_release() .name("dummy") .version("0.2.1") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .yanked(true) .create()?; @@ -941,8 +966,9 @@ mod test { }) } - #[test] - fn yanked_release_shows_warning_in_nav() { + #[test_case(true)] + #[test_case(false)] + fn yanked_release_shows_warning_in_nav(archive_storage: bool) { fn has_yanked_warning(path: &str, web: &TestFrontend) -> Result { assert_success(path, web)?; let data = web.get(path).send()?.text()?; @@ -959,6 +985,7 @@ mod test { env.fake_release() .name("dummy") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .yanked(true) .create()?; @@ -968,6 +995,7 @@ mod test { env.fake_release() .name("dummy") .version("0.2.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .yanked(true) .create()?; @@ -1022,12 +1050,14 @@ mod test { }) } - #[test] - fn crate_name_percent_decoded_redirect() { + #[test_case(true)] + #[test_case(false)] + fn crate_name_percent_decoded_redirect(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("fake-crate") .version("0.0.1") + .archive_storage(archive_storage) .rustdoc_file("fake_crate/index.html") .create()?; @@ -1038,8 +1068,9 @@ mod test { }); } - #[test] - fn base_redirect_handles_mismatched_separators() { + #[test_case(true)] + #[test_case(false)] + fn base_redirect_handles_mismatched_separators(archive_storage: bool) { wrapper(|env| { let rels = [ ("dummy-dash", "0.1.0"), @@ -1054,6 +1085,7 @@ mod test { env.fake_release() .name(name) .version(version) + .archive_storage(archive_storage) .rustdoc_file(&(name.replace("-", "_") + "/index.html")) .create()?; } @@ -1098,18 +1130,21 @@ mod test { }) } - #[test] - fn specific_pages_do_not_handle_mismatched_separators() { + #[test_case(true)] + #[test_case(false)] + fn specific_pages_do_not_handle_mismatched_separators(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("dummy-dash") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("dummy_dash/index.html") .create()?; env.fake_release() .name("dummy_mixed-separators") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("dummy_mixed_separators/index.html") .create()?; @@ -1168,8 +1203,9 @@ mod test { }) } - #[test] - fn platform_links_go_to_current_path() { + #[test_case(true)] + #[test_case(false)] + fn platform_links_go_to_current_path(archive_storage: bool) { fn get_platform_links( path: &str, web: &TestFrontend, @@ -1214,6 +1250,7 @@ mod test { env.fake_release() .name("dummy") .version("0.1.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .rustdoc_file("dummy/struct.Dummy.html") .add_target("x86_64-unknown-linux-gnu") @@ -1244,6 +1281,7 @@ mod test { env.fake_release() .name("dummy") .version("0.2.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .rustdoc_file("dummy/struct.Dummy.html") .default_target("x86_64-pc-windows-msvc") @@ -1274,6 +1312,7 @@ mod test { env.fake_release() .name("dummy") .version("0.3.0") + .archive_storage(archive_storage) .rustdoc_file("dummy/index.html") .rustdoc_file("dummy/struct.Dummy.html") .default_target("x86_64-unknown-linux-gnu") @@ -1304,6 +1343,7 @@ mod test { env.fake_release() .name("dummy") .version("0.4.0") + .archive_storage(archive_storage) .rustdoc_file("settings.html") .rustdoc_file("dummy/index.html") .rustdoc_file("dummy/struct.Dummy.html") @@ -1442,12 +1482,14 @@ mod test { }) } - #[test] - fn test_fully_yanked_crate_404s() { + #[test_case(true)] + #[test_case(false)] + fn test_fully_yanked_crate_404s(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("dummy") .version("1.0.0") + .archive_storage(archive_storage) .yanked(true) .create()?; @@ -1465,11 +1507,16 @@ mod test { }) } - #[test] - // regression test for https://github.com/rust-lang/docs.rs/issues/856 - fn test_no_trailing_target_slash() { + #[test_case(true)] + #[test_case(false)] + fn test_no_trailing_target_slash(archive_storage: bool) { + // regression test for https://github.com/rust-lang/docs.rs/issues/856 wrapper(|env| { - env.fake_release().name("dummy").version("0.1.0").create()?; + env.fake_release() + .name("dummy") + .version("0.1.0") + .archive_storage(archive_storage) + .create()?; let web = env.frontend(); assert_redirect( "/crate/dummy/0.1.0/target-redirect/x86_64-apple-darwin", @@ -1479,6 +1526,7 @@ mod test { env.fake_release() .name("dummy") .version("0.2.0") + .archive_storage(archive_storage) .add_platform("x86_64-apple-darwin") .create()?; assert_redirect( @@ -1574,12 +1622,14 @@ mod test { }) } - #[test] - fn test_no_trailing_rustdoc_slash() { + #[test_case(true)] + #[test_case(false)] + fn test_no_trailing_rustdoc_slash(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("tokio") .version("0.2.21") + .archive_storage(archive_storage) .rustdoc_file("tokio/time/index.html") .create()?; assert_redirect( @@ -1590,12 +1640,14 @@ mod test { }) } - #[test] - fn test_non_ascii() { + #[test_case(true)] + #[test_case(false)] + fn test_non_ascii(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("const_unit_poc") .version("1.0.0") + .archive_storage(archive_storage) .rustdoc_file("const_unit_poc/units/constant.Ω.html") .create()?; assert_success( @@ -1605,17 +1657,20 @@ mod test { }) } - #[test] - fn test_latest_version_keeps_query() { + #[test_case(true)] + #[test_case(false)] + fn test_latest_version_keeps_query(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("tungstenite") .version("0.10.0") + .archive_storage(archive_storage) .rustdoc_file("tungstenite/index.html") .create()?; env.fake_release() .name("tungstenite") .version("0.11.0") + .archive_storage(archive_storage) .rustdoc_file("tungstenite/index.html") .create()?; assert_eq!( @@ -1629,12 +1684,14 @@ mod test { }); } - #[test] - fn latest_version_works_when_source_deleted() { + #[test_case(true)] + #[test_case(false)] + fn latest_version_works_when_source_deleted(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("pyo3") .version("0.2.7") + .archive_storage(archive_storage) .source_file("src/objects/exc.rs", b"//! some docs") .create()?; env.fake_release().name("pyo3").version("0.13.2").create()?; @@ -1655,17 +1712,20 @@ mod test { }) } - #[test] - fn test_version_link_goes_to_docs() { + #[test_case(true)] + #[test_case(false)] + fn test_version_link_goes_to_docs(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("hexponent") .version("0.3.0") + .archive_storage(archive_storage) .rustdoc_file("hexponent/index.html") .create()?; env.fake_release() .name("hexponent") .version("0.3.1") + .archive_storage(archive_storage) .rustdoc_file("hexponent/index.html") .create()?; @@ -1759,12 +1819,14 @@ mod test { }) } - #[test] - fn test_missing_target_redirects_to_search() { + #[test_case(true)] + #[test_case(false)] + fn test_missing_target_redirects_to_search(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("winapi") .version("0.3.9") + .archive_storage(archive_storage) .rustdoc_file("winapi/macro.ENUM.html") .create()?; @@ -1779,18 +1841,21 @@ mod test { }) } - #[test] - fn test_redirect_source_not_rust() { + #[test_case(true)] + #[test_case(false)] + fn test_redirect_source_not_rust(archive_storage: bool) { wrapper(|env| { env.fake_release() .name("winapi") .version("0.3.8") + .archive_storage(archive_storage) .source_file("src/docs.md", b"created by Peter Rabbit") .create()?; env.fake_release() .name("winapi") .version("0.3.9") + .archive_storage(archive_storage) .create()?; assert_success("/winapi/0.3.8/src/winapi/docs.md.html", env.frontend())?; diff --git a/src/web/source.rs b/src/web/source.rs index fdcbb1609..32995456a 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -7,7 +7,7 @@ use crate::{ error::Nope, file::File as DbFile, match_version, page::WebPage, redirect_base, MatchSemver, MetaData, Url, }, - Config, Storage, + Storage, }; use iron::{IronResult, Request, Response}; use postgres::Client; @@ -198,7 +198,12 @@ pub fn source_browser_handler(req: &mut Request) -> IronResult { // get path (req_path) for FileList::from_path and actual path for super::file::File::from_path let (req_path, file_path) = { - let file_path = format!("sources/{}/{}/{}", crate_name, version, req_path.join("/")); + let mut req_path = req.url.path(); + // remove first elements from path which is /crate/:name/:version/source + for _ in 0..4 { + req_path.remove(0); + } + let file_path = req_path.join("/"); // FileList::from_path is only working for directories // remove file name if it's not a directory @@ -217,24 +222,46 @@ pub fn source_browser_handler(req: &mut Request) -> IronResult { }; let storage = extension!(req, Storage); - let config = extension!(req, Config); + let archive_storage: bool = { + let rows = ctry!( + req, + conn.query( + " + SELECT archive_storage + FROM releases + INNER JOIN crates ON releases.crate_id = crates.id + WHERE + name = $1 AND + version = $2 + ", + &[&crate_name, &version] + ) + ); + // this unwrap is safe because `match_version` guarantees that the `crate_name`/`version` + // combination exists. + let row = rows.get(0).unwrap(); + + row.get::<_, bool>(0) + }; // try to get actual file first // skip if request is a directory - let file = if !file_path.ends_with('/') { - DbFile::from_path(storage, &file_path, config).ok() + let blob = if !file_path.ends_with('/') { + storage + .fetch_source_file(crate_name, &version, &file_path, archive_storage) + .ok() } else { None }; - let (file_content, is_rust_source) = if let Some(file) = file { + let (file_content, is_rust_source) = if let Some(blob) = blob { // serve the file with DatabaseFileHandler if file isn't text and not empty - if !file.0.mime.starts_with("text") && !file.is_empty() { - return Ok(file.serve()); - } else if file.0.mime.starts_with("text") && !file.is_empty() { + if !blob.mime.starts_with("text") && !blob.is_empty() { + return Ok(DbFile(blob).serve()); + } else if blob.mime.starts_with("text") && !blob.is_empty() { ( - String::from_utf8(file.0.content).ok(), - file.0.path.ends_with(".rs"), + String::from_utf8(blob.content).ok(), + blob.path.ends_with(".rs"), ) } else { (None, false) @@ -258,11 +285,35 @@ pub fn source_browser_handler(req: &mut Request) -> IronResult { #[cfg(test)] mod tests { use crate::test::*; + use test_case::test_case; + + #[test_case(true)] + #[test_case(false)] + fn fetch_source_file_content(archive_storage: bool) { + wrapper(|env| { + env.fake_release() + .archive_storage(archive_storage) + .name("fake") + .version("0.1.0") + .source_file("some_filename.rs", b"some_random_content") + .create()?; + let web = env.frontend(); + assert_success("/crate/fake/0.1.0/source/", web)?; + let response = web + .get("/crate/fake/0.1.0/source/some_filename.rs") + .send()?; + assert!(response.status().is_success()); + assert!(response.text()?.contains("some_random_content")); + Ok(()) + }); + } - #[test] - fn cargo_ok_not_skipped() { + #[test_case(true)] + #[test_case(false)] + fn cargo_ok_not_skipped(archive_storage: bool) { wrapper(|env| { env.fake_release() + .archive_storage(archive_storage) .name("fake") .version("0.1.0") .source_file(".cargo-ok", b"ok") @@ -274,10 +325,12 @@ mod tests { }); } - #[test] - fn directory_not_found() { + #[test_case(true)] + #[test_case(false)] + fn directory_not_found(archive_storage: bool) { wrapper(|env| { env.fake_release() + .archive_storage(archive_storage) .name("mbedtls") .version("0.2.0") .create()?; @@ -287,10 +340,12 @@ mod tests { }) } - #[test] - fn semver_handled() { + #[test_case(true)] + #[test_case(false)] + fn semver_handled(archive_storage: bool) { wrapper(|env| { env.fake_release() + .archive_storage(archive_storage) .name("mbedtls") .version("0.2.0") .source_file("README.md", b"hello") @@ -305,10 +360,13 @@ mod tests { Ok(()) }) } - #[test] - fn literal_krate_description() { + + #[test_case(true)] + #[test_case(false)] + fn literal_krate_description(archive_storage: bool) { wrapper(|env| { env.fake_release() + .archive_storage(archive_storage) .name("rustc-ap-syntax") .version("178.0.0") .description("some stuff with krate")