Update tests

rust-lang · Apr 7, 2022 · 92698e3 · 92698e3
1 parent 5c02bae
commit 92698e3
Show file tree

Hide file tree

Showing 40 changed files with 2,838 additions and 25 deletions.
diff --git a/cargo-registry-index/lib.rs b/cargo-registry-index/lib.rs
@@ -297,10 +297,10 @@ impl Repository {
     /// Does not perform conversion to lowercase.
     fn relative_index_file_helper(name: &str) -> Vec<&str> {
         match name.len() {
-            1 => vec!["1", &name],
-            2 => vec!["2", &name],
-            3 => vec!["3", &name[..1], &name],
-            _ => vec![&name[0..2], &name[2..4], &name],
+            1 => vec!["1", name],
+            2 => vec!["2", name],
+            3 => vec!["3", &name[..1], name],
+            _ => vec![&name[0..2], &name[2..4], name],
         }
     }
 
@@ -359,18 +359,35 @@ impl Repository {
     }
 
     /// Gets a list of files that have been modified since a given commit (or None for all files).
-    pub fn get_files_modified_since(&self, starting_commit: Option<&str>) -> anyhow::Result<Vec<PathBuf>>{
+    pub fn get_files_modified_since(
+        &self,
+        starting_commit: Option<&str>,
+    ) -> anyhow::Result<Vec<PathBuf>> {
         let starting_commit = match starting_commit {
             Some(starting_commit) => {
-                let oid = git2::Oid::from_str(starting_commit).context("failed to parse commit into Oid")?;
-                let commit = self.repository.find_commit(oid).context("failed to find commit")?;
-                Some(commit.as_object().peel_to_tree().context("failed to find tree for commit")?)
+                let oid = git2::Oid::from_str(starting_commit)
+                    .context("failed to parse commit into Oid")?;
+                let commit = self
+                    .repository
+                    .find_commit(oid)
+                    .context("failed to find commit")?;
+                Some(
+                    commit
+                        .as_object()
+                        .peel_to_tree()
+                        .context("failed to find tree for commit")?,
+                )
             }
             None => None,
         };
 
-        let head = self.repository.find_commit(self.head_oid()?)?.as_object().peel_to_tree()?;
-        let diff = self.repository
+        let head = self
+            .repository
+            .find_commit(self.head_oid()?)?
+            .as_object()
+            .peel_to_tree()?;
+        let diff = self
+            .repository
             .diff_tree_to_tree(starting_commit.as_ref(), Some(&head), None)
             .context("failed to run diff")?;
         let mut files = Vec::new();

diff --git a/src/admin/mod.rs b/src/admin/mod.rs
@@ -7,5 +7,5 @@ pub mod populate;
 pub mod render_readmes;
 pub mod test_pagerduty;
 pub mod transfer_crates;
-pub mod verify_token;
 pub mod upload_index;
+pub mod verify_token;
diff --git a/src/admin/render_readmes.rs b/src/admin/render_readmes.rs
@@ -113,12 +113,7 @@ pub fn run(opts: Opts) -> anyhow::Result<()> {
 
                 base_config
                     .uploader()
-                    .upload_readme(
-                        &client,
-                        &krate_name,
-                        &version.num,
-                        readme,
-                    )
+                    .upload_readme(&client, &krate_name, &version.num, readme)
                     .context("Failed to upload rendered README file to S3")?;
 
                 Ok(())

diff --git a/src/admin/upload_index.rs b/src/admin/upload_index.rs
@@ -0,0 +1,50 @@
+use crate::admin::dialoguer;
+use cargo_registry_index::{Repository, RepositoryConfig};
+use reqwest::blocking::Client;
+
+use crate::config;
+
+#[derive(clap::Parser, Debug)]
+#[clap(
+    name = "upload-index",
+    about = "Upload index from git to S3 (http-based index)"
+)]
+pub struct Opts {
+    /// Incremental commit. Any changed files made after this commit will be uploaded.
+    incremental_commit: Option<String>,
+}
+
+pub fn run(opts: Opts) -> anyhow::Result<()> {
+    let config = config::Base::from_environment();
+    let uploader = config.uploader();
+    let client = Client::new();
+
+    println!("fetching git repo");
+    let config = RepositoryConfig::from_environment();
+    let repo = Repository::open(&config)?;
+    repo.reset_head()?;
+    println!("HEAD is at {}", repo.head_oid()?);
+
+    let files = repo.get_files_modified_since(opts.incremental_commit.as_deref())?;
+    println!("found {} files to upload", files.len());
+    if !dialoguer::confirm("continue with upload?") {
+        return Ok(());
+    }
+
+    for file in files {
+        let crate_name = file.file_name().unwrap().to_str().unwrap();
+        let path = repo.index_file(crate_name);
+        if !path.exists() {
+            println!("skipping file `{}`", crate_name);
+            continue;
+        }
+        let contents = std::fs::read_to_string(&path)?;
+        uploader.upload_index(&client, crate_name, contents)?;
+    }
+
+    println!(
+        "uploading completed; use `upload-index {}` for an incremental run",
+        repo.head_oid()?
+    );
+    Ok(())
+}
diff --git a/src/bin/crates-admin.rs b/src/bin/crates-admin.rs
@@ -2,7 +2,7 @@
 
 use cargo_registry::admin::{
     delete_crate, delete_version, migrate, populate, render_readmes, test_pagerduty,
-    transfer_crates, verify_token, upload_index,
+    transfer_crates, upload_index, verify_token,
 };
 
 #[derive(clap::Parser, Debug)]

diff --git a/src/tests/http-data/krate_publish_features_version_2 b/src/tests/http-data/krate_publish_features_version_2
@@ -65,5 +65,72 @@
       ],
       "body": ""
     }
+  },
+  {
+    "request": {
+      "uri": "http://alexcrichton-test.s3.amazonaws.com/index/3/f/foo",
+      "method": "PUT",
+      "headers": [
+        [
+          "accept-encoding",
+          "gzip"
+        ],
+        [
+          "accept",
+          "*/*"
+        ],
+        [
+          "content-length",
+          "336"
+        ],
+        [
+          "date",
+          "Fri, 15 Sep 2017 07:53:06 -0700"
+        ],
+        [
+          "authorization",
+          "AWS AKIAICL5IWUZYWWKA7JA:uDc39eNdF6CcwB+q+JwKsoDLQc4="
+        ],
+        [
+          "content-type",
+          "text/plain"
+        ],
+        [
+          "host",
+          "alexcrichton-test.s3.amazonaws.com"
+        ]
+      ],
+      "body": "eyJuYW1lIjoiZm9vIiwidmVycyI6IjEuMC4wIiwiZGVwcyI6W3sibmFtZSI6ImJhciIsInJlcSI6Ij4gMCIsImZlYXR1cmVzIjpbXSwib3B0aW9uYWwiOmZhbHNlLCJkZWZhdWx0X2ZlYXR1cmVzIjp0cnVlLCJ0YXJnZXQiOm51bGwsImtpbmQiOiJub3JtYWwifV0sImNrc3VtIjoiYWNiNTYwNGIxMjZhYzg5NGMxZWIxMWM0NTc1YmYyMDcyZmVhNjEyMzJhODg4ZTQ1Mzc3MGM3OWQ3ZWQ1NjQxOSIsImZlYXR1cmVzIjp7Im9sZF9mZWF0IjpbXX0sImZlYXR1cmVzMiI6eyJuZXdfZmVhdCI6WyJkZXA6YmFyIiwiYmFyPy9mZWF0Il19LCJ5YW5rZWQiOmZhbHNlLCJsaW5rcyI6bnVsbCwidiI6Mn0K"
+    },
+    "response": {
+      "status": 200,
+      "headers": [
+        [
+          "x-amz-request-id",
+          "26589A5E52F8395C"
+        ],
+        [
+          "x-amz-id-2",
+          "JdIvnNTw53aqXjBIqBLNuN4kxf/w1XWX+xuIiGBDYy7yzOSDuAMtBSrTW4ZWetcCIdqCUHuQ51A="
+        ],
+        [
+          "content-length",
+          "0"
+        ],
+        [
+          "Server",
+          "AmazonS3"
+        ],
+        [
+          "date",
+          "Fri,15 Sep 2017 14:53:07 GMT"
+        ],
+        [
+          "ETag",
+          "\"f9016ad360cebb4fe2e6e96e5949f022\""
+        ]
+      ],
+      "body": ""
+    }
   }
 ]
diff --git a/src/tests/http-data/krate_publish_good_badges b/src/tests/http-data/krate_publish_good_badges
@@ -65,5 +65,72 @@
       ],
       "body": ""
     }
+  },
+  {
+    "request": {
+      "uri": "http://alexcrichton-test.s3.amazonaws.com/index/fo/ob/foobadger",
+      "method": "PUT",
+      "headers": [
+        [
+          "accept-encoding",
+          "gzip"
+        ],
+        [
+          "accept",
+          "*/*"
+        ],
+        [
+          "content-length",
+          "163"
+        ],
+        [
+          "date",
+          "Fri, 15 Sep 2017 07:53:06 -0700"
+        ],
+        [
+          "authorization",
+          "AWS AKIAICL5IWUZYWWKA7JA:uDc39eNdF6CcwB+q+JwKsoDLQc4="
+        ],
+        [
+          "content-type",
+          "text/plain"
+        ],
+        [
+          "host",
+          "alexcrichton-test.s3.amazonaws.com"
+        ]
+      ],
+      "body": "eyJuYW1lIjoiZm9vYmFkZ2VyIiwidmVycyI6IjEuMC4wIiwiZGVwcyI6W10sImNrc3VtIjoiYWNiNTYwNGIxMjZhYzg5NGMxZWIxMWM0NTc1YmYyMDcyZmVhNjEyMzJhODg4ZTQ1Mzc3MGM3OWQ3ZWQ1NjQxOSIsImZlYXR1cmVzIjp7fSwieWFua2VkIjpmYWxzZSwibGlua3MiOm51bGx9Cg=="
+    },
+    "response": {
+      "status": 200,
+      "headers": [
+        [
+          "x-amz-request-id",
+          "26589A5E52F8395C"
+        ],
+        [
+          "x-amz-id-2",
+          "JdIvnNTw53aqXjBIqBLNuN4kxf/w1XWX+xuIiGBDYy7yzOSDuAMtBSrTW4ZWetcCIdqCUHuQ51A="
+        ],
+        [
+          "content-length",
+          "0"
+        ],
+        [
+          "Server",
+          "AmazonS3"
+        ],
+        [
+          "date",
+          "Fri,15 Sep 2017 14:53:07 GMT"
+        ],
+        [
+          "ETag",
+          "\"f9016ad360cebb4fe2e6e96e5949f022\""
+        ]
+      ],
+      "body": ""
+    }
   }
 ]
diff --git a/src/tests/http-data/krate_publish_good_categories b/src/tests/http-data/krate_publish_good_categories
@@ -65,5 +65,72 @@
       ],
       "body": ""
     }
+  },
+  {
+    "request": {
+      "uri": "http://alexcrichton-test.s3.amazonaws.com/index/fo/o_/foo_good_cat",
+      "method": "PUT",
+      "headers": [
+        [
+          "accept-encoding",
+          "gzip"
+        ],
+        [
+          "accept",
+          "*/*"
+        ],
+        [
+          "content-length",
+          "166"
+        ],
+        [
+          "date",
+          "Fri, 15 Sep 2017 07:53:06 -0700"
+        ],
+        [
+          "authorization",
+          "AWS AKIAICL5IWUZYWWKA7JA:uDc39eNdF6CcwB+q+JwKsoDLQc4="
+        ],
+        [
+          "content-type",
+          "text/plain"
+        ],
+        [
+          "host",
+          "alexcrichton-test.s3.amazonaws.com"
+        ]
+      ],
+      "body": "eyJuYW1lIjoiZm9vX2dvb2RfY2F0IiwidmVycyI6IjEuMC4wIiwiZGVwcyI6W10sImNrc3VtIjoiYWNiNTYwNGIxMjZhYzg5NGMxZWIxMWM0NTc1YmYyMDcyZmVhNjEyMzJhODg4ZTQ1Mzc3MGM3OWQ3ZWQ1NjQxOSIsImZlYXR1cmVzIjp7fSwieWFua2VkIjpmYWxzZSwibGlua3MiOm51bGx9Cg=="
+    },
+    "response": {
+      "status": 200,
+      "headers": [
+        [
+          "x-amz-request-id",
+          "26589A5E52F8395C"
+        ],
+        [
+          "x-amz-id-2",
+          "JdIvnNTw53aqXjBIqBLNuN4kxf/w1XWX+xuIiGBDYy7yzOSDuAMtBSrTW4ZWetcCIdqCUHuQ51A="
+        ],
+        [
+          "content-length",
+          "0"
+        ],
+        [
+          "Server",
+          "AmazonS3"
+        ],
+        [
+          "date",
+          "Fri,15 Sep 2017 14:53:07 GMT"
+        ],
+        [
+          "ETag",
+          "\"f9016ad360cebb4fe2e6e96e5949f022\""
+        ]
+      ],
+      "body": ""
+    }
   }
 ]