Auto merge of #6497 - ehuss:test-require-output-check, r=dwijnand

testsuite: Require failing commands to check output.

This requires that commands that expect an error (using `with_status()`) also check some part of the output, to ensure that the test is working correctly. As evidence by the few tests fixed here, it is dangerous to assume that the correct error will trigger.

Some of these check rustc error messages which is generally undesirable since they aren't always stable, but I think in the few cases they shouldn't be much of a problem. If that seems too risky, I can make them more generic.

tests/testsuite/alt_registry.rs

@@ -310,6 +310,7 @@ fn cannot_publish_to_crates_io_with_registry_dependency() {
         .arg(registry::registry().to_string())
         .masquerade_as_nightly_cargo()
         .with_status(101)
+        .with_stderr_contains("[ERROR] crates cannot be published to crates.io[..]")
         .run();
 }
 

tests/testsuite/build.rs

@@ -542,15 +542,19 @@ fn cargo_compile_with_invalid_code_in_deps() {
         .build();
     let _bar = project()
         .at("bar")
-        .file("Cargo.toml", &basic_bin_manifest("bar"))
+        .file("Cargo.toml", &basic_manifest("bar", "0.1.0"))
         .file("src/lib.rs", "invalid rust code!")
         .build();
     let _baz = project()
         .at("baz")
-        .file("Cargo.toml", &basic_bin_manifest("baz"))
+        .file("Cargo.toml", &basic_manifest("baz", "0.1.0"))
         .file("src/lib.rs", "invalid rust code!")
         .build();
-    p.cargo("build").with_status(101).run();
+    p.cargo("build")
+        .with_status(101)
+        .with_stderr_contains("[..]invalid rust code[..]")
+        .with_stderr_contains("[ERROR] Could not compile [..]")
+        .run();
 }
 
 #[test]
@@ -1266,7 +1270,10 @@ fn compile_offline_while_transitive_dep_not_cached() {
         .build();
 
     // simulate download bar, but fail to download baz
-    p.cargo("build").with_status(101).run();
+    p.cargo("build")
+        .with_status(101)
+        .with_stderr_contains("[..]failed to verify the checksum of `baz[..]")
+        .run();
 
     drop(File::create(baz_path).ok().unwrap().write_all(&content));
 
@@ -2169,7 +2176,10 @@ fn deletion_causes_failure() {
 
     p.cargo("build").run();
     p.change_file("Cargo.toml", &basic_manifest("foo", "0.0.1"));
-    p.cargo("build").with_status(101).run();
+    p.cargo("build")
+        .with_status(101)
+        .with_stderr_contains("[..]can't find crate for `bar`")
+        .run();
 }
 
 #[test]
@@ -2541,11 +2551,14 @@ fn bad_platform_specific_dependency() {
         .file("bar/Cargo.toml", &basic_manifest("bar", "0.5.0"))
         .file(
             "bar/src/lib.rs",
-            r#"extern crate baz; pub fn gimme() -> String { format!("") }"#,
+            r#"pub fn gimme() -> String { format!("") }"#,
         )
         .build();
 
-    p.cargo("build").with_status(101).run();
+    p.cargo("build")
+        .with_status(101)
+        .with_stderr_contains("[..]can't find crate for `bar`")
+        .run();
 }
 
 #[test]
@@ -2858,7 +2871,17 @@ fn dashes_in_crate_name_bad() {
         .file("src/main.rs", "extern crate foo_bar; fn main() {}")
         .build();
 
-    p.cargo("build -v").with_status(101).run();
+    p.cargo("build -v")
+        .with_status(101)
+        .with_stderr(
+            "\
+[ERROR] failed to parse manifest at `[..]/foo/Cargo.toml`
+
+Caused by:
+  library target names cannot contain hyphens: foo-bar
+",
+        )
+        .run();
 }
 
 #[test]
@@ -4513,7 +4536,17 @@ fn avoid_dev_deps() {
         .file("src/main.rs", "fn main() {}")
         .build();
 
-    p.cargo("build").with_status(101).run();
+    p.cargo("build")
+        .with_status(101)
+        .with_stderr(
+            "\
+[UPDATING] [..]
+[ERROR] no matching package named `baz` found
+location searched: registry `https://github.com/rust-lang/crates.io-index`
+required by package `bar v0.1.0 ([..]/foo)`
+",
+        )
+        .run();
     p.cargo("build -Zavoid-dev-deps")
         .masquerade_as_nightly_cargo()
         .run();

tests/testsuite/check.rs

@@ -61,7 +61,10 @@ fn check_fail() {
         .file("src/lib.rs", "pub fn baz() {}")
         .build();
 
-    foo.cargo("check").with_status(101).run();
+    foo.cargo("check")
+        .with_status(101)
+        .with_stderr_contains("[..]this function takes 0 parameters but 1 parameter was supplied")
+        .run();
 }
 
 #[test]
@@ -363,6 +366,9 @@ fn rustc_check_err() {
 
     foo.cargo("rustc --profile check -- --emit=metadata")
         .with_status(101)
+        .with_stderr_contains("[CHECKING] bar [..]")
+        .with_stderr_contains("[CHECKING] foo [..]")
+        .with_stderr_contains("[..]cannot find function `qux` in module `bar`")
         .run();
 }
 

tests/testsuite/features.rs

@@ -1171,7 +1171,10 @@ fn dep_feature_in_cmd_line() {
 
     // The foo project requires that feature "some-feat" in "bar" is enabled.
     // Building without any features enabled should fail:
-    p.cargo("build").with_status(101).run();
+    p.cargo("build")
+        .with_status(101)
+        .with_stderr_contains("[..]unresolved import `bar::test`")
+        .run();
 
     // We should be able to enable the feature "derived-feat", which enables "some-feat",
     // on the command line. The feature is enabled, thus building should be successful:

tests/testsuite/fix.rs

@@ -29,6 +29,7 @@ fn do_not_fix_broken_builds() {
     p.cargo("fix --allow-no-vcs")
         .env("__CARGO_FIX_YOLO", "1")
         .with_status(101)
+        .with_stderr_contains("[ERROR] Could not compile `foo`.")
         .run();
     assert!(p.read_file("src/lib.rs").contains("let mut x = 3;"));
 }
@@ -1247,6 +1248,7 @@ fn fix_to_broken_code() {
         .cwd(p.root().join("bar"))
         .env("RUSTC", p.root().join("foo/target/debug/foo"))
         .with_status(101)
+        .with_stderr_contains("[WARNING] failed to automatically apply fixes [..]")
         .run();
 
     assert_eq!(

tests/testsuite/freshness.rs

@@ -40,7 +40,10 @@ fn modifying_and_moving() {
         .run();
 
     fs::rename(&p.root().join("src/a.rs"), &p.root().join("src/b.rs")).unwrap();
-    p.cargo("build").with_status(101).run();
+    p.cargo("build")
+        .with_status(101)
+        .with_stderr_contains("[..]file not found[..]")
+        .run();
 }
 
 #[test]
@@ -513,7 +516,10 @@ fn rebuild_tests_if_lib_changes() {
     File::create(&p.root().join("src/lib.rs")).unwrap();
 
     p.cargo("build -v").run();
-    p.cargo("test -v").with_status(101).run();
+    p.cargo("test -v")
+        .with_status(101)
+        .with_stderr_contains("[..]cannot find function `foo`[..]")
+        .run();
 }
 
 #[test]

tests/testsuite/install.rs

@@ -1014,7 +1014,10 @@ fn dev_dependencies_no_check() {
         .file("src/main.rs", "fn main() {}")
         .build();
 
-    p.cargo("build").with_status(101).run();
+    p.cargo("build")
+        .with_status(101)
+        .with_stderr_contains("[..] no matching package named `baz` found")
+        .run();
     p.cargo("install").run();
 }
 

tests/testsuite/path.rs

@@ -138,7 +138,10 @@ fn cargo_compile_with_root_dev_deps() {
         )
         .build();
 
-    p.cargo("build").with_status(101).run();
+    p.cargo("build")
+        .with_status(101)
+        .with_stderr_contains("[..]can't find crate for `bar`")
+        .run();
 }
 
 #[test]

tests/testsuite/publish.rs

@@ -597,6 +597,10 @@ fn new_crate_rejected() {
     p.cargo("publish --index")
         .arg(publish::registry().to_string())
         .with_status(101)
+        .with_stderr_contains(
+            "[ERROR] 3 files in the working directory contain \
+             changes that were not yet committed into git:",
+        )
         .run();
 }
 

tests/testsuite/rename_deps.rs

@@ -228,7 +228,10 @@ fn rename_affects_fingerprint() {
         "#,
     );
 
-    p.cargo("build -v").with_status(101).run();
+    p.cargo("build -v")
+        .with_status(101)
+        .with_stderr_contains("[..]can't find crate for `foo`")
+        .run();
 }
 
 #[test]

tests/testsuite/rustdocflags.rs

@@ -35,6 +35,7 @@ fn bad_flags() {
     p.cargo("doc")
         .env("RUSTDOCFLAGS", "--bogus")
         .with_status(101)
+        .with_stderr_contains("[..]bogus[..]")
         .run();
 }