RalfJung
commented
3 weeks ago Turns out this is a lot harder than I thought, for two reasons:
- Many of our tests have
-Zmiriflags which rustc of course chokes on. We should ideally ignore those flags when building with rustc. - When building and running the tests with rustc, the output from the program happens at runtime, not at "build" time. So for ui-test, the stdout/stderr files we have do not match, since it expects that output to be generated at build time -- after all, ui-test thinks it is just building the program when Miri actually runs it.
We'll also want some way to have a test opt-out from this, some of our tests fundamentally can only work in Miri.
So this will probably require some pretty advanced ui_test surgery or so, not quite sure how to best do this.
Here's my WIP patch, I will not work further on this:
```diff
diff --git a/tests/ui.rs b/tests/ui.rs
index 9cbcf6e42..f523130b3 100644
--- a/tests/ui.rs
+++ b/tests/ui.rs
@@ -99,14 +99,14 @@ fn miri_config(target: &str, path: &str, mode: Mode, with_dependencies: bool) ->
config
}
-fn run_tests(
+fn run_tests_miri(
mode: Mode,
path: &str,
target: &str,
- with_dependencies: bool,
+ with_dependencies: Dependencies,
tmpdir: &Path,
) -> Result<()> {
- let mut config = miri_config(target, path, mode, with_dependencies);
+ let mut config = miri_config(target, path, mode, matches!(with_dependencies, WithDependencies));
// Add a test env var to do environment communication tests.
config.program.envs.push(("MIRI_ENV_VAR_TEST".into(), Some("0".into())));
@@ -176,6 +176,73 @@ fn run_tests(
)
}
+fn run_tests_rustc(
+ mode: Mode,
+ path: &str,
+ target: &str,
+ with_dependencies: Dependencies,
+ tmpdir: &Path,
+) -> Result<()> {
+ let mut config = Config {
+ target: Some(target.to_owned()),
+ stderr_filters: stderr_filters().into(),
+ stdout_filters: stdout_filters().into(),
+ mode,
+ program: CommandBuilder::rustc(),
+ out_dir: PathBuf::from(std::env::var_os("CARGO_TARGET_DIR").unwrap()).join("miri_ui_rustc"),
+ edition: Some("2021".into()), // keep in sync with `./miri run`
+ threads: std::env::var("MIRI_TEST_THREADS")
+ .ok()
+ .map(|threads| NonZero::new(threads.parse().unwrap()).unwrap()),
+ ..Config::rustc(path)
+ };
+
+ if matches!(with_dependencies, WithDependencies) {
+ config.dependencies_crate_manifest_path =
+ Some(Path::new("test_dependencies").join("Cargo.toml"));
+ // Reset `RUSTFLAGS` to work around .
+ config.dependency_builder.envs.push(("RUSTFLAGS".into(), None));
+ }
+
+ // Add a test env var to do environment communication tests.
+ config.program.envs.push(("MIRI_ENV_VAR_TEST".into(), Some("0".into())));
+ // Let the tests know where to store temp files (they might run for a different target, which can make this hard to find).
+ config.program.envs.push(("MIRI_TEMP".into(), Some(tmpdir.to_owned().into())));
+ // If a test ICEs, we want to see a backtrace.
+ config.program.envs.push(("RUST_BACKTRACE".into(), Some("1".into())));
+
+ // Add some flags we always want.
+ config.program.args.push("-Dwarnings".into());
+ config.program.args.push("-Dunused".into());
+ config.program.args.push("-Ainternal_features".into());
+ config.program.args.push("-Zui-testing".into());
+ config.program.args.push("--target".into());
+ config.program.args.push(target.into());
+
+ // Handle command-line arguments.
+ let args = ui_test::Args::test()?;
+ config.with_args(&args, /*default_bless*/ false);
+ eprintln!(" Compiler: {}", config.program.display());
+ ui_test::run_tests_generic(
+ // Only run one test suite. In the future we can add all test suites to one `Vec` and run
+ // them all at once, making best use of systems with high parallelism.
+ vec![config],
+ // The files we're actually interested in (all `.rs` files).
+ ui_test::default_file_filter,
+ // This could be used to overwrite the `Config` on a per-test basis.
+ |_, _, _| {},
+ (
+ match args.format {
+ Format::Terse => status_emitter::Text::quiet(),
+ Format::Pretty => status_emitter::Text::verbose(),
+ },
+ status_emitter::Gha:: {
+ name: format!("{mode:?} {path} ({target})"),
+ },
+ ),
+ )
+}
+
macro_rules! regexes {
($name:ident: $($regex:expr => $replacement:expr,)*) => {
fn $name() -> &'static [(Match, &'static [u8])] {
@@ -239,7 +306,7 @@ enum Dependencies {
use Dependencies::*;
-fn ui(
+fn ui_miri(
mode: Mode,
path: &str,
target: &str,
@@ -249,14 +316,24 @@ fn ui(
let msg = format!("## Running ui tests in {path} for {target}");
eprintln!("{}", msg.green().bold());
- let with_dependencies = match with_dependencies {
- WithDependencies => true,
- WithoutDependencies => false,
- };
- run_tests(mode, path, target, with_dependencies, tmpdir)
+ run_tests_miri(mode, path, target, with_dependencies, tmpdir)
.with_context(|| format!("ui tests in {path} for {target} failed"))
}
+fn ui_rustc(
+ mode: Mode,
+ path: &str,
+ target: &str,
+ with_dependencies: Dependencies,
+ tmpdir: &Path,
+) -> Result<()> {
+ let msg = format!("## Running ui tests in {path} with rustc for {target}");
+ eprintln!("{}", msg.green().bold());
+
+ run_tests_rustc(mode, path, target, with_dependencies, tmpdir)
+ .with_context(|| format!("ui tests in {path} with rustc for {target} failed"))
+}
+
fn get_target() -> String {
env::var("MIRI_TEST_TARGET").ok().unwrap_or_else(get_host)
}
@@ -276,17 +353,29 @@ fn main() -> Result<()> {
}
}
- ui(Mode::Pass, "tests/pass", &target, WithoutDependencies, tmpdir.path())?;
- ui(Mode::Pass, "tests/pass-dep", &target, WithDependencies, tmpdir.path())?;
- ui(Mode::Panic, "tests/panic", &target, WithDependencies, tmpdir.path())?;
- ui(
+ // If requested, build and run the tests with rustc instead.
+ if env::var("MIRI_TEST_WITH_RUSTC").is_ok_and(|v| v != "0") {
+ ui_rustc(
+ Mode::Run { exit_code: 0 },
+ "tests/pass-dep",
+ &target,
+ WithDependencies,
+ tmpdir.path(),
+ )?;
+ return Ok(());
+ }
+
+ ui_miri(Mode::Pass, "tests/pass", &target, WithoutDependencies, tmpdir.path())?;
+ ui_miri(Mode::Pass, "tests/pass-dep", &target, WithDependencies, tmpdir.path())?;
+ ui_miri(Mode::Panic, "tests/panic", &target, WithDependencies, tmpdir.path())?;
+ ui_miri(
Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
"tests/fail",
&target,
WithoutDependencies,
tmpdir.path(),
)?;
- ui(
+ ui_miri(
Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
"tests/fail-dep",
&target,
@@ -294,8 +383,8 @@ fn main() -> Result<()> {
tmpdir.path(),
)?;
if cfg!(target_os = "linux") {
- ui(Mode::Pass, "tests/native-lib/pass", &target, WithoutDependencies, tmpdir.path())?;
- ui(
+ ui_miri(Mode::Pass, "tests/native-lib/pass", &target, WithoutDependencies, tmpdir.path())?;
+ ui_miri(
Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
"tests/native-lib/fail",
&target,
```
Some of our pass-dep tests currently do not pass when they are run against real rustc, e.g.
tests/pass-dep/libc/libc-epoll.rs. That's quite strange, but I guess it's not surprising given that we are not even running these tests against real rustc in CI.