From 6acd8159b4321d14e2c52deedc7bf759c5fdb49e Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sat, 3 May 2025 15:53:52 +0300 Subject: [PATCH 1/2] compiletest: Improve diagnostics for line annotation mismatches --- src/tools/compiletest/src/errors.rs | 36 +++++---- src/tools/compiletest/src/header.rs | 2 +- src/tools/compiletest/src/runtest.rs | 79 +++++++++++++------ .../line-annotation-mismatches.rs | 42 ++++++++++ .../line-annotation-mismatches.stderr | 61 ++++++++++++++ 5 files changed, 180 insertions(+), 40 deletions(-) create mode 100644 tests/ui/compiletest-self-test/line-annotation-mismatches.rs create mode 100644 tests/ui/compiletest-self-test/line-annotation-mismatches.stderr diff --git a/src/tools/compiletest/src/errors.rs b/src/tools/compiletest/src/errors.rs index b5a2b7feac9d6..bf03a40dc8eff 100644 --- a/src/tools/compiletest/src/errors.rs +++ b/src/tools/compiletest/src/errors.rs @@ -16,6 +16,8 @@ pub enum ErrorKind { Suggestion, Warning, Raw, + /// Used for better recovery and diagnostics in compiletest. + Unknown, } impl ErrorKind { @@ -31,21 +33,25 @@ impl ErrorKind { /// Either the canonical uppercase string, or some additional versions for compatibility. /// FIXME: consider keeping only the canonical versions here. - pub fn from_user_str(s: &str) -> ErrorKind { - match s { + fn from_user_str(s: &str) -> Option { + Some(match s { "HELP" | "help" => ErrorKind::Help, "ERROR" | "error" => ErrorKind::Error, - // `MONO_ITEM` makes annotations in `codegen-units` tests syntactically correct, - // but those tests never use the error kind later on. - "NOTE" | "note" | "MONO_ITEM" => ErrorKind::Note, + "NOTE" | "note" => ErrorKind::Note, "SUGGESTION" => ErrorKind::Suggestion, "WARN" | "WARNING" | "warn" | "warning" => ErrorKind::Warning, "RAW" => ErrorKind::Raw, - _ => panic!( + _ => return None, + }) + } + + pub fn expect_from_user_str(s: &str) -> ErrorKind { + ErrorKind::from_user_str(s).unwrap_or_else(|| { + panic!( "unexpected diagnostic kind `{s}`, expected \ - `ERROR`, `WARN`, `NOTE`, `HELP` or `SUGGESTION`" - ), - } + `ERROR`, `WARN`, `NOTE`, `HELP`, `SUGGESTION` or `RAW`" + ) + }) } } @@ -58,6 +64,7 @@ impl fmt::Display for ErrorKind { ErrorKind::Suggestion => write!(f, "SUGGESTION"), ErrorKind::Warning => write!(f, "WARN"), ErrorKind::Raw => write!(f, "RAW"), + ErrorKind::Unknown => write!(f, "UNKNOWN"), } } } @@ -75,11 +82,6 @@ pub struct Error { } impl Error { - pub fn render_for_expected(&self) -> String { - use colored::Colorize; - format!("{: <10}line {: >3}: {}", self.kind, self.line_num_str(), self.msg.cyan()) - } - pub fn line_num_str(&self) -> String { self.line_num.map_or("?".to_string(), |line_num| line_num.to_string()) } @@ -168,8 +170,10 @@ fn parse_expected( let rest = line[tag.end()..].trim_start(); let (kind_str, _) = rest.split_once(|c: char| c != '_' && !c.is_ascii_alphabetic()).unwrap_or((rest, "")); - let kind = ErrorKind::from_user_str(kind_str); - let untrimmed_msg = &rest[kind_str.len()..]; + let (kind, untrimmed_msg) = match ErrorKind::from_user_str(kind_str) { + Some(kind) => (kind, &rest[kind_str.len()..]), + None => (ErrorKind::Unknown, rest), + }; let msg = untrimmed_msg.strip_prefix(':').unwrap_or(untrimmed_msg).trim().to_owned(); let line_num_adjust = &captures["adjust"]; diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs index 8bee9caacc949..2b203bb309c63 100644 --- a/src/tools/compiletest/src/header.rs +++ b/src/tools/compiletest/src/header.rs @@ -593,7 +593,7 @@ impl TestProps { config.parse_name_value_directive(ln, DONT_REQUIRE_ANNOTATIONS) { self.dont_require_annotations - .insert(ErrorKind::from_user_str(err_kind.trim())); + .insert(ErrorKind::expect_from_user_str(err_kind.trim())); } }, ); diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 75f24adb70fa5..8fe038eae9c0c 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -704,6 +704,7 @@ impl<'test> TestCx<'test> { .map(|e| Error { msg: self.normalize_output(&e.msg, &[]), ..e }); let mut unexpected = Vec::new(); + let mut unimportant = Vec::new(); let mut found = vec![false; expected_errors.len()]; for actual_error in actual_errors { for pattern in &self.props.error_patterns { @@ -738,14 +739,9 @@ impl<'test> TestCx<'test> { && expected_kinds.contains(&actual_error.kind) && !self.props.dont_require_annotations.contains(&actual_error.kind) { - self.error(&format!( - "{}:{}: unexpected {}: '{}'", - file_name, - actual_error.line_num_str(), - actual_error.kind, - actual_error.msg - )); unexpected.push(actual_error); + } else { + unimportant.push(actual_error); } } } @@ -755,39 +751,77 @@ impl<'test> TestCx<'test> { // anything not yet found is a problem for (index, expected_error) in expected_errors.iter().enumerate() { if !found[index] { - self.error(&format!( - "{}:{}: expected {} not found: {}", - file_name, - expected_error.line_num_str(), - expected_error.kind, - expected_error.msg - )); not_found.push(expected_error); } } if !unexpected.is_empty() || !not_found.is_empty() { self.error(&format!( - "{} unexpected errors found, {} expected errors not found", + "{} unexpected diagnostics reported, {} expected diagnostics not reported", unexpected.len(), not_found.len() )); - println!("status: {}\ncommand: {}\n", proc_res.status, proc_res.cmdline); + let print = |e: &Error| { + println!("{file_name}:{}: {}: {}", e.line_num_str(), e.kind, e.msg.cyan()) + }; + // Fuzzy matching quality: + // - message and line / message and kind - great, suggested + // - only message - good, suggested + // - known line and kind - ok, suggested + // - only known line - meh, but suggested + // - others are not worth suggesting if !unexpected.is_empty() { - println!("{}", "--- unexpected errors (from JSON output) ---".green()); + println!("{}", "--- reported but not expected (from JSON output) ---".green()); for error in &unexpected { - println!("{}", error.render_for_expected()); + print(error); + for candidate in ¬_found { + if error.msg.contains(&candidate.msg) { + let prefix = if candidate.line_num != error.line_num { + "expected on a different line" + } else { + "expected with a different kind" + } + .red(); + print!(" {prefix}: "); + print(candidate); + } else if candidate.line_num.is_some() + && candidate.line_num == error.line_num + { + print!(" {}: ", "expected with a different message".red()); + print(candidate); + } + } } println!("{}", "---".green()); } if !not_found.is_empty() { - println!("{}", "--- not found errors (from test file) ---".red()); + println!("{}", "--- expected but not reported (from test file) ---".red()); for error in ¬_found { - println!("{}", error.render_for_expected()); + print(error); + for candidate in unexpected.iter().chain(&unimportant) { + if candidate.msg.contains(&error.msg) { + let prefix = if candidate.line_num != error.line_num { + "reported on a different line" + } else { + "reported with a different kind" + } + .green(); + print!(" {prefix}: "); + print(candidate); + } else if candidate.line_num.is_some() + && candidate.line_num == error.line_num + { + print!(" {}: ", "reported with a different message".green()); + print(candidate); + } + } } - println!("{}", "---\n".red()); + println!("{}", "---".red()); } - panic!("errors differ from expected"); + panic!( + "errors differ from expected\nstatus: {}\ncommand: {}\n", + proc_res.status, proc_res.cmdline + ); } } @@ -2073,7 +2107,6 @@ impl<'test> TestCx<'test> { println!("{}", String::from_utf8_lossy(&output.stdout)); eprintln!("{}", String::from_utf8_lossy(&output.stderr)); } else { - use colored::Colorize; eprintln!("warning: no pager configured, falling back to unified diff"); eprintln!( "help: try configuring a git pager (e.g. `delta`) with `git config --global core.pager delta`" diff --git a/tests/ui/compiletest-self-test/line-annotation-mismatches.rs b/tests/ui/compiletest-self-test/line-annotation-mismatches.rs new file mode 100644 index 0000000000000..d2a14374ed4c7 --- /dev/null +++ b/tests/ui/compiletest-self-test/line-annotation-mismatches.rs @@ -0,0 +1,42 @@ +//@ should-fail + +// The warning is reported with unknown line +//@ compile-flags: -D raw_pointer_derive +//~? WARN kind and unknown line match the reported warning, but we do not suggest it + +// The error is expected but not reported at all. +//~ ERROR this error does not exist + +// The error is reported but not expected at all. +// "`main` function not found in crate" (the main function is intentionally not added) + +// An "unimportant" diagnostic is expected on a wrong line. +//~ ERROR aborting due to + +// An "unimportant" diagnostic is expected with a wrong kind. +//~? ERROR For more information about an error + +fn wrong_line_or_kind() { + // A diagnostic expected on a wrong line. + unresolved1; + //~ ERROR cannot find value `unresolved1` in this scope + + // A diagnostic expected with a wrong kind. + unresolved2; //~ WARN cannot find value `unresolved2` in this scope + + // A diagnostic expected with a missing kind (treated as a wrong kind). + unresolved3; //~ cannot find value `unresolved3` in this scope + + // A diagnostic expected with a wrong line and kind. + unresolved4; + //~ WARN cannot find value `unresolved4` in this scope +} + +fn wrong_message() { + // A diagnostic expected with a wrong message, but the line is known and right. + unresolvedA; //~ ERROR stub message 1 + + // A diagnostic expected with a wrong message, but the line is known and right, + // even if the kind doesn't match. + unresolvedB; //~ WARN stub message 2 +} diff --git a/tests/ui/compiletest-self-test/line-annotation-mismatches.stderr b/tests/ui/compiletest-self-test/line-annotation-mismatches.stderr new file mode 100644 index 0000000000000..7ca3bfaf396c9 --- /dev/null +++ b/tests/ui/compiletest-self-test/line-annotation-mismatches.stderr @@ -0,0 +1,61 @@ +warning: lint `raw_pointer_derive` has been removed: using derive with raw pointers is ok + | + = note: requested on the command line with `-D raw_pointer_derive` + = note: `#[warn(renamed_and_removed_lints)]` on by default + +error[E0425]: cannot find value `unresolved1` in this scope + --> $DIR/line-annotation-mismatches.rs:21:5 + | +LL | unresolved1; + | ^^^^^^^^^^^ not found in this scope + +error[E0425]: cannot find value `unresolved2` in this scope + --> $DIR/line-annotation-mismatches.rs:25:5 + | +LL | unresolved2; + | ^^^^^^^^^^^ not found in this scope + +error[E0425]: cannot find value `unresolved3` in this scope + --> $DIR/line-annotation-mismatches.rs:28:5 + | +LL | unresolved3; + | ^^^^^^^^^^^ not found in this scope + +error[E0425]: cannot find value `unresolved4` in this scope + --> $DIR/line-annotation-mismatches.rs:31:5 + | +LL | unresolved4; + | ^^^^^^^^^^^ not found in this scope + +error[E0425]: cannot find value `unresolvedA` in this scope + --> $DIR/line-annotation-mismatches.rs:37:5 + | +LL | unresolvedA; + | ^^^^^^^^^^^ not found in this scope + +error[E0425]: cannot find value `unresolvedB` in this scope + --> $DIR/line-annotation-mismatches.rs:41:5 + | +LL | unresolvedB; + | ^^^^^^^^^^^ not found in this scope + +warning: lint `raw_pointer_derive` has been removed: using derive with raw pointers is ok + | + = note: requested on the command line with `-D raw_pointer_derive` + = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no` + +error[E0601]: `main` function not found in crate `line_annotation_mismatches` + --> $DIR/line-annotation-mismatches.rs:42:2 + | +LL | } + | ^ consider adding a `main` function to `$DIR/line-annotation-mismatches.rs` + +warning: lint `raw_pointer_derive` has been removed: using derive with raw pointers is ok + | + = note: requested on the command line with `-D raw_pointer_derive` + = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no` + +error: aborting due to 7 previous errors; 3 warnings emitted + +Some errors have detailed explanations: E0425, E0601. +For more information about an error, try `rustc --explain E0425`. From 776730ec45d59122405aa8cb182fa3e21dcb940f Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Wed, 21 May 2025 20:33:27 +0300 Subject: [PATCH 2/2] WIP --- src/tools/compiletest/src/errors.rs | 10 ++--- src/tools/compiletest/src/json.rs | 40 +++++++------------ src/tools/compiletest/src/runtest.rs | 8 +++- .../line-annotation-mismatches.rs | 2 +- 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/src/tools/compiletest/src/errors.rs b/src/tools/compiletest/src/errors.rs index bf03a40dc8eff..9fa26305f6b0b 100644 --- a/src/tools/compiletest/src/errors.rs +++ b/src/tools/compiletest/src/errors.rs @@ -72,6 +72,7 @@ impl fmt::Display for ErrorKind { #[derive(Debug)] pub struct Error { pub line_num: Option, + pub column_num: Option, /// What kind of message we expect (e.g., warning, error, suggestion). pub kind: ErrorKind, pub msg: String, @@ -81,12 +82,6 @@ pub struct Error { pub require_annotation: bool, } -impl Error { - pub fn line_num_str(&self) -> String { - self.line_num.map_or("?".to_string(), |line_num| line_num.to_string()) - } -} - /// Looks for either "//~| KIND MESSAGE" or "//~^^... KIND MESSAGE" /// The former is a "follow" that inherits its target from the preceding line; /// the latter is an "adjusts" that goes that many lines up. @@ -186,6 +181,7 @@ fn parse_expected( } else { (false, Some(line_num - line_num_adjust.len())) }; + let column_num = Some(tag.start() + 1); debug!( "line={:?} tag={:?} follow_prev={:?} kind={:?} msg={:?}", @@ -195,7 +191,7 @@ fn parse_expected( kind, msg ); - Some((follow_prev, Error { line_num, kind, msg, require_annotation: true })) + Some((follow_prev, Error { line_num, column_num, kind, msg, require_annotation: true })) } #[cfg(test)] diff --git a/src/tools/compiletest/src/json.rs b/src/tools/compiletest/src/json.rs index 6ed2b52c66d21..a8e6416e56c84 100644 --- a/src/tools/compiletest/src/json.rs +++ b/src/tools/compiletest/src/json.rs @@ -36,9 +36,7 @@ struct UnusedExternNotification { struct DiagnosticSpan { file_name: String, line_start: usize, - line_end: usize, column_start: usize, - column_end: usize, is_primary: bool, label: Option, suggested_replacement: Option, @@ -148,6 +146,7 @@ pub fn parse_output(file_name: &str, output: &str) -> Vec { Ok(diagnostic) => push_actual_errors(&mut errors, &diagnostic, &[], file_name), Err(_) => errors.push(Error { line_num: None, + column_num: None, kind: ErrorKind::Raw, msg: line.to_string(), require_annotation: false, @@ -193,25 +192,9 @@ fn push_actual_errors( // also ensure that `//~ ERROR E123` *always* works. The // assumption is that these multi-line error messages are on their // way out anyhow. - let with_code = |span: Option<&DiagnosticSpan>, text: &str| { - // FIXME(#33000) -- it'd be better to use a dedicated - // UI harness than to include the line/col number like - // this, but some current tests rely on it. - // - // Note: Do NOT include the filename. These can easily - // cause false matches where the expected message - // appears in the filename, and hence the message - // changes but the test still passes. - let span_str = match span { - Some(DiagnosticSpan { line_start, column_start, line_end, column_end, .. }) => { - format!("{line_start}:{column_start}: {line_end}:{column_end}") - } - None => format!("?:?: ?:?"), - }; - match &diagnostic.code { - Some(code) => format!("{span_str}: {text} [{}]", code.code), - None => format!("{span_str}: {text}"), - } + let with_code = |text| match &diagnostic.code { + Some(code) => format!("{text} [{}]", code.code), + None => format!("{text}"), }; // Convert multi-line messages into multiple errors. @@ -225,8 +208,9 @@ fn push_actual_errors( || Regex::new(r"aborting due to \d+ previous errors?|\d+ warnings? emitted").unwrap(); errors.push(Error { line_num: None, + column_num: None, kind, - msg: with_code(None, first_line), + msg: with_code(first_line), require_annotation: diagnostic.level != "failure-note" && !RE.get_or_init(re_init).is_match(first_line), }); @@ -234,8 +218,9 @@ fn push_actual_errors( for span in primary_spans { errors.push(Error { line_num: Some(span.line_start), + column_num: Some(span.column_start), kind, - msg: with_code(Some(span), first_line), + msg: with_code(first_line), require_annotation: true, }); } @@ -244,16 +229,18 @@ fn push_actual_errors( if primary_spans.is_empty() { errors.push(Error { line_num: None, + column_num: None, kind, - msg: with_code(None, next_line), + msg: with_code(next_line), require_annotation: false, }); } else { for span in primary_spans { errors.push(Error { line_num: Some(span.line_start), + column_num: Some(span.column_start), kind, - msg: with_code(Some(span), next_line), + msg: with_code(next_line), require_annotation: false, }); } @@ -266,6 +253,7 @@ fn push_actual_errors( for (index, line) in suggested_replacement.lines().enumerate() { errors.push(Error { line_num: Some(span.line_start + index), + column_num: Some(span.column_start), kind: ErrorKind::Suggestion, msg: line.to_string(), // Empty suggestions (suggestions to remove something) are common @@ -288,6 +276,7 @@ fn push_actual_errors( if let Some(label) = &span.label { errors.push(Error { line_num: Some(span.line_start), + column_num: Some(span.column_start), kind: ErrorKind::Note, msg: label.clone(), // Empty labels (only underlining spans) are common and do not need annotations. @@ -310,6 +299,7 @@ fn push_backtrace( if Path::new(&expansion.span.file_name) == Path::new(&file_name) { errors.push(Error { line_num: Some(expansion.span.line_start), + column_num: Some(expansion.span.column_start), kind: ErrorKind::Note, msg: format!("in this expansion of {}", expansion.macro_decl_name), require_annotation: true, diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 8fe038eae9c0c..2369ea6b1cce0 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -762,7 +762,13 @@ impl<'test> TestCx<'test> { not_found.len() )); let print = |e: &Error| { - println!("{file_name}:{}: {}: {}", e.line_num_str(), e.kind, e.msg.cyan()) + let line_num = e.line_num.map_or("?".to_string(), |line_num| line_num.to_string()); + // `file:?:NUM` may be confusing to editors and unclickable. + let opt_col_num = match e.column_num { + Some(col_num) if line_num != "?" => format!("{col_num}:"), + _ => "".to_string(), + }; + println!("{file_name}:{line_num}:{opt_col_num} {}: {}", e.kind, e.msg.cyan()) }; // Fuzzy matching quality: // - message and line / message and kind - great, suggested diff --git a/tests/ui/compiletest-self-test/line-annotation-mismatches.rs b/tests/ui/compiletest-self-test/line-annotation-mismatches.rs index d2a14374ed4c7..fb6d74ac25d98 100644 --- a/tests/ui/compiletest-self-test/line-annotation-mismatches.rs +++ b/tests/ui/compiletest-self-test/line-annotation-mismatches.rs @@ -1,4 +1,4 @@ -//@ should-fail +// should-fail // The warning is reported with unknown line //@ compile-flags: -D raw_pointer_derive