From 4771618573f233d04ff9d30964565cf27a095b61 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 1 Sep 2026 11:35:29 -0700 Subject: [PATCH 01/10] Add some test cases These were identified by matheww in https://github.com/rust-lang/reference/pull/2325#issuecomment-5309031418 and we should have captured them when they were fixed. --- tools/grammar-check/src/test_cases.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/grammar-check/src/test_cases.rs b/tools/grammar-check/src/test_cases.rs index da3a13e19c..ef950859c3 100644 --- a/tools/grammar-check/src/test_cases.rs +++ b/tools/grammar-check/src/test_cases.rs @@ -41,6 +41,8 @@ cases! { "/// ☃" comment::outer_block_doc => "/** outer block doc */" + comment::cr_starting_block_doc => + "/**\r CR starting block doc comment */" reserved::pounds => "##" @@ -54,6 +56,9 @@ cases! { "'x'" string => "\"string\"" + string::continuation::bare_carriage => + "\"string\\\n\n\r\tcontinuation\"" + raw_string => "r\"raw string\"" "r#\"raw string\"#" @@ -81,4 +86,7 @@ cases! { identifier => "ident" "fn" + + shebang::doc_comment => + "#! /** doc */ [attr]\n" } From a2f45cf4845f769ca1155c09ac86e60c24551130 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 1 Sep 2026 12:10:12 -0700 Subject: [PATCH 02/10] Make it easier to see CR output in test errors When the grammar check fails, it prints the input on the screen. However, when there are embedded CRs, it disrupts the output. This replaces it with the Unicode character for CR Symbol. --- tools/grammar-check/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/grammar-check/src/main.rs b/tools/grammar-check/src/main.rs index 12512d26dd..ea571d59e3 100644 --- a/tools/grammar-check/src/main.rs +++ b/tools/grammar-check/src/main.rs @@ -419,6 +419,7 @@ fn translate_position(input: &str, index: usize) -> (&str, usize, usize) { fn display_line(src: &str, range: &Range) -> String { let (line, line_no, col_no) = translate_position(src, range.start); + let line = line.replace('\r', "␍"); let prefix = format!("{line_no}: "); let indent = col_no.saturating_sub(1); let len = (range.end - range.start).min(line.len().saturating_sub(indent)); From 721395442167d284fbdf4c993f21cf7ce1e51e52 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 1 Sep 2026 12:10:44 -0700 Subject: [PATCH 03/10] Add some tests for nested CR in comments These are some problems identified in https://github.com/rust-lang/reference/issues/2333. --- tools/grammar-check/src/test_cases.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/grammar-check/src/test_cases.rs b/tools/grammar-check/src/test_cases.rs index ef950859c3..6d7a298e0e 100644 --- a/tools/grammar-check/src/test_cases.rs +++ b/tools/grammar-check/src/test_cases.rs @@ -44,6 +44,19 @@ cases! { comment::cr_starting_block_doc => "/**\r CR starting block doc comment */" + comment::block::nested_cr1 => + "/* /**\r*/ */" + comment::block::nested_cr2 => + "/* /*!\r*/ */" + comment::block::nested_cr3 => + "/* /** x\r y */ */" + comment::block::nested_cr4 => + "/** /*\r*/ */" + comment::block::nested_cr5 => + "/*! /*\r*/ */" + comment::block::nested_cr6 => + "/** /* x\r y */ */" + reserved::pounds => "##" "###" From 1c79291c5979e5e5dd234e3e79088d24ab769129 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 1 Sep 2026 12:38:28 -0700 Subject: [PATCH 04/10] Add separate BLOCK_CHAR and DOC_BLOCK_CHAR productions This adds a BLOCK_CHAR rule so that both block comments and doc-block comments match each other. This is just done for aesthetics. --- src/comments.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/comments.md b/src/comments.md index 7102bf5825..5ff508532b 100644 --- a/src/comments.md +++ b/src/comments.md @@ -18,7 +18,7 @@ LINE_COMMENT -> BLOCK_COMMENT -> `/*` !(`!` | `*` ![`*` `/`]) ^ - ( BLOCK_COMMENT_OR_DOC | (!`*/` CHAR) )* + ( BLOCK_COMMENT_OR_DOC | BLOCK_CHAR )* `*/` INNER_LINE_DOC -> @@ -27,7 +27,7 @@ INNER_LINE_DOC -> LINE_DOC_COMMENT_CONTENT -> (!CR ~LF)* INNER_BLOCK_DOC -> - `/*!` ^ ( BLOCK_COMMENT_OR_DOC | BLOCK_CHAR )* `*/` + `/*!` ^ ( BLOCK_COMMENT_OR_DOC | DOC_BLOCK_CHAR )* `*/` OUTER_LINE_DOC -> `///` ^ LINE_DOC_COMMENT_CONTENT (LF | EOF) @@ -36,10 +36,12 @@ OUTER_BLOCK_DOC -> `/**` ![`*` `/`] ^ ( ~[`*` CR] | BLOCK_COMMENT_OR_DOC ) - ( BLOCK_COMMENT_OR_DOC | BLOCK_CHAR )* + ( BLOCK_COMMENT_OR_DOC | DOC_BLOCK_CHAR )* `*/` -BLOCK_CHAR -> (!(`*/` | CR) CHAR) +BLOCK_CHAR -> !`*/` CHAR + +DOC_BLOCK_CHAR -> (!(`*/` | CR) CHAR) BLOCK_COMMENT_OR_DOC -> INNER_BLOCK_DOC From 54298a58f6db5c78c8ddf526a6be610e04161b33 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 1 Sep 2026 12:42:12 -0700 Subject: [PATCH 05/10] Revert BLOCK_COMMENT change for shebang support This reverts https://github.com/rust-lang/reference/commit/5b328bcf7e5e56c5cd3c660655348c2d85dea5cf which is no longer needed due to https://github.com/rust-lang/reference/pull/2331 changing shebang to use its own dedicated comment rule. --- src/comments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comments.md b/src/comments.md index 5ff508532b..aae4bde9e6 100644 --- a/src/comments.md +++ b/src/comments.md @@ -17,7 +17,7 @@ LINE_COMMENT -> | `//` _immediately followed by LF_ BLOCK_COMMENT -> - `/*` !(`!` | `*` ![`*` `/`]) ^ + `/*` ^ ( BLOCK_COMMENT_OR_DOC | BLOCK_CHAR )* `*/` From 0078db3f45a9748b60f184d59f29a327174efade Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 1 Sep 2026 12:47:03 -0700 Subject: [PATCH 06/10] Fix nested comments with respect to carriage returns This fixes some issues with nested block comments when they contain carriage returns. The old logic was that a block comment could contain nested block comments of the form of either regular block comments or doc-block comments. This wasn't correct, though, because when inter-mixing regular block comments and doc-block comments, the rules are different with respect to things like carriage returns. However, in reality, when rustc encounters a regular block comment, it just eats everything that looks block-like without any validation. And conversely, for a doc-block comment, it requires that everything inside does not contain a carriage return, even if it is a nested regular block comment. The logic for how nested comments are handled is at https://github.com/rust-lang/rust/blob/a4330234a776684c36428d001721d0320d24dd77/compiler/rustc_lexer/src/lib.rs#L783-L815. --- src/comments.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/comments.md b/src/comments.md index aae4bde9e6..1e50d5cba1 100644 --- a/src/comments.md +++ b/src/comments.md @@ -18,7 +18,7 @@ LINE_COMMENT -> BLOCK_COMMENT -> `/*` ^ - ( BLOCK_COMMENT_OR_DOC | BLOCK_CHAR )* + ( BLOCK_COMMENT | BLOCK_CHAR )* `*/` INNER_LINE_DOC -> @@ -27,7 +27,7 @@ INNER_LINE_DOC -> LINE_DOC_COMMENT_CONTENT -> (!CR ~LF)* INNER_BLOCK_DOC -> - `/*!` ^ ( BLOCK_COMMENT_OR_DOC | DOC_BLOCK_CHAR )* `*/` + `/*!` ^ ( NESTED_BLOCK_DOC_COMMENT | DOC_BLOCK_CHAR )* `*/` OUTER_LINE_DOC -> `///` ^ LINE_DOC_COMMENT_CONTENT (LF | EOF) @@ -35,18 +35,16 @@ OUTER_LINE_DOC -> OUTER_BLOCK_DOC -> `/**` ![`*` `/`] ^ - ( ~[`*` CR] | BLOCK_COMMENT_OR_DOC ) - ( BLOCK_COMMENT_OR_DOC | DOC_BLOCK_CHAR )* + ( ~[`*` CR] | NESTED_BLOCK_DOC_COMMENT ) + ( NESTED_BLOCK_DOC_COMMENT | DOC_BLOCK_CHAR )* `*/` BLOCK_CHAR -> !`*/` CHAR DOC_BLOCK_CHAR -> (!(`*/` | CR) CHAR) -BLOCK_COMMENT_OR_DOC -> - INNER_BLOCK_DOC - | OUTER_BLOCK_DOC - | BLOCK_COMMENT +NESTED_BLOCK_DOC_COMMENT -> + `/*` (NESTED_BLOCK_DOC_COMMENT | DOC_BLOCK_CHAR)* `*/` ``` r[comments.normal] From 231f77165d94c827408d2840c0a96233dcd2e574 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 1 Sep 2026 12:49:56 -0700 Subject: [PATCH 07/10] Also include the hard-coded test cases for grammar check This includes the tests from the `cases` module. Eventually this should be changed in some way. Either these tests should be moved to rust-lang/rust, or there should be separate files that contain the tests. For now it doesn't hurt to run some of these that don't show up in either the permutation or the rust-lang/rust tests. --- .github/workflows/daily-grammar-check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/daily-grammar-check.yml b/.github/workflows/daily-grammar-check.yml index 7cf4417528..8b90c2e53b 100644 --- a/.github/workflows/daily-grammar-check.yml +++ b/.github/workflows/daily-grammar-check.yml @@ -41,6 +41,7 @@ jobs: cargo run --release -p grammar-check -- lex-compare --path rust cargo run --release -p grammar-check -- lex-compare --permute Token --tool rustc_parse cargo run --release -p grammar-check -- lex-compare --permute three + cargo run --release -p grammar-check -- lex-compare --tool rustc_parse - name: Check for existing open issues if: steps.grammar-check.outcome == 'failure' From 1ca96671f17201b3f2fd4d81d4eddab414d6f8ea Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Tue, 1 Sep 2026 20:27:57 +0000 Subject: [PATCH 08/10] Fix spacing in `NESTED_BLOCK_DOC_COMMENT` rule In the grammar rules, we put spaces between parentheses and what comes inside; let's do that here. --- src/comments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comments.md b/src/comments.md index 1e50d5cba1..bf09389838 100644 --- a/src/comments.md +++ b/src/comments.md @@ -44,7 +44,7 @@ BLOCK_CHAR -> !`*/` CHAR DOC_BLOCK_CHAR -> (!(`*/` | CR) CHAR) NESTED_BLOCK_DOC_COMMENT -> - `/*` (NESTED_BLOCK_DOC_COMMENT | DOC_BLOCK_CHAR)* `*/` + `/*` ( NESTED_BLOCK_DOC_COMMENT | DOC_BLOCK_CHAR )* `*/` ``` r[comments.normal] From 6cdb81d7c9cc18fea6628e189acfb83cc36f3d1c Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Tue, 1 Sep 2026 20:28:47 +0000 Subject: [PATCH 09/10] Add test cases for deeper nesting and inner doc CRs Before the fix in this branch, our grammar disagreed with rustc on comments nested three layers deep. Let's add tests for that and for an inner block doc variant of the CR-starting case. --- tools/grammar-check/src/test_cases.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/grammar-check/src/test_cases.rs b/tools/grammar-check/src/test_cases.rs index 6d7a298e0e..7b7b7f6606 100644 --- a/tools/grammar-check/src/test_cases.rs +++ b/tools/grammar-check/src/test_cases.rs @@ -43,6 +43,8 @@ cases! { "/** outer block doc */" comment::cr_starting_block_doc => "/**\r CR starting block doc comment */" + comment::cr_starting_inner_block_doc => + "/*!\r CR starting inner block doc comment */" comment::block::nested_cr1 => "/* /**\r*/ */" @@ -56,6 +58,10 @@ cases! { "/*! /*\r*/ */" comment::block::nested_cr6 => "/** /* x\r y */ */" + comment::block::nested_cr7 => + "/** /* /*\r*/ */ */" + comment::block::nested_cr8 => + "/* /* /**\r*/ */ */" reserved::pounds => "##" From c49c5600a4403b070c99ea6806a3d1fe9d08e123 Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Tue, 1 Sep 2026 20:29:49 +0000 Subject: [PATCH 10/10] Remove unreachable alternative in `OUTER_BLOCK_DOC` The only way to match the `NESTED_BLOCK_DOC_COMMENT` rule is for the string to start with `/`, but we've already ruled that out with a negative lookahead, making this alternative dead. Let's remove it. --- src/comments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comments.md b/src/comments.md index bf09389838..44ef06fc33 100644 --- a/src/comments.md +++ b/src/comments.md @@ -35,7 +35,7 @@ OUTER_LINE_DOC -> OUTER_BLOCK_DOC -> `/**` ![`*` `/`] ^ - ( ~[`*` CR] | NESTED_BLOCK_DOC_COMMENT ) + ~[`*` CR] ( NESTED_BLOCK_DOC_COMMENT | DOC_BLOCK_CHAR )* `*/`