From b2f138fded55c2f6981ae2f1ed8374acf01b656f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Erdo=C4=9Fan?= Date: Sun, 18 Oct 2026 14:24:46 +0300 Subject: [PATCH] fix: read wrapped list items and escapes in Sigma YAML Running the import over the SigmaHQ rules folder found two reader bugs: plain list items continued on a deeper line failed to parse, and double-quoted \u, \x and \U escapes came out as literal letters. Record the measured conversion coverage in the changelog. --- CHANGELOG.md | 8 +++-- src/rules/sigma.rs | 71 +++++++++++++++++++++++++++++----------- src/rules/sigma/tests.rs | 54 ++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5e28c..ac37736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,8 +40,12 @@ changes must be called out explicitly. Tags are immutable. names are ignored), `or` and `1 of` branches become separate rules, and constructs without a content equivalent (`not`, aggregations, `timeframe`, `base64`, `cidr`, null checks) skip the rule with a reason. Levels, status, - ATT&CK tags, references and the log source carry over as metadata. Tested - with hand-written rules; not yet run against the SigmaHQ repository. + ATT&CK tags, references and the log source carry over as metadata. Against + the SigmaHQ `rules` folder (3,155 files, October 2026) 2,425 files convert + into 3,755 Banshee rules (902 of 1,191 in `windows/process_creation`); the + rest are skipped for `not` (597), null or empty values (92), `cidr`, + `base64offset` and `fieldref` modifiers (33), more than 16 branches (6) or + patterns over 32 KiB (2), with no parse errors. - `timeline PATH...` merges timestamps from local files into one sorted timeline (text, JSON or CSV): file modification times, PE build stamps, email Date headers, shortcut target times, event log and capture ranges diff --git a/src/rules/sigma.rs b/src/rules/sigma.rs index 8ba5eb2..a795753 100644 --- a/src/rules/sigma.rs +++ b/src/rules/sigma.rs @@ -102,16 +102,42 @@ fn unquote(value: &str) -> Result { let mut output = String::new(); let mut characters = inner.chars(); while let Some(character) = characters.next() { - if character == '\\' { - match characters.next() { - Some('n') => output.push('\n'), - Some('t') => output.push('\t'), - Some(other) => output.push(other), - None => output.push('\\'), - } - } else { + if character != '\\' { output.push(character); + continue; } + let escaped = match characters.next() { + Some('0') => '\0', + Some('a') => '\u{7}', + Some('b') => '\u{8}', + Some('t' | '\t') => '\t', + Some('n') => '\n', + Some('v') => '\u{b}', + Some('f') => '\u{c}', + Some('r') => '\r', + Some('e') => '\u{1b}', + Some('N') => '\u{85}', + Some('_') => '\u{a0}', + Some('L') => '\u{2028}', + Some('P') => '\u{2029}', + Some(digits @ ('x' | 'u' | 'U')) => { + let length = match digits { + 'x' => 2, + 'u' => 4, + _ => 8, + }; + let hex: String = characters.by_ref().take(length).collect(); + u32::from_str_radix(&hex, 16) + .ok() + .filter(|_| hex.len() == length) + .and_then(char::from_u32) + .ok_or_else(|| anyhow!("invalid escape \\{digits}{hex} in {value}"))? + } + Some(other @ (' ' | '"' | '/' | '\\')) => other, + Some(other) => bail!("invalid escape \\{other} in {value}"), + None => bail!("unterminated escape in {value}"), + }; + output.push(escaped); } return Ok(output); } @@ -220,12 +246,27 @@ impl Parser { items.push(self.map(indent + offset)?); } else { self.position += 1; - items.push(scalar(content)?); + let text = self.continuation(content, indent); + items.push(scalar(&text)?); } } Ok(Yaml::List(items)) } + /// Joins the deeper lines that continue a plain or quoted scalar. + fn continuation(&mut self, first: &str, indent: usize) -> String { + let mut text = first.to_owned(); + while let Some(next) = self.peek() { + if next.indent <= indent { + break; + } + text.push(' '); + text.push_str(&next.text); + self.position += 1; + } + text + } + fn map(&mut self, indent: usize) -> Result { let mut entries = Vec::new(); while let Some(line) = self.peek().cloned() { @@ -261,17 +302,7 @@ impl Parser { _ => Yaml::Null, } } else { - // Plain and quoted scalars may continue on deeper lines. - let mut text = value.to_owned(); - while let Some(next) = self.peek() { - if next.indent <= indent { - break; - } - text.push(' '); - text.push_str(&next.text); - self.position += 1; - } - scalar(&text)? + scalar(&self.continuation(value, indent))? }; entries.push((key, value)); } diff --git a/src/rules/sigma/tests.rs b/src/rules/sigma/tests.rs index e11ad53..7a82bd2 100644 --- a/src/rules/sigma/tests.rs +++ b/src/rules/sigma/tests.rs @@ -209,3 +209,57 @@ fn yaml_reader_handles_block_scalars_quotes_and_comments() { ); assert!(import_source("title: x\n\tdetection: y\n").is_err()); } + +#[test] +fn yaml_reader_decodes_double_quoted_unicode_escapes() { + let source = "title: Homoglyphs\n\ + detection:\n\ + \x20 selection:\n\ + \x20 Field|contains:\n\ + \x20 - \"\\u0410\" # Cyrillic A\n\ + \x20 - \"tab\\x09\\U0001F600\"\n\ + \x20 condition: selection\n"; + let conversion = import_source(source).expect("import"); + assert_eq!(conversion.converted, 1, "{:?}", conversion.skipped); + for (input, matched) in [("p\u{0430}ypal", 1), ("tab\t\u{1F600}", 1), ("u0410", 0)] { + assert_eq!( + scan_source(&conversion.output, input.as_bytes()) + .unwrap() + .matched_rules, + matched, + "{input:?}" + ); + } + assert!(import_source("title: \"bad \\q escape\"\n").is_err()); + assert!(import_source("title: \"short \\u04\"\n").is_err()); +} + +#[test] +fn yaml_reader_joins_list_items_wrapped_onto_deeper_lines() { + let source = "title: Wrapped List Item\n\ + detection:\n\ + \x20 selection:\n\ + \x20 - first value\n\ + \x20 continued here\n\ + \x20 - 'quoted value\n\ + \x20 also wrapped'\n\ + \x20 condition: selection\n\ + falsepositives:\n\ + \x20 - Admins who are part of the group.\n\ + \x20 These users shouldn't be flagged.\n\ + level: low\n"; + let conversion = import_source(source).expect("import"); + assert_eq!(conversion.converted, 1, "{:?}", conversion.skipped); + for (input, matched) in [ + (&b"first value continued here"[..], 1), + (b"quoted value also wrapped", 1), + (b"first value", 0), + ] { + assert_eq!( + scan_source(&conversion.output, input) + .unwrap() + .matched_rules, + matched + ); + } +}