From 91c7786cea38a80654a1ccc99b7bf6fb269337e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Stefa=C5=84ski?= Date: Sat, 23 May 2026 15:48:34 +0200 Subject: [PATCH] Logos lexer --- Cargo.lock | 33 ++++++++ libsvld/Cargo.toml | 1 + libsvld/src/asc.rs | 186 +++++++++++++++++++++++++++------------------ libsvld/src/lib.rs | 46 ++++++----- 4 files changed, 172 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b96e826..ee00941 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2728,6 +2728,7 @@ dependencies = [ "crossbeam-channel", "dbc-codegen", "hashbrown 0.17.1", + "logos", "memchr", "memmap2", "nom 8.0.0", @@ -2798,6 +2799,38 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "logos" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" +dependencies = [ + "fnv", + "proc-macro2", + "quote", + "regex-automata", + "regex-syntax", + "syn 2.0.117", +] + +[[package]] +name = "logos-derive" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" +dependencies = [ + "logos-codegen", +] + [[package]] name = "loop9" version = "0.1.5" diff --git a/libsvld/Cargo.toml b/libsvld/Cargo.toml index dc7b217..684cce8 100644 --- a/libsvld/Cargo.toml +++ b/libsvld/Cargo.toml @@ -25,3 +25,4 @@ atoi_simd = "0.18.1" voracious_radix_sort = { version = "1.2.0", features = ["voracious_multithread"] } rayon = "1.12.0" chrono = { version = "0.4", default-features = false, features = ["std"] } +logos = "0.16.1" diff --git a/libsvld/src/asc.rs b/libsvld/src/asc.rs index d3390f5..92aad81 100644 --- a/libsvld/src/asc.rs +++ b/libsvld/src/asc.rs @@ -1,84 +1,120 @@ use crate::can_frame::{CanFrame, Direction}; -#[inline] -fn next_token<'a>(input: &mut &'a [u8]) -> Option<&'a [u8]> { - *input = input.trim_ascii_start(); - if input.is_empty() { - return None; - } +use logos::{Lexer, Logos}; - // Stop on space, tab, or CR. Lines end with CRLF, so stopping on CR keeps - // the cursor on the line; the leading \r\n is then consumed by the next - // call's trim_ascii_start — no need to ever search for '\n'. - let end = memchr::memchr3(b' ', b'\t', b'\r', input).unwrap_or(input.len()); - let token = &input[..end]; - *input = &input[end..]; - Some(token) -} - -pub enum AscLineType { - Frame(CanFrame), - Errorframe, -} - -/// Parse an ASC header date line of the form: -/// `date Sun May 3 08:14:59 pm 2026` -/// -/// Returns a naive (timezone-less) datetime. Format it as ISO 8601 with -/// `dt.format("%Y-%m-%dT%H:%M:%S").to_string()`. pub fn parse_asc_date(input: &[u8]) -> Option { let s = std::str::from_utf8(input).ok()?.trim(); chrono::NaiveDateTime::parse_from_str(s, "date %a %b %e %I:%M:%S %P %Y").ok() } -/// Parses one ASC line starting at `input[0]`. -/// -/// Returns the parsed line type and the number of bytes consumed. On success -/// the consumed count leaves `input[consumed..]` pointing at the trailing -/// `\r\n`, which the next call's `trim_ascii_start` will eat. -/// -/// `None` means the line could not be parsed (header line, truncated tail, -/// corrupt frame). The caller is responsible for advancing past the bad line. -pub fn parse_asc_line(input: &[u8]) -> Option<(AscLineType, usize)> { - let mut cursor = input; - - let timestamp: f64 = { - let tok = next_token(&mut cursor)?; - unsafe { std::str::from_utf8_unchecked(tok) }.parse().ok()? - }; - let channel = atoi_simd::parse_pos::(next_token(&mut cursor)?).ok()?; - - let tok = next_token(&mut cursor)?; - if tok[0] == b'E' { - return Some((AscLineType::Errorframe, input.len() - cursor.len())); - } - let can_id = atoi_simd::parse_pos::(tok).ok()?; - - let direction = match next_token(&mut cursor)? { - b"Rx" => Direction::Rx, - b"Tx" => Direction::Tx, - _ => return None, - }; - - // 'D' frame type - next_token(&mut cursor)?; - - let dlc = atoi_simd::parse_pos::(next_token(&mut cursor)?).ok()?; - - let mut data = [0u8; 8]; - for byte in data.iter_mut().take(dlc as usize) { - *byte = atoi_simd::parse_pos::(next_token(&mut cursor)?).ok()?; - } - - Some(( - AscLineType::Frame(CanFrame { - timestamp, - channel, - can_id, - direction, - dlc, - data, - }), - input.len() - cursor.len(), - )) +fn parse_float(lex: &mut Lexer) -> Option { + unsafe { + std::str::from_utf8_unchecked(lex.slice()) + }.parse().ok() +} + +// fn parse_int8(lex: &mut Lexer) -> Option { +// atoi_simd::parse_pos::(lex.slice()).ok() +// } + +fn parse_int32(lex: &mut Lexer) -> Option { + atoi_simd::parse_pos::(lex.slice()).ok() +} + +#[derive(Logos, Debug)] +#[logos(skip r"[ \t\n\f]+")] +#[logos(utf8 = false)] +enum Token { + #[regex(r"\d+\.\d+", parse_float)] + Float(f64), + // #[regex(r"\d+", parse_int8)] + // Int8(u8), + #[regex(r"\d+", parse_int32)] + Int32(u32), + #[token(b"Rx")] + Rx, + #[token(b"D")] + D, + #[token(b"Errorframe ECC:")] + Errorframe, +} + +#[derive(Debug)] +pub struct ErrorFrame { + timestamp: f64, + channel: u8, + data: u8, +} + +#[derive(Debug)] +pub enum AscLine { + ErrorFrame(ErrorFrame), + CanFrame(CanFrame), +} + +// 0.001029 1 595 Rx D 3 232 242 45 +// 0.643971 2 Errorframe ECC: 10100010 +pub fn parse_asc_line(line: &[u8]) -> Option<(AscLine, usize)> { + let mut lex = Token::lexer(line); + + let timestamp = lex.next()?.ok()?; + let channel = lex.next()?.ok()?; + let canid_or_error = lex.next()?.ok()?; + + match canid_or_error { + Token::Int32(can_id) => { + let _ = lex.next()?.ok()?; + let _ = lex.next()?.ok()?; + let dlc = lex.next()?.ok()?; + let mut data = [0u8; 8]; + if let Token::Float(timestamp) = timestamp && + let Token::Int32(channel) = channel && + let Token::Int32(dlc) = dlc { + let channel = channel as u8; + let dlc = dlc as u8; + for i in 0..(dlc as usize) { + data[i] = match lex.next()?.ok()? { + Token::Int32(val) => val as u8, + _ => panic!() + }; + } + + Some(( + AscLine::CanFrame( + CanFrame {timestamp, channel, can_id, direction: Direction::Rx, dlc, data }), + lex.span().end + )) + } else { + None + } + }, + Token::Errorframe => { + let err_byte = lex.next()?.ok()?; + if let Token::Float(timestamp) = timestamp && + let Token::Int32(channel) = channel && + let Token::Int32(err_byte) = err_byte { + let channel = channel as u8; + let data = binary_decimal_to_u8(err_byte); + Some(( + AscLine::ErrorFrame( + ErrorFrame {timestamp, channel, data}), + lex.span().end + )) + } else { + None + } + }, + _ => panic!("Expected CanId or Errorframe") + } +} + +fn binary_decimal_to_u8(mut n: u32) -> u8 { + let mut result = 0u8; + let mut bit = 0; + while n > 0 { + result |= ((n % 10) as u8) << bit; + n /= 10; + bit += 1; + } + result } diff --git a/libsvld/src/lib.rs b/libsvld/src/lib.rs index d1c6246..029380f 100644 --- a/libsvld/src/lib.rs +++ b/libsvld/src/lib.rs @@ -10,6 +10,7 @@ use std::{ use can_dbc::Dbc; use hashbrown::HashMap; +use memchr::memchr_iter; #[cfg(unix)] use memmap2::Advice; use memmap2::Mmap; @@ -106,29 +107,36 @@ fn frame_worker( let mut line_start = 0usize; while line_start < mmap.len() { - match parse_asc_line(&mmap[line_start..]) { - Some((asc::AscLineType::Frame(frame), consumed)) => { - match channel_data.get_mut(&frame.can_id) { - Some(vec) => vec.push(frame), - None => { - unsafe { - channel_data.insert_unique_unchecked(frame.can_id, vec![frame]); + if let Some((val, _read)) = parse_asc_line(&mmap[line_start..]) { + match val { + asc::AscLine::CanFrame(frame) => { + match channel_data.get_mut(&frame.can_id) { + Some(vec) => vec.push(frame), + None => { + unsafe { + channel_data.insert_unique_unchecked(frame.can_id, vec![frame]); + } + add_metadata(frame.can_id); } - add_metadata(frame.can_id); } - } - line_start += consumed; + }, + asc::AscLine::ErrorFrame(frame) => { + eprintln!("Got Errorframe: {:?}", frame); + }, } - Some((asc::AscLineType::Errorframe, consumed)) => { - line_start += consumed; + match memchr::memchr(b'\n', &mmap[line_start..]) { + Some(val) => line_start += val + 1, + None => break, } - None => { - // Header lines (chunk 0), corrupt frames, or a truncated tail. - // Walk to the next '\n' by hand — no memchr. - while line_start < mmap.len() && mmap[line_start] != b'\n' { - line_start += 1; - } - line_start += 1; + } else { + let next = memchr::memchr(b'\n', &mmap[line_start..]); + match next { + Some(val) => { + eprintln!("Couldn't parse line: '{}'", + std::str::from_utf8(&mmap[line_start..(line_start + val)]).unwrap()); + line_start += val + 1; + }, + None => break, } } }