Logos lexer

Cargo.lock

@@ -2728,6 +2728,7 @@ dependencies = [
  "crossbeam-channel",
  "dbc-codegen",
  "hashbrown 0.17.1",
+ "logos",
  "memchr",
  "memmap2",
  "nom 8.0.0",
@@ -2798,6 +2799,38 @@ version = "0.4.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 
+[[package]]
+name = "logos"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f"
+dependencies = [
+ "logos-derive",
+]
+
+[[package]]
+name = "logos-codegen"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970"
+dependencies = [
+ "fnv",
+ "proc-macro2",
+ "quote",
+ "regex-automata",
+ "regex-syntax",
+ "syn 2.0.117",
+]
+
+[[package]]
+name = "logos-derive"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31"
+dependencies = [
+ "logos-codegen",
+]
+
 [[package]]
 name = "loop9"
 version = "0.1.5"

libsvld/Cargo.toml

@@ -25,3 +25,4 @@ atoi_simd = "0.18.1"
 voracious_radix_sort = { version = "1.2.0", features = ["voracious_multithread"] }
 rayon = "1.12.0"
 chrono = { version = "0.4", default-features = false, features = ["std"] }
+logos = "0.16.1"

libsvld/src/asc.rs

@@ -1,84 +1,120 @@
 use crate::can_frame::{CanFrame, Direction};
 
-#[inline]
+use logos::{Lexer, Logos};
-fn next_token<'a>(input: &mut &'a [u8]) -> Option<&'a [u8]> {
-    *input = input.trim_ascii_start();
-    if input.is_empty() {
-        return None;
-    }
 
-    // Stop on space, tab, or CR. Lines end with CRLF, so stopping on CR keeps
-    // the cursor on the line; the leading \r\n is then consumed by the next
-    // call's trim_ascii_start — no need to ever search for '\n'.
-    let end = memchr::memchr3(b' ', b'\t', b'\r', input).unwrap_or(input.len());
-    let token = &input[..end];
-    *input = &input[end..];
-    Some(token)
-}
-
-pub enum AscLineType {
-    Frame(CanFrame),
-    Errorframe,
-}
-
-/// Parse an ASC header date line of the form:
-/// `date Sun May 3 08:14:59 pm 2026`
-///
-/// Returns a naive (timezone-less) datetime. Format it as ISO 8601 with
-/// `dt.format("%Y-%m-%dT%H:%M:%S").to_string()`.
 pub fn parse_asc_date(input: &[u8]) -> Option<chrono::NaiveDateTime> {
     let s = std::str::from_utf8(input).ok()?.trim();
     chrono::NaiveDateTime::parse_from_str(s, "date %a %b %e %I:%M:%S %P %Y").ok()
 }
 
-/// Parses one ASC line starting at `input[0]`.
+fn parse_float(lex: &mut Lexer<Token>) -> Option<f64> {
-///
+    unsafe {
-/// Returns the parsed line type and the number of bytes consumed. On success
+        std::str::from_utf8_unchecked(lex.slice())
-/// the consumed count leaves `input[consumed..]` pointing at the trailing
+    }.parse().ok()
-/// `\r\n`, which the next call's `trim_ascii_start` will eat.
+}
-///
+
-/// `None` means the line could not be parsed (header line, truncated tail,
+// fn parse_int8(lex: &mut Lexer<Token>) -> Option<u8> {
-/// corrupt frame). The caller is responsible for advancing past the bad line.
+//     atoi_simd::parse_pos::<u8, false>(lex.slice()).ok()
-pub fn parse_asc_line(input: &[u8]) -> Option<(AscLineType, usize)> {
+// }
-    let mut cursor = input;
+
-
+fn parse_int32(lex: &mut Lexer<Token>) -> Option<u32> {
-    let timestamp: f64 = {
+    atoi_simd::parse_pos::<u32, false>(lex.slice()).ok()
-        let tok = next_token(&mut cursor)?;
+}
-        unsafe { std::str::from_utf8_unchecked(tok) }.parse().ok()?
+
-    };
+#[derive(Logos, Debug)]
-    let channel = atoi_simd::parse_pos::<u8, false>(next_token(&mut cursor)?).ok()?;
+#[logos(skip r"[ \t\n\f]+")]
-
+#[logos(utf8 = false)]
-    let tok = next_token(&mut cursor)?;
+enum Token {
-    if tok[0] == b'E' {
+    #[regex(r"\d+\.\d+", parse_float)]
-        return Some((AscLineType::Errorframe, input.len() - cursor.len()));
+    Float(f64),
-    }
+    // #[regex(r"\d+", parse_int8)]
-    let can_id = atoi_simd::parse_pos::<u32, false>(tok).ok()?;
+    // Int8(u8),
-
+    #[regex(r"\d+", parse_int32)]
-    let direction = match next_token(&mut cursor)? {
+    Int32(u32),
-        b"Rx" => Direction::Rx,
+    #[token(b"Rx")]
-        b"Tx" => Direction::Tx,
+    Rx,
-        _ => return None,
+    #[token(b"D")]
-    };
+    D,
-
+    #[token(b"Errorframe ECC:")]
-    // 'D' frame type
+    Errorframe,
-    next_token(&mut cursor)?;
+}
-
+
-    let dlc = atoi_simd::parse_pos::<u8, false>(next_token(&mut cursor)?).ok()?;
+#[derive(Debug)]
-
+pub struct ErrorFrame {
-    let mut data = [0u8; 8];
+    timestamp: f64,
-    for byte in data.iter_mut().take(dlc as usize) {
+    channel: u8,
-        *byte = atoi_simd::parse_pos::<u8, false>(next_token(&mut cursor)?).ok()?;
+    data: u8,
-    }
+}
-
+
-    Some((
+#[derive(Debug)]
-        AscLineType::Frame(CanFrame {
+pub enum AscLine {
-            timestamp,
+    ErrorFrame(ErrorFrame),
-            channel,
+    CanFrame(CanFrame),
-            can_id,
+}
-            direction,
+
-            dlc,
+// 0.001029 1  595        Rx D 3 232 242  45
-            data,
+// 0.643971 2  Errorframe ECC: 10100010
-        }),
+pub fn parse_asc_line(line: &[u8]) -> Option<(AscLine, usize)> {
-        input.len() - cursor.len(),
+    let mut lex = Token::lexer(line);
-    ))
+
+    let timestamp = lex.next()?.ok()?;
+    let channel = lex.next()?.ok()?;
+    let canid_or_error = lex.next()?.ok()?;
+
+    match canid_or_error {
+        Token::Int32(can_id) => {
+            let _ = lex.next()?.ok()?;
+            let _ = lex.next()?.ok()?;
+            let dlc = lex.next()?.ok()?;
+            let mut data = [0u8; 8];
+            if let Token::Float(timestamp) = timestamp &&
+                let Token::Int32(channel) = channel &&
+                let Token::Int32(dlc) = dlc {
+                    let channel = channel as u8;
+                    let dlc = dlc as u8;
+                    for i in 0..(dlc as usize) {
+                        data[i] = match lex.next()?.ok()? {
+                            Token::Int32(val) => val as u8,
+                            _ => panic!()
+                        };
+                    }
+
+                    Some((
+                        AscLine::CanFrame(
+                            CanFrame {timestamp, channel, can_id, direction: Direction::Rx, dlc, data }),
+                        lex.span().end
+                    ))
+                } else {
+                    None
+                }
+        },
+        Token::Errorframe => {
+            let err_byte = lex.next()?.ok()?;
+            if let Token::Float(timestamp) = timestamp &&
+                let Token::Int32(channel) = channel &&
+                let Token::Int32(err_byte) = err_byte {
+                    let channel = channel as u8;
+                    let data = binary_decimal_to_u8(err_byte);
+                    Some((
+                        AscLine::ErrorFrame(
+                            ErrorFrame {timestamp, channel, data}),
+                        lex.span().end
+                    ))
+                } else {
+                    None
+                }
+        },
+        _ => panic!("Expected CanId or Errorframe")
+    }
+}
+
+fn binary_decimal_to_u8(mut n: u32) -> u8 {
+    let mut result = 0u8;
+    let mut bit = 0;
+    while n > 0 {
+        result |= ((n % 10) as u8) << bit;
+        n /= 10;
+        bit += 1;
+    }
+    result
 }

libsvld/src/lib.rs

@@ -10,6 +10,7 @@ use std::{
 
 use can_dbc::Dbc;
 use hashbrown::HashMap;
+use memchr::memchr_iter;
 #[cfg(unix)]
 use memmap2::Advice;
 use memmap2::Mmap;
@@ -106,29 +107,36 @@ fn frame_worker(
     let mut line_start = 0usize;
 
     while line_start < mmap.len() {
-        match parse_asc_line(&mmap[line_start..]) {
+        if let Some((val, _read)) = parse_asc_line(&mmap[line_start..]) {
-            Some((asc::AscLineType::Frame(frame), consumed)) => {
+            match val {
-                match channel_data.get_mut(&frame.can_id) {
+                asc::AscLine::CanFrame(frame) => {
-                    Some(vec) => vec.push(frame),
+                    match channel_data.get_mut(&frame.can_id) {
-                    None => {
+                        Some(vec) => vec.push(frame),
-                        unsafe {
+                        None => {
-                            channel_data.insert_unique_unchecked(frame.can_id, vec![frame]);
+                            unsafe {
+                                channel_data.insert_unique_unchecked(frame.can_id, vec![frame]);
+                            }
+                            add_metadata(frame.can_id);
                         }
-                        add_metadata(frame.can_id);
                     }
-                }
+                },
-                line_start += consumed;
+                asc::AscLine::ErrorFrame(frame) => {
+                    eprintln!("Got Errorframe: {:?}", frame);
+                },
             }
-            Some((asc::AscLineType::Errorframe, consumed)) => {
+            match memchr::memchr(b'\n', &mmap[line_start..]) {
-                line_start += consumed;
+                Some(val) => line_start += val + 1,
+                None => break,
             }
-            None => {
+        } else {
-                // Header lines (chunk 0), corrupt frames, or a truncated tail.
+            let next = memchr::memchr(b'\n', &mmap[line_start..]);
-                // Walk to the next '\n' by hand — no memchr.
+            match next {
-                while line_start < mmap.len() && mmap[line_start] != b'\n' {
+                Some(val) => {
-                    line_start += 1;
+                    eprintln!("Couldn't parse line: '{}'",
-                }
+                              std::str::from_utf8(&mmap[line_start..(line_start + val)]).unwrap());
-                line_start += 1;
+                    line_start += val + 1;
+                },
+                None => break,
             }
         }
     }