Logos lexer
This commit is contained in:
@@ -25,3 +25,4 @@ atoi_simd = "0.18.1"
|
||||
voracious_radix_sort = { version = "1.2.0", features = ["voracious_multithread"] }
|
||||
rayon = "1.12.0"
|
||||
chrono = { version = "0.4", default-features = false, features = ["std"] }
|
||||
logos = "0.16.1"
|
||||
|
||||
+111
-75
@@ -1,84 +1,120 @@
|
||||
use crate::can_frame::{CanFrame, Direction};
|
||||
|
||||
#[inline]
|
||||
fn next_token<'a>(input: &mut &'a [u8]) -> Option<&'a [u8]> {
|
||||
*input = input.trim_ascii_start();
|
||||
if input.is_empty() {
|
||||
return None;
|
||||
}
|
||||
use logos::{Lexer, Logos};
|
||||
|
||||
// Stop on space, tab, or CR. Lines end with CRLF, so stopping on CR keeps
|
||||
// the cursor on the line; the leading \r\n is then consumed by the next
|
||||
// call's trim_ascii_start — no need to ever search for '\n'.
|
||||
let end = memchr::memchr3(b' ', b'\t', b'\r', input).unwrap_or(input.len());
|
||||
let token = &input[..end];
|
||||
*input = &input[end..];
|
||||
Some(token)
|
||||
}
|
||||
|
||||
pub enum AscLineType {
|
||||
Frame(CanFrame),
|
||||
Errorframe,
|
||||
}
|
||||
|
||||
/// Parse an ASC header date line of the form:
|
||||
/// `date Sun May 3 08:14:59 pm 2026`
|
||||
///
|
||||
/// Returns a naive (timezone-less) datetime. Format it as ISO 8601 with
|
||||
/// `dt.format("%Y-%m-%dT%H:%M:%S").to_string()`.
|
||||
pub fn parse_asc_date(input: &[u8]) -> Option<chrono::NaiveDateTime> {
|
||||
let s = std::str::from_utf8(input).ok()?.trim();
|
||||
chrono::NaiveDateTime::parse_from_str(s, "date %a %b %e %I:%M:%S %P %Y").ok()
|
||||
}
|
||||
|
||||
/// Parses one ASC line starting at `input[0]`.
|
||||
///
|
||||
/// Returns the parsed line type and the number of bytes consumed. On success
|
||||
/// the consumed count leaves `input[consumed..]` pointing at the trailing
|
||||
/// `\r\n`, which the next call's `trim_ascii_start` will eat.
|
||||
///
|
||||
/// `None` means the line could not be parsed (header line, truncated tail,
|
||||
/// corrupt frame). The caller is responsible for advancing past the bad line.
|
||||
pub fn parse_asc_line(input: &[u8]) -> Option<(AscLineType, usize)> {
|
||||
let mut cursor = input;
|
||||
|
||||
let timestamp: f64 = {
|
||||
let tok = next_token(&mut cursor)?;
|
||||
unsafe { std::str::from_utf8_unchecked(tok) }.parse().ok()?
|
||||
};
|
||||
let channel = atoi_simd::parse_pos::<u8, false>(next_token(&mut cursor)?).ok()?;
|
||||
|
||||
let tok = next_token(&mut cursor)?;
|
||||
if tok[0] == b'E' {
|
||||
return Some((AscLineType::Errorframe, input.len() - cursor.len()));
|
||||
}
|
||||
let can_id = atoi_simd::parse_pos::<u32, false>(tok).ok()?;
|
||||
|
||||
let direction = match next_token(&mut cursor)? {
|
||||
b"Rx" => Direction::Rx,
|
||||
b"Tx" => Direction::Tx,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
// 'D' frame type
|
||||
next_token(&mut cursor)?;
|
||||
|
||||
let dlc = atoi_simd::parse_pos::<u8, false>(next_token(&mut cursor)?).ok()?;
|
||||
|
||||
let mut data = [0u8; 8];
|
||||
for byte in data.iter_mut().take(dlc as usize) {
|
||||
*byte = atoi_simd::parse_pos::<u8, false>(next_token(&mut cursor)?).ok()?;
|
||||
}
|
||||
|
||||
Some((
|
||||
AscLineType::Frame(CanFrame {
|
||||
timestamp,
|
||||
channel,
|
||||
can_id,
|
||||
direction,
|
||||
dlc,
|
||||
data,
|
||||
}),
|
||||
input.len() - cursor.len(),
|
||||
))
|
||||
fn parse_float(lex: &mut Lexer<Token>) -> Option<f64> {
|
||||
unsafe {
|
||||
std::str::from_utf8_unchecked(lex.slice())
|
||||
}.parse().ok()
|
||||
}
|
||||
|
||||
// fn parse_int8(lex: &mut Lexer<Token>) -> Option<u8> {
|
||||
// atoi_simd::parse_pos::<u8, false>(lex.slice()).ok()
|
||||
// }
|
||||
|
||||
fn parse_int32(lex: &mut Lexer<Token>) -> Option<u32> {
|
||||
atoi_simd::parse_pos::<u32, false>(lex.slice()).ok()
|
||||
}
|
||||
|
||||
#[derive(Logos, Debug)]
|
||||
#[logos(skip r"[ \t\n\f]+")]
|
||||
#[logos(utf8 = false)]
|
||||
enum Token {
|
||||
#[regex(r"\d+\.\d+", parse_float)]
|
||||
Float(f64),
|
||||
// #[regex(r"\d+", parse_int8)]
|
||||
// Int8(u8),
|
||||
#[regex(r"\d+", parse_int32)]
|
||||
Int32(u32),
|
||||
#[token(b"Rx")]
|
||||
Rx,
|
||||
#[token(b"D")]
|
||||
D,
|
||||
#[token(b"Errorframe ECC:")]
|
||||
Errorframe,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ErrorFrame {
|
||||
timestamp: f64,
|
||||
channel: u8,
|
||||
data: u8,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum AscLine {
|
||||
ErrorFrame(ErrorFrame),
|
||||
CanFrame(CanFrame),
|
||||
}
|
||||
|
||||
// 0.001029 1 595 Rx D 3 232 242 45
|
||||
// 0.643971 2 Errorframe ECC: 10100010
|
||||
pub fn parse_asc_line(line: &[u8]) -> Option<(AscLine, usize)> {
|
||||
let mut lex = Token::lexer(line);
|
||||
|
||||
let timestamp = lex.next()?.ok()?;
|
||||
let channel = lex.next()?.ok()?;
|
||||
let canid_or_error = lex.next()?.ok()?;
|
||||
|
||||
match canid_or_error {
|
||||
Token::Int32(can_id) => {
|
||||
let _ = lex.next()?.ok()?;
|
||||
let _ = lex.next()?.ok()?;
|
||||
let dlc = lex.next()?.ok()?;
|
||||
let mut data = [0u8; 8];
|
||||
if let Token::Float(timestamp) = timestamp &&
|
||||
let Token::Int32(channel) = channel &&
|
||||
let Token::Int32(dlc) = dlc {
|
||||
let channel = channel as u8;
|
||||
let dlc = dlc as u8;
|
||||
for i in 0..(dlc as usize) {
|
||||
data[i] = match lex.next()?.ok()? {
|
||||
Token::Int32(val) => val as u8,
|
||||
_ => panic!()
|
||||
};
|
||||
}
|
||||
|
||||
Some((
|
||||
AscLine::CanFrame(
|
||||
CanFrame {timestamp, channel, can_id, direction: Direction::Rx, dlc, data }),
|
||||
lex.span().end
|
||||
))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
},
|
||||
Token::Errorframe => {
|
||||
let err_byte = lex.next()?.ok()?;
|
||||
if let Token::Float(timestamp) = timestamp &&
|
||||
let Token::Int32(channel) = channel &&
|
||||
let Token::Int32(err_byte) = err_byte {
|
||||
let channel = channel as u8;
|
||||
let data = binary_decimal_to_u8(err_byte);
|
||||
Some((
|
||||
AscLine::ErrorFrame(
|
||||
ErrorFrame {timestamp, channel, data}),
|
||||
lex.span().end
|
||||
))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
},
|
||||
_ => panic!("Expected CanId or Errorframe")
|
||||
}
|
||||
}
|
||||
|
||||
fn binary_decimal_to_u8(mut n: u32) -> u8 {
|
||||
let mut result = 0u8;
|
||||
let mut bit = 0;
|
||||
while n > 0 {
|
||||
result |= ((n % 10) as u8) << bit;
|
||||
n /= 10;
|
||||
bit += 1;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
+27
-19
@@ -10,6 +10,7 @@ use std::{
|
||||
|
||||
use can_dbc::Dbc;
|
||||
use hashbrown::HashMap;
|
||||
use memchr::memchr_iter;
|
||||
#[cfg(unix)]
|
||||
use memmap2::Advice;
|
||||
use memmap2::Mmap;
|
||||
@@ -106,29 +107,36 @@ fn frame_worker(
|
||||
let mut line_start = 0usize;
|
||||
|
||||
while line_start < mmap.len() {
|
||||
match parse_asc_line(&mmap[line_start..]) {
|
||||
Some((asc::AscLineType::Frame(frame), consumed)) => {
|
||||
match channel_data.get_mut(&frame.can_id) {
|
||||
Some(vec) => vec.push(frame),
|
||||
None => {
|
||||
unsafe {
|
||||
channel_data.insert_unique_unchecked(frame.can_id, vec![frame]);
|
||||
if let Some((val, _read)) = parse_asc_line(&mmap[line_start..]) {
|
||||
match val {
|
||||
asc::AscLine::CanFrame(frame) => {
|
||||
match channel_data.get_mut(&frame.can_id) {
|
||||
Some(vec) => vec.push(frame),
|
||||
None => {
|
||||
unsafe {
|
||||
channel_data.insert_unique_unchecked(frame.can_id, vec![frame]);
|
||||
}
|
||||
add_metadata(frame.can_id);
|
||||
}
|
||||
add_metadata(frame.can_id);
|
||||
}
|
||||
}
|
||||
line_start += consumed;
|
||||
},
|
||||
asc::AscLine::ErrorFrame(frame) => {
|
||||
eprintln!("Got Errorframe: {:?}", frame);
|
||||
},
|
||||
}
|
||||
Some((asc::AscLineType::Errorframe, consumed)) => {
|
||||
line_start += consumed;
|
||||
match memchr::memchr(b'\n', &mmap[line_start..]) {
|
||||
Some(val) => line_start += val + 1,
|
||||
None => break,
|
||||
}
|
||||
None => {
|
||||
// Header lines (chunk 0), corrupt frames, or a truncated tail.
|
||||
// Walk to the next '\n' by hand — no memchr.
|
||||
while line_start < mmap.len() && mmap[line_start] != b'\n' {
|
||||
line_start += 1;
|
||||
}
|
||||
line_start += 1;
|
||||
} else {
|
||||
let next = memchr::memchr(b'\n', &mmap[line_start..]);
|
||||
match next {
|
||||
Some(val) => {
|
||||
eprintln!("Couldn't parse line: '{}'",
|
||||
std::str::from_utf8(&mmap[line_start..(line_start + val)]).unwrap());
|
||||
line_start += val + 1;
|
||||
},
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user