diff --git a/libsvld/src/asc.rs b/libsvld/src/asc.rs index 92aad81..a6bd81b 100644 --- a/libsvld/src/asc.rs +++ b/libsvld/src/asc.rs @@ -41,9 +41,9 @@ enum Token { #[derive(Debug)] pub struct ErrorFrame { - timestamp: f64, - channel: u8, - data: u8, + pub timestamp: f64, + pub channel: u8, + pub data: u8, } #[derive(Debug)] diff --git a/libsvld/src/lib.rs b/libsvld/src/lib.rs index 029380f..fa04a80 100644 --- a/libsvld/src/lib.rs +++ b/libsvld/src/lib.rs @@ -9,29 +9,37 @@ use std::{ }; use can_dbc::Dbc; -use hashbrown::HashMap; -use memchr::memchr_iter; +use chrono::NaiveDateTime; +use hashbrown::{HashMap, HashSet}; #[cfg(unix)] use memmap2::Advice; use memmap2::Mmap; use voracious_radix_sort::RadixSort; use crate::{ - asc::{parse_asc_date, parse_asc_line}, + asc::{ErrorFrame, parse_asc_date, parse_asc_line}, can_frame::{CanFrame, CanId}, motec::{MessageMetadata, SignalMetadata}, }; +struct FrameOutput { + data: HashMap>, + metadata: HashMap, + + unknown_ids: HashSet, + error_frames: Vec, +} + fn frame_worker( mmap: &[u8], dbc: &Dbc, -) -> ( - HashMap>, - HashMap, -) { +) -> FrameOutput { let mut channel_data = HashMap::>::new(); let mut metadatas = HashMap::::new(); + let mut unknown_ids = HashSet::::new(); + let mut error_frames = Vec::::new(); + let get_default = |name: &str| match dbc .attribute_defaults .iter() @@ -52,7 +60,7 @@ fn frame_worker( let msg = match dbc.messages.iter().find(|x| x.id.raw() == id) { Some(msg) => msg, None => { - eprintln!("Could not find msg with id: {}", id); + unknown_ids.insert(id); return; } }; @@ -121,7 +129,7 @@ fn frame_worker( } }, asc::AscLine::ErrorFrame(frame) => { - eprintln!("Got Errorframe: {:?}", frame); + error_frames.push(frame); }, } match memchr::memchr(b'\n', &mmap[line_start..]) { @@ -141,12 +149,24 @@ fn frame_worker( } } - (channel_data, metadatas) + FrameOutput { data: channel_data, metadata: metadatas, unknown_ids, error_frames } +} + +pub struct DecodeProcessInfo { + pub unknown_ids: HashSet, + pub error_frames: Vec, + + pub date: NaiveDateTime, + + pub thread_count: usize, + pub page_size: usize, + pub pages_per_chunk: usize, + pub chunk_size: usize, } pub fn decode(dbc: &Dbc, asc_path: &Path, - output_dir: &Path) { + output_dir: &Path) -> DecodeProcessInfo { let thread_count = std::thread::available_parallelism().unwrap().get(); let file = File::open(asc_path).unwrap(); @@ -159,19 +179,19 @@ pub fn decode(dbc: &Dbc, let date = parse_asc_date(&mmap[..memchr::memchr(b'\r', &mmap).unwrap()]).unwrap(); - let date = date + let date_str = date .and_utc() .to_rfc3339_opts(chrono::SecondsFormat::Secs, true); - println!("Date: {}", date); + // println!("Date: {}", date); let page_size = page_size::get(); let pages_per_chunk = (mmap.len() / page_size) / thread_count; let chunk_size = pages_per_chunk * page_size; - println!("Page size: {}", page_size); - println!("Pages per chunk: {}", pages_per_chunk); - println!("Chunk size: {}", chunk_size); - println!("Thread count: {}", thread_count); + // println!("Page size: {}", page_size); + // println!("Pages per chunk: {}", pages_per_chunk); + // println!("Chunk size: {}", chunk_size); + // println!("Thread count: {}", thread_count); // Align chunk ends to '\n' so every line is wholly contained in one chunk. // Without this, the line spanning each boundary is lost on both sides. @@ -195,6 +215,9 @@ pub fn decode(dbc: &Dbc, let mut data = HashMap::>::new(); let mut metadata = HashMap::::new(); + let mut unknown_ids = HashSet::::new(); + let mut error_frames = Vec::::new(); + thread::scope(|s| { let mut handles = vec![]; for w in bounds.windows(2) { @@ -204,8 +227,8 @@ pub fn decode(dbc: &Dbc, } for handle in handles { - let (datah, metadatah) = handle.join().unwrap(); - for (id, mut datap) in datah { + let mut frame_out = handle.join().unwrap(); + for (id, mut datap) in frame_out.data { match data.get_mut(&id) { Some(out) => out.append(&mut datap), None => unsafe { @@ -213,16 +236,23 @@ pub fn decode(dbc: &Dbc, }, } } - for (id, datap) in metadatah { + for (id, datap) in frame_out.metadata { if !metadata.contains_key(&id) { unsafe { metadata.insert_unique_unchecked(id, datap); }; } } + for id in frame_out.unknown_ids { + unknown_ids.insert(id); + } + error_frames.append(&mut frame_out.error_frames); } }); + // println!("{:?}", unknown_ids); + // println!("{:?}", error_frames); + println!("Post processing"); for (_, frames) in &mut data { @@ -232,12 +262,22 @@ pub fn decode(dbc: &Dbc, let msgs: Vec = metadata.values().cloned().collect(); - println!("Messages: {}", msgs.len()); - println!("Data: {}", data.len()); + // println!("Messages: {}", msgs.len()); + // println!("Data: {}", data.len()); println!("Write to file..."); - let out_file_name = format!("{}.ld", date.replace(":", "-")); + let out_file_name = format!("{}.ld", date_str.replace(":", "-")); let out_name = Path::new(&out_file_name); motec::write(&output_dir.join(out_name), &msgs, data); println!("Finished!"); + + DecodeProcessInfo { + unknown_ids, + error_frames, + date, + thread_count, + page_size, + pages_per_chunk, + chunk_size, + } }