Propagate some useful info

libsvld/src/asc.rs

@@ -41,9 +41,9 @@ enum Token {
 
 #[derive(Debug)]
 pub struct ErrorFrame {
-    timestamp: f64,
-    channel: u8,
-    data: u8,
+    pub timestamp: f64,
+    pub channel: u8,
+    pub data: u8,
 }
 
 #[derive(Debug)]

libsvld/src/lib.rs

@@ -9,29 +9,37 @@ use std::{
 };
 
 use can_dbc::Dbc;
-use hashbrown::HashMap;
-use memchr::memchr_iter;
+use chrono::NaiveDateTime;
+use hashbrown::{HashMap, HashSet};
 #[cfg(unix)]
 use memmap2::Advice;
 use memmap2::Mmap;
 use voracious_radix_sort::RadixSort;
 
 use crate::{
-    asc::{parse_asc_date, parse_asc_line},
+    asc::{ErrorFrame, parse_asc_date, parse_asc_line},
     can_frame::{CanFrame, CanId},
     motec::{MessageMetadata, SignalMetadata},
 };
 
+struct FrameOutput {
+    data: HashMap<CanId, Vec<CanFrame>>,
+    metadata: HashMap<CanId, MessageMetadata>,
+
+    unknown_ids: HashSet<CanId>,
+    error_frames: Vec<ErrorFrame>,
+}
+
 fn frame_worker(
     mmap: &[u8],
     dbc: &Dbc,
-) -> (
-    HashMap<CanId, Vec<CanFrame>>,
-    HashMap<CanId, MessageMetadata>,
-) {
+) -> FrameOutput {
     let mut channel_data = HashMap::<CanId, Vec<CanFrame>>::new();
     let mut metadatas = HashMap::<CanId, MessageMetadata>::new();
 
+    let mut unknown_ids = HashSet::<CanId>::new();
+    let mut error_frames = Vec::<ErrorFrame>::new();
+
     let get_default = |name: &str| match dbc
         .attribute_defaults
         .iter()
@@ -52,7 +60,7 @@ fn frame_worker(
         let msg = match dbc.messages.iter().find(|x| x.id.raw() == id) {
             Some(msg) => msg,
             None => {
-                eprintln!("Could not find msg with id: {}", id);
+                unknown_ids.insert(id);
                 return;
             }
         };
@@ -121,7 +129,7 @@ fn frame_worker(
                     }
                 },
                 asc::AscLine::ErrorFrame(frame) => {
-                    eprintln!("Got Errorframe: {:?}", frame);
+                    error_frames.push(frame);
                 },
             }
             match memchr::memchr(b'\n', &mmap[line_start..]) {
@@ -141,12 +149,24 @@ fn frame_worker(
         }
     }
 
-    (channel_data, metadatas)
+    FrameOutput { data: channel_data, metadata: metadatas, unknown_ids, error_frames }
+}
+
+pub struct DecodeProcessInfo {
+    pub unknown_ids: HashSet<CanId>,
+    pub error_frames: Vec<ErrorFrame>,
+
+    pub date: NaiveDateTime,
+
+    pub thread_count: usize,
+    pub page_size: usize,
+    pub pages_per_chunk: usize,
+    pub chunk_size: usize,
 }
 
 pub fn decode(dbc: &Dbc,
               asc_path: &Path,
-              output_dir: &Path) {
+              output_dir: &Path) -> DecodeProcessInfo {
     let thread_count = std::thread::available_parallelism().unwrap().get();
 
     let file = File::open(asc_path).unwrap();
@@ -159,19 +179,19 @@ pub fn decode(dbc: &Dbc,
 
     let date = parse_asc_date(&mmap[..memchr::memchr(b'\r', &mmap).unwrap()]).unwrap();
 
-    let date = date
+    let date_str = date
         .and_utc()
         .to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
-    println!("Date: {}", date);
+    // println!("Date: {}", date);
 
     let page_size = page_size::get();
     let pages_per_chunk = (mmap.len() / page_size) / thread_count;
     let chunk_size = pages_per_chunk * page_size;
 
-    println!("Page size: {}", page_size);
-    println!("Pages per chunk: {}", pages_per_chunk);
-    println!("Chunk size: {}", chunk_size);
-    println!("Thread count: {}", thread_count);
+    // println!("Page size: {}", page_size);
+    // println!("Pages per chunk: {}", pages_per_chunk);
+    // println!("Chunk size: {}", chunk_size);
+    // println!("Thread count: {}", thread_count);
 
     // Align chunk ends to '\n' so every line is wholly contained in one chunk.
     // Without this, the line spanning each boundary is lost on both sides.
@@ -195,6 +215,9 @@ pub fn decode(dbc: &Dbc,
 
     let mut data = HashMap::<u32, Vec<CanFrame>>::new();
     let mut metadata = HashMap::<CanId, MessageMetadata>::new();
+    let mut unknown_ids = HashSet::<CanId>::new();
+    let mut error_frames = Vec::<ErrorFrame>::new();
+
     thread::scope(|s| {
         let mut handles = vec![];
         for w in bounds.windows(2) {
@@ -204,8 +227,8 @@ pub fn decode(dbc: &Dbc,
         }
 
         for handle in handles {
-            let (datah, metadatah) = handle.join().unwrap();
-            for (id, mut datap) in datah {
+            let mut frame_out = handle.join().unwrap();
+            for (id, mut datap) in frame_out.data {
                 match data.get_mut(&id) {
                     Some(out) => out.append(&mut datap),
                     None => unsafe {
@@ -213,16 +236,23 @@ pub fn decode(dbc: &Dbc,
                     },
                 }
             }
-            for (id, datap) in metadatah {
+            for (id, datap) in frame_out.metadata {
                 if !metadata.contains_key(&id) {
                     unsafe {
                         metadata.insert_unique_unchecked(id, datap);
                     };
                 }
             }
+            for id in frame_out.unknown_ids {
+                unknown_ids.insert(id);
+            }
+            error_frames.append(&mut frame_out.error_frames);
         }
     });
 
+    // println!("{:?}", unknown_ids);
+    // println!("{:?}", error_frames);
+
     println!("Post processing");
 
     for (_, frames) in &mut data {
@@ -232,12 +262,22 @@ pub fn decode(dbc: &Dbc,
 
     let msgs: Vec<MessageMetadata> = metadata.values().cloned().collect();
 
-    println!("Messages: {}", msgs.len());
-    println!("Data: {}", data.len());
+    // println!("Messages: {}", msgs.len());
+    // println!("Data: {}", data.len());
 
     println!("Write to file...");
-    let out_file_name = format!("{}.ld", date.replace(":", "-"));
+    let out_file_name = format!("{}.ld", date_str.replace(":", "-"));
     let out_name = Path::new(&out_file_name);
     motec::write(&output_dir.join(out_name), &msgs, data);
     println!("Finished!");
+
+    DecodeProcessInfo {
+        unknown_ids,
+        error_frames,
+        date,
+        thread_count,
+        page_size,
+        pages_per_chunk,
+        chunk_size,
+    }
 }