New work

Author: friendlymatthew

src/classifier.rs

@@ -84,7 +84,13 @@ impl<'a> CsvClassifier<'a> {
 
         let mut temp = [0u8; 16];
         temp[..slice.len()].copy_from_slice(slice);
-
+      
+        let last = self.data[self.data.len() - 1];
+      
+        if last != 0x0A && last != 0x0D {
+            temp[slice.len()] = 0x0A; 
+        }
+       
         (u8x16::from_slice_unchecked(&temp), false)
     }
 }
@@ -115,11 +121,11 @@ mod tests {
                 QUOTATION_CLASS,
                 0,
                 QUOTATION_CLASS,
+                WHITESPACE_CLASS, // we always add a \n at the end
                 0,
                 0,
                 0,
                 0,
-                0
             ]
         );
     }

src/reader.rs

@@ -80,15 +80,14 @@ impl CsvReader {
                 if first_whitespace < first_comma {
                     rows.push(Row::from(current_row.clone()));
                     current_row.clear();
-                    break;
                 }
 
                 bitset_cursor += bits_traveled + 1;
                 valid_commas <<= bits_traveled + 1;
                 valid_whitespace <<= bits_traveled + 1;
             }
         }
-
+        
         Ok(rows)
     }
 }
@@ -152,7 +151,7 @@ mod tests {
 
     #[test]
     fn read_basic() -> Result<()> {
-        let data = b"aaa,bbb,ccc\n";
+        let data = b"aaa,bbb,ccc";
 
         check_line(
             data,
@@ -168,7 +167,7 @@ mod tests {
 
     #[test]
     fn read_basic2() -> Result<()> {
-        let data = b"aaa,\"bbb\",ccc\n";
+        let data = b"aaa,\"bbb\",ccc";
 
         check_line(
             data,
@@ -184,7 +183,7 @@ mod tests {
 
     #[test]
     fn read_nested() -> Result<()> {
-        let data = b"\"aaa,howdy\",\"b\"\"bb\",\"ccc\"\n";
+        let data = b"\"aaa,howdy\",\"b\"\"bb\",\"ccc\"";
 
         check_line(
             data,
@@ -198,18 +197,9 @@ mod tests {
         Ok(())
     }
 
-    // #[test]
-    // fn read_nested2() -> Result<()> {
-    //     let data = b"aaa,\"b\"\"\"\n";
-    // 
-    //     check_line(data, vec![vec!["aaa".to_string(), "\"b\"\"\"".to_string()]])?;
-    // 
-    //     Ok(())
-    // }
-
     #[test]
     fn read_newline_field() -> Result<()> {
-        let data = b"\"aaa,ho\nwdy\",\"b\"\"bb\",\"ccc\"\n";
+        let data = b"\"aaa,ho\nwdy\",\"b\"\"bb\",\"ccc\"";
 
         check_line(
             data,
@@ -225,7 +215,7 @@ mod tests {
 
     #[test]
     fn read_crlf_field() -> Result<()> {
-        let data = b"\"aaa,ho\r\nwdy\",\"b\"\"bb\",\"ccc\"\n";
+        let data = b"\"aaa,ho\r\nwdy\",\"b\"\"bb\",\"ccc\"";
 
         check_line(
             data,
@@ -241,8 +231,8 @@ mod tests {
 
     #[test]
     fn read_taxi_zone_lookup_header() -> Result<()> {
-        let data = b"\"LocationID\",\"Borough\",\"Zone\",\"service_zone\"\r\n";
-
+        let data = b"\"LocationID\",\"Borough\",\"Zone\",\"service_zone\"\n";
+    
         check_line(
             data,
             vec![vec![
@@ -252,7 +242,7 @@ mod tests {
                 r#""service_zone""#.to_string(),
             ]],
         )?;
-
+    
         Ok(())
     }