parse newlines created in MacOS MS Excel (#55)

* parse newlines created in MacOS MS Excel

* version bump to 1.0.0.2

Author: dmvianna

changelog.md

@@ -1,3 +1,8 @@
+1.0.0.2
+* Fixed [#17](https://github.com/ozataman/csv-conduit/issues/17),
+  where CSV created with Excel in Mac OS failed to parse due to its
+  newline characters.
+
 1.0.0.1
 * Removed dependencies: mmorph, monad-control, mtl,
   unordered-containers, primitive

csv-conduit.cabal

@@ -1,5 +1,5 @@
 name:               csv-conduit
-version:            1.0.0.1
+version:            1.0.0.2
 synopsis:
   A flexible, fast, conduit-based CSV parser library for Haskell.
 
@@ -57,6 +57,7 @@ description:
 extra-source-files:
   changelog.md
   README.md
+  test/test-mac-excel.csv
   test/test.csv
   test/Test.hs
 

src/Data/CSV/Conduit/Parser/ByteString.hs

@@ -58,15 +58,17 @@ csv s = do
 row :: CSVSettings -> Parser (Maybe (Row ByteString))
 row csvs = csvrow csvs <|> badrow
 
+csvEndOfLine :: Parser ()
+csvEndOfLine = (word8 10 >> return ()) <|> (C8.string (B8.pack "\r\n") >> return ()) <|> (word8 13 >> return ())
 
 badrow :: Parser (Maybe (Row ByteString))
 badrow = P.takeWhile (not . C8.isEndOfLine) *>
-         (C8.endOfLine <|> C8.endOfInput) *> return Nothing
+         (csvEndOfLine <|> C8.endOfInput) *> return Nothing
 
 csvrow :: CSVSettings -> Parser (Maybe (Row ByteString))
 csvrow c =
   let rowbody = (quotedField' <|> field c) `sepBy` C8.char (csvSep c)
-      properrow = rowbody <* (C8.endOfLine <|> P.endOfInput)
+      properrow = rowbody <* (csvEndOfLine <|> P.endOfInput)
       quotedField' = case csvQuoteChar c of
           Nothing -> mzero
           Just q' -> try (quotedField q')

src/Data/CSV/Conduit/Parser/Text.hs

@@ -58,15 +58,17 @@ csv s = do
 row :: CSVSettings -> Parser (Maybe (Row Text))
 row csvs = csvrow csvs <|> badrow
 
+csvEndOfLine :: Parser ()
+csvEndOfLine = (char '\n' >> return ()) <|> (string (T.pack "\r\n") >> return ()) <|> (char '\r' >> return ())
 
 badrow :: Parser (Maybe (Row Text))
 badrow = P.takeWhile (not . T.isEndOfLine) *>
-         (T.endOfLine <|> T.endOfInput) *> return Nothing
+         (csvEndOfLine <|> T.endOfInput) *> return Nothing
 
 csvrow :: CSVSettings -> Parser (Maybe (Row Text))
 csvrow c =
   let rowbody = (quotedField' <|> field c) `sepBy` T.char (csvSep c)
-      properrow = rowbody <* (T.endOfLine <|> P.endOfInput)
+      properrow = rowbody <* (csvEndOfLine <|> P.endOfInput)
       quotedField' = case csvQuoteChar c of
           Nothing -> mzero
           Just q' -> try (quotedField q')

test/Test.hs

@@ -32,7 +32,8 @@ tests =
 baseTests :: [Test]
 baseTests =
   [ testCase "mapping with id works" test_identityMap,
-    testCase "simple parsing works" test_simpleParse,
+    testCase "simple parsing works" (test_simpleParse testFile1),
+    testCase "simple parsing works for Mac-Excel" (test_simpleParse testFile3),
     testCase "OrderedMap" test_orderedMap
   ]
 
@@ -74,9 +75,9 @@ test_identityMap = do
     f :: Row Text -> [Row Text]
     f = return
 
-test_simpleParse :: IO ()
-test_simpleParse = do
-  (d :: V.Vector (MapRow B.ByteString)) <- readCSVFile csvSettings testFile1
+test_simpleParse :: FilePath -> IO ()
+test_simpleParse fp = do
+  (d :: V.Vector (MapRow B.ByteString)) <- readCSVFile csvSettings fp
   V.mapM_ assertRow d
   where
     assertRow r = v3 @=? (v1 + v2)
@@ -109,9 +110,10 @@ test_orderedMap = do
 csvSettings :: CSVSettings
 csvSettings = defCSVSettings {csvQuoteCharAndStyle = Just ('`', DontQuoteEmpty)}
 
-testFile1, testFile2 :: FilePath
+testFile1, testFile2, testFile3 :: FilePath
 testFile1 = "test/test.csv"
 testFile2 = "test/test.csv"
+testFile3 = "test/test-mac-excel.csv"
 
 readBS :: B.ByteString -> Int
 readBS = read . B.unpack

test/test-mac-excel.csv

@@ -0,0 +1 @@
+`Col1`,`Col2`,`Col3`,`Sum``A`,`2`,`3`,`5``B`,`3`,`4`,`7``Field using the quote char ``this is the in-quoted value```,`4`,`5`,`9`