stripBOM

leeoniya · Feb 14, 2025 · 480afcb · 480afcb
1 parent 349de02
commit 480afcb
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 1 deletion.
diff --git a/dist/uDSV.cjs.js b/dist/uDSV.cjs.js
@@ -20,6 +20,10 @@ const BOOL_RE = /^(?:t(?:rue)?|f(?:alse)?|y(?:es)?|n(?:o)?|0|1)$/i;
 
 const COL_DELIMS = [tab, pipe, semi, comma];
 
+function stripBOM(str) {
+	return str.charCodeAt(0) === 0xFEFF ? str.slice(1) : str;
+}
+
 function boolTrue(v) {
 	let [c0, c1 = ''] = v;
 
@@ -177,6 +181,8 @@ function inferSchema(csvStr, opts, maxRows) {
 
 	maxRows ??= 10;
 
+	csvStr = stripBOM(csvStr);
+
 	// will fail if header contains line breaks in quoted value
 	// will fail if single line without line breaks
 	const rowRE         = new RegExp(`(.*)(${rowDelim ?? '\r\n|\r|\n'})`);
@@ -376,6 +382,8 @@ function initParser(schema) {
 // _maxCols is cols estimated by simple delimiter detection and split()
 // returns [unparsed tail, shouldHalt]
 function parse(csvStr, schema, skip = 0, each = () => true, withEOF = true, _maxCols) {
+	csvStr = stripBOM(csvStr);
+
 	let {
 		row:  rowDelim,
 		col:  colDelim,

diff --git a/dist/uDSV.iife.js b/dist/uDSV.iife.js
@@ -21,6 +21,10 @@ var uDSV = (function (exports) {
 
 	const COL_DELIMS = [tab, pipe, semi, comma];
 
+	function stripBOM(str) {
+		return str.charCodeAt(0) === 0xFEFF ? str.slice(1) : str;
+	}
+
 	function boolTrue(v) {
 		let [c0, c1 = ''] = v;
 
@@ -178,6 +182,8 @@ var uDSV = (function (exports) {
 
 		maxRows ??= 10;
 
+		csvStr = stripBOM(csvStr);
+
 		// will fail if header contains line breaks in quoted value
 		// will fail if single line without line breaks
 		const rowRE         = new RegExp(`(.*)(${rowDelim ?? '\r\n|\r|\n'})`);
@@ -377,6 +383,8 @@ var uDSV = (function (exports) {
 	// _maxCols is cols estimated by simple delimiter detection and split()
 	// returns [unparsed tail, shouldHalt]
 	function parse(csvStr, schema, skip = 0, each = () => true, withEOF = true, _maxCols) {
+		csvStr = stripBOM(csvStr);
+
 		let {
 			row:  rowDelim,
 			col:  colDelim,

diff --git a/dist/uDSV.iife.min.js b/dist/uDSV.iife.min.js
diff --git a/dist/uDSV.mjs b/dist/uDSV.mjs
@@ -18,6 +18,10 @@ const BOOL_RE = /^(?:t(?:rue)?|f(?:alse)?|y(?:es)?|n(?:o)?|0|1)$/i;
 
 const COL_DELIMS = [tab, pipe, semi, comma];
 
+function stripBOM(str) {
+	return str.charCodeAt(0) === 0xFEFF ? str.slice(1) : str;
+}
+
 function boolTrue(v) {
 	let [c0, c1 = ''] = v;
 
@@ -175,6 +179,8 @@ function inferSchema(csvStr, opts, maxRows) {
 
 	maxRows ??= 10;
 
+	csvStr = stripBOM(csvStr);
+
 	// will fail if header contains line breaks in quoted value
 	// will fail if single line without line breaks
 	const rowRE         = new RegExp(`(.*)(${rowDelim ?? '\r\n|\r|\n'})`);
@@ -374,6 +380,8 @@ function initParser(schema) {
 // _maxCols is cols estimated by simple delimiter detection and split()
 // returns [unparsed tail, shouldHalt]
 function parse(csvStr, schema, skip = 0, each = () => true, withEOF = true, _maxCols) {
+	csvStr = stripBOM(csvStr);
+
 	let {
 		row:  rowDelim,
 		col:  colDelim,

diff --git a/src/uDSV.mjs b/src/uDSV.mjs
@@ -11,6 +11,10 @@ const BOOL_RE = /^(?:t(?:rue)?|f(?:alse)?|y(?:es)?|n(?:o)?|0|1)$/i;
 
 const COL_DELIMS = [tab, pipe, semi, comma];
 
+function stripBOM(str) {
+	return str.charCodeAt(0) === 0xFEFF ? str.slice(1) : str;
+}
+
 function boolTrue(v) {
 	let [c0, c1 = ''] = v;
 
@@ -168,6 +172,8 @@ export function inferSchema(csvStr, opts, maxRows) {
 
 	maxRows ??= 10;
 
+	csvStr = stripBOM(csvStr);
+
 	// will fail if header contains line breaks in quoted value
 	// will fail if single line without line breaks
 	const rowRE         = new RegExp(`(.*)(${rowDelim ?? '\r\n|\r|\n'})`);
@@ -368,6 +374,8 @@ export function initParser(schema) {
 // _maxCols is cols estimated by simple delimiter detection and split()
 // returns [unparsed tail, shouldHalt]
 function parse(csvStr, schema, skip = 0, each = () => true, withEOF = true, _maxCols) {
+	csvStr = stripBOM(csvStr);
+
 	let {
 		row:  rowDelim,
 		col:  colDelim,

diff --git a/test/parse.spec.mjs b/test/parse.spec.mjs
@@ -1085,6 +1085,20 @@ test('repl injection guard', (t) => {
     }
 });
 
+test('strip BOM', (t) => {
+    const csvStr = '\uFEFFa,b,c\n1,2,3';
+
+    let schema = inferSchema(csvStr);
+    schema.skip = 0;
+    let parser = initParser(schema);
+    let arrs = parser.stringArrs(csvStr);
+
+    assert.deepEqual(arrs, [
+        ['a','b','c'],
+        ['1','2','3'],
+    ]);
+});
+
 test('transform stream', (t) => {
     class ParseCSVTransform extends Transform {
         #parser = null;