Only skip first n lines in the first chunk and don't take the first line as header (mholt#1045) (mholt#1046)

* Only skip first n lines in the first chunk and don't take the first line as header (mholt#1045)

* use newline from config (or guess if unset)

---------

Co-authored-by: Jonas Krukenberg <[email protected]>

Author: jkruke

docs/resources/js/papaparse.js

@@ -491,6 +491,16 @@ License: MIT
 		this.parseChunk = function(chunk, isFakeChunk)
 		{
 			// First chunk pre-processing
+			const skipFirstNLines = parseInt(this._config.skipFirstNLines) || 0;
+			if (this.isFirstChunk && skipFirstNLines > 0) {
+				let _newline = this._config.newline;
+				if (!_newline) {
+					const quoteChar = this._config.quoteChar || '"';
+					_newline = this._handle.guessLineEndings(chunk, quoteChar);
+				}
+				const splitChunk = chunk.split(_newline);
+				chunk = [...splitChunk.slice(skipFirstNLines)].join(_newline);
+			}
 			if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk))
 			{
 				var modifiedChunk = this._config.beforeFirstChunk(chunk);
@@ -503,7 +513,6 @@ License: MIT
 			// Rejoin the line we likely just split in two by chunking the file
 			var aggregate = this._partialLine + chunk;
 			this._partialLine = '';
-
 			var results = this._handle.parse(aggregate, this._baseIndex, !this._finished);
 
 			if (this._handle.paused() || this._handle.aborted()) {
@@ -1048,7 +1057,7 @@ License: MIT
 		{
 			var quoteChar = _config.quoteChar || '"';
 			if (!_config.newline)
-				_config.newline = guessLineEndings(input, quoteChar);
+				_config.newline = this.guessLineEndings(input, quoteChar);
 
 			_delimiterError = false;
 			if (!_config.delimiter)
@@ -1119,6 +1128,32 @@ License: MIT
 			_input = '';
 		};
 
+		this.guessLineEndings = function(input, quoteChar)
+		{
+			input = input.substr(0, 1024 * 1024);	// max length 1 MB
+			// Replace all the text inside quotes
+			var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm');
+			input = input.replace(re, '');
+
+			var r = input.split('\r');
+
+			var n = input.split('\n');
+
+			var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length);
+
+			if (r.length === 1 || nAppearsFirst)
+				return '\n';
+
+			var numWithN = 0;
+			for (var i = 0; i < r.length; i++)
+			{
+				if (r[i][0] === '\n')
+					numWithN++;
+			}
+
+			return numWithN >= r.length / 2 ? '\r\n' : '\r';
+		};
+
 		function testEmptyLine(s) {
 			return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0;
 		}
@@ -1321,32 +1356,6 @@ License: MIT
 			};
 		}
 
-		function guessLineEndings(input, quoteChar)
-		{
-			input = input.substr(0, 1024 * 1024);	// max length 1 MB
-			// Replace all the text inside quotes
-			var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm');
-			input = input.replace(re, '');
-
-			var r = input.split('\r');
-
-			var n = input.split('\n');
-
-			var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length);
-
-			if (r.length === 1 || nAppearsFirst)
-				return '\n';
-
-			var numWithN = 0;
-			for (var i = 0; i < r.length; i++)
-			{
-				if (r[i][0] === '\n')
-					numWithN++;
-			}
-
-			return numWithN >= r.length / 2 ? '\r\n' : '\r';
-		}
-
 		function addError(type, code, msg, row)
 		{
 			_results.errors.push({

papaparse.js

@@ -511,6 +511,16 @@ License: MIT
 		this.parseChunk = function(chunk, isFakeChunk)
 		{
 			// First chunk pre-processing
+			const skipFirstNLines = parseInt(this._config.skipFirstNLines) || 0;
+			if (this.isFirstChunk && skipFirstNLines > 0) {
+				let _newline = this._config.newline;
+				if (!_newline) {
+					const quoteChar = this._config.quoteChar || '"';
+					_newline = this._handle.guessLineEndings(chunk, quoteChar);
+				}
+				const splitChunk = chunk.split(_newline);
+				chunk = [...splitChunk.slice(skipFirstNLines)].join(_newline);
+			}
 			if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk))
 			{
 				var modifiedChunk = this._config.beforeFirstChunk(chunk);
@@ -522,22 +532,6 @@ License: MIT
 
 			// Rejoin the line we likely just split in two by chunking the file
 			var aggregate = this._partialLine + chunk;
-			this._pendingSkip = parseInt(this._config.skipFirstNLines) || 0;
-			this._skipHeader = 0;
-			if (this._config.header) {
-				this._skipHeader++;
-			}
-			if (this._pendingSkip > 0) {
-				var splitChunk = aggregate.split('\n');
-				var currentChunkLength = splitChunk.length;
-				if (currentChunkLength <= this._pendingSkip) {
-					aggregate = this._partialLine;
-				}
-				else{
-					aggregate = this._partialLine + [...splitChunk.slice(0, this._skipHeader), ...splitChunk.slice(this._skipHeader + this._pendingSkip)].join('\n');
-				}
-				this._pendingSkip -= currentChunkLength;
-			}
 			this._partialLine = '';
 			var results = this._handle.parse(aggregate, this._baseIndex, !this._finished);
 
@@ -1093,7 +1087,7 @@ License: MIT
 		{
 			var quoteChar = _config.quoteChar || '"';
 			if (!_config.newline)
-				_config.newline = guessLineEndings(input, quoteChar);
+				_config.newline = this.guessLineEndings(input, quoteChar);
 
 			_delimiterError = false;
 			if (!_config.delimiter)
@@ -1167,6 +1161,32 @@ License: MIT
 			_input = '';
 		};
 
+		this.guessLineEndings = function(input, quoteChar)
+		{
+			input = input.substring(0, 1024 * 1024);	// max length 1 MB
+			// Replace all the text inside quotes
+			var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm');
+			input = input.replace(re, '');
+
+			var r = input.split('\r');
+
+			var n = input.split('\n');
+
+			var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length);
+
+			if (r.length === 1 || nAppearsFirst)
+				return '\n';
+
+			var numWithN = 0;
+			for (var i = 0; i < r.length; i++)
+			{
+				if (r[i][0] === '\n')
+					numWithN++;
+			}
+
+			return numWithN >= r.length / 2 ? '\r\n' : '\r';
+		};
+
 		function testEmptyLine(s) {
 			return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0;
 		}
@@ -1373,32 +1393,6 @@ License: MIT
 			};
 		}
 
-		function guessLineEndings(input, quoteChar)
-		{
-			input = input.substring(0, 1024 * 1024);	// max length 1 MB
-			// Replace all the text inside quotes
-			var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm');
-			input = input.replace(re, '');
-
-			var r = input.split('\r');
-
-			var n = input.split('\n');
-
-			var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length);
-
-			if (r.length === 1 || nAppearsFirst)
-				return '\n';
-
-			var numWithN = 0;
-			for (var i = 0; i < r.length; i++)
-			{
-				if (r[i][0] === '\n')
-					numWithN++;
-			}
-
-			return numWithN >= r.length / 2 ? '\r\n' : '\r';
-		}
-
 		function addError(type, code, msg, row)
 		{
 			var error = {

tests/test-cases.js

@@ -1585,11 +1585,11 @@ var PARSE_TESTS = [
 		}
 	},
 	{
-		description: "Skip First N number of lines , with header and 3 rows",
-		input: 'a,b,c,d\n1,2,3,4\n4,5,6,7',
+		description: "Skip First N number of lines , with header and 2 rows",
+		input: 'to-be-ignored\na,b,c,d\n1,2,3,4',
 		config: { header: true, skipFirstNLines: 1 },
 		expected: {
-			data: [{a: '4', b: '5', c: '6', d: '7'}],
+			data: [{a: '1', b: '2', c: '3', d: '4'}],
 			errors: []
 		}
 	},
@@ -1610,6 +1610,15 @@ var PARSE_TESTS = [
 			data: [['a','b','c','d'],['1','2','3','4'],['4','5','6','7']],
 			errors: []
 		}
+	},
+	{
+		description: "Skip first 2 lines , with custom newline character",
+		input: 'skip-this\rskip-this\r1,2,3,4',
+		config: { header: false, skipFirstNLines: 2, newline: '\r' },
+		expected: {
+			data: [['1','2','3','4']],
+			errors: []
+		}
 	}
 ];