commit b604b5b84be32fe132f9f70c7a6e4d51e2589056 Author: upstream Date: Thu Sep 3 14:09:51 2026 +0200 Upstream mholt/PapaParse state of the touched files before 3deba2fd1e diff --git a/papaparse.js b/papaparse.js new file mode 100644 index 0000000..5229936 --- /dev/null +++ b/papaparse.js @@ -0,0 +1,1880 @@ +/* @license +Papa Parse +v5.7.0 +https://github.com/mholt/PapaParse +License: MIT +*/ + +(function(root, factory) +{ + /* globals define */ + if (typeof define === 'function' && define.amd) + { + // AMD. Register as an anonymous module. + define([], factory); + } + else if (typeof module === 'object' && typeof exports !== 'undefined') + { + // Node. Does not work with strict CommonJS, but + // only CommonJS-like environments that support module.exports, + // like Node. + module.exports = factory(); + } + else + { + // Browser globals (root is window) + root.Papa = factory(); + } + // in strict mode we cannot access arguments.callee, so we need a named reference to + // stringify the factory method for the blob worker + // eslint-disable-next-line func-name +}(this, function moduleFactory() +{ + 'use strict'; + + var global = (function() { + // alternative method, similar to `Function('return this')()` + // but without using `eval` (which is disabled when + // using Content Security Policy). + + if (typeof self !== 'undefined') { return self; } + if (typeof window !== 'undefined') { return window; } + if (typeof global !== 'undefined') { return global; } + + // When running tests none of the above have been defined + return {}; + })(); + + + function getWorkerBlob() { + var URL = global.URL || global.webkitURL || null; + var code = moduleFactory.toString(); + return Papa.BLOB_URL || (Papa.BLOB_URL = URL.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ", '(', code, ')();'], {type: 'text/javascript'}))); + } + + var IS_WORKER = !global.document && !!global.postMessage, + IS_PAPA_WORKER = global.IS_PAPA_WORKER || false; + + var workers = {}, workerIdCounter = 0; + + var Papa = {}; + + Papa.parse = CsvToJson; + Papa.unparse = JsonToCsv; + + Papa.RECORD_SEP = String.fromCharCode(30); + Papa.UNIT_SEP = String.fromCharCode(31); + Papa.BYTE_ORDER_MARK = '\ufeff'; + Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK]; + Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker; + Papa.NODE_STREAM_INPUT = 1; + + // Configurable chunk sizes for local and remote files, respectively + Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB + Papa.RemoteChunkSize = 1024 * 1024 * 5; // 5 MB + Papa.DefaultDelimiter = ','; // Used if not specified and detection fails + + // Exposed for testing and development only + Papa.Parser = Parser; + Papa.ParserHandle = ParserHandle; + Papa.NetworkStreamer = NetworkStreamer; + Papa.FileStreamer = FileStreamer; + Papa.StringStreamer = StringStreamer; + Papa.ReadableStreamStreamer = ReadableStreamStreamer; + if (typeof PAPA_BROWSER_CONTEXT === 'undefined') { + Papa.DuplexStreamStreamer = DuplexStreamStreamer; + } + + if (IS_PAPA_WORKER) + { + global.onmessage = workerThreadReceivedMessage; + } + + // Strip character from UTF-8 BOM encoded files that cause issue parsing the file + function stripBom(string) { + if (string.charCodeAt(0) === 0xfeff) { + return string.slice(1); + } + return string; + } + + function CsvToJson(_input, _config) + { + _config = _config || {}; + var dynamicTyping = _config.dynamicTyping || false; + if (isFunction(dynamicTyping)) { + _config.dynamicTypingFunction = dynamicTyping; + // Will be filled on first row call + dynamicTyping = {}; + } + _config.dynamicTyping = dynamicTyping; + + _config.transform = isFunction(_config.transform) ? _config.transform : false; + + if (_config.downloadTimeout !== undefined) + { + var downloadTimeout = parseInt(_config.downloadTimeout); + if (isNaN(downloadTimeout)) + { + throw new Error('Config downloadTimeout value (' + _config.downloadTimeout + ') not parsable by parseInt(val).'); + } + _config.downloadTimeout = downloadTimeout; + } + + if (_config.worker && Papa.WORKERS_SUPPORTED) + { + var w = newWorker(); + + w.userStep = _config.step; + w.userChunk = _config.chunk; + w.userComplete = _config.complete; + w.userError = _config.error; + + _config.step = isFunction(_config.step); + _config.chunk = isFunction(_config.chunk); + _config.complete = isFunction(_config.complete); + _config.error = isFunction(_config.error); + delete _config.worker; // prevent infinite loop + + w.postMessage({ + input: _input, + config: _config, + workerId: w.id + }); + + return; + } + + var streamer = null; + if (_input === Papa.NODE_STREAM_INPUT && typeof PAPA_BROWSER_CONTEXT === 'undefined') + { + // create a node Duplex stream for use + // with .pipe + streamer = new DuplexStreamStreamer(_config); + return streamer.getStream(); + } + else if (typeof _input === 'string') + { + _input = stripBom(_input); + if (_config.download) + streamer = new NetworkStreamer(_config); + else + streamer = new StringStreamer(_config); + } + else if (_input.readable === true && isFunction(_input.read) && isFunction(_input.on)) + { + streamer = new ReadableStreamStreamer(_config); + } + else if ((global.File && _input instanceof File) || _input instanceof Object) // ...Safari. (see issue #106) + streamer = new FileStreamer(_config); + + return streamer.stream(_input); + } + + + + + + + function JsonToCsv(_input, _config) + { + // Default configuration + + /** whether to surround every datum with quotes */ + var _quotes = false; + + /** whether to write headers */ + var _writeHeader = true; + + /** delimiting character(s) */ + var _delimiter = ','; + + /** newline character(s) */ + var _newline = '\r\n'; + + /** quote character */ + var _quoteChar = '"'; + + /** escaped quote character, either "" or " */ + var _escapedQuote = _quoteChar + _quoteChar; + + /** whether to skip empty lines */ + var _skipEmptyLines = false; + + /** the columns (keys) we expect when we unparse objects */ + var _columns = null; + + /** whether to prevent outputting cells that can be parsed as formulae by spreadsheet software (Excel and LibreOffice) */ + var _escapeFormulae = false; + + unpackConfig(); + + var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g'); + + if (typeof _input === 'string') + _input = JSON.parse(_input); + + if (Array.isArray(_input)) + { + if (!_input.length || Array.isArray(_input[0])) + return serialize(null, _input, _skipEmptyLines); + else if (typeof _input[0] === 'object') + return serialize(_columns || Object.keys(_input[0]), _input, _skipEmptyLines); + } + else if (typeof _input === 'object') + { + if (typeof _input.data === 'string') + _input.data = JSON.parse(_input.data); + + if (Array.isArray(_input.data)) + { + if (!_input.fields) + _input.fields = _input.meta && _input.meta.fields || _columns; + + if (!_input.fields) + _input.fields = Array.isArray(_input.data[0]) + ? _input.fields + : typeof _input.data[0] === 'object' + ? Object.keys(_input.data[0]) + : []; + + if (!(Array.isArray(_input.data[0])) && typeof _input.data[0] !== 'object') + _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf'] + } + + return serialize(_input.fields || [], _input.data || [], _skipEmptyLines); + } + + // Default (any valid paths should return before this) + throw new Error('Unable to serialize unrecognized input'); + + + function unpackConfig() + { + if (typeof _config !== 'object') + return; + + if (typeof _config.delimiter === 'string' + && !Papa.BAD_DELIMITERS.filter(function(value) { return _config.delimiter.indexOf(value) !== -1; }).length) + { + _delimiter = _config.delimiter; + } + + if (typeof _config.quotes === 'boolean' + || typeof _config.quotes === 'function' + || Array.isArray(_config.quotes)) + _quotes = _config.quotes; + + if (typeof _config.skipEmptyLines === 'boolean' + || typeof _config.skipEmptyLines === 'string') + _skipEmptyLines = _config.skipEmptyLines; + + if (typeof _config.newline === 'string') + _newline = _config.newline; + + if (typeof _config.quoteChar === 'string') { + _quoteChar = _config.quoteChar; + _escapedQuote = _quoteChar + _quoteChar; + } + + if (typeof _config.header === 'boolean') + _writeHeader = _config.header; + + if (Array.isArray(_config.columns)) { + + if (_config.columns.length === 0) throw new Error('Option columns is empty'); + + _columns = _config.columns; + } + + if (_config.escapeChar !== undefined) { + _escapedQuote = _config.escapeChar + _quoteChar; + } + + if (_config.escapeFormulae instanceof RegExp) { + _escapeFormulae = _config.escapeFormulae; + } else if (typeof _config.escapeFormulae === 'boolean' && _config.escapeFormulae) { + _escapeFormulae = /^[=+\-@\t\r].*$/; + } + } + + /** The double for loop that iterates the data and writes out a CSV string including header row */ + function serialize(fields, data, skipEmptyLines) + { + var csv = ''; + + if (typeof fields === 'string') + fields = JSON.parse(fields); + if (typeof data === 'string') + data = JSON.parse(data); + + var hasHeader = Array.isArray(fields) && fields.length > 0; + var dataKeyedByField = !(Array.isArray(data[0])); + + // If there a header row, write it first + if (hasHeader && _writeHeader) + { + for (var i = 0; i < fields.length; i++) + { + if (i > 0) + csv += _delimiter; + csv += safe(fields[i], i); + } + if (data.length > 0) + csv += _newline; + } + + // Then write out the data + for (var row = 0; row < data.length; row++) + { + var maxCol = hasHeader ? fields.length : data[row].length; + + var emptyLine = false; + var nullLine = hasHeader ? Object.keys(data[row]).length === 0 : data[row].length === 0; + if (skipEmptyLines && !hasHeader) + { + emptyLine = skipEmptyLines === 'greedy' ? data[row].join('').trim() === '' : data[row].length === 1 && data[row][0].length === 0; + } + if (skipEmptyLines === 'greedy' && hasHeader) { + var line = []; + for (var c = 0; c < maxCol; c++) { + var cx = dataKeyedByField ? fields[c] : c; + line.push(data[row][cx]); + } + emptyLine = line.join('').trim() === ''; + } + if (!emptyLine) + { + for (var col = 0; col < maxCol; col++) + { + if (col > 0 && !nullLine) + csv += _delimiter; + var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; + csv += safe(data[row][colIdx], col); + } + if (row < data.length - 1 && (!skipEmptyLines || (maxCol > 0 && !nullLine))) + { + csv += _newline; + } + } + } + return csv; + } + + /** Encloses a value around quotes if needed (makes a value safe for CSV insertion) */ + function safe(str, col) + { + if (typeof str === 'undefined' || str === null) + return ''; + + if (str.constructor === Date) + { + // Return empty string for invalid dates + if (isNaN(str.getTime())) + return ''; + return str.toISOString(); + } + + var needsQuotes = false; + + if (_escapeFormulae && typeof str === "string" && _escapeFormulae.test(str)) { + str = "'" + str; + needsQuotes = true; + } + + var strValue = str.toString(); + var escapedQuoteStr = strValue.replace(quoteCharRegex, _escapedQuote); + + needsQuotes = needsQuotes + || _quotes === true + || (typeof _quotes === 'function' && _quotes(str, col)) + || (Array.isArray(_quotes) && _quotes[col]) + || hasAny(escapedQuoteStr, Papa.BAD_DELIMITERS) + || escapedQuoteStr.indexOf(_delimiter) > -1 + || strValue.indexOf(_quoteChar) > -1 + || escapedQuoteStr.charAt(0) === ' ' + || escapedQuoteStr.charAt(escapedQuoteStr.length - 1) === ' '; + + return needsQuotes ? _quoteChar + escapedQuoteStr + _quoteChar : escapedQuoteStr; + } + + function hasAny(str, substrings) + { + for (var i = 0; i < substrings.length; i++) + if (str.indexOf(substrings[i]) > -1) + return true; + return false; + } + } + + + /** ChunkStreamer is the base prototype for various streamer implementations. */ + function ChunkStreamer(config) + { + this._handle = null; + this._finished = false; + this._completed = false; + this._halted = false; + this._input = null; + this._baseIndex = 0; + this._partialLine = ''; + this._rowCount = 0; + this._start = 0; + this._nextChunk = null; + this.isFirstChunk = true; + this._completeResults = { + data: [], + errors: [], + meta: {} + }; + replaceConfig.call(this, config); + + this.parseChunk = function(chunk, isFakeChunk) + { + // First chunk pre-processing + const skipFirstNLines = parseInt(this._config.skipFirstNLines) || 0; + if (this.isFirstChunk && skipFirstNLines > 0) { + let _newline = this._config.newline; + if (!_newline) { + const quoteChar = this._config.quoteChar || '"'; + _newline = this._handle.guessLineEndings(chunk, quoteChar); + } + const splitChunk = chunk.split(_newline); + chunk = [...splitChunk.slice(skipFirstNLines)].join(_newline); + } + if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) + { + var modifiedChunk = this._config.beforeFirstChunk(chunk); + if (modifiedChunk !== undefined) + chunk = modifiedChunk; + } + this.isFirstChunk = false; + this._halted = false; + + // Rejoin the line we likely just split in two by chunking the file + var aggregate = this._partialLine + chunk; + this._partialLine = ''; + var results = this._handle.parse(aggregate, this._baseIndex, !this._finished); + + if (this._handle.paused() || this._handle.aborted()) { + this._halted = true; + return; + } + + var lastIndex = results.meta.cursor; + + if (!this._finished) + { + this._partialLine = aggregate.substring(lastIndex - this._baseIndex); + this._baseIndex = lastIndex; + } + + if (results && results.data) + this._rowCount += results.data.length; + + var finishedIncludingPreview = this._finished || (this._config.preview && this._rowCount >= this._config.preview); + + if (IS_PAPA_WORKER) + { + global.postMessage({ + results: results, + workerId: Papa.WORKER_ID, + finished: finishedIncludingPreview + }); + } + else if (isFunction(this._config.chunk) && !isFakeChunk) + { + this._config.chunk(results, this._handle); + if (this._handle.paused() || this._handle.aborted()) { + this._halted = true; + return; + } + results = undefined; + this._completeResults = undefined; + } + + if (!this._config.step && !this._config.chunk) { + this._completeResults.data = this._completeResults.data.concat(results.data); + this._completeResults.errors = this._completeResults.errors.concat(results.errors); + this._completeResults.meta = results.meta; + } + + if (!this._completed && finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) { + this._config.complete(this._completeResults, this._input); + this._completed = true; + } + + if (!finishedIncludingPreview && (!results || !results.meta.paused)) + this._nextChunk(); + + return results; + }; + + this._sendError = function(error) + { + if (isFunction(this._config.error)) + this._config.error(error); + else if (IS_PAPA_WORKER && this._config.error) + { + global.postMessage({ + workerId: Papa.WORKER_ID, + error: error, + finished: false + }); + } + }; + + function replaceConfig(config) + { + // Deep-copy the config so we can edit it + var configCopy = copy(config); + configCopy.chunkSize = parseInt(configCopy.chunkSize); // parseInt VERY important so we don't concatenate strings! + if (!config.step && !config.chunk) + configCopy.chunkSize = null; // disable Range header if not streaming; bad values break IIS - see issue #196 + this._handle = new ParserHandle(configCopy); + this._handle.streamer = this; + this._config = configCopy; // persist the copy to the caller + } + } + + + function NetworkStreamer(config) + { + config = config || {}; + if (!config.chunkSize) + config.chunkSize = Papa.RemoteChunkSize; + ChunkStreamer.call(this, config); + + var xhr; + + if (IS_WORKER) + { + this._nextChunk = function() + { + this._readChunk(); + this._chunkLoaded(); + }; + } + else + { + this._nextChunk = function() + { + this._readChunk(); + }; + } + + this.stream = function(url) + { + this._input = url; + this._nextChunk(); // Starts streaming + }; + + this._readChunk = function() + { + if (this._finished) + { + this._chunkLoaded(); + return; + } + + xhr = new XMLHttpRequest(); + + if (this._config.withCredentials) + { + xhr.withCredentials = this._config.withCredentials; + } + + if (!IS_WORKER) + { + xhr.onload = bindFunction(this._chunkLoaded, this); + xhr.onerror = bindFunction(this._chunkError, this); + } + + xhr.ontimeout = bindFunction(this._chunkTimeout, this); + + xhr.open(this._config.downloadRequestBody ? 'POST' : 'GET', this._input, !IS_WORKER); + // Timeout is only supported for asynchronous requests + if (this._config.downloadTimeout && !IS_WORKER) + { + xhr.timeout = this._config.downloadTimeout; + } + // Headers can only be set when once the request state is OPENED + if (this._config.downloadRequestHeaders) + { + var headers = this._config.downloadRequestHeaders; + + for (var headerName in headers) + { + xhr.setRequestHeader(headerName, headers[headerName]); + } + } + + if (this._config.chunkSize) + { + var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive + xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end); + } + + try { + xhr.send(this._config.downloadRequestBody); + } + catch (err) { + this._chunkError(err.message); + } + + if (IS_WORKER && xhr.status === 0) + this._chunkError(); + }; + + this._chunkLoaded = function() + { + if (xhr.readyState !== 4) + return; + + if (xhr.status < 200 || xhr.status >= 400) + { + this._chunkError(); + return; + } + + // Use chunckSize as it may be a diference on reponse lentgh due to characters with more than 1 byte + this._start += this._config.chunkSize ? this._config.chunkSize : xhr.responseText.length; + this._finished = !this._config.chunkSize || this._start >= getFileSize(xhr); + this.parseChunk(xhr.responseText); + }; + + this._chunkError = function(errorMessage) + { + var errorText = xhr.statusText || errorMessage; + this._sendError(new Error(errorText)); + }; + + this._chunkTimeout = function() + { + this._chunkError('Request timed out after ' + this._config.downloadTimeout + 'ms'); + }; + + function getFileSize(xhr) + { + var contentRange = xhr.getResponseHeader('Content-Range'); + if (contentRange === null) { // no content range, then finish! + return -1; + } + return parseInt(contentRange.substring(contentRange.lastIndexOf('/') + 1)); + } + } + NetworkStreamer.prototype = Object.create(ChunkStreamer.prototype); + NetworkStreamer.prototype.constructor = NetworkStreamer; + + + function FileStreamer(config) + { + config = config || {}; + if (!config.chunkSize) + config.chunkSize = Papa.LocalChunkSize; + ChunkStreamer.call(this, config); + + var reader, slice; + + // FileReader is better than FileReaderSync (even in worker) - see http://stackoverflow.com/q/24708649/1048862 + // But Firefox is a pill, too - see issue #76: https://github.com/mholt/PapaParse/issues/76 + var usingAsyncReader = typeof FileReader !== 'undefined'; // Safari doesn't consider it a function - see issue #105 + + this.stream = function(file) + { + this._input = file; + slice = file.slice || file.webkitSlice || file.mozSlice; + + if (usingAsyncReader) + { + reader = new FileReader(); // Preferred method of reading files, even in workers + reader.onload = bindFunction(this._chunkLoaded, this); + reader.onerror = bindFunction(this._chunkError, this); + } + else + reader = new FileReaderSync(); // Hack for running in a web worker in Firefox + + this._nextChunk(); // Starts streaming + }; + + this._nextChunk = function() + { + if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview)) + this._readChunk(); + }; + + this._readChunk = function() + { + var input = this._input; + if (this._config.chunkSize) + { + var end = Math.min(this._start + this._config.chunkSize, this._input.size); + input = slice.call(input, this._start, end); + } + var txt = reader.readAsText(input, this._config.encoding); + if (!usingAsyncReader) + this._chunkLoaded({ target: { result: txt } }); // mimic the async signature + }; + + this._chunkLoaded = function(event) + { + // Very important to increment start each time before handling results + this._start += this._config.chunkSize; + this._finished = !this._config.chunkSize || this._start >= this._input.size; + this.parseChunk(event.target.result); + }; + + this._chunkError = function() + { + this._sendError(reader.error); + }; + + } + FileStreamer.prototype = Object.create(ChunkStreamer.prototype); + FileStreamer.prototype.constructor = FileStreamer; + + + function StringStreamer(config) + { + config = config || {}; + ChunkStreamer.call(this, config); + + var remaining; + this.stream = function(s) + { + remaining = s; + return this._nextChunk(); + }; + this._nextChunk = function() + { + if (this._finished) return; + var size = this._config.chunkSize; + var chunk; + if(size) { + chunk = remaining.substring(0, size); + remaining = remaining.substring(size); + } else { + chunk = remaining; + remaining = ''; + } + this._finished = !remaining; + return this.parseChunk(chunk); + }; + } + StringStreamer.prototype = Object.create(StringStreamer.prototype); + StringStreamer.prototype.constructor = StringStreamer; + + + function ReadableStreamStreamer(config) + { + config = config || {}; + + ChunkStreamer.call(this, config); + + var queue = []; + var parseOnData = true; + var streamHasEnded = false; + + this.pause = function() + { + ChunkStreamer.prototype.pause.apply(this, arguments); + this._input.pause(); + }; + + this.resume = function() + { + ChunkStreamer.prototype.resume.apply(this, arguments); + this._input.resume(); + }; + + this.stream = function(stream) + { + this._input = stream; + + this._input.on('data', this._streamData); + this._input.on('end', this._streamEnd); + this._input.on('error', this._streamError); + }; + + this._checkIsFinished = function() + { + if (streamHasEnded && queue.length === 1) { + this._finished = true; + } + }; + + this._nextChunk = function() + { + this._checkIsFinished(); + if (queue.length) + { + this.parseChunk(queue.shift()); + } + else + { + parseOnData = true; + } + }; + + this._streamData = bindFunction(function(chunk) + { + try + { + queue.push(typeof chunk === 'string' ? chunk : chunk.toString(this._config.encoding)); + + if (parseOnData) + { + parseOnData = false; + this._checkIsFinished(); + this.parseChunk(queue.shift()); + } + } + catch (error) + { + this._streamError(error); + } + }, this); + + this._streamError = bindFunction(function(error) + { + this._streamCleanUp(); + this._sendError(error); + }, this); + + this._streamEnd = bindFunction(function() + { + this._streamCleanUp(); + streamHasEnded = true; + this._streamData(''); + }, this); + + this._streamCleanUp = bindFunction(function() + { + this._input.removeListener('data', this._streamData); + this._input.removeListener('end', this._streamEnd); + this._input.removeListener('error', this._streamError); + }, this); + } + ReadableStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); + ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer; + + + function DuplexStreamStreamer(_config) { + var Duplex = require('stream').Duplex; + var config = copy(_config); + var parseOnWrite = true; + var writeStreamHasFinished = false; + var parseCallbackQueue = []; + var stream = null; + + this._onCsvData = function(results) + { + var data = results.data; + if (!stream.push(data) && !this._handle.paused()) { + // the writeable consumer buffer has filled up + // so we need to pause until more items + // can be processed + this._handle.pause(); + } + }; + + this._onCsvComplete = function() + { + // node will finish the read stream when + // null is pushed + stream.push(null); + }; + + config.step = bindFunction(this._onCsvData, this); + config.complete = bindFunction(this._onCsvComplete, this); + ChunkStreamer.call(this, config); + + this._nextChunk = function() + { + if (writeStreamHasFinished && parseCallbackQueue.length === 1) { + this._finished = true; + } + if (parseCallbackQueue.length) { + parseCallbackQueue.shift()(); + } else { + parseOnWrite = true; + } + }; + + this._addToParseQueue = function(chunk, callback) + { + // add to queue so that we can indicate + // completion via callback + // node will automatically pause the incoming stream + // when too many items have been added without their + // callback being invoked + parseCallbackQueue.push(bindFunction(function() { + this.parseChunk(typeof chunk === 'string' ? chunk : chunk.toString(config.encoding)); + if (isFunction(callback)) { + return callback(); + } + }, this)); + if (parseOnWrite) { + parseOnWrite = false; + this._nextChunk(); + } + }; + + this._onRead = function() + { + if (this._handle.paused()) { + // the writeable consumer can handle more data + // so resume the chunk parsing + this._handle.resume(); + } + }; + + this._onWrite = function(chunk, encoding, callback) + { + this._addToParseQueue(chunk, callback); + }; + + this._onWriteComplete = function() + { + writeStreamHasFinished = true; + // have to write empty string + // so parser knows its done + this._addToParseQueue(''); + }; + + this.getStream = function() + { + return stream; + }; + stream = new Duplex({ + readableObjectMode: true, + decodeStrings: false, + read: bindFunction(this._onRead, this), + write: bindFunction(this._onWrite, this) + }); + stream.once('finish', bindFunction(this._onWriteComplete, this)); + } + if (typeof PAPA_BROWSER_CONTEXT === 'undefined') { + DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); + DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer; + } + + + // Use one ParserHandle per entire CSV file or string + function ParserHandle(_config) + { + // One goal is to minimize the use of regular expressions... + var MAX_FLOAT = Math.pow(2, 53); + var MIN_FLOAT = -MAX_FLOAT; + var FLOAT = /^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/; + var ISO_DATE = /^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/; + var self = this; + var _stepCounter = 0; // Number of times step was called (number of rows parsed) + var _rowCounter = 0; // Number of rows that have been parsed so far + var _input; // The input being parsed + var _parser; // The core parser being used + var _paused = false; // Whether we are paused or not + var _aborted = false; // Whether the parser has aborted or not + var _delimiterError; // Temporary state between delimiter detection and processing results + var _fields = []; // Fields are from the header row of the input, if there is one + var _results = { // The last results returned from the parser + data: [], + errors: [], + meta: {} + }; + + if (isFunction(_config.step)) + { + var userStep = _config.step; + _config.step = function(results) + { + _results = results; + + if (needsHeaderRow()) + processResults(); + else // only call user's step function after header row + { + processResults(); + + // It's possbile that this line was empty and there's no row here after all + if (_results.data.length === 0) + return; + + _stepCounter += results.data.length; + if (_config.preview && _stepCounter > _config.preview) + _parser.abort(); + else { + _results.data = _results.data[0]; + userStep(_results, self); + } + } + }; + } + + /** + * Parses input. Most users won't need, and shouldn't mess with, the baseIndex + * and ignoreLastRow parameters. They are used by streamers (wrapper functions) + * when an input comes in multiple chunks, like from a file. + */ + this.parse = function(input, baseIndex, ignoreLastRow) + { + var quoteChar = _config.quoteChar || '"'; + if (!_config.newline) + _config.newline = this.guessLineEndings(input, quoteChar); + + _delimiterError = false; + if (!_config.delimiter) + { + var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments, _config.delimitersToGuess); + if (delimGuess.successful) + _config.delimiter = delimGuess.bestDelimiter; + else + { + _delimiterError = true; // add error after parsing (otherwise it would be overwritten) + _config.delimiter = Papa.DefaultDelimiter; + } + _results.meta.delimiter = _config.delimiter; + } + else if(isFunction(_config.delimiter)) + { + _config.delimiter = _config.delimiter(input); + _results.meta.delimiter = _config.delimiter; + } + + var parserConfig = copy(_config); + // Tell the parser the header instead of reguessing on each chunk + parserConfig.header = needsHeaderRow(); + if (_config.preview && _config.header) + parserConfig.preview++; // to compensate for header row + + _input = input; + _parser = new Parser(parserConfig); + _results = _parser.parse(_input, baseIndex, ignoreLastRow); + processResults(); + return _paused ? { meta: { paused: true } } : (_results || { meta: { paused: false } }); + }; + + this.paused = function() + { + return _paused; + }; + + this.pause = function() + { + _paused = true; + _parser.abort(); + + // If it is streaming via "chunking", the reader will start appending correctly already so no need to substring, + // otherwise we can get duplicate content within a row + _input = isFunction(_config.chunk) ? "" : _input.substring(_parser.getCharIndex()); + }; + + this.resume = function() + { + if(self.streamer._halted) { + _paused = false; + self.streamer.parseChunk(_input, true); + } else { + // Bugfix: #636 In case the processing hasn't halted yet + // wait for it to halt in order to resume + setTimeout(self.resume, 3); + } + }; + + this.aborted = function() + { + return _aborted; + }; + + this.abort = function() + { + _aborted = true; + _parser.abort(); + _results.meta.aborted = true; + if (isFunction(_config.complete)) + _config.complete(_results); + _input = ''; + }; + + this.guessLineEndings = function(input, quoteChar) + { + input = input.substring(0, 1024 * 1024); // max length 1 MB + // Replace all the text inside quotes + var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm'); + input = input.replace(re, ''); + + var r = input.split('\r'); + + var n = input.split('\n'); + + var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length); + + if (r.length === 1 || nAppearsFirst) + return '\n'; + + var numWithN = 0; + for (var i = 0; i < r.length; i++) + { + if (r[i][0] === '\n') + numWithN++; + } + + return numWithN >= r.length / 2 ? '\r\n' : '\r'; + }; + + function testEmptyLine(s) { + return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0; + } + + function testFloat(s) { + if (FLOAT.test(s)) { + var floatValue = parseFloat(s); + if (floatValue > MIN_FLOAT && floatValue < MAX_FLOAT) { + return true; + } + } + return false; + } + + function processResults() + { + if (_results && _delimiterError) + { + addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \'' + Papa.DefaultDelimiter + '\''); + _delimiterError = false; + } + + if (_config.skipEmptyLines) + { + _results.data = _results.data.filter(function(d) { + return !testEmptyLine(d); + }); + } + + if (needsHeaderRow()) + fillHeaderFields(); + + return applyHeaderAndDynamicTypingAndTransformation(); + } + + function needsHeaderRow() + { + return _config.header && _fields.length === 0; + } + + function fillHeaderFields() + { + if (!_results) + return; + + function addHeader(header) + { + _fields.push(header); + } + + if (Array.isArray(_results.data[0])) + { + for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) + _results.data[i].forEach(addHeader); + + _results.data.splice(0, 1); + } + // if _results.data[0] is not an array, we are in a step where _results.data is the row. + else + _results.data.forEach(addHeader); + } + + function shouldApplyDynamicTyping(field) { + // Cache function values to avoid calling it for each row + if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === undefined) { + _config.dynamicTyping[field] = _config.dynamicTypingFunction(field); + } + return (_config.dynamicTyping[field] || _config.dynamicTyping) === true; + } + + function parseDynamic(field, value) + { + if (shouldApplyDynamicTyping(field)) + { + if (value === 'true' || value === 'TRUE') + return true; + else if (value === 'false' || value === 'FALSE') + return false; + else if (testFloat(value)) + return parseFloat(value); + else if (ISO_DATE.test(value)) + return new Date(value); + else + return (value === '' ? null : value); + } + return value; + } + + function applyHeaderAndDynamicTypingAndTransformation() + { + if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform)) + return _results; + + function processRow(rowSource, i) + { + var row = _config.header ? {} : []; + + var j; + for (j = 0; j < rowSource.length; j++) + { + var field = j; + var value = rowSource[j]; + + if (_config.header) + field = j >= _fields.length ? '__parsed_extra' : _fields[j]; + + if (_config.transform) + value = _config.transform(value,field); + + value = parseDynamic(field, value); + + if (field === '__parsed_extra') + { + row[field] = row[field] || []; + row[field].push(value); + } + else + row[field] = value; + } + + + if (_config.header) + { + if (j > _fields.length) + addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); + else if (j < _fields.length) + addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); + } + + return row; + } + + var incrementBy = 1; + if (!_results.data.length || Array.isArray(_results.data[0])) + { + _results.data = _results.data.map(processRow); + incrementBy = _results.data.length; + } + else + _results.data = processRow(_results.data, 0); + + + if (_config.header && _results.meta) + _results.meta.fields = _fields; + + _rowCounter += incrementBy; + return _results; + } + + function guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess) { + var bestDelim, bestDelta, fieldCountPrevRow, maxFieldCount; + + delimitersToGuess = delimitersToGuess || [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; + + for (var i = 0; i < delimitersToGuess.length; i++) { + var delim = delimitersToGuess[i]; + var delta = 0, avgFieldCount = 0, emptyLinesCount = 0; + fieldCountPrevRow = undefined; + + var preview = new Parser({ + comments: comments, + delimiter: delim, + newline: newline, + preview: 10 + }).parse(input); + + for (var j = 0; j < preview.data.length; j++) { + if (skipEmptyLines && testEmptyLine(preview.data[j])) { + emptyLinesCount++; + continue; + } + var fieldCount = preview.data[j].length; + avgFieldCount += fieldCount; + + if (typeof fieldCountPrevRow === 'undefined') { + fieldCountPrevRow = fieldCount; + continue; + } + else if (fieldCount > 0) { + delta += Math.abs(fieldCount - fieldCountPrevRow); + fieldCountPrevRow = fieldCount; + } + } + + if (preview.data.length > 0) + avgFieldCount /= (preview.data.length - emptyLinesCount); + + // Lowest delta (most consistent field count across rows) wins; average + // field count only breaks ties between equally consistent delimiters. + if (avgFieldCount > 1.99 && (typeof bestDelta === 'undefined' + || delta < bestDelta + || (delta === bestDelta && avgFieldCount > maxFieldCount))) { + bestDelta = delta; + bestDelim = delim; + maxFieldCount = avgFieldCount; + } + } + + _config.delimiter = bestDelim; + + return { + successful: !!bestDelim, + bestDelimiter: bestDelim + }; + } + + function addError(type, code, msg, row) + { + var error = { + type: type, + code: code, + message: msg + }; + if(row !== undefined) { + error.row = row; + } + _results.errors.push(error); + } + } + + /** https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */ + function escapeRegExp(string) + { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string + } + + /** The core parser implements speedy and correct CSV parsing */ + function Parser(config) + { + // Unpack the config object + config = config || {}; + var delim = config.delimiter; + var newline = config.newline; + var comments = config.comments; + var step = config.step; + var preview = config.preview; + var fastMode = config.fastMode; + var quoteChar; + var renamedHeaders = null; + var headerParsed = false; + + if (config.quoteChar === undefined || config.quoteChar === null) { + quoteChar = '"'; + } else { + quoteChar = config.quoteChar; + } + var escapeChar = quoteChar; + if (config.escapeChar !== undefined) { + escapeChar = config.escapeChar; + } + + // Delimiter must be valid + if (typeof delim !== 'string' + || Papa.BAD_DELIMITERS.indexOf(delim) > -1) + delim = ','; + + // Comment character must be valid + if (comments === delim) + throw new Error('Comment character same as delimiter'); + else if (comments === true) + comments = '#'; + else if (typeof comments !== 'string' + || Papa.BAD_DELIMITERS.indexOf(comments) > -1) + comments = false; + + // Newline must be valid: \r, \n, or \r\n + if (newline !== '\n' && newline !== '\r' && newline !== '\r\n') + newline = '\n'; + + // We're gonna need these at the Parser scope + var cursor = 0; + var aborted = false; + + this.parse = function(input, baseIndex, ignoreLastRow) + { + // For some reason, in Chrome, this speeds things up (!?) + if (typeof input !== 'string') + throw new Error('Input must be a string'); + + // We don't need to compute some of these every time parse() is called, + // but having them in a more local scope seems to perform better + var inputLen = input.length, + delimLen = delim.length, + newlineLen = newline.length, + commentsLen = comments.length; + var stepIsFunction = isFunction(step); + + // Establish starting state + cursor = 0; + var data = [], errors = [], row = [], lastCursor = 0; + + if (!input) + return returnable(); + + if (fastMode || (fastMode !== false && input.indexOf(quoteChar) === -1)) + { + var rows = input.split(newline); + for (var i = 0; i < rows.length; i++) + { + row = rows[i]; + cursor += row.length; + + if (i !== rows.length - 1) + cursor += newline.length; + else if (ignoreLastRow) + return returnable(); + if (comments && row.substring(0, commentsLen) === comments) + continue; + if (stepIsFunction) + { + data = []; + pushRow(row.split(delim)); + doStep(); + if (aborted) + return returnable(); + } + else + pushRow(row.split(delim)); + if (preview && i >= preview) + { + data = data.slice(0, preview); + return returnable(true); + } + } + return returnable(); + } + + var nextDelim = input.indexOf(delim, cursor); + var nextNewline = input.indexOf(newline, cursor); + var quoteCharRegex = new RegExp(escapeRegExp(escapeChar) + escapeRegExp(quoteChar), 'g'); + var quoteSearch = input.indexOf(quoteChar, cursor); + + // Parser loop + for (;;) + { + // Field has opening quote + if (input[cursor] === quoteChar) + { + // Start our search for the closing quote where the cursor is + quoteSearch = cursor; + + // Skip the opening quote + cursor++; + + for (;;) + { + // Find closing quote + quoteSearch = input.indexOf(quoteChar, quoteSearch + 1); + + //No other quotes are found - no other delimiters + if (quoteSearch === -1) + { + if (!ignoreLastRow) { + // No closing quote... what a pity + errors.push({ + type: 'Quotes', + code: 'MissingQuotes', + message: 'Quoted field unterminated', + row: data.length, // row has yet to be inserted + index: cursor + }); + } + return finish(); + } + + // Closing quote at EOF + if (quoteSearch === inputLen - 1) + { + var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar); + return finish(value); + } + + // If this quote is escaped, it's part of the data; skip it + // If the quote character is the escape character, then check if the next character is the escape character + if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar) + { + quoteSearch++; + continue; + } + + // If the quote character is not the escape character, then check if the previous character was the escape character + if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch - 1] === escapeChar) + { + continue; + } + + if(nextDelim !== -1 && nextDelim < (quoteSearch + 1)) { + nextDelim = input.indexOf(delim, (quoteSearch + 1)); + } + if(nextNewline !== -1 && nextNewline < (quoteSearch + 1)) { + nextNewline = input.indexOf(newline, (quoteSearch + 1)); + } + // Check up to nextDelim or nextNewline, whichever is closest + var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline); + var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo); + + // Closing quote followed by delimiter or 'unnecessary spaces + delimiter' + if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndDelimiter, delimLen) === delim) + { + row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); + cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen; + + // If char after following delimiter is not quoteChar, we find next quote char position + if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen] !== quoteChar) + { + quoteSearch = input.indexOf(quoteChar, cursor); + } + nextDelim = input.indexOf(delim, cursor); + nextNewline = input.indexOf(newline, cursor); + break; + } + + var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline); + + // Closing quote followed by newline or 'unnecessary spaces + newLine' + if (input.substring(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen) === newline) + { + row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); + saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen); + nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field + quoteSearch = input.indexOf(quoteChar, cursor); // we search for first quote in next line + + if (stepIsFunction) + { + doStep(); + if (aborted) + return returnable(); + } + + if (preview && data.length >= preview) + return returnable(true); + + break; + } + + + // Checks for valid closing quotes are complete (escaped quotes or quote followed by EOF/delimiter/newline) -- assume these quotes are part of an invalid text string + errors.push({ + type: 'Quotes', + code: 'InvalidQuotes', + message: 'Trailing quote on quoted field is malformed', + row: data.length, // row has yet to be inserted + index: cursor + }); + + quoteSearch++; + continue; + + } + + continue; + } + + // Comment found at start of new line + if (comments && row.length === 0 && input.substring(cursor, cursor + commentsLen) === comments) + { + if (nextNewline === -1) // Comment ends at EOF + return returnable(); + cursor = nextNewline + newlineLen; + nextNewline = input.indexOf(newline, cursor); + nextDelim = input.indexOf(delim, cursor); + continue; + } + + // Next delimiter comes before next newline, so we've reached end of field + if (nextDelim !== -1 && (nextDelim < nextNewline || nextNewline === -1)) + { + row.push(input.substring(cursor, nextDelim)); + cursor = nextDelim + delimLen; + // we look for next delimiter char + nextDelim = input.indexOf(delim, cursor); + continue; + } + + // End of row + if (nextNewline !== -1) + { + row.push(input.substring(cursor, nextNewline)); + saveRow(nextNewline + newlineLen); + + if (stepIsFunction) + { + doStep(); + if (aborted) + return returnable(); + } + + if (preview && data.length >= preview) + return returnable(true); + + continue; + } + + break; + } + + return finish(); + + + function pushRow(row) + { + data.push(row); + lastCursor = cursor; + } + + /** + * checks if there are extra spaces after closing quote and given index without any text + * if Yes, returns the number of spaces + */ + function extraSpaces(index) { + var spaceLength = 0; + if (index !== -1) { + var textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index); + if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === '') { + spaceLength = textBetweenClosingQuoteAndIndex.length; + } + } + return spaceLength; + } + + /** + * Appends the remaining input from cursor to the end into + * row, saves the row, calls step, and returns the results. + */ + function finish(value) + { + if (ignoreLastRow) + return returnable(); + if (typeof value === 'undefined') + value = input.substring(cursor); + row.push(value); + cursor = inputLen; // important in case parsing is paused + pushRow(row); + if (stepIsFunction) + doStep(); + return returnable(); + } + + /** + * Appends the current row to the results. It sets the cursor + * to newCursor and finds the nextNewline. The caller should + * take care to execute user's step function and check for + * preview and end parsing if necessary. + */ + function saveRow(newCursor) + { + cursor = newCursor; + pushRow(row); + row = []; + nextNewline = input.indexOf(newline, cursor); + } + + /** Returns an object with the results, errors, and meta. */ + function returnable(stopped) + { + if (config.header && !baseIndex && data.length && !headerParsed) + { + const result = data[0]; + const headerCount = Object.create(null); // To track the count of each base header + const usedHeaders = new Set(result); // To track used headers and avoid duplicates + let duplicateHeaders = false; + + for (let i = 0; i < result.length; i++) { + let header = stripBom(result[i]); + if (isFunction(config.transformHeader)) + header = config.transformHeader(header, i); + + if (!headerCount[header]) { + headerCount[header] = 1; + result[i] = header; + } else { + let newHeader; + let suffixCount = headerCount[header]; + + // Find a unique new header + do { + newHeader = `${header}_${suffixCount}`; + suffixCount++; + } while (usedHeaders.has(newHeader)); + + usedHeaders.add(newHeader); // Mark this new Header as used + result[i] = newHeader; + headerCount[header]++; + duplicateHeaders = true; + if (renamedHeaders === null) { + renamedHeaders = {}; + } + renamedHeaders[newHeader] = header; + } + + usedHeaders.add(header); // Ensure the original header is marked as used + } + if (duplicateHeaders) { + console.warn('Duplicate headers found and renamed.'); + } + headerParsed = true; + } + return { + data: data, + errors: errors, + meta: { + delimiter: delim, + linebreak: newline, + aborted: aborted, + truncated: !!stopped, + cursor: lastCursor + (baseIndex || 0), + renamedHeaders: renamedHeaders + } + }; + } + + /** Executes the user's step function and resets data & errors. */ + function doStep() + { + step(returnable()); + data = []; + errors = []; + } + }; + + /** Sets the abort flag */ + this.abort = function() + { + aborted = true; + }; + + /** Gets the cursor position */ + this.getCharIndex = function() + { + return cursor; + }; + } + + + function newWorker() + { + if (!Papa.WORKERS_SUPPORTED) + return false; + + var workerUrl = getWorkerBlob(); + var w = new global.Worker(workerUrl); + w.onmessage = mainThreadReceivedMessage; + w.id = workerIdCounter++; + workers[w.id] = w; + return w; + } + + /** Callback when main thread receives a message */ + function mainThreadReceivedMessage(e) + { + var msg = e.data; + var worker = workers[msg.workerId]; + var aborted = false; + + if (msg.error) + worker.userError(msg.error, msg.file); + else if (msg.results && msg.results.data) + { + var abort = function() { + aborted = true; + completeWorker(msg.workerId, { data: [], errors: [], meta: { aborted: true } }); + }; + + var handle = { + abort: abort, + pause: notImplemented, + resume: notImplemented + }; + + if (isFunction(worker.userStep)) + { + for (var i = 0; i < msg.results.data.length; i++) + { + worker.userStep({ + data: msg.results.data[i], + errors: msg.results.errors, + meta: msg.results.meta + }, handle); + if (aborted) + break; + } + delete msg.results; // free memory ASAP + } + else if (isFunction(worker.userChunk)) + { + worker.userChunk(msg.results, handle, msg.file); + delete msg.results; + } + } + + if (msg.finished && !aborted) + completeWorker(msg.workerId, msg.results); + } + + function completeWorker(workerId, results) { + var worker = workers[workerId]; + if (isFunction(worker.userComplete)) + worker.userComplete(results); + worker.terminate(); + delete workers[workerId]; + } + + function notImplemented() { + throw new Error('Not implemented.'); + } + + /** Callback when worker thread receives a message */ + function workerThreadReceivedMessage(e) + { + var msg = e.data; + + if (typeof Papa.WORKER_ID === 'undefined' && msg) + Papa.WORKER_ID = msg.workerId; + + if (typeof msg.input === 'string') + { + global.postMessage({ + workerId: Papa.WORKER_ID, + results: Papa.parse(msg.input, msg.config), + finished: true + }); + } + else if ((global.File && msg.input instanceof File) || msg.input instanceof Object) // thank you, Safari (see issue #106) + { + var results = Papa.parse(msg.input, msg.config); + if (results) + global.postMessage({ + workerId: Papa.WORKER_ID, + results: results, + finished: true + }); + } + } + + /** Makes a deep copy of an array or object (mostly) */ + function copy(obj) + { + if (typeof obj !== 'object' || obj === null) + return obj; + var cpy = Array.isArray(obj) ? [] : {}; + for (var key in obj) + cpy[key] = copy(obj[key]); + return cpy; + } + + function bindFunction(f, self) + { + return function() { f.apply(self, arguments); }; + } + function isFunction(func) + { + return typeof func === 'function'; + } + + return Papa; +})); diff --git a/tests/test-cases.js b/tests/test-cases.js new file mode 100644 index 0000000..72c540a --- /dev/null +++ b/tests/test-cases.js @@ -0,0 +1,3032 @@ +var chai; +var Papa; +if (typeof module !== 'undefined' && module.exports) { + chai = require('chai'); + Papa = require('../papaparse.js'); +} + +var assert = chai.assert; + +var BASE_PATH = (typeof document === 'undefined') ? './' : document.getElementById('test-cases').src.replace(/test-cases\.js$/, ''); +var RECORD_SEP = String.fromCharCode(30); +var UNIT_SEP = String.fromCharCode(31); +var FILES_ENABLED = false; +try { + new File([""], ""); // eslint-disable-line no-new + // Required since Node20 as it ads a FileGlobal but not a FileReaderSync + new FileReaderSync(); // eslint-disable-line no-new + FILES_ENABLED = true; +} catch (e) {} // safari, ie + +var XHR_ENABLED = false; +try { + new XMLHttpRequest(); // eslint-disable-line no-new + XHR_ENABLED = true; +} catch (e) {} // safari, ie + +// Tests for the core parser using new Papa.Parser().parse() (CSV to JSON) +var CORE_PARSER_TESTS = [ + { + description: "One row", + input: 'A,b,c', + expected: { + data: [['A', 'b', 'c']], + errors: [], + meta: {delimiter: ',', renamedHeaders: null} + } + }, + { + description: "Two rows", + input: 'A,b,c\nd,E,f', + expected: { + data: [['A', 'b', 'c'], ['d', 'E', 'f']], + errors: [] + } + }, + { + description: "Three rows", + input: 'A,b,c\nd,E,f\nG,h,i', + expected: { + data: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']], + errors: [] + } + }, + { + description: "Whitespace at edges of unquoted field", + input: 'a, b ,c', + notes: "Extra whitespace should graciously be preserved", + expected: { + data: [['a', ' b ', 'c']], + errors: [] + } + }, + { + description: "Quoted field", + input: 'A,"B",C', + expected: { + data: [['A', 'B', 'C']], + errors: [] + } + }, + { + description: "Quoted field with extra whitespace on edges", + input: 'A," B ",C', + expected: { + data: [['A', ' B ', 'C']], + errors: [] + } + }, + { + description: "Quoted field with delimiter", + input: 'A,"B,B",C', + expected: { + data: [['A', 'B,B', 'C']], + errors: [] + } + }, + { + description: "Quoted field with line break", + input: 'A,"B\nB",C', + expected: { + data: [['A', 'B\nB', 'C']], + errors: [] + } + }, + { + description: "Quoted fields with line breaks", + input: 'A,"B\nB","C\nC\nC"', + expected: { + data: [['A', 'B\nB', 'C\nC\nC']], + errors: [] + } + }, + { + description: "Quoted fields at end of row with delimiter and line break", + input: 'a,b,"c,c\nc"\nd,e,f', + expected: { + data: [['a', 'b', 'c,c\nc'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Quoted field with escaped quotes", + input: 'A,"B""B""B",C', + expected: { + data: [['A', 'B"B"B', 'C']], + errors: [] + } + }, + { + description: "Quoted field with escaped quotes at boundaries", + input: 'A,"""B""",C', + expected: { + data: [['A', '"B"', 'C']], + errors: [] + } + }, + { + description: "Unquoted field with quotes at end of field", + notes: "The quotes character is misplaced, but shouldn't generate an error or break the parser", + input: 'A,B",C', + expected: { + data: [['A', 'B"', 'C']], + errors: [] + } + }, + { + description: "Quoted field with quotes around delimiter", + input: 'A,""",""",C', + notes: "For a boundary to exist immediately before the quotes, we must not already be in quotes", + expected: { + data: [['A', '","', 'C']], + errors: [] + } + }, + { + description: "Quoted field with quotes on right side of delimiter", + input: 'A,",""",C', + notes: "Similar to the test above but with quotes only after the comma", + expected: { + data: [['A', ',"', 'C']], + errors: [] + } + }, + { + description: "Quoted field with quotes on left side of delimiter", + input: 'A,""",",C', + notes: "Similar to the test above but with quotes only before the comma", + expected: { + data: [['A', '",', 'C']], + errors: [] + } + }, + { + description: "Quoted field with 5 quotes in a row and a delimiter in there, too", + input: '"1","cnonce="""",nc=""""","2"', + notes: "Actual input reported in issue #121", + expected: { + data: [['1', 'cnonce="",nc=""', '2']], + errors: [] + } + }, + { + description: "Quoted field with whitespace around quotes", + input: 'A, "B" ,C', + notes: "The quotes must be immediately adjacent to the delimiter to indicate a quoted field", + expected: { + data: [['A', ' "B" ', 'C']], + errors: [] + } + }, + { + description: "Misplaced quotes in data, not as opening quotes", + input: 'A,B "B",C', + notes: "The input is technically malformed, but this syntax should not cause an error", + expected: { + data: [['A', 'B "B"', 'C']], + errors: [] + } + }, + { + description: "Quoted field has no closing quote", + input: 'a,"b,c\nd,e,f', + expected: { + data: [['a', 'b,c\nd,e,f']], + errors: [{ + "type": "Quotes", + "code": "MissingQuotes", + "message": "Quoted field unterminated", + "row": 0, + "index": 3 + }] + } + }, + { + description: "Quoted field has invalid trailing quote after delimiter with a valid closer", + input: '"a,"b,c"\nd,e,f', + notes: "The input is malformed, opening quotes identified, trailing quote is malformed. Trailing quote should be escaped or followed by valid new line or delimiter to be valid", + expected: { + data: [['a,"b,c'], ['d', 'e', 'f']], + errors: [{ + "type": "Quotes", + "code": "InvalidQuotes", + "message": "Trailing quote on quoted field is malformed", + "row": 0, + "index": 1 + }] + } + }, + { + description: "Quoted field has invalid trailing quote after delimiter", + input: 'a,"b,"c\nd,e,f', + notes: "The input is malformed, opening quotes identified, trailing quote is malformed. Trailing quote should be escaped or followed by valid new line or delimiter to be valid", + expected: { + data: [['a', 'b,"c\nd,e,f']], + errors: [{ + "type": "Quotes", + "code": "InvalidQuotes", + "message": "Trailing quote on quoted field is malformed", + "row": 0, + "index": 3 + }, + { + "type": "Quotes", + "code": "MissingQuotes", + "message": "Quoted field unterminated", + "row": 0, + "index": 3 + }] + } + }, + { + description: "Quoted field has invalid trailing quote before delimiter", + input: 'a,"b"c,d\ne,f,g', + notes: "The input is malformed, opening quotes identified, trailing quote is malformed. Trailing quote should be escaped or followed by valid new line or delimiter to be valid", + expected: { + data: [['a', 'b"c,d\ne,f,g']], + errors: [{ + "type": "Quotes", + "code": "InvalidQuotes", + "message": "Trailing quote on quoted field is malformed", + "row": 0, + "index": 3 + }, + { + "type": "Quotes", + "code": "MissingQuotes", + "message": "Quoted field unterminated", + "row": 0, + "index": 3 + }] + } + }, + { + description: "Quoted field has invalid trailing quote after new line", + input: 'a,"b,c\nd"e,f,g', + notes: "The input is malformed, opening quotes identified, trailing quote is malformed. Trailing quote should be escaped or followed by valid new line or delimiter to be valid", + expected: { + data: [['a', 'b,c\nd"e,f,g']], + errors: [{ + "type": "Quotes", + "code": "InvalidQuotes", + "message": "Trailing quote on quoted field is malformed", + "row": 0, + "index": 3 + }, + { + "type": "Quotes", + "code": "MissingQuotes", + "message": "Quoted field unterminated", + "row": 0, + "index": 3 + }] + } + }, + { + description: "Quoted field has valid trailing quote via delimiter", + input: 'a,"b",c\nd,e,f', + notes: "Trailing quote is valid due to trailing delimiter", + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Quoted field has valid trailing quote via \\n", + input: 'a,b,"c"\nd,e,f', + notes: "Trailing quote is valid due to trailing new line delimiter", + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Quoted field has valid trailing quote via EOF", + input: 'a,b,c\nd,e,"f"', + notes: "Trailing quote is valid due to EOF", + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Quoted field contains delimiters and \\n with valid trailing quote", + input: 'a,"b,c\nd,e,f"', + notes: "Trailing quote is valid due to trailing delimiter", + expected: { + data: [['a', 'b,c\nd,e,f']], + errors: [] + } + }, + { + description: "Line starts with quoted field", + input: 'a,b,c\n"d",e,f', + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Line starts with unquoted empty field", + input: ',b,c\n"d",e,f', + expected: { + data: [['', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Line ends with quoted field", + input: 'a,b,c\nd,e,f\n"g","h","i"\n"j","k","l"', + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i'], ['j', 'k', 'l']], + errors: [] + } + }, + { + description: "Line ends with quoted field, first field of next line is empty, \\n", + input: 'a,b,c\n,e,f\n,"h","i"\n,"k","l"', + config: { + newline: '\n', + }, + expected: { + data: [['a', 'b', 'c'], ['', 'e', 'f'], ['', 'h', 'i'], ['', 'k', 'l']], + errors: [] + } + }, + { + description: "Quoted field at end of row (but not at EOF) has quotes", + input: 'a,b,"c""c"""\nd,e,f', + expected: { + data: [['a', 'b', 'c"c"'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Empty quoted field at EOF is empty", + input: 'a,b,""\na,b,""', + expected: { + data: [['a', 'b', ''], ['a', 'b', '']], + errors: [] + } + }, + { + description: "Multiple consecutive empty fields", + input: 'a,b,,,c,d\n,,e,,,f', + expected: { + data: [['a', 'b', '', '', 'c', 'd'], ['', '', 'e', '', '', 'f']], + errors: [] + } + }, + { + description: "Empty input string", + input: '', + expected: { + data: [], + errors: [] + } + }, + { + description: "Input is just the delimiter (2 empty fields)", + input: ',', + expected: { + data: [['', '']], + errors: [] + } + }, + { + description: "Input is just empty fields", + input: ',,\n,,,', + expected: { + data: [['', '', ''], ['', '', '', '']], + errors: [] + } + }, + { + description: "Input is just a string (a single field)", + input: 'Abc def', + expected: { + data: [['Abc def']], + errors: [] + } + }, + { + description: "Commented line at beginning", + input: '# Comment!\na,b,c', + config: { comments: true }, + expected: { + data: [['a', 'b', 'c']], + errors: [] + } + }, + { + description: "Commented line in middle", + input: 'a,b,c\n# Comment\nd,e,f', + config: { comments: true }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Commented line at end", + input: 'a,true,false\n# Comment', + config: { comments: true }, + expected: { + data: [['a', 'true', 'false']], + errors: [] + } + }, + { + description: "Two comment lines consecutively", + input: 'a,b,c\n#comment1\n#comment2\nd,e,f', + config: { comments: true }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Two comment lines consecutively at end of file", + input: 'a,b,c\n#comment1\n#comment2', + config: { comments: true }, + expected: { + data: [['a', 'b', 'c']], + errors: [] + } + }, + { + description: "Three comment lines consecutively at beginning of file", + input: '#comment1\n#comment2\n#comment3\na,b,c', + config: { comments: true }, + expected: { + data: [['a', 'b', 'c']], + errors: [] + } + }, + { + description: "Entire file is comment lines", + input: '#comment1\n#comment2\n#comment3', + config: { comments: true }, + expected: { + data: [], + errors: [] + } + }, + { + description: "Comment with non-default character", + input: 'a,b,c\n!Comment goes here\nd,e,f', + config: { comments: '!' }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Bad comments value specified", + notes: "Should silently disable comment parsing", + input: 'a,b,c\n5comment\nd,e,f', + config: { comments: 5 }, + expected: { + data: [['a', 'b', 'c'], ['5comment'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Multi-character comment string", + input: 'a,b,c\n=N(Comment)\nd,e,f', + config: { comments: "=N(" }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Input with only a commented line", + input: '#commented line', + config: { comments: true, delimiter: ',' }, + expected: { + data: [], + errors: [] + } + }, + { + description: "Input with only a commented line and blank line after", + input: '#commented line\n', + config: { comments: true, delimiter: ',' }, + expected: { + data: [['']], + errors: [] + } + }, + { + description: "Input with only a commented line, without comments enabled", + input: '#commented line', + config: { delimiter: ',' }, + expected: { + data: [['#commented line']], + errors: [] + } + }, + { + description: "Input without comments with line starting with whitespace", + input: 'a\n b\nc', + config: { delimiter: ',' }, + notes: "\" \" == false, but \" \" !== false, so === comparison is required", + expected: { + data: [['a'], [' b'], ['c']], + errors: [] + } + }, + { + description: "Multiple rows, one column (no delimiter found)", + input: 'a\nb\nc\nd\ne', + expected: { + data: [['a'], ['b'], ['c'], ['d'], ['e']], + errors: [] + } + }, + { + description: "One column input with empty fields", + input: 'a\nb\n\n\nc\nd\ne\n', + expected: { + data: [['a'], ['b'], [''], [''], ['c'], ['d'], ['e'], ['']], + errors: [] + } + }, + { + description: "Fast mode, basic", + input: 'a,b,c\nd,e,f', + config: { fastMode: true }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Fast mode with comments", + input: '// Commented line\na,b,c', + config: { fastMode: true, comments: "//" }, + expected: { + data: [['a', 'b', 'c']], + errors: [] + } + }, + { + description: "Fast mode with preview", + input: 'a,b,c\nd,e,f\nh,j,i\n', + config: { fastMode: true, preview: 2 }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Fast mode with blank line at end", + input: 'a,b,c\n', + config: { fastMode: true }, + expected: { + data: [['a', 'b', 'c'], ['']], + errors: [] + } + }, + { + description: "Simple duplicated header names", + input: 'A,A,A,A\n1,2,3,4', + config: { header: true }, + expected: { + data: [['A', 'A_1', 'A_2', 'A_3'], ['1', '2', '3', '4']], + errors: [], + meta: { + renamedHeaders: {A_1: 'A', A_2: 'A', A_3: 'A'}, + cursor: 15 + } + } + }, + { + description: "Duplicated header names with headerTransform", + input: 'A,A,A,A\n1,2,3,4', + config: { header: true, transformHeader: function(header) { return header.toLowerCase(); } }, + expected: { + data: [['a', 'a_1', 'a_2', 'a_3'], ['1', '2', '3', '4']], + errors: [], + meta: { + renamedHeaders: {a_1: 'a', a_2: 'a', a_3: 'a'}, + cursor: 15 + } + } + }, + { + description: "Duplicated header names existing column", + input: 'c,c,c,c_1\n1,2,3,4', + config: { header: true }, + expected: { + data: [['c', 'c_2', 'c_3', 'c_1'], ['1', '2', '3', '4']], + errors: [], + meta: { + renamedHeaders: {c_2: 'c', c_3: 'c'}, + cursor: 17 + } + } + }, + { + description: "Duplicate header names with __proto__ field", + input: '__proto__,__proto__,__proto__\n1,2,3', + config: { header: true }, + expected: { + data: [['__proto__', '__proto___1', '__proto___2'], ['1', '2', '3']], + errors: [], + meta: { + renamedHeaders: {__proto___1: '__proto__', __proto___2: '__proto__'}, + cursor: 35 + } + } + }, + +]; + +describe('Core Parser Tests', function() { + function generateTest(test) { + (test.disabled ? it.skip : it)(test.description, function() { + var actual = new Papa.Parser(test.config).parse(test.input); + assert.deepEqual(actual.errors, test.expected.errors); + assert.deepEqual(actual.data, test.expected.data); + assert.deepNestedInclude(actual.meta, test.expected.meta || {}); + }); + } + + for (var i = 0; i < CORE_PARSER_TESTS.length; i++) { + generateTest(CORE_PARSER_TESTS[i]); + } +}); + + + +// Tests for Papa.parse() function -- high-level wrapped parser (CSV to JSON) +var PARSE_TESTS = [ + { + description: "Two rows, just \\r", + input: 'A,b,c\rd,E,f', + expected: { + data: [['A', 'b', 'c'], ['d', 'E', 'f']], + errors: [] + } + }, + { + description: "Two rows, \\r\\n", + input: 'A,b,c\r\nd,E,f', + expected: { + data: [['A', 'b', 'c'], ['d', 'E', 'f']], + errors: [] + } + }, + { + description: "Quoted field with \\r\\n", + input: 'A,"B\r\nB",C', + expected: { + data: [['A', 'B\r\nB', 'C']], + errors: [] + } + }, + { + description: "Quoted field with \\r", + input: 'A,"B\rB",C', + expected: { + data: [['A', 'B\rB', 'C']], + errors: [] + } + }, + { + description: "Quoted field with \\n", + input: 'A,"B\nB",C', + expected: { + data: [['A', 'B\nB', 'C']], + errors: [] + } + }, + { + description: "Quoted fields with spaces between closing quote and next delimiter", + input: 'A,"B" ,C,D\r\nE,F,"G" ,H', + expected: { + data: [['A', 'B', 'C','D'],['E', 'F', 'G','H']], + errors: [] + } + }, + { + description: "Quoted fields with spaces between closing quote and next new line", + input: 'A,B,C,"D" \r\nE,F,G,"H" \r\nQ,W,E,R', + expected: { + data: [['A', 'B', 'C','D'],['E', 'F', 'G','H'],['Q', 'W', 'E','R']], + errors: [] + } + }, + { + description: "Quoted fields with spaces after closing quote", + input: 'A,"B" ,C,"D" \r\nE,F,"G" ,"H" \r\nQ,W,"E" ,R', + expected: { + data: [['A', 'B', 'C','D'],['E', 'F', 'G','H'],['Q', 'W', 'E','R']], + errors: [] + } + }, + { + description: "Misplaced quotes in data twice, not as opening quotes", + input: 'A,B",C\nD,E",F', + expected: { + data: [['A', 'B"', 'C'], ['D', 'E"', 'F']], + errors: [] + } + }, + { + description: "Mixed slash n and slash r should choose first as precident", + input: 'a,b,c\nd,e,f\rg,h,i\n', + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f\rg', 'h', 'i'], ['']], + errors: [] + } + }, + { + description: "Header row with one row of data", + input: 'A,B,C\r\na,b,c', + config: { header: true }, + expected: { + data: [{"A": "a", "B": "b", "C": "c"}], + errors: [] + } + }, + { + description: "Header row only", + input: 'A,B,C', + config: { header: true }, + expected: { + data: [], + errors: [] + } + }, + { + description: "Row with too few fields", + input: 'A,B,C\r\na,b', + config: { header: true }, + expected: { + data: [{"A": "a", "B": "b"}], + errors: [{ + "type": "FieldMismatch", + "code": "TooFewFields", + "message": "Too few fields: expected 3 fields but parsed 2", + "row": 0 + }] + } + }, + { + description: "Row with too many fields", + input: 'A,B,C\r\na,b,c,d,e\r\nf,g,h', + config: { header: true }, + expected: { + data: [{"A": "a", "B": "b", "C": "c", "__parsed_extra": ["d", "e"]}, {"A": "f", "B": "g", "C": "h"}], + errors: [{ + "type": "FieldMismatch", + "code": "TooManyFields", + "message": "Too many fields: expected 3 fields but parsed 5", + "row": 0 + }] + } + }, + { + description: "Row with enough fields but blank field in the begining", + input: 'A,B,C\r\n,b1,c1\r\na2,b2,c2', + expected: { + data: [["A", "B", "C"], ['', 'b1', 'c1'], ['a2', 'b2', 'c2']], + errors: [] + } + }, + { + description: "Row with enough fields but blank field in the begining using headers", + input: 'A,B,C\r\n,b1,c1\r\n,b2,c2', + config: { header: true }, + expected: { + data: [{"A": "", "B": "b1", "C": "c1"}, {"A": "", "B": "b2", "C": "c2"}], + errors: [] + } + }, + { + description: "Row with enough fields but blank field at end", + input: 'A,B,C\r\na,b,', + config: { header: true }, + expected: { + data: [{"A": "a", "B": "b", "C": ""}], + errors: [] + } + }, + { + description: "Header rows are transformed when transformHeader function is provided", + input: 'A,B,C\r\na,b,c', + config: { header: true, transformHeader: function(header) { return header.toLowerCase(); } }, + expected: { + data: [{"a": "a", "b": "b", "c": "c"}], + errors: [] + } + }, + { + description: "transformHeader accepts and optional index attribute", + input: 'A,B,C\r\na,b,c', + config: { header: true, transformHeader: function(header, i) { return i % 2 ? header.toLowerCase() : header; } }, + expected: { + data: [{"A": "a", "b": "b", "C": "c"}], + errors: [] + } + }, + { + description: "Line ends with quoted field, first field of next line is empty using headers", + input: 'a,b,"c"\r\nd,e,"f"\r\n,"h","i"\r\n,"k","l"', + config: { + header: true, + newline: '\r\n', + }, + expected: { + data: [ + {a: 'd', b: 'e', c: 'f'}, + {a: '', b: 'h', c: 'i'}, + {a: '', b: 'k', c: 'l'} + ], + errors: [] + } + }, + { + description: "Tab delimiter", + input: 'a\tb\tc\r\nd\te\tf', + config: { delimiter: "\t" }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Pipe delimiter", + input: 'a|b|c\r\nd|e|f', + config: { delimiter: "|" }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "ASCII 30 delimiter", + input: 'a' + RECORD_SEP + 'b' + RECORD_SEP + 'c\r\nd' + RECORD_SEP + 'e' + RECORD_SEP + 'f', + config: { delimiter: RECORD_SEP }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "ASCII 31 delimiter", + input: 'a' + UNIT_SEP + 'b' + UNIT_SEP + 'c\r\nd' + UNIT_SEP + 'e' + UNIT_SEP + 'f', + config: { delimiter: UNIT_SEP }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Bad delimiter (\\n)", + input: 'a,b,c', + config: { delimiter: "\n" }, + notes: "Should silently default to comma", + expected: { + data: [['a', 'b', 'c']], + errors: [] + } + }, + { + description: "Multi-character delimiter", + input: 'a, b, c', + config: { delimiter: ", " }, + expected: { + data: [['a', 'b', 'c']], + errors: [] + } + }, + { + description: "Multi-character delimiter (length 2) with quoted field", + input: 'a, b, "c, e", d', + config: { delimiter: ", " }, + notes: "The quotes must be immediately adjacent to the delimiter to indicate a quoted field", + expected: { + data: [['a', 'b', 'c, e', 'd']], + errors: [] + } + }, + { + description: "Callback delimiter", + input: 'a$ b$ c', + config: { delimiter: function(input) { return input[1] + ' '; } }, + expected: { + data: [['a', 'b', 'c']], + errors: [] + } + }, + { + description: "Dynamic typing converts numeric literals and maintains precision", + input: '1,2.2,1e3\r\n-4,-4.5,-4e-5\r\n-,5a,5-2\r\n16142028098527942586,9007199254740991,-9007199254740992', + config: { dynamicTyping: true }, + expected: { + data: [[1, 2.2, 1000], [-4, -4.5, -0.00004], ["-", "5a", "5-2"], ["16142028098527942586", 9007199254740991, "-9007199254740992"]], + errors: [] + } + }, + { + description: "Dynamic typing converts boolean literals", + input: 'true,false,T,F,TRUE,FALSE,True,False', + config: { dynamicTyping: true }, + expected: { + data: [[true, false, "T", "F", true, false, "True", "False"]], + errors: [] + } + }, + { + description: "Dynamic typing doesn't convert other types", + input: 'A,B,C\r\nundefined,null,[\r\nvar,float,if', + config: { dynamicTyping: true }, + expected: { + data: [["A", "B", "C"], ["undefined", "null", "["], ["var", "float", "if"]], + errors: [] + } + }, + { + description: "Dynamic typing applies to specific columns", + input: 'A,B,C\r\n1,2.2,1e3\r\n-4,-4.5,-4e-5', + config: { header: true, dynamicTyping: { A: true, C: true } }, + expected: { + data: [{"A": 1, "B": "2.2", "C": 1000}, {"A": -4, "B": "-4.5", "C": -0.00004}], + errors: [] + } + }, + { + description: "Dynamic typing applies to specific columns by index", + input: '1,2.2,1e3\r\n-4,-4.5,-4e-5\r\n-,5a,5-2', + config: { dynamicTyping: { 1: true } }, + expected: { + data: [["1", 2.2, "1e3"], ["-4", -4.5, "-4e-5"], ["-", "5a", "5-2"]], + errors: [] + } + }, + { + description: "Dynamic typing can be applied to `__parsed_extra`", + input: 'A,B,C\r\n1,2.2,1e3,5.5\r\n-4,-4.5,-4e-5', + config: { header: true, dynamicTyping: { A: true, C: true, __parsed_extra: true } }, + expected: { + data: [{"A": 1, "B": "2.2", "C": 1000, "__parsed_extra": [5.5]}, {"A": -4, "B": "-4.5", "C": -0.00004}], + errors: [{ + "type": "FieldMismatch", + "code": "TooManyFields", + "message": "Too many fields: expected 3 fields but parsed 4", + "row": 0 + }] + } + }, + { + description: "Dynamic typing by indices can be determined by function", + input: '001,002,003', + config: { dynamicTyping: function(field) { return (field % 2) === 0; } }, + expected: { + data: [[1, "002", 3]], + errors: [] + } + }, + { + description: "Dynamic typing by headers can be determined by function", + input: 'A_as_int,B,C_as_int\r\n001,002,003', + config: { header: true, dynamicTyping: function(field) { return /_as_int$/.test(field); } }, + expected: { + data: [{"A_as_int": 1, "B": "002", "C_as_int": 3}], + errors: [] + } + }, + { + description: "Dynamic typing converts empty values into NULL", + input: '1,2.2,1e3\r\n,NULL,\r\n-,5a,null', + config: { dynamicTyping: true }, + expected: { + data: [[1, 2.2, 1000], [null, "NULL", null], ["-", "5a", "null"]], + errors: [] + } + }, + { + description: "Custom transform function is applied to values", + input: 'A,B,C\r\nd,e,f', + config: { + transform: function(value) { + return value.toLowerCase(); + } + }, + expected: { + data: [["a","b","c"], ["d","e","f"]], + errors: [] + } + }, + { + description: "Custom transform accepts column number also", + input: 'A,B,C\r\nd,e,f', + config: { + transform: function(value, column) { + if (column % 2) { + value = value.toLowerCase(); + } + return value; + } + }, + expected: { + data: [["A","b","C"], ["d","e","f"]], + errors: [] + } + }, + { + description: "Custom transform accepts header name when using header", + input: 'A,B,C\r\nd,e,f', + config: { + header: true, + transform: function(value, name) { + if (name === 'B') { + value = value.toUpperCase(); + } + return value; + } + }, + expected: { + data: [{'A': "d", 'B': "E", 'C': "f"}], + errors: [] + } + }, + { + description: "Dynamic typing converts ISO date strings to Dates", + input: 'ISO date,long date\r\n2018-05-04T21:08:03.269Z,Fri May 04 2018 14:08:03 GMT-0700 (PDT)\r\n2018-05-08T15:20:22.642Z,Tue May 08 2018 08:20:22 GMT-0700 (PDT)', + config: { dynamicTyping: true }, + expected: { + data: [["ISO date", "long date"], [new Date("2018-05-04T21:08:03.269Z"), "Fri May 04 2018 14:08:03 GMT-0700 (PDT)"], [new Date("2018-05-08T15:20:22.642Z"), "Tue May 08 2018 08:20:22 GMT-0700 (PDT)"]], + errors: [] + } + }, + { + description: "Dynamic typing skips ISO date strings ocurring in other strings", + input: 'ISO date,String with ISO date\r\n2018-05-04T21:08:03.269Z,The date is 2018-05-04T21:08:03.269Z\r\n2018-05-08T15:20:22.642Z,The date is 2018-05-08T15:20:22.642Z', + config: { dynamicTyping: true }, + expected: { + data: [["ISO date", "String with ISO date"], [new Date("2018-05-04T21:08:03.269Z"), "The date is 2018-05-04T21:08:03.269Z"], [new Date("2018-05-08T15:20:22.642Z"), "The date is 2018-05-08T15:20:22.642Z"]], + errors: [] + } + }, + { + description: "Blank line at beginning", + input: '\r\na,b,c\r\nd,e,f', + config: { newline: '\r\n' }, + expected: { + data: [[''], ['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Blank line in middle", + input: 'a,b,c\r\n\r\nd,e,f', + config: { newline: '\r\n' }, + expected: { + data: [['a', 'b', 'c'], [''], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Blank lines at end", + input: 'a,b,c\nd,e,f\n\n', + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f'], [''], ['']], + errors: [] + } + }, + { + description: "Blank line in middle with whitespace", + input: 'a,b,c\r\n \r\nd,e,f', + expected: { + data: [['a', 'b', 'c'], [" "], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "First field of a line is empty", + input: 'a,b,c\r\n,e,f', + expected: { + data: [['a', 'b', 'c'], ['', 'e', 'f']], + errors: [] + } + }, + { + description: "Last field of a line is empty", + input: 'a,b,\r\nd,e,f', + expected: { + data: [['a', 'b', ''], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Other fields are empty", + input: 'a,,c\r\n,,', + expected: { + data: [['a', '', 'c'], ['', '', '']], + errors: [] + } + }, + { + description: "Empty input string", + input: '', + expected: { + data: [], + errors: [{ + "type": "Delimiter", + "code": "UndetectableDelimiter", + "message": "Unable to auto-detect delimiting character; defaulted to ','" + }] + } + }, + { + description: "Input is just the delimiter (2 empty fields)", + input: ',', + expected: { + data: [['', '']], + errors: [] + } + }, + { + description: "Input is just a string (a single field)", + input: 'Abc def', + expected: { + data: [['Abc def']], + errors: [ + { + "type": "Delimiter", + "code": "UndetectableDelimiter", + "message": "Unable to auto-detect delimiting character; defaulted to ','" + } + ] + } + }, + { + description: "Preview 0 rows should default to parsing all", + input: 'a,b,c\r\nd,e,f\r\ng,h,i', + config: { preview: 0 }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']], + errors: [] + } + }, + { + description: "Preview 1 row", + input: 'a,b,c\r\nd,e,f\r\ng,h,i', + config: { preview: 1 }, + expected: { + data: [['a', 'b', 'c']], + errors: [] + } + }, + { + description: "Preview 2 rows", + input: 'a,b,c\r\nd,e,f\r\ng,h,i', + config: { preview: 2 }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Preview all (3) rows", + input: 'a,b,c\r\nd,e,f\r\ng,h,i', + config: { preview: 3 }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']], + errors: [] + } + }, + { + description: "Preview more rows than input has", + input: 'a,b,c\r\nd,e,f\r\ng,h,i', + config: { preview: 4 }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']], + errors: [] + } + }, + { + description: "Preview should count rows, not lines", + input: 'a,b,c\r\nd,e,"f\r\nf",g,h,i', + config: { preview: 2 }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f\r\nf', 'g', 'h', 'i']], + errors: [] + } + }, + { + description: "Preview with header row", + notes: "Preview is defined to be number of rows of input not including header row", + input: 'a,b,c\r\nd,e,f\r\ng,h,i\r\nj,k,l', + config: { header: true, preview: 2 }, + expected: { + data: [{"a": "d", "b": "e", "c": "f"}, {"a": "g", "b": "h", "c": "i"}], + errors: [] + } + }, + { + description: "Empty lines", + input: '\na,b,c\n\nd,e,f\n\n', + config: { delimiter: ',' }, + expected: { + data: [[''], ['a', 'b', 'c'], [''], ['d', 'e', 'f'], [''], ['']], + errors: [] + } + }, + { + description: "Skip empty lines", + input: 'a,b,c\n\nd,e,f', + config: { skipEmptyLines: true }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Skip empty lines, with newline at end of input", + input: 'a,b,c\r\n\r\nd,e,f\r\n', + config: { skipEmptyLines: true }, + expected: { + data: [['a', 'b', 'c'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Skip empty lines, with empty input", + input: '', + config: { skipEmptyLines: true }, + expected: { + data: [], + errors: [ + { + "type": "Delimiter", + "code": "UndetectableDelimiter", + "message": "Unable to auto-detect delimiting character; defaulted to ','" + } + ] + } + }, + { + description: "Skip empty lines, with first line only whitespace", + notes: "A line must be absolutely empty to be considered empty", + input: ' \na,b,c', + config: { skipEmptyLines: true, delimiter: ',' }, + expected: { + data: [[" "], ['a', 'b', 'c']], + errors: [] + } + }, + { + description: "Skip empty lines while detecting delimiter", + notes: "Parsing correctly newline-terminated short data with delimiter:auto and skipEmptyLines:true", + input: 'a,b\n1,2\n3,4\n', + config: { header: true, skipEmptyLines: true }, + expected: { + data: [{'a': '1', 'b': '2'}, {'a': '3', 'b': '4'}], + errors: [] + } + }, + { + description: "Lines with comments are not used when guessing the delimiter in an escaped file", + notes: "Guessing the delimiter should work even if there are many lines of comments at the start of the file", + input: '#1\n#2\n#3\n#4\n#5\n#6\n#7\n#8\n#9\n#10\none,"t,w,o",three\nfour,five,six', + config: { comments: '#' }, + expected: { + data: [['one','t,w,o','three'],['four','five','six']], + errors: [] + } + }, + { + description: "Lines with comments are not used when guessing the delimiter in a non-escaped file", + notes: "Guessing the delimiter should work even if there are many lines of comments at the start of the file", + input: '#1\n#2\n#3\n#4\n#5\n#6\n#7\n#8\n#9\n#10\n#11\none,two,three\nfour,five,six', + config: { comments: '#' }, + expected: { + data: [['one','two','three'],['four','five','six']], + errors: [] + } + }, + { + description: "Pipe delimiter is guessed correctly when mixed with comas", + notes: "Guessing the delimiter should work even if there are many lines of comments at the start of the file", + input: 'one|two,two|three\nfour|five,five|six', + config: {}, + expected: { + data: [['one','two,two','three'],['four','five,five','six']], + errors: [] + } + }, + { + description: "Pipe delimiter is guessed correctly choose avgFildCount max one", + notes: "Guessing the delimiter should work choose the min delta one and the max one", + config: {}, + input: 'a,b,c\na,b,c|d|e|f', + expected: { + data: [['a', 'b', 'c'], ['a','b','c|d|e|f']], + errors: [] + } + }, + { + description: "Pipe delimiter is guessed correctly when first field are enclosed in quotes and contain delimiter characters", + notes: "Guessing the delimiter should work if the first field is enclosed in quotes, but others are not", + input: '"Field1,1,1";Field2;"Field3";Field4;Field5;Field6', + config: {}, + expected: { + data: [['Field1,1,1','Field2','Field3', 'Field4', 'Field5', 'Field6']], + errors: [] + } + }, + { + description: "Pipe delimiter is guessed correctly when some fields are enclosed in quotes and contain delimiter characters and escaoped quotes", + notes: "Guessing the delimiter should work even if the first field is not enclosed in quotes, but others are", + input: 'Field1;Field2;"Field,3,""3,3";Field4;Field5;"Field6,6"', + config: {}, + expected: { + data: [['Field1','Field2','Field,3,"3,3', 'Field4', 'Field5', 'Field6,6']], + errors: [] + } + }, + { + description: "Delimiter is guessed correctly when a quoted field holds newlines and other delimiter characters", + notes: "A wrong delimiter that splits the quoted field into many pieces can report a higher average field count; the most consistent delimiter (lowest delta) should still win. See issue #1077", + input: 'id;title;data\r\n1;row one;"{\n ""a"": 1,\n ""b"": 2,\n ""c"": 3,\n ""d"": 4\n}"\r\n2;row two;"{\n ""a"": 1,\n ""b"": 2,\n ""c"": 3,\n ""d"": 4\n}"', + config: {}, + expected: { + data: [ + ['id', 'title', 'data'], + ['1', 'row one', '{\n "a": 1,\n "b": 2,\n "c": 3,\n "d": 4\n}'], + ['2', 'row two', '{\n "a": 1,\n "b": 2,\n "c": 3,\n "d": 4\n}'] + ], + errors: [] + } + }, + { + description: "Single quote as quote character", + notes: "Must parse correctly when single quote is specified as a quote character", + input: "a,b,'c,d'", + config: { quoteChar: "'" }, + expected: { + data: [['a', 'b', 'c,d']], + errors: [] + } + }, + { + description: "Custom escape character in the middle", + notes: "Must parse correctly if the backslash sign (\\) is configured as a custom escape character", + input: 'a,b,"c\\"d\\"f"', + config: { escapeChar: '\\' }, + expected: { + data: [['a', 'b', 'c"d"f']], + errors: [] + } + }, + { + description: "Custom escape character at the end", + notes: "Must parse correctly if the backslash sign (\\) is configured as a custom escape character and the escaped quote character appears at the end of the column", + input: 'a,b,"c\\"d\\""', + config: { escapeChar: '\\' }, + expected: { + data: [['a', 'b', 'c"d"']], + errors: [] + } + }, + { + description: "Custom escape character not used for escaping", + notes: "Must parse correctly if the backslash sign (\\) is configured as a custom escape character and appears as regular character in the text", + input: 'a,b,"c\\d"', + config: { escapeChar: '\\' }, + expected: { + data: [['a', 'b', 'c\\d']], + errors: [] + } + }, + { + description: "Header row with preceding comment", + notes: "Must parse correctly headers if they are preceded by comments", + input: '#Comment\na,b\nc,d\n', + config: { header: true, comments: '#', skipEmptyLines: true, delimiter: ',' }, + expected: { + data: [{'a': 'c', 'b': 'd'}], + errors: [] + } + }, + { + description: "Carriage return in header inside quotes, with line feed endings", + input: '"a\r\na","b"\n"c","d"\n"e","f"\n"g","h"\n"i","j"', + config: {}, + expected: { + data: [['a\r\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [] + } + }, + { + description: "Line feed in header inside quotes, with carriage return + line feed endings", + input: '"a\na","b"\r\n"c","d"\r\n"e","f"\r\n"g","h"\r\n"i","j"', + config: {}, + expected: { + data: [['a\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [] + } + }, + { + description: "Using \\r\\n endings uses \\r\\n linebreak", + input: 'a,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', + config: {}, + expected: { + data: [['a', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\r\n', + delimiter: ',', + cursor: 23, + aborted: false, + truncated: false, + renamedHeaders: null + } + } + }, + { + description: "Using \\n endings uses \\n linebreak", + input: 'a,b\nc,d\ne,f\ng,h\ni,j', + config: {}, + expected: { + data: [['a', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\n', + delimiter: ',', + cursor: 19, + aborted: false, + truncated: false, + renamedHeaders: null + } + } + }, + { + description: "Using \\r\\n endings with \\r\\n in header field uses \\r\\n linebreak", + input: '"a\r\na",b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', + config: {}, + expected: { + data: [['a\r\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\r\n', + delimiter: ',', + cursor: 28, + aborted: false, + truncated: false, + renamedHeaders: null + } + } + }, + { + description: "Using \\r\\n endings with \\n in header field uses \\r\\n linebreak", + input: '"a\na",b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', + config: {}, + expected: { + data: [['a\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\r\n', + delimiter: ',', + cursor: 27, + aborted: false, + truncated: false, + renamedHeaders: null + } + } + }, + { + description: "Using \\r\\n endings with \\n in header field with skip empty lines uses \\r\\n linebreak", + input: '"a\na",b\r\nc,d\r\ne,f\r\ng,h\r\ni,j\r\n', + config: {skipEmptyLines: true}, + expected: { + data: [['a\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\r\n', + delimiter: ',', + cursor: 29, + aborted: false, + truncated: false, + renamedHeaders: null + } + } + }, + { + description: "Using \\n endings with \\r\\n in header field uses \\n linebreak", + input: '"a\r\na",b\nc,d\ne,f\ng,h\ni,j', + config: {}, + expected: { + data: [['a\r\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\n', + delimiter: ',', + cursor: 24, + aborted: false, + truncated: false, + renamedHeaders: null + } + } + }, + { + description: "Using reserved regex character . as quote character", + input: '.a\na.,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', + config: { quoteChar: '.' }, + expected: { + data: [['a\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\r\n', + delimiter: ',', + cursor: 27, + aborted: false, + truncated: false, + renamedHeaders: null + } + } + }, + { + description: "Using reserved regex character | as quote character", + input: '|a\na|,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', + config: { quoteChar: '|' }, + expected: { + data: [['a\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\r\n', + delimiter: ',', + cursor: 27, + aborted: false, + truncated: false, + renamedHeaders: null + } + } + }, + { + description: "UTF-8 BOM encoded input is stripped from invisible BOM character", + input: '\ufeffA,B\nX,Y', + config: {}, + expected: { + data: [['A', 'B'], ['X', 'Y']], + errors: [], + } + }, + { + description: "UTF-8 BOM encoded input with header produces column key stripped from invisible BOM character", + input: '\ufeffA,B\nX,Y', + config: { header: true }, + expected: { + data: [{A: 'X', B: 'Y'}], + errors: [], + } + }, + { + description: "Parsing with skipEmptyLines set to 'greedy'", + notes: "Must parse correctly without lines with no content", + input: 'a,b\n\n,\nc,d\n , \n""," "\n , \n,,,,\n', + config: { skipEmptyLines: 'greedy' }, + expected: { + data: [['a', 'b'], ['c', 'd']], + errors: [] + } + }, + { + description: "Parsing with skipEmptyLines set to 'greedy' with quotes and delimiters as content", + notes: "Must include lines with escaped delimiters and quotes", + input: 'a,b\n\n,\nc,d\n" , ",","\n""" """,""""""\n\n\n', + config: { skipEmptyLines: 'greedy' }, + expected: { + data: [['a', 'b'], ['c', 'd'], [' , ', ','], ['" "', '""']], + errors: [] + } + }, + { + description: "Quoted fields with spaces between closing quote and next delimiter and contains delimiter", + input: 'A,",B" ,C,D\nE,F,G,H', + expected: { + data: [['A', ',B', 'C', 'D'],['E', 'F', 'G', 'H']], + errors: [] + } + }, + { + description: "Quoted fields with spaces between closing quote and newline and contains newline", + input: 'a,b,"c\n" \nd,e,f', + expected: { + data: [['a', 'b', 'c\n'], ['d', 'e', 'f']], + errors: [] + } + }, + { + description: "Skip First N number of lines , with header and 2 rows", + input: 'a,b,c,d\n1,2,3,4', + config: { header: true, skipFirstNLines: 1 }, + expected: { + data: [], + errors: [] + } + }, + { + description: "Skip First N number of lines , with header and 2 rows", + input: 'to-be-ignored\na,b,c,d\n1,2,3,4', + config: { header: true, skipFirstNLines: 1 }, + expected: { + data: [{a: '1', b: '2', c: '3', d: '4'}], + errors: [] + } + }, + { + description: "Skip First N number of lines , with header false", + input: 'a,b,c,d\n1,2,3,4\n4,5,6,7', + config: { header: false, skipFirstNLines: 1 }, + expected: { + data: [['1','2','3','4'],['4','5','6','7']], + errors: [] + } + }, + { + description: "Skip First N number of lines , with header false and skipFirstNLines as negative value", + input: 'a,b,c,d\n1,2,3,4\n4,5,6,7', + config: { header: false, skipFirstNLines: -2 }, + expected: { + data: [['a','b','c','d'],['1','2','3','4'],['4','5','6','7']], + errors: [] + } + }, + { + description: "Skip first 2 lines , with custom newline character", + input: 'skip-this\rskip-this\r1,2,3,4', + config: { header: false, skipFirstNLines: 2, newline: '\r' }, + expected: { + data: [['1','2','3','4']], + errors: [] + } + }, + +]; + +describe('Parse Tests', function() { + function generateTest(test) { + (test.disabled ? it.skip : it)(test.description, function() { + var actual = Papa.parse(test.input, test.config); + // allows for testing the meta object if present in the test + if (test.expected.meta) { + assert.deepEqual(actual.meta, test.expected.meta); + } + assert.deepEqual(actual.errors, test.expected.errors); + assert.deepEqual(actual.data, test.expected.data); + }); + } + + for (var i = 0; i < PARSE_TESTS.length; i++) { + generateTest(PARSE_TESTS[i]); + } + + // Custom test for Issue 1024 - renamedHeaders regression test + it('Issue 1024: renamedHeaders returned for simple duplicate headers (regression test)', function() { + var result = Papa.parse('Column,Column\n1-1,1-2\n2-1,2-2\n3-1,3-2', { header: true }); + + // Test data structure + assert.deepEqual(result.data, [ + {Column: '1-1', Column_1: '1-2'}, + {Column: '2-1', Column_1: '2-2'}, + {Column: '3-1', Column_1: '3-2'} + ]); + + // Test errors + assert.deepEqual(result.errors, []); + + // Test that renamedHeaders is present and correct + assert.isNotNull(result.meta.renamedHeaders, 'renamedHeaders should not be null'); + assert.isObject(result.meta.renamedHeaders, 'renamedHeaders should be an object'); + assert.deepEqual(result.meta.renamedHeaders, {Column_1: 'Column'}, 'renamedHeaders should contain the renamed header mapping'); + }); + + it('downloadTimeout throws when set to a value not parsable by parseInt', function() { + assert.throws(function() { + Papa.parse('A,B,C', { downloadTimeout: 'not-a-number' }); + }, /downloadTimeout/); + }); + + it('downloadTimeout is coerced to a number when parsable', function() { + var result = Papa.parse('A,B,C', { downloadTimeout: '5000' }); + + assert.deepEqual(result.data, [['A', 'B', 'C']]); + }); +}); + + + +// Tests for Papa.parse() that involve asynchronous operation +var PARSE_ASYNC_TESTS = [ + { + description: "Simple worker", + input: "A,B,C\nX,Y,Z", + config: { + worker: true, + }, + expected: { + data: [['A','B','C'],['X','Y','Z']], + errors: [] + } + }, + { + description: "Simple download", + input: BASE_PATH + "sample.csv", + config: { + download: true + }, + disabled: !XHR_ENABLED, + expected: { + data: [['A','B','C'],['X','Y','Z']], + errors: [] + } + }, + { + description: "Simple download + worker", + input: BASE_PATH + "sample.csv", + config: { + worker: true, + download: true + }, + disabled: !XHR_ENABLED, + expected: { + data: [['A','B','C'],['X','Y','Z']], + errors: [] + } + }, + { + description: "Simple file", + disabled: !FILES_ENABLED, + input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z"], "sample.csv") : false, + config: { + }, + expected: { + data: [['A','B','C'],['X','Y','Z']], + errors: [] + } + }, + { + description: "Simple file + worker", + disabled: !FILES_ENABLED, + input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z"], "sample.csv") : false, + config: { + worker: true, + }, + expected: { + data: [['A','B','C'],['X','Y','Z']], + errors: [] + } + }, + { + description: "File with a few regular and lots of empty lines", + disabled: !FILES_ENABLED, + input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z\n" + new Array(500000).fill(",,").join("\n")], "sample.csv") : false, + config: { + skipEmptyLines: "greedy" + }, + expected: { + data: [['A','B','C'],['X','Y','Z']], + errors: [] + } + }, + { + description: "File with a few regular and lots of empty lines + worker", + disabled: !FILES_ENABLED, + input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z\n" + new Array(500000).fill(",,").join("\n")], "sample.csv") : false, + config: { + worker: true, + skipEmptyLines: "greedy" + }, + expected: { + data: [['A','B','C'],['X','Y','Z']], + errors: [] + } + } +]; + +describe('Parse Async Tests', function() { + function generateTest(test) { + (test.disabled ? it.skip : it)(test.description, function(done) { + var config = test.config; + + config.complete = function(actual) { + assert.deepEqual(actual.errors, test.expected.errors); + assert.deepEqual(actual.data, test.expected.data); + done(); + }; + + config.error = function(err) { + throw err; + }; + + Papa.parse(test.input, config); + }); + } + + for (var i = 0; i < PARSE_ASYNC_TESTS.length; i++) { + generateTest(PARSE_ASYNC_TESTS[i]); + } +}); + + + +// Tests for Papa.unparse() function (JSON to CSV) +var UNPARSE_TESTS = [ + { + description: "A simple row", + notes: "Comma should be default delimiter", + input: [['A', 'b', 'c']], + expected: 'A,b,c' + }, + { + description: "Two rows", + input: [['A', 'b', 'c'], ['d', 'E', 'f']], + expected: 'A,b,c\r\nd,E,f' + }, + { + description: "Data with quotes", + input: [['a', '"b"', 'c'], ['"d"', 'e', 'f']], + expected: 'a,"""b""",c\r\n"""d""",e,f' + }, + { + description: "Data with newlines", + input: [['a', 'b\nb', 'c'], ['d', 'e', 'f\r\nf']], + expected: 'a,"b\nb",c\r\nd,e,"f\r\nf"' + }, + { + description: "Array of objects (header row)", + input: [{ "Col1": "a", "Col2": "b", "Col3": "c" }, { "Col1": "d", "Col2": "e", "Col3": "f" }], + expected: 'Col1,Col2,Col3\r\na,b,c\r\nd,e,f' + }, + { + description: "With header row, missing a field in a row", + input: [{ "Col1": "a", "Col2": "b", "Col3": "c" }, { "Col1": "d", "Col3": "f" }], + expected: 'Col1,Col2,Col3\r\na,b,c\r\nd,,f' + }, + { + description: "With header row, with extra field in a row", + notes: "Extra field should be ignored; first object in array dictates header row", + input: [{ "Col1": "a", "Col2": "b", "Col3": "c" }, { "Col1": "d", "Col2": "e", "Extra": "g", "Col3": "f" }], + expected: 'Col1,Col2,Col3\r\na,b,c\r\nd,e,f' + }, + { + description: "Specifying column names and data separately", + input: { fields: ["Col1", "Col2", "Col3"], data: [["a", "b", "c"], ["d", "e", "f"]] }, + expected: 'Col1,Col2,Col3\r\na,b,c\r\nd,e,f' + }, + { + description: "Specifying column names only (no data)", + notes: "Papa should add a data property that is an empty array to prevent errors (no copy is made)", + input: { fields: ["Col1", "Col2", "Col3"] }, + expected: 'Col1,Col2,Col3' + }, + { + description: "Specifying data only (no field names), improperly", + notes: "A single array for a single row is wrong, but it can be compensated.
Papa should add empty fields property to prevent errors.", + input: { data: ["abc", "d", "ef"] }, + expected: 'abc,d,ef' + }, + { + description: "Specifying data only (no field names), properly", + notes: "An array of arrays, even if just a single row.
Papa should add empty fields property to prevent errors.", + input: { data: [["a", "b", "c"]] }, + expected: 'a,b,c' + }, + { + description: "Custom delimiter (semicolon)", + input: [['A', 'b', 'c'], ['d', 'e', 'f']], + config: { delimiter: ';' }, + expected: 'A;b;c\r\nd;e;f' + }, + { + description: "Custom delimiter (tab)", + input: [['Ab', 'cd', 'ef'], ['g', 'h', 'ij']], + config: { delimiter: '\t' }, + expected: 'Ab\tcd\tef\r\ng\th\tij' + }, + { + description: "Custom delimiter (ASCII 30)", + input: [['a', 'b', 'c'], ['d', 'e', 'f']], + config: { delimiter: RECORD_SEP }, + expected: 'a' + RECORD_SEP + 'b' + RECORD_SEP + 'c\r\nd' + RECORD_SEP + 'e' + RECORD_SEP + 'f' + }, + { + description: "Custom delimiter (Multi-character)", + input: [['A', 'b', 'c'], ['d', 'e', 'f']], + config: { delimiter: ', ' }, + expected: 'A, b, c\r\nd, e, f' + }, + { + description: "Custom delimiter (Multi-character), field contains custom delimiter", + input: [['A', 'b', 'c'], ['d', 'e', 'f, g']], + config: { delimiter: ', ' }, + expected: 'A, b, c\r\nd, e, "f, g"' + }, + { + description: "Bad delimiter (\\n)", + notes: "Should default to comma", + input: [['a', 'b', 'c'], ['d', 'e', 'f']], + config: { delimiter: '\n' }, + expected: 'a,b,c\r\nd,e,f' + }, + { + description: "Custom line ending (\\r)", + input: [['a', 'b', 'c'], ['d', 'e', 'f']], + config: { newline: '\r' }, + expected: 'a,b,c\rd,e,f' + }, + { + description: "Custom line ending (\\n)", + input: [['a', 'b', 'c'], ['d', 'e', 'f']], + config: { newline: '\n' }, + expected: 'a,b,c\nd,e,f' + }, + { + description: "Custom, but strange, line ending ($)", + input: [['a', 'b', 'c'], ['d', 'e', 'f']], + config: { newline: '$' }, + expected: 'a,b,c$d,e,f' + }, + { + description: "Force quotes around all fields", + input: [['a', 'b', 'c'], ['d', 'e', 'f']], + config: { quotes: true }, + expected: '"a","b","c"\r\n"d","e","f"' + }, + { + description: "Force quotes around all fields (with header row)", + input: [{ "Col1": "a", "Col2": "b", "Col3": "c" }, { "Col1": "d", "Col2": "e", "Col3": "f" }], + config: { quotes: true }, + expected: '"Col1","Col2","Col3"\r\n"a","b","c"\r\n"d","e","f"' + }, + { + description: "Force quotes around certain fields only", + input: [['a', 'b', 'c'], ['d', 'e', 'f']], + config: { quotes: [true, false, true] }, + expected: '"a",b,"c"\r\n"d",e,"f"' + }, + { + description: "Force quotes around certain fields only (with header row)", + input: [{ "Col1": "a", "Col2": "b", "Col3": "c" }, { "Col1": "d", "Col2": "e", "Col3": "f" }], + config: { quotes: [true, false, true] }, + expected: '"Col1",Col2,"Col3"\r\n"a",b,"c"\r\n"d",e,"f"' + }, + { + description: "Force quotes around string fields only", + input: [['a', 'b', 'c'], ['d', 10, true]], + config: { quotes: function(value) { return typeof value === 'string'; } }, + expected: '"a","b","c"\r\n"d",10,true' + }, + { + description: "Force quotes around string fields only (with header row)", + input: [{ "Col1": "a", "Col2": "b", "Col3": "c" }, { "Col1": "d", "Col2": 10, "Col3": true }], + config: { quotes: function(value) { return typeof value === 'string'; } }, + expected: '"Col1","Col2","Col3"\r\n"a","b","c"\r\n"d",10,true' + }, + { + description: "Empty input", + input: [], + expected: '' + }, + { + description: "Mismatched field counts in rows", + input: [['a', 'b', 'c'], ['d', 'e'], ['f']], + expected: 'a,b,c\r\nd,e\r\nf' + }, + { + description: "JSON null is treated as empty value", + input: [{ "Col1": "a", "Col2": null, "Col3": "c" }], + expected: 'Col1,Col2,Col3\r\na,,c' + }, + { + description: "Custom quote character (single quote)", + input: [['a,d','b','c']], + config: { quoteChar: "'"}, + expected: "'a,d',b,c" + }, + { + description: "Don't print header if header:false option specified", + input: [{"Col1": "a", "Col2": "b", "Col3": "c"}, {"Col1": "d", "Col2": "e", "Col3": "f"}], + config: {header: false}, + expected: 'a,b,c\r\nd,e,f' + }, + { + description: "Date objects are exported in its ISO representation", + input: [{date: new Date("2018-05-04T21:08:03.269Z"), "not a date": 16}, {date: new Date("Tue May 08 2018 08:20:22 GMT-0700 (PDT)"), "not a date": 32}], + expected: 'date,not a date\r\n2018-05-04T21:08:03.269Z,16\r\n2018-05-08T15:20:22.000Z,32' + }, + { + description: "Invalid Date objects are exported as an empty field", + input: [{date: new Date("not a date"), "not a date": 16}], + expected: 'date,not a date\r\n,16' + }, + { + description: "Date objects with an expanded year are exported in its full ISO representation", + input: [{date: new Date("+010000-01-01T00:00:00.000Z"), "not a date": 16}], + expected: 'date,not a date\r\n+010000-01-01T00:00:00.000Z,16' + }, + { + description: "Returns empty rows when empty rows are passed and skipEmptyLines is false", + input: [[null, ' '], [], ['1', '2']], + config: {skipEmptyLines: false}, + expected: '," "\r\n\r\n1,2' + }, + { + description: "Returns without empty rows when skipEmptyLines is true", + input: [[null, ' '], [], ['1', '2']], + config: {skipEmptyLines: true}, + expected: '," "\r\n1,2' + }, + { + description: "Returns without rows with no content when skipEmptyLines is 'greedy'", + input: [[null, ' '], [], ['1', '2']].concat(new Array(500000).fill(['', ''])).concat([['3', '4']]), + config: {skipEmptyLines: 'greedy'}, + expected: '1,2\r\n3,4' + }, + { + description: "Returns empty rows when empty rows are passed and skipEmptyLines is false with headers", + input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], + config: {skipEmptyLines: false, header: true}, + expected: 'a,b\r\n," "\r\n\r\n1,2' + }, + { + description: "Returns without empty rows when skipEmptyLines is true with headers", + input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], + config: {skipEmptyLines: true, header: true}, + expected: 'a,b\r\n," "\r\n1,2' + }, + { + description: "Returns without rows with no content when skipEmptyLines is 'greedy' with headers", + input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], + config: {skipEmptyLines: 'greedy', header: true}, + expected: 'a,b\r\n1,2' + }, + { + description: "Column option used to manually specify keys", + notes: "Should not throw any error when attempting to serialize key not present in object. Columns are different than keys of the first object. When an object is missing a key then the serialized value should be an empty string.", + input: [{a: 1, b: '2'}, {}, {a: 3, d: 'd', c: 4,}], + config: {columns: ['a', 'b', 'c']}, + expected: 'a,b,c\r\n1,2,\r\n\r\n3,,4' + }, + { + description: "Column option used to manually specify keys with input type object", + notes: "Should not throw any error when attempting to serialize key not present in object. Columns are different than keys of the first object. When an object is missing a key then the serialized value should be an empty string.", + input: { data: [{a: 1, b: '2'}, {}, {a: 3, d: 'd', c: 4,}] }, + config: {columns: ['a', 'b', 'c']}, + expected: 'a,b,c\r\n1,2,\r\n\r\n3,,4' + }, + { + description: "Use different escapeChar", + input: [{a: 'foo', b: '"quoted"'}], + config: {header: false, escapeChar: '\\'}, + expected: 'foo,"\\"quoted\\""' + }, + { + description: "test defeault escapeChar", + input: [{a: 'foo', b: '"quoted"'}], + config: {header: false}, + expected: 'foo,"""quoted"""' + }, + { + description: "Custom quoteChar escapes and quotes embedded quoteChar (no escapeChar)", + input: [['a', 'x+y']], + config: {header: false, quoteChar: '+'}, + expected: 'a,+x++y+' + }, + { + description: "Custom quoteChar with custom escapeChar quotes and escapes embedded quoteChar", + input: [['a', 'x+y']], + config: {header: false, quoteChar: '+', escapeChar: '\\'}, + expected: 'a,+x\\+y+' + }, + { + description: "Escape formulae", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + config: { escapeFormulae: true }, + expected: 'Col1,Col2,Col3\r\n"\'=danger","\'@danger",safe\r\nsafe=safe,"\'+danger","\'-danger, danger"\r\n\'+safe,\'@safe,"safe, safe"' + }, + { + description: "Don't escape formulae by default", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + expected: 'Col1,Col2,Col3\r\n=danger,@danger,safe\r\nsafe=safe,+danger,"-danger, danger"\r\n\'+safe,\'@safe,"safe, safe"' + }, + { + description: "Escape formulae with forced quotes", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + config: { escapeFormulae: true, quotes: true }, + expected: '"Col1","Col2","Col3"\r\n"\'=danger","\'@danger","safe"\r\n"safe=safe","\'+danger","\'-danger, danger"\r\n"\'+safe","\'@safe","safe, safe"' + }, + { + description: "Escape formulae with single-quote quoteChar and escapeChar", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + config: { escapeFormulae: true, quoteChar: "'", escapeChar: "'" }, + expected: 'Col1,Col2,Col3\r\n\'\'\'=danger\',\'\'\'@danger\',safe\r\nsafe=safe,\'\'\'+danger\',\'\'\'-danger, danger\'\r\n\'\'\'+safe\',\'\'\'@safe\',\'safe, safe\'' + }, + { + description: "Escape formulae with single-quote quoteChar and escapeChar and forced quotes", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + config: { escapeFormulae: true, quotes: true, quoteChar: "'", escapeChar: "'" }, + expected: '\'Col1\',\'Col2\',\'Col3\'\r\n\'\'\'=danger\',\'\'\'@danger\',\'safe\'\r\n\'safe=safe\',\'\'\'+danger\',\'\'\'-danger, danger\'\r\n\'\'\'+safe\',\'\'\'@safe\',\'safe, safe\'' + }, + // new escapeFormulae values: + { + description: "Escape formulae with tab and carriage-return", + input: [{ "Col1": "\tdanger", "Col2": "\rdanger,", "Col3": "safe\t\r" }], + config: { escapeFormulae: true }, + expected: 'Col1,Col2,Col3\r\n"\'\tdanger","\'\rdanger,","safe\t\r"' + }, + { + description: "Escape formulae with tab and carriage-return, with forced quotes", + input: [{ "Col1": " danger", "Col2": "\rdanger,", "Col3": "safe\t\r" }], + config: { escapeFormulae: true, quotes: true }, + expected: '"Col1","Col2","Col3"\r\n"\'\tdanger","\'\rdanger,","safe\t\r"' + }, + { + description: "Escape formulae with tab and carriage-return, with single-quote quoteChar and escapeChar", + input: [{ "Col1": " danger", "Col2": "\rdanger,", "Col3": "safe, \t\r" }], + config: { escapeFormulae: true, quoteChar: "'", escapeChar: "'" }, + expected: 'Col1,Col2,Col3\r\n\'\'\'\tdanger\',\'\'\'\rdanger,\',\'safe, \t\r\'' + }, + { + description: "Escape formulae with tab and carriage-return, with single-quote quoteChar and escapeChar and forced quotes", + input: [{ "Col1": " danger", "Col2": "\rdanger,", "Col3": "safe, \t\r" }], + config: { escapeFormulae: true, quotes: true, quoteChar: "'", escapeChar: "'" }, + expected: '\'Col1\',\'Col2\',\'Col3\'\r\n\'\'\'\tdanger\',\'\'\'\rdanger,\',\'safe, \t\r\'' + }, +]; + +describe('Unparse Tests', function() { + function generateTest(test) { + (test.disabled ? it.skip : it)(test.description, function() { + var actual; + + try { + actual = Papa.unparse(test.input, test.config); + } catch (e) { + if (e instanceof Error) { + throw e; + } + actual = e; + } + + assert.strictEqual(actual, test.expected); + }); + } + + for (var i = 0; i < UNPARSE_TESTS.length; i++) { + generateTest(UNPARSE_TESTS[i]); + } +}); + + + +var CUSTOM_TESTS = [ + { + description: "Pause and resume works (Regression Test for Bug #636)", + disabled: !XHR_ENABLED, + timeout: 30000, + expected: [2001, [ + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Lorem ipsum dolor sit","42"] + ], 0], + run: function(callback) { + var stepped = 0; + var dataRows = []; + var errorCount = 0; + var output = []; + Papa.parse(BASE_PATH + "verylong-sample.csv", { + download: true, + step: function(results, parser) { + stepped++; + if (results) + { + parser.pause(); + parser.resume(); + if (results.data && stepped % 200 === 0) { + dataRows.push(results.data); + } + } + }, + complete: function() { + output.push(stepped); + output.push(dataRows); + output.push(errorCount); + callback(output); + } + }); + } + }, + { + description: "Pause and resume works for chunks with NetworkStreamer", + disabled: !XHR_ENABLED, + timeout: 30000, + expected: ["Etiam a dolor vitae est vestibulum", "84", "DEF"], + run: function(callback) { + var chunkNum = 0; + Papa.parse(BASE_PATH + "verylong-sample.csv", { + download: true, + chunkSize: 1000, + chunk: function(results, parser) { + chunkNum++; + parser.pause(); + + if (chunkNum === 2) { + callback(results.data[0]); + return; + } + + parser.resume(); + }, + complete: function() { + callback(new Error("Should have found matched row before parsing whole file")); + } + }); + } + }, + { + description: "Pause and resume works for chunks with FileStreamer", + disabled: !XHR_ENABLED, + timeout: 30000, + expected: ["Etiam a dolor vitae est vestibulum", "84", "DEF"], + run: function(callback) { + var chunkNum = 0; + var xhr = new XMLHttpRequest(); + xhr.onload = function() { + Papa.parse(new File([xhr.responseText], './verylong-sample.csv'), { + chunkSize: 1000, + chunk: function(results, parser) { + chunkNum++; + parser.pause(); + + if (chunkNum === 2) { + callback(results.data[0]); + return; + } + + parser.resume(); + }, + complete: function() { + callback(new Error("Should have found matched row before parsing whole file")); + } + }); + }; + + xhr.open("GET", BASE_PATH + "verylong-sample.csv"); + try { + xhr.send(); + } catch (err) { + callback(err); + return; + } + } + }, + { + description: "Pause and resume works for chunks with StringStreamer", + disabled: !XHR_ENABLED, + timeout: 30000, + // Test also with string as byte size may be diferent + expected: ["Etiam a dolor vitae est vestibulum", "84", "DEF"], + run: function(callback) { + var chunkNum = 0; + var xhr = new XMLHttpRequest(); + xhr.onload = function() { + Papa.parse(xhr.responseText, { + chunkSize: 1000, + chunk: function(results, parser) { + chunkNum++; + parser.pause(); + + if (chunkNum === 2) { + callback(results.data[0]); + return; + } + + parser.resume(); + }, + complete: function() { + callback(new Error("Should have found matched row before parsing whole file")); + } + }); + }; + + xhr.open("GET", BASE_PATH + "verylong-sample.csv"); + try { + xhr.send(); + } catch (err) { + callback(err); + return; + } + } + }, + { + description: "Duplicate values in a data row are not renamed after resume (Regression Test for Issue #998)", + expected: [ + {c1: 'foo, bar', c2: 'foo, bar', c3: 'v1'}, + {c1: 'foo, bar', c2: 'foo, bar', c3: 'v2'} + ], + run: function(callback) { + var rows = []; + Papa.parse('c1,c2,c3\n"foo, bar","foo, bar",v1\n"foo, bar","foo, bar",v2\n', { + header: true, + step: function(results, parser) { + parser.pause(); + rows.push(results.data); + parser.resume(); + }, + complete: function() { + callback(rows); + } + }); + } + }, + { + description: "Empty trailing fields stay empty after resume (Regression Test for Issue #985)", + expected: [ + {a: '1', b: 'x', c: '', d: ''}, + {a: '2', b: 'y', c: '', d: ''} + ], + run: function(callback) { + var rows = []; + Papa.parse('a,b,c,d\n1,x,,\n2,y,,\n', { + header: true, + step: function(results, parser) { + parser.pause(); + rows.push(results.data); + parser.resume(); + }, + complete: function() { + callback(rows); + } + }); + } + }, + { + description: "Complete is called with all results if neither step nor chunk is defined", + expected: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']], + disabled: !FILES_ENABLED, + run: function(callback) { + Papa.parse(new File(['A,b,c\nd,E,f\nG,h,i'], 'sample.csv'), { + chunkSize: 3, + complete: function(response) { + callback(response.data); + } + }); + } + }, + { + description: "Step is called for each row", + expected: 2, + run: function(callback) { + var callCount = 0; + Papa.parse('A,b,c\nd,E,f', { + step: function() { + callCount++; + }, + complete: function() { + callback(callCount); + } + }); + } + }, + { + description: "Data is correctly parsed with steps", + expected: [['A', 'b', 'c'], ['d', 'E', 'f']], + run: function(callback) { + var data = []; + Papa.parse('A,b,c\nd,E,f', { + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Data is correctly parsed with steps (headers)", + expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}], + run: function(callback) { + var data = []; + Papa.parse('One,Two,Three\nA,b,c\nd,E,f', { + header: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Data is correctly parsed with steps and worker (headers)", + expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}], + run: function(callback) { + var data = []; + Papa.parse('One,Two,Three\nA,b,c\nd,E,f', { + header: true, + worker: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Data is correctly parsed with steps and worker", + expected: [['A', 'b', 'c'], ['d', 'E', 'f']], + run: function(callback) { + var data = []; + Papa.parse('A,b,c\nd,E,f', { + worker: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Data is correctly parsed with steps when skipping empty lines", + expected: [['A', 'b', 'c'], ['d', 'E', 'f']], + run: function(callback) { + var data = []; + Papa.parse('A,b,c\n\nd,E,f', { + skipEmptyLines: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Data is correctly parsed with steps when there are empty values", + expected: [{A: 'a', B: 'b', C: 'c', D: 'd'}, {A: 'a', B: '', C: '', D: ''}], + run: function(callback) { + var data = []; + Papa.parse('A,B,C,D\na,b,c,d\na,,,', { + header: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Step is called with the contents of the row", + expected: ['A', 'b', 'c'], + run: function(callback) { + Papa.parse('A,b,c', { + step: function(response) { + callback(response.data); + } + }); + } + }, + { + description: "Step is called with the last cursor position", + expected: [6, 12, 17], + run: function(callback) { + var updates = []; + Papa.parse('A,b,c\nd,E,f\nG,h,i', { + step: function(response) { + updates.push(response.meta.cursor); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Step exposes cursor for downloads", + expected: [129, 287, 452, 595, 727, 865, 1031, 1209], + disabled: !XHR_ENABLED, + run: function(callback) { + var updates = []; + Papa.parse(BASE_PATH + "long-sample.csv", { + download: true, + step: function(response) { + updates.push(response.meta.cursor); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Step exposes cursor for chunked downloads", + expected: [129, 287, 452, 595, 727, 865, 1031, 1209], + disabled: !XHR_ENABLED, + run: function(callback) { + var updates = []; + Papa.parse(BASE_PATH + "long-sample.csv", { + download: true, + chunkSize: 500, + step: function(response) { + updates.push(response.meta.cursor); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Step exposes cursor for workers", + expected: [452, 452, 452, 865, 865, 865, 1209, 1209], + disabled: !XHR_ENABLED, + run: function(callback) { + var updates = []; + Papa.parse(BASE_PATH + "long-sample.csv", { + download: true, + chunkSize: 500, + worker: true, + step: function(response) { + updates.push(response.meta.cursor); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Chunk is called for each chunk", + expected: [3, 3, 2], + disabled: !XHR_ENABLED, + run: function(callback) { + var updates = []; + Papa.parse(BASE_PATH + "long-sample.csv", { + download: true, + chunkSize: 500, + chunk: function(response) { + updates.push(response.data.length); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Chunk is called with cursor position", + expected: [452, 865, 1209], + disabled: !XHR_ENABLED, + run: function(callback) { + var updates = []; + Papa.parse(BASE_PATH + "long-sample.csv", { + download: true, + chunkSize: 500, + chunk: function(response) { + updates.push(response.meta.cursor); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Chunk functions can pause parsing", + expected: [ + [['A', 'b', 'c']] + ], + run: function(callback) { + var updates = []; + Papa.parse('A,b,c\nd,E,f\nG,h,i', { + chunkSize: 10, + chunk: function(response, handle) { + updates.push(response.data); + handle.pause(); + callback(updates); + }, + complete: function() { + callback(new Error('incorrect complete callback')); + } + }); + } + }, + { + description: "Chunk functions can resume parsing", + expected: [ + [['A', 'b', 'c']], + [['d', 'E', 'f'], ['G', 'h', 'i']] + ], + run: function(callback) { + var updates = []; + var handle = null; + var first = true; + Papa.parse('A,b,c\nd,E,f\nG,h,i', { + chunkSize: 10, + chunk: function(response, h) { + updates.push(response.data); + if (!first) return; + handle = h; + handle.pause(); + first = false; + }, + complete: function() { + callback(updates); + } + }); + setTimeout(function() { + handle.resume(); + }, 500); + } + }, + { + description: "Chunk functions can abort parsing", + expected: [ + [['A', 'b', 'c']] + ], + run: function(callback) { + var updates = []; + Papa.parse('A,b,c\nd,E,f\nG,h,i', { + chunkSize: 1, + chunk: function(response, handle) { + if (response.data.length) { + updates.push(response.data); + handle.abort(); + } + }, + complete: function(response) { + callback(updates); + } + }); + } + }, + { + description: "Step exposes indexes for files", + expected: [6, 12, 17], + disabled: !FILES_ENABLED, + run: function(callback) { + var updates = []; + Papa.parse(new File(['A,b,c\nd,E,f\nG,h,i'], 'sample.csv'), { + download: true, + step: function(response) { + updates.push(response.meta.cursor); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Step exposes indexes for chunked files", + expected: [6, 12, 17], + disabled: !FILES_ENABLED, + run: function(callback) { + var updates = []; + Papa.parse(new File(['A,b,c\nd,E,f\nG,h,i'], 'sample.csv'), { + chunkSize: 3, + step: function(response) { + updates.push(response.meta.cursor); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Quoted line breaks near chunk boundaries are handled", + expected: [['A', 'B', 'C'], ['X', 'Y\n1\n2\n3', 'Z']], + disabled: !FILES_ENABLED, + run: function(callback) { + var updates = []; + Papa.parse(new File(['A,B,C\nX,"Y\n1\n2\n3",Z'], 'sample.csv'), { + chunkSize: 3, + step: function(response) { + updates.push(response.data); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Step functions can abort parsing", + expected: [['A', 'b', 'c']], + run: function(callback) { + var updates = []; + Papa.parse('A,b,c\nd,E,f\nG,h,i', { + step: function(response, handle) { + updates.push(response.data); + handle.abort(); + callback(updates); + }, + chunkSize: 6 + }); + } + }, + { + description: "Complete is called after aborting", + expected: true, + run: function(callback) { + Papa.parse('A,b,c\nd,E,f\nG,h,i', { + step: function(response, handle) { + handle.abort(); + }, + chunkSize: 6, + complete: function(response) { + callback(response.meta.aborted); + } + }); + } + }, + { + description: "Step functions can pause parsing", + expected: [['A', 'b', 'c']], + run: function(callback) { + var updates = []; + Papa.parse('A,b,c\nd,E,f\nG,h,i', { + step: function(response, handle) { + updates.push(response.data); + handle.pause(); + callback(updates); + }, + complete: function() { + callback('incorrect complete callback'); + } + }); + } + }, + { + description: "Step functions can resume parsing", + expected: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']], + run: function(callback) { + var updates = []; + var handle = null; + var first = true; + Papa.parse('A,b,c\nd,E,f\nG,h,i', { + step: function(response, h) { + updates.push(response.data); + if (!first) return; + handle = h; + handle.pause(); + first = false; + }, + complete: function() { + callback(updates); + } + }); + setTimeout(function() { + handle.resume(); + }, 500); + } + }, + { + description: "Step functions can abort workers", + expected: 1, + disabled: !XHR_ENABLED, + run: function(callback) { + var updates = 0; + Papa.parse(BASE_PATH + "long-sample.csv", { + worker: true, + download: true, + chunkSize: 500, + step: function(response, handle) { + updates++; + handle.abort(); + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "beforeFirstChunk manipulates only first chunk", + expected: 7, + disabled: !XHR_ENABLED, + run: function(callback) { + var updates = 0; + Papa.parse(BASE_PATH + "long-sample.csv", { + download: true, + chunkSize: 500, + beforeFirstChunk: function(chunk) { + return chunk.replace(/.*?\n/, ''); + }, + step: function(response) { + updates++; + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "First chunk not modified if beforeFirstChunk returns nothing", + expected: 8, + disabled: !XHR_ENABLED, + run: function(callback) { + var updates = 0; + Papa.parse(BASE_PATH + "long-sample.csv", { + download: true, + chunkSize: 500, + beforeFirstChunk: function(chunk) { + }, + step: function(response) { + updates++; + }, + complete: function() { + callback(updates); + } + }); + } + }, + { + description: "Should correctly guess custom delimiter when passed delimiters to guess.", + expected: "~", + run: function(callback) { + var results = Papa.parse('"A"~"B"~"C"~"D"', { + delimitersToGuess: ['~', '@', '%'] + }); + callback(results.meta.delimiter); + } + }, + { + description: "Should still correctly guess default delimiters when delimiters to guess are not given.", + expected: ",", + run: function(callback) { + var results = Papa.parse('"A","B","C","D"'); + callback(results.meta.delimiter); + } + }, + { + description: "Data is correctly parsed with chunks and duplicated headers", + expected: [{h0: 'a', h1: 'a'}, {h0: 'b', h1: 'b'}], + run: function(callback) { + var data = []; + Papa.parse('h0,h1\na,a\nb,b', { + header: true, + chunkSize: 10, + chunk: function(results) { + data.push(results.data[0]); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "downloadTimeout aborts a slow remote request with a timeout error", + disabled: !XHR_ENABLED, + timeout: 5000, + expected: true, + run: function(callback) { + Papa.parse(BASE_PATH + "slow-sample.csv", { + download: true, + downloadTimeout: 200, + error: function(err) { + callback(/timed out/i.test(err.message)); + }, + complete: function() { + callback(false); + } + }); + } + }, + { + description: "downloadTimeout does not interfere with a request that finishes in time", + disabled: !XHR_ENABLED, + timeout: 5000, + expected: [['A', 'B', 'C'], ['X', 'Y', 'Z']], + run: function(callback) { + Papa.parse(BASE_PATH + "sample.csv", { + download: true, + downloadTimeout: 10000, + complete: function(results) { + callback(results.data); + }, + error: function(err) { + callback(err); + } + }); + } + }, + { + description: "transformHeader is called exactly once per header when streaming in chunks", + // Each header must be passed to transformHeader exactly once, with its + // original (untransformed) value. Regression test for #1083. + expected: ['Col1', 'Col2', 'Col3'], + run: function(callback) { + var seen = []; + var rows = []; + for (var i = 0; i < 20; i++) + rows.push('a' + i + ',b' + i + ',c' + i); + var input = 'Col1,Col2,Col3\r\n' + rows.join('\r\n'); + Papa.parse(input, { + header: true, + chunkSize: 10, // force many chunks so the header row is crossed once + transformHeader: function(header) { + seen.push(header); + return header.toLowerCase(); + }, + complete: function() { + callback(seen); + } + }); + } + }, +]; + +describe('Custom Tests', function() { + function generateTest(test) { + (test.disabled ? it.skip : it)(test.description, function(done) { + if(test.timeout) { + this.timeout(test.timeout); + } + test.run(function(actual) { + assert.deepEqual(actual, test.expected); + done(); + }); + }); + } + + for (var i = 0; i < CUSTOM_TESTS.length; i++) { + generateTest(CUSTOM_TESTS[i]); + } +}); + +(typeof window !== "undefined" ? describe : describe.skip)("Browser Tests", () => { + it("When parsing synchronously inside a web-worker not owned by PapaParse we should not invoke postMessage", async() => { + // Arrange + const papaParseScriptPath = new URL("../papaparse.js", window.document.baseURI).href; + + // Define our custom web-worker that loads PapaParse and executes a synchronous parse + const blob = new Blob([ + ` + importScripts('${papaParseScriptPath}'); + + self.addEventListener("message", function(event) { + if (event.data === "ExecuteParse") { + // Perform our synchronous parse, as requested + const results = Papa.parse('x\\ny\\n'); + postMessage({type: "ParseExecutedSuccessfully", results}); + } else { + // Otherwise, send whatever we received back. We shouldn't be hitting this (!) If we're reached + // this it means PapaParse thinks it is running inside a web-worker that it owns + postMessage(event.data); + } + }); + ` + ], {type: 'text/javascript'}); + + const blobURL = window.URL.createObjectURL(blob); + const webWorker = new Worker(blobURL); + + const receiveMessagePromise = new Promise((resolve, reject) => { + webWorker.addEventListener("message", (event) => { + if (event.data.type === "ParseExecutedSuccessfully") { + resolve(event.data); + } else { + const error = new Error(`Received unexpected message: ${JSON.stringify(event.data, null, 2)}`); + error.data = event.data; + reject(error); + } + }); + }); + + // Act + webWorker.postMessage("ExecuteParse"); + const webWorkerMessage = await receiveMessagePromise; + + // Assert + assert.equal("ParseExecutedSuccessfully", webWorkerMessage.type); + assert.equal(3, webWorkerMessage.results.data.length); + }); +});