knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve csv parsing code

Browse Source
This commit is contained in:
Max Ogden 2011-08-31 19:56:56 -04:00
parent 48dbdf58f5
commit 935e0d15a0
2 changed files with 66 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
attachments/script/costco-csv-worker.js
View File

@ -1,19 +1,56 @@
importScripts('lib/underscore.js');
onmessage = function(message) {
var rows = message.data.data.split('\n');
var docs = [];
_.each(rows, function(row) {
if (row.length == 0) return;
postMessage(JSON.stringify({size: rows.length}));
var doc = {};
_.each(row.split(','), function(field, index) { doc['field' + index] = field });
docs.push(doc);
})
function parseCSV(rawCSV) {
var patterns = new RegExp((
// Delimiters.
"(\\,|\\r?\\n|\\r|^)" +
// Quoted fields.
"(?:\"([^\"]*(?:\"\"[^\"]*)*)\"|" +
// Standard fields.
"([^\"\\,\\r\\n]*))"
), "gi");
var rows = [[]], matches = null;
while (matches = patterns.exec(rawCSV)) {
var delimiter = matches[1];
if (delimiter.length && (delimiter !== ",")) rows.push([]);
if (matches[2]) {
var value = matches[2].replace(new RegExp("\"\"", "g"), "\"");
} else {
var value = matches[3];
}
rows[rows.length - 1].push(value);
}
if(_.isEqual(rows[rows.length -1], [""])) rows.pop();
var docs = [];
var headers = _.first(rows);
_.each(_.rest(rows), function(row, rowIDX) {
var doc = {};
_.each(row, function(cell, idx) {
doc[headers[idx]] = cell;
})
docs.push(doc);
})
return docs;
}
var docs = parseCSV(message.data.data);
var req = new XMLHttpRequest();
req.onreadystatechange = function() {
if (req.readyState == 4) postMessage(JSON.stringify({done: true}))
req.onprogress = req.upload.onprogress = function(e) {
if(e.lengthComputable) postMessage(JSON.stringify({ percent: (e.loaded / e.total) * 100 }));
};
req.onreadystatechange = function() { if (req.readyState == 4) postMessage(JSON.stringify( {done: true} )) };
req.open('POST', message.data.url);
req.setRequestHeader('Content-Type', 'application/json');
req.send(JSON.stringify({docs: docs}));

30
attachments/script/costco.js
View File

@ -120,18 +120,24 @@ var costco = function() {
};
var worker = new Worker('script/costco-csv-worker.js');
worker.onmessage = function(message) {
message = JSON.parse(message.data);
if(message.done) {
util.hide('dialog');
util.notify("Data uploaded successfully!");
recline.initializeTable(app.offset);
} else if (message.size) {
util.notify("Processing " + message.size + " rows. This could take a while...", {persist: true, loader: true});
} else {
util.notify(JSON.stringify(message));
}
};
worker.postMessage(payload);
message = JSON.parse(message.data);
console.log(message)
if (message.done) {
util.hide('dialog');
util.notify("Data uploaded successfully!");
recline.initializeTable(app.offset);
} else if (message.percent) {
if (message.percent === 100) {
util.notify("Waiting for CouchDB...", {persist: true, loader: true})
} else {
util.notify("Uploading... " + message.percent + "%");
}
} else {
util.notify(JSON.stringify(message));
}
};
worker.postMessage(payload);
};
} else {
util.notify('File not selected. Please try again');