knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#128,backend/gdoc][s]: refactor gdocs to standard backend setup.

Browse Source
This commit is contained in:
Rufus Pollock 2012-05-26 17:46:51 +01:00
parent 5890007c59
commit e9d1b8a55a
2 changed files with 123 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

214
src/backend/gdocs.js
View File

@ -1,7 +1,9 @@
this.recline = this.recline || {};
this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
(function($, my) {
// ## Google spreadsheet backend
//
// Connect to Google Docs spreadsheet.
@ -16,50 +18,25 @@ this.recline.Backend = this.recline.Backend || {};
// 'gdocs'
// );
// </pre>
my.GDoc = my.Base.extend({
__type__: 'gdoc',
readonly: true,
getUrl: function(dataset) {
var url = dataset.get('url');
if (url.indexOf('feeds/list') != -1) {
return url;
} else {
// https://docs.google.com/spreadsheet/ccc?key=XXXX#gid=0
var regex = /.*spreadsheet\/ccc?.*key=([^#?&+]+).*/;
var matches = url.match(regex);
if (matches) {
var key = matches[1];
var worksheet = 1;
var out = 'https://spreadsheets.google.com/feeds/list/' + key + '/' + worksheet + '/public/values?alt=json';
return out;
} else {
alert('Failed to extract gdocs key from ' + url);
}
}
},
sync: function(method, model, options) {
my.Backbone = function() {
this.__type__ = 'gdocs';
this.readonly = true;
this.sync = function(method, model, options) {
var self = this;
if (method === "read") {
var dfd = $.Deferred();
var dataset = model;
var url = this.getUrl(model);
$.getJSON(url, function(d) {
result = self.gdocsToJavascript(d);
model.fields.reset(_.map(result.field, function(fieldId) {
return {id: fieldId};
})
);
loadData(model.get('url')).done(function(result) {
model.fields.reset(result.fields);
// cache data onto dataset (we have loaded whole gdoc it seems!)
model._dataCache = result.data;
dfd.resolve(model);
});
return dfd.promise();
}
},
};
query: function(dataset, queryObj) {
this.query = function(dataset, queryObj) {
var dfd = $.Deferred();
var fields = _.pluck(dataset.fields.toJSON(), 'id');
@ -72,70 +49,115 @@ this.recline.Backend = this.recline.Backend || {};
});
return obj;
});
dfd.resolve(this._docsToQueryResult(objs));
var out = {
total: objs.length,
hits: _.map(objs, function(row) {
return { _source: row }
})
}
dfd.resolve(out);
return dfd;
},
gdocsToJavascript: function(gdocsSpreadsheet) {
/*
:options: (optional) optional argument dictionary:
columnsToUse: list of columns to use (specified by field names)
colTypes: dictionary (with column names as keys) specifying types (e.g. range, percent for use in conversion).
:return: tabular data object (hash with keys: field and data).
};
};
Issues: seems google docs return columns in rows in random order and not even sure whether consistent across rows.
*/
var options = {};
if (arguments.length > 1) {
options = arguments[1];
}
var results = {
'field': [],
'data': []
};
// default is no special info on type of columns
var colTypes = {};
if (options.colTypes) {
colTypes = options.colTypes;
}
// either extract column headings from spreadsheet directly, or used supplied ones
if (options.columnsToUse) {
// columns set to subset supplied
results.field = options.columnsToUse;
} else {
// set columns to use to be all available
if (gdocsSpreadsheet.feed.entry.length > 0) {
for (var k in gdocsSpreadsheet.feed.entry[0]) {
if (k.substr(0, 3) == 'gsx') {
var col = k.substr(4);
results.field.push(col);
}
}
}
}
// converts non numberical values that should be numerical (22.3%[string] -> 0.223[float])
var rep = /^([\d\.\-]+)\%$/;
$.each(gdocsSpreadsheet.feed.entry, function (i, entry) {
var row = [];
for (var k in results.field) {
var col = results.field[k];
var _keyname = 'gsx$' + col;
var value = entry[_keyname]['$t'];
// if labelled as % and value contains %, convert
if (colTypes[col] == 'percent') {
if (rep.test(value)) {
var value2 = rep.exec(value);
var value3 = parseFloat(value2);
value = value3 / 100;
}
}
row.push(value);
}
results.data.push(row);
});
return results;
// ## loadData
//
// loadData from a google docs URL
//
// @return object with two attributes
//
// * fields: array of objects
// * data: array of arrays
var loadData = function(url) {
var dfd = $.Deferred();
var url = my.getSpreadsheetAPIUrl(url);
var out = {
fields: [],
data: []
}
});
$.getJSON(url, function(d) {
result = my.parseData(d);
result.fields = _.map(result.fields, function(fieldId) {
return {id: fieldId};
});
dfd.resolve(result);
});
return dfd.promise();
};
}(jQuery, this.recline.Backend));
// ## parseData
//
// Parse data from Google Docs API into a reasonable form
//
// :options: (optional) optional argument dictionary:
// columnsToUse: list of columns to use (specified by field names)
// colTypes: dictionary (with column names as keys) specifying types (e.g. range, percent for use in conversion).
// :return: tabular data object (hash with keys: field and data).
//
// Issues: seems google docs return columns in rows in random order and not even sure whether consistent across rows.
my.parseData = function(gdocsSpreadsheet) {
var options = {};
if (arguments.length > 1) {
options = arguments[1];
}
var results = {
'fields': [],
'data': []
};
// default is no special info on type of columns
var colTypes = {};
if (options.colTypes) {
colTypes = options.colTypes;
}
if (gdocsSpreadsheet.feed.entry.length > 0) {
for (var k in gdocsSpreadsheet.feed.entry[0]) {
if (k.substr(0, 3) == 'gsx') {
var col = k.substr(4);
results.fields.push(col);
}
}
}
// converts non numberical values that should be numerical (22.3%[string] -> 0.223[float])
var rep = /^([\d\.\-]+)\%$/;
$.each(gdocsSpreadsheet.feed.entry, function (i, entry) {
var row = [];
for (var k in results.fields) {
var col = results.fields[k];
var _keyname = 'gsx$' + col;
var value = entry[_keyname]['$t'];
// if labelled as % and value contains %, convert
if (colTypes[col] == 'percent') {
if (rep.test(value)) {
var value2 = rep.exec(value);
var value3 = parseFloat(value2);
value = value3 / 100;
}
}
row.push(value);
}
results.data.push(row);
});
return results;
};
// Convenience function to get GDocs JSON API Url from standard URL
my.getSpreadsheetAPIUrl = function(url) {
if (url.indexOf('feeds/list') != -1) {
return url;
} else {
// https://docs.google.com/spreadsheet/ccc?key=XXXX#gid=0
var regex = /.*spreadsheet\/ccc?.*key=([^#?&+]+).*/;
var matches = url.match(regex);
if (matches) {
var key = matches[1];
var worksheet = 1;
var out = 'https://spreadsheets.google.com/feeds/list/' + key + '/' + worksheet + '/public/values?alt=json';
return out;
} else {
alert('Failed to extract gdocs key from ' + url);
}
}
};
}(jQuery, this.recline.Backend.GDocs));

13
test/backend.test.js
View File

@ -273,8 +273,8 @@ var sample_gdocs_spreadsheet_data = {
"encoding": "UTF-8"
}
test("GDoc Backend", function() {
var backend = new recline.Backend.GDoc();
test("GDocs Backend", function() {
var backend = new recline.Backend.GDocs.Backbone();
var dataset = new recline.Model.Dataset({
url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
},
@ -301,13 +301,10 @@ test("GDoc Backend", function() {
$.getJSON.restore();
});
test("GDoc Backend.getUrl", function() {
test("GDocs Backend.getUrl", function() {
var key = 'Abc_dajkdkjdafkj';
var dataset = new recline.Model.Dataset({
url: 'https://docs.google.com/spreadsheet/ccc?key=' + key + '#gid=0'
});
var backend = new recline.Backend.GDoc();
var out = backend.getUrl(dataset);
var url = 'https://docs.google.com/spreadsheet/ccc?key=' + key + '#gid=0'
var out = recline.Backend.GDocs.getSpreadsheetAPIUrl(url);
var exp = 'https://spreadsheets.google.com/feeds/list/' + key + '/1/public/values?alt=json'
equal(exp, out);
});