knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#162,backend,model][l]: major commit addressing several parts of the backend / model refactor in #162.

Browse Source 
* Now have Dataset setup and manage "memory store"

  * New fetch API as per issue #162 spec
  * dataproxy utilizes useMemoryStore attribute and just implements fetch
  * Switch gdocs to use Memory.Store properly via new useMemoryStore + fetch methodology
  * Memory backend: query function now follows promise API, remove fetch,upsert,delete and add save function to Store object

* Also refactor to remove _source in QueryResult "hits" attribute on all backends but ElasticSearch - cf #159 (note this means ES currently broken)
This commit is contained in:
Rufus Pollock
2012-06-23 20:23:24 +01:00
parent bda4797ed8
commit 6e5c15a816
8 changed files with 153 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/backend/dataproxy.js
View File

@@ -50,11 +50,11 @@ this.recline.Backend.DataProxy = this.recline.Backend.DataProxy || {};
}); });
return tmp; return tmp;
}); });
var store = new recline.Backend.Memory.Store(records, fields); dfd.resolve({
dataset._dataCache = store; records: records,
dataset.fields.reset(fields); fields: fields,
dataset.query(); useMemoryStore: true
dfd.resolve(dataset); });
}) })
.fail(function(arguments) { .fail(function(arguments) {
dfd.reject(arguments); dfd.reject(arguments);
@@ -62,21 +62,6 @@ this.recline.Backend.DataProxy = this.recline.Backend.DataProxy || {};
return dfd.promise(); return dfd.promise();
}; };
my.query = function(dataset, queryObj) {
var dfd = $.Deferred();
var results = dataset._dataCache.query(queryObj);
var hits = _.map(results.records, function(row) {
return { _source: row };
});
var out = {
total: results.total,
hits: hits,
facets: results.facets
};
dfd.resolve(out);
return dfd.promise();
};
// ## _wrapInTimeout // ## _wrapInTimeout
// //
// Convenience method providing a crude way to catch backend errors on JSONP calls. // Convenience method providing a crude way to catch backend errors on JSONP calls.

98
src/backend/gdocs.js
View File

@@ -3,14 +3,13 @@ this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.GDocs = this.recline.Backend.GDocs || {}; this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
(function($, my) { (function($, my) {
my.__type__ = 'gdocs';
// ## Google spreadsheet backend // ## Google spreadsheet backend
// //
// Connect to Google Docs spreadsheet. // Fetch data from a Google Docs spreadsheet.
//
// Dataset must have a url attribute pointing to the Gdocs
// spreadsheet's JSON feed e.g.
// //
// Dataset must have a url attribute pointing to the Gdocs or its JSON feed e.g.
// <pre> // <pre>
// var dataset = new recline.Model.Dataset({ // var dataset = new recline.Model.Dataset({
// url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json' // url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
@@ -18,77 +17,25 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
// 'gdocs' // 'gdocs'
// ); // );
// </pre> // </pre>
my.Backbone = function() {
var self = this;
this.__type__ = 'gdocs';
this.readonly = true;
this.sync = function(method, model, options) {
var self = this;
if (method === "read") {
var dfd = $.Deferred();
dfd.resolve(model);
return dfd.promise();
}
};
this.query = function(dataset, queryObj) {
var dfd = $.Deferred();
if (dataset._dataCache) {
dfd.resolve(dataset._dataCache);
} else {
loadData(dataset.get('url')).done(function(result) {
dataset.fields.reset(result.fields);
// cache data onto dataset (we have loaded whole gdoc it seems!)
dataset._dataCache = self._formatResults(dataset, result.data);
dfd.resolve(dataset._dataCache);
});
}
return dfd.promise();
};
this._formatResults = function(dataset, data) {
var fields = _.pluck(dataset.fields.toJSON(), 'id');
// zip the fields with the data rows to produce js objs
// TODO: factor this out as a common method with other backends
var objs = _.map(data, function (d) {
var obj = {};
_.each(_.zip(fields, d), function (x) {
obj[x[0]] = x[1];
});
return obj;
});
var out = {
total: objs.length,
hits: _.map(objs, function(row) {
return { _source: row }
})
}
return out;
};
};
// ## loadData
//
// loadData from a google docs URL
// //
// @return object with two attributes // @return object with two attributes
// //
// * fields: array of objects // * fields: array of Field objects
// * data: array of arrays // * records: array of objects for each row
var loadData = function(url) { //
my.fetch = function(dataset) {
var dfd = $.Deferred(); var dfd = $.Deferred();
var url = my.getSpreadsheetAPIUrl(url); var url = my.getSpreadsheetAPIUrl(dataset.get('url'));
var out = {
fields: [],
data: []
}
$.getJSON(url, function(d) { $.getJSON(url, function(d) {
result = my.parseData(d); result = my.parseData(d);
result.fields = _.map(result.fields, function(fieldId) { var fields = _.map(result.fields, function(fieldId) {
return {id: fieldId}; return {id: fieldId};
}); });
dfd.resolve(result); dfd.resolve({
records: result.records,
fields: fields,
useMemoryStore: true
});
}); });
return dfd.promise(); return dfd.promise();
}; };
@@ -109,8 +56,8 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
options = arguments[1]; options = arguments[1];
} }
var results = { var results = {
'fields': [], fields: [],
'data': [] records: []
}; };
// default is no special info on type of columns // default is no special info on type of columns
var colTypes = {}; var colTypes = {};
@@ -128,10 +75,9 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
// converts non numberical values that should be numerical (22.3%[string] -> 0.223[float]) // converts non numberical values that should be numerical (22.3%[string] -> 0.223[float])
var rep = /^([\d\.\-]+)\%$/; var rep = /^([\d\.\-]+)\%$/;
$.each(gdocsSpreadsheet.feed.entry, function (i, entry) { results.records = _.map(gdocsSpreadsheet.feed.entry, function(entry) {
var row = []; var row = {};
for (var k in results.fields) { _.each(results.fields, function(col) {
var col = results.fields[k];
var _keyname = 'gsx$' + col; var _keyname = 'gsx$' + col;
var value = entry[_keyname]['$t']; var value = entry[_keyname]['$t'];
// if labelled as % and value contains %, convert // if labelled as % and value contains %, convert
@@ -142,9 +88,9 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
value = value3 / 100; value = value3 / 100;
} }
} }
row.push(value); row[col] = value;
} });
results.data.push(row); return row;
}); });
return results; return results;
}; };

65
src/backend/memory.js
View File

@@ -20,49 +20,9 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
var dataset = new recline.Model.Dataset( var dataset = new recline.Model.Dataset(
_.extend({}, metadata, {records: data, fields: fields}) _.extend({}, metadata, {records: data, fields: fields})
); );
dataset.fetch();
return dataset; return dataset;
}; };
my.fetch = function(dataset) {
var dfd = $.Deferred();
var store = new my.Store(dataset.get('records'), dataset.get('fields'));
dataset._dataCache = store;
dataset.fields.reset(store.fields);
dataset.query();
dfd.resolve(dataset);
return dfd.promise();
};
my.save = function(dataset, changes) {
var dfd = $.Deferred();
// TODO
// _.each(changes.creates) { ... }
_.each(changes.updates, function(record) {
dataset._dataCache.update(record);
});
_.each(changes.deletes, function(record) {
dataset._dataCache.delete(record);
});
dfd.resolve(dataset);
return dfd.promise();
},
my.query = function(dataset, queryObj) {
var dfd = $.Deferred();
var results = dataset._dataCache.query(queryObj);
var hits = _.map(results.records, function(row) {
return { _source: row };
});
var out = {
total: results.total,
hits: hits,
facets: results.facets
};
dfd.resolve(out);
return dfd.promise();
};
// ## Data Wrapper // ## Data Wrapper
// //
// Turn a simple array of JS objects into a mini data-store with // Turn a simple array of JS objects into a mini data-store with
@@ -102,7 +62,22 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
this.data = newdocs; this.data = newdocs;
}; };
this.save = function(changes, dataset) {
var self = this;
var dfd = $.Deferred();
// TODO _.each(changes.creates) { ... }
_.each(changes.updates, function(record) {
self.update(record);
});
_.each(changes.deletes, function(record) {
self.delete(record);
});
dfd.resolve(this);
return dfd.promise();
},
this.query = function(queryObj) { this.query = function(queryObj) {
var dfd = $.Deferred();
var numRows = queryObj.size || this.data.length; var numRows = queryObj.size || this.data.length;
var start = queryObj.from || 0; var start = queryObj.from || 0;
var results = this.data; var results = this.data;
@@ -119,14 +94,14 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
results.reverse(); results.reverse();
} }
}); });
var total = results.length;
var facets = this.computeFacets(results, queryObj); var facets = this.computeFacets(results, queryObj);
results = results.slice(start, start+numRows); var out = {
return { total: results.length,
total: total, hits: results.slice(start, start+numRows),
records: results,
facets: facets facets: facets
}; };
dfd.resolve(out);
return dfd.promise();
}; };
// in place filtering // in place filtering

110
src/model.js
View File

@@ -65,17 +65,49 @@ my.Dataset = Backbone.Model.extend({
this.queryState = new my.Query(); this.queryState = new my.Query();
this.queryState.bind('change', this.query); this.queryState.bind('change', this.query);
this.queryState.bind('facet:add', this.query); this.queryState.bind('facet:add', this.query);
this._store = this.backend;
if (this.backend == recline.Backend.Memory) {
this.fetch();
}
}, },
// ### fetch // ### fetch
// //
// Retrieve dataset and (some) records from the backend. // Retrieve dataset and (some) records from the backend.
fetch: function() { fetch: function() {
return this.backend.fetch(this); var self = this;
var dfd = $.Deferred();
// TODO: fail case;
if (this.backend !== recline.Backend.Memory) {
this.backend.fetch(this).then(handleResults)
} else {
// special case where we have been given data directly
handleResults({
records: this.get('records'),
fields: this.get('fields'),
useMemoryStore: true
});
}
function handleResults(results) {
self.set(results.metadata);
if (results.useMemoryStore) {
self._store = new recline.Backend.Memory.Store(results.records, results.fields);
self.query();
// store will have extracted fields if not provided
self.fields.reset(self._store.fields);
} else {
self.fields.reset(results.fields);
}
// TODO: parsing the processing of fields
dfd.resolve(this);
}
return dfd.promise();
}, },
save: function() { save: function() {
return this.backend.save(this, this._changes); var self = this;
return this._store.save(this._changes, this);
}, },
// ### query // ### query
@@ -89,48 +121,48 @@ my.Dataset = Backbone.Model.extend({
// also returned. // also returned.
query: function(queryObj) { query: function(queryObj) {
var self = this; var self = this;
this.trigger('query:start');
var actualQuery = self._prepareQuery(queryObj);
var dfd = $.Deferred(); var dfd = $.Deferred();
this.backend.query(this, actualQuery).done(function(queryResult) { this.trigger('query:start');
self.docCount = queryResult.total;
var docs = _.map(queryResult.hits, function(hit) { if (queryObj) {
var _doc = new my.Record(hit._source); this.queryState.set(queryObj);
_doc.backend = self.backend; }
_doc.dataset = self; var actualQuery = this.queryState.toJSON();
_doc.bind('change', function(doc) {
self._changes.updates.push(doc.toJSON()); this._store.query(actualQuery, this)
}); .done(function(queryResult) {
_doc.bind('destroy', function(doc) { self._handleQueryResult(queryResult);
self._changes.deletes.push(doc.toJSON()); self.trigger('query:done');
}); dfd.resolve(self.currentRecords);
return _doc; })
.fail(function(arguments) {
self.trigger('query:fail', arguments);
dfd.reject(arguments);
}); });
self.currentRecords.reset(docs);
if (queryResult.facets) {
var facets = _.map(queryResult.facets, function(facetResult, facetId) {
facetResult.id = facetId;
return new my.Facet(facetResult);
});
self.facets.reset(facets);
}
self.trigger('query:done');
dfd.resolve(self.currentRecords);
})
.fail(function(arguments) {
self.trigger('query:fail', arguments);
dfd.reject(arguments);
});
return dfd.promise(); return dfd.promise();
}, },
_handleQueryResult: function(queryResult) {
_prepareQuery: function(newQueryObj) { var self = this;
if (newQueryObj) { self.docCount = queryResult.total;
this.queryState.set(newQueryObj); var docs = _.map(queryResult.hits, function(hit) {
var _doc = new my.Record(hit);
_doc.bind('change', function(doc) {
self._changes.updates.push(doc.toJSON());
});
_doc.bind('destroy', function(doc) {
self._changes.deletes.push(doc.toJSON());
});
return _doc;
});
self.currentRecords.reset(docs);
if (queryResult.facets) {
var facets = _.map(queryResult.facets, function(facetResult, facetId) {
facetResult.id = facetId;
return new my.Facet(facetResult);
});
self.facets.reset(facets);
} }
var out = this.queryState.toJSON();
return out;
}, },
toTemplateJSON: function() { toTemplateJSON: function() {
@@ -151,7 +183,7 @@ my.Dataset = Backbone.Model.extend({
query.addFacet(field.id); query.addFacet(field.id);
}); });
var dfd = $.Deferred(); var dfd = $.Deferred();
this.backend.query(this, query.toJSON()).done(function(queryResult) { this._store.query(query.toJSON(), this).done(function(queryResult) {
if (queryResult.facets) { if (queryResult.facets) {
_.each(queryResult.facets, function(facetResult, facetId) { _.each(queryResult.facets, function(facetResult, facetId) {
facetResult.id = facetId; facetResult.id = facetId;

6
test/backend/gdocs.test.js
View File

@@ -168,11 +168,10 @@ var sample_gdocs_spreadsheet_data = {
} }
test("GDocs Backend", function() { test("GDocs Backend", function() {
var backend = new recline.Backend.GDocs.Backbone();
var dataset = new recline.Model.Dataset({ var dataset = new recline.Model.Dataset({
url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json' url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
}, },
backend 'gdocs'
); );
var stub = sinon.stub($, 'getJSON', function(options, cb) { var stub = sinon.stub($, 'getJSON', function(options, cb) {
@@ -182,7 +181,8 @@ test("GDocs Backend", function() {
} }
}); });
dataset.query().then(function(docList) { dataset.fetch().then(function() {
var docList = dataset.currentRecords;
deepEqual(['column-2', 'column-1'], _.pluck(dataset.fields.toJSON(), 'id')); deepEqual(['column-2', 'column-1'], _.pluck(dataset.fields.toJSON(), 'id'));
equal(3, docList.length); equal(3, docList.length);
equal("A", docList.models[0].get('column-1')); equal("A", docList.models[0].get('column-1'));

55
test/backend/memory.test.js
View File

@@ -29,10 +29,11 @@ test('query', function () {
size: 4 size: 4
, from: 2 , from: 2
}; };
var out = data.query(queryObj); data.query(queryObj).then(function(out) {
deepEqual(out.records[0], memoryData[2]); deepEqual(out.hits[0], memoryData[2]);
equal(out.records.length, 4); equal(out.hits.length, 4);
equal(out.total, 6); equal(out.total, 6);
});
}); });
test('query sort', function () { test('query sort', function () {
@@ -42,44 +43,50 @@ test('query sort', function () {
{'y': {order: 'desc'}} {'y': {order: 'desc'}}
] ]
}; };
var out = data.query(queryObj); data.query(queryObj).then(function(out) {
equal(out.records[0].x, 6); equal(out.hits[0].x, 6);
});
var queryObj = { var queryObj = {
sort: [ sort: [
{'country': {order: 'desc'}} {'country': {order: 'desc'}}
] ]
}; };
var out = data.query(queryObj); data.query(queryObj).then(function(out) {
equal(out.records[0].country, 'US'); equal(out.hits[0].country, 'US');
});
var queryObj = { var queryObj = {
sort: [ sort: [
{'country': {order: 'asc'}} {'country': {order: 'asc'}}
] ]
}; };
var out = data.query(queryObj); data.query(queryObj).then(function(out) {
equal(out.records[0].country, 'DE'); equal(out.hits[0].country, 'DE');
});
}); });
test('query string', function () { test('query string', function () {
var data = _wrapData(); var data = _wrapData();
var out = data.query({q: 'UK'}); data.query({q: 'UK'}).then(function(out) {
equal(out.total, 3); equal(out.total, 3);
deepEqual(_.pluck(out.records, 'country'), ['UK', 'UK', 'UK']); deepEqual(_.pluck(out.hits, 'country'), ['UK', 'UK', 'UK']);
});
var out = data.query({q: 'UK 6'}) data.query({q: 'UK 6'}).then(function(out) {
equal(out.total, 1); equal(out.total, 1);
deepEqual(out.records[0].id, 1); deepEqual(out.hits[0].id, 1);
});
}); });
test('filters', function () { test('filters', function () {
var data = _wrapData(); var data = _wrapData();
var query = new recline.Model.Query(); var query = new recline.Model.Query();
query.addFilter({type: 'term', field: 'country', term: 'UK'}); query.addFilter({type: 'term', field: 'country', term: 'UK'});
var out = data.query(query.toJSON()); data.query(query.toJSON()).then(function(out) {
equal(out.total, 3); equal(out.total, 3);
deepEqual(_.pluck(out.records, 'country'), ['UK', 'UK', 'UK']); deepEqual(_.pluck(out.hits, 'country'), ['UK', 'UK', 'UK']);
});
}); });
test('facet', function () { test('facet', function () {
@@ -167,7 +174,7 @@ test('basics', function () {
var dataset = makeBackendDataset(); var dataset = makeBackendDataset();
expect(3); expect(3);
// convenience for tests - get the data that should get changed // convenience for tests - get the data that should get changed
var data = dataset._dataCache; var data = dataset._store;
dataset.fetch().then(function(datasetAgain) { dataset.fetch().then(function(datasetAgain) {
equal(dataset.get('name'), memoryData.metadata.name); equal(dataset.get('name'), memoryData.metadata.name);
deepEqual(_.pluck(dataset.fields.toJSON(), 'id'), _.pluck(data.fields, 'id')); deepEqual(_.pluck(dataset.fields.toJSON(), 'id'), _.pluck(data.fields, 'id'));
@@ -178,21 +185,21 @@ test('basics', function () {
test('query', function () { test('query', function () {
var dataset = makeBackendDataset(); var dataset = makeBackendDataset();
// convenience for tests - get the data that should get changed // convenience for tests - get the data that should get changed
var data = dataset._dataCache.data; var data = dataset._store.data;
var dataset = makeBackendDataset(); var dataset = makeBackendDataset();
var queryObj = { var queryObj = {
size: 4 size: 4
, from: 2 , from: 2
}; };
dataset.query(queryObj).then(function(recordList) { dataset.query(queryObj).then(function(recordList) {
deepEqual(data[2], recordList.models[0].toJSON()); deepEqual(recordList.models[0].toJSON(), data[2]);
}); });
}); });
test('query sort', function () { test('query sort', function () {
var dataset = makeBackendDataset(); var dataset = makeBackendDataset();
// convenience for tests - get the data that should get changed // convenience for tests - get the data that should get changed
var data = dataset._dataCache.data; var data = dataset._store.data;
var queryObj = { var queryObj = {
sort: [ sort: [
{'y': {order: 'desc'}} {'y': {order: 'desc'}}
@@ -253,7 +260,7 @@ test('facet', function () {
test('update and delete', function () { test('update and delete', function () {
var dataset = makeBackendDataset(); var dataset = makeBackendDataset();
// convenience for tests - get the data that should get changed // convenience for tests - get the data that should get changed
var data = dataset._dataCache; var data = dataset._store;
dataset.query().then(function(docList) { dataset.query().then(function(docList) {
equal(docList.length, Math.min(100, data.data.length)); equal(docList.length, Math.min(100, data.data.length));
var doc1 = docList.models[0]; var doc1 = docList.models[0];

2
test/base.js
View File

@@ -19,7 +19,7 @@ var Fixture = {
{id: 4, date: '2011-05-04', x: 5, y: 10, z: 15, country: 'UK', title: 'fifth', lat:51.58, lon:0}, {id: 4, date: '2011-05-04', x: 5, y: 10, z: 15, country: 'UK', title: 'fifth', lat:51.58, lon:0},
{id: 5, date: '2011-06-02', x: 6, y: 12, z: 18, country: 'DE', title: 'sixth', lat:51.04, lon:7.9} {id: 5, date: '2011-06-02', x: 6, y: 12, z: 18, country: 'DE', title: 'sixth', lat:51.04, lon:7.9}
]; ];
var dataset = recline.Backend.Memory.createDataset(documents, fields); var dataset = new recline.Model.Dataset({records: documents, fields: fields});
return dataset; return dataset;
} }
}; };

9
test/model.test.js
View File

@@ -116,15 +116,6 @@ test('Dataset', function () {
equal(out.fields.length, 2); equal(out.fields.length, 2);
}); });
test('Dataset _prepareQuery', function () {
var meta = {id: 'test', title: 'xyz'};
var dataset = new recline.Model.Dataset(meta);
var out = dataset._prepareQuery();
var exp = new recline.Model.Query().toJSON();
deepEqual(out, exp);
});
test('Dataset getFieldsSummary', function () { test('Dataset getFieldsSummary', function () {
var dataset = Fixture.getDataset(); var dataset = Fixture.getDataset();
dataset.getFieldsSummary().done(function() { dataset.getFieldsSummary().done(function() {