knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#162,backend,model][l]: major commit addressing several parts of the backend / model refactor in #162.

Browse Source 
* Now have Dataset setup and manage "memory store"

  * New fetch API as per issue #162 spec
  * dataproxy utilizes useMemoryStore attribute and just implements fetch
  * Switch gdocs to use Memory.Store properly via new useMemoryStore + fetch methodology
  * Memory backend: query function now follows promise API, remove fetch,upsert,delete and add save function to Store object

* Also refactor to remove _source in QueryResult "hits" attribute on all backends but ElasticSearch - cf #159 (note this means ES currently broken)
This commit is contained in:
Rufus Pollock 2012-06-23 20:23:24 +01:00
parent bda4797ed8
commit 6e5c15a816
8 changed files with 153 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/backend/dataproxy.js
View File

@ -50,11 +50,11 @@ this.recline.Backend.DataProxy = this.recline.Backend.DataProxy || {};
});
return tmp;
});
var store = new recline.Backend.Memory.Store(records, fields);
dataset._dataCache = store;
dataset.fields.reset(fields);
dataset.query();
dfd.resolve(dataset);
dfd.resolve({
records: records,
fields: fields,
useMemoryStore: true
});
})
.fail(function(arguments) {
dfd.reject(arguments);
@ -62,21 +62,6 @@ this.recline.Backend.DataProxy = this.recline.Backend.DataProxy || {};
return dfd.promise();
};
my.query = function(dataset, queryObj) {
var dfd = $.Deferred();
var results = dataset._dataCache.query(queryObj);
var hits = _.map(results.records, function(row) {
return { _source: row };
});
var out = {
total: results.total,
hits: hits,
facets: results.facets
};
dfd.resolve(out);
return dfd.promise();
};
// ## _wrapInTimeout
//
// Convenience method providing a crude way to catch backend errors on JSONP calls.

98
src/backend/gdocs.js
View File

@ -3,14 +3,13 @@ this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
(function($, my) {
my.__type__ = 'gdocs';
// ## Google spreadsheet backend
//
// Connect to Google Docs spreadsheet.
//
// Dataset must have a url attribute pointing to the Gdocs
// spreadsheet's JSON feed e.g.
// Fetch data from a Google Docs spreadsheet.
//
// Dataset must have a url attribute pointing to the Gdocs or its JSON feed e.g.
// <pre>
// var dataset = new recline.Model.Dataset({
// url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
@ -18,77 +17,25 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
// 'gdocs'
// );
// </pre>
my.Backbone = function() {
var self = this;
this.__type__ = 'gdocs';
this.readonly = true;
this.sync = function(method, model, options) {
var self = this;
if (method === "read") {
var dfd = $.Deferred();
dfd.resolve(model);
return dfd.promise();
}
};
this.query = function(dataset, queryObj) {
var dfd = $.Deferred();
if (dataset._dataCache) {
dfd.resolve(dataset._dataCache);
} else {
loadData(dataset.get('url')).done(function(result) {
dataset.fields.reset(result.fields);
// cache data onto dataset (we have loaded whole gdoc it seems!)
dataset._dataCache = self._formatResults(dataset, result.data);
dfd.resolve(dataset._dataCache);
});
}
return dfd.promise();
};
this._formatResults = function(dataset, data) {
var fields = _.pluck(dataset.fields.toJSON(), 'id');
// zip the fields with the data rows to produce js objs
// TODO: factor this out as a common method with other backends
var objs = _.map(data, function (d) {
var obj = {};
_.each(_.zip(fields, d), function (x) {
obj[x[0]] = x[1];
});
return obj;
});
var out = {
total: objs.length,
hits: _.map(objs, function(row) {
return { _source: row }
})
}
return out;
};
};
// ## loadData
//
// loadData from a google docs URL
//
// @return object with two attributes
//
// * fields: array of objects
// * data: array of arrays
var loadData = function(url) {
// * fields: array of Field objects
// * records: array of objects for each row
//
my.fetch = function(dataset) {
var dfd = $.Deferred();
var url = my.getSpreadsheetAPIUrl(url);
var out = {
fields: [],
data: []
}
var url = my.getSpreadsheetAPIUrl(dataset.get('url'));
$.getJSON(url, function(d) {
result = my.parseData(d);
result.fields = _.map(result.fields, function(fieldId) {
var fields = _.map(result.fields, function(fieldId) {
return {id: fieldId};
});
dfd.resolve(result);
dfd.resolve({
records: result.records,
fields: fields,
useMemoryStore: true
});
});
return dfd.promise();
};
@ -109,8 +56,8 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
options = arguments[1];
}
var results = {
'fields': [],
'data': []
fields: [],
records: []
};
// default is no special info on type of columns
var colTypes = {};
@ -128,10 +75,9 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
// converts non numberical values that should be numerical (22.3%[string] -> 0.223[float])
var rep = /^([\d\.\-]+)\%$/;
$.each(gdocsSpreadsheet.feed.entry, function (i, entry) {
var row = [];
for (var k in results.fields) {
var col = results.fields[k];
results.records = _.map(gdocsSpreadsheet.feed.entry, function(entry) {
var row = {};
_.each(results.fields, function(col) {
var _keyname = 'gsx$' + col;
var value = entry[_keyname]['$t'];
// if labelled as % and value contains %, convert
@ -142,9 +88,9 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
value = value3 / 100;
}
}
row.push(value);
}
results.data.push(row);
row[col] = value;
});
return row;
});
return results;
};

65
src/backend/memory.js
View File

@ -20,49 +20,9 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
var dataset = new recline.Model.Dataset(
_.extend({}, metadata, {records: data, fields: fields})
);
dataset.fetch();
return dataset;
};
my.fetch = function(dataset) {
var dfd = $.Deferred();
var store = new my.Store(dataset.get('records'), dataset.get('fields'));
dataset._dataCache = store;
dataset.fields.reset(store.fields);
dataset.query();
dfd.resolve(dataset);
return dfd.promise();
};
my.save = function(dataset, changes) {
var dfd = $.Deferred();
// TODO
// _.each(changes.creates) { ... }
_.each(changes.updates, function(record) {
dataset._dataCache.update(record);
});
_.each(changes.deletes, function(record) {
dataset._dataCache.delete(record);
});
dfd.resolve(dataset);
return dfd.promise();
},
my.query = function(dataset, queryObj) {
var dfd = $.Deferred();
var results = dataset._dataCache.query(queryObj);
var hits = _.map(results.records, function(row) {
return { _source: row };
});
var out = {
total: results.total,
hits: hits,
facets: results.facets
};
dfd.resolve(out);
return dfd.promise();
};
// ## Data Wrapper
//
// Turn a simple array of JS objects into a mini data-store with
@ -102,7 +62,22 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
this.data = newdocs;
};
this.save = function(changes, dataset) {
var self = this;
var dfd = $.Deferred();
// TODO _.each(changes.creates) { ... }
_.each(changes.updates, function(record) {
self.update(record);
});
_.each(changes.deletes, function(record) {
self.delete(record);
});
dfd.resolve(this);
return dfd.promise();
},
this.query = function(queryObj) {
var dfd = $.Deferred();
var numRows = queryObj.size || this.data.length;
var start = queryObj.from || 0;
var results = this.data;
@ -119,14 +94,14 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
results.reverse();
}
});
var total = results.length;
var facets = this.computeFacets(results, queryObj);
results = results.slice(start, start+numRows);
return {
total: total,
records: results,
var out = {
total: results.length,
hits: results.slice(start, start+numRows),
facets: facets
};
dfd.resolve(out);
return dfd.promise();
};
// in place filtering

110
src/model.js
View File

@ -65,17 +65,49 @@ my.Dataset = Backbone.Model.extend({
this.queryState = new my.Query();
this.queryState.bind('change', this.query);
this.queryState.bind('facet:add', this.query);
this._store = this.backend;
if (this.backend == recline.Backend.Memory) {
this.fetch();
}
},
// ### fetch
//
// Retrieve dataset and (some) records from the backend.
fetch: function() {
return this.backend.fetch(this);
var self = this;
var dfd = $.Deferred();
// TODO: fail case;
if (this.backend !== recline.Backend.Memory) {
this.backend.fetch(this).then(handleResults)
} else {
// special case where we have been given data directly
handleResults({
records: this.get('records'),
fields: this.get('fields'),
useMemoryStore: true
});
}
function handleResults(results) {
self.set(results.metadata);
if (results.useMemoryStore) {
self._store = new recline.Backend.Memory.Store(results.records, results.fields);
self.query();
// store will have extracted fields if not provided
self.fields.reset(self._store.fields);
} else {
self.fields.reset(results.fields);
}
// TODO: parsing the processing of fields
dfd.resolve(this);
}
return dfd.promise();
},
save: function() {
return this.backend.save(this, this._changes);
var self = this;
return this._store.save(this._changes, this);
},
// ### query
@ -89,48 +121,48 @@ my.Dataset = Backbone.Model.extend({
// also returned.
query: function(queryObj) {
var self = this;
this.trigger('query:start');
var actualQuery = self._prepareQuery(queryObj);
var dfd = $.Deferred();
this.backend.query(this, actualQuery).done(function(queryResult) {
self.docCount = queryResult.total;
var docs = _.map(queryResult.hits, function(hit) {
var _doc = new my.Record(hit._source);
_doc.backend = self.backend;
_doc.dataset = self;
_doc.bind('change', function(doc) {
self._changes.updates.push(doc.toJSON());
});
_doc.bind('destroy', function(doc) {
self._changes.deletes.push(doc.toJSON());
});
return _doc;
this.trigger('query:start');
if (queryObj) {
this.queryState.set(queryObj);
}
var actualQuery = this.queryState.toJSON();
this._store.query(actualQuery, this)
.done(function(queryResult) {
self._handleQueryResult(queryResult);
self.trigger('query:done');
dfd.resolve(self.currentRecords);
})
.fail(function(arguments) {
self.trigger('query:fail', arguments);
dfd.reject(arguments);
});
self.currentRecords.reset(docs);
if (queryResult.facets) {
var facets = _.map(queryResult.facets, function(facetResult, facetId) {
facetResult.id = facetId;
return new my.Facet(facetResult);
});
self.facets.reset(facets);
}
self.trigger('query:done');
dfd.resolve(self.currentRecords);
})
.fail(function(arguments) {
self.trigger('query:fail', arguments);
dfd.reject(arguments);
});
return dfd.promise();
},
_prepareQuery: function(newQueryObj) {
if (newQueryObj) {
this.queryState.set(newQueryObj);
_handleQueryResult: function(queryResult) {
var self = this;
self.docCount = queryResult.total;
var docs = _.map(queryResult.hits, function(hit) {
var _doc = new my.Record(hit);
_doc.bind('change', function(doc) {
self._changes.updates.push(doc.toJSON());
});
_doc.bind('destroy', function(doc) {
self._changes.deletes.push(doc.toJSON());
});
return _doc;
});
self.currentRecords.reset(docs);
if (queryResult.facets) {
var facets = _.map(queryResult.facets, function(facetResult, facetId) {
facetResult.id = facetId;
return new my.Facet(facetResult);
});
self.facets.reset(facets);
}
var out = this.queryState.toJSON();
return out;
},
toTemplateJSON: function() {
@ -151,7 +183,7 @@ my.Dataset = Backbone.Model.extend({
query.addFacet(field.id);
});
var dfd = $.Deferred();
this.backend.query(this, query.toJSON()).done(function(queryResult) {
this._store.query(query.toJSON(), this).done(function(queryResult) {
if (queryResult.facets) {
_.each(queryResult.facets, function(facetResult, facetId) {
facetResult.id = facetId;

6
test/backend/gdocs.test.js
View File

@ -168,11 +168,10 @@ var sample_gdocs_spreadsheet_data = {
}
test("GDocs Backend", function() {
var backend = new recline.Backend.GDocs.Backbone();
var dataset = new recline.Model.Dataset({
url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
},
backend
'gdocs'
);
var stub = sinon.stub($, 'getJSON', function(options, cb) {
@ -182,7 +181,8 @@ test("GDocs Backend", function() {
}
});
dataset.query().then(function(docList) {
dataset.fetch().then(function() {
var docList = dataset.currentRecords;
deepEqual(['column-2', 'column-1'], _.pluck(dataset.fields.toJSON(), 'id'));
equal(3, docList.length);
equal("A", docList.models[0].get('column-1'));

55
test/backend/memory.test.js
View File

@ -29,10 +29,11 @@ test('query', function () {
size: 4
, from: 2
};
var out = data.query(queryObj);
deepEqual(out.records[0], memoryData[2]);
equal(out.records.length, 4);
equal(out.total, 6);
data.query(queryObj).then(function(out) {
deepEqual(out.hits[0], memoryData[2]);
equal(out.hits.length, 4);
equal(out.total, 6);
});
});
test('query sort', function () {
@ -42,44 +43,50 @@ test('query sort', function () {
{'y': {order: 'desc'}}
]
};
var out = data.query(queryObj);
equal(out.records[0].x, 6);
data.query(queryObj).then(function(out) {
equal(out.hits[0].x, 6);
});
var queryObj = {
sort: [
{'country': {order: 'desc'}}
]
};
var out = data.query(queryObj);
equal(out.records[0].country, 'US');
data.query(queryObj).then(function(out) {
equal(out.hits[0].country, 'US');
});
var queryObj = {
sort: [
{'country': {order: 'asc'}}
]
};
var out = data.query(queryObj);
equal(out.records[0].country, 'DE');
data.query(queryObj).then(function(out) {
equal(out.hits[0].country, 'DE');
});
});
test('query string', function () {
var data = _wrapData();
var out = data.query({q: 'UK'});
equal(out.total, 3);
deepEqual(_.pluck(out.records, 'country'), ['UK', 'UK', 'UK']);
data.query({q: 'UK'}).then(function(out) {
equal(out.total, 3);
deepEqual(_.pluck(out.hits, 'country'), ['UK', 'UK', 'UK']);
});
var out = data.query({q: 'UK 6'})
equal(out.total, 1);
deepEqual(out.records[0].id, 1);
data.query({q: 'UK 6'}).then(function(out) {
equal(out.total, 1);
deepEqual(out.hits[0].id, 1);
});
});
test('filters', function () {
var data = _wrapData();
var query = new recline.Model.Query();
query.addFilter({type: 'term', field: 'country', term: 'UK'});
var out = data.query(query.toJSON());
equal(out.total, 3);
deepEqual(_.pluck(out.records, 'country'), ['UK', 'UK', 'UK']);
data.query(query.toJSON()).then(function(out) {
equal(out.total, 3);
deepEqual(_.pluck(out.hits, 'country'), ['UK', 'UK', 'UK']);
});
});
test('facet', function () {
@ -167,7 +174,7 @@ test('basics', function () {
var dataset = makeBackendDataset();
expect(3);
// convenience for tests - get the data that should get changed
var data = dataset._dataCache;
var data = dataset._store;
dataset.fetch().then(function(datasetAgain) {
equal(dataset.get('name'), memoryData.metadata.name);
deepEqual(_.pluck(dataset.fields.toJSON(), 'id'), _.pluck(data.fields, 'id'));
@ -178,21 +185,21 @@ test('basics', function () {
test('query', function () {
var dataset = makeBackendDataset();
// convenience for tests - get the data that should get changed
var data = dataset._dataCache.data;
var data = dataset._store.data;
var dataset = makeBackendDataset();
var queryObj = {
size: 4
, from: 2
};
dataset.query(queryObj).then(function(recordList) {
deepEqual(data[2], recordList.models[0].toJSON());
deepEqual(recordList.models[0].toJSON(), data[2]);
});
});
test('query sort', function () {
var dataset = makeBackendDataset();
// convenience for tests - get the data that should get changed
var data = dataset._dataCache.data;
var data = dataset._store.data;
var queryObj = {
sort: [
{'y': {order: 'desc'}}
@ -253,7 +260,7 @@ test('facet', function () {
test('update and delete', function () {
var dataset = makeBackendDataset();
// convenience for tests - get the data that should get changed
var data = dataset._dataCache;
var data = dataset._store;
dataset.query().then(function(docList) {
equal(docList.length, Math.min(100, data.data.length));
var doc1 = docList.models[0];

2
test/base.js
View File

@ -19,7 +19,7 @@ var Fixture = {
{id: 4, date: '2011-05-04', x: 5, y: 10, z: 15, country: 'UK', title: 'fifth', lat:51.58, lon:0},
{id: 5, date: '2011-06-02', x: 6, y: 12, z: 18, country: 'DE', title: 'sixth', lat:51.04, lon:7.9}
];
var dataset = recline.Backend.Memory.createDataset(documents, fields);
var dataset = new recline.Model.Dataset({records: documents, fields: fields});
return dataset;
}
};

9
test/model.test.js
View File

@ -116,15 +116,6 @@ test('Dataset', function () {
equal(out.fields.length, 2);
});
test('Dataset _prepareQuery', function () {
var meta = {id: 'test', title: 'xyz'};
var dataset = new recline.Model.Dataset(meta);
var out = dataset._prepareQuery();
var exp = new recline.Model.Query().toJSON();
deepEqual(out, exp);
});
test('Dataset getFieldsSummary', function () {
var dataset = Fixture.getDataset();
dataset.getFieldsSummary().done(function() {