knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#128,backend/memory,misc][l]: commence major backend refactor by converting recline.Backend.Memory to module and splitting existing code into BackboneSyncer and DataWrapper.

Browse Source 
* Lots of other changes to having passing tests (note some actual functionality is likely a little broken esp around state serialization and the app)
This commit is contained in:
Rufus Pollock 2012-05-25 23:42:58 +01:00
parent 39c72aef13
commit 23b32dff1c
9 changed files with 250 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/backend/localcsv.js
View File

@ -33,7 +33,7 @@ this.recline.Backend = this.recline.Backend || {};
});
return _doc;
});
var dataset = recline.Backend.createDataset(data, fields);
var dataset = recline.Backend.Memory.createDataset(data, fields);
return dataset;
};

226
src/backend/memory.js
View File

@ -1,5 +1,6 @@
this.recline = this.recline || {};
this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.Memory = this.recline.Backend.Memory || {};
(function($, my) {
// ## createDataset
@ -14,115 +15,54 @@ this.recline.Backend = this.recline.Backend || {};
// @param metadata: (optional) dataset metadata - see recline.Model.Dataset.
// If not defined (or id not provided) id will be autogenerated.
my.createDataset = function(data, fields, metadata) {
if (!metadata) {
metadata = {};
}
if (!metadata.id) {
metadata.id = String(Math.floor(Math.random() * 100000000) + 1);
}
var backend = new recline.Backend.Memory();
var datasetInfo = {
documents: data,
metadata: metadata
};
if (fields) {
datasetInfo.fields = fields;
} else {
if (data) {
datasetInfo.fields = _.map(data[0], function(value, key) {
return {id: key};
});
}
}
backend.addDataset(datasetInfo);
var dataset = new recline.Model.Dataset({id: metadata.id}, backend);
var wrapper = new my.DataWrapper(data, fields);
var syncer = new my.BackboneSyncer();
var dataset = new recline.Model.Dataset(metadata, syncer);
dataset._dataCache = wrapper;
dataset.fetch();
dataset.query();
return dataset;
};
// ## Memory Backend - uses in-memory data
// ## Data Wrapper
//
// To use it you should provide in your constructor data:
//
// * metadata (including fields array)
// * documents: list of hashes, each hash being one doc. A doc *must* have an id attribute which is unique.
//
// Example:
//
// <pre>
// // Backend setup
// var backend = recline.Backend.Memory();
// backend.addDataset({
// metadata: {
// id: 'my-id',
// title: 'My Title'
// },
// fields: [{id: 'x'}, {id: 'y'}, {id: 'z'}],
// documents: [
// {id: 0, x: 1, y: 2, z: 3},
// {id: 1, x: 2, y: 4, z: 6}
// ]
// });
// // later ...
// var dataset = Dataset({id: 'my-id'}, 'memory');
// dataset.fetch();
// etc ...
// </pre>
my.Memory = my.Base.extend({
__type__: 'memory',
readonly: false,
initialize: function() {
this.datasets = {};
},
addDataset: function(data) {
this.datasets[data.metadata.id] = $.extend(true, {}, data);
},
sync: function(method, model, options) {
var self = this;
var dfd = $.Deferred();
if (method === "read") {
if (model.__type__ == 'Dataset') {
var rawDataset = this.datasets[model.id];
model.set(rawDataset.metadata);
model.fields.reset(rawDataset.fields);
model.docCount = rawDataset.documents.length;
dfd.resolve(model);
}
return dfd.promise();
} else if (method === 'update') {
if (model.__type__ == 'Document') {
_.each(self.datasets[model.dataset.id].documents, function(doc, idx) {
if(doc.id === model.id) {
self.datasets[model.dataset.id].documents[idx] = model.toJSON();
}
});
dfd.resolve(model);
}
return dfd.promise();
} else if (method === 'delete') {
if (model.__type__ == 'Document') {
var rawDataset = self.datasets[model.dataset.id];
var newdocs = _.reject(rawDataset.documents, function(doc) {
return (doc.id === model.id);
});
rawDataset.documents = newdocs;
dfd.resolve(model);
}
return dfd.promise();
} else {
alert('Not supported: sync on Memory backend with method ' + method + ' and model ' + model);
// Turn a simple array of JS objects into a mini data-store with
// functionality like querying, faceting, updating (by ID) and deleting (by
// ID).
my.DataWrapper = function(data, fields) {
var self = this;
this.data = data;
if (fields) {
this.fields = fields;
} else {
if (data) {
this.fields = _.map(data[0], function(value, key) {
return {id: key};
});
}
},
query: function(model, queryObj) {
var dfd = $.Deferred();
var out = {};
var numRows = queryObj.size;
var start = queryObj.from;
var results = this.datasets[model.id].documents;
}
this.update = function(doc) {
_.each(self.data, function(internalDoc, idx) {
if(doc.id === internalDoc.id) {
self.data[idx] = doc;
}
});
};
this.delete = function(doc) {
var newdocs = _.reject(self.data, function(internalDoc) {
return (doc.id === internalDoc.id);
});
this.data = newdocs;
};
this.query = function(queryObj) {
var numRows = queryObj.size || this.data.length;
var start = queryObj.from || 0;
var results = this.data;
results = this._applyFilters(results, queryObj);
results = this._applyFreeTextQuery(model, results, queryObj);
results = this._applyFreeTextQuery(results, queryObj);
// not complete sorting!
_.each(queryObj.sort, function(sortObj) {
var fieldName = _.keys(sortObj)[0];
@ -131,17 +71,18 @@ this.recline.Backend = this.recline.Backend || {};
return (sortObj[fieldName].order == 'asc') ? _out : -1*_out;
});
});
out.facets = this._computeFacets(results, queryObj);
var total = results.length;
resultsObj = this._docsToQueryResult(results.slice(start, start+numRows));
_.extend(out, resultsObj);
out.total = total;
dfd.resolve(out);
return dfd.promise();
},
var facets = this.computeFacets(results, queryObj);
results = results.slice(start, start+numRows);
return {
total: total,
documents: results,
facets: facets
};
};
// in place filtering
_applyFilters: function(results, queryObj) {
this._applyFilters = function(results, queryObj) {
_.each(queryObj.filters, function(filter) {
results = _.filter(results, function(doc) {
var fieldId = _.keys(filter.term)[0];
@ -149,17 +90,17 @@ this.recline.Backend = this.recline.Backend || {};
});
});
return results;
},
};
// we OR across fields but AND across terms in query string
_applyFreeTextQuery: function(dataset, results, queryObj) {
this._applyFreeTextQuery = function(results, queryObj) {
if (queryObj.q) {
var terms = queryObj.q.split(' ');
results = _.filter(results, function(rawdoc) {
var matches = true;
_.each(terms, function(term) {
var foundmatch = false;
dataset.fields.each(function(field) {
_.each(self.fields, function(field) {
var value = rawdoc[field.id];
if (value !== null) { value = value.toString(); }
// TODO regexes?
@ -175,14 +116,15 @@ this.recline.Backend = this.recline.Backend || {};
});
}
return results;
},
};
_computeFacets: function(documents, queryObj) {
this.computeFacets = function(documents, queryObj) {
var facetResults = {};
if (!queryObj.facets) {
return facetsResults;
return facetResults;
}
_.each(queryObj.facets, function(query, facetId) {
// TODO: remove dependency on recline.Model
facetResults[facetId] = new recline.Model.Facet({id: facetId}).toJSON();
facetResults[facetId].termsall = {};
});
@ -211,7 +153,55 @@ this.recline.Backend = this.recline.Backend || {};
tmp.terms = tmp.terms.slice(0, 10);
});
return facetResults;
}
});
};
};
}(jQuery, this.recline.Backend));
// ## BackboneSyncer
//
// Provide a Backbone Sync interface to a DataWrapper data backend attached
// to a Dataset object
my.BackboneSyncer = function() {
this.sync = function(method, model, options) {
var self = this;
var dfd = $.Deferred();
if (method === "read") {
if (model.__type__ == 'Dataset') {
model.fields.reset(model._dataCache.fields);
dfd.resolve(model);
}
return dfd.promise();
} else if (method === 'update') {
if (model.__type__ == 'Document') {
model.dataset._dataCache.update(model.toJSON());
dfd.resolve(model);
}
return dfd.promise();
} else if (method === 'delete') {
if (model.__type__ == 'Document') {
model.dataset._dataCache.delete(model.toJSON());
dfd.resolve(model);
}
return dfd.promise();
} else {
alert('Not supported: sync on Memory backend with method ' + method + ' and model ' + model);
}
};
this.query = function(model, queryObj) {
var dfd = $.Deferred();
var results = model._dataCache.query(queryObj);
var hits = _.map(results.documents, function(row) {
return { _source: row };
});
var out = {
total: results.total,
hits: hits,
facets: results.facets
};
dfd.resolve(out);
return dfd.promise();
};
};
}(jQuery, this.recline.Backend.Memory));

4
src/model.js
View File

@ -44,6 +44,8 @@ my.Dataset = Backbone.Model.extend({
initialize: function(model, backend) {
_.bindAll(this, 'query');
this.backend = backend;
this.backendType = 'memory';
this.backendURL = null;
if (typeof(backend) === 'string') {
this.backend = this._backendFromString(backend);
}
@ -162,7 +164,7 @@ my.Dataset.restore = function(state) {
state.dataset = {url: state.url};
}
if (state.backend === 'memory') {
dataset = recline.Backend.createDataset(
dataset = recline.Backend.Memory.createDataset(
[{stub: 'this is a stub dataset because we do not restore memory datasets'}],
[],
state.dataset // metadata

2
src/view.js
View File

@ -362,7 +362,7 @@ my.DataExplorer = Backbone.View.extend({
var stateData = _.extend({
query: query,
'view-graph': graphState,
backend: this.model.backend.__type__,
backend: this.model.backendType,
dataset: this.model.toJSON(),
currentView: null,
readOnly: false

168
test/backend/memory.js
View File

@ -1,4 +1,115 @@
module("Backend Memory");
(function ($) {
module("Backend Memory - DataWrapper");
var memoryData = [
{id: 0, x: 1, y: 2, z: 3, country: 'DE', label: 'first'}
, {id: 1, x: 2, y: 4, z: 6, country: 'UK', label: 'second'}
, {id: 2, x: 3, y: 6, z: 9, country: 'US', label: 'third'}
, {id: 3, x: 4, y: 8, z: 12, country: 'UK', label: 'fourth'}
, {id: 4, x: 5, y: 10, z: 15, country: 'UK', label: 'fifth'}
, {id: 5, x: 6, y: 12, z: 18, country: 'DE', label: 'sixth'}
];
var _wrapData = function() {
var dataCopy = $.extend(true, [], memoryData);
return new recline.Backend.Memory.DataWrapper(dataCopy);
}
test('basics', function () {
var data = _wrapData();
equal(data.fields.length, 6);
deepEqual(['id', 'x', 'y', 'z', 'country', 'label'], _.pluck(data.fields, 'id'));
equal(memoryData.length, data.data.length);
});
test('query', function () {
var data = _wrapData();
var queryObj = {
size: 4
, from: 2
};
var out = data.query(queryObj);
deepEqual(out.documents[0], memoryData[2]);
equal(out.documents.length, 4);
equal(out.total, 6);
});
test('query sort', function () {
var data = _wrapData();
var queryObj = {
sort: [
{'y': {order: 'desc'}}
]
};
var out = data.query(queryObj);
equal(out.documents[0].x, 6);
});
test('query string', function () {
var data = _wrapData();
var out = data.query({q: 'UK'});
equal(out.total, 3);
deepEqual(_.pluck(out.documents, 'country'), ['UK', 'UK', 'UK']);
var out = data.query({q: 'UK 6'})
equal(out.total, 1);
deepEqual(out.documents[0].id, 1);
});
test('filters', function () {
var data = _wrapData();
var query = new recline.Model.Query();
query.addTermFilter('country', 'UK');
var out = data.query(query.toJSON());
equal(out.total, 3);
deepEqual(_.pluck(out.documents, 'country'), ['UK', 'UK', 'UK']);
});
test('facet', function () {
var data = _wrapData();
var query = new recline.Model.Query();
query.addFacet('country');
var out = data.computeFacets(data.data, query.toJSON());
var exp = [
{
term: 'UK',
count: 3
},
{
term: 'DE',
count: 2
},
{
term: 'US',
count: 1
}
];
deepEqual(out['country'].terms, exp);
});
test('update and delete', function () {
var data = _wrapData();
// Test UPDATE
var newVal = 10;
doc1 = $.extend(true, {}, memoryData[0]);
doc1.x = newVal;
data.update(doc1);
equal(data.data[0].x, newVal);
// Test Delete
data.delete(doc1);
equal(data.data.length, 5);
equal(data.data[0].x, memoryData[1].x);
});
})(this.jQuery);
// ======================================
(function ($) {
module("Backend Memory - BackboneSyncer");
var memoryData = {
metadata: {
@ -18,60 +129,48 @@ var memoryData = {
};
function makeBackendDataset() {
var backend = new recline.Backend.Memory();
backend.addDataset(memoryData);
var dataset = new recline.Model.Dataset({id: memoryData.metadata.id}, backend);
var dataset = new recline.Backend.Memory.createDataset(memoryData.documents, null, memoryData.metadata);
return dataset;
}
test('Memory Backend: readonly', function () {
var backend = new recline.Backend.Memory();
equal(backend.readonly, false);
});
test('Memory Backend: createDataset', function () {
var dataset = recline.Backend.createDataset(memoryData.documents, memoryData.fields, memoryData.metadata);
equal(memoryData.metadata.id, dataset.id);
});
test('Memory Backend: createDataset 2', function () {
var dataset = recline.Backend.createDataset(memoryData.documents);
test('createDataset', function () {
var dataset = recline.Backend.Memory.createDataset(memoryData.documents);
equal(dataset.fields.length, 6);
deepEqual(['id', 'x', 'y', 'z', 'country', 'label'], dataset.fields.pluck('id'));
dataset.query();
equal(memoryData.documents.length, dataset.currentDocuments.length);
});
test('Memory Backend: basics', function () {
test('basics', function () {
var dataset = makeBackendDataset();
expect(3);
// convenience for tests - get the data that should get changed
var data = dataset.backend.datasets[memoryData.metadata.id];
var data = dataset._dataCache;
dataset.fetch().then(function(datasetAgain) {
equal(dataset.get('name'), data.metadata.name);
equal(dataset.get('name'), memoryData.metadata.name);
deepEqual(_.pluck(dataset.fields.toJSON(), 'id'), _.pluck(data.fields, 'id'));
equal(dataset.docCount, 6);
});
});
test('Memory Backend: query', function () {
test('query', function () {
var dataset = makeBackendDataset();
// convenience for tests - get the data that should get changed
var data = dataset.backend.datasets[memoryData.metadata.id];
var data = dataset._dataCache.data;
var dataset = makeBackendDataset();
var queryObj = {
size: 4
, from: 2
};
dataset.query(queryObj).then(function(documentList) {
deepEqual(data.documents[2], documentList.models[0].toJSON());
deepEqual(data[2], documentList.models[0].toJSON());
});
});
test('Memory Backend: query sort', function () {
test('query sort', function () {
var dataset = makeBackendDataset();
// convenience for tests - get the data that should get changed
var data = dataset.backend.datasets[memoryData.metadata.id];
var data = dataset._dataCache.data;
var queryObj = {
sort: [
{'y': {order: 'desc'}}
@ -83,7 +182,7 @@ test('Memory Backend: query sort', function () {
});
});
test('Memory Backend: query string', function () {
test('query string', function () {
var dataset = makeBackendDataset();
dataset.fetch();
dataset.query({q: 'UK'}).then(function() {
@ -97,7 +196,7 @@ test('Memory Backend: query string', function () {
});
});
test('Memory Backend: filters', function () {
test('filters', function () {
var dataset = makeBackendDataset();
dataset.queryState.addTermFilter('country', 'UK');
dataset.query().then(function() {
@ -106,7 +205,7 @@ test('Memory Backend: filters', function () {
});
});
test('Memory Backend: facet', function () {
test('facet', function () {
var dataset = makeBackendDataset();
dataset.queryState.addFacet('country');
dataset.query().then(function() {
@ -129,27 +228,28 @@ test('Memory Backend: facet', function () {
});
});
test('Memory Backend: update and delete', function () {
test('update and delete', function () {
var dataset = makeBackendDataset();
// convenience for tests - get the data that should get changed
var data = dataset.backend.datasets[memoryData.metadata.id];
var data = dataset._dataCache;
dataset.query().then(function(docList) {
equal(docList.length, Math.min(100, data.documents.length));
equal(docList.length, Math.min(100, data.data.length));
var doc1 = docList.models[0];
deepEqual(doc1.toJSON(), data.documents[0]);
deepEqual(doc1.toJSON(), data.data[0]);
// Test UPDATE
var newVal = 10;
doc1.set({x: newVal});
doc1.save().then(function() {
equal(data.documents[0].x, newVal);
equal(data.data[0].x, newVal);
})
// Test Delete
doc1.destroy().then(function() {
equal(data.documents.length, 5);
equal(data.documents[0].x, memoryData.documents[1].x);
equal(data.data.length, 5);
equal(data.data[0].x, memoryData.documents[1].x);
});
});
});
})(this.jQuery);

2
test/base.js
View File

@ -19,7 +19,7 @@ var Fixture = {
{id: 4, date: '2011-05-04', x: 5, y: 10, z: 15, country: 'UK', label: 'fifth', lat:51.58, lon:0},
{id: 5, date: '2011-06-02', x: 6, y: 12, z: 18, country: 'DE', label: 'sixth', lat:51.04, lon:7.9}
];
var dataset = recline.Backend.createDataset(documents, fields);
var dataset = recline.Backend.Memory.createDataset(documents, fields);
return dataset;
}
};

11
test/model.test.js
View File

@ -125,17 +125,6 @@ test('Dataset _prepareQuery', function () {
deepEqual(out, exp);
});
test('Dataset _backendFromString', function () {
var dataset = new recline.Model.Dataset();
var out = dataset._backendFromString('recline.Backend.Memory');
equal(out.__type__, 'memory');
var out = dataset._backendFromString('dataproxy');
equal(out.__type__, 'dataproxy');
});
// =================================
// Query

2
test/view-map.test.js
View File

@ -16,7 +16,7 @@ var GeoJSONFixture = {
{id: 1, x: 2, y: 4, z: 6, geom: {type:"Point",coordinates:[13.40,52.35]}},
{id: 2, x: 3, y: 6, z: 9, geom: {type:"LineString",coordinates:[[100.0, 0.0],[101.0, 1.0]]}}
];
var dataset = recline.Backend.createDataset(documents, fields);
var dataset = recline.Backend.Memory.createDataset(documents, fields);
return dataset;
}
};

2
test/view-timeline.test.js
View File

@ -1,7 +1,7 @@
module("View - Timeline");
test('extract dates and timelineJSON', function () {
var dataset = recline.Backend.createDataset([
var dataset = recline.Backend.Memory.createDataset([
{'Date': '2012-03-20', 'title': '1'},
{'Date': '2012-03-25', 'title': '2'},
]);