knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#162,backend,model][l]: major commit addressing several parts of the backend / model refactor in #162.

Browse Source 
* Now have Dataset setup and manage "memory store"

  * New fetch API as per issue #162 spec
  * dataproxy utilizes useMemoryStore attribute and just implements fetch
  * Switch gdocs to use Memory.Store properly via new useMemoryStore + fetch methodology
  * Memory backend: query function now follows promise API, remove fetch,upsert,delete and add save function to Store object

* Also refactor to remove _source in QueryResult "hits" attribute on all backends but ElasticSearch - cf #159 (note this means ES currently broken)
This commit is contained in:
Rufus Pollock
2012-06-23 20:23:24 +01:00
parent bda4797ed8
commit 6e5c15a816
8 changed files with 153 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/backend/dataproxy.js
View File

@@ -50,11 +50,11 @@ this.recline.Backend.DataProxy = this.recline.Backend.DataProxy || {};
});
return tmp;
});
var store = new recline.Backend.Memory.Store(records, fields);
dataset._dataCache = store;
dataset.fields.reset(fields);
dataset.query();
dfd.resolve(dataset);
dfd.resolve({
records: records,
fields: fields,
useMemoryStore: true
});
})
.fail(function(arguments) {
dfd.reject(arguments);
@@ -62,21 +62,6 @@ this.recline.Backend.DataProxy = this.recline.Backend.DataProxy || {};
return dfd.promise();
};
my.query = function(dataset, queryObj) {
var dfd = $.Deferred();
var results = dataset._dataCache.query(queryObj);
var hits = _.map(results.records, function(row) {
return { _source: row };
});
var out = {
total: results.total,
hits: hits,
facets: results.facets
};
dfd.resolve(out);
return dfd.promise();
};
// ## _wrapInTimeout
//
// Convenience method providing a crude way to catch backend errors on JSONP calls.

98
src/backend/gdocs.js
View File

@@ -3,14 +3,13 @@ this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
(function($, my) {
my.__type__ = 'gdocs';
// ## Google spreadsheet backend
//
// Connect to Google Docs spreadsheet.
//
// Dataset must have a url attribute pointing to the Gdocs
// spreadsheet's JSON feed e.g.
// Fetch data from a Google Docs spreadsheet.
//
// Dataset must have a url attribute pointing to the Gdocs or its JSON feed e.g.
// <pre>
// var dataset = new recline.Model.Dataset({
// url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
@@ -18,77 +17,25 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
// 'gdocs'
// );
// </pre>
my.Backbone = function() {
var self = this;
this.__type__ = 'gdocs';
this.readonly = true;
this.sync = function(method, model, options) {
var self = this;
if (method === "read") {
var dfd = $.Deferred();
dfd.resolve(model);
return dfd.promise();
}
};
this.query = function(dataset, queryObj) {
var dfd = $.Deferred();
if (dataset._dataCache) {
dfd.resolve(dataset._dataCache);
} else {
loadData(dataset.get('url')).done(function(result) {
dataset.fields.reset(result.fields);
// cache data onto dataset (we have loaded whole gdoc it seems!)
dataset._dataCache = self._formatResults(dataset, result.data);
dfd.resolve(dataset._dataCache);
});
}
return dfd.promise();
};
this._formatResults = function(dataset, data) {
var fields = _.pluck(dataset.fields.toJSON(), 'id');
// zip the fields with the data rows to produce js objs
// TODO: factor this out as a common method with other backends
var objs = _.map(data, function (d) {
var obj = {};
_.each(_.zip(fields, d), function (x) {
obj[x[0]] = x[1];
});
return obj;
});
var out = {
total: objs.length,
hits: _.map(objs, function(row) {
return { _source: row }
})
}
return out;
};
};
// ## loadData
//
// loadData from a google docs URL
//
// @return object with two attributes
//
// * fields: array of objects
// * data: array of arrays
var loadData = function(url) {
// * fields: array of Field objects
// * records: array of objects for each row
//
my.fetch = function(dataset) {
var dfd = $.Deferred();
var url = my.getSpreadsheetAPIUrl(url);
var out = {
fields: [],
data: []
}
var url = my.getSpreadsheetAPIUrl(dataset.get('url'));
$.getJSON(url, function(d) {
result = my.parseData(d);
result.fields = _.map(result.fields, function(fieldId) {
var fields = _.map(result.fields, function(fieldId) {
return {id: fieldId};
});
dfd.resolve(result);
dfd.resolve({
records: result.records,
fields: fields,
useMemoryStore: true
});
});
return dfd.promise();
};
@@ -109,8 +56,8 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
options = arguments[1];
}
var results = {
'fields': [],
'data': []
fields: [],
records: []
};
// default is no special info on type of columns
var colTypes = {};
@@ -128,10 +75,9 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
// converts non numberical values that should be numerical (22.3%[string] -> 0.223[float])
var rep = /^([\d\.\-]+)\%$/;
$.each(gdocsSpreadsheet.feed.entry, function (i, entry) {
var row = [];
for (var k in results.fields) {
var col = results.fields[k];
results.records = _.map(gdocsSpreadsheet.feed.entry, function(entry) {
var row = {};
_.each(results.fields, function(col) {
var _keyname = 'gsx$' + col;
var value = entry[_keyname]['$t'];
// if labelled as % and value contains %, convert
@@ -142,9 +88,9 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
value = value3 / 100;
}
}
row.push(value);
}
results.data.push(row);
row[col] = value;
});
return row;
});
return results;
};

65
src/backend/memory.js
View File

@@ -20,49 +20,9 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
var dataset = new recline.Model.Dataset(
_.extend({}, metadata, {records: data, fields: fields})
);
dataset.fetch();
return dataset;
};
my.fetch = function(dataset) {
var dfd = $.Deferred();
var store = new my.Store(dataset.get('records'), dataset.get('fields'));
dataset._dataCache = store;
dataset.fields.reset(store.fields);
dataset.query();
dfd.resolve(dataset);
return dfd.promise();
};
my.save = function(dataset, changes) {
var dfd = $.Deferred();
// TODO
// _.each(changes.creates) { ... }
_.each(changes.updates, function(record) {
dataset._dataCache.update(record);
});
_.each(changes.deletes, function(record) {
dataset._dataCache.delete(record);
});
dfd.resolve(dataset);
return dfd.promise();
},
my.query = function(dataset, queryObj) {
var dfd = $.Deferred();
var results = dataset._dataCache.query(queryObj);
var hits = _.map(results.records, function(row) {
return { _source: row };
});
var out = {
total: results.total,
hits: hits,
facets: results.facets
};
dfd.resolve(out);
return dfd.promise();
};
// ## Data Wrapper
//
// Turn a simple array of JS objects into a mini data-store with
@@ -102,7 +62,22 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
this.data = newdocs;
};
this.save = function(changes, dataset) {
var self = this;
var dfd = $.Deferred();
// TODO _.each(changes.creates) { ... }
_.each(changes.updates, function(record) {
self.update(record);
});
_.each(changes.deletes, function(record) {
self.delete(record);
});
dfd.resolve(this);
return dfd.promise();
},
this.query = function(queryObj) {
var dfd = $.Deferred();
var numRows = queryObj.size || this.data.length;
var start = queryObj.from || 0;
var results = this.data;
@@ -119,14 +94,14 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
results.reverse();
}
});
var total = results.length;
var facets = this.computeFacets(results, queryObj);
results = results.slice(start, start+numRows);
return {
total: total,
records: results,
var out = {
total: results.length,
hits: results.slice(start, start+numRows),
facets: facets
};
dfd.resolve(out);
return dfd.promise();
};
// in place filtering

110
src/model.js
View File

@@ -65,17 +65,49 @@ my.Dataset = Backbone.Model.extend({
this.queryState = new my.Query();
this.queryState.bind('change', this.query);
this.queryState.bind('facet:add', this.query);
this._store = this.backend;
if (this.backend == recline.Backend.Memory) {
this.fetch();
}
},
// ### fetch
//
// Retrieve dataset and (some) records from the backend.
fetch: function() {
return this.backend.fetch(this);
var self = this;
var dfd = $.Deferred();
// TODO: fail case;
if (this.backend !== recline.Backend.Memory) {
this.backend.fetch(this).then(handleResults)
} else {
// special case where we have been given data directly
handleResults({
records: this.get('records'),
fields: this.get('fields'),
useMemoryStore: true
});
}
function handleResults(results) {
self.set(results.metadata);
if (results.useMemoryStore) {
self._store = new recline.Backend.Memory.Store(results.records, results.fields);
self.query();
// store will have extracted fields if not provided
self.fields.reset(self._store.fields);
} else {
self.fields.reset(results.fields);
}
// TODO: parsing the processing of fields
dfd.resolve(this);
}
return dfd.promise();
},
save: function() {
return this.backend.save(this, this._changes);
var self = this;
return this._store.save(this._changes, this);
},
// ### query
@@ -89,48 +121,48 @@ my.Dataset = Backbone.Model.extend({
// also returned.
query: function(queryObj) {
var self = this;
this.trigger('query:start');
var actualQuery = self._prepareQuery(queryObj);
var dfd = $.Deferred();
this.backend.query(this, actualQuery).done(function(queryResult) {
self.docCount = queryResult.total;
var docs = _.map(queryResult.hits, function(hit) {
var _doc = new my.Record(hit._source);
_doc.backend = self.backend;
_doc.dataset = self;
_doc.bind('change', function(doc) {
self._changes.updates.push(doc.toJSON());
});
_doc.bind('destroy', function(doc) {
self._changes.deletes.push(doc.toJSON());
});
return _doc;
this.trigger('query:start');
if (queryObj) {
this.queryState.set(queryObj);
}
var actualQuery = this.queryState.toJSON();
this._store.query(actualQuery, this)
.done(function(queryResult) {
self._handleQueryResult(queryResult);
self.trigger('query:done');
dfd.resolve(self.currentRecords);
})
.fail(function(arguments) {
self.trigger('query:fail', arguments);
dfd.reject(arguments);
});
self.currentRecords.reset(docs);
if (queryResult.facets) {
var facets = _.map(queryResult.facets, function(facetResult, facetId) {
facetResult.id = facetId;
return new my.Facet(facetResult);
});
self.facets.reset(facets);
}
self.trigger('query:done');
dfd.resolve(self.currentRecords);
})
.fail(function(arguments) {
self.trigger('query:fail', arguments);
dfd.reject(arguments);
});
return dfd.promise();
},
_prepareQuery: function(newQueryObj) {
if (newQueryObj) {
this.queryState.set(newQueryObj);
_handleQueryResult: function(queryResult) {
var self = this;
self.docCount = queryResult.total;
var docs = _.map(queryResult.hits, function(hit) {
var _doc = new my.Record(hit);
_doc.bind('change', function(doc) {
self._changes.updates.push(doc.toJSON());
});
_doc.bind('destroy', function(doc) {
self._changes.deletes.push(doc.toJSON());
});
return _doc;
});
self.currentRecords.reset(docs);
if (queryResult.facets) {
var facets = _.map(queryResult.facets, function(facetResult, facetId) {
facetResult.id = facetId;
return new my.Facet(facetResult);
});
self.facets.reset(facets);
}
var out = this.queryState.toJSON();
return out;
},
toTemplateJSON: function() {
@@ -151,7 +183,7 @@ my.Dataset = Backbone.Model.extend({
query.addFacet(field.id);
});
var dfd = $.Deferred();
this.backend.query(this, query.toJSON()).done(function(queryResult) {
this._store.query(query.toJSON(), this).done(function(queryResult) {
if (queryResult.facets) {
_.each(queryResult.facets, function(facetResult, facetId) {
facetResult.id = facetId;