knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#162,be/elasticsearch][s]: convert elasticsearch to the new setup (remove all Backbone from the Backend!).

Browse Source 
* Also convert to new QueryResult format in which no _source, _type etc - cf #159
This commit is contained in:
Rufus Pollock 2012-06-23 22:29:51 +01:00
parent dd45991321
commit 31980857a9
4 changed files with 132 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

200
src/backend/elasticsearch.js
View File

@ -3,12 +3,12 @@ this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
(function($, my) {
my.fetch = function(dataset) {
};
my.__type__ = 'elasticsearch';
// ## ElasticSearch Wrapper
//
// Connecting to [ElasticSearch](http://www.elasticsearch.org/) endpoints.
// A simple JS wrapper around an [ElasticSearch](http://www.elasticsearch.org/) endpoints.
//
// @param {String} endpoint: url for ElasticSearch type/table, e.g. for ES running
// on http://localhost:9200 with index twitter and type tweet it would be:
//
@ -33,7 +33,7 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
// @return promise compatible deferred object.
this.mapping = function() {
var schemaUrl = self.endpoint + '/_mapping';
var jqxhr = recline.Backend.makeRequest({
var jqxhr = makeRequest({
url: schemaUrl,
dataType: this.options.dataType
});
@ -47,7 +47,7 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
// @return promise compatible deferred object.
this.get = function(id) {
var base = this.endpoint + '/' + id;
return recline.Backend.makeRequest({
return makeRequest({
url: base,
dataType: 'json'
});
@ -65,7 +65,7 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
if (doc.id) {
url += '/' + doc.id;
}
return recline.Backend.makeRequest({
return makeRequest({
url: url,
type: 'POST',
data: data,
@ -82,7 +82,7 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
this.delete = function(id) {
url = this.endpoint;
url += '/' + id;
return recline.Backend.makeRequest({
return makeRequest({
url: url,
type: 'DELETE',
dataType: 'json'
@ -143,7 +143,7 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
esQuery.query = queryNormalized;
var data = {source: JSON.stringify(esQuery)};
var url = this.endpoint + '/_search';
var jqxhr = recline.Backend.makeRequest({
var jqxhr = makeRequest({
url: url,
data: data,
dataType: this.options.dataType
@ -152,94 +152,110 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
}
};
// ## ElasticSearch Backbone Backend
//
// Backbone connector for an ES backend.
//
// Usage:
//
// var backend = new recline.Backend.ElasticSearch(options);
//
// `options` are passed through to Wrapper
my.Backbone = function(options) {
var self = this;
var esOptions = options;
this.__type__ = 'elasticsearch';
// ### sync
//
// Backbone sync implementation for this backend.
//
// URL of ElasticSearch endpoint to use must be specified on the dataset
// (and on a Record via its dataset attribute) by the dataset having a
// url attribute.
this.sync = function(method, model, options) {
if (model.__type__ == 'Dataset') {
var endpoint = model.get('url');
} else {
var endpoint = model.dataset.get('url');
}
var es = new my.Wrapper(endpoint, esOptions);
if (method === "read") {
if (model.__type__ == 'Dataset') {
var dfd = $.Deferred();
es.mapping().done(function(schema) {
// only one top level key in ES = the type so we can ignore it
var key = _.keys(schema)[0];
var fieldData = _.map(schema[key].properties, function(dict, fieldName) {
dict.id = fieldName;
return dict;
});
model.fields.reset(fieldData);
dfd.resolve(model);
})
.fail(function(arguments) {
dfd.reject(arguments);
});
return dfd.promise();
} else if (model.__type__ == 'Record') {
return es.get(model.dataset.id);
}
} else if (method === 'update') {
if (model.__type__ == 'Record') {
return es.upsert(model.toJSON());
}
} else if (method === 'delete') {
if (model.__type__ == 'Record') {
return es.delete(model.id);
}
}
};
// ## Recline Connectors
//
// Requires URL of ElasticSearch endpoint to be specified on the dataset
// via the url attribute.
// ### query
//
// query the ES backend
this.query = function(model, queryObj) {
var dfd = $.Deferred();
var url = model.get('url');
var es = new my.Wrapper(url, esOptions);
var jqxhr = es.query(queryObj);
// TODO: fail case
jqxhr.done(function(results) {
_.each(results.hits.hits, function(hit) {
if (!('id' in hit._source) && hit._id) {
hit._source.id = hit._id;
}
});
if (results.facets) {
results.hits.facets = results.facets;
}
dfd.resolve(results.hits);
}).fail(function(errorObj) {
var out = {
title: 'Failed: ' + errorObj.status + ' code',
message: errorObj.responseText
};
dfd.reject(out);
// ES options which are passed through to `options` on Wrapper (see Wrapper for details)
my.esOptions = {};
// ### fetch
my.fetch = function(dataset) {
var es = new my.Wrapper(dataset.url, my.esOptions);
var dfd = $.Deferred();
es.mapping().done(function(schema) {
// only one top level key in ES = the type so we can ignore it
var key = _.keys(schema)[0];
var fieldData = _.map(schema[key].properties, function(dict, fieldName) {
dict.id = fieldName;
return dict;
});
return dfd.promise();
};
dfd.resolve({
fields: fieldData
});
})
.fail(function(arguments) {
dfd.reject(arguments);
});
return dfd.promise();
};
// ### save
my.save = function(changes, dataset) {
var es = new my.Wrapper(dataset.url, my.esOptions);
if (changes.creates.length + changes.updates.length + changes.deletes.length > 1) {
var dfd = $.Deferred();
msg = 'Saving more than one item at a time not yet supported';
alert(msg);
dfd.reject(msg);
return dfd.promise();
}
if (changes.creates.length > 0) {
return es.upsert(changes.creates[0]);
}
else if (changes.updates.length >0) {
return es.upsert(changes.updates[0]);
} else if (changes.deletes.length > 0) {
return es.delete(changes.deletes[0].id);
}
};
// ### query
my.query = function(queryObj, dataset) {
var dfd = $.Deferred();
var es = new my.Wrapper(dataset.url, my.esOptions);
var jqxhr = es.query(queryObj);
jqxhr.done(function(results) {
var out = {
total: results.hits.total,
};
out.hits = _.map(results.hits.hits, function(hit) {
if (!('id' in hit._source) && hit._id) {
hit._source.id = hit._id;
}
return hit._source;
});
if (results.facets) {
out.facets = results.facets;
}
dfd.resolve(out);
}).fail(function(errorObj) {
var out = {
title: 'Failed: ' + errorObj.status + ' code',
message: errorObj.responseText
};
dfd.reject(out);
});
return dfd.promise();
};
// ### makeRequest
//
// Just $.ajax but in any headers in the 'headers' attribute of this
// Backend instance. Example:
//
// <pre>
// var jqxhr = this._makeRequest({
// url: the-url
// });
// </pre>
var makeRequest = function(data, headers) {
var extras = {};
if (headers) {
extras = {
beforeSend: function(req) {
_.each(headers, function(value, key) {
req.setRequestHeader(key, value);
});
}
};
}
var data = _.extend(extras, data);
return $.ajax(data);
};
}(jQuery, this.recline.Backend.ElasticSearch));

2
src/backend/memory.js
View File

@ -54,7 +54,7 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
_.each(changes.deletes, function(record) {
self.delete(record);
});
dfd.resolve(this);
dfd.resolve();
return dfd.promise();
},

11
src/model.js
View File

@ -79,7 +79,11 @@ my.Dataset = Backbone.Model.extend({
var dfd = $.Deferred();
// TODO: fail case;
if (this.backend !== recline.Backend.Memory) {
this.backend.fetch(this.toJSON()).then(handleResults)
this.backend.fetch(this.toJSON())
.done(handleResults)
.fail(function(arguments) {
dfd.reject(arguments);
});
} else {
// special case where we have been given data directly
handleResults({
@ -100,14 +104,15 @@ my.Dataset = Backbone.Model.extend({
self.fields.reset(results.fields);
}
// TODO: parsing the processing of fields
dfd.resolve(this);
dfd.resolve(self);
}
return dfd.promise();
},
save: function() {
var self = this;
return this._store.save(this._changes, this);
// TODO: need to reset the changes ...
return this._store.save(this._changes, this.toJSON());
},
// ### query

31
test/backend/elasticsearch.test.js
View File

@ -246,14 +246,13 @@ test("write", function() {
// ==================================================
module("Backend ElasticSearch - Backbone");
module("Backend ElasticSearch - Recline");
test("query", function() {
var backend = new recline.Backend.ElasticSearch.Backbone();
var dataset = new recline.Model.Dataset({
url: 'https://localhost:9200/my-es-db/my-es-type'
},
backend
'elasticsearch'
);
var stub = sinon.stub($, 'ajax', function(options) {
@ -292,11 +291,10 @@ test("query", function() {
});
test("write", function() {
var backend = new recline.Backend.ElasticSearch.Backbone();
var dataset = new recline.Model.Dataset({
url: 'http://localhost:9200/recline-test/es-write'
},
backend
'elasticsearch'
);
stop();
@ -306,10 +304,10 @@ test("write", function() {
id: id,
title: 'my title'
});
rec.backend = backend;
rec.dataset = dataset;
dataset.currentRecords.add(rec);
var jqxhr = rec.save();
// have to do this explicitly as we not really supporting adding new items atm
dataset._changes.creates.push(rec.toJSON());
var jqxhr = dataset.save();
jqxhr.done(function(data) {
ok(data.ok);
equal(data._id, id);
@ -318,28 +316,29 @@ test("write", function() {
// update
rec.set({title: 'new title'});
var jqxhr = rec.save();
// again set up by hand ...
dataset._changes.creates = [];
dataset._changes.updates.push(rec.toJSON());
var jqxhr = dataset.save();
jqxhr.done(function(data) {
equal(data._version, 2);
// delete
var jqxhr = rec.destroy();
dataset._changes.updates = 0;
dataset._changes.deletes.push(rec.toJSON());
var jqxhr = dataset.save();
jqxhr.done(function(data) {
ok(data.ok);
rec = null;
// try to get ...
var oldrec = new recline.Model.Record({id: id});
equal(oldrec.get('title'), null);
oldrec.dataset = dataset;
oldrec.backend = backend;
var jqxhr = oldrec.fetch();
var es = new recline.Backend.ElasticSearch.Wrapper(dataset.get('url'));
var jqxhr = es.get(id);
jqxhr.done(function(data) {
// should not be here
ok(false, 'Should have got 404');
}).error(function(error) {
equal(error.status, 404);
equal(typeof oldrec.get('title'), 'undefined');
start();
});
});