knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#128,backend/elasticsearch][m]: rework elasticsearch model to new cleaner setup.

Browse Source
This commit is contained in:
Rufus Pollock 2012-05-26 15:53:59 +01:00
parent 23b32dff1c
commit 1bc8c77098
3 changed files with 273 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/backend/base.js
View File

@ -155,5 +155,31 @@ this.recline.Backend = this.recline.Backend || {};
}
});
// ### makeRequest
//
// Just $.ajax but in any headers in the 'headers' attribute of this
// Backend instance. Example:
//
// <pre>
// var jqxhr = this._makeRequest({
// url: the-url
// });
// </pre>
my.makeRequest = function(data, headers) {
var extras = {};
if (headers) {
extras = {
beforeSend: function(req) {
_.each(headers, function(value, key) {
req.setRequestHeader(key, value);
});
}
};
}
var data = _.extend(extras, data);
return $.ajax(data);
};
}(jQuery, this.recline.Backend));

246
src/backend/elasticsearch.js
View File

@ -1,80 +1,44 @@
this.recline = this.recline || {};
this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
(function($, my) {
// ## ElasticSearch Backend
// ## ElasticSearch Wrapper
//
// Connecting to [ElasticSearch](http://www.elasticsearch.org/).
//
// Usage:
//
// <pre>
// var backend = new recline.Backend.ElasticSearch({
// // optional as can also be provided by Dataset/Document
// url: {url to ElasticSearch endpoint i.e. ES 'type/table' url - more info below}
// // optional
// headers: {dict of headers to add to each request}
// });
//
// @param {String} url: url for ElasticSearch type/table, e.g. for ES running
// Connecting to [ElasticSearch](http://www.elasticsearch.org/) endpoints.
// @param {String} endpoint: url for ElasticSearch type/table, e.g. for ES running
// on localhost:9200 with index // twitter and type tweet it would be:
//
// <pre>http://localhost:9200/twitter/tweet</pre>
// @param {Object} options: set of options such as:
//
// This url is optional since the ES endpoint url may be specified on the the
// dataset (and on a Document by the document having a dataset attribute) by
// having one of the following (see also `_getESUrl` function):
//
// <pre>
// elasticsearch_url
// webstore_url
// url
// </pre>
my.ElasticSearch = my.Base.extend({
__type__: 'elasticsearch',
readonly: false,
sync: function(method, model, options) {
var self = this;
if (method === "read") {
if (model.__type__ == 'Dataset') {
var schemaUrl = self._getESUrl(model) + '/_mapping';
var jqxhr = this._makeRequest({
url: schemaUrl,
dataType: 'jsonp'
});
var dfd = $.Deferred();
this._wrapInTimeout(jqxhr).done(function(schema) {
// only one top level key in ES = the type so we can ignore it
var key = _.keys(schema)[0];
var fieldData = _.map(schema[key].properties, function(dict, fieldName) {
dict.id = fieldName;
return dict;
});
model.fields.reset(fieldData);
dfd.resolve(model, jqxhr);
})
.fail(function(arguments) {
dfd.reject(arguments);
});
return dfd.promise();
} else if (model.__type__ == 'Document') {
var base = this._getESUrl(model.dataset) + '/' + model.id;
return this._makeRequest({
url: base,
dataType: 'json'
});
}
} else if (method === 'update') {
if (model.__type__ == 'Document') {
return this.upsert(model.toJSON(), this._getESUrl(model.dataset));
}
} else if (method === 'delete') {
if (model.__type__ == 'Document') {
var url = this._getESUrl(model.dataset);
return this.delete(model.id, url);
}
}
},
// * headers - {dict of headers to add to each request}
my.Wrapper = function(endpoint, options) {
var self = this;
this.endpoint = endpoint;
this.options = _.extend({
dataType: 'json'
},
options);
this.mapping = function() {
var schemaUrl = self.endpoint + '/_mapping';
var jqxhr = recline.Backend.makeRequest({
url: schemaUrl,
dataType: this.options.dataType
});
return jqxhr;
};
this.get = function(id, error, success) {
var base = this.endpoint + '/' + id;
return recline.Backend.makeRequest({
url: base,
dataType: 'json',
error: error,
success: success
});
}
// ### upsert
//
@ -83,19 +47,21 @@ this.recline.Backend = this.recline.Backend || {};
// @param {Object} doc an object to insert to the index.
// @param {string} url (optional) url for ElasticSearch endpoint (if not
// defined called this._getESUrl()
upsert: function(doc, url) {
this.upsert = function(doc, error, success) {
var data = JSON.stringify(doc);
url = url ? url : this._getESUrl();
url = this.endpoint;
if (doc.id) {
url += '/' + doc.id;
}
return this._makeRequest({
return recline.Backend.makeRequest({
url: url,
type: 'POST',
data: data,
dataType: 'json'
dataType: 'json',
error: error,
success: success
});
},
};
// ### delete
//
@ -104,32 +70,18 @@ this.recline.Backend = this.recline.Backend || {};
// @param {Object} id id of object to delete
// @param {string} url (optional) url for ElasticSearch endpoint (if not
// provided called this._getESUrl()
delete: function(id, url) {
url = url ? url : this._getESUrl();
this.delete = function(id, error, success) {
url = this.endpoint;
url += '/' + id;
return this._makeRequest({
return recline.Backend.makeRequest({
url: url,
type: 'DELETE',
dataType: 'json'
});
},
};
// ### _getESUrl
//
// get url to ElasticSearch endpoint (see above)
_getESUrl: function(dataset) {
if (dataset) {
var out = dataset.get('elasticsearch_url');
if (out) return out;
out = dataset.get('webstore_url');
if (out) return out;
out = dataset.get('url');
return out;
}
return this.get('url');
},
_normalizeQuery: function(queryObj) {
var out = queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj);
this._normalizeQuery = function(queryObj) {
var out = queryObj && queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj);
if (out.q !== undefined && out.q.trim() === '') {
delete out.q;
}
@ -159,17 +111,107 @@ this.recline.Backend = this.recline.Backend || {};
delete out.filters;
}
return out;
},
query: function(model, queryObj) {
};
this.query = function(queryObj) {
var queryNormalized = this._normalizeQuery(queryObj);
var data = {source: JSON.stringify(queryNormalized)};
var base = this._getESUrl(model);
var jqxhr = this._makeRequest({
url: base + '/_search',
var url = this.endpoint + '/_search';
var jqxhr = recline.Backend.makeRequest({
url: url,
data: data,
dataType: 'jsonp'
dataType: this.options.dataType
});
return jqxhr;
}
};
// ## ElasticSearch Backbone Backend
//
// Backbone connector for an ES backend.
//
// Usage:
//
// var backend = new recline.Backend.ElasticSearch(options);
//
// `options` are passed through to Wrapper
my.Backbone = function(options) {
var self = this;
var esOptions = options;
this.__type__ = 'elasticsearch';
// ### sync
//
// Backbone sync implementation for this backend.
//
// URL of ElasticSearch endpoint to use must be specified on the
// dataset (and on a Document by the document having a dataset
// attribute) by the dataset having one of the following data
// attributes (see also `_getESUrl` function):
//
// <pre>
// elasticsearch_url
// webstore_url
// url
// </pre>
this.sync = function(method, model, options) {
if (model.__type__ == 'Dataset') {
var endpoint = self._getESUrl(model);
} else {
var endpoint = self._getESUrl(model.dataset);
}
var es = new my.Wrapper(endpoint, esOptions);
if (method === "read") {
if (model.__type__ == 'Dataset') {
var dfd = $.Deferred();
es.mapping().done(function(schema) {
// only one top level key in ES = the type so we can ignore it
var key = _.keys(schema)[0];
var fieldData = _.map(schema[key].properties, function(dict, fieldName) {
dict.id = fieldName;
return dict;
});
model.fields.reset(fieldData);
dfd.resolve(model);
})
.fail(function(arguments) {
dfd.reject(arguments);
});
return dfd.promise();
} else if (model.__type__ == 'Document') {
return es.get(model.dataset.id);
}
} else if (method === 'update') {
if (model.__type__ == 'Document') {
return es.upsert(model.toJSON());
}
} else if (method === 'delete') {
if (model.__type__ == 'Document') {
return es.delete(model.id);
}
}
};
// ### _getESUrl
//
// get url to ElasticSearch endpoint (see above)
this._getESUrl = function(dataset) {
if (dataset) {
var out = dataset.get('elasticsearch_url');
if (out) return out;
out = dataset.get('webstore_url');
if (out) return out;
out = dataset.get('url');
return out;
}
return this.get('url');
};
this.query = function(model, queryObj) {
var dfd = $.Deferred();
var url = this._getESUrl(model);
var es = new my.Wrapper(url, esOptions);
var jqxhr = es.query(queryObj);
// TODO: fail case
jqxhr.done(function(results) {
_.each(results.hits.hits, function(hit) {
@ -183,8 +225,8 @@ this.recline.Backend = this.recline.Backend || {};
dfd.resolve(results.hits);
});
return dfd.promise();
}
});
};
};
}(jQuery, this.recline.Backend));
}(jQuery, this.recline.Backend.ElasticSearch));

112
test/backend.elasticsearch.test.js
View File

@ -1,15 +1,18 @@
(function ($) {
module("Backend ElasticSearch");
module("Backend ElasticSearch - Wrapper");
test("queryNormalize", function() {
var backend = new recline.Backend.ElasticSearch.Wrapper();
var in_ = new recline.Model.Query();
var out = backend._normalizeQuery(in_);
equal(out.size, 100);
test("ElasticSearch queryNormalize", function() {
var backend = new recline.Backend.ElasticSearch();
var in_ = new recline.Model.Query();
in_.set({q: ''});
var out = backend._normalizeQuery(in_);
equal(out.q, undefined);
deepEqual(out.query.match_all, {});
var backend = new recline.Backend.ElasticSearch();
var in_ = new recline.Model.Query().toJSON();
in_.q = '';
var out = backend._normalizeQuery(in_);
@ -107,8 +110,99 @@ var sample_data = {
"took": 2
};
test("ElasticSearch query", function() {
var backend = new recline.Backend.ElasticSearch();
test("query", function() {
var backend = new recline.Backend.ElasticSearch.Wrapper('https://localhost:9200/my-es-db/my-es-type');
var stub = sinon.stub($, 'ajax', function(options) {
if (options.url.indexOf('_mapping') != -1) {
return {
done: function(callback) {
callback(mapping_data);
return this;
},
fail: function() {
return this;
}
}
} else {
return {
done: function(callback) {
callback(sample_data);
},
fail: function() {
}
}
}
});
backend.mapping().done(function(data) {
var fields = _.keys(data.note.properties);
deepEqual(['_created', '_last_modified', 'end', 'owner', 'start', 'title'], fields);
});
backend.query().done(function(queryResult) {
equal(3, queryResult.hits.total);
equal(3, queryResult.hits.hits.length);
equal('Note 1', queryResult.hits.hits[0]._source['title']);
start();
});
$.ajax.restore();
});
test("write", function() {
var url = 'http://localhost:9200/recline-test/es-write';
var backend = new recline.Backend.ElasticSearch.Wrapper(url);
stop();
var id = parseInt(Math.random()*100000000).toString();
var doc = {
id: id,
title: 'my title'
};
var jqxhr = backend.upsert(doc);
jqxhr.done(function(data) {
ok(data.ok);
equal(data._id, id);
equal(data._type, 'es-write');
equal(data._version, 1);
// update
doc.title = 'new title';
var jqxhr = backend.upsert(doc);
jqxhr.done(function(data) {
equal(data._version, 2);
// delete
var jqxhr = backend.delete(doc.id);
jqxhr.done(function(data) {
ok(data.ok);
doc = null;
// try to get ...
var jqxhr = backend.get(id);
jqxhr.done(function(data) {
// should not be here
ok(false, 'Should have got 404');
}).error(function(error) {
equal(error.status, 404);
start();
});
});
});
}).fail(function(error) {
console.log(error);
ok(false, 'Basic request failed - is ElasticSearch running locally on port 9200 (required for this test!)');
start();
});
});
// ==================================================
module("Backend ElasticSearch - Backbone");
test("query", function() {
var backend = new recline.Backend.ElasticSearch.Backbone();
var dataset = new recline.Model.Dataset({
url: 'https://localhost:9200/my-es-db/my-es-type'
},
@ -137,7 +231,7 @@ test("ElasticSearch query", function() {
}
});
dataset.fetch().then(function(dataset) {
dataset.fetch().done(function(dataset) {
deepEqual(['_created', '_last_modified', 'end', 'owner', 'start', 'title'], _.pluck(dataset.fields.toJSON(), 'id'));
dataset.query().then(function(docList) {
equal(3, dataset.docCount);
@ -149,8 +243,8 @@ test("ElasticSearch query", function() {
$.ajax.restore();
});
test("ElasticSearch write", function() {
var backend = new recline.Backend.ElasticSearch();
test("write", function() {
var backend = new recline.Backend.ElasticSearch.Backbone();
var dataset = new recline.Model.Dataset({
url: 'http://localhost:9200/recline-test/es-write'
},