diff --git a/_includes/backend-list.html b/_includes/backend-list.html index 88e5f38d..315f856f 100644 --- a/_includes/backend-list.html +++ b/_includes/backend-list.html @@ -2,7 +2,7 @@
http://localhost:9200/twitter/tweet- // - // @param {Object} options: set of options such as: - // - // * headers - {dict of headers to add to each request} - // * dataType: dataType for AJAx requests e.g. set to jsonp to make jsonp requests (default is json requests) - my.Wrapper = function(endpoint, options) { - var self = this; - this.endpoint = endpoint; - this.options = _.extend({ - dataType: 'json' - }, - options); - - // ### mapping - // - // Get ES mapping for this type/table - // - // @return promise compatible deferred object. - this.mapping = function() { - var schemaUrl = self.endpoint + '/_mapping'; - var jqxhr = makeRequest({ - url: schemaUrl, - dataType: this.options.dataType - }); - return jqxhr; - }; - - // ### get - // - // Get record corresponding to specified id - // - // @return promise compatible deferred object. - this.get = function(id) { - var base = this.endpoint + '/' + id; - return makeRequest({ - url: base, - dataType: 'json' - }); - }; - - // ### upsert - // - // create / update a record to ElasticSearch backend - // - // @param {Object} doc an object to insert to the index. - // @return deferred supporting promise API - this.upsert = function(doc) { - var data = JSON.stringify(doc); - url = this.endpoint; - if (doc.id) { - url += '/' + doc.id; - } - return makeRequest({ - url: url, - type: 'POST', - data: data, - dataType: 'json' - }); - }; - - // ### delete - // - // Delete a record from the ElasticSearch backend. - // - // @param {Object} id id of object to delete - // @return deferred supporting promise API - this.remove = function(id) { - url = this.endpoint; - url += '/' + id; - return makeRequest({ - url: url, - type: 'DELETE', - dataType: 'json' - }); - }; - - this._normalizeQuery = function(queryObj) { - var self = this; - var queryInfo = (queryObj && queryObj.toJSON) ? queryObj.toJSON() : _.extend({}, queryObj); - var out = { - constant_score: { - query: {} - } - }; - if (!queryInfo.q) { - out.constant_score.query = { - match_all: {} - }; - } else { - out.constant_score.query = { - query_string: { - query: queryInfo.q - } - }; - } - if (queryInfo.filters && queryInfo.filters.length) { - out.constant_score.filter = { - and: [] - }; - _.each(queryInfo.filters, function(filter) { - out.constant_score.filter.and.push(self._convertFilter(filter)); - }); - } - return out; - }, - - // convert from Recline sort structure to ES form - // http://www.elasticsearch.org/guide/reference/api/search/sort.html - this._normalizeSort = function(sort) { - var out = _.map(sort, function(sortObj) { - var _tmp = {}; - var _tmp2 = _.clone(sortObj); - delete _tmp2['field']; - _tmp[sortObj.field] = _tmp2; - return _tmp; - }); - return out; - }, - - this._convertFilter = function(filter) { - var out = {}; - out[filter.type] = {}; - if (filter.type === 'term') { - out.term[filter.field] = filter.term.toLowerCase(); - } else if (filter.type === 'geo_distance') { - out.geo_distance[filter.field] = filter.point; - out.geo_distance.distance = filter.distance; - out.geo_distance.unit = filter.unit; - } - return out; - }, - - // ### query - // - // @return deferred supporting promise API - this.query = function(queryObj) { - var esQuery = (queryObj && queryObj.toJSON) ? queryObj.toJSON() : _.extend({}, queryObj); - esQuery.query = this._normalizeQuery(queryObj); - delete esQuery.q; - delete esQuery.filters; - if (esQuery.sort && esQuery.sort.length > 0) { - esQuery.sort = this._normalizeSort(esQuery.sort); - } - var data = {source: JSON.stringify(esQuery)}; - var url = this.endpoint + '/_search'; - var jqxhr = makeRequest({ - url: url, - data: data, - dataType: this.options.dataType - }); - return jqxhr; - }; - }; - - - // ## Recline Connectors - // - // Requires URL of ElasticSearch endpoint to be specified on the dataset - // via the url attribute. - - // ES options which are passed through to `options` on Wrapper (see Wrapper for details) - my.esOptions = {}; - - // ### fetch - my.fetch = function(dataset) { - var es = new my.Wrapper(dataset.url, my.esOptions); - var dfd = new Deferred(); - es.mapping().done(function(schema) { - - if (!schema){ - dfd.reject({'message':'Elastic Search did not return a mapping'}); - return; - } - - // only one top level key in ES = the type so we can ignore it - var key = _.keys(schema)[0]; - var fieldData = _.map(schema[key].properties, function(dict, fieldName) { - dict.id = fieldName; - return dict; - }); - dfd.resolve({ - fields: fieldData - }); - }) - .fail(function(args) { - dfd.reject(args); - }); - return dfd.promise(); - }; - - // ### save - my.save = function(changes, dataset) { - var es = new my.Wrapper(dataset.url, my.esOptions); - if (changes.creates.length + changes.updates.length + changes.deletes.length > 1) { - var dfd = new Deferred(); - msg = 'Saving more than one item at a time not yet supported'; - alert(msg); - dfd.reject(msg); - return dfd.promise(); - } - if (changes.creates.length > 0) { - return es.upsert(changes.creates[0]); - } - else if (changes.updates.length >0) { - return es.upsert(changes.updates[0]); - } else if (changes.deletes.length > 0) { - return es.remove(changes.deletes[0].id); - } - }; - - // ### query - my.query = function(queryObj, dataset) { - var dfd = new Deferred(); - var es = new my.Wrapper(dataset.url, my.esOptions); - var jqxhr = es.query(queryObj); - jqxhr.done(function(results) { - var out = { - total: results.hits.total - }; - out.hits = _.map(results.hits.hits, function(hit) { - if (!('id' in hit._source) && hit._id) { - hit._source.id = hit._id; - } - return hit._source; - }); - if (results.facets) { - out.facets = results.facets; - } - dfd.resolve(out); - }).fail(function(errorObj) { - var out = { - title: 'Failed: ' + errorObj.status + ' code', - message: errorObj.responseText - }; - dfd.reject(out); - }); - return dfd.promise(); - }; - - -// ### makeRequest -// -// Just $.ajax but in any headers in the 'headers' attribute of this -// Backend instance. Example: -// -//
-// var jqxhr = this._makeRequest({
-// url: the-url
-// });
-//
-var makeRequest = function(data, headers) {
- var extras = {};
- if (headers) {
- extras = {
- beforeSend: function(req) {
- _.each(headers, function(value, key) {
- req.setRequestHeader(key, value);
- });
- }
- };
- }
- var data = _.extend(extras, data);
- return $.ajax(data);
-};
-
-}(jQuery, this.recline.Backend.ElasticSearch));
-
this.recline = this.recline || {};
this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
diff --git a/docs/tutorial-backends.markdown b/docs/tutorial-backends.markdown
index 2f8c3594..032452ba 100644
--- a/docs/tutorial-backends.markdown
+++ b/docs/tutorial-backends.markdown
@@ -111,21 +111,12 @@ a bespoke chooser and a Kartograph (svg-only) map.
-## Loading Data from ElasticSearch and the DataHub
+## Loading Data from ElasticSearch
-Recline supports ElasticSearch as a full read/write/query backend. It also means that Recline can load data from the [DataHub's](http://datahub.io/) data API as that is ElasticSearch compatible. Here's an example, using [this dataset about Rendition flights](http://datahub.io/dataset/rendition-on-record/ac5a28ea-eb52-4b0a-a399-5dcc1becf9d9') on the DataHub:
+Recline supports ElasticSearch as a full read/write/query backend via the
+[ElasticSearch.js library][esjs]. See the library for examples.
-{% highlight javascript %}
-{% include example-backends-elasticsearch.js %}
-{% endhighlight %}
-
-### Result
-
-http://localhost:9200/twitter/tweet- // - // @param {Object} options: set of options such as: - // - // * headers - {dict of headers to add to each request} - // * dataType: dataType for AJAx requests e.g. set to jsonp to make jsonp requests (default is json requests) - my.Wrapper = function(endpoint, options) { - var self = this; - this.endpoint = endpoint; - this.options = _.extend({ - dataType: 'json' - }, - options); - - // ### mapping - // - // Get ES mapping for this type/table - // - // @return promise compatible deferred object. - this.mapping = function() { - var schemaUrl = self.endpoint + '/_mapping'; - var jqxhr = makeRequest({ - url: schemaUrl, - dataType: this.options.dataType - }); - return jqxhr; - }; - - // ### get - // - // Get record corresponding to specified id - // - // @return promise compatible deferred object. - this.get = function(id) { - var base = this.endpoint + '/' + id; - return makeRequest({ - url: base, - dataType: 'json' - }); - }; - - // ### upsert - // - // create / update a record to ElasticSearch backend - // - // @param {Object} doc an object to insert to the index. - // @return deferred supporting promise API - this.upsert = function(doc) { - var data = JSON.stringify(doc); - url = this.endpoint; - if (doc.id) { - url += '/' + doc.id; - } - return makeRequest({ - url: url, - type: 'POST', - data: data, - dataType: 'json' - }); - }; - - // ### delete - // - // Delete a record from the ElasticSearch backend. - // - // @param {Object} id id of object to delete - // @return deferred supporting promise API - this.remove = function(id) { - url = this.endpoint; - url += '/' + id; - return makeRequest({ - url: url, - type: 'DELETE', - dataType: 'json' - }); - }; - - this._normalizeQuery = function(queryObj) { - var self = this; - var queryInfo = (queryObj && queryObj.toJSON) ? queryObj.toJSON() : _.extend({}, queryObj); - var out = { - constant_score: { - query: {} - } - }; - if (!queryInfo.q) { - out.constant_score.query = { - match_all: {} - }; - } else { - out.constant_score.query = { - query_string: { - query: queryInfo.q - } - }; - } - if (queryInfo.filters && queryInfo.filters.length) { - out.constant_score.filter = { - and: [] - }; - _.each(queryInfo.filters, function(filter) { - out.constant_score.filter.and.push(self._convertFilter(filter)); - }); - } - return out; - }, - - // convert from Recline sort structure to ES form - // http://www.elasticsearch.org/guide/reference/api/search/sort.html - this._normalizeSort = function(sort) { - var out = _.map(sort, function(sortObj) { - var _tmp = {}; - var _tmp2 = _.clone(sortObj); - delete _tmp2['field']; - _tmp[sortObj.field] = _tmp2; - return _tmp; - }); - return out; - }, - - this._convertFilter = function(filter) { - var out = {}; - out[filter.type] = {}; - if (filter.type === 'term') { - out.term[filter.field] = filter.term.toLowerCase(); - } else if (filter.type === 'geo_distance') { - out.geo_distance[filter.field] = filter.point; - out.geo_distance.distance = filter.distance; - out.geo_distance.unit = filter.unit; - } - return out; - }, - - // ### query - // - // @return deferred supporting promise API - this.query = function(queryObj) { - var esQuery = (queryObj && queryObj.toJSON) ? queryObj.toJSON() : _.extend({}, queryObj); - esQuery.query = this._normalizeQuery(queryObj); - delete esQuery.q; - delete esQuery.filters; - if (esQuery.sort && esQuery.sort.length > 0) { - esQuery.sort = this._normalizeSort(esQuery.sort); - } - var data = {source: JSON.stringify(esQuery)}; - var url = this.endpoint + '/_search'; - var jqxhr = makeRequest({ - url: url, - data: data, - dataType: this.options.dataType - }); - return jqxhr; - }; - }; - - - // ## Recline Connectors - // - // Requires URL of ElasticSearch endpoint to be specified on the dataset - // via the url attribute. - - // ES options which are passed through to `options` on Wrapper (see Wrapper for details) - my.esOptions = {}; - - // ### fetch - my.fetch = function(dataset) { - var es = new my.Wrapper(dataset.url, my.esOptions); - var dfd = new Deferred(); - es.mapping().done(function(schema) { - - if (!schema){ - dfd.reject({'message':'Elastic Search did not return a mapping'}); - return; - } - - // only one top level key in ES = the type so we can ignore it - var key = _.keys(schema)[0]; - var fieldData = _.map(schema[key].properties, function(dict, fieldName) { - dict.id = fieldName; - return dict; - }); - dfd.resolve({ - fields: fieldData - }); - }) - .fail(function(args) { - dfd.reject(args); - }); - return dfd.promise(); - }; - - // ### save - my.save = function(changes, dataset) { - var es = new my.Wrapper(dataset.url, my.esOptions); - if (changes.creates.length + changes.updates.length + changes.deletes.length > 1) { - var dfd = new Deferred(); - msg = 'Saving more than one item at a time not yet supported'; - alert(msg); - dfd.reject(msg); - return dfd.promise(); - } - if (changes.creates.length > 0) { - return es.upsert(changes.creates[0]); - } - else if (changes.updates.length >0) { - return es.upsert(changes.updates[0]); - } else if (changes.deletes.length > 0) { - return es.remove(changes.deletes[0].id); - } - }; - - // ### query - my.query = function(queryObj, dataset) { - var dfd = new Deferred(); - var es = new my.Wrapper(dataset.url, my.esOptions); - var jqxhr = es.query(queryObj); - jqxhr.done(function(results) { - var out = { - total: results.hits.total - }; - out.hits = _.map(results.hits.hits, function(hit) { - if (!('id' in hit._source) && hit._id) { - hit._source.id = hit._id; - } - return hit._source; - }); - if (results.facets) { - out.facets = results.facets; - } - dfd.resolve(out); - }).fail(function(errorObj) { - var out = { - title: 'Failed: ' + errorObj.status + ' code', - message: errorObj.responseText - }; - dfd.reject(out); - }); - return dfd.promise(); - }; - - -// ### makeRequest -// -// Just $.ajax but in any headers in the 'headers' attribute of this -// Backend instance. Example: -// -//
-// var jqxhr = this._makeRequest({
-// url: the-url
-// });
-//
-var makeRequest = function(data, headers) {
- var extras = {};
- if (headers) {
- extras = {
- beforeSend: function(req) {
- _.each(headers, function(value, key) {
- req.setRequestHeader(key, value);
- });
- }
- };
- }
- var data = _.extend(extras, data);
- return $.ajax(data);
-};
-
-}(jQuery, this.recline.Backend.ElasticSearch));
-
diff --git a/test/backend.elasticsearch.test.js b/test/backend.elasticsearch.test.js
deleted file mode 100644
index c97bc744..00000000
--- a/test/backend.elasticsearch.test.js
+++ /dev/null
@@ -1,351 +0,0 @@
-(function ($) {
-module("Backend ElasticSearch - Wrapper");
-
-test("queryNormalize", function() {
- var backend = new recline.Backend.ElasticSearch.Wrapper();
-
- var in_ = new recline.Model.Query();
- var out = backend._normalizeQuery(in_);
- var exp = {
- constant_score: {
- query: {
- match_all: {}
- }
- }
- };
- deepEqual(out, exp);
-
- var in_ = new recline.Model.Query();
- in_.set({q: ''});
- var out = backend._normalizeQuery(in_);
- deepEqual(out, exp);
-
- var in_ = new recline.Model.Query();
- in_.attributes.q = 'abc';
- var out = backend._normalizeQuery(in_);
- equal(out.constant_score.query.query_string.query, 'abc');
-
- var in_ = new recline.Model.Query();
- in_.addFilter({
- type: 'term',
- field: 'xyz',
- term: 'XXX'
- });
- var out = backend._normalizeQuery(in_);
- var exp = {
- constant_score: {
- query: {
- match_all: {}
- },
- filter: {
- and: [
- {
- term: {
- xyz: 'xxx'
- }
- }
- ]
- }
- }
- };
- deepEqual(out, exp);
-
- var in_ = new recline.Model.Query();
- in_.addFilter({
- type: 'geo_distance',
- field: 'xyz'
- });
- var out = backend._normalizeQuery(in_);
- var exp = {
- constant_score: {
- query: {
- match_all: {}
- },
- filter: {
- and: [
- {
- geo_distance: {
- distance: 10,
- unit: 'km',
- 'xyz': { lon: 0, lat: 0 }
- }
- }
- ]
- }
- }
- };
- deepEqual(out, exp);
-});
-
-var mapping_data = {
- "note": {
- "properties": {
- "_created": {
- "format": "dateOptionalTime",
- "type": "date"
- },
- "_last_modified": {
- "format": "dateOptionalTime",
- "type": "date"
- },
- "end": {
- "type": "string"
- },
- "owner": {
- "type": "string"
- },
- "start": {
- "type": "string"
- },
- "title": {
- "type": "string"
- }
- }
- }
-};
-
-var sample_data = {
- "_shards": {
- "failed": 0,
- "successful": 5,
- "total": 5
- },
- "hits": {
- "hits": [
- {
- "_id": "u3rpLyuFS3yLNXrtxWkMwg",
- "_index": "hypernotes",
- "_score": 1.0,
- "_source": {
- "_created": "2012-02-24T17:53:57.286Z",
- "_last_modified": "2012-02-24T17:53:57.286Z",
- "owner": "tester",
- "title": "Note 1"
- },
- "_type": "note"
- },
- {
- "_id": "n7JMkFOHSASJCVTXgcpqkA",
- "_index": "hypernotes",
- "_score": 1.0,
- "_source": {
- "_created": "2012-02-24T17:53:57.290Z",
- "_last_modified": "2012-02-24T17:53:57.290Z",
- "owner": "tester",
- "title": "Note 3"
- },
- "_type": "note"
- },
- {
- "_id": "g7UMA55gTJijvsB3dFitzw",
- "_index": "hypernotes",
- "_score": 1.0,
- "_source": {
- "_created": "2012-02-24T17:53:57.289Z",
- "_last_modified": "2012-02-24T17:53:57.289Z",
- "owner": "tester",
- "title": "Note 2"
- },
- "_type": "note"
- }
- ],
- "max_score": 1.0,
- "total": 3
- },
- "timed_out": false,
- "took": 2
-};
-
-test("query", function() {
- var backend = new recline.Backend.ElasticSearch.Wrapper('https://localhost:9200/my-es-db/my-es-type');
-
- var stub = sinon.stub($, 'ajax', function(options) {
- if (options.url.indexOf('_mapping') != -1) {
- return {
- done: function(callback) {
- callback(mapping_data);
- return this;
- },
- fail: function() {
- return this;
- }
- }
- } else {
- return {
- done: function(callback) {
- callback(sample_data);
- return this;
- },
- fail: function() {
- }
- }
- }
- });
-
- backend.mapping().done(function(data) {
- var fields = _.keys(data.note.properties);
- deepEqual(['_created', '_last_modified', 'end', 'owner', 'start', 'title'], fields);
- });
-
- backend.query().done(function(queryResult) {
- equal(3, queryResult.hits.total);
- equal(3, queryResult.hits.hits.length);
- equal('Note 1', queryResult.hits.hits[0]._source['title']);
- });
- $.ajax.restore();
-});
-
-// DISABLED - this test requires ElasticSearch to be running locally
-// test("write", function() {
-// var url = 'http://localhost:9200/recline-test/es-write';
-// var backend = new recline.Backend.ElasticSearch.Wrapper(url);
-// stop();
-//
-// var id = parseInt(Math.random()*100000000).toString();
-// var rec = {
-// id: id,
-// title: 'my title'
-// };
-// var jqxhr = backend.upsert(rec);
-// jqxhr.done(function(data) {
-// ok(data.ok);
-// equal(data._id, id);
-// equal(data._type, 'es-write');
-// equal(data._version, 1);
-//
-// // update
-// rec.title = 'new title';
-// var jqxhr = backend.upsert(rec);
-// jqxhr.done(function(data) {
-// equal(data._version, 2);
-//
-// // delete
-// var jqxhr = backend.remove(rec.id);
-// jqxhr.done(function(data) {
-// ok(data.ok);
-// rec = null;
-//
-// // try to get ...
-// var jqxhr = backend.get(id);
-// jqxhr.done(function(data) {
-// // should not be here
-// ok(false, 'Should have got 404');
-// }).error(function(error) {
-// equal(error.status, 404);
-// start();
-// });
-// });
-// });
-// }).fail(function(error) {
-// console.log(error);
-// ok(false, 'Basic request failed - is ElasticSearch running locally on port 9200 (required for this test!)');
-// start();
-// });
-// });
-
-
-// ==================================================
-
-module("Backend ElasticSearch - Recline");
-
-test("query", function() {
- var dataset = new recline.Model.Dataset({
- url: 'https://localhost:9200/my-es-db/my-es-type',
- backend: 'elasticsearch'
- });
-
- var stub = sinon.stub($, 'ajax', function(options) {
- if (options.url.indexOf('_mapping') != -1) {
- return {
- done: function(callback) {
- callback(mapping_data);
- return this;
- },
- fail: function() {
- return this;
- }
- }
- } else {
- return {
- done: function(callback) {
- callback(sample_data);
- return this;
- },
- fail: function() {
- }
- };
- }
- });
-
- dataset.fetch().done(function(dataset) {
- deepEqual(['_created', '_last_modified', 'end', 'owner', 'start', 'title'], _.pluck(dataset.fields.toJSON(), 'id'));
- dataset.query().then(function(recList) {
- equal(3, dataset.recordCount);
- equal(3, recList.length);
- equal('Note 1', recList.models[0].get('title'));
- });
- });
- $.ajax.restore();
-});
-
-// DISABLED - this test requires ElasticSearch to be running locally
-// test("write", function() {
-// var dataset = new recline.Model.Dataset({
-// url: 'http://localhost:9200/recline-test/es-write',
-// backend: 'elasticsearch'
-// });
-//
-// stop();
-//
-// var id = parseInt(Math.random()*100000000).toString();
-// var rec = new recline.Model.Record({
-// id: id,
-// title: 'my title'
-// });
-// dataset.records.add(rec);
-// // have to do this explicitly as we not really supporting adding new items atm
-// dataset._changes.creates.push(rec.toJSON());
-// var jqxhr = dataset.save();
-// jqxhr.done(function(data) {
-// ok(data.ok);
-// equal(data._id, id);
-// equal(data._type, 'es-write');
-// equal(data._version, 1);
-//
-// // update
-// rec.set({title: 'new title'});
-// // again set up by hand ...
-// dataset._changes.creates = [];
-// dataset._changes.updates.push(rec.toJSON());
-// var jqxhr = dataset.save();
-// jqxhr.done(function(data) {
-// equal(data._version, 2);
-//
-// // delete
-// dataset._changes.updates = 0;
-// dataset._changes.deletes.push(rec.toJSON());
-// var jqxhr = dataset.save();
-// jqxhr.done(function(data) {
-// ok(data.ok);
-// rec = null;
-//
-// // try to get ...
-// var es = new recline.Backend.ElasticSearch.Wrapper(dataset.get('url'));
-// var jqxhr = es.get(id);
-// jqxhr.done(function(data) {
-// // should not be here
-// ok(false, 'Should have got 404');
-// }).error(function(error) {
-// equal(error.status, 404);
-// start();
-// });
-// });
-// });
-// }).fail(function(error) {
-// console.log(error);
-// ok(false, 'Basic request failed - is ElasticSearch running locally on port 9200 (required for this test!)');
-// start();
-// });
-// });
-
-})(this.jQuery);
diff --git a/test/index.html b/test/index.html
index ddb4aa92..1c103cac 100644
--- a/test/index.html
+++ b/test/index.html
@@ -41,14 +41,12 @@
-
-