From 1bc8c770982559c4dd6227f220f88fc699fa33f9 Mon Sep 17 00:00:00 2001
From: Rufus Pollock <rufus.pollock@okfn.org>
Date: Sat, 26 May 2012 15:53:59 +0100
Subject: [PATCH] [#128,backend/elasticsearch][m]: rework elasticsearch model
 to new cleaner setup.

---
 src/backend/base.js                |  26 +++
 src/backend/elasticsearch.js       | 246 +++++++++++++++++------------
 test/backend.elasticsearch.test.js | 112 +++++++++++--
 3 files changed, 273 insertions(+), 111 deletions(-)

diff --git a/src/backend/base.js b/src/backend/base.js
index a8dc9f8d..9c027d87 100644
--- a/src/backend/base.js
+++ b/src/backend/base.js
@@ -155,5 +155,31 @@ this.recline.Backend = this.recline.Backend || {};
     }
   });
 
+  // ### makeRequest
+  // 
+  // Just $.ajax but in any headers in the 'headers' attribute of this
+  // Backend instance. Example:
+  //
+  // <pre>
+  // var jqxhr = this._makeRequest({
+  //   url: the-url
+  // });
+  // </pre>
+  my.makeRequest = function(data, headers) {
+    var extras = {};
+    if (headers) {
+      extras = {
+        beforeSend: function(req) {
+          _.each(headers, function(value, key) {
+            req.setRequestHeader(key, value);
+          });
+        }
+      };
+    }
+    var data = _.extend(extras, data);
+    return $.ajax(data);
+  };
+
+
 }(jQuery, this.recline.Backend));
 
diff --git a/src/backend/elasticsearch.js b/src/backend/elasticsearch.js
index 546c0c8a..d035d138 100644
--- a/src/backend/elasticsearch.js
+++ b/src/backend/elasticsearch.js
@@ -1,80 +1,44 @@
 this.recline = this.recline || {};
 this.recline.Backend = this.recline.Backend || {};
+this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
 
 (function($, my) {
-  // ## ElasticSearch Backend
+  // ## ElasticSearch Wrapper
   //
-  // Connecting to [ElasticSearch](http://www.elasticsearch.org/).
-  //
-  // Usage:
-  //
-  // <pre>
-  // var backend = new recline.Backend.ElasticSearch({
-  //   // optional as can also be provided by Dataset/Document
-  //   url: {url to ElasticSearch endpoint i.e. ES 'type/table' url - more info below}
-  //   // optional
-  //   headers: {dict of headers to add to each request}
-  // });
-  //
-  // @param {String} url: url for ElasticSearch type/table, e.g. for ES running
+  // Connecting to [ElasticSearch](http://www.elasticsearch.org/) endpoints.
+  // @param {String} endpoint: url for ElasticSearch type/table, e.g. for ES running
   // on localhost:9200 with index // twitter and type tweet it would be:
   // 
   // <pre>http://localhost:9200/twitter/tweet</pre>
+  // @param {Object} options: set of options such as:
   //
-  // This url is optional since the ES endpoint url may be specified on the the
-  // dataset (and on a Document by the document having a dataset attribute) by
-  // having one of the following (see also `_getESUrl` function):
-  //
-  // <pre>
-  // elasticsearch_url
-  // webstore_url
-  // url
-  // </pre>
-  my.ElasticSearch = my.Base.extend({
-    __type__: 'elasticsearch',
-    readonly: false,
-    sync: function(method, model, options) {
-      var self = this;
-      if (method === "read") {
-        if (model.__type__ == 'Dataset') {
-          var schemaUrl = self._getESUrl(model) + '/_mapping';
-          var jqxhr = this._makeRequest({
-            url: schemaUrl,
-            dataType: 'jsonp'
-          });
-          var dfd = $.Deferred();
-          this._wrapInTimeout(jqxhr).done(function(schema) {
-            // only one top level key in ES = the type so we can ignore it
-            var key = _.keys(schema)[0];
-            var fieldData = _.map(schema[key].properties, function(dict, fieldName) {
-              dict.id = fieldName;
-              return dict;
-            });
-            model.fields.reset(fieldData);
-            dfd.resolve(model, jqxhr);
-          })
-          .fail(function(arguments) {
-            dfd.reject(arguments);
-          });
-          return dfd.promise();
-        } else if (model.__type__ == 'Document') {
-          var base = this._getESUrl(model.dataset) + '/' + model.id;
-          return this._makeRequest({
-            url: base,
-            dataType: 'json'
-          });
-        }
-      } else if (method === 'update') {
-        if (model.__type__ == 'Document') {
-          return this.upsert(model.toJSON(), this._getESUrl(model.dataset));
-        }
-      } else if (method === 'delete') {
-        if (model.__type__ == 'Document') {
-          var url = this._getESUrl(model.dataset);
-          return this.delete(model.id, url);
-        }
-      }
-    },
+  // * headers - {dict of headers to add to each request}
+  my.Wrapper = function(endpoint, options) { 
+    var self = this;
+    this.endpoint = endpoint;
+    this.options = _.extend({
+        dataType: 'json'
+      },
+      options);
+
+    this.mapping = function() {
+      var schemaUrl = self.endpoint + '/_mapping';
+      var jqxhr = recline.Backend.makeRequest({
+        url: schemaUrl,
+        dataType: this.options.dataType
+      });
+      return jqxhr;
+    };
+
+    this.get = function(id, error, success) {
+      var base = this.endpoint + '/' + id;
+      return recline.Backend.makeRequest({
+        url: base,
+        dataType: 'json',
+        error: error,
+        success: success
+      });
+    }
 
     // ### upsert
     //
@@ -83,19 +47,21 @@ this.recline.Backend = this.recline.Backend || {};
     // @param {Object} doc an object to insert to the index.
     // @param {string} url (optional) url for ElasticSearch endpoint (if not
     // defined called this._getESUrl()
-    upsert: function(doc, url) {
+    this.upsert = function(doc, error, success) {
       var data = JSON.stringify(doc);
-      url = url ? url : this._getESUrl();
+      url = this.endpoint;
       if (doc.id) {
         url += '/' + doc.id;
       }
-      return this._makeRequest({
+      return recline.Backend.makeRequest({
         url: url,
         type: 'POST',
         data: data,
-        dataType: 'json'
+        dataType: 'json',
+        error: error,
+        success: success
       });
-    },
+    };
 
     // ### delete
     //
@@ -104,32 +70,18 @@ this.recline.Backend = this.recline.Backend || {};
     // @param {Object} id id of object to delete
     // @param {string} url (optional) url for ElasticSearch endpoint (if not
     // provided called this._getESUrl()
-    delete: function(id, url) {
-      url = url ? url : this._getESUrl();
+    this.delete = function(id, error, success) {
+      url = this.endpoint;
       url += '/' + id;
-      return this._makeRequest({
+      return recline.Backend.makeRequest({
         url: url,
         type: 'DELETE',
         dataType: 'json'
       });
-    },
+    };
 
-    // ### _getESUrl
-    //
-    // get url to ElasticSearch endpoint (see above)
-    _getESUrl: function(dataset) {
-      if (dataset) {
-        var out = dataset.get('elasticsearch_url');
-        if (out) return out;
-        out = dataset.get('webstore_url');
-        if (out) return out;
-        out = dataset.get('url');
-        return out;
-      }
-      return this.get('url');
-    },
-    _normalizeQuery: function(queryObj) {
-      var out = queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj);
+    this._normalizeQuery = function(queryObj) {
+      var out = queryObj && queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj);
       if (out.q !== undefined && out.q.trim() === '') {
         delete out.q;
       }
@@ -159,17 +111,107 @@ this.recline.Backend = this.recline.Backend || {};
         delete out.filters;
       }
       return out;
-    },
-    query: function(model, queryObj) {
+    };
+
+    this.query = function(queryObj) {
       var queryNormalized = this._normalizeQuery(queryObj);
       var data = {source: JSON.stringify(queryNormalized)};
-      var base = this._getESUrl(model);
-      var jqxhr = this._makeRequest({
-        url: base + '/_search',
+      var url = this.endpoint + '/_search';
+      var jqxhr = recline.Backend.makeRequest({
+        url: url,
         data: data,
-        dataType: 'jsonp'
+        dataType: this.options.dataType
       });
+      return jqxhr;
+    }
+  };
+
+  // ## ElasticSearch Backbone Backend
+  //
+  // Backbone connector for an ES backend.
+  //
+  // Usage:
+  //
+  // var backend = new recline.Backend.ElasticSearch(options);
+  //
+  // `options` are passed through to Wrapper
+  my.Backbone = function(options) {
+    var self = this;
+    var esOptions = options;
+    this.__type__ = 'elasticsearch';
+
+    // ### sync
+    //
+    // Backbone sync implementation for this backend.
+    //
+    // URL of ElasticSearch endpoint to use must be specified on the
+    // dataset (and on a Document by the document having a dataset
+    // attribute) by the dataset having one of the following data
+    // attributes (see also `_getESUrl` function):
+    //
+    // <pre>
+    // elasticsearch_url
+    // webstore_url
+    // url
+    // </pre>
+    this.sync = function(method, model, options) {
+      if (model.__type__ == 'Dataset') {
+        var endpoint = self._getESUrl(model);
+      } else {
+        var endpoint = self._getESUrl(model.dataset);
+      }
+      var es = new my.Wrapper(endpoint, esOptions);
+      if (method === "read") {
+        if (model.__type__ == 'Dataset') {
+          var dfd = $.Deferred();
+          es.mapping().done(function(schema) {
+            // only one top level key in ES = the type so we can ignore it
+            var key = _.keys(schema)[0];
+            var fieldData = _.map(schema[key].properties, function(dict, fieldName) {
+              dict.id = fieldName;
+              return dict;
+            });
+            model.fields.reset(fieldData);
+            dfd.resolve(model);
+          })
+          .fail(function(arguments) {
+            dfd.reject(arguments);
+          });
+          return dfd.promise();
+        } else if (model.__type__ == 'Document') {
+          return es.get(model.dataset.id);
+        }
+      } else if (method === 'update') {
+        if (model.__type__ == 'Document') {
+          return es.upsert(model.toJSON());
+        }
+      } else if (method === 'delete') {
+        if (model.__type__ == 'Document') {
+          return es.delete(model.id);
+        }
+      }
+    };
+
+    // ### _getESUrl
+    //
+    // get url to ElasticSearch endpoint (see above)
+    this._getESUrl = function(dataset) {
+      if (dataset) {
+        var out = dataset.get('elasticsearch_url');
+        if (out) return out;
+        out = dataset.get('webstore_url');
+        if (out) return out;
+        out = dataset.get('url');
+        return out;
+      }
+      return this.get('url');
+    };
+
+    this.query = function(model, queryObj) {
       var dfd = $.Deferred();
+      var url = this._getESUrl(model);
+      var es = new my.Wrapper(url, esOptions);
+      var jqxhr = es.query(queryObj);
       // TODO: fail case
       jqxhr.done(function(results) {
         _.each(results.hits.hits, function(hit) {
@@ -183,8 +225,8 @@ this.recline.Backend = this.recline.Backend || {};
         dfd.resolve(results.hits);
       });
       return dfd.promise();
-    }
-  });
+    };
+  };
 
-}(jQuery, this.recline.Backend));
+}(jQuery, this.recline.Backend.ElasticSearch));
 
diff --git a/test/backend.elasticsearch.test.js b/test/backend.elasticsearch.test.js
index 9a0206fc..c0a19831 100644
--- a/test/backend.elasticsearch.test.js
+++ b/test/backend.elasticsearch.test.js
@@ -1,15 +1,18 @@
 (function ($) {
-module("Backend ElasticSearch");
+module("Backend ElasticSearch - Wrapper");
+
+test("queryNormalize", function() { 
+  var backend = new recline.Backend.ElasticSearch.Wrapper();
+  var in_ = new recline.Model.Query();
+  var out = backend._normalizeQuery(in_);
+  equal(out.size, 100);
 
-test("ElasticSearch queryNormalize", function() { 
-  var backend = new recline.Backend.ElasticSearch();
   var in_ = new recline.Model.Query();
   in_.set({q: ''});
   var out = backend._normalizeQuery(in_);
   equal(out.q, undefined);
   deepEqual(out.query.match_all, {});
 
-  var backend = new recline.Backend.ElasticSearch();
   var in_ = new recline.Model.Query().toJSON();
   in_.q = '';
   var out = backend._normalizeQuery(in_);
@@ -107,8 +110,99 @@ var sample_data = {
   "took": 2
 };
 
-test("ElasticSearch query", function() { 
-  var backend = new recline.Backend.ElasticSearch();
+test("query", function() { 
+  var backend = new recline.Backend.ElasticSearch.Wrapper('https://localhost:9200/my-es-db/my-es-type');
+
+  var stub = sinon.stub($, 'ajax', function(options) {
+    if (options.url.indexOf('_mapping') != -1) {
+      return {
+        done: function(callback) {
+          callback(mapping_data);
+          return this;
+        },
+        fail: function() {
+          return this;
+        }
+      }
+    } else {
+      return {
+        done: function(callback) {
+          callback(sample_data);
+        },
+        fail: function() {
+        }
+      }
+    }
+  });
+
+  backend.mapping().done(function(data) {
+    var fields = _.keys(data.note.properties);
+    deepEqual(['_created', '_last_modified', 'end', 'owner', 'start', 'title'], fields);
+  });
+
+  backend.query().done(function(queryResult) {
+    equal(3, queryResult.hits.total);
+    equal(3, queryResult.hits.hits.length);
+    equal('Note 1', queryResult.hits.hits[0]._source['title']);
+    start();
+  });
+  $.ajax.restore();
+});
+
+test("write", function() { 
+  var url = 'http://localhost:9200/recline-test/es-write';
+  var backend = new recline.Backend.ElasticSearch.Wrapper(url);
+  stop();
+
+  var id = parseInt(Math.random()*100000000).toString();
+  var doc = {
+    id: id,
+    title: 'my title'
+  };
+  var jqxhr = backend.upsert(doc);
+  jqxhr.done(function(data) {
+    ok(data.ok);
+    equal(data._id, id);
+    equal(data._type, 'es-write');
+    equal(data._version, 1);
+    
+    // update
+    doc.title = 'new title';
+    var jqxhr = backend.upsert(doc);
+    jqxhr.done(function(data) {
+      equal(data._version, 2);
+
+      // delete
+      var jqxhr = backend.delete(doc.id);
+      jqxhr.done(function(data) {
+        ok(data.ok);
+        doc = null;
+
+        // try to get ...
+        var jqxhr = backend.get(id);
+        jqxhr.done(function(data) {
+          // should not be here
+          ok(false, 'Should have got 404');
+        }).error(function(error) {
+          equal(error.status, 404);
+          start();
+        });
+      });
+    });
+  }).fail(function(error) {
+    console.log(error);
+    ok(false, 'Basic request failed - is ElasticSearch running locally on port 9200 (required for this test!)');
+    start();
+  });
+});
+
+
+// ==================================================
+
+module("Backend ElasticSearch - Backbone");
+
+test("query", function() { 
+  var backend = new recline.Backend.ElasticSearch.Backbone();
   var dataset = new recline.Model.Dataset({
       url: 'https://localhost:9200/my-es-db/my-es-type'
     },
@@ -137,7 +231,7 @@ test("ElasticSearch query", function() {
     }
   });
 
-  dataset.fetch().then(function(dataset) {
+  dataset.fetch().done(function(dataset) {
     deepEqual(['_created', '_last_modified', 'end', 'owner', 'start', 'title'], _.pluck(dataset.fields.toJSON(), 'id'));
     dataset.query().then(function(docList) {
       equal(3, dataset.docCount);
@@ -149,8 +243,8 @@ test("ElasticSearch query", function() {
   $.ajax.restore();
 });
 
-test("ElasticSearch write", function() { 
-  var backend = new recline.Backend.ElasticSearch();
+test("write", function() { 
+  var backend = new recline.Backend.ElasticSearch.Backbone();
   var dataset = new recline.Model.Dataset({
       url: 'http://localhost:9200/recline-test/es-write'
     },