From d19337eda4cbb90b01eec419990e29146e11695b Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sun, 22 Apr 2012 18:23:38 +0100 Subject: [PATCH 1/4] [build][xs]: regular build of recline.js. --- recline.js | 88 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/recline.js b/recline.js index e404d78b..a0efe02e 100644 --- a/recline.js +++ b/recline.js @@ -285,7 +285,8 @@ my.DocumentList = Backbone.Collection.extend({ // * format: (optional) used to indicate how the data should be formatted. For example: // * type=date, format=yyyy-mm-dd // * type=float, format=percentage -// * type=float, format='###,###.##' +// * type=string, format=link (render as hyperlink) +// * type=string, format=markdown (render as markdown if Showdown available) // * is_derived: (default: false) attribute indicating this field has no backend data but is just derived from other fields (see below). // // Following additional instance properties: @@ -341,6 +342,22 @@ my.Field = Backbone.Model.extend({ if (format === 'percentage') { return val + '%'; } + return val; + }, + 'string': function(val, field, doc) { + var format = field.get('format'); + if (format === 'link') { + return 'VAL'.replace(/VAL/g, val); + } else if (format === 'markdown') { + if (typeof Showdown !== 'undefined') { + var showdown = new Showdown.converter(); + out = showdown.makeHtml(val); + return out; + } else { + return val; + } + } + return val; } } }); @@ -1626,9 +1643,12 @@ my.Map = Backbone.View.extend({ if (!(docs instanceof Array)) docs = [docs]; + var count = 0; + var wrongSoFar = 0; _.every(docs,function(doc){ + count += 1; var feature = self._getGeometryFromDocument(doc); - if (typeof feature === 'undefined'){ + if (typeof feature === 'undefined' || feature === null){ // Empty field return true; } else if (feature instanceof Object){ @@ -1645,16 +1665,20 @@ my.Map = Backbone.View.extend({ feature.properties.cid = doc.cid; try { - self.features.addGeoJSON(feature); + self.features.addGeoJSON(feature); } catch (except) { - var msg = 'Wrong geometry value'; - if (except.message) msg += ' (' + except.message + ')'; + wrongSoFar += 1; + var msg = 'Wrong geometry value'; + if (except.message) msg += ' (' + except.message + ')'; + if (wrongSoFar <= 10) { my.notify(msg,{category:'error'}); - return false; + } } } else { - my.notify('Wrong geometry value',{category:'error'}); - return false; + wrongSoFar += 1 + if (wrongSoFar <= 10) { + my.notify('Wrong geometry value',{category:'error'}); + } } return true; }); @@ -1687,13 +1711,17 @@ my.Map = Backbone.View.extend({ return doc.attributes[this.state.get('geomField')]; } else if (this.state.get('lonField') && this.state.get('latField')){ // We'll create a GeoJSON like point object from the two lat/lon fields - return { - type: 'Point', - coordinates: [ - doc.attributes[this.state.get('lonField')], - doc.attributes[this.state.get('latField')] - ] - }; + var lon = doc.get(this.state.get('lonField')); + var lat = doc.get(this.state.get('latField')); + if (lon && lat) { + return { + type: 'Point', + coordinates: [ + doc.attributes[this.state.get('lonField')], + doc.attributes[this.state.get('latField')] + ] + }; + } } return null; } @@ -1705,12 +1733,16 @@ my.Map = Backbone.View.extend({ // If not found, the user can define them via the UI form. _setupGeometryField: function(){ var geomField, latField, lonField; - this.state.set({ - geomField: this._checkField(this.geometryFieldNames), - latField: this._checkField(this.latitudeFieldNames), - lonField: this._checkField(this.longitudeFieldNames) - }); this.geomReady = (this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField'))); + // should not overwrite if we have already set this (e.g. explicitly via state) + if (!this.geomReady) { + this.state.set({ + geomField: this._checkField(this.geometryFieldNames), + latField: this._checkField(this.latitudeFieldNames), + lonField: this._checkField(this.longitudeFieldNames) + }); + this.geomReady = (this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField'))); + } }, // Private: Check if a field in the current model exists in the provided @@ -2172,8 +2204,8 @@ my.DataExplorer = Backbone.View.extend({ initialize: function(options) { var self = this; this.el = $(this.el); - // Hash of 'page' views (i.e. those for whole page) keyed by page name this._setupState(options.state); + // Hash of 'page' views (i.e. those for whole page) keyed by page name if (options.views) { this.pageViews = options.views; } else { @@ -2772,6 +2804,13 @@ this.recline.Backend = this.recline.Backend || {}; // backends (see recline.Model.Dataset.initialize). __type__: 'base', + + // ### readonly + // + // Class level attribute indicating that this backend is read-only (that + // is, cannot be written to). + readonly: true, + // ### sync // // An implementation of Backbone.sync that will be used to override @@ -2891,6 +2930,7 @@ this.recline.Backend = this.recline.Backend || {}; // Note that this is a **read-only** backend. my.DataProxy = my.Base.extend({ __type__: 'dataproxy', + readonly: true, defaults: { dataproxy_url: 'http://jsonpdataproxy.appspot.com' }, @@ -2970,6 +3010,7 @@ this.recline.Backend = this.recline.Backend || {}; // 
http://localhost:9200/twitter/tweet
my.ElasticSearch = my.Base.extend({ __type__: 'elasticsearch', + readonly: true, _getESUrl: function(dataset) { var out = dataset.get('elasticsearch_url'); if (out) return out; @@ -3088,6 +3129,7 @@ this.recline.Backend = this.recline.Backend || {}; // my.GDoc = my.Base.extend({ __type__: 'gdoc', + readonly: true, getUrl: function(dataset) { var url = dataset.get('url'); if (url.indexOf('feeds/list') != -1) { @@ -3450,6 +3492,7 @@ this.recline.Backend = this.recline.Backend || {}; // my.Memory = my.Base.extend({ __type__: 'memory', + readonly: false, initialize: function() { this.datasets = {}; }, @@ -3537,7 +3580,8 @@ this.recline.Backend = this.recline.Backend || {}; _.each(terms, function(term) { var foundmatch = false; dataset.fields.each(function(field) { - var value = rawdoc[field.id].toString(); + var value = rawdoc[field.id]; + if (value !== null) { value = value.toString(); } // TODO regexes? foundmatch = foundmatch || (value === term); // TODO: early out (once we are true should break to spare unnecessary testing) From 1bf64c5f9475c73ec1d5306731eeb54ae3c0fc5f Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sun, 22 Apr 2012 22:17:47 +0100 Subject: [PATCH 2/4] [#61,backend/elasticsearch][m]: create, update and delete support in elasticsearch backend -- fixes #61. --- src/backend/elasticsearch.js | 33 ++++++++++++++-- test/backend.elasticsearch.test.js | 62 +++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/src/backend/elasticsearch.js b/src/backend/elasticsearch.js index 0ccb5295..03448277 100644 --- a/src/backend/elasticsearch.js +++ b/src/backend/elasticsearch.js @@ -21,7 +21,7 @@ this.recline.Backend = this.recline.Backend || {}; // 
http://localhost:9200/twitter/tweet
my.ElasticSearch = my.Base.extend({ __type__: 'elasticsearch', - readonly: true, + readonly: false, _getESUrl: function(dataset) { var out = dataset.get('elasticsearch_url'); if (out) return out; @@ -55,9 +55,36 @@ this.recline.Backend = this.recline.Backend || {}; dfd.reject(arguments); }); return dfd.promise(); + } else if (model.__type__ == 'Document') { + var base = this._getESUrl(model.dataset) + '/' + model.id; + return $.ajax({ + url: base, + dataType: 'json' + }); + } + } else if (method === 'update') { + if (model.__type__ == 'Document') { + var data = JSON.stringify(model.toJSON()); + var base = this._getESUrl(model.dataset); + if (model.id) { + base += '/' + model.id; + } + return $.ajax({ + url: base, + type: 'POST', + data: data, + dataType: 'json' + }); + } + } else if (method === 'delete') { + if (model.__type__ == 'Document') { + var base = this._getESUrl(model.dataset) + '/' + model.id; + return $.ajax({ + url: base, + type: 'DELETE', + dataType: 'json' + }); } - } else { - alert('This backend currently only supports read operations'); } }, _normalizeQuery: function(queryObj) { diff --git a/test/backend.elasticsearch.test.js b/test/backend.elasticsearch.test.js index 0a132d0c..9a0206fc 100644 --- a/test/backend.elasticsearch.test.js +++ b/test/backend.elasticsearch.test.js @@ -107,7 +107,7 @@ var sample_data = { "took": 2 }; -test("ElasticSearch", function() { +test("ElasticSearch query", function() { var backend = new recline.Backend.ElasticSearch(); var dataset = new recline.Model.Dataset({ url: 'https://localhost:9200/my-es-db/my-es-type' @@ -149,4 +149,64 @@ test("ElasticSearch", function() { $.ajax.restore(); }); +test("ElasticSearch write", function() { + var backend = new recline.Backend.ElasticSearch(); + var dataset = new recline.Model.Dataset({ + url: 'http://localhost:9200/recline-test/es-write' + }, + backend + ); + + stop(); + + var id = parseInt(Math.random()*100000000).toString(); + var doc = new recline.Model.Document({ + id: id, + title: 'my title' + }); + doc.backend = backend; + doc.dataset = dataset; + dataset.currentDocuments.add(doc); + var jqxhr = doc.save(); + jqxhr.done(function(data) { + ok(data.ok); + equal(data._id, id); + equal(data._type, 'es-write'); + equal(data._version, 1); + + // update + doc.set({title: 'new title'}); + var jqxhr = doc.save(); + jqxhr.done(function(data) { + equal(data._version, 2); + + // delete + var jqxhr = doc.destroy(); + jqxhr.done(function(data) { + ok(data.ok); + doc = null; + + // try to get ... + var olddoc = new recline.Model.Document({id: id}); + equal(olddoc.get('title'), null); + olddoc.dataset = dataset; + olddoc.backend = backend; + var jqxhr = olddoc.fetch(); + jqxhr.done(function(data) { + // should not be here + ok(false, 'Should have got 404'); + }).error(function(error) { + equal(error.status, 404); + equal(typeof olddoc.get('title'), 'undefined'); + start(); + }); + }); + }); + }).fail(function(error) { + console.log(error); + ok(false, 'Basic request failed - is ElasticSearch running locally on port 9200 (required for this test!)'); + start(); + }); +}); + })(this.jQuery); From a577866932a6b1ae0c482d2f4c67b3155b96af20 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Mon, 23 Apr 2012 02:30:16 +0100 Subject: [PATCH 3/4] [#61,backend/elasticearch][s]: refactor plus support for setting request headers (e.g. Authorization) plus can set backend url on initialization. * support for headers went into new _makeRequest method on backend base class * support for setting backend url on initialization (rather than depending on it being on Dataset/Document objects) * move upsert and delete methods out into distinct methods from being inside backbone sync The two last of these pave the way for use of ElasticSearch backend standalone (both independent from Recline and independent of Backbone) --- src/backend/base.js | 26 ++++++++ src/backend/elasticsearch.js | 115 ++++++++++++++++++++++++----------- 2 files changed, 104 insertions(+), 37 deletions(-) diff --git a/src/backend/base.js b/src/backend/base.js index 1b06dc01..94cccf4f 100644 --- a/src/backend/base.js +++ b/src/backend/base.js @@ -99,6 +99,32 @@ this.recline.Backend = this.recline.Backend || {}; query: function(model, queryObj) { }, + // ### _makeRequest + // + // Just $.ajax but in any headers in the 'headers' attribute of this + // Backend instance. Example: + // + // 
+    // var jqxhr = this._makeRequest({
+    //   url: the-url
+    // });
+    //
+ _makeRequest: function(data) { + var headers = this.get('headers'); + var extras = {}; + if (headers) { + extras = { + beforeSend: function(req) { + _.each(headers, function(value, key) { + req.setRequestHeader(key, value); + }); + } + }; + } + var data = _.extend(extras, data); + return $.ajax(data); + }, + // convenience method to convert simple set of documents / rows to a QueryResult _docsToQueryResult: function(rows) { var hits = _.map(rows, function(row) { diff --git a/src/backend/elasticsearch.js b/src/backend/elasticsearch.js index 03448277..546c0c8a 100644 --- a/src/backend/elasticsearch.js +++ b/src/backend/elasticsearch.js @@ -6,37 +6,39 @@ this.recline.Backend = this.recline.Backend || {}; // // Connecting to [ElasticSearch](http://www.elasticsearch.org/). // - // To use this backend ensure your Dataset has one of the following - // attributes (first one found is used): + // Usage: + // + // 
+  // var backend = new recline.Backend.ElasticSearch({
+  //   // optional as can also be provided by Dataset/Document
+  //   url: {url to ElasticSearch endpoint i.e. ES 'type/table' url - more info below}
+  //   // optional
+  //   headers: {dict of headers to add to each request}
+  // });
+  //
+  // @param {String} url: url for ElasticSearch type/table, e.g. for ES running
+  // on localhost:9200 with index // twitter and type tweet it would be:
+  // 
+  // 
http://localhost:9200/twitter/tweet

+  //
+  // This url is optional since the ES endpoint url may be specified on the the
+  // dataset (and on a Document by the document having a dataset attribute) by
+  // having one of the following (see also `_getESUrl` function):
   //
   // 
   // elasticsearch_url
   // webstore_url
   // url
   // 

-  //
-  // This should point to the ES type url. E.G. for ES running on
-  // localhost:9200 with index twitter and type tweet it would be
-  //
-  // 
http://localhost:9200/twitter/tweet

   my.ElasticSearch = my.Base.extend({
     __type__: 'elasticsearch',
     readonly: false,
-    _getESUrl: function(dataset) {
-      var out = dataset.get('elasticsearch_url');
-      if (out) return out;
-      out = dataset.get('webstore_url');
-      if (out) return out;
-      out = dataset.get('url');
-      return out;
-    },
     sync: function(method, model, options) {
       var self = this;
       if (method === "read") {
         if (model.__type__ == 'Dataset') {
-          var base = self._getESUrl(model);
-          var schemaUrl = base + '/_mapping';
-          var jqxhr = $.ajax({
+          var schemaUrl = self._getESUrl(model) + '/_mapping';
+          var jqxhr = this._makeRequest({
             url: schemaUrl,
             dataType: 'jsonp'
           });
@@ -57,36 +59,75 @@ this.recline.Backend = this.recline.Backend || {};
           return dfd.promise();
         } else if (model.__type__ == 'Document') {
           var base = this._getESUrl(model.dataset) + '/' + model.id;
-          return $.ajax({
+          return this._makeRequest({
             url: base,
             dataType: 'json'
           });
         }
       } else if (method === 'update') {
         if (model.__type__ == 'Document') {
-          var data = JSON.stringify(model.toJSON());
-          var base = this._getESUrl(model.dataset);
-          if (model.id) {
-            base += '/' + model.id;
-          }
-          return $.ajax({
-            url: base,
-            type: 'POST',
-            data: data,
-            dataType: 'json'
-          });
+          return this.upsert(model.toJSON(), this._getESUrl(model.dataset));
         }
       } else if (method === 'delete') {
         if (model.__type__ == 'Document') {
-          var base = this._getESUrl(model.dataset) + '/' + model.id;
-          return $.ajax({
-            url: base,
-            type: 'DELETE',
-            dataType: 'json'
-          });
+          var url = this._getESUrl(model.dataset);
+          return this.delete(model.id, url);
         }
       }
     },
+
+    // ### upsert
+    //
+    // create / update a document to ElasticSearch backend
+    //
+    // @param {Object} doc an object to insert to the index.
+    // @param {string} url (optional) url for ElasticSearch endpoint (if not
+    // defined called this._getESUrl()
+    upsert: function(doc, url) {
+      var data = JSON.stringify(doc);
+      url = url ? url : this._getESUrl();
+      if (doc.id) {
+        url += '/' + doc.id;
+      }
+      return this._makeRequest({
+        url: url,
+        type: 'POST',
+        data: data,
+        dataType: 'json'
+      });
+    },
+
+    // ### delete
+    //
+    // Delete a document from the ElasticSearch backend.
+    //
+    // @param {Object} id id of object to delete
+    // @param {string} url (optional) url for ElasticSearch endpoint (if not
+    // provided called this._getESUrl()
+    delete: function(id, url) {
+      url = url ? url : this._getESUrl();
+      url += '/' + id;
+      return this._makeRequest({
+        url: url,
+        type: 'DELETE',
+        dataType: 'json'
+      });
+    },
+
+    // ### _getESUrl
+    //
+    // get url to ElasticSearch endpoint (see above)
+    _getESUrl: function(dataset) {
+      if (dataset) {
+        var out = dataset.get('elasticsearch_url');
+        if (out) return out;
+        out = dataset.get('webstore_url');
+        if (out) return out;
+        out = dataset.get('url');
+        return out;
+      }
+      return this.get('url');
+    },
     _normalizeQuery: function(queryObj) {
       var out = queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj);
       if (out.q !== undefined && out.q.trim() === '') {
@@ -123,7 +164,7 @@ this.recline.Backend = this.recline.Backend || {};
       var queryNormalized = this._normalizeQuery(queryObj);
       var data = {source: JSON.stringify(queryNormalized)};
       var base = this._getESUrl(model);
-      var jqxhr = $.ajax({
+      var jqxhr = this._makeRequest({
         url: base + '/_search',
         data: data,
         dataType: 'jsonp'

From e5316e03cf1e1f72bbcf2c200149cc0e41f2b5c6 Mon Sep 17 00:00:00 2001
From: Rufus Pollock 
Date: Mon, 23 Apr 2012 02:36:11 +0100
Subject: [PATCH 4/4] [build][s]: regular build of docs and library.

---
 docs/backend/base.html          |  34 +++++++-
 docs/backend/dataproxy.html     |   1 +
 docs/backend/elasticsearch.html | 108 +++++++++++++++++++------
 docs/backend/gdocs.html         |   1 +
 docs/backend/memory.html        |   4 +-
 docs/model.html                 |  19 ++++-
 docs/view-map.html              |  59 ++++++++------
 docs/view.html                  |   4 +-
 recline.js                      | 138 +++++++++++++++++++++++++++-----
 9 files changed, 291 insertions(+), 77 deletions(-)

diff --git a/docs/backend/base.html b/docs/backend/base.html
index 9f022974..38e13e3a 100644
--- a/docs/backend/base.html
+++ b/docs/backend/base.html
@@ -22,7 +22,10 @@ object as a string (e.g. recline.Backend.Memory) or for Backends within
 recline.Backend module it may be their class name.

 
 
This value is used as an identifier for this backend when initializing
-backends (see recline.Model.Dataset.initialize).
                                         
    __type__: 'base',
                                                                                   
                                
               
sync

+backends (see recline.Model.Dataset.initialize).
                                         
    __type__: 'base',
                                                                                   
                                
               
readonly

+
+
Class level attribute indicating that this backend is read-only (that
+is, cannot be written to).
                                         
    readonly: true,
                                                                                   
                                
               
sync

 
 
An implementation of Backbone.sync that will be used to override
 Backbone.sync on operations for Datasets and Documents which are using this backend.

@@ -36,7 +39,7 @@ for Documents because they are loaded in bulk by the query method.

 
 
All code paths should return an object conforming to the jquery promise API.
                                         
    sync: function(method, model, options) {
     },
-    
                                                                                   
                                
               
query

+

query

Query the backend for documents returning them in bulk. This method will be used by the Dataset.query method to search the backend for documents, @@ -77,7 +80,30 @@ details):

} } 
    query: function(model, queryObj) {
-    },

convenience method to convert simple set of documents / rows to a QueryResult

    _docsToQueryResult: function(rows) {
+    },

_makeRequest

+ +

Just $.ajax but in any headers in the 'headers' attribute of this +Backend instance. Example:

+ +
+var jqxhr = this._makeRequest({
+  url: the-url
+});
+
    _makeRequest: function(data) {
+      var headers = this.get('headers');
+      var extras = {};
+      if (headers) {
+        extras = {
+          beforeSend: function(req) {
+            _.each(headers, function(value, key) {
+              req.setRequestHeader(key, value);
+            });
+          }
+        };
+      }
+      var data = _.extend(extras, data);
+      return $.ajax(data);
+    },

convenience method to convert simple set of documents / rows to a QueryResult

    _docsToQueryResult: function(rows) {
       var hits = _.map(rows, function(row) {
         return { _source: row };
       });
@@ -85,7 +111,7 @@ details):

         total: null,
         hits: hits
       };
-    },

_wrapInTimeout

+ },

_wrapInTimeout

Convenience method providing a crude way to catch backend errors on JSONP calls. Many of backends use JSONP and so will not get error messages and this is diff --git a/docs/backend/dataproxy.html b/docs/backend/dataproxy.html index b65ef570..0ff29c43 100644 --- a/docs/backend/dataproxy.html +++ b/docs/backend/dataproxy.html @@ -20,6 +20,7 @@

Note that this is a read-only backend.

  my.DataProxy = my.Base.extend({
     __type__: 'dataproxy',
+    readonly: true,
     defaults: {
       dataproxy_url: 'http://jsonpdataproxy.appspot.com'
     },
diff --git a/docs/backend/elasticsearch.html b/docs/backend/elasticsearch.html
index 0257f24c..11bb4b3a 100644
--- a/docs/backend/elasticsearch.html
+++ b/docs/backend/elasticsearch.html
@@ -5,35 +5,38 @@
 
 
Connecting to ElasticSearch.

 
-
To use this backend ensure your Dataset has one of the following
-attributes (first one found is used):

+
Usage:

+
+
+var backend = new recline.Backend.ElasticSearch({
+  // optional as can also be provided by Dataset/Document
+  url: {url to ElasticSearch endpoint i.e. ES 'type/table' url - more info below}
+  // optional
+  headers: {dict of headers to add to each request}
+});
+
+@param {String} url: url for ElasticSearch type/table, e.g. for ES running
+on localhost:9200 with index // twitter and type tweet it would be:
+
+
http://localhost:9200/twitter/tweet

+
+This url is optional since the ES endpoint url may be specified on the the
+dataset (and on a Document by the document having a dataset attribute) by
+having one of the following (see also `_getESUrl` function):
 
 
 elasticsearch_url
 webstore_url
 url
-

-
-
This should point to the ES type url. E.G. for ES running on
-localhost:9200 with index twitter and type tweet it would be

-
-
http://localhost:9200/twitter/tweet
                                         
  my.ElasticSearch = my.Base.extend({
+
                                         
  my.ElasticSearch = my.Base.extend({
     __type__: 'elasticsearch',
-    _getESUrl: function(dataset) {
-      var out = dataset.get('elasticsearch_url');
-      if (out) return out;
-      out = dataset.get('webstore_url');
-      if (out) return out;
-      out = dataset.get('url');
-      return out;
-    },
+    readonly: false,
     sync: function(method, model, options) {
       var self = this;
       if (method === "read") {
         if (model.__type__ == 'Dataset') {
-          var base = self._getESUrl(model);
-          var schemaUrl = base + '/_mapping';
-          var jqxhr = $.ajax({
+          var schemaUrl = self._getESUrl(model) + '/_mapping';
+          var jqxhr = this._makeRequest({
             url: schemaUrl,
             dataType: 'jsonp'
           });
@@ -50,10 +53,67 @@ localhost:9200 with index twitter and type tweet it would be

             dfd.reject(arguments);
           });
           return dfd.promise();
+        } else if (model.__type__ == 'Document') {
+          var base = this._getESUrl(model.dataset) + '/' + model.id;
+          return this._makeRequest({
+            url: base,
+            dataType: 'json'
+          });
+        }
+      } else if (method === 'update') {
+        if (model.__type__ == 'Document') {
+          return this.upsert(model.toJSON(), this._getESUrl(model.dataset));
+        }
+      } else if (method === 'delete') {
+        if (model.__type__ == 'Document') {
+          var url = this._getESUrl(model.dataset);
+          return this.delete(model.id, url);
         }
-      } else {
-        alert('This backend currently only supports read operations');
       }
+    },
                                                                                   
                                
               
upsert

+
+
create / update a document to ElasticSearch backend

+
+
@param {Object} doc an object to insert to the index.
+@param {string} url (optional) url for ElasticSearch endpoint (if not
+defined called this._getESUrl()
                                         
    upsert: function(doc, url) {
+      var data = JSON.stringify(doc);
+      url = url ? url : this._getESUrl();
+      if (doc.id) {
+        url += '/' + doc.id;
+      }
+      return this._makeRequest({
+        url: url,
+        type: 'POST',
+        data: data,
+        dataType: 'json'
+      });
+    },
                                                                                   
                                
               
delete

+
+
Delete a document from the ElasticSearch backend.

+
+
@param {Object} id id of object to delete
+@param {string} url (optional) url for ElasticSearch endpoint (if not
+provided called this._getESUrl()
                                         
    delete: function(id, url) {
+      url = url ? url : this._getESUrl();
+      url += '/' + id;
+      return this._makeRequest({
+        url: url,
+        type: 'DELETE',
+        dataType: 'json'
+      });
+    },
                                                                                   
                                
               
_getESUrl

+
+
get url to ElasticSearch endpoint (see above)
                                         
    _getESUrl: function(dataset) {
+      if (dataset) {
+        var out = dataset.get('elasticsearch_url');
+        if (out) return out;
+        out = dataset.get('webstore_url');
+        if (out) return out;
+        out = dataset.get('url');
+        return out;
+      }
+      return this.get('url');
     },
     _normalizeQuery: function(queryObj) {
       var out = queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj);
@@ -71,7 +131,7 @@ localhost:9200 with index twitter and type tweet it would be

           }
         };
         delete out.q;
-      }
                                                                                   
                                
               
now do filters (note the plural)
                                         
      if (out.filters && out.filters.length) {
+      }
                                                                                   
                                
               
now do filters (note the plural)
                                         
      if (out.filters && out.filters.length) {
         if (!out.filter) {
           out.filter = {};
         }
@@ -89,12 +149,12 @@ localhost:9200 with index twitter and type tweet it would be

       var queryNormalized = this._normalizeQuery(queryObj);
       var data = {source: JSON.stringify(queryNormalized)};
       var base = this._getESUrl(model);
-      var jqxhr = $.ajax({
+      var jqxhr = this._makeRequest({
         url: base + '/_search',
         data: data,
         dataType: 'jsonp'
       });
-      var dfd = $.Deferred();
                                                                                   
                                
               
TODO: fail case
                                         
      jqxhr.done(function(results) {
+      var dfd = $.Deferred();
                                                                                   
                                
               
TODO: fail case
                                         
      jqxhr.done(function(results) {
         _.each(results.hits.hits, function(hit) {
           if (!('id' in hit._source) && hit._id) {
             hit._source.id = hit._id;
diff --git a/docs/backend/gdocs.html b/docs/backend/gdocs.html
index 4d23ae12..b84397fa 100644
--- a/docs/backend/gdocs.html
+++ b/docs/backend/gdocs.html
@@ -16,6 +16,7 @@ var dataset = new recline.Model.Dataset({
 );
 
                                         
  my.GDoc = my.Base.extend({
     __type__: 'gdoc',
+    readonly: true,
     getUrl: function(dataset) {
       var url = dataset.get('url');
       if (url.indexOf('feeds/list') != -1) {
diff --git a/docs/backend/memory.html b/docs/backend/memory.html
index b55f36f9..6304cee6 100644
--- a/docs/backend/memory.html
+++ b/docs/backend/memory.html
@@ -67,6 +67,7 @@ If not defined (or id not provided) id will be autogenerated.
             
                                         
  my.Memory = my.Base.extend({
     __type__: 'memory',
+    readonly: false,
     initialize: function() {
       this.datasets = {};
     },
@@ -146,7 +147,8 @@ If not defined (or id not provided) id will be autogenerated.
             _.each(terms, function(term) {
             var foundmatch = false;
             dataset.fields.each(function(field) {
-              var value = rawdoc[field.id].toString();
                                                                                   
                                
               
TODO regexes?
                                         
              foundmatch = foundmatch || (value === term);
                                                                                   
                                
               
TODO: early out (once we are true should break to spare unnecessary testing)
+              var value = rawdoc[field.id];
+              if (value !== null) { value = value.toString(); }
                                                                                   
                                
               
TODO regexes?
                                         
              foundmatch = foundmatch || (value === term);
                                                                                   
                                
               
TODO: early out (once we are true should break to spare unnecessary testing)
 if (foundmatch) return true;
                                         
            });
             matches = matches && foundmatch;
                                                                                   
                                
               
TODO: early out (once false should break to spare unnecessary testing)
 if (!matches) return false;
                                         
          });
diff --git a/docs/model.html b/docs/model.html
index c43556ca..3009088a 100644
--- a/docs/model.html
+++ b/docs/model.html
@@ -183,7 +183,8 @@ for this document.
                          
 
  • format: (optional) used to indicate how the data should be formatted. For example:
 
    • type=date, format=yyyy-mm-dd
      • 
 
    • type=float, format=percentage
      • 
-
    • type=float, format='###,###.##'
    • 
+
  • type=string, format=link (render as hyperlink)
    • 
+
  • type=string, format=markdown (render as markdown if Showdown available)
    • 
 
  • is_derived: (default: false) attribute indicating this field has no backend data but is just derived from other fields (see below).
    • 
 
 
@@ -233,6 +234,22 @@ value of this field prior to rendering.
                              if (format === 'percentage') {
         return val + '%';
       }
+      return val;
+    },
+    'string': function(val, field, doc) {
+      var format = field.get('format');
+      if (format === 'link') {
+        return '<a href="VAL">VAL</a>'.replace(/VAL/g, val);
+      } else if (format === 'markdown') {
+        if (typeof Showdown !== 'undefined') {
+          var showdown = new Showdown.converter();
+          out = showdown.makeHtml(val);
+          return out;
+        } else {
+          return val;
+        }
+      }
+      return val;
     }
   }
 });
diff --git a/docs/view-map.html b/docs/view-map.html
index b865f6e7..73faab44 100644
--- a/docs/view-map.html
+++ b/docs/view-map.html
@@ -217,9 +217,12 @@ stopped and an error notification shown.
    
 
     if (!(docs instanceof Array)) docs = [docs];
 
+    var count = 0;
+    var wrongSoFar = 0;
     _.every(docs,function(doc){
+      count += 1;
       var feature = self._getGeometryFromDocument(doc);
-      if (typeof feature === 'undefined'){
                                                                                       
                                    
                   
    Empty field
                                             
            return true;
+      if (typeof feature === 'undefined' || feature === null){
                                                                                       
                                    
                   
    Empty field
                                             
            return true;
       } else if (feature instanceof Object){
                                                                                       
                                    
                   
    Build popup contents
 TODO: mustache?
                                             
            html = ''
         for (key in doc.attributes){
@@ -229,16 +232,20 @@ TODO: mustache?
                              
 link this Leaflet layer to a Recline doc
                                             
            feature.properties.cid = doc.cid;
 
         try {
-            self.features.addGeoJSON(feature);
+          self.features.addGeoJSON(feature);
         } catch (except) {
-            var msg = 'Wrong geometry value';
-            if (except.message) msg += ' (' + except.message + ')';
+          wrongSoFar += 1;
+          var msg = 'Wrong geometry value';
+          if (except.message) msg += ' (' + except.message + ')';
+          if (wrongSoFar <= 10) {
             my.notify(msg,{category:'error'});
-            return false;
+          }
         }
       } else {
-        my.notify('Wrong geometry value',{category:'error'});
-        return false;
+        wrongSoFar += 1
+        if (wrongSoFar <= 10) {
+          my.notify('Wrong geometry value',{category:'error'});
+        }
       }
       return true;
     });
@@ -259,13 +266,17 @@ link this Leaflet layer to a Recline doc
                              },
                                                                                       
                                    
                   
    Private: Return a GeoJSON geomtry extracted from the document fields
                                             
      _getGeometryFromDocument: function(doc){
     if (this.geomReady){
       if (this.state.get('geomField')){
                                                                                       
                                    
                   
    We assume that the contents of the field are a valid GeoJSON object
                                             
            return doc.attributes[this.state.get('geomField')];
-      } else if (this.state.get('lonField') && this.state.get('latField')){
                                                                                       
                                    
                   
    We'll create a GeoJSON like point object from the two lat/lon fields
                                             
            return {
-          type: 'Point',
-          coordinates: [
-            doc.attributes[this.state.get('lonField')],
-            doc.attributes[this.state.get('latField')]
-            ]
-        };
+      } else if (this.state.get('lonField') && this.state.get('latField')){
                                                                                       
                                    
                   
    We'll create a GeoJSON like point object from the two lat/lon fields
                                             
            var lon = doc.get(this.state.get('lonField'));
+        var lat = doc.get(this.state.get('latField'));
+        if (lon && lat) {
+          return {
+            type: 'Point',
+            coordinates: [
+              doc.attributes[this.state.get('lonField')],
+              doc.attributes[this.state.get('latField')]
+              ]
+          };
+        }
       }
       return null;
     }
@@ -274,13 +285,15 @@ two fields with lat/lon values.
    
 
 
    If not found, the user can define them via the UI form.
                                             
      _setupGeometryField: function(){
     var geomField, latField, lonField;
-    this.state.set({
-      geomField: this._checkField(this.geometryFieldNames),
-      latField: this._checkField(this.latitudeFieldNames),
-      lonField: this._checkField(this.longitudeFieldNames)
-    });
-    this.geomReady = (this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField')));
-  },
                                                                                       
                                    
                   
    Private: Check if a field in the current model exists in the provided
+    this.geomReady = (this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField')));
                                                                                       
                                    
                   
    should not overwrite if we have already set this (e.g. explicitly via state)
                                             
        if (!this.geomReady) {
+      this.state.set({
+        geomField: this._checkField(this.geometryFieldNames),
+        latField: this._checkField(this.latitudeFieldNames),
+        lonField: this._checkField(this.longitudeFieldNames)
+      });
+      this.geomReady = (this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField')));
+    }
+  },
                                                                                       
                                    
                   
    Private: Check if a field in the current model exists in the provided
 list of names.
                                             
      _checkField: function(fieldNames){
     var field;
     var modelFieldNames = this.model.fields.pluck('id');
@@ -291,7 +304,7 @@ list of names.
                              
       }
     }
     return null;
-  },
                                                                                       
                                    
                   
    Private: Sets up the Leaflet map control and the features layer.
    
+  },
                                                                                       
                                    
                   
    Private: Sets up the Leaflet map control and the features layer.
    
 
 
    The map uses a base layer from MapQuest based
 on OpenStreetMap.
                                             
      _setupMap: function(){
@@ -318,7 +331,7 @@ on OpenStreetMap.
                 
     this.map.setView(new L.LatLng(0, 0), 2);
 
     this.mapReady = true;
-  },
                                                                                       
                                    
                   
    Private: Helper function to select an option from a select list
                                             
      _selectOption: function(id,value){
+  },
                                                                                       
                                    
                   
    Private: Helper function to select an option from a select list
                                             
      _selectOption: function(id,value){
     var options = $('.' + id + ' > select > option');
     if (options){
       options.each(function(opt){
diff --git a/docs/view.html b/docs/view.html
index 0ca33a89..34f9cb34 100644
--- a/docs/view.html
+++ b/docs/view.html
@@ -180,8 +180,8 @@ initialized the DataExplorer with the relevant views themselves.
    
 
   initialize: function(options) {
     var self = this;
-    this.el = $(this.el);
                                                                                       
                                    
                   
    Hash of 'page' views (i.e. those for whole page) keyed by page name
                                             
        this._setupState(options.state);
-    if (options.views) {
+    this.el = $(this.el);
+    this._setupState(options.state);
                                                                                       
                                    
                   
    Hash of 'page' views (i.e. those for whole page) keyed by page name
                                             
        if (options.views) {
       this.pageViews = options.views;
     } else {
       this.pageViews = [{
diff --git a/recline.js b/recline.js
index a0efe02e..391296aa 100644
--- a/recline.js
+++ b/recline.js
@@ -2871,6 +2871,32 @@ this.recline.Backend = this.recline.Backend || {};
     query: function(model, queryObj) {
     },
 
+    // ### _makeRequest
+    // 
+    // Just $.ajax but in any headers in the 'headers' attribute of this
+    // Backend instance. Example:
+    //
+    // 
    +    // var jqxhr = this._makeRequest({
+    //   url: the-url
+    // });
+    // 
    
+    _makeRequest: function(data) {
+      var headers = this.get('headers');
+      var extras = {};
+      if (headers) {
+        extras = {
+          beforeSend: function(req) {
+            _.each(headers, function(value, key) {
+              req.setRequestHeader(key, value);
+            });
+          }
+        };
+      }
+      var data = _.extend(extras, data);
+      return $.ajax(data);
+    },
+
     // convenience method to convert simple set of documents / rows to a QueryResult
     _docsToQueryResult: function(rows) {
       var hits = _.map(rows, function(row) {
@@ -2995,37 +3021,39 @@ this.recline.Backend = this.recline.Backend || {};
   //
   // Connecting to [ElasticSearch](http://www.elasticsearch.org/).
   //
-  // To use this backend ensure your Dataset has one of the following
-  // attributes (first one found is used):
+  // Usage:
+  //
+  // 
    +  // var backend = new recline.Backend.ElasticSearch({
+  //   // optional as can also be provided by Dataset/Document
+  //   url: {url to ElasticSearch endpoint i.e. ES 'type/table' url - more info below}
+  //   // optional
+  //   headers: {dict of headers to add to each request}
+  // });
+  //
+  // @param {String} url: url for ElasticSearch type/table, e.g. for ES running
+  // on localhost:9200 with index // twitter and type tweet it would be:
+  // 
+  // 
    http://localhost:9200/twitter/tweet
    
+  //
+  // This url is optional since the ES endpoint url may be specified on the the
+  // dataset (and on a Document by the document having a dataset attribute) by
+  // having one of the following (see also `_getESUrl` function):
   //
   // 
       // elasticsearch_url
   // webstore_url
   // url
   // 
    
-  //
-  // This should point to the ES type url. E.G. for ES running on
-  // localhost:9200 with index twitter and type tweet it would be
-  //
-  // 
    http://localhost:9200/twitter/tweet
    
   my.ElasticSearch = my.Base.extend({
     __type__: 'elasticsearch',
-    readonly: true,
-    _getESUrl: function(dataset) {
-      var out = dataset.get('elasticsearch_url');
-      if (out) return out;
-      out = dataset.get('webstore_url');
-      if (out) return out;
-      out = dataset.get('url');
-      return out;
-    },
+    readonly: false,
     sync: function(method, model, options) {
       var self = this;
       if (method === "read") {
         if (model.__type__ == 'Dataset') {
-          var base = self._getESUrl(model);
-          var schemaUrl = base + '/_mapping';
-          var jqxhr = $.ajax({
+          var schemaUrl = self._getESUrl(model) + '/_mapping';
+          var jqxhr = this._makeRequest({
             url: schemaUrl,
             dataType: 'jsonp'
           });
@@ -3044,11 +3072,77 @@ this.recline.Backend = this.recline.Backend || {};
             dfd.reject(arguments);
           });
           return dfd.promise();
+        } else if (model.__type__ == 'Document') {
+          var base = this._getESUrl(model.dataset) + '/' + model.id;
+          return this._makeRequest({
+            url: base,
+            dataType: 'json'
+          });
+        }
+      } else if (method === 'update') {
+        if (model.__type__ == 'Document') {
+          return this.upsert(model.toJSON(), this._getESUrl(model.dataset));
+        }
+      } else if (method === 'delete') {
+        if (model.__type__ == 'Document') {
+          var url = this._getESUrl(model.dataset);
+          return this.delete(model.id, url);
         }
-      } else {
-        alert('This backend currently only supports read operations');
       }
     },
+
+    // ### upsert
+    //
+    // create / update a document to ElasticSearch backend
+    //
+    // @param {Object} doc an object to insert to the index.
+    // @param {string} url (optional) url for ElasticSearch endpoint (if not
+    // defined called this._getESUrl()
+    upsert: function(doc, url) {
+      var data = JSON.stringify(doc);
+      url = url ? url : this._getESUrl();
+      if (doc.id) {
+        url += '/' + doc.id;
+      }
+      return this._makeRequest({
+        url: url,
+        type: 'POST',
+        data: data,
+        dataType: 'json'
+      });
+    },
+
+    // ### delete
+    //
+    // Delete a document from the ElasticSearch backend.
+    //
+    // @param {Object} id id of object to delete
+    // @param {string} url (optional) url for ElasticSearch endpoint (if not
+    // provided called this._getESUrl()
+    delete: function(id, url) {
+      url = url ? url : this._getESUrl();
+      url += '/' + id;
+      return this._makeRequest({
+        url: url,
+        type: 'DELETE',
+        dataType: 'json'
+      });
+    },
+
+    // ### _getESUrl
+    //
+    // get url to ElasticSearch endpoint (see above)
+    _getESUrl: function(dataset) {
+      if (dataset) {
+        var out = dataset.get('elasticsearch_url');
+        if (out) return out;
+        out = dataset.get('webstore_url');
+        if (out) return out;
+        out = dataset.get('url');
+        return out;
+      }
+      return this.get('url');
+    },
     _normalizeQuery: function(queryObj) {
       var out = queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj);
       if (out.q !== undefined && out.q.trim() === '') {
@@ -3085,7 +3179,7 @@ this.recline.Backend = this.recline.Backend || {};
       var queryNormalized = this._normalizeQuery(queryObj);
       var data = {source: JSON.stringify(queryNormalized)};
       var base = this._getESUrl(model);
-      var jqxhr = $.ajax({
+      var jqxhr = this._makeRequest({
         url: base + '/_search',
         data: data,
         dataType: 'jsonp'