[#154,model/query][m]: refactor to new filter structure (see ticket) updating FilterEditor widget and backends.

* ElasticSearch changes represents a significant refactor and now support filters and query via constant_score (did not support this before!)

src/backend/elasticsearch.js

@@ -87,44 +87,57 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
     };
 
     this._normalizeQuery = function(queryObj) {
-      var out = queryObj && queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj);
+      var self = this;
-      if (out.q !== undefined && out.q.trim() === '') {
+      var queryInfo = (queryObj && queryObj.toJSON) ? queryObj.toJSON() : _.extend({}, queryObj);
-        delete out.q;
+      var out = {
+        constant_score: {
+          query: {}
         }
-      if (!out.q) {
+      };
-        out.query = {
+      if (!queryInfo.q) {
+        out.constant_score.query = {
           match_all: {}
         };
       } else {
-        out.query = {
+        out.constant_score.query = {
           query_string: {
-            query: out.q
+            query: queryInfo.q
           }
         };
-        delete out.q;
       }
-      // now do filters (note the *plural*)
+      if (queryInfo.filters && queryInfo.filters.length) {
-      if (out.filters && out.filters.length) {
+        out.constant_score.filter = {
-        if (!out.filter) {
+          and: []
-          out.filter = {};
+        };
-        }
+        _.each(queryInfo.filters, function(filter) {
-        if (!out.filter.and) {
+          out.constant_score.filter.and.push(self._convertFilter(filter));
-          out.filter.and = [];
+        });
-        }
-        out.filter.and = out.filter.and.concat(out.filters);
-      }
-      if (out.filters !== undefined) {
-        delete out.filters;
       }
       return out;
-    };
+    },
+
+    this._convertFilter = function(filter) {
+      var out = {};
+      out[filter.type] = {}
+      if (filter.type === 'term') {
+        out.term[filter.field] = filter.term;
+      } else if (filter.type === 'geo_point') {
+        out.geo_point[filter.field] = filter.point;
+        out.geo_point[filter.distance] = filter.distance;
+      }
+      return out;
+    },
 
     // ### query
     //
     // @return deferred supporting promise API
     this.query = function(queryObj) {
+      var esQuery = (queryObj && queryObj.toJSON) ? queryObj.toJSON() : _.extend({}, queryObj);
       var queryNormalized = this._normalizeQuery(queryObj);
-      var data = {source: JSON.stringify(queryNormalized)};
+      delete esQuery.q;
+      delete esQuery.filters;
+      esQuery.query = queryNormalized;
+      var data = {source: JSON.stringify(esQuery)};
       var url = this.endpoint + '/_search';
       var jqxhr = recline.Backend.makeRequest({
         url: url,

src/backend/memory.js

@@ -91,10 +91,9 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
     this._applyFilters = function(results, queryObj) {
       _.each(queryObj.filters, function(filter) {
         // if a term filter ...
-        if (filter.term) {
+        if (filter.type === 'term') {
           results = _.filter(results, function(doc) {
-            var fieldId = _.keys(filter.term)[0];
+            return (doc[filter.field] == filter.term);
-            return (doc[fieldId] == filter.term[fieldId]);
           });
         }
       });

src/model.js

@@ -411,19 +411,20 @@ my.Query = Backbone.Model.extend({
     return {
       size: 100,
       from: 0,
+      q: '',
       facets: {},
-      // <http://www.elasticsearch.org/guide/reference/query-dsl/and-filter.html>
-      // , filter: {}
       filters: []
     };
   },
   _filterTemplates: {
     term: {
-      '{{fieldId}}': ''
+      type: 'term',
+      field: '',
+      term: ''
     },
     geo_distance: {
       distance: '10km',
-      '{{fieldId}}': {
+      point: {
         lon: 0,
         lat: 0
       }
@@ -431,19 +432,18 @@ my.Query = Backbone.Model.extend({
   },  
   // ### addFilter
   //
-  // Add a new filter (appended to the list of filters) with default
+  // Add a new filter (appended to the list of filters)
-  // value. To set value for the filter use updateFilter.
   //
-  // @param type type of this filter e.g. term, geo_distance etc
+  // @param filter an object specifying the filter - see _filterTemplates for examples. If only type is provided will generate a filter by cloning _filterTemplates
-  // @param fieldId the field to add this filter on
+  addFilter: function(filter) {
-  addFilter: function(type, fieldId) {
+    // crude deep copy
-    var tmpl = JSON.stringify(this._filterTemplates[type]);
+    var ourfilter = JSON.parse(JSON.stringify(filter));
+    // not full specified so use template and over-write
+    if (_.keys(filter).length <= 2) {
+      ourfilter = _.extend(this._filterTemplates[filter.type], ourfilter);
+    }
     var filters = this.get('filters');
-    var filter = {};
+    filters.push(ourfilter);
-    filter[type] = JSON.parse(Mustache.render(tmpl, {type: type, fieldId: fieldId}));
-    filter[type]._type = type;
-    filter[type]._field = fieldId;
-    filters.push(filter);
     this.trigger('change:filters:new-blank');
   },
   updateFilter: function(index, value) {

src/widget.filtereditor.js

@@ -39,22 +39,22 @@ my.FilterEditor = Backbone.View.extend({
   ',
   filterTemplates: {
     term: ' \
-      <div class="control-group filter-{{_type}} filter"> \
+      <div class="control-group filter-{{type}} filter"> \
-        <label class="control-label" for="">{{_field}}</label> \
+        <label class="control-label" for="">{{field}}</label> \
         <div class="controls"> \
-            <input type="text" value="{{_value}}" name="term" data-filter-field="{{_field}}" data-filter-id="{{id}}" data-filter-type="{{_type}}" /> \
+            <input type="text" value="{{term}}" name="term" data-filter-field="{{field}}" data-filter-id="{{id}}" data-filter-type="{{type}}" /> \
             <a class="js-remove-filter" href="#">&times;</a> \
         </div> \
       </div> \
     ',
     geo_distance: ' \
-      <div class="control-group filter-{{_type}} filter"> \
+      <div class="control-group filter-{{type}} filter"> \
-        <label class="control-label" for="">{{_field}}</label> \
+        <label class="control-label" for="">{{field}}</label> \
         <a class="js-remove-filter" href="#">&times;</a> \
         <div class="controls"> \
-            <input type="text" value="{{_value.lon}}" name="lon" data-filter-field="{{_field}}" data-filter-id="{{id}}" data-filter-type="{{_type}}" /> \
+            <input type="text" value="{{point.lon}}" name="lon" data-filter-field="{{field}}" data-filter-id="{{id}}" data-filter-type="{{type}}" /> \
-            <input type="text" value="{{_value.lat}}" name="lat" data-filter-field="{{_field}}" data-filter-id="{{id}}" data-filter-type="{{_type}}" /> \
+            <input type="text" value="{{point.lat}}" name="lat" data-filter-field="{{field}}" data-filter-id="{{id}}" data-filter-type="{{type}}" /> \
-            <input type="text" value="{{distance}}" name="distance" data-filter-field="{{_field}}" data-filter-id="{{id}}" data-filter-type="{{_type}}" /> \
+            <input type="text" value="{{distance}}" name="distance" data-filter-field="{{field}}" data-filter-id="{{id}}" data-filter-type="{{type}}" /> \
         </div> \
       </div> \
     '
@@ -83,12 +83,7 @@ my.FilterEditor = Backbone.View.extend({
     });
     tmplData.fields = this.model.fields.toJSON();
     tmplData.filterRender = function() {
-      var filterType = _.keys(this)[0];
+      return Mustache.render(self.filterTemplates[this.type], this);
-      var _data = this[filterType];
-      _data.id = this.id;
-      _data._type = filterType;
-      _data._value = _data[_data._field];
-      return Mustache.render(self.filterTemplates[filterType], _data);
     };
     var out = Mustache.render(this.template, tmplData);
     this.el.html(out);
@@ -105,7 +100,7 @@ my.FilterEditor = Backbone.View.extend({
     $target.hide();
     var filterType = $target.find('select.filterType').val();
     var field = $target.find('select.fields').val();
-    this.model.queryState.addFilter(filterType, field);
+    this.model.queryState.addFilter({type: filterType, field: field});
     // trigger render explicitly as queryState change will not be triggered (as blank value for filter)
     this.render();
   },
@@ -128,12 +123,12 @@ my.FilterEditor = Backbone.View.extend({
       var name = $input.attr('name');
       var value = $input.val();
       if (filterType === 'term') {
-        filters[filterIndex].term[fieldId] = value;
+        filters[filterIndex].term = value;
       } else if (filterType === 'geo_distance') {
         if (name === 'distance') {
-          filters[filterIndex].geo_distance.distance = parseInt(value);
+          filters[filterIndex].distance = parseInt(value);
         } else {
-          filters[filterIndex].geo_distance[fieldId][name] = parseFloat(value);
+          filters[filterIndex].point[name] = parseFloat(value);
         }
       }
     });

test/backend/elasticsearch.test.js

@@ -3,32 +3,52 @@ module("Backend ElasticSearch - Wrapper");
 
 test("queryNormalize", function() { 
   var backend = new recline.Backend.ElasticSearch.Wrapper();
+
   var in_ = new recline.Model.Query();
   var out = backend._normalizeQuery(in_);
-  equal(out.size, 100);
+  var exp = {
+    constant_score: {
+      query: {
+        match_all: {}
+      }
+    }
+  };
+  deepEqual(out, exp);
 
   var in_ = new recline.Model.Query();
   in_.set({q: ''});
   var out = backend._normalizeQuery(in_);
-  equal(out.q, undefined);
+  deepEqual(out, exp);
-  deepEqual(out.query.match_all, {});
-
-  var in_ = new recline.Model.Query().toJSON();
-  in_.q = '';
-  var out = backend._normalizeQuery(in_);
-  equal(out.q, undefined);
-  deepEqual(out.query.match_all, {});
-
-  var in_ = new recline.Model.Query().toJSON();
-  in_.q = 'abc';
-  var out = backend._normalizeQuery(in_);
-  equal(out.query.query_string.query, 'abc');
 
   var in_ = new recline.Model.Query();
-  in_.addTermFilter('xyz', 'XXX');
+  in_.attributes.q = 'abc';
-  in_ = in_.toJSON();
   var out = backend._normalizeQuery(in_);
-  deepEqual(out.filter.and[0], {term: { xyz: 'XXX'}});
+  equal(out.constant_score.query.query_string.query, 'abc');
+
+  var in_ = new recline.Model.Query();
+  in_.addFilter({
+    type: 'term',
+    field: 'xyz',
+    term: 'XXX'
+  });
+  var out = backend._normalizeQuery(in_);
+  var exp = {
+    constant_score: {
+      query: {
+        match_all: {}
+      },
+      filter: {
+        and: [
+          {
+            term: {
+              xyz: 'XXX'
+            }
+          }
+        ]
+      }
+    }
+  };
+  deepEqual(out, exp);
 });
 
 var mapping_data = {

test/backend/memory.test.js

@@ -60,7 +60,7 @@ test('query string', function () {
 test('filters', function () {
   var data = _wrapData();
   var query = new recline.Model.Query();
-  query.addTermFilter('country', 'UK');
+  query.addFilter({type: 'term', field: 'country', term: 'UK'});
   var out = data.query(query.toJSON());
   equal(out.total, 3);
   deepEqual(_.pluck(out.records, 'country'), ['UK', 'UK', 'UK']);
@@ -198,7 +198,7 @@ test('query string', function () {
 
 test('filters', function () {
   var dataset = makeBackendDataset();
-  dataset.queryState.addTermFilter('country', 'UK');
+  dataset.queryState.addFilter({type: 'term', field: 'country', term: 'UK'});
   dataset.query().then(function() {
     equal(dataset.currentRecords.length, 3);
     deepEqual(dataset.currentRecords.pluck('country'), ['UK', 'UK', 'UK']);

test/model.test.js

@@ -151,27 +151,23 @@ test('Query', function () {
 
 test('Query.addFilter', function () {
   var query = new recline.Model.Query();
-  query.addFilter('term', 'xyz');
+  query.addFilter({type: 'term', field: 'xyz'});
   var exp = {
-    term: {
+    field: 'xyz',
-      xyz: '',
+    type: 'term',
-      _field: 'xyz',
+    term: ''
-      _type: 'term'
-    }
   };
-  deepEqual(exp, query.get('filters')[0]);
+  deepEqual(query.get('filters')[0], exp);
 
-  query.addFilter('geo_distance', 'xyz');
+  query.addFilter({type: 'geo_distance', field: 'xyz'});
   var exp = {
-    geo_distance: {
     distance: '10km',
-      xyz: {
+    point: {
       lon: 0,
       lat: 0
     },
-      _field: 'xyz',
+    field: 'xyz',
-      _type: 'geo_distance'
+    type: 'geo_distance'
-    }
   };
   deepEqual(exp, query.get('filters')[1]);
 });

test/widget.filtereditor.test.js

@@ -21,12 +21,12 @@ test('basics', function () {
   ok(!$addForm.is(":visible"));
   $editForm = view.el.find('form.js-edit');
   equal($editForm.find('.filter-term').length, 1)
-  equal(_.keys(dataset.queryState.attributes.filters[0].term)[0], 'country');
+  equal(dataset.queryState.attributes.filters[0].field, 'country');
 
   // now set filter value and apply
   $editForm.find('input').val('UK');
   $editForm.submit();
-  equal(dataset.queryState.attributes.filters[0].term.country, 'UK');
+  equal(dataset.queryState.attributes.filters[0].term, 'UK');
   equal(dataset.currentRecords.length, 3);
 
   // now remove filter
@@ -55,12 +55,13 @@ test('geo_distance', function () {
   // now check we have new filter
   $editForm = view.el.find('form.js-edit');
   equal($editForm.find('.filter-geo_distance').length, 1)
-  deepEqual(_.keys(dataset.queryState.attributes.filters[0].geo_distance), ['distance', 'lon', '_type', '_field']);
+  deepEqual(_.keys(dataset.queryState.attributes.filters[0]), ['distance',
+    'point', 'type', 'field']);
 
   // now set filter value and apply
   $editForm.find('input[name="lat"]').val(10);
   $editForm.submit();
-  equal(dataset.queryState.attributes.filters[0].geo_distance.lon.lat, 10);
+  equal(dataset.queryState.attributes.filters[0].point.lat, 10);
 
   view.remove();
 });