From e3819d8f4ef8abddda8dbbe972b5ddf92a785098 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 31 Mar 2012 11:09:10 +0100 Subject: [PATCH 01/11] [#51,bugfix][xs]: fix bug in pulling of fields from document in createDataset (corrects error in 5fc486d8083a27db21993fc7d58c2c2dae75c17f). --- src/backend/memory.js | 4 ++-- test/backend.test.js | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/backend/memory.js b/src/backend/memory.js index 7ca76932..be86c936 100644 --- a/src/backend/memory.js +++ b/src/backend/memory.js @@ -29,8 +29,8 @@ this.recline.Backend = this.recline.Backend || {}; datasetInfo.fields = fields; } else { if (data) { - datasetInfo.fields = _.map(data[0], function(cell) { - return {id: cell}; + datasetInfo.fields = _.map(data[0], function(value, key) { + return {id: key}; }); } } diff --git a/test/backend.test.js b/test/backend.test.js index 27e0bd53..332b8eb1 100644 --- a/test/backend.test.js +++ b/test/backend.test.js @@ -7,7 +7,7 @@ var memoryData = { , name: '1-my-test-dataset' , id: 'test-dataset' }, - fields: [{id: 'x'}, {id: 'y'}, {id: 'z'}], + fields: [{id: 'id'}, {id: 'x'}, {id: 'y'}, {id: 'z'}], documents: [ {id: 0, x: 1, y: 2, z: 3} , {id: 1, x: 2, y: 4, z: 6} @@ -32,6 +32,8 @@ test('Memory Backend: createDataset', function () { test('Memory Backend: createDataset 2', function () { var dataset = recline.Backend.createDataset(memoryData.documents); + equal(dataset.fields.length, 4); + deepEqual(['id', 'x', 'y', 'z'], dataset.fields.pluck('id')); dataset.query(); equal(memoryData.documents.length, dataset.currentDocuments.length); }); From 905659d86fdb900360bbd27147346e365870a5f9 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 31 Mar 2012 15:08:27 +0100 Subject: [PATCH 02/11] [#62,view/query][m]: start of faceting support with QueryFacetEditor. * QueryFacetEditor in DataExplorer which displays facets but nothing yet happens. * NB: very much less than half way through (things look worse rather than better atm). --- css/data-explorer.css | 4 +++ demo/js/app.js | 33 ++++++++++++++++----- src/model.js | 21 ++++++++++++++ src/view.js | 67 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 7 deletions(-) diff --git a/css/data-explorer.css b/css/data-explorer.css index 0fab70e1..a88f53cd 100644 --- a/css/data-explorer.css +++ b/css/data-explorer.css @@ -60,6 +60,10 @@ clear: both; } +.recline-query-facet-editor { + clear: both; +} + /********************************************************** * Notifications *********************************************************/ diff --git a/demo/js/app.js b/demo/js/app.js index 9198a913..e921031c 100755 --- a/demo/js/app.js +++ b/demo/js/app.js @@ -81,19 +81,38 @@ function localDataset() { , name: '1-my-test-dataset' , id: datasetId }, - fields: [{id: 'x'}, {id: 'y'}, {id: 'z'}, {id: 'label'}], + fields: [{id: 'x'}, {id: 'y'}, {id: 'z'}, {id: 'country'}, {id: 'label'}], documents: [ - {id: 0, x: 1, y: 2, z: 3, label: 'first'} - , {id: 1, x: 2, y: 4, z: 6, label: 'second'} - , {id: 2, x: 3, y: 6, z: 9, label: 'third'} - , {id: 3, x: 4, y: 8, z: 12, label: 'fourth'} - , {id: 4, x: 5, y: 10, z: 15, label: 'fifth'} - , {id: 5, x: 6, y: 12, z: 18, label: 'sixth'} + {id: 0, x: 1, y: 2, z: 3, country: 'DE', label: 'first'} + , {id: 1, x: 2, y: 4, z: 6, country: 'UK', label: 'second'} + , {id: 2, x: 3, y: 6, z: 9, country: 'US', label: 'third'} + , {id: 3, x: 4, y: 8, z: 12, country: 'UK', label: 'fourth'} + , {id: 4, x: 5, y: 10, z: 15, country: 'UK', label: 'fifth'} + , {id: 5, x: 6, y: 12, z: 18, country: 'DE', label: 'sixth'} ] }; var backend = new recline.Backend.Memory(); backend.addDataset(inData); var dataset = new recline.Model.Dataset({id: datasetId}, backend); + // TODO: auto-compute in Memory backend ?? + dataset.facets = { + 'country': { + terms: [ + { + term: 'UK', + count: 3 + }, + { + term: 'DE', + count: 2 + }, + { + term: 'US', + count: 1 + } + ] + } + }; return dataset; } diff --git a/src/model.js b/src/model.js index 207a9872..41934e67 100644 --- a/src/model.js +++ b/src/model.js @@ -23,6 +23,7 @@ my.Dataset = Backbone.Model.extend({ } this.fields = new my.FieldList(); this.currentDocuments = new my.DocumentList(); + this.facets = {}; this.docCount = null; this.queryState = new my.Query(); this.queryState.bind('change', this.query); @@ -116,6 +117,26 @@ my.Query = Backbone.Model.extend({ defaults: { size: 100 , from: 0 + , facets: {} + }, + initialize: function() { + _.bindAll(this, 'addFacet'); + }, + + addFacet: function(fieldId) { + // TODO: utilize field info to determine facet type ?? + var facets = this.get('facets'); + if (fieldId in facets) { + return; + } + facets[fieldId] = { + terms: { + field: fieldId + } + } + this.set({facets: facets}); + // trigger change event (does not seem to be triggered o/w) + // this.change(); } }); diff --git a/src/view.js b/src/view.js index 72b1c0d3..ae9c24f4 100644 --- a/src/view.js +++ b/src/view.js @@ -168,6 +168,12 @@ my.DataExplorer = Backbone.View.extend({ model: this.model.queryState }); this.el.find('.header').append(queryEditor.el); + var queryFacetEditor = new my.QueryFacetEditor({ + model: this.model.queryState + }, + this.model + ); + this.el.find('.header').append(queryFacetEditor.el); }, setupRouting: function() { @@ -275,6 +281,67 @@ my.QueryEditor = Backbone.View.extend({ } }); +my.QueryFacetEditor = Backbone.View.extend({ + className: 'recline-query-facet-editor', + template: ' \ + Add filter on \ + \ +
\ + {{#facets}} \ + {{label}} \ + \ + {{/facets}} \ +
\ + ', + + events: { + 'change .js-faceton': 'onAddFilter', + 'click .js-facet-show-toggle': 'onFacetShowToggle' + }, + initialize: function(model, dataset) { + _.bindAll(this, 'render'); + this.el = $(this.el); + this.model.bind('change', this.render); + this.dataset = dataset; + this.dataset.fields.bind('add', this.render); + this.dataset.fields.bind('reset', this.render); + this.dataset.fields.bind('remove', this.render); + this.render(); + }, + render: function() { + var tmplData = { + query: this.model.toJSON(), + fields: this.dataset.fields.toJSON() + }; + tmplData.facets = _.map(this.dataset.facets, function(data, key) { + var out = _.extend({label: key}, data); + return out; + }) + var templated = $.mustache(this.template, tmplData); + this.el.html(templated); + }, + onAddFilter: function(e) { + var fieldId = $(e.target).find('option:selected').attr('name'); + this.model.addFacet(fieldId); + // calculate facets for that field (if not already there??) + // re-render will happen automatically as dataset will have updated + }, + onFacetShowToggle: function(e) { + e.preventDefault(); + var $a = $(e.target); + var facetId = $a.attr('data-facet'); + var $ul = this.el.find('.facet-items[data-facet="' + facetId + '"]'); + $ul.toggle(); + } +}); /* ========================================================== */ // ## Miscellaneous Utilities From 3412962a358f11907794e96f8523ec52c54c38ea Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 31 Mar 2012 18:06:41 +0100 Subject: [PATCH 03/11] [#62,faceting][s]: introduce Facet and FacetList models and use them. * Conceptual breakthrough on how Faceting and Filtering interact (have updated issue #62 as a result). --- demo/js/app.js | 9 +++---- src/model.js | 40 +++++++++++++++++++++----------- src/view.js | 58 ++++++++++++++++++++-------------------------- test/model.test.js | 7 ++++++ 4 files changed, 63 insertions(+), 51 deletions(-) diff --git a/demo/js/app.js b/demo/js/app.js index e921031c..a6ca9f63 100755 --- a/demo/js/app.js +++ b/demo/js/app.js @@ -95,9 +95,10 @@ function localDataset() { backend.addDataset(inData); var dataset = new recline.Model.Dataset({id: datasetId}, backend); // TODO: auto-compute in Memory backend ?? - dataset.facets = { - 'country': { - terms: [ + dataset.facets = new recline.Model.FacetList([ + { + id: 'country', + result: [ { term: 'UK', count: 3 @@ -112,7 +113,7 @@ function localDataset() { } ] } - }; + ]); return dataset; } diff --git a/src/model.js b/src/model.js index 41934e67..b3743381 100644 --- a/src/model.js +++ b/src/model.js @@ -23,7 +23,7 @@ my.Dataset = Backbone.Model.extend({ } this.fields = new my.FieldList(); this.currentDocuments = new my.DocumentList(); - this.facets = {}; + this.facets = new my.FacetList(); this.docCount = null; this.queryState = new my.Query(); this.queryState.bind('change', this.query); @@ -119,24 +119,36 @@ my.Query = Backbone.Model.extend({ , from: 0 , facets: {} }, - initialize: function() { - _.bindAll(this, 'addFacet'); - }, + // Set (update or add) a terms filter + // http://www.elasticsearch.org/guide/reference/query-dsl/terms-filter.html + setFilter: function(fieldId, values) { + } +}); +my.Facet = Backbone.Model.extend({ + defaults: { + query: null, + result: null + } +}); + +my.FacetList = Backbone.Collection.extend({ + model: my.Facet, addFacet: function(fieldId) { - // TODO: utilize field info to determine facet type ?? - var facets = this.get('facets'); - if (fieldId in facets) { + // Assume id and fieldId should be the same (TODO: this need not be true if we want to add two different type of facets on same field) + if (fieldId in this) { return; } - facets[fieldId] = { - terms: { - field: fieldId + // TODO: utilize field info to determine facet type ?? + var facet = new my.Facet({ + id: fieldId, + query: { + terms: { + field: fieldId + } } - } - this.set({facets: facets}); - // trigger change event (does not seem to be triggered o/w) - // this.change(); + }); + this.add(facet); } }); diff --git a/src/view.js b/src/view.js index ae9c24f4..a463d8b5 100644 --- a/src/view.js +++ b/src/view.js @@ -168,11 +168,9 @@ my.DataExplorer = Backbone.View.extend({ model: this.model.queryState }); this.el.find('.header').append(queryEditor.el); - var queryFacetEditor = new my.QueryFacetEditor({ - model: this.model.queryState - }, - this.model - ); + var queryFacetEditor = new my.FacetQueryEditor({ + model: this.model + }); this.el.find('.header').append(queryFacetEditor.el); }, @@ -281,58 +279,52 @@ my.QueryEditor = Backbone.View.extend({ } }); -my.QueryFacetEditor = Backbone.View.extend({ +my.FacetQueryEditor = Backbone.View.extend({ className: 'recline-query-facet-editor', template: ' \ - Add filter on \ -
    \ - {{#fields}} \ -
  • {{label}}
    • \ - {{/fields}} \ +
    \ + Add Facet On \ +
      \ + {{#fields}} \ +
    • {{label}}
      • \ + {{/fields}} \
    \ +
    \
    \ {{#facets}} \ - {{label}} \ -
    \ ', events: { - 'change .js-faceton': 'onAddFilter', + 'click .js-add-facet .dropdown-menu a': 'onAddFacet', 'click .js-facet-show-toggle': 'onFacetShowToggle' }, - initialize: function(model, dataset) { + initialize: function(model) { _.bindAll(this, 'render'); this.el = $(this.el); - this.model.bind('change', this.render); - this.dataset = dataset; - this.dataset.fields.bind('add', this.render); - this.dataset.fields.bind('reset', this.render); - this.dataset.fields.bind('remove', this.render); + this.model.facets.bind('all', this.render); + this.model.fields.bind('all', this.render); this.render(); }, render: function() { var tmplData = { - query: this.model.toJSON(), - fields: this.dataset.fields.toJSON() + facets: this.model.facets.toJSON(), + fields: this.model.fields.toJSON() }; - tmplData.facets = _.map(this.dataset.facets, function(data, key) { - var out = _.extend({label: key}, data); - return out; - }) var templated = $.mustache(this.template, tmplData); this.el.html(templated); }, - onAddFilter: function(e) { - var fieldId = $(e.target).find('option:selected').attr('name'); - this.model.addFacet(fieldId); - // calculate facets for that field (if not already there??) - // re-render will happen automatically as dataset will have updated + onAddFacet: function(e) { + e.preventDefault(); + var fieldId = $(e.target).attr('href').slice(1); + this.model.facets.addFacet(fieldId); }, onFacetShowToggle: function(e) { e.preventDefault(); diff --git a/test/model.test.js b/test/model.test.js index a3c3b4c6..61587bd9 100644 --- a/test/model.test.js +++ b/test/model.test.js @@ -43,4 +43,11 @@ test('Dataset', function () { equal(out.fields.length, 2); }); +test('Facet', function () { + var facets = new recline.Model.FacetList(); + facets.addFacet('xyz'); + equal(1, facets.length); + deepEqual({terms: {field: 'xyz'}}, facets.get('xyz').get('query')); +}); + })(this.jQuery); From 767f9a23d659f3c6d794e20096aa2474ee0d9d48 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sun, 1 Apr 2012 10:49:22 +0100 Subject: [PATCH 04/11] [#57,backend,refactor][s]: create a recline.Backend.Base model which other backends inherit from. * At the moment this added little other than a cleaner structure but will help with change for #57. --- src/backend/base.js | 60 +++++++++++++++++++++--------------- src/backend/dataproxy.js | 4 +-- src/backend/elasticsearch.js | 4 +-- src/backend/gdocs.js | 2 +- src/backend/memory.js | 2 +- 5 files changed, 42 insertions(+), 30 deletions(-) diff --git a/src/backend/base.js b/src/backend/base.js index caf317f9..94ab7fb1 100644 --- a/src/backend/base.js +++ b/src/backend/base.js @@ -14,29 +14,41 @@ this.recline.Backend = this.recline.Backend || {}; return model.backend.sync(method, model, options); } - // ## wrapInTimeout - // - // Crude way to catch backend errors - // Many of backends use JSONP and so will not get error messages and this is - // a crude way to catch those errors. - my.wrapInTimeout = function(ourFunction) { - var dfd = $.Deferred(); - var timeout = 5000; - var timer = setTimeout(function() { - dfd.reject({ - message: 'Request Error: Backend did not respond after ' + (timeout / 1000) + ' seconds' - }); - }, timeout); - ourFunction.done(function(arguments) { - clearTimeout(timer); - dfd.resolve(arguments); - }) - .fail(function(arguments) { - clearTimeout(timer); - dfd.reject(arguments); - }) - ; - return dfd.promise(); - } + // ## recline.Backend.Base + // + // Base class for backends providing a template and convenience functions. + // You do not have to inherit from this class but even when not it does provide guidance on the functions you must implement. + my.Base = Backbone.Model.extend({ + sync: function(method, model, options) { + }, + query: function(model, queryObj) { + }, + + // ## _wrapInTimeout + // + // Convenience method providing a crude way to catch backend errors on JSONP calls. + // Many of backends use JSONP and so will not get error messages and this is + // a crude way to catch those errors. + _wrapInTimeout: function(ourFunction) { + var dfd = $.Deferred(); + var timeout = 5000; + var timer = setTimeout(function() { + dfd.reject({ + message: 'Request Error: Backend did not respond after ' + (timeout / 1000) + ' seconds' + }); + }, timeout); + ourFunction.done(function(arguments) { + clearTimeout(timer); + dfd.resolve(arguments); + }) + .fail(function(arguments) { + clearTimeout(timer); + dfd.reject(arguments); + }) + ; + return dfd.promise(); + } + }); + }(jQuery, this.recline.Backend)); diff --git a/src/backend/dataproxy.js b/src/backend/dataproxy.js index 45abc89a..e2ddb767 100644 --- a/src/backend/dataproxy.js +++ b/src/backend/dataproxy.js @@ -16,7 +16,7 @@ this.recline.Backend = this.recline.Backend || {}; // * format: (optional) csv | xls (defaults to csv if not specified) // // Note that this is a **read-only** backend. - my.DataProxy = Backbone.Model.extend({ + my.DataProxy = my.Base.extend({ defaults: { dataproxy_url: 'http://jsonpdataproxy.appspot.com' }, @@ -47,7 +47,7 @@ this.recline.Backend = this.recline.Backend || {}; , dataType: 'jsonp' }); var dfd = $.Deferred(); - my.wrapInTimeout(jqxhr).done(function(results) { + this._wrapInTimeout(jqxhr).done(function(results) { if (results.error) { dfd.reject(results.error); } diff --git a/src/backend/elasticsearch.js b/src/backend/elasticsearch.js index 78e27af6..5668202c 100644 --- a/src/backend/elasticsearch.js +++ b/src/backend/elasticsearch.js @@ -19,7 +19,7 @@ this.recline.Backend = this.recline.Backend || {}; // localhost:9200 with index twitter and type tweet it would be // // 
    http://localhost:9200/twitter/tweet
    - my.ElasticSearch = Backbone.Model.extend({ + my.ElasticSearch = my.Base.extend({ _getESUrl: function(dataset) { var out = dataset.get('elasticsearch_url'); if (out) return out; @@ -39,7 +39,7 @@ this.recline.Backend = this.recline.Backend || {}; dataType: 'jsonp' }); var dfd = $.Deferred(); - my.wrapInTimeout(jqxhr).done(function(schema) { + this._wrapInTimeout(jqxhr).done(function(schema) { // only one top level key in ES = the type so we can ignore it var key = _.keys(schema)[0]; var fieldData = _.map(schema[key].properties, function(dict, fieldName) { diff --git a/src/backend/gdocs.js b/src/backend/gdocs.js index ff2c6444..63ce2eb7 100644 --- a/src/backend/gdocs.js +++ b/src/backend/gdocs.js @@ -16,7 +16,7 @@ this.recline.Backend = this.recline.Backend || {}; // 'gdocs' // ); // - my.GDoc = Backbone.Model.extend({ + my.GDoc = my.Base.extend({ getUrl: function(dataset) { var url = dataset.get('url'); if (url.indexOf('feeds/list') != -1) { diff --git a/src/backend/memory.js b/src/backend/memory.js index be86c936..97b585e0 100644 --- a/src/backend/memory.js +++ b/src/backend/memory.js @@ -69,7 +69,7 @@ this.recline.Backend = this.recline.Backend || {}; // dataset.fetch(); // etc ... // - my.Memory = Backbone.Model.extend({ + my.Memory = my.Base.extend({ initialize: function() { this.datasets = {}; }, From e5e24f72debaabfc4551fd489d574677f2b18b50 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sun, 1 Apr 2012 11:11:38 +0100 Subject: [PATCH 05/11] [#52,doc,refactor][s]: move instructions re creating a backend into backend/base source docs and tidy doc page a bit. --- index.html | 67 ++++++++++----------------------------------- src/backend/base.js | 30 +++++++++++++++++++- 2 files changed, 43 insertions(+), 54 deletions(-) diff --git a/index.html b/index.html index 1f1c298a..c43582b4 100644 --- a/index.html +++ b/index.html @@ -9,9 +9,9 @@ - + - +