From e2b1b9f545a810ea5760e1d788189c0263c8f9f1 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Thu, 24 May 2012 21:53:04 +0100 Subject: [PATCH 01/22] [#73,view/timeline][m]: many improvements to timeline - now pretty functional. * reload data on query updates * attempting to guess start date and end date fields. * avoid initting multiple times * refactor to be cleaner --- src/view-timeline.js | 78 ++++++++++++++++++++++++++++++-------- test/view-timeline.test.js | 38 ++++++++++++++++++- 2 files changed, 98 insertions(+), 18 deletions(-) diff --git a/src/view-timeline.js b/src/view-timeline.js index ef9bc09e..39962807 100644 --- a/src/view-timeline.js +++ b/src/view-timeline.js @@ -12,16 +12,37 @@ my.Timeline = Backbone.View.extend({
\ ', - initialize: function() { + // These are the default (case-insensitive) names of field that are used if found. + // If not found, the user will need to define these fields on initialization + startFieldNames: ['date','startdate', 'start', 'start-date'], + endFieldNames: ['end','endDate'], + elementId: '#vmm-timeline-id', + + initialize: function(options) { var self = this; this.el = $(this.el); - this.render(); + this.timeline = new VMM.Timeline(); + this._timelineIsInitialized = false; this.bind('view:show', function() { - // set width explicitly o/w timeline goes wider that screen for some reason - self.el.find('#vmm-timeline-id').width(self.el.parent().width()); - // only call initTimeline once in DOM as Timeline uses $ internally to look up element - self.initTimeline(); + if (self._timelineIsInitialized === false) { + self._initTimeline(); + } }); + this.model.fields.bind('change', function() { + self._setupTemporalField(); + }); + this.model.currentDocuments.bind('all', function() { + self.reloadData(); + }); + var stateData = _.extend({ + startField: null, + endField: null + }, + options.state + ); + this.state = new recline.Model.ObjectState(stateData); + this._setupTemporalField(); + this.render(); }, render: function() { @@ -30,15 +51,21 @@ my.Timeline = Backbone.View.extend({ this.el.html(htmls); }, - initTimeline: function() { - var config = { - width: "300px", - height: "50%" - }; + _initTimeline: function() { + // set width explicitly o/w timeline goes wider that screen for some reason + this.el.find(this.elementId).width(this.el.parent().width()); + // only call _initTimeline once view in DOM as Timeline uses $ internally to look up element + var config = {}; var data = this._timelineJSON(); - var elementId = '#vmm-timeline-id'; - this.timeline = new VMM.Timeline(); - this.timeline.init(data, elementId, config); + this.timeline.init(data, this.elementId, config); + this._timelineIsInitialized = true + }, + + reloadData: function() { + if (this._timelineIsInitialized) { + var data = this._timelineJSON(); + this.timeline.reload(data); + } }, _timelineJSON: function() { @@ -46,14 +73,15 @@ my.Timeline = Backbone.View.extend({ var out = { 'timeline': { 'type': 'default', - 'headline': ' ', + 'headline': '', 'date': [ ] } }; this.model.currentDocuments.each(function(doc) { var tlEntry = { - "startDate": doc.get('date'), + "startDate": doc.get(self.state.get('startField')), + "endDate": doc.get(self.state.get('endField')) || null, "headline": String(doc.get(self.model.fields.models[0].id)), "text": '' }; @@ -62,6 +90,24 @@ my.Timeline = Backbone.View.extend({ } }); return out; + }, + + _setupTemporalField: function() { + this.state.set({ + startField: this._checkField(this.startFieldNames), + endField: this._checkField(this.endFieldNames) + }); + }, + + _checkField: function(possibleFieldNames) { + var modelFieldNames = this.model.fields.pluck('id'); + for (var i = 0; i < possibleFieldNames.length; i++){ + for (var j = 0; j < modelFieldNames.length; j++){ + if (modelFieldNames[j].toLowerCase() == possibleFieldNames[i].toLowerCase()) + return modelFieldNames[j]; + } + } + return null; } }); diff --git a/test/view-timeline.test.js b/test/view-timeline.test.js index 15ca17fc..3161d4e8 100644 --- a/test/view-timeline.test.js +++ b/test/view-timeline.test.js @@ -1,12 +1,46 @@ module("View - Timeline"); -test('basics', function () { +test('extract dates and timelineJSON', function () { + var dataset = recline.Backend.createDataset([ + {'Date': '2012-03-20', 'title': '1'}, + {'Date': '2012-03-25', 'title': '2'}, + ]); + var view = new recline.View.Timeline({ + model: dataset + }); + equal(view.state.get('startField'), 'Date'); + + var out = view._timelineJSON(); + var exp = { + 'timeline': { + 'type': 'default', + 'headline': '', + 'date': [ + { + 'startDate': '2012-03-20', + 'endDate': null, + 'headline': '2012-03-20', + 'text': '' + }, + { + 'startDate': '2012-03-25', + 'endDate': null, + 'headline': '2012-03-25', + 'text': '' + } + ] + } + }; + deepEqual(out, exp); +}); + +test('render etc', function () { var dataset = Fixture.getDataset(); var view = new recline.View.Timeline({ model: dataset }); $('.fixtures').append(view.el); - view.initTimeline(); + view._initTimeline(); assertPresent('.vmm-timeline', view.el); assertPresent('.timenav', view.el); assertPresent('.timenav', view.el); From c16b636a40236587dfd7e222102fb0b04fbf6224 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Thu, 24 May 2012 21:57:50 +0100 Subject: [PATCH 02/22] [doc,view][xs]: mention flash messages / notifications. --- src/view.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/view.js b/src/view.js index 3731de62..4324994e 100644 --- a/src/view.js +++ b/src/view.js @@ -71,6 +71,21 @@ // State is available not only for individual views (as described above) but // for the dataset (e.g. the current query). For an example of pulling together // state from across multiple components see `recline.View.DataExplorer`. +// +// ### Flash Messages / Notifications +// +// To send 'flash messages' or notifications the convention is that views +// should fire an event named `recline:flash` with a payload that is a +// flash object with the following attributes (all optional): +// +// * message: message to show. +// * category: warning (default), success, error +// * persist: if true alert is persistent, o/w hidden after 3s (default=false) +// * loader: if true show a loading message +// +// Objects or views wishing to bind to flash messages may then subscribe to +// these events and take some action such as displaying them to the user. For +// an example of such behaviour see the DataExplorer view. // // ### Writing your own Views // From 59e7b93b785799c1fb0665c6be537d020c757a30 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Thu, 24 May 2012 22:30:57 +0100 Subject: [PATCH 03/22] [view-graph][xs]: remove help section in sidebar editor as not needed (self-explanatory). --- css/graph.css | 13 ------------- src/view-graph.js | 16 ++-------------- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/css/graph.css b/css/graph.css index 413ac14e..d88168c4 100644 --- a/css/graph.css +++ b/css/graph.css @@ -28,14 +28,6 @@ padding-left: 0px; } -.recline-graph .editor-info { - padding-left: 4px; -} - -.recline-graph .editor-info { - cursor: pointer; -} - .recline-graph .editor form { padding-left: 4px; } @@ -44,11 +36,6 @@ width: 100%; } -.recline-graph .editor-info { - border-bottom: 1px solid #ddd; - margin-bottom: 10px; -} - .recline-graph .editor-hide-info p { display: none; } diff --git a/src/view-graph.js b/src/view-graph.js index c10c31a5..78eb12d8 100644 --- a/src/view-graph.js +++ b/src/view-graph.js @@ -27,13 +27,6 @@ my.Graph = Backbone.View.extend({ template: ' \
\ -
\ -

Help »

\ -

To create a chart select a column (group) to use as the x-axis \ - then another column (Series A) to plot against it.

\ -

You can add add \ - additional series by clicking the "Add series" button

\ -
\
\
\ \ @@ -93,8 +86,7 @@ my.Graph = Backbone.View.extend({ events: { 'change form select': 'onEditorSubmit', 'click .editor-add': '_onAddSeries', - 'click .action-remove-series': 'removeSeries', - 'click .action-toggle-help': 'toggleHelp' + 'click .action-remove-series': 'removeSeries' }, initialize: function(options) { @@ -401,11 +393,7 @@ my.Graph = Backbone.View.extend({ var $el = $(e.target); $el.parent().parent().remove(); this.onEditorSubmit(); - }, - - toggleHelp: function() { - this.el.find('.editor-info').toggleClass('editor-hide-info'); - }, + } }); })(jQuery, recline.View); From fc22e8650949e58b5df751b90e5be07e3ba14a60 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Fri, 25 May 2012 08:04:01 +0100 Subject: [PATCH 04/22] [app][xs]: micro tidy. --- app/js/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/js/app.js b/app/js/app.js index 145ec05b..7c976fcf 100755 --- a/app/js/app.js +++ b/app/js/app.js @@ -1,5 +1,5 @@ jQuery(function($) { - var app = new ExplorerApp({ + window.ReclineDataExplorer = new ExplorerApp({ el: $('.recline-app') }) }); @@ -12,7 +12,7 @@ var ExplorerApp = Backbone.View.extend({ initialize: function() { this.el = $(this.el); - this.explorer = null; + this.dataExplorer = null; this.explorerDiv = $('.data-explorer-here'); _.bindAll(this, 'viewExplorer', 'viewHome'); From 9518d2483dd3b0dfb73b8a6118f1031005739ea7 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Fri, 25 May 2012 09:01:03 +0100 Subject: [PATCH 05/22] [backend,refactor][xs]: move Backbone.sync override into recline.Model from recline.Backend.base. --- src/backend/base.js | 7 ------- src/model.js | 8 +++++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/backend/base.js b/src/backend/base.js index 94cccf4f..a8dc9f8d 100644 --- a/src/backend/base.js +++ b/src/backend/base.js @@ -7,13 +7,6 @@ this.recline = this.recline || {}; this.recline.Backend = this.recline.Backend || {}; (function($, my) { - // ## Backbone.sync - // - // Override Backbone.sync to hand off to sync function in relevant backend - Backbone.sync = function(method, model, options) { - return model.backend.sync(method, model, options); - }; - // ## recline.Backend.Base // // Base class for backends providing a template and convenience functions. diff --git a/src/model.js b/src/model.js index aca6c1e3..b99b8053 100644 --- a/src/model.js +++ b/src/model.js @@ -493,10 +493,12 @@ my.ObjectState = Backbone.Model.extend({ }); -// ## Backend registry +// ## Backbone.sync // -// Backends will register themselves by id into this registry -my.backends = {}; +// Override Backbone.sync to hand off to sync function in relevant backend +Backbone.sync = function(method, model, options) { + return model.backend.sync(method, model, options); +}; }(jQuery, this.recline.Model)); From 39c72aef134b0a26e8e90d9026dd12b7bc594ec7 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Fri, 25 May 2012 21:26:35 +0100 Subject: [PATCH 06/22] [test,refactor][xs]: move backend tests to test/backend/memory.js. --- test/backend.test.js | 154 ---------------------------------------- test/backend/memory.js | 155 +++++++++++++++++++++++++++++++++++++++++ test/index.html | 1 + 3 files changed, 156 insertions(+), 154 deletions(-) create mode 100644 test/backend/memory.js diff --git a/test/backend.test.js b/test/backend.test.js index 69686b8a..3dcce7fd 100644 --- a/test/backend.test.js +++ b/test/backend.test.js @@ -1,160 +1,6 @@ (function ($) { module("Backend"); -var memoryData = { - metadata: { - title: 'My Test Dataset' - , name: '1-my-test-dataset' - , id: 'test-dataset' - }, - fields: [{id: 'x'}, {id: 'y'}, {id: 'z'}, {id: 'country'}, {id: 'label'}], - documents: [ - {id: 0, x: 1, y: 2, z: 3, country: 'DE', label: 'first'} - , {id: 1, x: 2, y: 4, z: 6, country: 'UK', label: 'second'} - , {id: 2, x: 3, y: 6, z: 9, country: 'US', label: 'third'} - , {id: 3, x: 4, y: 8, z: 12, country: 'UK', label: 'fourth'} - , {id: 4, x: 5, y: 10, z: 15, country: 'UK', label: 'fifth'} - , {id: 5, x: 6, y: 12, z: 18, country: 'DE', label: 'sixth'} - ] -}; - -function makeBackendDataset() { - var backend = new recline.Backend.Memory(); - backend.addDataset(memoryData); - var dataset = new recline.Model.Dataset({id: memoryData.metadata.id}, backend); - return dataset; -} - -test('Memory Backend: readonly', function () { - var backend = new recline.Backend.Memory(); - equal(backend.readonly, false); -}); - -test('Memory Backend: createDataset', function () { - var dataset = recline.Backend.createDataset(memoryData.documents, memoryData.fields, memoryData.metadata); - equal(memoryData.metadata.id, dataset.id); -}); - -test('Memory Backend: createDataset 2', function () { - var dataset = recline.Backend.createDataset(memoryData.documents); - equal(dataset.fields.length, 6); - deepEqual(['id', 'x', 'y', 'z', 'country', 'label'], dataset.fields.pluck('id')); - dataset.query(); - equal(memoryData.documents.length, dataset.currentDocuments.length); -}); - -test('Memory Backend: basics', function () { - var dataset = makeBackendDataset(); - expect(3); - // convenience for tests - get the data that should get changed - var data = dataset.backend.datasets[memoryData.metadata.id]; - dataset.fetch().then(function(datasetAgain) { - equal(dataset.get('name'), data.metadata.name); - deepEqual(_.pluck(dataset.fields.toJSON(), 'id'), _.pluck(data.fields, 'id')); - equal(dataset.docCount, 6); - }); -}); - -test('Memory Backend: query', function () { - var dataset = makeBackendDataset(); - // convenience for tests - get the data that should get changed - var data = dataset.backend.datasets[memoryData.metadata.id]; - var dataset = makeBackendDataset(); - var queryObj = { - size: 4 - , from: 2 - }; - dataset.query(queryObj).then(function(documentList) { - deepEqual(data.documents[2], documentList.models[0].toJSON()); - }); -}); - -test('Memory Backend: query sort', function () { - var dataset = makeBackendDataset(); - // convenience for tests - get the data that should get changed - var data = dataset.backend.datasets[memoryData.metadata.id]; - var queryObj = { - sort: [ - {'y': {order: 'desc'}} - ] - }; - dataset.query(queryObj).then(function() { - var doc0 = dataset.currentDocuments.models[0].toJSON(); - equal(doc0.x, 6); - }); -}); - -test('Memory Backend: query string', function () { - var dataset = makeBackendDataset(); - dataset.fetch(); - dataset.query({q: 'UK'}).then(function() { - equal(dataset.currentDocuments.length, 3); - deepEqual(dataset.currentDocuments.pluck('country'), ['UK', 'UK', 'UK']); - }); - - dataset.query({q: 'UK 6'}).then(function() { - equal(dataset.currentDocuments.length, 1); - deepEqual(dataset.currentDocuments.models[0].id, 1); - }); -}); - -test('Memory Backend: filters', function () { - var dataset = makeBackendDataset(); - dataset.queryState.addTermFilter('country', 'UK'); - dataset.query().then(function() { - equal(dataset.currentDocuments.length, 3); - deepEqual(dataset.currentDocuments.pluck('country'), ['UK', 'UK', 'UK']); - }); -}); - -test('Memory Backend: facet', function () { - var dataset = makeBackendDataset(); - dataset.queryState.addFacet('country'); - dataset.query().then(function() { - equal(dataset.facets.length, 1); - var exp = [ - { - term: 'UK', - count: 3 - }, - { - term: 'DE', - count: 2 - }, - { - term: 'US', - count: 1 - } - ]; - deepEqual(dataset.facets.get('country').toJSON().terms, exp); - }); -}); - -test('Memory Backend: update and delete', function () { - var dataset = makeBackendDataset(); - // convenience for tests - get the data that should get changed - var data = dataset.backend.datasets[memoryData.metadata.id]; - dataset.query().then(function(docList) { - equal(docList.length, Math.min(100, data.documents.length)); - var doc1 = docList.models[0]; - deepEqual(doc1.toJSON(), data.documents[0]); - - // Test UPDATE - var newVal = 10; - doc1.set({x: newVal}); - doc1.save().then(function() { - equal(data.documents[0].x, newVal); - }) - - // Test Delete - doc1.destroy().then(function() { - equal(data.documents.length, 5); - equal(data.documents[0].x, memoryData.documents[1].x); - }); - }); -}); - - var dataProxyData = { "data": [ [ diff --git a/test/backend/memory.js b/test/backend/memory.js new file mode 100644 index 00000000..0563faf8 --- /dev/null +++ b/test/backend/memory.js @@ -0,0 +1,155 @@ +module("Backend Memory"); + +var memoryData = { + metadata: { + title: 'My Test Dataset' + , name: '1-my-test-dataset' + , id: 'test-dataset' + }, + fields: [{id: 'x'}, {id: 'y'}, {id: 'z'}, {id: 'country'}, {id: 'label'}], + documents: [ + {id: 0, x: 1, y: 2, z: 3, country: 'DE', label: 'first'} + , {id: 1, x: 2, y: 4, z: 6, country: 'UK', label: 'second'} + , {id: 2, x: 3, y: 6, z: 9, country: 'US', label: 'third'} + , {id: 3, x: 4, y: 8, z: 12, country: 'UK', label: 'fourth'} + , {id: 4, x: 5, y: 10, z: 15, country: 'UK', label: 'fifth'} + , {id: 5, x: 6, y: 12, z: 18, country: 'DE', label: 'sixth'} + ] +}; + +function makeBackendDataset() { + var backend = new recline.Backend.Memory(); + backend.addDataset(memoryData); + var dataset = new recline.Model.Dataset({id: memoryData.metadata.id}, backend); + return dataset; +} + +test('Memory Backend: readonly', function () { + var backend = new recline.Backend.Memory(); + equal(backend.readonly, false); +}); + +test('Memory Backend: createDataset', function () { + var dataset = recline.Backend.createDataset(memoryData.documents, memoryData.fields, memoryData.metadata); + equal(memoryData.metadata.id, dataset.id); +}); + +test('Memory Backend: createDataset 2', function () { + var dataset = recline.Backend.createDataset(memoryData.documents); + equal(dataset.fields.length, 6); + deepEqual(['id', 'x', 'y', 'z', 'country', 'label'], dataset.fields.pluck('id')); + dataset.query(); + equal(memoryData.documents.length, dataset.currentDocuments.length); +}); + +test('Memory Backend: basics', function () { + var dataset = makeBackendDataset(); + expect(3); + // convenience for tests - get the data that should get changed + var data = dataset.backend.datasets[memoryData.metadata.id]; + dataset.fetch().then(function(datasetAgain) { + equal(dataset.get('name'), data.metadata.name); + deepEqual(_.pluck(dataset.fields.toJSON(), 'id'), _.pluck(data.fields, 'id')); + equal(dataset.docCount, 6); + }); +}); + +test('Memory Backend: query', function () { + var dataset = makeBackendDataset(); + // convenience for tests - get the data that should get changed + var data = dataset.backend.datasets[memoryData.metadata.id]; + var dataset = makeBackendDataset(); + var queryObj = { + size: 4 + , from: 2 + }; + dataset.query(queryObj).then(function(documentList) { + deepEqual(data.documents[2], documentList.models[0].toJSON()); + }); +}); + +test('Memory Backend: query sort', function () { + var dataset = makeBackendDataset(); + // convenience for tests - get the data that should get changed + var data = dataset.backend.datasets[memoryData.metadata.id]; + var queryObj = { + sort: [ + {'y': {order: 'desc'}} + ] + }; + dataset.query(queryObj).then(function() { + var doc0 = dataset.currentDocuments.models[0].toJSON(); + equal(doc0.x, 6); + }); +}); + +test('Memory Backend: query string', function () { + var dataset = makeBackendDataset(); + dataset.fetch(); + dataset.query({q: 'UK'}).then(function() { + equal(dataset.currentDocuments.length, 3); + deepEqual(dataset.currentDocuments.pluck('country'), ['UK', 'UK', 'UK']); + }); + + dataset.query({q: 'UK 6'}).then(function() { + equal(dataset.currentDocuments.length, 1); + deepEqual(dataset.currentDocuments.models[0].id, 1); + }); +}); + +test('Memory Backend: filters', function () { + var dataset = makeBackendDataset(); + dataset.queryState.addTermFilter('country', 'UK'); + dataset.query().then(function() { + equal(dataset.currentDocuments.length, 3); + deepEqual(dataset.currentDocuments.pluck('country'), ['UK', 'UK', 'UK']); + }); +}); + +test('Memory Backend: facet', function () { + var dataset = makeBackendDataset(); + dataset.queryState.addFacet('country'); + dataset.query().then(function() { + equal(dataset.facets.length, 1); + var exp = [ + { + term: 'UK', + count: 3 + }, + { + term: 'DE', + count: 2 + }, + { + term: 'US', + count: 1 + } + ]; + deepEqual(dataset.facets.get('country').toJSON().terms, exp); + }); +}); + +test('Memory Backend: update and delete', function () { + var dataset = makeBackendDataset(); + // convenience for tests - get the data that should get changed + var data = dataset.backend.datasets[memoryData.metadata.id]; + dataset.query().then(function(docList) { + equal(docList.length, Math.min(100, data.documents.length)); + var doc1 = docList.models[0]; + deepEqual(doc1.toJSON(), data.documents[0]); + + // Test UPDATE + var newVal = 10; + doc1.set({x: newVal}); + doc1.save().then(function() { + equal(data.documents[0].x, newVal); + }) + + // Test Delete + doc1.destroy().then(function() { + equal(data.documents.length, 5); + equal(data.documents[0].x, memoryData.documents[1].x); + }); + }); +}); + diff --git a/test/index.html b/test/index.html index 1d2e4ee5..b1ac33c1 100644 --- a/test/index.html +++ b/test/index.html @@ -33,6 +33,7 @@ + From 23b32dff1c00861256cf65f75b1b1197d162544e Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Fri, 25 May 2012 23:42:58 +0100 Subject: [PATCH 07/22] [#128,backend/memory,misc][l]: commence major backend refactor by converting recline.Backend.Memory to module and splitting existing code into BackboneSyncer and DataWrapper. * Lots of other changes to having passing tests (note some actual functionality is likely a little broken esp around state serialization and the app) --- src/backend/localcsv.js | 2 +- src/backend/memory.js | 226 ++++++++++++++++++------------------- src/model.js | 4 +- src/view.js | 2 +- test/backend/memory.js | 168 +++++++++++++++++++++------ test/base.js | 2 +- test/model.test.js | 11 -- test/view-map.test.js | 2 +- test/view-timeline.test.js | 2 +- 9 files changed, 250 insertions(+), 169 deletions(-) diff --git a/src/backend/localcsv.js b/src/backend/localcsv.js index d1969fa2..0510feb6 100644 --- a/src/backend/localcsv.js +++ b/src/backend/localcsv.js @@ -33,7 +33,7 @@ this.recline.Backend = this.recline.Backend || {}; }); return _doc; }); - var dataset = recline.Backend.createDataset(data, fields); + var dataset = recline.Backend.Memory.createDataset(data, fields); return dataset; }; diff --git a/src/backend/memory.js b/src/backend/memory.js index 4783c20d..e117769a 100644 --- a/src/backend/memory.js +++ b/src/backend/memory.js @@ -1,5 +1,6 @@ this.recline = this.recline || {}; this.recline.Backend = this.recline.Backend || {}; +this.recline.Backend.Memory = this.recline.Backend.Memory || {}; (function($, my) { // ## createDataset @@ -14,115 +15,54 @@ this.recline.Backend = this.recline.Backend || {}; // @param metadata: (optional) dataset metadata - see recline.Model.Dataset. // If not defined (or id not provided) id will be autogenerated. my.createDataset = function(data, fields, metadata) { - if (!metadata) { - metadata = {}; - } - if (!metadata.id) { - metadata.id = String(Math.floor(Math.random() * 100000000) + 1); - } - var backend = new recline.Backend.Memory(); - var datasetInfo = { - documents: data, - metadata: metadata - }; - if (fields) { - datasetInfo.fields = fields; - } else { - if (data) { - datasetInfo.fields = _.map(data[0], function(value, key) { - return {id: key}; - }); - } - } - backend.addDataset(datasetInfo); - var dataset = new recline.Model.Dataset({id: metadata.id}, backend); + var wrapper = new my.DataWrapper(data, fields); + var syncer = new my.BackboneSyncer(); + var dataset = new recline.Model.Dataset(metadata, syncer); + dataset._dataCache = wrapper; dataset.fetch(); dataset.query(); return dataset; }; - - // ## Memory Backend - uses in-memory data + // ## Data Wrapper // - // To use it you should provide in your constructor data: - // - // * metadata (including fields array) - // * documents: list of hashes, each hash being one doc. A doc *must* have an id attribute which is unique. - // - // Example: - // - //
-  //  // Backend setup
-  //  var backend = recline.Backend.Memory();
-  //  backend.addDataset({
-  //    metadata: {
-  //      id: 'my-id',
-  //      title: 'My Title'
-  //    },
-  //    fields: [{id: 'x'}, {id: 'y'}, {id: 'z'}],
-  //    documents: [
-  //        {id: 0, x: 1, y: 2, z: 3},
-  //        {id: 1, x: 2, y: 4, z: 6}
-  //      ]
-  //  });
-  //  // later ...
-  //  var dataset = Dataset({id: 'my-id'}, 'memory');
-  //  dataset.fetch();
-  //  etc ...
-  //  
- my.Memory = my.Base.extend({ - __type__: 'memory', - readonly: false, - initialize: function() { - this.datasets = {}; - }, - addDataset: function(data) { - this.datasets[data.metadata.id] = $.extend(true, {}, data); - }, - sync: function(method, model, options) { - var self = this; - var dfd = $.Deferred(); - if (method === "read") { - if (model.__type__ == 'Dataset') { - var rawDataset = this.datasets[model.id]; - model.set(rawDataset.metadata); - model.fields.reset(rawDataset.fields); - model.docCount = rawDataset.documents.length; - dfd.resolve(model); - } - return dfd.promise(); - } else if (method === 'update') { - if (model.__type__ == 'Document') { - _.each(self.datasets[model.dataset.id].documents, function(doc, idx) { - if(doc.id === model.id) { - self.datasets[model.dataset.id].documents[idx] = model.toJSON(); - } - }); - dfd.resolve(model); - } - return dfd.promise(); - } else if (method === 'delete') { - if (model.__type__ == 'Document') { - var rawDataset = self.datasets[model.dataset.id]; - var newdocs = _.reject(rawDataset.documents, function(doc) { - return (doc.id === model.id); - }); - rawDataset.documents = newdocs; - dfd.resolve(model); - } - return dfd.promise(); - } else { - alert('Not supported: sync on Memory backend with method ' + method + ' and model ' + model); + // Turn a simple array of JS objects into a mini data-store with + // functionality like querying, faceting, updating (by ID) and deleting (by + // ID). + my.DataWrapper = function(data, fields) { + var self = this; + this.data = data; + if (fields) { + this.fields = fields; + } else { + if (data) { + this.fields = _.map(data[0], function(value, key) { + return {id: key}; + }); } - }, - query: function(model, queryObj) { - var dfd = $.Deferred(); - var out = {}; - var numRows = queryObj.size; - var start = queryObj.from; - var results = this.datasets[model.id].documents; + } + + this.update = function(doc) { + _.each(self.data, function(internalDoc, idx) { + if(doc.id === internalDoc.id) { + self.data[idx] = doc; + } + }); + }; + + this.delete = function(doc) { + var newdocs = _.reject(self.data, function(internalDoc) { + return (doc.id === internalDoc.id); + }); + this.data = newdocs; + }; + + this.query = function(queryObj) { + var numRows = queryObj.size || this.data.length; + var start = queryObj.from || 0; + var results = this.data; results = this._applyFilters(results, queryObj); - results = this._applyFreeTextQuery(model, results, queryObj); + results = this._applyFreeTextQuery(results, queryObj); // not complete sorting! _.each(queryObj.sort, function(sortObj) { var fieldName = _.keys(sortObj)[0]; @@ -131,17 +71,18 @@ this.recline.Backend = this.recline.Backend || {}; return (sortObj[fieldName].order == 'asc') ? _out : -1*_out; }); }); - out.facets = this._computeFacets(results, queryObj); var total = results.length; - resultsObj = this._docsToQueryResult(results.slice(start, start+numRows)); - _.extend(out, resultsObj); - out.total = total; - dfd.resolve(out); - return dfd.promise(); - }, + var facets = this.computeFacets(results, queryObj); + results = results.slice(start, start+numRows); + return { + total: total, + documents: results, + facets: facets + }; + }; // in place filtering - _applyFilters: function(results, queryObj) { + this._applyFilters = function(results, queryObj) { _.each(queryObj.filters, function(filter) { results = _.filter(results, function(doc) { var fieldId = _.keys(filter.term)[0]; @@ -149,17 +90,17 @@ this.recline.Backend = this.recline.Backend || {}; }); }); return results; - }, + }; // we OR across fields but AND across terms in query string - _applyFreeTextQuery: function(dataset, results, queryObj) { + this._applyFreeTextQuery = function(results, queryObj) { if (queryObj.q) { var terms = queryObj.q.split(' '); results = _.filter(results, function(rawdoc) { var matches = true; _.each(terms, function(term) { var foundmatch = false; - dataset.fields.each(function(field) { + _.each(self.fields, function(field) { var value = rawdoc[field.id]; if (value !== null) { value = value.toString(); } // TODO regexes? @@ -175,14 +116,15 @@ this.recline.Backend = this.recline.Backend || {}; }); } return results; - }, + }; - _computeFacets: function(documents, queryObj) { + this.computeFacets = function(documents, queryObj) { var facetResults = {}; if (!queryObj.facets) { - return facetsResults; + return facetResults; } _.each(queryObj.facets, function(query, facetId) { + // TODO: remove dependency on recline.Model facetResults[facetId] = new recline.Model.Facet({id: facetId}).toJSON(); facetResults[facetId].termsall = {}; }); @@ -211,7 +153,55 @@ this.recline.Backend = this.recline.Backend || {}; tmp.terms = tmp.terms.slice(0, 10); }); return facetResults; - } - }); + }; + }; + -}(jQuery, this.recline.Backend)); + // ## BackboneSyncer + // + // Provide a Backbone Sync interface to a DataWrapper data backend attached + // to a Dataset object + my.BackboneSyncer = function() { + this.sync = function(method, model, options) { + var self = this; + var dfd = $.Deferred(); + if (method === "read") { + if (model.__type__ == 'Dataset') { + model.fields.reset(model._dataCache.fields); + dfd.resolve(model); + } + return dfd.promise(); + } else if (method === 'update') { + if (model.__type__ == 'Document') { + model.dataset._dataCache.update(model.toJSON()); + dfd.resolve(model); + } + return dfd.promise(); + } else if (method === 'delete') { + if (model.__type__ == 'Document') { + model.dataset._dataCache.delete(model.toJSON()); + dfd.resolve(model); + } + return dfd.promise(); + } else { + alert('Not supported: sync on Memory backend with method ' + method + ' and model ' + model); + } + }; + + this.query = function(model, queryObj) { + var dfd = $.Deferred(); + var results = model._dataCache.query(queryObj); + var hits = _.map(results.documents, function(row) { + return { _source: row }; + }); + var out = { + total: results.total, + hits: hits, + facets: results.facets + }; + dfd.resolve(out); + return dfd.promise(); + }; + }; + +}(jQuery, this.recline.Backend.Memory)); diff --git a/src/model.js b/src/model.js index b99b8053..180fb926 100644 --- a/src/model.js +++ b/src/model.js @@ -44,6 +44,8 @@ my.Dataset = Backbone.Model.extend({ initialize: function(model, backend) { _.bindAll(this, 'query'); this.backend = backend; + this.backendType = 'memory'; + this.backendURL = null; if (typeof(backend) === 'string') { this.backend = this._backendFromString(backend); } @@ -162,7 +164,7 @@ my.Dataset.restore = function(state) { state.dataset = {url: state.url}; } if (state.backend === 'memory') { - dataset = recline.Backend.createDataset( + dataset = recline.Backend.Memory.createDataset( [{stub: 'this is a stub dataset because we do not restore memory datasets'}], [], state.dataset // metadata diff --git a/src/view.js b/src/view.js index 4324994e..82fb1525 100644 --- a/src/view.js +++ b/src/view.js @@ -362,7 +362,7 @@ my.DataExplorer = Backbone.View.extend({ var stateData = _.extend({ query: query, 'view-graph': graphState, - backend: this.model.backend.__type__, + backend: this.model.backendType, dataset: this.model.toJSON(), currentView: null, readOnly: false diff --git a/test/backend/memory.js b/test/backend/memory.js index 0563faf8..d9263d6b 100644 --- a/test/backend/memory.js +++ b/test/backend/memory.js @@ -1,4 +1,115 @@ -module("Backend Memory"); +(function ($) { + +module("Backend Memory - DataWrapper"); + +var memoryData = [ + {id: 0, x: 1, y: 2, z: 3, country: 'DE', label: 'first'} + , {id: 1, x: 2, y: 4, z: 6, country: 'UK', label: 'second'} + , {id: 2, x: 3, y: 6, z: 9, country: 'US', label: 'third'} + , {id: 3, x: 4, y: 8, z: 12, country: 'UK', label: 'fourth'} + , {id: 4, x: 5, y: 10, z: 15, country: 'UK', label: 'fifth'} + , {id: 5, x: 6, y: 12, z: 18, country: 'DE', label: 'sixth'} +]; + +var _wrapData = function() { + var dataCopy = $.extend(true, [], memoryData); + return new recline.Backend.Memory.DataWrapper(dataCopy); +} + +test('basics', function () { + var data = _wrapData(); + equal(data.fields.length, 6); + deepEqual(['id', 'x', 'y', 'z', 'country', 'label'], _.pluck(data.fields, 'id')); + equal(memoryData.length, data.data.length); +}); + +test('query', function () { + var data = _wrapData(); + var queryObj = { + size: 4 + , from: 2 + }; + var out = data.query(queryObj); + deepEqual(out.documents[0], memoryData[2]); + equal(out.documents.length, 4); + equal(out.total, 6); +}); + +test('query sort', function () { + var data = _wrapData(); + var queryObj = { + sort: [ + {'y': {order: 'desc'}} + ] + }; + var out = data.query(queryObj); + equal(out.documents[0].x, 6); +}); + +test('query string', function () { + var data = _wrapData(); + var out = data.query({q: 'UK'}); + equal(out.total, 3); + deepEqual(_.pluck(out.documents, 'country'), ['UK', 'UK', 'UK']); + + var out = data.query({q: 'UK 6'}) + equal(out.total, 1); + deepEqual(out.documents[0].id, 1); +}); + +test('filters', function () { + var data = _wrapData(); + var query = new recline.Model.Query(); + query.addTermFilter('country', 'UK'); + var out = data.query(query.toJSON()); + equal(out.total, 3); + deepEqual(_.pluck(out.documents, 'country'), ['UK', 'UK', 'UK']); +}); + +test('facet', function () { + var data = _wrapData(); + var query = new recline.Model.Query(); + query.addFacet('country'); + var out = data.computeFacets(data.data, query.toJSON()); + var exp = [ + { + term: 'UK', + count: 3 + }, + { + term: 'DE', + count: 2 + }, + { + term: 'US', + count: 1 + } + ]; + deepEqual(out['country'].terms, exp); +}); + +test('update and delete', function () { + var data = _wrapData(); + // Test UPDATE + var newVal = 10; + doc1 = $.extend(true, {}, memoryData[0]); + doc1.x = newVal; + data.update(doc1); + equal(data.data[0].x, newVal); + + // Test Delete + data.delete(doc1); + equal(data.data.length, 5); + equal(data.data[0].x, memoryData[1].x); +}); + +})(this.jQuery); + +// ====================================== + +(function ($) { + +module("Backend Memory - BackboneSyncer"); var memoryData = { metadata: { @@ -18,60 +129,48 @@ var memoryData = { }; function makeBackendDataset() { - var backend = new recline.Backend.Memory(); - backend.addDataset(memoryData); - var dataset = new recline.Model.Dataset({id: memoryData.metadata.id}, backend); + var dataset = new recline.Backend.Memory.createDataset(memoryData.documents, null, memoryData.metadata); return dataset; } -test('Memory Backend: readonly', function () { - var backend = new recline.Backend.Memory(); - equal(backend.readonly, false); -}); - -test('Memory Backend: createDataset', function () { - var dataset = recline.Backend.createDataset(memoryData.documents, memoryData.fields, memoryData.metadata); - equal(memoryData.metadata.id, dataset.id); -}); - -test('Memory Backend: createDataset 2', function () { - var dataset = recline.Backend.createDataset(memoryData.documents); +test('createDataset', function () { + var dataset = recline.Backend.Memory.createDataset(memoryData.documents); equal(dataset.fields.length, 6); deepEqual(['id', 'x', 'y', 'z', 'country', 'label'], dataset.fields.pluck('id')); dataset.query(); equal(memoryData.documents.length, dataset.currentDocuments.length); }); -test('Memory Backend: basics', function () { +test('basics', function () { var dataset = makeBackendDataset(); expect(3); // convenience for tests - get the data that should get changed - var data = dataset.backend.datasets[memoryData.metadata.id]; + var data = dataset._dataCache; dataset.fetch().then(function(datasetAgain) { - equal(dataset.get('name'), data.metadata.name); + equal(dataset.get('name'), memoryData.metadata.name); deepEqual(_.pluck(dataset.fields.toJSON(), 'id'), _.pluck(data.fields, 'id')); equal(dataset.docCount, 6); }); }); -test('Memory Backend: query', function () { +test('query', function () { var dataset = makeBackendDataset(); // convenience for tests - get the data that should get changed - var data = dataset.backend.datasets[memoryData.metadata.id]; + var data = dataset._dataCache.data; var dataset = makeBackendDataset(); var queryObj = { size: 4 , from: 2 }; dataset.query(queryObj).then(function(documentList) { - deepEqual(data.documents[2], documentList.models[0].toJSON()); + deepEqual(data[2], documentList.models[0].toJSON()); }); }); -test('Memory Backend: query sort', function () { +test('query sort', function () { var dataset = makeBackendDataset(); // convenience for tests - get the data that should get changed - var data = dataset.backend.datasets[memoryData.metadata.id]; + var data = dataset._dataCache.data; var queryObj = { sort: [ {'y': {order: 'desc'}} @@ -83,7 +182,7 @@ test('Memory Backend: query sort', function () { }); }); -test('Memory Backend: query string', function () { +test('query string', function () { var dataset = makeBackendDataset(); dataset.fetch(); dataset.query({q: 'UK'}).then(function() { @@ -97,7 +196,7 @@ test('Memory Backend: query string', function () { }); }); -test('Memory Backend: filters', function () { +test('filters', function () { var dataset = makeBackendDataset(); dataset.queryState.addTermFilter('country', 'UK'); dataset.query().then(function() { @@ -106,7 +205,7 @@ test('Memory Backend: filters', function () { }); }); -test('Memory Backend: facet', function () { +test('facet', function () { var dataset = makeBackendDataset(); dataset.queryState.addFacet('country'); dataset.query().then(function() { @@ -129,27 +228,28 @@ test('Memory Backend: facet', function () { }); }); -test('Memory Backend: update and delete', function () { +test('update and delete', function () { var dataset = makeBackendDataset(); // convenience for tests - get the data that should get changed - var data = dataset.backend.datasets[memoryData.metadata.id]; + var data = dataset._dataCache; dataset.query().then(function(docList) { - equal(docList.length, Math.min(100, data.documents.length)); + equal(docList.length, Math.min(100, data.data.length)); var doc1 = docList.models[0]; - deepEqual(doc1.toJSON(), data.documents[0]); + deepEqual(doc1.toJSON(), data.data[0]); // Test UPDATE var newVal = 10; doc1.set({x: newVal}); doc1.save().then(function() { - equal(data.documents[0].x, newVal); + equal(data.data[0].x, newVal); }) // Test Delete doc1.destroy().then(function() { - equal(data.documents.length, 5); - equal(data.documents[0].x, memoryData.documents[1].x); + equal(data.data.length, 5); + equal(data.data[0].x, memoryData.documents[1].x); }); }); }); +})(this.jQuery); diff --git a/test/base.js b/test/base.js index f80977ac..7163131b 100644 --- a/test/base.js +++ b/test/base.js @@ -19,7 +19,7 @@ var Fixture = { {id: 4, date: '2011-05-04', x: 5, y: 10, z: 15, country: 'UK', label: 'fifth', lat:51.58, lon:0}, {id: 5, date: '2011-06-02', x: 6, y: 12, z: 18, country: 'DE', label: 'sixth', lat:51.04, lon:7.9} ]; - var dataset = recline.Backend.createDataset(documents, fields); + var dataset = recline.Backend.Memory.createDataset(documents, fields); return dataset; } }; diff --git a/test/model.test.js b/test/model.test.js index b7a0e3d8..fef946e7 100644 --- a/test/model.test.js +++ b/test/model.test.js @@ -125,17 +125,6 @@ test('Dataset _prepareQuery', function () { deepEqual(out, exp); }); -test('Dataset _backendFromString', function () { - var dataset = new recline.Model.Dataset(); - - var out = dataset._backendFromString('recline.Backend.Memory'); - equal(out.__type__, 'memory'); - - var out = dataset._backendFromString('dataproxy'); - equal(out.__type__, 'dataproxy'); -}); - - // ================================= // Query diff --git a/test/view-map.test.js b/test/view-map.test.js index cf709a53..adc12940 100644 --- a/test/view-map.test.js +++ b/test/view-map.test.js @@ -16,7 +16,7 @@ var GeoJSONFixture = { {id: 1, x: 2, y: 4, z: 6, geom: {type:"Point",coordinates:[13.40,52.35]}}, {id: 2, x: 3, y: 6, z: 9, geom: {type:"LineString",coordinates:[[100.0, 0.0],[101.0, 1.0]]}} ]; - var dataset = recline.Backend.createDataset(documents, fields); + var dataset = recline.Backend.Memory.createDataset(documents, fields); return dataset; } }; diff --git a/test/view-timeline.test.js b/test/view-timeline.test.js index 3161d4e8..562fc982 100644 --- a/test/view-timeline.test.js +++ b/test/view-timeline.test.js @@ -1,7 +1,7 @@ module("View - Timeline"); test('extract dates and timelineJSON', function () { - var dataset = recline.Backend.createDataset([ + var dataset = recline.Backend.Memory.createDataset([ {'Date': '2012-03-20', 'title': '1'}, {'Date': '2012-03-25', 'title': '2'}, ]); From 1bc8c770982559c4dd6227f220f88fc699fa33f9 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 15:53:59 +0100 Subject: [PATCH 08/22] [#128,backend/elasticsearch][m]: rework elasticsearch model to new cleaner setup. --- src/backend/base.js | 26 +++ src/backend/elasticsearch.js | 246 +++++++++++++++++------------ test/backend.elasticsearch.test.js | 112 +++++++++++-- 3 files changed, 273 insertions(+), 111 deletions(-) diff --git a/src/backend/base.js b/src/backend/base.js index a8dc9f8d..9c027d87 100644 --- a/src/backend/base.js +++ b/src/backend/base.js @@ -155,5 +155,31 @@ this.recline.Backend = this.recline.Backend || {}; } }); + // ### makeRequest + // + // Just $.ajax but in any headers in the 'headers' attribute of this + // Backend instance. Example: + // + //
+  // var jqxhr = this._makeRequest({
+  //   url: the-url
+  // });
+  // 
+ my.makeRequest = function(data, headers) { + var extras = {}; + if (headers) { + extras = { + beforeSend: function(req) { + _.each(headers, function(value, key) { + req.setRequestHeader(key, value); + }); + } + }; + } + var data = _.extend(extras, data); + return $.ajax(data); + }; + + }(jQuery, this.recline.Backend)); diff --git a/src/backend/elasticsearch.js b/src/backend/elasticsearch.js index 546c0c8a..d035d138 100644 --- a/src/backend/elasticsearch.js +++ b/src/backend/elasticsearch.js @@ -1,80 +1,44 @@ this.recline = this.recline || {}; this.recline.Backend = this.recline.Backend || {}; +this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {}; (function($, my) { - // ## ElasticSearch Backend + // ## ElasticSearch Wrapper // - // Connecting to [ElasticSearch](http://www.elasticsearch.org/). - // - // Usage: - // - //
-  // var backend = new recline.Backend.ElasticSearch({
-  //   // optional as can also be provided by Dataset/Document
-  //   url: {url to ElasticSearch endpoint i.e. ES 'type/table' url - more info below}
-  //   // optional
-  //   headers: {dict of headers to add to each request}
-  // });
-  //
-  // @param {String} url: url for ElasticSearch type/table, e.g. for ES running
+  // Connecting to [ElasticSearch](http://www.elasticsearch.org/) endpoints.
+  // @param {String} endpoint: url for ElasticSearch type/table, e.g. for ES running
   // on localhost:9200 with index // twitter and type tweet it would be:
   // 
   // 
http://localhost:9200/twitter/tweet
+ // @param {Object} options: set of options such as: // - // This url is optional since the ES endpoint url may be specified on the the - // dataset (and on a Document by the document having a dataset attribute) by - // having one of the following (see also `_getESUrl` function): - // - //
-  // elasticsearch_url
-  // webstore_url
-  // url
-  // 
- my.ElasticSearch = my.Base.extend({ - __type__: 'elasticsearch', - readonly: false, - sync: function(method, model, options) { - var self = this; - if (method === "read") { - if (model.__type__ == 'Dataset') { - var schemaUrl = self._getESUrl(model) + '/_mapping'; - var jqxhr = this._makeRequest({ - url: schemaUrl, - dataType: 'jsonp' - }); - var dfd = $.Deferred(); - this._wrapInTimeout(jqxhr).done(function(schema) { - // only one top level key in ES = the type so we can ignore it - var key = _.keys(schema)[0]; - var fieldData = _.map(schema[key].properties, function(dict, fieldName) { - dict.id = fieldName; - return dict; - }); - model.fields.reset(fieldData); - dfd.resolve(model, jqxhr); - }) - .fail(function(arguments) { - dfd.reject(arguments); - }); - return dfd.promise(); - } else if (model.__type__ == 'Document') { - var base = this._getESUrl(model.dataset) + '/' + model.id; - return this._makeRequest({ - url: base, - dataType: 'json' - }); - } - } else if (method === 'update') { - if (model.__type__ == 'Document') { - return this.upsert(model.toJSON(), this._getESUrl(model.dataset)); - } - } else if (method === 'delete') { - if (model.__type__ == 'Document') { - var url = this._getESUrl(model.dataset); - return this.delete(model.id, url); - } - } - }, + // * headers - {dict of headers to add to each request} + my.Wrapper = function(endpoint, options) { + var self = this; + this.endpoint = endpoint; + this.options = _.extend({ + dataType: 'json' + }, + options); + + this.mapping = function() { + var schemaUrl = self.endpoint + '/_mapping'; + var jqxhr = recline.Backend.makeRequest({ + url: schemaUrl, + dataType: this.options.dataType + }); + return jqxhr; + }; + + this.get = function(id, error, success) { + var base = this.endpoint + '/' + id; + return recline.Backend.makeRequest({ + url: base, + dataType: 'json', + error: error, + success: success + }); + } // ### upsert // @@ -83,19 +47,21 @@ this.recline.Backend = this.recline.Backend || {}; // @param {Object} doc an object to insert to the index. // @param {string} url (optional) url for ElasticSearch endpoint (if not // defined called this._getESUrl() - upsert: function(doc, url) { + this.upsert = function(doc, error, success) { var data = JSON.stringify(doc); - url = url ? url : this._getESUrl(); + url = this.endpoint; if (doc.id) { url += '/' + doc.id; } - return this._makeRequest({ + return recline.Backend.makeRequest({ url: url, type: 'POST', data: data, - dataType: 'json' + dataType: 'json', + error: error, + success: success }); - }, + }; // ### delete // @@ -104,32 +70,18 @@ this.recline.Backend = this.recline.Backend || {}; // @param {Object} id id of object to delete // @param {string} url (optional) url for ElasticSearch endpoint (if not // provided called this._getESUrl() - delete: function(id, url) { - url = url ? url : this._getESUrl(); + this.delete = function(id, error, success) { + url = this.endpoint; url += '/' + id; - return this._makeRequest({ + return recline.Backend.makeRequest({ url: url, type: 'DELETE', dataType: 'json' }); - }, + }; - // ### _getESUrl - // - // get url to ElasticSearch endpoint (see above) - _getESUrl: function(dataset) { - if (dataset) { - var out = dataset.get('elasticsearch_url'); - if (out) return out; - out = dataset.get('webstore_url'); - if (out) return out; - out = dataset.get('url'); - return out; - } - return this.get('url'); - }, - _normalizeQuery: function(queryObj) { - var out = queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj); + this._normalizeQuery = function(queryObj) { + var out = queryObj && queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj); if (out.q !== undefined && out.q.trim() === '') { delete out.q; } @@ -159,17 +111,107 @@ this.recline.Backend = this.recline.Backend || {}; delete out.filters; } return out; - }, - query: function(model, queryObj) { + }; + + this.query = function(queryObj) { var queryNormalized = this._normalizeQuery(queryObj); var data = {source: JSON.stringify(queryNormalized)}; - var base = this._getESUrl(model); - var jqxhr = this._makeRequest({ - url: base + '/_search', + var url = this.endpoint + '/_search'; + var jqxhr = recline.Backend.makeRequest({ + url: url, data: data, - dataType: 'jsonp' + dataType: this.options.dataType }); + return jqxhr; + } + }; + + // ## ElasticSearch Backbone Backend + // + // Backbone connector for an ES backend. + // + // Usage: + // + // var backend = new recline.Backend.ElasticSearch(options); + // + // `options` are passed through to Wrapper + my.Backbone = function(options) { + var self = this; + var esOptions = options; + this.__type__ = 'elasticsearch'; + + // ### sync + // + // Backbone sync implementation for this backend. + // + // URL of ElasticSearch endpoint to use must be specified on the + // dataset (and on a Document by the document having a dataset + // attribute) by the dataset having one of the following data + // attributes (see also `_getESUrl` function): + // + //
+    // elasticsearch_url
+    // webstore_url
+    // url
+    // 
+ this.sync = function(method, model, options) { + if (model.__type__ == 'Dataset') { + var endpoint = self._getESUrl(model); + } else { + var endpoint = self._getESUrl(model.dataset); + } + var es = new my.Wrapper(endpoint, esOptions); + if (method === "read") { + if (model.__type__ == 'Dataset') { + var dfd = $.Deferred(); + es.mapping().done(function(schema) { + // only one top level key in ES = the type so we can ignore it + var key = _.keys(schema)[0]; + var fieldData = _.map(schema[key].properties, function(dict, fieldName) { + dict.id = fieldName; + return dict; + }); + model.fields.reset(fieldData); + dfd.resolve(model); + }) + .fail(function(arguments) { + dfd.reject(arguments); + }); + return dfd.promise(); + } else if (model.__type__ == 'Document') { + return es.get(model.dataset.id); + } + } else if (method === 'update') { + if (model.__type__ == 'Document') { + return es.upsert(model.toJSON()); + } + } else if (method === 'delete') { + if (model.__type__ == 'Document') { + return es.delete(model.id); + } + } + }; + + // ### _getESUrl + // + // get url to ElasticSearch endpoint (see above) + this._getESUrl = function(dataset) { + if (dataset) { + var out = dataset.get('elasticsearch_url'); + if (out) return out; + out = dataset.get('webstore_url'); + if (out) return out; + out = dataset.get('url'); + return out; + } + return this.get('url'); + }; + + this.query = function(model, queryObj) { var dfd = $.Deferred(); + var url = this._getESUrl(model); + var es = new my.Wrapper(url, esOptions); + var jqxhr = es.query(queryObj); // TODO: fail case jqxhr.done(function(results) { _.each(results.hits.hits, function(hit) { @@ -183,8 +225,8 @@ this.recline.Backend = this.recline.Backend || {}; dfd.resolve(results.hits); }); return dfd.promise(); - } - }); + }; + }; -}(jQuery, this.recline.Backend)); +}(jQuery, this.recline.Backend.ElasticSearch)); diff --git a/test/backend.elasticsearch.test.js b/test/backend.elasticsearch.test.js index 9a0206fc..c0a19831 100644 --- a/test/backend.elasticsearch.test.js +++ b/test/backend.elasticsearch.test.js @@ -1,15 +1,18 @@ (function ($) { -module("Backend ElasticSearch"); +module("Backend ElasticSearch - Wrapper"); + +test("queryNormalize", function() { + var backend = new recline.Backend.ElasticSearch.Wrapper(); + var in_ = new recline.Model.Query(); + var out = backend._normalizeQuery(in_); + equal(out.size, 100); -test("ElasticSearch queryNormalize", function() { - var backend = new recline.Backend.ElasticSearch(); var in_ = new recline.Model.Query(); in_.set({q: ''}); var out = backend._normalizeQuery(in_); equal(out.q, undefined); deepEqual(out.query.match_all, {}); - var backend = new recline.Backend.ElasticSearch(); var in_ = new recline.Model.Query().toJSON(); in_.q = ''; var out = backend._normalizeQuery(in_); @@ -107,8 +110,99 @@ var sample_data = { "took": 2 }; -test("ElasticSearch query", function() { - var backend = new recline.Backend.ElasticSearch(); +test("query", function() { + var backend = new recline.Backend.ElasticSearch.Wrapper('https://localhost:9200/my-es-db/my-es-type'); + + var stub = sinon.stub($, 'ajax', function(options) { + if (options.url.indexOf('_mapping') != -1) { + return { + done: function(callback) { + callback(mapping_data); + return this; + }, + fail: function() { + return this; + } + } + } else { + return { + done: function(callback) { + callback(sample_data); + }, + fail: function() { + } + } + } + }); + + backend.mapping().done(function(data) { + var fields = _.keys(data.note.properties); + deepEqual(['_created', '_last_modified', 'end', 'owner', 'start', 'title'], fields); + }); + + backend.query().done(function(queryResult) { + equal(3, queryResult.hits.total); + equal(3, queryResult.hits.hits.length); + equal('Note 1', queryResult.hits.hits[0]._source['title']); + start(); + }); + $.ajax.restore(); +}); + +test("write", function() { + var url = 'http://localhost:9200/recline-test/es-write'; + var backend = new recline.Backend.ElasticSearch.Wrapper(url); + stop(); + + var id = parseInt(Math.random()*100000000).toString(); + var doc = { + id: id, + title: 'my title' + }; + var jqxhr = backend.upsert(doc); + jqxhr.done(function(data) { + ok(data.ok); + equal(data._id, id); + equal(data._type, 'es-write'); + equal(data._version, 1); + + // update + doc.title = 'new title'; + var jqxhr = backend.upsert(doc); + jqxhr.done(function(data) { + equal(data._version, 2); + + // delete + var jqxhr = backend.delete(doc.id); + jqxhr.done(function(data) { + ok(data.ok); + doc = null; + + // try to get ... + var jqxhr = backend.get(id); + jqxhr.done(function(data) { + // should not be here + ok(false, 'Should have got 404'); + }).error(function(error) { + equal(error.status, 404); + start(); + }); + }); + }); + }).fail(function(error) { + console.log(error); + ok(false, 'Basic request failed - is ElasticSearch running locally on port 9200 (required for this test!)'); + start(); + }); +}); + + +// ================================================== + +module("Backend ElasticSearch - Backbone"); + +test("query", function() { + var backend = new recline.Backend.ElasticSearch.Backbone(); var dataset = new recline.Model.Dataset({ url: 'https://localhost:9200/my-es-db/my-es-type' }, @@ -137,7 +231,7 @@ test("ElasticSearch query", function() { } }); - dataset.fetch().then(function(dataset) { + dataset.fetch().done(function(dataset) { deepEqual(['_created', '_last_modified', 'end', 'owner', 'start', 'title'], _.pluck(dataset.fields.toJSON(), 'id')); dataset.query().then(function(docList) { equal(3, dataset.docCount); @@ -149,8 +243,8 @@ test("ElasticSearch query", function() { $.ajax.restore(); }); -test("ElasticSearch write", function() { - var backend = new recline.Backend.ElasticSearch(); +test("write", function() { + var backend = new recline.Backend.ElasticSearch.Backbone(); var dataset = new recline.Model.Dataset({ url: 'http://localhost:9200/recline-test/es-write' }, From fae1496eb2431081e7bdbb850bb2fd768d74e016 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 16:06:14 +0100 Subject: [PATCH 09/22] [test,refactor][xs]: rename memory test to memory.test.js. --- test/backend/{memory.js => memory.test.js} | 2 +- test/index.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename test/backend/{memory.js => memory.test.js} (99%) diff --git a/test/backend/memory.js b/test/backend/memory.test.js similarity index 99% rename from test/backend/memory.js rename to test/backend/memory.test.js index d9263d6b..20ce290a 100644 --- a/test/backend/memory.js +++ b/test/backend/memory.test.js @@ -109,7 +109,7 @@ test('update and delete', function () { (function ($) { -module("Backend Memory - BackboneSyncer"); +module("Backend Memory - Backbone"); var memoryData = { metadata: { diff --git a/test/index.html b/test/index.html index b1ac33c1..a97c37b6 100644 --- a/test/index.html +++ b/test/index.html @@ -33,7 +33,7 @@ - + From ad7fc1a0294f5356eb2689956b9428cebd361156 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 16:07:05 +0100 Subject: [PATCH 10/22] [#128,backend][xs]: minor tweaks to memory backend to follow style in ES - all tests passing again (though stuff still broken in app!). --- src/backend/memory.js | 12 ++++++------ src/view.js | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/backend/memory.js b/src/backend/memory.js index e117769a..f7fa8afb 100644 --- a/src/backend/memory.js +++ b/src/backend/memory.js @@ -16,8 +16,8 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {}; // If not defined (or id not provided) id will be autogenerated. my.createDataset = function(data, fields, metadata) { var wrapper = new my.DataWrapper(data, fields); - var syncer = new my.BackboneSyncer(); - var dataset = new recline.Model.Dataset(metadata, syncer); + var backend = new my.Backbone(); + var dataset = new recline.Model.Dataset(metadata, backend); dataset._dataCache = wrapper; dataset.fetch(); dataset.query(); @@ -157,11 +157,11 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {}; }; - // ## BackboneSyncer + // ## Backbone // - // Provide a Backbone Sync interface to a DataWrapper data backend attached - // to a Dataset object - my.BackboneSyncer = function() { + // Backbone connector for memory store attached to a Dataset object + my.Backbone = function() { + this.__type__ = 'memory'; this.sync = function(method, model, options) { var self = this; var dfd = $.Deferred(); diff --git a/src/view.js b/src/view.js index 82fb1525..4324994e 100644 --- a/src/view.js +++ b/src/view.js @@ -362,7 +362,7 @@ my.DataExplorer = Backbone.View.extend({ var stateData = _.extend({ query: query, 'view-graph': graphState, - backend: this.model.backendType, + backend: this.model.backend.__type__, dataset: this.model.toJSON(), currentView: null, readOnly: false From 5890007c590daf8d1f4e990bba7b3a2e00c9dc6b Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 16:20:18 +0100 Subject: [PATCH 11/22] [#128,backend/dataproxy][s]: refactor to new setup. --- src/backend/dataproxy.js | 68 +++++++++++++++++++++++++++++----------- test/backend.test.js | 6 ++-- 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/backend/dataproxy.js b/src/backend/dataproxy.js index 16db3db6..a2731f00 100644 --- a/src/backend/dataproxy.js +++ b/src/backend/dataproxy.js @@ -1,12 +1,14 @@ this.recline = this.recline || {}; this.recline.Backend = this.recline.Backend || {}; +this.recline.Backend.DataProxy = this.recline.Backend.DataProxy || {}; (function($, my) { // ## DataProxy Backend // // For connecting to [DataProxy-s](http://github.com/okfn/dataproxy). // - // When initializing the DataProxy backend you can set the following attributes: + // When initializing the DataProxy backend you can set the following + // attributes in the options object: // // * dataproxy: {url-to-proxy} (optional). Defaults to http://jsonpdataproxy.appspot.com // @@ -16,14 +18,14 @@ this.recline.Backend = this.recline.Backend || {}; // * format: (optional) csv | xls (defaults to csv if not specified) // // Note that this is a **read-only** backend. - my.DataProxy = my.Base.extend({ - __type__: 'dataproxy', - readonly: true, - defaults: { - dataproxy_url: 'http://jsonpdataproxy.appspot.com' - }, - sync: function(method, model, options) { - var self = this; + my.Backbone = function(options) { + var self = this; + this.__type__ = 'dataproxy'; + this.readonly = true; + + this.dataproxy_url = options && options.dataproxy_url ? options.dataproxy_url : 'http://jsonpdataproxy.appspot.com'; + + this.sync = function(method, model, options) { if (method === "read") { if (model.__type__ == 'Dataset') { // Do nothing as we will get fields in query step (and no metadata to @@ -35,22 +37,22 @@ this.recline.Backend = this.recline.Backend || {}; } else { alert('This backend only supports read operations'); } - }, - query: function(dataset, queryObj) { + }; + + this.query = function(dataset, queryObj) { var self = this; - var base = this.get('dataproxy_url'); var data = { url: dataset.get('url'), 'max-results': queryObj.size, type: dataset.get('format') }; var jqxhr = $.ajax({ - url: base, + url: this.dataproxy_url, data: data, dataType: 'jsonp' }); var dfd = $.Deferred(); - this._wrapInTimeout(jqxhr).done(function(results) { + _wrapInTimeout(jqxhr).done(function(results) { if (results.error) { dfd.reject(results.error); } @@ -65,13 +67,43 @@ this.recline.Backend = this.recline.Backend || {}; }); return tmp; }); - dfd.resolve(self._docsToQueryResult(_out)); + dfd.resolve({ + total: null, + hits: _.map(_out, function(row) { + return { _source: row }; + }) + }); }) .fail(function(arguments) { dfd.reject(arguments); }); return dfd.promise(); - } - }); + }; + }; -}(jQuery, this.recline.Backend)); + // ## _wrapInTimeout + // + // Convenience method providing a crude way to catch backend errors on JSONP calls. + // Many of backends use JSONP and so will not get error messages and this is + // a crude way to catch those errors. + var _wrapInTimeout = function(ourFunction) { + var dfd = $.Deferred(); + var timeout = 5000; + var timer = setTimeout(function() { + dfd.reject({ + message: 'Request Error: Backend did not respond after ' + (timeout / 1000) + ' seconds' + }); + }, timeout); + ourFunction.done(function(arguments) { + clearTimeout(timer); + dfd.resolve(arguments); + }) + .fail(function(arguments) { + clearTimeout(timer); + dfd.reject(arguments); + }) + ; + return dfd.promise(); + } + +}(jQuery, this.recline.Backend.DataProxy)); diff --git a/test/backend.test.js b/test/backend.test.js index 3dcce7fd..134b2d62 100644 --- a/test/backend.test.js +++ b/test/backend.test.js @@ -67,8 +67,10 @@ var dataProxyData = { test('DataProxy Backend', function() { // needed only if not stubbing // stop(); - var backend = new recline.Backend.DataProxy(); - ok(backend.readonly, false); + var backend = new recline.Backend.DataProxy.Backbone(); + console.log(backend.readonly); + ok(backend.readonly); + equal(backend.__type__, 'dataproxy'); var dataset = new recline.Model.Dataset({ url: 'http://webstore.thedatahub.org/rufuspollock/gold_prices/data.csv' From e9d1b8a55a700b8e77e8bf7126225b805eccd762 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 17:46:51 +0100 Subject: [PATCH 12/22] [#128,backend/gdoc][s]: refactor gdocs to standard backend setup. --- src/backend/gdocs.js | 214 ++++++++++++++++++++++++------------------- test/backend.test.js | 13 +-- 2 files changed, 123 insertions(+), 104 deletions(-) diff --git a/src/backend/gdocs.js b/src/backend/gdocs.js index c9b5b551..0dd0e669 100644 --- a/src/backend/gdocs.js +++ b/src/backend/gdocs.js @@ -1,7 +1,9 @@ this.recline = this.recline || {}; this.recline.Backend = this.recline.Backend || {}; +this.recline.Backend.GDocs = this.recline.Backend.GDocs || {}; (function($, my) { + // ## Google spreadsheet backend // // Connect to Google Docs spreadsheet. @@ -16,50 +18,25 @@ this.recline.Backend = this.recline.Backend || {}; // 'gdocs' // ); //
- my.GDoc = my.Base.extend({ - __type__: 'gdoc', - readonly: true, - getUrl: function(dataset) { - var url = dataset.get('url'); - if (url.indexOf('feeds/list') != -1) { - return url; - } else { - // https://docs.google.com/spreadsheet/ccc?key=XXXX#gid=0 - var regex = /.*spreadsheet\/ccc?.*key=([^#?&+]+).*/; - var matches = url.match(regex); - if (matches) { - var key = matches[1]; - var worksheet = 1; - var out = 'https://spreadsheets.google.com/feeds/list/' + key + '/' + worksheet + '/public/values?alt=json'; - return out; - } else { - alert('Failed to extract gdocs key from ' + url); - } - } - }, - sync: function(method, model, options) { + my.Backbone = function() { + this.__type__ = 'gdocs'; + this.readonly = true; + + this.sync = function(method, model, options) { var self = this; if (method === "read") { var dfd = $.Deferred(); - var dataset = model; - - var url = this.getUrl(model); - - $.getJSON(url, function(d) { - result = self.gdocsToJavascript(d); - model.fields.reset(_.map(result.field, function(fieldId) { - return {id: fieldId}; - }) - ); + loadData(model.get('url')).done(function(result) { + model.fields.reset(result.fields); // cache data onto dataset (we have loaded whole gdoc it seems!) model._dataCache = result.data; dfd.resolve(model); }); return dfd.promise(); } - }, + }; - query: function(dataset, queryObj) { + this.query = function(dataset, queryObj) { var dfd = $.Deferred(); var fields = _.pluck(dataset.fields.toJSON(), 'id'); @@ -72,70 +49,115 @@ this.recline.Backend = this.recline.Backend || {}; }); return obj; }); - dfd.resolve(this._docsToQueryResult(objs)); + var out = { + total: objs.length, + hits: _.map(objs, function(row) { + return { _source: row } + }) + } + dfd.resolve(out); return dfd; - }, - gdocsToJavascript: function(gdocsSpreadsheet) { - /* - :options: (optional) optional argument dictionary: - columnsToUse: list of columns to use (specified by field names) - colTypes: dictionary (with column names as keys) specifying types (e.g. range, percent for use in conversion). - :return: tabular data object (hash with keys: field and data). + }; + }; - Issues: seems google docs return columns in rows in random order and not even sure whether consistent across rows. - */ - var options = {}; - if (arguments.length > 1) { - options = arguments[1]; - } - var results = { - 'field': [], - 'data': [] - }; - // default is no special info on type of columns - var colTypes = {}; - if (options.colTypes) { - colTypes = options.colTypes; - } - // either extract column headings from spreadsheet directly, or used supplied ones - if (options.columnsToUse) { - // columns set to subset supplied - results.field = options.columnsToUse; - } else { - // set columns to use to be all available - if (gdocsSpreadsheet.feed.entry.length > 0) { - for (var k in gdocsSpreadsheet.feed.entry[0]) { - if (k.substr(0, 3) == 'gsx') { - var col = k.substr(4); - results.field.push(col); - } - } - } - } - - // converts non numberical values that should be numerical (22.3%[string] -> 0.223[float]) - var rep = /^([\d\.\-]+)\%$/; - $.each(gdocsSpreadsheet.feed.entry, function (i, entry) { - var row = []; - for (var k in results.field) { - var col = results.field[k]; - var _keyname = 'gsx$' + col; - var value = entry[_keyname]['$t']; - // if labelled as % and value contains %, convert - if (colTypes[col] == 'percent') { - if (rep.test(value)) { - var value2 = rep.exec(value); - var value3 = parseFloat(value2); - value = value3 / 100; - } - } - row.push(value); - } - results.data.push(row); - }); - return results; + // ## loadData + // + // loadData from a google docs URL + // + // @return object with two attributes + // + // * fields: array of objects + // * data: array of arrays + var loadData = function(url) { + var dfd = $.Deferred(); + var url = my.getSpreadsheetAPIUrl(url); + var out = { + fields: [], + data: [] } - }); + $.getJSON(url, function(d) { + result = my.parseData(d); + result.fields = _.map(result.fields, function(fieldId) { + return {id: fieldId}; + }); + dfd.resolve(result); + }); + return dfd.promise(); + }; -}(jQuery, this.recline.Backend)); + // ## parseData + // + // Parse data from Google Docs API into a reasonable form + // + // :options: (optional) optional argument dictionary: + // columnsToUse: list of columns to use (specified by field names) + // colTypes: dictionary (with column names as keys) specifying types (e.g. range, percent for use in conversion). + // :return: tabular data object (hash with keys: field and data). + // + // Issues: seems google docs return columns in rows in random order and not even sure whether consistent across rows. + my.parseData = function(gdocsSpreadsheet) { + var options = {}; + if (arguments.length > 1) { + options = arguments[1]; + } + var results = { + 'fields': [], + 'data': [] + }; + // default is no special info on type of columns + var colTypes = {}; + if (options.colTypes) { + colTypes = options.colTypes; + } + if (gdocsSpreadsheet.feed.entry.length > 0) { + for (var k in gdocsSpreadsheet.feed.entry[0]) { + if (k.substr(0, 3) == 'gsx') { + var col = k.substr(4); + results.fields.push(col); + } + } + } + + // converts non numberical values that should be numerical (22.3%[string] -> 0.223[float]) + var rep = /^([\d\.\-]+)\%$/; + $.each(gdocsSpreadsheet.feed.entry, function (i, entry) { + var row = []; + for (var k in results.fields) { + var col = results.fields[k]; + var _keyname = 'gsx$' + col; + var value = entry[_keyname]['$t']; + // if labelled as % and value contains %, convert + if (colTypes[col] == 'percent') { + if (rep.test(value)) { + var value2 = rep.exec(value); + var value3 = parseFloat(value2); + value = value3 / 100; + } + } + row.push(value); + } + results.data.push(row); + }); + return results; + }; + + // Convenience function to get GDocs JSON API Url from standard URL + my.getSpreadsheetAPIUrl = function(url) { + if (url.indexOf('feeds/list') != -1) { + return url; + } else { + // https://docs.google.com/spreadsheet/ccc?key=XXXX#gid=0 + var regex = /.*spreadsheet\/ccc?.*key=([^#?&+]+).*/; + var matches = url.match(regex); + if (matches) { + var key = matches[1]; + var worksheet = 1; + var out = 'https://spreadsheets.google.com/feeds/list/' + key + '/' + worksheet + '/public/values?alt=json'; + return out; + } else { + alert('Failed to extract gdocs key from ' + url); + } + } + }; +}(jQuery, this.recline.Backend.GDocs)); diff --git a/test/backend.test.js b/test/backend.test.js index 134b2d62..e65e69bd 100644 --- a/test/backend.test.js +++ b/test/backend.test.js @@ -273,8 +273,8 @@ var sample_gdocs_spreadsheet_data = { "encoding": "UTF-8" } -test("GDoc Backend", function() { - var backend = new recline.Backend.GDoc(); +test("GDocs Backend", function() { + var backend = new recline.Backend.GDocs.Backbone(); var dataset = new recline.Model.Dataset({ url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json' }, @@ -301,13 +301,10 @@ test("GDoc Backend", function() { $.getJSON.restore(); }); -test("GDoc Backend.getUrl", function() { +test("GDocs Backend.getUrl", function() { var key = 'Abc_dajkdkjdafkj'; - var dataset = new recline.Model.Dataset({ - url: 'https://docs.google.com/spreadsheet/ccc?key=' + key + '#gid=0' - }); - var backend = new recline.Backend.GDoc(); - var out = backend.getUrl(dataset); + var url = 'https://docs.google.com/spreadsheet/ccc?key=' + key + '#gid=0' + var out = recline.Backend.GDocs.getSpreadsheetAPIUrl(url); var exp = 'https://spreadsheets.google.com/feeds/list/' + key + '/1/public/values?alt=json' equal(exp, out); }); From feaaf786393fb9bb62e69a47d660d7909785a9fd Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 18:07:36 +0100 Subject: [PATCH 13/22] [#128,backend/gdocs][s]: minor refactoring to make cleaner and have it actually work. --- src/backend/gdocs.js | 30 +++++++++++++++++++----------- test/backend.test.js | 12 +++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/backend/gdocs.js b/src/backend/gdocs.js index 0dd0e669..c9449916 100644 --- a/src/backend/gdocs.js +++ b/src/backend/gdocs.js @@ -19,6 +19,7 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {}; // ); // my.Backbone = function() { + var self = this; this.__type__ = 'gdocs'; this.readonly = true; @@ -26,23 +27,31 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {}; var self = this; if (method === "read") { var dfd = $.Deferred(); - loadData(model.get('url')).done(function(result) { - model.fields.reset(result.fields); - // cache data onto dataset (we have loaded whole gdoc it seems!) - model._dataCache = result.data; - dfd.resolve(model); - }); + dfd.resolve(model); return dfd.promise(); } }; this.query = function(dataset, queryObj) { var dfd = $.Deferred(); - var fields = _.pluck(dataset.fields.toJSON(), 'id'); + if (dataset._dataCache) { + dfd.resolve(dataset._dataCache); + } else { + loadData(dataset.get('url')).done(function(result) { + dataset.fields.reset(result.fields); + // cache data onto dataset (we have loaded whole gdoc it seems!) + dataset._dataCache = self._formatResults(dataset, result.data); + dfd.resolve(dataset._dataCache); + }); + } + return dfd.promise(); + }; + this._formatResults = function(dataset, data) { + var fields = _.pluck(dataset.fields.toJSON(), 'id'); // zip the fields with the data rows to produce js objs // TODO: factor this out as a common method with other backends - var objs = _.map(dataset._dataCache, function (d) { + var objs = _.map(data, function (d) { var obj = {}; _.each(_.zip(fields, d), function (x) { obj[x[0]] = x[1]; @@ -55,8 +64,7 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {}; return { _source: row } }) } - dfd.resolve(out); - return dfd; + return out; }; }; @@ -78,7 +86,7 @@ this.recline.Backend.GDocs = this.recline.Backend.GDocs || {}; $.getJSON(url, function(d) { result = my.parseData(d); result.fields = _.map(result.fields, function(fieldId) { - return {id: fieldId}; + return {id: fieldId}; }); dfd.resolve(result); }); diff --git a/test/backend.test.js b/test/backend.test.js index e65e69bd..332ab6b9 100644 --- a/test/backend.test.js +++ b/test/backend.test.js @@ -68,7 +68,6 @@ test('DataProxy Backend', function() { // needed only if not stubbing // stop(); var backend = new recline.Backend.DataProxy.Backbone(); - console.log(backend.readonly); ok(backend.readonly); equal(backend.__type__, 'dataproxy'); @@ -288,15 +287,10 @@ test("GDocs Backend", function() { } }); - dataset.fetch().then(function(dataset) { + dataset.query().then(function(docList) { deepEqual(['column-2', 'column-1'], _.pluck(dataset.fields.toJSON(), 'id')); - //equal(null, dataset.docCount) - dataset.query().then(function(docList) { - equal(3, docList.length); - equal("A", docList.models[0].get('column-1')); - // needed only if not stubbing - start(); - }); + equal(3, docList.length); + equal("A", docList.models[0].get('column-1')); }); $.getJSON.restore(); }); From 72ed877ae249ec18658ce307eb4a63958e6bbfb6 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 18:26:47 +0100 Subject: [PATCH 14/22] [#128,misc][s]: final wiring up to have everything working again -- fixes #128. --- app/index.html | 2 +- src/backend/base.js | 262 ++++++++++++++++---------------------------- src/model.js | 4 +- 3 files changed, 97 insertions(+), 171 deletions(-) diff --git a/app/index.html b/app/index.html index 6d9d78c2..6f52251e 100644 --- a/app/index.html +++ b/app/index.html @@ -157,7 +157,7 @@
diff --git a/src/backend/base.js b/src/backend/base.js index 9c027d87..2758f51e 100644 --- a/src/backend/base.js +++ b/src/backend/base.js @@ -6,180 +6,108 @@ this.recline = this.recline || {}; this.recline.Backend = this.recline.Backend || {}; -(function($, my) { - // ## recline.Backend.Base +// ## recline.Backend.Base +// +// Exemplar 'class' for backends showing what a base class would look like. +this.recline.Backend.Base = function() { + // ### __type__ // - // Base class for backends providing a template and convenience functions. - // You do not have to inherit from this class but even when not it does - // provide guidance on the functions you must implement. + // 'type' of this backend. This should be either the class path for this + // object as a string (e.g. recline.Backend.Memory) or for Backends within + // recline.Backend module it may be their class name. // - // Note also that while this (and other Backends) are implemented as Backbone models this is just a convenience. - my.Base = Backbone.Model.extend({ - // ### __type__ - // - // 'type' of this backend. This should be either the class path for this - // object as a string (e.g. recline.Backend.Memory) or for Backends within - // recline.Backend module it may be their class name. - // - // This value is used as an identifier for this backend when initializing - // backends (see recline.Model.Dataset.initialize). - __type__: 'base', + // This value is used as an identifier for this backend when initializing + // backends (see recline.Model.Dataset.initialize). + this.__type__ = 'base'; + // ### readonly + // + // Class level attribute indicating that this backend is read-only (that + // is, cannot be written to). + this.readonly = true; - // ### readonly - // - // Class level attribute indicating that this backend is read-only (that - // is, cannot be written to). - readonly: true, - - // ### sync - // - // An implementation of Backbone.sync that will be used to override - // Backbone.sync on operations for Datasets and Documents which are using this backend. - // - // For read-only implementations you will need only to implement read method - // for Dataset models (and even this can be a null operation). The read method - // should return relevant metadata for the Dataset. We do not require read support - // for Documents because they are loaded in bulk by the query method. - // - // For backends supporting write operations you must implement update and delete support for Document objects. - // - // All code paths should return an object conforming to the jquery promise API. - sync: function(method, model, options) { - }, - - // ### query - // - // Query the backend for documents returning them in bulk. This method will - // be used by the Dataset.query method to search the backend for documents, - // retrieving the results in bulk. - // - // @param {recline.model.Dataset} model: Dataset model. - // - // @param {Object} queryObj: object describing a query (usually produced by - // using recline.Model.Query and calling toJSON on it). - // - // The structure of data in the Query object or - // Hash should follow that defined in issue 34. - // (Of course, if you are writing your own backend, and hence - // have control over the interpretation of the query object, you - // can use whatever structure you like). - // - // @returns {Promise} promise API object. The promise resolve method will - // be called on query completion with a QueryResult object. - // - // A QueryResult has the following structure (modelled closely on - // ElasticSearch - see this issue for more - // details): - // - //
-    // {
-    //   total: // (required) total number of results (can be null)
-    //   hits: [ // (required) one entry for each result document
-    //     {
-    //        _score:   // (optional) match score for document
-    //        _type: // (optional) document type
-    //        _source: // (required) document/row object
-    //     } 
-    //   ],
-    //   facets: { // (optional) 
-    //     // facet results (as per )
-    //   }
-    // }
-    // 
- query: function(model, queryObj) { - }, - - // ### _makeRequest - // - // Just $.ajax but in any headers in the 'headers' attribute of this - // Backend instance. Example: - // - //
-    // var jqxhr = this._makeRequest({
-    //   url: the-url
-    // });
-    // 
- _makeRequest: function(data) { - var headers = this.get('headers'); - var extras = {}; - if (headers) { - extras = { - beforeSend: function(req) { - _.each(headers, function(value, key) { - req.setRequestHeader(key, value); - }); - } - }; - } - var data = _.extend(extras, data); - return $.ajax(data); - }, - - // convenience method to convert simple set of documents / rows to a QueryResult - _docsToQueryResult: function(rows) { - var hits = _.map(rows, function(row) { - return { _source: row }; - }); - return { - total: null, - hits: hits - }; - }, - - // ## _wrapInTimeout - // - // Convenience method providing a crude way to catch backend errors on JSONP calls. - // Many of backends use JSONP and so will not get error messages and this is - // a crude way to catch those errors. - _wrapInTimeout: function(ourFunction) { - var dfd = $.Deferred(); - var timeout = 5000; - var timer = setTimeout(function() { - dfd.reject({ - message: 'Request Error: Backend did not respond after ' + (timeout / 1000) + ' seconds' - }); - }, timeout); - ourFunction.done(function(arguments) { - clearTimeout(timer); - dfd.resolve(arguments); - }) - .fail(function(arguments) { - clearTimeout(timer); - dfd.reject(arguments); - }) - ; - return dfd.promise(); - } - }); - - // ### makeRequest + // ### sync + // + // An implementation of Backbone.sync that will be used to override + // Backbone.sync on operations for Datasets and Documents which are using this backend. + // + // For read-only implementations you will need only to implement read method + // for Dataset models (and even this can be a null operation). The read method + // should return relevant metadata for the Dataset. We do not require read support + // for Documents because they are loaded in bulk by the query method. + // + // For backends supporting write operations you must implement update and delete support for Document objects. + // + // All code paths should return an object conforming to the jquery promise API. + this.sync = function(method, model, options) { + }, + + // ### query + // + // Query the backend for documents returning them in bulk. This method will + // be used by the Dataset.query method to search the backend for documents, + // retrieving the results in bulk. + // + // @param {recline.model.Dataset} model: Dataset model. + // + // @param {Object} queryObj: object describing a query (usually produced by + // using recline.Model.Query and calling toJSON on it). + // + // The structure of data in the Query object or + // Hash should follow that defined in issue 34. + // (Of course, if you are writing your own backend, and hence + // have control over the interpretation of the query object, you + // can use whatever structure you like). + // + // @returns {Promise} promise API object. The promise resolve method will + // be called on query completion with a QueryResult object. // - // Just $.ajax but in any headers in the 'headers' attribute of this - // Backend instance. Example: + // A QueryResult has the following structure (modelled closely on + // ElasticSearch - see this issue for more + // details): // //
-  // var jqxhr = this._makeRequest({
-  //   url: the-url
-  // });
+  // {
+  //   total: // (required) total number of results (can be null)
+  //   hits: [ // (required) one entry for each result document
+  //     {
+  //        _score:   // (optional) match score for document
+  //        _type: // (optional) document type
+  //        _source: // (required) document/row object
+  //     } 
+  //   ],
+  //   facets: { // (optional) 
+  //     // facet results (as per )
+  //   }
+  // }
   // 
- my.makeRequest = function(data, headers) { - var extras = {}; - if (headers) { - extras = { - beforeSend: function(req) { - _.each(headers, function(value, key) { - req.setRequestHeader(key, value); - }); - } - }; - } - var data = _.extend(extras, data); - return $.ajax(data); - }; + this.query = function(model, queryObj) {} +}; - -}(jQuery, this.recline.Backend)); +// ### makeRequest +// +// Just $.ajax but in any headers in the 'headers' attribute of this +// Backend instance. Example: +// +//
+// var jqxhr = this._makeRequest({
+//   url: the-url
+// });
+// 
+this.recline.Backend.makeRequest = function(data, headers) { + var extras = {}; + if (headers) { + extras = { + beforeSend: function(req) { + _.each(headers, function(value, key) { + req.setRequestHeader(key, value); + }); + } + }; + } + var data = _.extend(extras, data); + return $.ajax(data); +}; diff --git a/src/model.js b/src/model.js index 180fb926..b1ec858b 100644 --- a/src/model.js +++ b/src/model.js @@ -44,8 +44,6 @@ my.Dataset = Backbone.Model.extend({ initialize: function(model, backend) { _.bindAll(this, 'query'); this.backend = backend; - this.backendType = 'memory'; - this.backendURL = null; if (typeof(backend) === 'string') { this.backend = this._backendFromString(backend); } @@ -135,7 +133,7 @@ my.Dataset = Backbone.Model.extend({ if (recline && recline.Backend) { _.each(_.keys(recline.Backend), function(name) { if (name.toLowerCase() === backendString.toLowerCase()) { - backend = new recline.Backend[name](); + backend = new recline.Backend[name].Backbone(); } }); } From ed4f315a97995056c2fd9bf81268063259b76956 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 19:00:39 +0100 Subject: [PATCH 15/22] [#128,backend/csv][s]: rename localcsv to csv and make into a module recline.Backend.CSv. --- app/js/app.js | 2 +- src/backend/{localcsv.js => csv.js} | 13 ++++++++++--- .../csv.test.js} | 10 +++++----- 3 files changed, 16 insertions(+), 9 deletions(-) rename src/backend/{localcsv.js => csv.js} (93%) rename test/{backend.localcsv.test.js => backend/csv.test.js} (79%) diff --git a/app/js/app.js b/app/js/app.js index 7c976fcf..b3ea3f22 100755 --- a/app/js/app.js +++ b/app/js/app.js @@ -154,7 +154,7 @@ var ExplorerApp = Backbone.View.extend({ delimiter : $form.find('input[name="delimiter"]').val(), encoding : $form.find('input[name="encoding"]').val() }; - recline.Backend.loadFromCSVFile(file, function(dataset) { + recline.Backend.CSV.load(file, function(dataset) { self.createExplorer(dataset) }, options diff --git a/src/backend/localcsv.js b/src/backend/csv.js similarity index 93% rename from src/backend/localcsv.js rename to src/backend/csv.js index 0510feb6..a680ef17 100644 --- a/src/backend/localcsv.js +++ b/src/backend/csv.js @@ -1,8 +1,15 @@ this.recline = this.recline || {}; this.recline.Backend = this.recline.Backend || {}; +this.recline.Backend.CSV = this.recline.Backend.CSV || {}; -(function($, my) { - my.loadFromCSVFile = function(file, callback, options) { +(function(my) { + // ## load + // + // Load data from a CSV file referenced in an HTMl5 file object returning the + // dataset in the callback + // + // @param options as for parseCSV below + my.load = function(file, callback, options) { var encoding = options.encoding || 'UTF-8'; var metadata = { @@ -168,4 +175,4 @@ this.recline.Backend = this.recline.Backend || {}; } -}(jQuery, this.recline.Backend)); +}(this.recline.Backend.CSV)); diff --git a/test/backend.localcsv.test.js b/test/backend/csv.test.js similarity index 79% rename from test/backend.localcsv.test.js rename to test/backend/csv.test.js index 8325e549..e47c46bf 100644 --- a/test/backend.localcsv.test.js +++ b/test/backend/csv.test.js @@ -6,7 +6,7 @@ test("parseCSV", function() { '"Xyz ""ABC"" O\'Brien",11:35\n' + '"Other, AN",12:35\n'; - var array = recline.Backend.parseCSV(csv); + var array = recline.Backend.CSV.parseCSV(csv); var exp = [ ['Jones, Jay', 10], ['Xyz "ABC" O\'Brien', '11:35' ], @@ -17,14 +17,14 @@ test("parseCSV", function() { var csv = '"Jones, Jay", 10\n' + '"Xyz ""ABC"" O\'Brien", 11:35\n' + '"Other, AN", 12:35\n'; - var array = recline.Backend.parseCSV(csv, {trim : true}); + var array = recline.Backend.CSV.parseCSV(csv, {trim : true}); deepEqual(exp, array); var csv = 'Name, Value\n' + '"Jones, Jay", 10\n' + '"Xyz ""ABC"" O\'Brien", 11:35\n' + '"Other, AN", 12:35\n'; - var dataset = recline.Backend.csvToDataset(csv); + var dataset = recline.Backend.CSV.csvToDataset(csv); dataset.query(); equal(dataset.currentDocuments.length, 3); }); @@ -34,7 +34,7 @@ test("parseCSVsemicolon", function() { '"Xyz ""ABC"" O\'Brien";11:35\n' + '"Other; AN";12:35\n'; - var array = recline.Backend.parseCSV(csv, {separator : ';'}); + var array = recline.Backend.CSV.parseCSV(csv, {separator : ';'}); var exp = [ ['Jones; Jay', 10], ['Xyz "ABC" O\'Brien', '11:35' ], @@ -49,7 +49,7 @@ test("parseCSVdelimiter", function() { "'Xyz \"ABC\" O''Brien',11:35\n" + "'Other; AN',12:35\n"; - var array = recline.Backend.parseCSV(csv, {delimiter:"'"}); + var array = recline.Backend.CSV.parseCSV(csv, {delimiter:"'"}); var exp = [ ["Jones, Jay", 10], ["Xyz \"ABC\" O'Brien", "11:35" ], From 13d1a9e0bde5fe41a4fd5a9f085b50c70c0bc7e3 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 22:51:50 +0100 Subject: [PATCH 16/22] [#128,state][s]: move to simpler state serialization for backend/dataset in which we just store url attribute of dataset (if present). --- README.md | 5 +++++ src/model.js | 10 +++++----- src/view.js | 2 +- test/view.test.js | 4 +++- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bcd93e3a..be7e8ce6 100755 --- a/README.md +++ b/README.md @@ -31,6 +31,11 @@ Running the tests by opening `test/index.html` in your browser. In progress. +Possible breaking changes: + +* State only stores backend (name) and dataset url (in url field) rather than entire dataset object +* Backends heavily reorganized + ### v0.4 - April 26th 2012 [23 closed issues](https://github.com/okfn/recline/issues?milestone=2&page=1&state=closed) including: diff --git a/src/model.js b/src/model.js index b1ec858b..9d4c690a 100644 --- a/src/model.js +++ b/src/model.js @@ -156,11 +156,8 @@ my.Dataset = Backbone.Model.extend({ // ... // } my.Dataset.restore = function(state) { - // hack-y - restoring a memory dataset does not mean much ... var dataset = null; - if (state.url && !state.dataset) { - state.dataset = {url: state.url}; - } + // hack-y - restoring a memory dataset does not mean much ... if (state.backend === 'memory') { dataset = recline.Backend.Memory.createDataset( [{stub: 'this is a stub dataset because we do not restore memory datasets'}], @@ -168,8 +165,11 @@ my.Dataset.restore = function(state) { state.dataset // metadata ); } else { + var datasetInfo = { + url: state.url + }; dataset = new recline.Model.Dataset( - state.dataset, + datasetInfo, state.backend ); } diff --git a/src/view.js b/src/view.js index 4324994e..89b74edf 100644 --- a/src/view.js +++ b/src/view.js @@ -363,7 +363,7 @@ my.DataExplorer = Backbone.View.extend({ query: query, 'view-graph': graphState, backend: this.model.backend.__type__, - dataset: this.model.toJSON(), + url: this.model.get('url'), currentView: null, readOnly: false }, diff --git a/test/view.test.js b/test/view.test.js index 4dec5d89..b00b7f1d 100644 --- a/test/view.test.js +++ b/test/view.test.js @@ -19,6 +19,8 @@ test('get State', function () { var $el = $('
'); $('.fixtures .data-explorer-here').append($el); var dataset = Fixture.getDataset(); + var url = 'xyz'; + dataset.set({url: url}); var explorer = new recline.View.DataExplorer({ model: dataset, el: $el @@ -31,7 +33,7 @@ test('get State', function () { deepEqual(state.get('view-grid').hiddenFields, []); deepEqual(state.get('view-graph').group, null); equal(state.get('backend'), 'memory'); - ok(state.get('dataset').id !== null); + ok(state.get('url') === url); $el.remove(); }); From 92543a76eec36c53b5a22f05269fc32e2b630aaa Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 26 May 2012 22:55:01 +0100 Subject: [PATCH 17/22] [#128,backend/elasticsearch][s]: standardize on url field on dataset to hold url for ES backend. * also ensure reasonable docs for all methods --- src/backend/elasticsearch.js | 72 ++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/src/backend/elasticsearch.js b/src/backend/elasticsearch.js index d035d138..a73d4140 100644 --- a/src/backend/elasticsearch.js +++ b/src/backend/elasticsearch.js @@ -10,9 +10,11 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {}; // on localhost:9200 with index // twitter and type tweet it would be: // //
http://localhost:9200/twitter/tweet
+ // // @param {Object} options: set of options such as: // // * headers - {dict of headers to add to each request} + // * dataType: dataType for AJAx requests e.g. set to jsonp to make jsonp requests (default is json requests) my.Wrapper = function(endpoint, options) { var self = this; this.endpoint = endpoint; @@ -21,6 +23,11 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {}; }, options); + // ### mapping + // + // Get ES mapping for this type/table + // + // @return promise compatible deferred object. this.mapping = function() { var schemaUrl = self.endpoint + '/_mapping'; var jqxhr = recline.Backend.makeRequest({ @@ -30,24 +37,26 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {}; return jqxhr; }; - this.get = function(id, error, success) { + // ### get + // + // Get document corresponding to specified id + // + // @return promise compatible deferred object. + this.get = function(id) { var base = this.endpoint + '/' + id; return recline.Backend.makeRequest({ url: base, - dataType: 'json', - error: error, - success: success + dataType: 'json' }); - } + }; // ### upsert // // create / update a document to ElasticSearch backend // // @param {Object} doc an object to insert to the index. - // @param {string} url (optional) url for ElasticSearch endpoint (if not - // defined called this._getESUrl() - this.upsert = function(doc, error, success) { + // @return deferred supporting promise API + this.upsert = function(doc) { var data = JSON.stringify(doc); url = this.endpoint; if (doc.id) { @@ -57,9 +66,7 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {}; url: url, type: 'POST', data: data, - dataType: 'json', - error: error, - success: success + dataType: 'json' }); }; @@ -68,9 +75,8 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {}; // Delete a document from the ElasticSearch backend. // // @param {Object} id id of object to delete - // @param {string} url (optional) url for ElasticSearch endpoint (if not - // provided called this._getESUrl() - this.delete = function(id, error, success) { + // @return deferred supporting promise API + this.delete = function(id) { url = this.endpoint; url += '/' + id; return recline.Backend.makeRequest({ @@ -113,6 +119,9 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {}; return out; }; + // ### query + // + // @return deferred supporting promise API this.query = function(queryObj) { var queryNormalized = this._normalizeQuery(queryObj); var data = {source: JSON.stringify(queryNormalized)}; @@ -144,21 +153,14 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {}; // // Backbone sync implementation for this backend. // - // URL of ElasticSearch endpoint to use must be specified on the - // dataset (and on a Document by the document having a dataset - // attribute) by the dataset having one of the following data - // attributes (see also `_getESUrl` function): - // - //
-    // elasticsearch_url
-    // webstore_url
-    // url
-    // 
+ // URL of ElasticSearch endpoint to use must be specified on the dataset + // (and on a Document via its dataset attribute) by the dataset having a + // url attribute. this.sync = function(method, model, options) { if (model.__type__ == 'Dataset') { - var endpoint = self._getESUrl(model); + var endpoint = model.get('url'); } else { - var endpoint = self._getESUrl(model.dataset); + var endpoint = model.dataset.get('url'); } var es = new my.Wrapper(endpoint, esOptions); if (method === "read") { @@ -192,24 +194,12 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {}; } }; - // ### _getESUrl + // ### query // - // get url to ElasticSearch endpoint (see above) - this._getESUrl = function(dataset) { - if (dataset) { - var out = dataset.get('elasticsearch_url'); - if (out) return out; - out = dataset.get('webstore_url'); - if (out) return out; - out = dataset.get('url'); - return out; - } - return this.get('url'); - }; - + // query the ES backend this.query = function(model, queryObj) { var dfd = $.Deferred(); - var url = this._getESUrl(model); + var url = model.get('url'); var es = new my.Wrapper(url, esOptions); var jqxhr = es.query(queryObj); // TODO: fail case From bc8e47c6cf1caa3a18c0ce021a65b36a3859a396 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sun, 27 May 2012 01:25:00 +0100 Subject: [PATCH 18/22] [#128,test][xs]: file renames in test/index.html to match csv rename in ed4f315a97995056c2fd9bf81268063259b76956. --- test/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/index.html b/test/index.html index a97c37b6..04dd88a4 100644 --- a/test/index.html +++ b/test/index.html @@ -30,12 +30,12 @@ - + - + From c6d4116cba7666d59fe73290cf7cad395f86925b Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sun, 27 May 2012 01:27:26 +0100 Subject: [PATCH 19/22] [#79,refactor][s]: switch from (old) jquery.mustache to latest mustache.js - fixes #79. * This mustache has support for nested values e.g. {{sub.x.y}} * Specifically mustache.js as of Fri Feb 24 09:58:31 2012 +0100 2f135e2e15dcc3c61385212261faaf00597bae10 --- app/index.html | 2 +- app/js/app.js | 2 +- src/view-graph.js | 4 +- src/view-grid.js | 8 +- src/view-map.js | 2 +- src/view-timeline.js | 2 +- src/view-transform-dialog.js | 4 +- src/view.js | 10 +- test/index.html | 2 +- vendor/jquery.mustache.js | 346 ----------------- vendor/mustache/0.5.0-dev/mustache.js | 536 ++++++++++++++++++++++++++ 11 files changed, 554 insertions(+), 364 deletions(-) delete mode 100755 vendor/jquery.mustache.js create mode 100644 vendor/mustache/0.5.0-dev/mustache.js diff --git a/app/index.html b/app/index.html index 6f52251e..818703a0 100644 --- a/app/index.html +++ b/app/index.html @@ -35,7 +35,7 @@ - + diff --git a/app/js/app.js b/app/js/app.js index b3ea3f22..f77375a0 100755 --- a/app/js/app.js +++ b/app/js/app.js @@ -102,7 +102,7 @@ var ExplorerApp = Backbone.View.extend({ function makeEmbedLink(state) { var link = self.makePermaLink(state); link = link + '&embed=true'; - var out = $.mustache('', {link: link}); + var out = Mustache.render('', {link: link}); return out; } explorer.state.bind('change', function() { diff --git a/src/view-graph.js b/src/view-graph.js index 78eb12d8..6b5374c4 100644 --- a/src/view-graph.js +++ b/src/view-graph.js @@ -120,7 +120,7 @@ my.Graph = Backbone.View.extend({ render: function() { var self = this; var tmplData = this.model.toTemplateJSON(); - var htmls = $.mustache(this.template, tmplData); + var htmls = Mustache.render(this.template, tmplData); $(this.el).html(htmls); this.$graph = this.el.find('.panel.graph'); @@ -375,7 +375,7 @@ my.Graph = Backbone.View.extend({ seriesName: String.fromCharCode(idx + 64 + 1), }, this.model.toTemplateJSON()); - var htmls = $.mustache(this.templateSeriesEditor, data); + var htmls = Mustache.render(this.templateSeriesEditor, data); this.el.find('.editor-series-group').append(htmls); return this; }, diff --git a/src/view-grid.js b/src/view-grid.js index ab8a2f39..fc9400b9 100644 --- a/src/view-grid.js +++ b/src/view-grid.js @@ -53,7 +53,7 @@ my.Grid = Backbone.View.extend({ {{#columns}} \
  • Show column: {{.}}
  • \ {{/columns}}'; - var tmp = $.mustache(tmpl, {'columns': this.state.get('hiddenFields')}); + var tmp = Mustache.render(tmpl, {'columns': this.state.get('hiddenFields')}); this.el.find('.root-header-menu .dropdown-menu').html(tmp); }, @@ -211,7 +211,7 @@ my.Grid = Backbone.View.extend({ field.set({width: width}); } }); - var htmls = $.mustache(this.template, this.toTemplateJSON()); + var htmls = Mustache.render(this.template, this.toTemplateJSON()); this.el.html(htmls); this.model.currentDocuments.forEach(function(doc) { var tr = $(''); @@ -308,7 +308,7 @@ my.GridRow = Backbone.View.extend({ render: function() { this.el.attr('data-id', this.model.id); - var html = $.mustache(this.template, this.toTemplateJSON()); + var html = Mustache.render(this.template, this.toTemplateJSON()); $(this.el).html(html); return this; }, @@ -336,7 +336,7 @@ my.GridRow = Backbone.View.extend({ $(e.target).addClass("hidden"); var cell = $(e.target).siblings('.data-table-cell-value'); cell.data("previousContents", cell.text()); - var templated = $.mustache(this.cellEditorTemplate, {value: cell.text()}); + var templated = Mustache.render(this.cellEditorTemplate, {value: cell.text()}); cell.html(templated); }, diff --git a/src/view-map.js b/src/view-map.js index 60be7bc4..412ab2c7 100644 --- a/src/view-map.js +++ b/src/view-map.js @@ -161,7 +161,7 @@ my.Map = Backbone.View.extend({ var self = this; - htmls = $.mustache(this.template, this.model.toTemplateJSON()); + htmls = Mustache.render(this.template, this.model.toTemplateJSON()); $(this.el).html(htmls); this.$map = this.el.find('.panel.map'); diff --git a/src/view-timeline.js b/src/view-timeline.js index 39962807..e3faca04 100644 --- a/src/view-timeline.js +++ b/src/view-timeline.js @@ -47,7 +47,7 @@ my.Timeline = Backbone.View.extend({ render: function() { var tmplData = {}; - var htmls = $.mustache(this.template, tmplData); + var htmls = Mustache.render(this.template, tmplData); this.el.html(htmls); }, diff --git a/src/view-transform-dialog.js b/src/view-transform-dialog.js index 6c69766b..e4bee9ae 100644 --- a/src/view-transform-dialog.js +++ b/src/view-transform-dialog.js @@ -76,7 +76,7 @@ my.ColumnTransform = Backbone.View.extend({ }, render: function() { - var htmls = $.mustache(this.template, + var htmls = Mustache.render(this.template, {name: this.state.currentColumn} ); this.el.html(htmls); @@ -163,7 +163,7 @@ my.ColumnTransform = Backbone.View.extend({ }); var previewData = costco.previewTransform(docs, editFunc, self.state.currentColumn); var $el = self.el.find('.expression-preview-container'); - var templated = $.mustache(self.editPreviewTemplate, {rows: previewData.slice(0,4)}); + var templated = Mustache.render(self.editPreviewTemplate, {rows: previewData.slice(0,4)}); $el.html(templated); } else { errors.text(editFunc.errorMessage); diff --git a/src/view.js b/src/view.js index 89b74edf..c49b6592 100644 --- a/src/view.js +++ b/src/view.js @@ -286,7 +286,7 @@ my.DataExplorer = Backbone.View.extend({ render: function() { var tmplData = this.model.toTemplateJSON(); tmplData.views = this.pageViews; - var template = $.mustache(this.template, tmplData); + var template = Mustache.render(this.template, tmplData); $(this.el).html(template); var $dataViewContainer = this.el.find('.data-view-container'); _.each(this.pageViews, function(view, pageName) { @@ -431,7 +431,7 @@ my.DataExplorer = Backbone.View.extend({ {{message}} \
    '; } - var _templated = $($.mustache(_template, tmplData)); + var _templated = $(Mustache.render(_template, tmplData)); _templated = $(_templated).appendTo($('.recline-data-explorer .alert-messages')); if (!flash.persist) { setTimeout(function() { @@ -516,7 +516,7 @@ my.QueryEditor = Backbone.View.extend({ render: function() { var tmplData = this.model.toJSON(); tmplData.to = this.model.get('from') + this.model.get('size'); - var templated = $.mustache(this.template, tmplData); + var templated = Mustache.render(this.template, tmplData); this.el.html(templated); } }); @@ -584,7 +584,7 @@ my.FilterEditor = Backbone.View.extend({ value: filter.term[fieldId] }; }); - var out = $.mustache(this.template, tmplData); + var out = Mustache.render(this.template, tmplData); this.el.html(out); // are there actually any facets to show? if (this.model.get('filters').length > 0) { @@ -669,7 +669,7 @@ my.FacetViewer = Backbone.View.extend({ } return facet; }); - var templated = $.mustache(this.template, tmplData); + var templated = Mustache.render(this.template, tmplData); this.el.html(templated); // are there actually any facets to show? if (this.model.facets.length > 0) { diff --git a/test/index.html b/test/index.html index 04dd88a4..b5dac4c5 100644 --- a/test/index.html +++ b/test/index.html @@ -12,7 +12,7 @@ - + diff --git a/vendor/jquery.mustache.js b/vendor/jquery.mustache.js deleted file mode 100755 index 5aa67def..00000000 --- a/vendor/jquery.mustache.js +++ /dev/null @@ -1,346 +0,0 @@ -/* -Shameless port of a shameless port -@defunkt => @janl => @aq - -See http://github.com/defunkt/mustache for more info. -*/ - -;(function($) { - -/* - mustache.js — Logic-less templates in JavaScript - - See http://mustache.github.com/ for more info. -*/ - -var Mustache = function() { - var Renderer = function() {}; - - Renderer.prototype = { - otag: "{{", - ctag: "}}", - pragmas: {}, - buffer: [], - pragmas_implemented: { - "IMPLICIT-ITERATOR": true - }, - context: {}, - - render: function(template, context, partials, in_recursion) { - // reset buffer & set context - if(!in_recursion) { - this.context = context; - this.buffer = []; // TODO: make this non-lazy - } - - // fail fast - if(!this.includes("", template)) { - if(in_recursion) { - return template; - } else { - this.send(template); - return; - } - } - - template = this.render_pragmas(template); - var html = this.render_section(template, context, partials); - if(in_recursion) { - return this.render_tags(html, context, partials, in_recursion); - } - - this.render_tags(html, context, partials, in_recursion); - }, - - /* - Sends parsed lines - */ - send: function(line) { - if(line != "") { - this.buffer.push(line); - } - }, - - /* - Looks for %PRAGMAS - */ - render_pragmas: function(template) { - // no pragmas - if(!this.includes("%", template)) { - return template; - } - - var that = this; - var regex = new RegExp(this.otag + "%([\\w-]+) ?([\\w]+=[\\w]+)?" + - this.ctag); - return template.replace(regex, function(match, pragma, options) { - if(!that.pragmas_implemented[pragma]) { - throw({message: - "This implementation of mustache doesn't understand the '" + - pragma + "' pragma"}); - } - that.pragmas[pragma] = {}; - if(options) { - var opts = options.split("="); - that.pragmas[pragma][opts[0]] = opts[1]; - } - return ""; - // ignore unknown pragmas silently - }); - }, - - /* - Tries to find a partial in the curent scope and render it - */ - render_partial: function(name, context, partials) { - name = this.trim(name); - if(!partials || partials[name] === undefined) { - throw({message: "unknown_partial '" + name + "'"}); - } - if(typeof(context[name]) != "object") { - return this.render(partials[name], context, partials, true); - } - return this.render(partials[name], context[name], partials, true); - }, - - /* - Renders inverted (^) and normal (#) sections - */ - render_section: function(template, context, partials) { - if(!this.includes("#", template) && !this.includes("^", template)) { - return template; - } - - var that = this; - // CSW - Added "+?" so it finds the tighest bound, not the widest - var regex = new RegExp(this.otag + "(\\^|\\#)\\s*(.+)\\s*" + this.ctag + - "\n*([\\s\\S]+?)" + this.otag + "\\/\\s*\\2\\s*" + this.ctag + - "\\s*", "mg"); - - // for each {{#foo}}{{/foo}} section do... - return template.replace(regex, function(match, type, name, content) { - var value = that.find(name, context); - if(type == "^") { // inverted section - if(!value || that.is_array(value) && value.length === 0) { - // false or empty list, render it - return that.render(content, context, partials, true); - } else { - return ""; - } - } else if(type == "#") { // normal section - if(that.is_array(value)) { // Enumerable, Let's loop! - return that.map(value, function(row) { - return that.render(content, that.create_context(row), - partials, true); - }).join(""); - } else if(that.is_object(value)) { // Object, Use it as subcontext! - return that.render(content, that.create_context(value), - partials, true); - } else if(typeof value === "function") { - // higher order section - return value.call(context, content, function(text) { - return that.render(text, context, partials, true); - }); - } else if(value) { // boolean section - return that.render(content, context, partials, true); - } else { - return ""; - } - } - }); - }, - - /* - Replace {{foo}} and friends with values from our view - */ - render_tags: function(template, context, partials, in_recursion) { - // tit for tat - var that = this; - - var new_regex = function() { - return new RegExp(that.otag + "(=|!|>|\\{|%)?([^\\/#\\^]+?)\\1?" + - that.ctag + "+", "g"); - }; - - var regex = new_regex(); - var tag_replace_callback = function(match, operator, name) { - switch(operator) { - case "!": // ignore comments - return ""; - case "=": // set new delimiters, rebuild the replace regexp - that.set_delimiters(name); - regex = new_regex(); - return ""; - case ">": // render partial - return that.render_partial(name, context, partials); - case "{": // the triple mustache is unescaped - return that.find(name, context); - default: // escape the value - return that.escape(that.find(name, context)); - } - }; - var lines = template.split("\n"); - for(var i = 0; i < lines.length; i++) { - lines[i] = lines[i].replace(regex, tag_replace_callback, this); - if(!in_recursion) { - this.send(lines[i]); - } - } - - if(in_recursion) { - return lines.join("\n"); - } - }, - - set_delimiters: function(delimiters) { - var dels = delimiters.split(" "); - this.otag = this.escape_regex(dels[0]); - this.ctag = this.escape_regex(dels[1]); - }, - - escape_regex: function(text) { - // thank you Simon Willison - if(!arguments.callee.sRE) { - var specials = [ - '/', '.', '*', '+', '?', '|', - '(', ')', '[', ']', '{', '}', '\\' - ]; - arguments.callee.sRE = new RegExp( - '(\\' + specials.join('|\\') + ')', 'g' - ); - } - return text.replace(arguments.callee.sRE, '\\$1'); - }, - - /* - find `name` in current `context`. That is find me a value - from the view object - */ - find: function(name, context) { - name = this.trim(name); - - // Checks whether a value is thruthy or false or 0 - function is_kinda_truthy(bool) { - return bool === false || bool === 0 || bool; - } - - var value; - if(is_kinda_truthy(context[name])) { - value = context[name]; - } else if(is_kinda_truthy(this.context[name])) { - value = this.context[name]; - } - - if(typeof value === "function") { - return value.apply(context); - } - if(value !== undefined) { - return value; - } - // silently ignore unkown variables - return ""; - }, - - // Utility methods - - /* includes tag */ - includes: function(needle, haystack) { - return haystack.indexOf(this.otag + needle) != -1; - }, - - /* - Does away with nasty characters - */ - escape: function(s) { - s = String(s === null ? "" : s); - return s.replace(/&(?!\w+;)|["<>\\]/g, function(s) { - switch(s) { - case "&": return "&"; - case "\\": return "\\\\"; - case '"': return '\"'; - case "<": return "<"; - case ">": return ">"; - default: return s; - } - }); - }, - - // by @langalex, support for arrays of strings - create_context: function(_context) { - if(this.is_object(_context)) { - return _context; - } else { - var iterator = "."; - if(this.pragmas["IMPLICIT-ITERATOR"]) { - iterator = this.pragmas["IMPLICIT-ITERATOR"].iterator; - } - var ctx = {}; - ctx[iterator] = _context; - return ctx; - } - }, - - is_object: function(a) { - return a && typeof a == "object"; - }, - - is_array: function(a) { - return Object.prototype.toString.call(a) === '[object Array]'; - }, - - /* - Gets rid of leading and trailing whitespace - */ - trim: function(s) { - return s.replace(/^\s*|\s*$/g, ""); - }, - - /* - Why, why, why? Because IE. Cry, cry cry. - */ - map: function(array, fn) { - if (typeof array.map == "function") { - return array.map(fn); - } else { - var r = []; - var l = array.length; - for(var i = 0; i < l; i++) { - r.push(fn(array[i])); - } - return r; - } - } - }; - - return({ - name: "mustache.js", - version: "0.3.1-dev", - - /* - Turns a template and view into HTML - */ - to_html: function(template, view, partials, send_fun) { - var renderer = new Renderer(); - if(send_fun) { - renderer.send = send_fun; - } - renderer.render(template, view, partials); - if(!send_fun) { - return renderer.buffer.join("\n"); - } - }, - escape : function(text) { - return new Renderer().escape(text); - } - }); -}(); - - $.mustache = function(template, view, partials) { - return Mustache.to_html(template, view, partials); - }; - - $.mustache.escape = function(text) { - return Mustache.escape(text); - }; - -})(jQuery); diff --git a/vendor/mustache/0.5.0-dev/mustache.js b/vendor/mustache/0.5.0-dev/mustache.js new file mode 100644 index 00000000..641cebd5 --- /dev/null +++ b/vendor/mustache/0.5.0-dev/mustache.js @@ -0,0 +1,536 @@ +/*! + * mustache.js - Logic-less {{mustache}} templates with JavaScript + * http://github.com/janl/mustache.js + */ +var Mustache = (typeof module !== "undefined" && module.exports) || {}; + +(function (exports) { + + exports.name = "mustache.js"; + exports.version = "0.5.0-dev"; + exports.tags = ["{{", "}}"]; + exports.parse = parse; + exports.compile = compile; + exports.render = render; + exports.clearCache = clearCache; + + // This is here for backwards compatibility with 0.4.x. + exports.to_html = function (template, view, partials, send) { + var result = render(template, view, partials); + + if (typeof send === "function") { + send(result); + } else { + return result; + } + }; + + var _toString = Object.prototype.toString; + var _isArray = Array.isArray; + var _forEach = Array.prototype.forEach; + var _trim = String.prototype.trim; + + var isArray; + if (_isArray) { + isArray = _isArray; + } else { + isArray = function (obj) { + return _toString.call(obj) === "[object Array]"; + }; + } + + var forEach; + if (_forEach) { + forEach = function (obj, callback, scope) { + return _forEach.call(obj, callback, scope); + }; + } else { + forEach = function (obj, callback, scope) { + for (var i = 0, len = obj.length; i < len; ++i) { + callback.call(scope, obj[i], i, obj); + } + }; + } + + var spaceRe = /^\s*$/; + + function isWhitespace(string) { + return spaceRe.test(string); + } + + var trim; + if (_trim) { + trim = function (string) { + return string == null ? "" : _trim.call(string); + }; + } else { + var trimLeft, trimRight; + + if (isWhitespace("\xA0")) { + trimLeft = /^\s+/; + trimRight = /\s+$/; + } else { + // IE doesn't match non-breaking spaces with \s, thanks jQuery. + trimLeft = /^[\s\xA0]+/; + trimRight = /[\s\xA0]+$/; + } + + trim = function (string) { + return string == null ? "" : + String(string).replace(trimLeft, "").replace(trimRight, ""); + }; + } + + var escapeMap = { + "&": "&", + "<": "<", + ">": ">", + '"': '"', + "'": ''' + }; + + function escapeHTML(string) { + return String(string).replace(/&(?!\w+;)|[<>"']/g, function (s) { + return escapeMap[s] || s; + }); + } + + /** + * Adds the `template`, `line`, and `file` properties to the given error + * object and alters the message to provide more useful debugging information. + */ + function debug(e, template, line, file) { + file = file || "