From 03c147cee185f0e0966a03ace80b0cc1453d9f21 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 18 Aug 2012 02:46:31 +0100 Subject: [PATCH 1/4] [#120,backend/solr][s]: start on a solr backend (read-only, basic testing and no proper query support yet!). --- src/backend.solr.js | 65 ++++++++++++++++++++++++++++++ test/backend.solr.test.js | 85 +++++++++++++++++++++++++++++++++++++++ test/index.html | 2 + 3 files changed, 152 insertions(+) create mode 100644 src/backend.solr.js create mode 100644 test/backend.solr.test.js diff --git a/src/backend.solr.js b/src/backend.solr.js new file mode 100644 index 00000000..fae6d353 --- /dev/null +++ b/src/backend.solr.js @@ -0,0 +1,65 @@ +this.recline = this.recline || {}; +this.recline.Backend = this.recline.Backend || {}; +this.recline.Backend.Solr = this.recline.Backend.Solr || {}; + +(function($, my) { + my.__type__ = 'solr'; + + // ### fetch + // + // dataset must have a solr or url attribute pointing to solr endpoint + my.fetch = function(dataset) { + var jqxhr = $.ajax({ + url: dataset.solr || dataset.url, + data: { + rows: 1, + wt: 'json' + }, + dataType: 'jsonp', + jsonp: 'json.wrf' + }); + var dfd = $.Deferred(); + jqxhr.done(function(results) { + // if we get 0 results we cannot get fields + var fields = [] + if (results.response.numFound > 0) { + fields = _.map(_.keys(results.response.docs[0]), function(fieldName) { + return { id: fieldName }; + }); + } + var out = { + fields: fields, + useMemoryStore: false + }; + dfd.resolve(out); + }); + return dfd.promise(); + } + + // TODO - much work on proper query support is needed!! + my.query = function(queryObj, dataset) { + var q = queryObj.q || '*:*'; + var data = { + q: q, + rows: queryObj.size, + start: queryObj.from, + wt: 'json' + }; + var jqxhr = $.ajax({ + url: dataset.solr || dataset.url, + data: data, + dataType: 'jsonp', + jsonp: 'json.wrf' + }); + var dfd = $.Deferred(); + jqxhr.done(function(results) { + var out = { + total: results.response.numFound, + hits: results.response.docs + }; + dfd.resolve(out); + }); + return dfd.promise(); + }; + +}(jQuery, this.recline.Backend.Solr)); diff --git a/test/backend.solr.test.js b/test/backend.solr.test.js new file mode 100644 index 00000000..2b7ec65a --- /dev/null +++ b/test/backend.solr.test.js @@ -0,0 +1,85 @@ +(function ($) { +module("Backend SOLR"); + +test("fetch", function() { + var dataset = new recline.Model.Dataset({ + url: 'http://openspending.org/api/search', + backend: 'solr' + }); + // stop(); + + var stub = sinon.stub($, 'ajax', function(options) { + return { + done: function(callback) { + callback(sample_data); + return this; + }, + fail: function() { + } + }; + }); + + dataset.fetch().done(function(dataset) { + var exp = [ + "_id", + "amount", + "category.label_facet", + "dataset", + "from.label_facet", + "id", + "subcategory.label_facet", + "time.label_facet", + "to.label_facet" + ]; + deepEqual( + exp, + _.pluck(dataset.fields.toJSON(), 'id') + ); + // check we've mapped types correctly + equal(dataset.fields.get('amount').get('type'), 'string'); + + // fetch does a query so we can check for records + equal(dataset.recordCount, 10342132); + equal(dataset.records.length, 2); + equal(dataset.records.at(0).get('id'), '3e3e25d7737634127b76d5ee4a7df280987013c7'); + // start(); + }); + $.ajax.restore(); +}); + +var sample_data = { + "response": { + "docs": [ + { + "_id": "south-african-national-gov-budget-2012-13::3e3e25d7737634127b76d5ee4a7df280987013c7", + "amount": 30905738200000.0, + "category.label_facet": "General public services", + "dataset": "south-african-national-gov-budget-2012-13", + "from.label_facet": "National Treasury", + "id": "3e3e25d7737634127b76d5ee4a7df280987013c7", + "subcategory.label_facet": "Transfers of a general character between different levels of government", + "time.label_facet": "01. April 2012", + "to.label_facet": "Provincial Equitable Share" + }, + { + "_id": "south-african-national-gov-budget-2012-13::738849e28e6b3c45e5b0001e142b51479b3a3e41", + "amount": 8938807300000.0, + "category.label_facet": "General public services", + "dataset": "south-african-national-gov-budget-2012-13", + "from.label_facet": "National Treasury", + "id": "738849e28e6b3c45e5b0001e142b51479b3a3e41", + "subcategory.label_facet": "Public debt transactions", + "time.label_facet": "01. April 2012", + "to.label_facet": "State Debt Costs" + } + ], + "numFound": 10342132, + "start": 0 + }, + "responseHeader": { + "QTime": 578, + "status": 0 + } +}; + +})(this.jQuery); diff --git a/test/index.html b/test/index.html index 8830b603..e1168150 100644 --- a/test/index.html +++ b/test/index.html @@ -37,6 +37,7 @@ + @@ -45,6 +46,7 @@ + From af37da4d97d670c2e58b3564fa765ef048701c5e Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Fri, 19 Oct 2012 08:16:39 +0100 Subject: [PATCH 2/4] [_includes][xs]: include solr backend. --- _includes/recline-deps.html | 1 + 1 file changed, 1 insertion(+) diff --git a/_includes/recline-deps.html b/_includes/recline-deps.html index 112121c6..2316cee3 100644 --- a/_includes/recline-deps.html +++ b/_includes/recline-deps.html @@ -50,6 +50,7 @@ + From 11c0a2b083e031bfa83349931753aaf3a3ef0f84 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Fri, 19 Oct 2012 09:03:06 +0100 Subject: [PATCH 3/4] [#180,demos/search][s]: support for getting config from url (not used yet) plus refactor so that template is just for one result. --- demos/search/app.js | 85 ++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/demos/search/app.js b/demos/search/app.js index 8e176330..f7e434ea 100644 --- a/demos/search/app.js +++ b/demos/search/app.js @@ -1,41 +1,66 @@ jQuery(function($) { var $el = $('.search-here'); - // var url = 'http://openspending.org/api/search'; - // var url = 'http://localhost:9200/tmp/sfpd-last-month'; + // Check for config from url query string + // (this allows us to point to specific data sources backends) + var config = recline.View.parseQueryString(decodeURIComponent(window.location.search)); + if (config.backend) { + setupMoreComplexExample(config); + return; + } - // Crate our Recline Dataset - // Here we are just using the local data + // the simple example case + + // Create our Recline Dataset + // We'll just use some sample local data (see end of this file) var dataset = new recline.Model.Dataset({ - records: simpleData + records: sampleData }); - // Optional - // Let's configure the initial query a bit and set up facets - dataset.queryState.set({ - size: 10 - }, - {silent: true} - ); - dataset.queryState.addFacet('Author'); - dataset.query(); - - // The search view allows us to customize the template used to render the - // list of results - var template = getTemplate(); + // Set up the search View + + // We give it a custom template for rendering the example records + var template = ' \ +
\ +

\ + {{title}} by {{Author}} \ +

\ +

{{description}}

\ +

${{price}}

\ +
\ + '; var searchView = new SearchView({ el: $el, model: dataset, template: template }); searchView.render(); + + // Optional - we configure the initial query a bit and set up facets + dataset.queryState.set({ + size: 10 + }, + {silent: true} + ); + dataset.queryState.addFacet('Author'); + // Now do the first query + // After this point the Search View will take over handling queries + dataset.query(); }); + // Simple Search View // // Pulls together various Recline UI components and the central Dataset and Query (state) object // // Plus support for customization e.g. of template for list of results +// +// +// var view = new SearchView({ +// el: $('some-element'), +// model: dataset +// template: mustache-template-or-function +// }); var SearchView = Backbone.View.extend({ initialize: function(options) { this.el = $(this.el); @@ -59,7 +84,9 @@ var SearchView = Backbone.View.extend({ ', render: function() { - var results = Mustache.render(this.templateResults, { + // templateResults is just for one result ... + var tmpl = '{{#records}}' + this.templateResults + '{{/records}}'; + var results = Mustache.render(tmpl, { records: this.model.records.toJSON() }); var html = Mustache.render(this.template, { @@ -86,24 +113,12 @@ var SearchView = Backbone.View.extend({ }); // -------------------------------------------------------- -// Stuff specific to this demo +// Stuff very specific to this demo -function getTemplate() { - template = ' \ - {{#records}} \ -
\ -

\ - {{title}} by {{Author}} \ -

\ -

{{description}}

\ -

${{price}}

\ -
\ - {{/records}} \ - '; - return template; -} +function setupMoreComplexExample(config) { +}; -var simpleData = [ +var sampleData = [ { title: 'War and Peace', description: 'The epic tale of love, war and history', From 672d151434d74fda45602e71dd5bc509f239cdb4 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Fri, 19 Oct 2012 09:53:19 +0100 Subject: [PATCH 4/4] [#180,demos/search][m]: working search demo using any backend and with special customization for openspending (the linked example). --- demos/search/app.js | 101 +++++++++++++++++++++++++++++++++++++--- demos/search/index.html | 2 +- 2 files changed, 96 insertions(+), 7 deletions(-) diff --git a/demos/search/app.js b/demos/search/app.js index f7e434ea..343bbb6b 100644 --- a/demos/search/app.js +++ b/demos/search/app.js @@ -82,13 +82,18 @@ var SearchView = Backbone.View.extend({ \
\ ', - + render: function() { - // templateResults is just for one result ... - var tmpl = '{{#records}}' + this.templateResults + '{{/records}}'; - var results = Mustache.render(tmpl, { - records: this.model.records.toJSON() - }); + var results = ''; + if (_.isFunction(this.templateResults)) { + var results = _.map(this.model.records.toJSON(), this.templateResults).join('\n'); + } else { + // templateResults is just for one result ... + var tmpl = '{{#records}}' + this.templateResults + '{{/records}}'; + var results = Mustache.render(tmpl, { + records: this.model.records.toJSON() + }); + } var html = Mustache.render(this.template, { results: results }); @@ -116,6 +121,73 @@ var SearchView = Backbone.View.extend({ // Stuff very specific to this demo function setupMoreComplexExample(config) { + var $el = $('.search-here'); + var dataset = new recline.Model.Dataset(config); + // async as may be fetching remote + dataset.fetch().done(function() { + if (dataset.get('url').indexOf('openspending') === -1) { + // generic template function + var template = function(record) { + var template = '
\ +
    \ + {{#data}} \ +
  • {{key}}: {{value}}
    • \ + {{/data}} \ +
\ + '; + var data = _.map(_.keys(record), function(key) { + return { key: key, value: record[key] }; + }); + return Mustache.render(template, { + data: data + }); + } + } else { + // generic template function + var template = function(record) { + record['time'] = record['time.label_facet'] + var template = '
\ +

\ + {{record.dataset}} {{record.time}} \ + – {{amount_formatted}} \ +

\ +
    \ + {{#data}} \ +
  • {{key}}: {{value}}
    • \ + {{/data}} \ +
\ + '; + var data = []; + _.each(_.keys(record), function(key) { + if (key !='_id' && key != 'id') { + data.push({ key: key, value: record[key] }); + } + }); + return Mustache.render(template, { + record: record, + amount_formatted: formatAmount(record['amount']), + data: data + }); + } + } + + var searchView = new SearchView({ + el: $el, + model: dataset, + template: template + }); + searchView.render(); + + dataset.queryState.set({ + size: 10 + }, + {silent: true} + ); + if (dataset.get('url').indexOf('openspending') != -1) { + dataset.queryState.addFacet('dataset'); + } + dataset.query(); + }); }; var sampleData = [ @@ -139,3 +211,20 @@ var sampleData = [ } ]; +var formatAmount = function (num) { + var billion = 1000000000; + var million = 1000000; + var thousand = 1000; + var numabs = Math.abs(num); + if (numabs > billion) { + return (num / billion).toFixed(0) + 'bn'; + } + if (numabs > (million / 2)) { + return (num / million).toFixed(0) + 'm'; + } + if (numabs > thousand) { + return (num / thousand).toFixed(0) + 'k'; + } else { + return num.toFixed(0); + } +}; diff --git a/demos/search/index.html b/demos/search/index.html index 3eb25414..aa94911c 100644 --- a/demos/search/index.html +++ b/demos/search/index.html @@ -81,7 +81,7 @@ ul.facet-items {

This demo shows how Recline can be used to build a search app. It includes faceting as well as seearch. You can find the source javascript here – please feel free to reuse!

-

The default version uses some local example data but you can also connect directly to any other backend supported by Recline, in particular SOLR or ElasticSearch.

+

The default version uses some local example data but you can also connect directly to any other backend supported by Recline, in particular SOLR or ElasticSearch. For example, here's an example of this app running off the OpenSpending SOLR-style search API.