Merge branch 'master' into gh-pages

demo/js/app.js

@@ -29,15 +29,15 @@ $(function() {
 
 function demoDataset() {
   var datasetId = 'test-dataset';
-  var metadata = {
+  var inData = {
-    title: 'My Test Dataset'
+    metadata: {
-    , name: '1-my-test-dataset' 
+      title: 'My Test Dataset'
-    , id: datasetId
+      , name: '1-my-test-dataset' 
-  };
+      , id: datasetId
-  var indata = {
+      , headers: ['x', 'y', 'z']
-      headers: ['x', 'y', 'z']
+    },
-    , rows: [
+    documents: [
-        {id: 0, x: 1, y: 2, z: 3}
+      {id: 0, x: 1, y: 2, z: 3}
       , {id: 1, x: 2, y: 4, z: 6}
       , {id: 2, x: 3, y: 6, z: 9}
       , {id: 3, x: 4, y: 8, z: 12}
@@ -45,12 +45,9 @@ function demoDataset() {
       , {id: 5, x: 6, y: 12, z: 18}
     ]
   };
-  var dataset = new recline.Model.Dataset(metadata);
+  var backend = new recline.Model.BackendMemory();
-  dataset.backendConfig = { 
+  backend.addDataset(inData);
-    type: 'memory'
+  var dataset = new recline.Model.Dataset({id: datasetId}, backend);
-    // deep copy so we do not touch original data ...
-    , data: $.extend(true, {}, indata)
-  };
   return dataset;
 }
 
@@ -62,11 +59,12 @@ function setupLoadFromWebstore(callback) {
     e.preventDefault();
     var $form = $(e.target);
     var source = $form.find('input[name="source"]').val();
-    var dataset = new recline.Model.Dataset();
+    var dataset = new recline.Model.Dataset({
-    dataset.backendConfig = {
+        id: 'gold-prices',
-      type: 'webstore',
+        webstore_url: source
-      url: source
+      },
-    };
+      'webstore'
+    );
     callback(dataset);
   });
 }

src/backend.js

@@ -9,13 +9,11 @@ this.recline = this.recline || {};
 this.recline.Model = this.recline.Model || {};
 
 (function($, my) {
-  my.backends = {};
-
   // ## Backbone.sync
   //
   // Override Backbone.sync to hand off to sync function in relevant backend
   Backbone.sync = function(method, model, options) {
-    return my.backends[model.backendConfig.type].sync(method, model, options);
+    return model.backend.sync(method, model, options);
   }
 
   // ## wrapInTimeout
@@ -45,83 +43,95 @@ this.recline.Model = this.recline.Model || {};
 
   // ## BackendMemory - uses in-memory data
   //
-  // To use you should:
+  // This is very artificial and is really only designed for testing
+  // purposes.
+  //
+  // To use it you should provide in your constructor data:
   // 
-  // A. provide metadata as model data to the Dataset
+  //   * metadata (including headers array)
+  //   * documents: list of hashes, each hash being one doc. A doc *must* have an id attribute which is unique.
   //
-  // B. Set backendConfig on your dataset with attributes:
+  //  Example:
-  //
-  //   - type: 'memory'
-  //   - data: hash with 2 keys:
-  //
-  //     * headers: list of header names/labels
-  //     * rows: list of hashes, each hash being one row. A row *must* have an id attribute which is unique.
-  //
-  //  Example of data:
   // 
   //  <pre>
-  //        {
+  //  // Backend setup
-  //            headers: ['x', 'y', 'z']
+  //  var backend = Backend();
-  //          , rows: [
+  //  backend.addDataset({
-  //              {id: 0, x: 1, y: 2, z: 3}
+  //    metadata: {
-  //            , {id: 1, x: 2, y: 4, z: 6}
+  //      id: 'my-id',
-  //          ]
+  //      title: 'My Title',
-  //        };
+  //      headers: ['x', 'y', 'z'],
+  //    },
+  //    documents: [
+  //        {id: 0, x: 1, y: 2, z: 3},
+  //        {id: 1, x: 2, y: 4, z: 6}
+  //      ]
+  //  });
+  //  // later ...
+  //  var dataset = Dataset({id: 'my-id'});
+  //  dataset.fetch();
+  //  etc ...
   //  </pre>
   my.BackendMemory = Backbone.Model.extend({
-      sync: function(method, model, options) {
+    initialize: function() {
-        var self = this;
+      this.datasets = {};
-        if (method === "read") {
+    },
-          var dfd = $.Deferred();
+    addDataset: function(data) {
-          if (model.__type__ == 'Dataset') {
+      this.datasets[data.metadata.id] = $.extend(true, {}, data);
-            var dataset = model;
+    },
-            dataset.set({
+    sync: function(method, model, options) {
-              headers: dataset.backendConfig.data.headers
+      var self = this;
-            });
+      if (method === "read") {
-            dataset.docCount = dataset.backendConfig.data.rows.length;
-            dfd.resolve(dataset);
-          }
-          return dfd.promise();
-        } else if (method === 'update') {
-          var dfd = $.Deferred();
-          if (model.__type__ == 'Document') {
-            _.each(model.backendConfig.data.rows, function(row, idx) {
-              if(row.id === model.id) {
-                model.backendConfig.data.rows[idx] = model.toJSON();
-              }
-            });
-            dfd.resolve(model);
-          }
-          return dfd.promise();
-        } else if (method === 'delete') {
-          var dfd = $.Deferred();
-          if (model.__type__ == 'Document') {
-            model.backendConfig.data.rows = _.reject(model.backendConfig.data.rows, function(row) {
-              return (row.id === model.id);
-            });
-            dfd.resolve(model);
-          }
-          return dfd.promise();
-        } else {
-          alert('Not supported: sync on BackendMemory with method ' + method + ' and model ' + model);
-        }
-      },
-      query: function(model, queryObj) {
-        var numRows = queryObj.size;
-        var start = queryObj.offset;
         var dfd = $.Deferred();
-        results = model.backendConfig.data.rows;
+        if (model.__type__ == 'Dataset') {
-        // not complete sorting!
+          var rawDataset = this.datasets[model.id];
-        _.each(queryObj.sort, function(item) {
+          model.set(rawDataset.metadata);
-          results = _.sortBy(results, function(row) {
+          model.docCount = rawDataset.documents.length;
-            var _out = row[item[0]];
+          dfd.resolve(model);
-            return (item[1] == 'asc') ? _out : -1*_out;
+        }
-          });
-        });
-        var results = results.slice(start, start+numRows);
-        dfd.resolve(results);
         return dfd.promise();
+      } else if (method === 'update') {
+        var dfd = $.Deferred();
+        if (model.__type__ == 'Document') {
+          _.each(self.datasets[model.dataset.id].documents, function(doc, idx) {
+            if(doc.id === model.id) {
+              self.datasets[model.dataset.id].documents[idx] = model.toJSON();
+            }
+          });
+          dfd.resolve(model);
+        }
+        return dfd.promise();
+      } else if (method === 'delete') {
+        var dfd = $.Deferred();
+        if (model.__type__ == 'Document') {
+          var rawDataset = self.datasets[model.dataset.id];
+          var newdocs = _.reject(rawDataset.documents, function(doc) {
+            return (doc.id === model.id);
+          });
+          rawDataset.documents = newdocs;
+          dfd.resolve(model);
+        }
+        return dfd.promise();
+      } else {
+        alert('Not supported: sync on BackendMemory with method ' + method + ' and model ' + model);
       }
+    },
+    query: function(model, queryObj) {
+      var numRows = queryObj.size;
+      var start = queryObj.offset;
+      var dfd = $.Deferred();
+      results = this.datasets[model.id].documents;
+      // not complete sorting!
+      _.each(queryObj.sort, function(item) {
+        results = _.sortBy(results, function(doc) {
+          var _out = doc[item[0]];
+          return (item[1] == 'asc') ? _out : -1*_out;
+        });
+      });
+      var results = results.slice(start, start+numRows);
+      dfd.resolve(results);
+      return dfd.promise();
+    }
   });
   my.backends['memory'] = new my.BackendMemory();
 
@@ -129,20 +139,12 @@ this.recline.Model = this.recline.Model || {};
   //
   // Connecting to [Webstores](http://github.com/okfn/webstore)
   //
-  // To use this backend set backendConfig on your Dataset as:
+  // To use this backend ensure your Dataset has a webstore_url in its attributes.
-  //
-  // <pre>
-  // {
-  //   'type': 'webstore',
-  //   'url': url to relevant Webstore table
-  // }
-  // </pre>
   my.BackendWebstore = Backbone.Model.extend({
     sync: function(method, model, options) {
       if (method === "read") {
         if (model.__type__ == 'Dataset') {
-          var dataset = model;
+          var base = model.get('webstore_url');
-          var base = dataset.backendConfig.url;
           var schemaUrl = base + '/schema.json';
           var jqxhr = $.ajax({
             url: schemaUrl,
@@ -154,11 +156,11 @@ this.recline.Model = this.recline.Model || {};
             headers = _.map(schema.data, function(item) {
               return item.name;
             });
-            dataset.set({
+            model.set({
               headers: headers
             });
-            dataset.docCount = schema.count;
+            model.docCount = schema.count;
-            dfd.resolve(dataset, jqxhr);
+            dfd.resolve(model, jqxhr);
           })
           .fail(function(arguments) {
             dfd.reject(arguments);
@@ -168,7 +170,7 @@ this.recline.Model = this.recline.Model || {};
       }
     },
     query: function(model, queryObj) {
-      var base = model.backendConfig.url;
+      var base = model.get('webstore_url');
       var data = {
         _limit:  queryObj.size
         , _offset: queryObj.offset
@@ -193,33 +195,30 @@ this.recline.Model = this.recline.Model || {};
   // 
   // For connecting to [DataProxy-s](http://github.com/okfn/dataproxy).
   //
-  // Set a Dataset to use this backend:
-  //
-  //     dataset.backendConfig = {
-  //       // required
-  //       url: {url-of-data-to-proxy},
-  //       format: csv | xls,
-  //     }
-  //
   // When initializing the DataProxy backend you can set the following attributes:
   //
   // * dataproxy: {url-to-proxy} (optional). Defaults to http://jsonpdataproxy.appspot.com
   //
+  // Datasets using using this backend should set the following attributes:
+  //
+  // * url: (required) url-of-data-to-proxy
+  // * format: (optional) csv | xls (defaults to csv if not specified)
+  //
   // Note that this is a **read-only** backend.
   my.BackendDataProxy = Backbone.Model.extend({
     defaults: {
-      dataproxy: 'http://jsonpdataproxy.appspot.com'
+      dataproxy_url: 'http://jsonpdataproxy.appspot.com'
     },
     sync: function(method, model, options) {
+      var self = this;
       if (method === "read") {
         if (model.__type__ == 'Dataset') {
-          var dataset = model;
+          var base = self.get('dataproxy_url');
-          var base = my.backends['dataproxy'].get('dataproxy');
           // TODO: should we cache for extra efficiency
           var data = {
-            url: dataset.backendConfig.url
+            url: model.get('url')
             , 'max-results':  1
-            , type: dataset.backendConfig.format
+            , type: model.get('format') || 'csv'
           };
           var jqxhr = $.ajax({
             url: base
@@ -228,10 +227,10 @@ this.recline.Model = this.recline.Model || {};
           });
           var dfd = $.Deferred();
           wrapInTimeout(jqxhr).done(function(results) {
-            dataset.set({
+            model.set({
               headers: results.fields
             });
-            dfd.resolve(dataset, jqxhr);
+            dfd.resolve(model, jqxhr);
           })
           .fail(function(arguments) {
             dfd.reject(arguments);
@@ -243,11 +242,11 @@ this.recline.Model = this.recline.Model || {};
       }
     },
     query: function(dataset, queryObj) {
-      var base = my.backends['dataproxy'].get('dataproxy');
+      var base = this.get('dataproxy_url');
       var data = {
-        url: dataset.backendConfig.url
+        url: dataset.get('url')
         , 'max-results':  queryObj.size
-        , type: dataset.backendConfig.format
+        , type: dataset.get('format')
       };
       var jqxhr = $.ajax({
         url: base
@@ -256,10 +255,10 @@ this.recline.Model = this.recline.Model || {};
       });
       var dfd = $.Deferred();
       jqxhr.done(function(results) {
-        var _out = _.map(results.data, function(row) {
+        var _out = _.map(results.data, function(doc) {
           var tmp = {};
           _.each(results.fields, function(key, idx) {
-            tmp[key] = row[idx];
+            tmp[key] = doc[idx];
           });
           return tmp;
         });
@@ -273,15 +272,27 @@ this.recline.Model = this.recline.Model || {};
 
   // ## Google spreadsheet backend
   // 
-  // Connect to Google Docs spreadsheet. For write operations
+  // Connect to Google Docs spreadsheet.
+  //
+  // Dataset must have a url attribute pointing to the Gdocs
+  // spreadsheet's JSON feed e.g.
+  //
+  // <pre>
+  // var dataset = new recline.Model.Dataset({
+  //     url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
+  //   },
+  //   'gdocs'
+  // );
+  // </pre>
   my.BackendGDoc = Backbone.Model.extend({
     sync: function(method, model, options) {
+      var self = this;
       if (method === "read") { 
         var dfd = $.Deferred(); 
         var dataset = model;
 
-        $.getJSON(model.backendConfig.url, function(d) {
+        $.getJSON(model.get('url'), function(d) {
-          result = my.backends['gdocs'].gdocsToJavascript(d);
+          result = self.gdocsToJavascript(d);
           model.set({'headers': result.header});
           // cache data onto dataset (we have loaded whole gdoc it seems!)
           model._dataCache = result.data;

src/model.js

@@ -11,10 +11,13 @@ this.recline.Model = this.recline.Model || {};
   // * docCount: total number of documents in this dataset (obtained on a fetch for this Dataset)
   my.Dataset = Backbone.Model.extend({
     __type__: 'Dataset',
-    initialize: function(options) {
+    initialize: function(model, backend) {
+      this.backend = backend;
+      if (backend && backend.constructor == String) {
+        this.backend = my.backends[backend];
+      }
       this.currentDocuments = new my.DocumentList();
       this.docCount = null;
-      this.backend = null;
       this.defaultQuery = {
         size: 100
         , offset: 0
@@ -36,15 +39,14 @@ this.recline.Model = this.recline.Model || {};
     // This also illustrates the limitations of separating the Dataset and the Backend
     query: function(queryObj) {
       var self = this;
-      var backend = my.backends[this.backendConfig.type];
       this.queryState = queryObj || this.defaultQuery;
       this.queryState = _.extend({size: 100, offset: 0}, this.queryState);
       var dfd = $.Deferred();
-      backend.query(this, this.queryState).done(function(rows) {
+      this.backend.query(this, this.queryState).done(function(rows) {
         var docs = _.map(rows, function(row) {
           var _doc = new my.Document(row);
-          _doc.backendConfig = self.backendConfig;
+          _doc.backend = self.backend;
-          _doc.backend = backend;
+          _doc.dataset = self;
           return _doc;
         });
         self.currentDocuments.reset(docs);
@@ -75,5 +77,11 @@ this.recline.Model = this.recline.Model || {};
     __type__: 'DocumentList',
     model: my.Document
   });
+
+  // ## Backend registry
+  //
+  // Backends will register themselves by id into this registry
+  my.backends = {};
+
 }(jQuery, this.recline.Model));
 

test/index.html

@@ -20,6 +20,9 @@
   <script type="text/javascript" src="../src/backend.js"></script>
   <script type="text/javascript" src="model.test.js"></script>
   <script type="text/javascript" src="../src/view.js"></script>
+  <script type="text/javascript" src="../src/view-data-explorer.js"></script>
+  <script type="text/javascript" src="../src/view-data-table.js"></script>
+  <script type="text/javascript" src="../src/view-transform-dialog.js"></script>
   <script type="text/javascript" src="view.test.js"></script>
 </head>
 <body>

test/model.test.js

@@ -1,494 +1,484 @@
 (function ($) {
+module("Dataset");
 
-  module("Dataset");
+test('Memory Backend', function () {
-
+  var datasetId = 'test-dataset';
-  test('new Dataset', function () {
+  var inData = {
-    var datasetId = 'test-dataset';
+    metadata: {
-    var metadata = {
       title: 'My Test Dataset'
-    , name: '1-my-test-dataset' 
+      , name: '1-my-test-dataset' 
-    , id: datasetId
+      , id: datasetId
-    };
+      , headers: ['x', 'y', 'z']
-    var indata = {
+    },
-      headers: ['x', 'y', 'z']
+    documents: [
-    , rows: [
       {id: 0, x: 1, y: 2, z: 3}
       , {id: 1, x: 2, y: 4, z: 6}
       , {id: 2, x: 3, y: 6, z: 9}
       , {id: 3, x: 4, y: 8, z: 12}
       , {id: 4, x: 5, y: 10, z: 15}
       , {id: 5, x: 6, y: 12, z: 18}
-      ]
-    };
-    var dataset = new recline.Model.Dataset(metadata);
-    dataset.backendConfig = { 
-      type: 'memory'
-      // deep copy so we do not touch original data ...
-      , data: $.extend(true, {}, indata)
-    };
-    expect(10);
-    dataset.fetch().then(function(dataset) {
-      equal(dataset.get('name'), metadata.name);
-      deepEqual(dataset.get('headers'), indata.headers);
-      equal(dataset.docCount, 6);
-      var queryObj = {
-        size: 4
-        , offset: 2
-      };
-      dataset.query(queryObj).then(function(documentList) {
-        deepEqual(indata.rows[2], documentList.models[0].toJSON());
-      });
-      var queryObj = {
-        sort: [
-          ['y', 'desc']
-        ]
-      };
-      dataset.query(queryObj).then(function(docs) {
-        var doc0 = dataset.currentDocuments.models[0].toJSON();
-        equal(doc0.x, 6);
-      });
-      dataset.query().then(function(docList) {
-        equal(docList.length, Math.min(100, indata.rows.length));
-        var doc1 = docList.models[0];
-        deepEqual(doc1.toJSON(), indata.rows[0]);
-
-        // Test UPDATE
-        var newVal = 10;
-        doc1.set({x: newVal});
-        doc1.save().then(function() {
-          equal(dataset.backendConfig.data.rows[0].x, newVal);
-        })
-
-        // Test Delete
-        doc1.destroy().then(function() {
-          equal(dataset.backendConfig.data.rows.length, 5);
-          equal(dataset.backendConfig.data.rows[0].x, indata.rows[1].x);
-        });
-      });
-    });
-  });
-
-  // TODO: move to fixtures
-  var webstoreSchema = {
-    "count": 3, 
-    "data": [
-    {
-      "name": "__id__", 
-      "type": "integer", 
-      "values_url": "/rufuspollock/demo/data/distinct/__id__"
-    }, 
-    {
-      "name": "date", 
-      "type": "text", 
-      "values_url": "/rufuspollock/demo/data/distinct/date"
-    }, 
-    {
-      "name": "geometry", 
-      "type": "text", 
-      "values_url": "/rufuspollock/demo/data/distinct/geometry"
-    }, 
-    {
-      "name": "amount", 
-      "type": "text", 
-      "values_url": "/rufuspollock/demo/data/distinct/amount"
-    }
-    ], 
-      "fields": [
-      {
-        "name": "type"
-      }, 
-      {
-        "name": "name"
-      }, 
-      {
-        "name": "values_url"
-      }
     ]
   };
-
+  var backend = new recline.Model.BackendMemory();
-  webstoreData = {
+  backend.addDataset(inData);
-    "count": null, 
+  var dataset = new recline.Model.Dataset({id: datasetId}, backend);
-    "data": [
+  // ### Start testing
-    {
+  expect(10);
-      "__id__": 1, 
+  // convenience for tests
-      "amount": "100", 
+  var data = backend.datasets[datasetId];
-      "date": "2009-01-01", 
+  dataset.fetch().then(function(datasetAgain) {
-      "geometry": null
+    equal(dataset.get('name'), data.metadata.name);
-    }, 
+    deepEqual(dataset.get('headers'), data.metadata.headers);
-    {
+    equal(dataset.docCount, 6);
-      "__id__": 2, 
+    var queryObj = {
-      "amount": "200", 
+      size: 4
-      "date": "2010-01-01", 
+      , offset: 2
-      "geometry": null
-    }, 
-    {
-      "__id__": 3, 
-      "amount": "300", 
-      "date": "2011-01-01", 
-      "geometry": null
-    }
-    ], 
-      "fields": [
-      {
-        "name": "__id__"
-      }, 
-      {
-        "name": "date"
-      }, 
-      {
-        "name": "geometry"
-      }, 
-      {
-        "name": "amount"
-      }
-    ]
-  };
-
-  test('Webstore Backend', function() {
-    var dataset = new recline.Model.Dataset();
-    dataset.backendConfig = {
-      type: 'webstore',
-      url: 'http://webstore.test.ckan.org/rufuspollock/demo/data'
     };
-
+    dataset.query(queryObj).then(function(documentList) {
-    var stub = sinon.stub($, 'ajax', function(options) {
+      deepEqual(data.documents[2], documentList.models[0].toJSON());
-      if (options.url.indexOf('schema.json') != -1) {
-        return {
-          done: function(callback) {
-            callback(webstoreSchema);
-            return this;
-          },
-          fail: function() {
-            return this;
-          }
-        }
-      } else {
-        return {
-          done: function(callback) {
-            callback(webstoreData);
-          },
-          fail: function() {
-          }
-        }
-      }
     });
-
+    var queryObj = {
-    dataset.fetch().done(function(dataset) {
+      sort: [
-      deepEqual(['__id__', 'date', 'geometry', 'amount'], dataset.get('headers'));
+        ['y', 'desc']
-      equal(3, dataset.docCount)
-      // dataset.query().done(function(docList) {
-      //  equal(3, docList.length)
-      //  equal("2009-01-01", docList.models[0].get('date'));
-      // });
-    });
-    $.ajax.restore();
-  });
-
-
-  var dataProxyData = {
-    "data": [
-      [
-      "1", 
-    "1950-01", 
-    "34.73"
-      ], 
-    [
-      "2", 
-    "1950-02", 
-    "34.73"
-      ], 
-    [
-      "3", 
-    "1950-03", 
-    "34.73"
-      ], 
-    [
-      "4", 
-    "1950-04", 
-    "34.73"
-      ], 
-    [
-      "5", 
-    "1950-05", 
-    "34.73"
-      ], 
-    [
-      "6", 
-    "1950-06", 
-    "34.73"
-      ], 
-    [
-      "7", 
-    "1950-07", 
-    "34.73"
-      ], 
-    [
-      "8", 
-    "1950-08", 
-    "34.73"
-      ], 
-    [
-      "9", 
-    "1950-09", 
-    "34.73"
-      ], 
-    [
-      "10", 
-    "1950-10", 
-    "34.73"
       ]
-      ], 
-    "fields": [
-      "__id__", 
-    "date", 
-    "price"
-      ], 
-    "length": null, 
-    "max_results": 10, 
-    "url": "http://webstore.thedatahub.org/rufuspollock/gold_prices/data.csv"
-  };
-
-  test('DataProxy Backend', function() {
-    // needed only if not stubbing
-    // stop();
-    var dataset = new recline.Model.Dataset();
-    dataset.backendConfig = {
-      type: 'dataproxy',
-      url: 'http://webstore.thedatahub.org/rufuspollock/gold_prices/data.csv'
     };
-
+    dataset.query(queryObj).then(function(docs) {
-    var stub = sinon.stub($, 'ajax', function(options) {
+      var doc0 = dataset.currentDocuments.models[0].toJSON();
-      var partialUrl = 'jsonpdataproxy.appspot.com';
+      equal(doc0.x, 6);
-      if (options.url.indexOf(partialUrl) != -1) {
-        return {
-          done: function(callback) {
-            callback(dataProxyData);
-            return this;
-          }, 
-          fail: function() {
-            return this;
-          }
-        }
-      }
     });
+    dataset.query().then(function(docList) {
+      equal(docList.length, Math.min(100, data.documents.length));
+      var doc1 = docList.models[0];
+      deepEqual(doc1.toJSON(), data.documents[0]);
 
-    dataset.fetch().done(function(dataset) {
+      // Test UPDATE
-      deepEqual(['__id__', 'date', 'price'], dataset.get('headers'));
+      var newVal = 10;
-      equal(null, dataset.docCount)
+      doc1.set({x: newVal});
-      dataset.query().done(function(docList) {
+      doc1.save().then(function() {
-        equal(10, docList.length)
+        equal(data.documents[0].x, newVal);
-        equal("1950-01", docList.models[0].get('date'));
+      })
-        // needed only if not stubbing
+
-        start();
+      // Test Delete
+      doc1.destroy().then(function() {
+        equal(data.documents.length, 5);
+        equal(data.documents[0].x, inData.documents[1].x);
       });
     });
-    $.ajax.restore();
   });
+});
 
-
+// TODO: move to fixtures
-  var sample_gdocs_spreadsheet_data = {
+var webstoreSchema = {
-    "feed": {
+  "count": 3, 
-      "category": [
+  "data": [
-      {
+  {
-        "term": "http://schemas.google.com/spreadsheets/2006#list", 
+    "name": "__id__", 
-        "scheme": "http://schemas.google.com/spreadsheets/2006"
+    "type": "integer", 
-      }
+    "values_url": "/rufuspollock/demo/data/distinct/__id__"
-      ], 
+  }, 
-        "updated": {
+  {
-          "$t": "2010-07-12T18:32:16.200Z"
+    "name": "date", 
-        }, 
+    "type": "text", 
-        "xmlns": "http://www.w3.org/2005/Atom", 
+    "values_url": "/rufuspollock/demo/data/distinct/date"
-        "xmlns$gsx": "http://schemas.google.com/spreadsheets/2006/extended", 
+  }, 
-        "title": {
+  {
-          "$t": "Sheet1", 
+    "name": "geometry", 
-          "type": "text"
+    "type": "text", 
-        }, 
+    "values_url": "/rufuspollock/demo/data/distinct/geometry"
-        "author": [
+  }, 
-        {
+  {
-          "name": {
+    "name": "amount", 
-            "$t": "okfn.rufus.pollock"
+    "type": "text", 
-          }, 
+    "values_url": "/rufuspollock/demo/data/distinct/amount"
-          "email": {
-            "$t": "okfn.rufus.pollock@gmail.com"
-          }
-        }
-      ], 
-        "openSearch$startIndex": {
-          "$t": "1"
-        }, 
-        "link": [
-        {
-          "href": "http://spreadsheets.google.com/pub?key=0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc", 
-          "type": "text/html", 
-          "rel": "alternate"
-        }, 
-        {
-          "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values", 
-          "type": "application/atom+xml", 
-          "rel": "http://schemas.google.com/g/2005#feed"
-        }, 
-        {
-          "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json-in-script", 
-          "type": "application/atom+xml", 
-          "rel": "self"
-        }
-      ], 
-        "xmlns$openSearch": "http://a9.com/-/spec/opensearchrss/1.0/", 
-        "entry": [
-        {
-          "category": [
-          {
-            "term": "http://schemas.google.com/spreadsheets/2006#list", 
-            "scheme": "http://schemas.google.com/spreadsheets/2006"
-          }
-          ], 
-            "updated": {
-              "$t": "2010-07-12T18:32:16.200Z"
-            }, 
-            "gsx$column-2": {
-              "$t": "1"
-            }, 
-            "gsx$column-1": {
-              "$t": "A"
-            }, 
-            "title": {
-              "$t": "A", 
-              "type": "text"
-            }, 
-            "content": {
-              "$t": "column-2: 1", 
-              "type": "text"
-            }, 
-            "link": [
-            {
-              "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cokwr", 
-              "type": "application/atom+xml", 
-              "rel": "self"
-            }
-          ], 
-            "id": {
-              "$t": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cokwr"
-            }
-        }, 
-        {
-          "category": [
-          {
-            "term": "http://schemas.google.com/spreadsheets/2006#list", 
-            "scheme": "http://schemas.google.com/spreadsheets/2006"
-          }
-          ], 
-            "updated": {
-              "$t": "2010-07-12T18:32:16.200Z"
-            }, 
-            "gsx$column-2": {
-              "$t": "2"
-            }, 
-            "gsx$column-1": {
-              "$t": "b"
-            }, 
-            "title": {
-              "$t": "b", 
-              "type": "text"
-            }, 
-            "content": {
-              "$t": "column-2: 2", 
-              "type": "text"
-            }, 
-            "link": [
-            {
-              "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cpzh4", 
-              "type": "application/atom+xml", 
-              "rel": "self"
-            }
-          ], 
-            "id": {
-              "$t": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cpzh4"
-            }
-        }, 
-        {
-          "category": [
-          {
-            "term": "http://schemas.google.com/spreadsheets/2006#list", 
-            "scheme": "http://schemas.google.com/spreadsheets/2006"
-          }
-          ], 
-            "updated": {
-              "$t": "2010-07-12T18:32:16.200Z"
-            }, 
-            "gsx$column-2": {
-              "$t": "3"
-            }, 
-            "gsx$column-1": {
-              "$t": "c"
-            }, 
-            "title": {
-              "$t": "c", 
-              "type": "text"
-            }, 
-            "content": {
-              "$t": "column-2: 3", 
-              "type": "text"
-            }, 
-            "link": [
-            {
-              "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cre1l", 
-              "type": "application/atom+xml", 
-              "rel": "self"
-            }
-          ], 
-            "id": {
-              "$t": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cre1l"
-            }
-        }
-      ], 
-        "openSearch$totalResults": {
-          "$t": "3"
-        }, 
-        "id": {
-          "$t": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values"
-        }
-    }, 
-    "version": "1.0", 
-    "encoding": "UTF-8"
   }
+  ], 
+    "fields": [
+    {
+      "name": "type"
+    }, 
+    {
+      "name": "name"
+    }, 
+    {
+      "name": "values_url"
+    }
+  ]
+};
 
+webstoreData = {
+  "count": null, 
+  "data": [
+  {
+    "__id__": 1, 
+    "amount": "100", 
+    "date": "2009-01-01", 
+    "geometry": null
+  }, 
+  {
+    "__id__": 2, 
+    "amount": "200", 
+    "date": "2010-01-01", 
+    "geometry": null
+  }, 
+  {
+    "__id__": 3, 
+    "amount": "300", 
+    "date": "2011-01-01", 
+    "geometry": null
+  }
+  ], 
+    "fields": [
+    {
+      "name": "__id__"
+    }, 
+    {
+      "name": "date"
+    }, 
+    {
+      "name": "geometry"
+    }, 
+    {
+      "name": "amount"
+    }
+  ]
+};
 
-
+test('Webstore Backend', function() {
-
+  var dataset = new recline.Model.Dataset({
-  test("GDoc Backend", function() { 
+    id: 'my-id',
-    var dataset = new recline.Model.Dataset();
+    webstore_url: 'http://webstore.test.ckan.org/rufuspollock/demo/data'
-    dataset.backendConfig = {
+    },
-      type: 'gdocs',
+    'webstore'
-      url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
+  );
-    };
+  var stub = sinon.stub($, 'ajax', function(options) {
-
+    if (options.url.indexOf('schema.json') != -1) {
-    console.log('got gdoc dataset', dataset);
+      return {
-
+        done: function(callback) {
-    var stub = sinon.stub($, 'getJSON', function(options, cb) {
+          callback(webstoreSchema);
-      console.log('options are', options, cb);
+          return this;
-      var partialUrl = 'spreadsheets.google.com';
+        },
-      if (options.indexOf(partialUrl) != -1) {
+        fail: function() {
-        cb(sample_gdocs_spreadsheet_data)
+          return this;
-
+        }
       }
-    });
+    } else {
-
+      return {
-    dataset.fetch().then(function(dataset) {
+        done: function(callback) {
-      console.log('inside dataset:', dataset, dataset.get('headers'), dataset.get('data'));
+          callback(webstoreData);
-      deepEqual(['column-2', 'column-1'], dataset.get('headers'));
+        },
-      //equal(null, dataset.docCount)
+        fail: function() {
-      dataset.query().then(function(docList) {
+        }
-        equal(3, docList.length);
+      }
-        console.log(docList.models[0]);
+    }
-        equal("A", docList.models[0].get('column-1'));
-        // needed only if not stubbing
-        start();
-      });
-    });
-    $.getJSON.restore();
-
-
   });
 
+  dataset.fetch().done(function(dataset) {
+    deepEqual(['__id__', 'date', 'geometry', 'amount'], dataset.get('headers'));
+    equal(3, dataset.docCount)
+    dataset.query().done(function(docList) {
+      equal(3, docList.length)
+      equal("2009-01-01", docList.models[0].get('date'));
+    });
+  });
+  $.ajax.restore();
+});
+
+
+var dataProxyData = {
+  "data": [
+    [
+    "1", 
+  "1950-01", 
+  "34.73"
+    ], 
+  [
+    "2", 
+  "1950-02", 
+  "34.73"
+    ], 
+  [
+    "3", 
+  "1950-03", 
+  "34.73"
+    ], 
+  [
+    "4", 
+  "1950-04", 
+  "34.73"
+    ], 
+  [
+    "5", 
+  "1950-05", 
+  "34.73"
+    ], 
+  [
+    "6", 
+  "1950-06", 
+  "34.73"
+    ], 
+  [
+    "7", 
+  "1950-07", 
+  "34.73"
+    ], 
+  [
+    "8", 
+  "1950-08", 
+  "34.73"
+    ], 
+  [
+    "9", 
+  "1950-09", 
+  "34.73"
+    ], 
+  [
+    "10", 
+  "1950-10", 
+  "34.73"
+    ]
+    ], 
+  "fields": [
+    "__id__", 
+  "date", 
+  "price"
+    ], 
+  "length": null, 
+  "max_results": 10, 
+  "url": "http://webstore.thedatahub.org/rufuspollock/gold_prices/data.csv"
+};
+
+test('DataProxy Backend', function() {
+  // needed only if not stubbing
+  // stop();
+  var dataset = new recline.Model.Dataset({
+      url: 'http://webstore.thedatahub.org/rufuspollock/gold_prices/data.csv'
+    },
+    'dataproxy'
+  );
+
+  var stub = sinon.stub($, 'ajax', function(options) {
+    var partialUrl = 'jsonpdataproxy.appspot.com';
+    if (options.url.indexOf(partialUrl) != -1) {
+      return {
+        done: function(callback) {
+          callback(dataProxyData);
+          return this;
+        }, 
+        fail: function() {
+          return this;
+        }
+      }
+    }
+  });
+
+  dataset.fetch().done(function(dataset) {
+    deepEqual(['__id__', 'date', 'price'], dataset.get('headers'));
+    equal(null, dataset.docCount)
+    dataset.query().done(function(docList) {
+      equal(10, docList.length)
+      equal("1950-01", docList.models[0].get('date'));
+      // needed only if not stubbing
+      start();
+    });
+  });
+  $.ajax.restore();
+});
+
+
+var sample_gdocs_spreadsheet_data = {
+  "feed": {
+    "category": [
+    {
+      "term": "http://schemas.google.com/spreadsheets/2006#list", 
+      "scheme": "http://schemas.google.com/spreadsheets/2006"
+    }
+    ], 
+      "updated": {
+        "$t": "2010-07-12T18:32:16.200Z"
+      }, 
+      "xmlns": "http://www.w3.org/2005/Atom", 
+      "xmlns$gsx": "http://schemas.google.com/spreadsheets/2006/extended", 
+      "title": {
+        "$t": "Sheet1", 
+        "type": "text"
+      }, 
+      "author": [
+      {
+        "name": {
+          "$t": "okfn.rufus.pollock"
+        }, 
+        "email": {
+          "$t": "okfn.rufus.pollock@gmail.com"
+        }
+      }
+    ], 
+      "openSearch$startIndex": {
+        "$t": "1"
+      }, 
+      "link": [
+      {
+        "href": "http://spreadsheets.google.com/pub?key=0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc", 
+        "type": "text/html", 
+        "rel": "alternate"
+      }, 
+      {
+        "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values", 
+        "type": "application/atom+xml", 
+        "rel": "http://schemas.google.com/g/2005#feed"
+      }, 
+      {
+        "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json-in-script", 
+        "type": "application/atom+xml", 
+        "rel": "self"
+      }
+    ], 
+      "xmlns$openSearch": "http://a9.com/-/spec/opensearchrss/1.0/", 
+      "entry": [
+      {
+        "category": [
+        {
+          "term": "http://schemas.google.com/spreadsheets/2006#list", 
+          "scheme": "http://schemas.google.com/spreadsheets/2006"
+        }
+        ], 
+          "updated": {
+            "$t": "2010-07-12T18:32:16.200Z"
+          }, 
+          "gsx$column-2": {
+            "$t": "1"
+          }, 
+          "gsx$column-1": {
+            "$t": "A"
+          }, 
+          "title": {
+            "$t": "A", 
+            "type": "text"
+          }, 
+          "content": {
+            "$t": "column-2: 1", 
+            "type": "text"
+          }, 
+          "link": [
+          {
+            "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cokwr", 
+            "type": "application/atom+xml", 
+            "rel": "self"
+          }
+        ], 
+          "id": {
+            "$t": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cokwr"
+          }
+      }, 
+      {
+        "category": [
+        {
+          "term": "http://schemas.google.com/spreadsheets/2006#list", 
+          "scheme": "http://schemas.google.com/spreadsheets/2006"
+        }
+        ], 
+          "updated": {
+            "$t": "2010-07-12T18:32:16.200Z"
+          }, 
+          "gsx$column-2": {
+            "$t": "2"
+          }, 
+          "gsx$column-1": {
+            "$t": "b"
+          }, 
+          "title": {
+            "$t": "b", 
+            "type": "text"
+          }, 
+          "content": {
+            "$t": "column-2: 2", 
+            "type": "text"
+          }, 
+          "link": [
+          {
+            "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cpzh4", 
+            "type": "application/atom+xml", 
+            "rel": "self"
+          }
+        ], 
+          "id": {
+            "$t": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cpzh4"
+          }
+      }, 
+      {
+        "category": [
+        {
+          "term": "http://schemas.google.com/spreadsheets/2006#list", 
+          "scheme": "http://schemas.google.com/spreadsheets/2006"
+        }
+        ], 
+          "updated": {
+            "$t": "2010-07-12T18:32:16.200Z"
+          }, 
+          "gsx$column-2": {
+            "$t": "3"
+          }, 
+          "gsx$column-1": {
+            "$t": "c"
+          }, 
+          "title": {
+            "$t": "c", 
+            "type": "text"
+          }, 
+          "content": {
+            "$t": "column-2: 3", 
+            "type": "text"
+          }, 
+          "link": [
+          {
+            "href": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cre1l", 
+            "type": "application/atom+xml", 
+            "rel": "self"
+          }
+        ], 
+          "id": {
+            "$t": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cre1l"
+          }
+      }
+    ], 
+      "openSearch$totalResults": {
+        "$t": "3"
+      }, 
+      "id": {
+        "$t": "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values"
+      }
+  }, 
+  "version": "1.0", 
+  "encoding": "UTF-8"
+}
+
+test("GDoc Backend", function() { 
+  var dataset = new recline.Model.Dataset({
+      url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
+    },
+    'gdocs'
+  );
+
+  var stub = sinon.stub($, 'getJSON', function(options, cb) {
+    console.log('options are', options, cb);
+    var partialUrl = 'spreadsheets.google.com';
+    if (options.indexOf(partialUrl) != -1) {
+      cb(sample_gdocs_spreadsheet_data)
+    }
+  });
+
+  dataset.fetch().then(function(dataset) {
+    console.log('inside dataset:', dataset, dataset.get('headers'), dataset.get('data'));
+    deepEqual(['column-2', 'column-1'], dataset.get('headers'));
+    //equal(null, dataset.docCount)
+    dataset.query().then(function(docList) {
+      equal(3, docList.length);
+      console.log(docList.models[0]);
+      equal("A", docList.models[0].get('column-1'));
+      // needed only if not stubbing
+      start();
+    });
+  });
+  $.getJSON.restore();
+});
 
 })(this.jQuery);