From 2cd91924807f68982d3a9ce1b545369ca80fac0b Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 7 Apr 2012 13:49:13 +0100 Subject: [PATCH 1/2] [#52,docs/model][m]: document all model elements in detail. * List more items in main index doc page * Major extension of source docs including * Query object with its spec * Facet * etc --- docs/model.html | 169 +++++++++++++++++++++++++++++++++++++++--------- index.html | 43 ++++++++++-- src/model.js | 144 +++++++++++++++++++++++++++++++++++------ 3 files changed, 298 insertions(+), 58 deletions(-) diff --git a/docs/model.html b/docs/model.html index 98c17439..61b73e8f 100644 --- a/docs/model.html +++ b/docs/model.html @@ -1,19 +1,31 @@ model.js
my.FieldList=Backbone.Collection.extend({model:my.Field -});

model.js

Recline Backbone Models

 
this.recline = this.recline || {};
 this.recline.Model = this.recline.Model || {};
 
-(function($, my) {

A Dataset model

+(function($, my) {

A Dataset model

A model has the following (non-Backbone) attributes:

-
    -
  • fields: (aka columns) is a FieldList listing all the fields on this -Dataset (this can be set explicitly, or, will be set by Dataset.fetch() or Dataset.query()
    • -
  • currentDocuments: a DocumentList containing the Documents we have -currently loaded for viewing (you update currentDocuments by calling query)
    • -
  • docCount: total number of documents in this dataset
    • -
 
my.Dataset = Backbone.Model.extend({
-  __type__: 'Dataset',
-  initialize: function(model, backend) {
+
@property {FieldList} fields: (aka columns) is a FieldList listing all the
+fields on this Dataset (this can be set explicitly, or, will be set by
+Dataset.fetch() or Dataset.query()

+
+
@property {DocumentList} currentDocuments: a DocumentList containing the
+Documents we have currently loaded for viewing (updated by calling query
+method)

+
+
@property {number} docCount: total number of documents in this dataset

+
+
@property {Backend} backend: the Backend (instance) for this Dataset

+
+
@property {Query} queryState: Query object which stores current
+queryState. queryState may be edited by other components (e.g. a query
+editor view) changes will trigger a Dataset query.

+
+
@property {FacetList} facets: FacetList object containing all current
+Facets.
             
my.Dataset = Backbone.Model.extend({
+  __type__: 'Dataset',

initialize

+ +

Sets up instance properties (see above)

 
  initialize: function(model, backend) {
     _.bindAll(this, 'query');
     this.backend = backend;
     if (backend && backend.constructor == String) {
@@ -26,7 +38,7 @@ currently loaded for viewing (you update currentDocuments by calling query)
     this.queryState = new my.Query();
     this.queryState.bind('change', this.query);
     this.queryState.bind('facet:add', this.query);
-  },

query

+ },

query

AJAX method with promise API to get documents from the backend.

@@ -79,14 +91,14 @@ also returned.

 data.fields = this.fields.toJSON(); return data; } -});

A Document (aka Row)

+});

A Document (aka Row)

A single entry or row in the dataset

 
my.Document = Backbone.Model.extend({
   __type__: 'Document'
-});

A Backbone collection of Documents

 
my.DocumentList = Backbone.Collection.extend({
+});

A Backbone collection of Documents

 
my.DocumentList = Backbone.Collection.extend({
   __type__: 'DocumentList',
   model: my.Document
-});

A Field (aka Column) on a Dataset

+});

A Field (aka Column) on a Dataset

Following attributes as standard:

@@ -99,8 +111,8 @@ also returned.

 id: null, label: null, type: 'String' - },

In addition to normal backbone initialization via a Hash you can also -just pass a single argument representing id to the ctor

 
  initialize: function(data) {

if a hash not passed in the first argument is set as value for key 0

 
    if ('0' in data) {
+  },
+  initialize: function(data) {

if a hash not passed in the first argument throw error

 
    if ('0' in data) {
       throw new Error('Looks like you did not pass a proper hash with id to Field constructor');
     }
     if (this.attributes.label == null) {
@@ -111,25 +123,81 @@ just pass a single argument representing id to the ctor

A Query object storing Dataset Query state

 
my.Query = Backbone.Model.extend({
+});

Query

+ +

Query instances encapsulate a query to the backend (see query method on backend). Useful both +for creating queries and for storing and manipulating query state - +e.g. from a query editor).

+ +

Query Structure and format

+ +

Query structure should follow that of ElasticSearch query +language.

+ +

NB: It is up to specific backends how to implement and support this query +structure. Different backends might choose to implement things differently +or not support certain features. Please check your backend for details.

+ +

Query object has the following key attributes:

+ + + +

Additions:

+ +
    +

  • q: either straight text or a hash will map directly onto a query_string +query +in backend

    + +
    • Of course this can be re-interpreted by different backends. E.g. some +may just pass this straight through e.g. for an SQL backend this could be +the full SQL query
    • +

  • filters: dict of ElasticSearch filters. These will be and-ed together for +execution.

    • +
+ +

Examples

+ +
+{
+   q: 'quick brown fox',
+   filters: [
+     { term: { 'owner': 'jones' } }
+   ]
+}
+
 
my.Query = Backbone.Model.extend({
   defaults: function() {
     return {
       size: 100
       , from: 0
-      , facets: {}

http://www.elasticsearch.org/guide/reference/query-dsl/and-filter.html -, filter: {} -list of simple filters which will be add to 'add' filter of filter

 
      , filters: []
+      , facets: {}

http://www.elasticsearch.org/guide/reference/query-dsl/and-filter.html +, filter: {}

list of simple filters which will be add to 'add' filter of filter

 
      , filters: []
     }
-  },

Set (update or add) a terms filter -http://www.elasticsearch.org/guide/reference/query-dsl/terms-filter.html

 
  addTermFilter: function(fieldId, value) {
+  },

addTermFilter

+ +

Set (update or add) a terms filter to filters

+ +

See http://www.elasticsearch.org/guide/reference/query-dsl/terms-filter.html

 
  addTermFilter: function(fieldId, value) {
     var filters = this.get('filters');
     var filter = { term: {} };
     filter.term[fieldId] = value;
     filters.push(filter);
-    this.set({filters: filters});

change does not seem to be triggered ...

 
    this.trigger('change');
-  },
-  addFacet: function(fieldId) {
-    var facets = this.get('facets');

Assume id and fieldId should be the same (TODO: this need not be true if we want to add two different type of facets on same field)

 
    if (_.contains(_.keys(facets), fieldId)) {
+    this.set({filters: filters});

change does not seem to be triggered ...

 
    this.trigger('change');
+  },

addFacet

+ +

Add a Facet to this query

+ +

See http://www.elasticsearch.org/guide/reference/api/search/facets/

 
  addFacet: function(fieldId) {
+    var facets = this.get('facets');

Assume id and fieldId should be the same (TODO: this need not be true if we want to add two different type of facets on same field)

 
    if (_.contains(_.keys(facets), fieldId)) {
       return;
     }
     facets[fieldId] = {
@@ -138,15 +206,56 @@ http://www.elasticsearch.org/guide/reference/query-dsl/terms-filter.html

     this.set({facets: facets}, {silent: true});
     this.trigger('facet:add', this);
   }
-});

A Facet (Result)

 
my.Facet = Backbone.Model.extend({
+});

A Facet (Result)

+ +

Object to store Facet information, that is summary information (e.g. values +and counts) about a field obtained by some faceting method on the +backend.

+ +

Structure of a facet follows that of Facet results in ElasticSearch, see: +http://www.elasticsearch.org/guide/reference/api/search/facets/

+ +

Specifically the object structure of a facet looks like (there is one +addition compared to ElasticSearch: the "id" field which corresponds to the +key used to specify this facet in the facet query):

+ +
+{
+  "id": "id-of-facet",
+  // type of this facet (terms, range, histogram etc)
+  "_type" : "terms",
+  // total number of tokens in the facet
+  "total": 5,
+  // @property {number} number of documents which have no value for the field
+  "missing" : 0,
+  // number of facet values not included in the returned facets
+  "other": 0,
+  // term object ({term: , count: ...})
+  "terms" : [ {
+      "term" : "foo",
+      "count" : 2
+    }, {
+      "term" : "bar",
+      "count" : 2
+    }, {
+      "term" : "baz",
+      "count" : 1
+    }
+  ]
+}
+
 
my.Facet = Backbone.Model.extend({
   defaults: function() {
     return {
-      _type: 'terms',

total number of tokens in the facet

 
      total: 0,

number of facet values not included in the returned facets

 
      other: 0,

number of documents which have no value for the field

 
      missing: 0,

term object ({term: , count: ...})

 
      terms: []
+      _type: 'terms',
+      total: 0,
+      other: 0,
+      missing: 0,
+      terms: []
     }
   }
-});

A Collection/List of Facets

 
my.FacetList = Backbone.Collection.extend({
+});

A Collection/List of Facets

 
my.FacetList = Backbone.Collection.extend({
   model: my.Facet
-});

Backend registry

+});

Backend registry

Backends will register themselves by id into this registry

 
my.backends = {};
 
diff --git a/index.html b/index.html
index a246a01f..e06043c6 100644
--- a/index.html
+++ b/index.html
@@ -84,11 +84,11 @@
   

     Recline Data Explorer and Library

     
-      A. Powerful data explorer using only javascript and html
+      A. Powerful data explorer built in pure javascript and html
       

       B. Suite of data components - grid, graphing and data connectors
       

-      — All built on Backbone
+      — All built with Backbone
@@ -207,20 +207,34 @@ var dataset = recline.Model.Dataset({

Recline has a simple structure layered on top of the basic Model/View distinction inherent in Backbone.

-

Models: there are two main model objects:

+

Models

+

There are two main model objects:

    -
  • Dataset: represents the dataset. +
  • Dataset: represents the dataset. Holds dataset info and a pointer to list of data items (Documents in our terminology) which it can load from the relevant Backend.
    • -
  • Document: an individual data item +
  • Document: an individual data item (e.g. a row from a relational database or a spreadsheet, a document from from a document DB like CouchDB or MongoDB).
+

Additional, related models:

+
    +
  • Field: a field/column on a + dataset.
    • +
  • Query: an object to encapsulate a + query to the backend (useful both for creating queries and for storing and + manipulating query state - e.g. from a query editor).
    • +
  • Facet: Object to store Facet + information, that is summary information (e.g. values and counts) about a + field obtained by some faceting method on the backend.
    • +
+

More detail of how these work can be found in the Model source docs.

-

Backends connect Dataset and Documents to data from a +

Backends

+

Backends connect Dataset and Documents to data from a specific 'Backend' data source. They provide methods for loading and saving Datasets and individuals Documents as well as for bulk loading via a query API and doing bulk transforms on the backend.

@@ -231,7 +245,8 @@ methods a Backend must have and (optionally) provides a base 'class' for inheritance. You can also find detailed examples of backend implementations in the source documentation below.

-

Views: complementing the model are various Views (you can +

Views

+

Complementing the model are various Views (you can also easily write your own). Each view holds a pointer to a Dataset:

  • DataExplorer: the parent view which manages the overall app and sets up @@ -239,14 +254,28 @@ also easily write your own). Each view holds a pointer to a Dataset:

  • DataGrid: the data grid view.
  • FlotGraph: a simple graphing view using Flot.
    • +
  • Map: a map view using Leaflet.
+

There are additional views which do not display a whole dataset but which +are useful:

+
    +
  • QueryEditor: a query editor view
    • +
  • FacetViewer: display facets
    • +
+ +

Source Docs (via Docco)

+

Models and Views (Widgets)

+ +

Backends

+
  • Backend: Base (base class providing a template for backends)
  • Backend: Memory (local data)
  • Backend: ElasticSearch
    • diff --git a/src/model.js b/src/model.js index 44de3689..a9c5270d 100644 --- a/src/model.js +++ b/src/model.js @@ -4,17 +4,33 @@ this.recline.Model = this.recline.Model || {}; (function($, my) { -// ## A Dataset model +// ## A Dataset model // // A model has the following (non-Backbone) attributes: // -// * fields: (aka columns) is a FieldList listing all the fields on this -// Dataset (this can be set explicitly, or, will be set by Dataset.fetch() or Dataset.query() -// * currentDocuments: a DocumentList containing the Documents we have -// currently loaded for viewing (you update currentDocuments by calling query) -// * docCount: total number of documents in this dataset +// @property {FieldList} fields: (aka columns) is a `FieldList` listing all the +// fields on this Dataset (this can be set explicitly, or, will be set by +// Dataset.fetch() or Dataset.query() +// +// @property {DocumentList} currentDocuments: a `DocumentList` containing the +// Documents we have currently loaded for viewing (updated by calling query +// method) +// +// @property {number} docCount: total number of documents in this dataset +// +// @property {Backend} backend: the Backend (instance) for this Dataset +// +// @property {Query} queryState: `Query` object which stores current +// queryState. queryState may be edited by other components (e.g. a query +// editor view) changes will trigger a Dataset query. +// +// @property {FacetList} facets: FacetList object containing all current +// Facets. my.Dataset = Backbone.Model.extend({ __type__: 'Dataset', + // ### initialize + // + // Sets up instance properties (see above) initialize: function(model, backend) { _.bindAll(this, 'query'); this.backend = backend; @@ -86,7 +102,7 @@ my.Dataset = Backbone.Model.extend({ } }); -// ## A Document (aka Row) +// ## A Document (aka Row) // // A single entry or row in the dataset my.Document = Backbone.Model.extend({ @@ -99,7 +115,7 @@ my.DocumentList = Backbone.Collection.extend({ model: my.Document }); -// ## A Field (aka Column) on a Dataset +// ## A Field (aka Column) on a Dataset // // Following attributes as standard: // @@ -112,10 +128,8 @@ my.Field = Backbone.Model.extend({ label: null, type: 'String' }, - // In addition to normal backbone initialization via a Hash you can also - // just pass a single argument representing id to the ctor initialize: function(data) { - // if a hash not passed in the first argument is set as value for key 0 + // if a hash not passed in the first argument throw error if ('0' in data) { throw new Error('Looks like you did not pass a proper hash with id to Field constructor'); } @@ -129,21 +143,71 @@ my.FieldList = Backbone.Collection.extend({ model: my.Field }); -// ## A Query object storing Dataset Query state +// ## Query +// +// Query instances encapsulate a query to the backend (see query method on backend). Useful both +// for creating queries and for storing and manipulating query state - +// e.g. from a query editor). +// +// **Query Structure and format** +// +// Query structure should follow that of [ElasticSearch query +// language](http://www.elasticsearch.org/guide/reference/api/search/). +// +// **NB: It is up to specific backends how to implement and support this query +// structure. Different backends might choose to implement things differently +// or not support certain features. Please check your backend for details.** +// +// Query object has the following key attributes: +// +// * size (=limit): number of results to return +// * from (=offset): offset into result set - http://www.elasticsearch.org/guide/reference/api/search/from-size.html +// * sort: sort order - +// * query: Query in ES Query DSL +// * filter: See filters and Filtered Query +// * fields: set of fields to return - http://www.elasticsearch.org/guide/reference/api/search/fields.html +// * facets: TODO - see http://www.elasticsearch.org/guide/reference/api/search/facets/ +// +// Additions: +// +// * q: either straight text or a hash will map directly onto a [query_string +// query](http://www.elasticsearch.org/guide/reference/query-dsl/query-string-query.html) +// in backend +// +// * Of course this can be re-interpreted by different backends. E.g. some +// may just pass this straight through e.g. for an SQL backend this could be +// the full SQL query +// +// * filters: dict of ElasticSearch filters. These will be and-ed together for +// execution. +// +// **Examples** +// +// 
    +// {
+//    q: 'quick brown fox',
+//    filters: [
+//      { term: { 'owner': 'jones' } }
+//    ]
+// }
+//
    my.Query = Backbone.Model.extend({ defaults: function() { return { size: 100 , from: 0 , facets: {} - // http://www.elasticsearch.org/guide/reference/query-dsl/and-filter.html + // // , filter: {} - // list of simple filters which will be add to 'add' filter of filter , filters: [] } }, - // Set (update or add) a terms filter - // http://www.elasticsearch.org/guide/reference/query-dsl/terms-filter.html + // #### addTermFilter + // + // Set (update or add) a terms filter to filters + // + // See addTermFilter: function(fieldId, value) { var filters = this.get('filters'); var filter = { term: {} }; @@ -153,6 +217,11 @@ my.Query = Backbone.Model.extend({ // change does not seem to be triggered ... this.trigger('change'); }, + // ### addFacet + // + // Add a Facet to this query + // + // See addFacet: function(fieldId) { var facets = this.get('facets'); // Assume id and fieldId should be the same (TODO: this need not be true if we want to add two different type of facets on same field) @@ -168,18 +237,51 @@ my.Query = Backbone.Model.extend({ }); -// ## A Facet (Result) +// ## A Facet (Result) +// +// Object to store Facet information, that is summary information (e.g. values +// and counts) about a field obtained by some faceting method on the +// backend. +// +// Structure of a facet follows that of Facet results in ElasticSearch, see: +// +// +// Specifically the object structure of a facet looks like (there is one +// addition compared to ElasticSearch: the "id" field which corresponds to the +// key used to specify this facet in the facet query): +// +// 
    +// {
+//   "id": "id-of-facet",
+//   // type of this facet (terms, range, histogram etc)
+//   "_type" : "terms",
+//   // total number of tokens in the facet
+//   "total": 5,
+//   // @property {number} number of documents which have no value for the field
+//   "missing" : 0,
+//   // number of facet values not included in the returned facets
+//   "other": 0,
+//   // term object ({term: , count: ...})
+//   "terms" : [ {
+//       "term" : "foo",
+//       "count" : 2
+//     }, {
+//       "term" : "bar",
+//       "count" : 2
+//     }, {
+//       "term" : "baz",
+//       "count" : 1
+//     }
+//   ]
+// }
+//
    my.Facet = Backbone.Model.extend({ defaults: function() { return { _type: 'terms', - // total number of tokens in the facet total: 0, - // number of facet values not included in the returned facets other: 0, - // number of documents which have no value for the field missing: 0, - // term object ({term: , count: ...}) terms: [] } } From debcfbbb7a7c8c2c64775737bcf24f6a66ca3d88 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Sat, 7 Apr 2012 13:52:47 +0100 Subject: [PATCH 2/2] [#52,docs][xs]: minor touch up to index page. --- index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.html b/index.html index e06043c6..e7b7d616 100644 --- a/index.html +++ b/index.html @@ -309,9 +309,9 @@ However, it has been rewritten from the ground up using Backbone.

    -

    Use It

    +

    Use the Explorer

    Visit the Data Explorer »

    -

    Get It

    +

    Get the Library

    Development Version
    v0.3 (67k)

    Dependencies