knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #171 from slmnhq/master

Browse Source 
[#162][s]: refactor couchdb.js to be inline with new backend style - from @slmnhq.
This commit is contained in:
Rufus Pollock
2012-06-26 12:33:02 -07:00
parent 3d6ad46cc5 fb9d154834
commit 3a2587bba0
1 changed files with 315 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

332
src/backend/couchdb.js
View File

@@ -3,6 +3,8 @@ this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {}; this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
(function($, my) { (function($, my) {
my.__type__ = 'couchdb';
// ## CouchDB Wrapper // ## CouchDB Wrapper
// //
// Connecting to [CouchDB] (http://www.couchdb.apache.org/) endpoints. // Connecting to [CouchDB] (http://www.couchdb.apache.org/) endpoints.
@@ -32,7 +34,7 @@ this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
} }
}; };
} }
var data = _.extend(extras, data); data = _.extend(extras, data);
return $.ajax(data); return $.ajax(data);
}; };
@@ -150,65 +152,54 @@ this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
// //
// Usage: // Usage:
// //
// var backend = new recline.Backend.CouchDB({ // var backend = new recline.Backend.CouchDB();
// var dataset = new recline.Model.Dataset({
// db_url: '/couchdb/mydb', // db_url: '/couchdb/mydb',
// view_url: '/couchdb/mydb/_design/design1/_views/view1', // view_url: '/couchdb/mydb/_design/design1/_views/view1',
// query_options: { // query_options: {
// 'key': 'some_document_key' // 'key': 'some_document_key'
// } // }
// }); // });
// backend.fetch(dataset.toJSON());
// backend.query(query, dataset.toJSON()).done(function () { ... });
// //
// If these options are not passed to the constructor, the model // Alternatively:
// object is checked for the presence of these arguments. // var dataset = new recline.Model.Dataset({ ... }, 'couchdb');
// dataset.fetch();
// var results = dataset.query(query_obj);
// //
// Additionally, the Dataset instance may define two methods: // Additionally, the Dataset instance may define three methods:
// function record_update (record, document) { ... } // function record_update (record, document) { ... }
// function record_delete (record, document) { ... } // function record_delete (record, document) { ... }
// function record_create (record, document) { ... }
// Where `record` is the JSON representation of the Record/Document instance // Where `record` is the JSON representation of the Record/Document instance
// and `document` is the JSON document stored in couchdb. // and `document` is the JSON document stored in couchdb.
// When _all_docs view is used (default), a record is the same as a document // When _all_docs view is used (default), a record is the same as a document
// so these methods need not be defined. // so these methods need not be defined.
// They are most useful when using a custom view that performs a map-reduce // They are most useful when using a custom view that performs a map-reduce
// operation on each document to yield a record. Hence, when the record is // operation on each document to yield a record. Hence, when the record is
// updated or deleted, an inverse operation must be performed on the original // created, updated or deleted, an inverse operation must be performed on the original
// document. // document.
// //
// @param {string} url of couchdb database. // @param {string} url of couchdb database.
// @param {string} (optional) url of couchdb view. default:`db_url`/_all_docs // @param {string} (optional) url of couchdb view. default:`db_url`/_all_docs
// @param {Object} (optional) query options accepted by couchdb views. // @param {Object} (optional) query options accepted by couchdb views.
// @param {string} (optional) url of Elastic Search engine.
// //
my.Backbone = function(options) {
var self = this;
this.__type__ ='couchdb';
this.db_url = options['db_url'] || '';
this.view_url = options['view_url'] || this.db_url + '/_all_docs';
this.query_options = options['query_options'] || {};
// ### sync my.couchOptions = {};
//
// Backbone sync implementation for this backend.
//
this.sync = function (method, model, options) {
var dataset = null
if (model.__type__ == 'Dataset')
dataset = model;
else
dataset = model.dataset;
var db_url = dataset.get('db_url') || self.db_url;
var view_url = dataset.get('view_url') || self.view_url;
var query_options = dataset.get('query_options') || self.query_options;
// ### fetch
// @param {object} dataset json object with the db_url, view_url, and query_options args.
// @return promise object that resolves to the document mapping.
my.fetch = function (dataset) {
var db_url = dataset.db_url;
var view_url = dataset.view_url;
var cdb = new my.CouchDBWrapper(db_url, view_url); var cdb = new my.CouchDBWrapper(db_url, view_url);
if (method === "read") {
if (model.__type__ == 'Dataset') {
var dfd = $.Deferred(); var dfd = $.Deferred();
// if 'doc' attribute is present, return schema of that // if 'doc' attribute is present, return schema of that
// else return schema of 'value' attribute which contains // else return schema of 'value' attribute which contains
// the map-reduce document. // the map-reduced document.
cdb.mapping().done(function(result) { cdb.mapping().done(function(result) {
var row = result.rows[0]; var row = result.rows[0];
var keys = []; var keys = [];
@@ -223,114 +214,74 @@ this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
var fieldData = _.map(keys, function(k) { var fieldData = _.map(keys, function(k) {
return { 'id' : k }; return { 'id' : k };
}); });
model.fields.reset(fieldData); dfd.resolve({
fields: fieldData
dfd.resolve(model); });
}) })
.fail(function(arguments) { .fail(function(arguments) {
dfd.reject(arguments); dfd.reject(arguments);
}); });
return dfd.promise(); return dfd.promise();
} else if (model.__type__ == 'Record' || model.__type__ == 'Document') { };
if (view_url.search("_all_docs") !== -1) {
return cdb.get(model.get('_id'));
}
else {
var dfd = $.Deferred();
// decompose _id
var id = model.get('_id').split('__')[0];
var key = model.get('_id').split('__')[1];
var jqxhr = cdb.query(model.dataset.get('query_options'));
jqxhr.done(function(records) { // ### save
var doc = {}; //
var rec = _.filter(records, function (record) { // Iterate through all the changes and save them to the server.
record['id'] == id && record['key'] == key; // N.B. This method is asynchronous and attempts to do multiple
}); // operation concurrently. This can be problematic when more than
doc = rec[0]; // XXX check len first // one operation is requested on the same document (as in the case
doc['id'] = doc['_id'] = model.get('_id'); // of bulk column transforms).
dfd.resolve(doc); //
}).fail(function(args) { // @param {object} lists of create, update, delete.
dfd.reject(args); // @param {object} dataset json object.
}); //
return dfd.promise(); //
} my.save = function (changes, dataset) {
}
} else if (method === 'update') {
if (model.__type__ == 'Record' || model.__type__ == 'Document') {
var dfd = $.Deferred(); var dfd = $.Deferred();
var _id = null; var total = changes.creates.length + changes.updates.length + changes.deletes.length;
var jqxhr; var results = {'done': [], 'fail': [] };
var new_doc = model.toJSON();
// couchdb uses _id to identify documents, Backbone models use id. var decr_cb = function () { total -= 1; }
// we should remove it before sending it to the server. var resolve_cb = function () { if (total == 0) dfd.resolve(results); }
delete new_doc['id'];
if (view_url.search('_all_docs') !== -1) { for (var i in changes.creates) {
_id = model.get('_id'); var new_doc = changes.creates[i];
jqxhr = cdb.get(_id); var succ_cb = function (msg) {results.done.push({'op': 'create', 'record': new_doc, 'reason': ''}); }
} var fail_cb = function (msg) {results.fail.push({'op': 'create', 'record': new_doc, 'reason': msg}); }
else {
_id = model.get('_id').split('__')[0]; _createDocument(new_doc, dataset).then([decr_cb, succ_cb, resolve_cb], [decr_cb, fail_cb, resolve_cb]);
jqxhr = cdb.get(_id);
} }
jqxhr.done(function(old_doc){ for (var i in changes.updates) {
if (model.dataset.record_update) var new_doc = changes.updates[i];
new_doc = model.dataset.record_update(new_doc, old_doc); var succ_cb = function (msg) {results.done.push({'op': 'update', 'record': new_doc, 'reason': ''}); }
new_doc = _.extend(old_doc, new_doc); var fail_cb = function (msg) {results.fail.push({'op': 'update', 'record': new_doc, 'reason': msg}); }
new_doc['_id'] = _id;
// XXX upsert can fail during a bulk column transform due to revision conflict. _updateDocument(new_doc, dataset).then([decr_cb, succ_cb, resolve_cb], [decr_cb, fail_cb, resolve_cb]);
// the correct way to handle bulk column transforms is to use }
// a queue which is processed by a web worker.
dfd.resolve(cdb.upsert(new_doc)); for (var i in changes.deletes) {
}).fail(function(args){ var old_doc = changes.deletes[i];
dfd.reject(args); var succ_cb = function (msg) {results.done.push({'op': 'delete', 'record': old_doc, 'reason': ''}); }
}); var fail_cb = function (msg) {results.fail.push({'op': 'delete', 'record': old_doc, 'reason': msg}); }
_deleteDocument(new_doc, dataset).then([decr_cb, succ_cb, resolve_cb], [decr_cb, fail_cb, resolve_cb]);
}
return dfd.promise(); return dfd.promise();
} };
} else if (method === 'delete') {
if (model.__type__ == 'Record' || model.__type__ == 'Document') {
if (view_url.search('_all_docs') !== -1) // ### query
return cdb.delete(model.get('_id')); //
else { // fetch the data from the couchdb view and filter it.
// @param {Object} recline.Dataset instance
// @param {Object} recline.Query instance.
my.query = function(queryObj, dataset) {
var dfd = $.Deferred(); var dfd = $.Deferred();
var _id = model.get('_id').split('__')[0]; var db_url = dataset.db_url;
var new_doc = null; var view_url = dataset.view_url;
var jqxhr = cdb.get(_id); var query_options = dataset.query_options;
jqxhr.done(function(old_doc){
if (model.dataset.record_delete)
new_doc = model.dataset.record_delete(model.toJSON(), old_doc);
if (_.isNull(new_doc))
dfd.resolve(cdb.delete(_id)); // XXX is this the right thing to do?
else {
// couchdb uses _id to identify documents, Backbone models use id.
// we should remove it before sending it to the server.
new_doc['_id'] = _id;
delete new_doc['id'];
dfd.resolve(cdb.upsert(new_doc));
}
}).fail(function(args){
dfd.reject(args);
});
return dfd.promise();
}
}
}
},
// ### query
//
// fetch the data from the couchdb view and filter it.
// @param {Object} recline.Dataset instance
// @param {Object} recline.Query instance.
this.query = function(model, queryObj) {
var dfd = $.Deferred();
var db_url = model.get('db_url') || self.db_url;
var view_url = model.get('view_url') || self.view_url;
var query_options = model.get('query_options') || self.query_options;
var cdb = new my.CouchDBWrapper(db_url, view_url); var cdb = new my.CouchDBWrapper(db_url, view_url);
var cdb_q = cdb._normalizeQuery(queryObj, query_options); var cdb_q = cdb._normalizeQuery(queryObj, query_options);
@@ -341,18 +292,17 @@ this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
_.each(records.rows, function(record) { _.each(records.rows, function(record) {
var doc = {}; var doc = {};
if (record.hasOwnProperty('doc')) { if (record.hasOwnProperty('doc')) {
doc['_source'] = record['doc']; doc = record['doc'];
// couchdb uses _id to identify documents, Backbone models use id. // couchdb uses _id to identify documents, Backbone models use id.
// we add this fix so backbone.Model works correctly. // we add this fix so backbone.Model works correctly.
doc['_source']['id'] = doc['_source']['_id']; doc['id'] = doc['_id'];
} }
else { else {
doc['_source'] = record['value']; doc = record['value'];
// using dunder to create compound id. need something more robust. // using dunder to create compound id. need something more robust.
doc['_source']['_id'] = record['id'] + '__' + record['key'];
// couchdb uses _id to identify documents, Backbone models use id. // couchdb uses _id to identify documents, Backbone models use id.
// we add this fix so backbone.Model works correctly. // we add this fix so backbone.Model works correctly.
doc['_source']['id'] = doc['_source']['_id']; doc['_id'] = doc['id'] = record['id'] + '__' + record['key'];
} }
query_result.total += 1; query_result.total += 1;
query_result.hits.push(doc); query_result.hits.push(doc);
@@ -361,8 +311,8 @@ this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
// the following block is borrowed verbatim from recline.backend.Memory // the following block is borrowed verbatim from recline.backend.Memory
// search (with filtering, faceting, and sorting) should be factored // search (with filtering, faceting, and sorting) should be factored
// out into a separate library. // out into a separate library.
query_result.hits = self._applyFilters(query_result.hits, queryObj); query_result.hits = _applyFilters(query_result.hits, queryObj);
query_result.hits = self._applyFreeTextQuery(query_result.hits, queryObj); query_result.hits = _applyFreeTextQuery(query_result.hits, queryObj);
// not complete sorting! // not complete sorting!
_.each(queryObj.sort, function(sortObj) { _.each(queryObj.sort, function(sortObj) {
var fieldName = _.keys(sortObj)[0]; var fieldName = _.keys(sortObj)[0];
@@ -372,36 +322,34 @@ this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
}); });
}); });
query_result.total = query_result.hits.length; query_result.total = query_result.hits.length;
query_result.facets = self.computeFacets(query_result.hits, queryObj); query_result.facets = _computeFacets(query_result.hits, queryObj);
query_result.hits = query_result.hits.slice(cdb_q.skip, cdb_q.skip + cdb_q.limit+1); query_result.hits = query_result.hits.slice(cdb_q.skip, cdb_q.skip + cdb_q.limit+1);
dfd.resolve(query_result); dfd.resolve(query_result);
}); });
return dfd.promise(); return dfd.promise();
}, };
// in place filtering // in place filtering
_applyFilters: function(results, queryObj) { _applyFilters = function(results, queryObj) {
_.each(queryObj.filters, function(filter) { _.each(queryObj.filters, function(filter) {
results = _.filter(results, function(doc) { results = _.filter(results, function(doc) {
var fieldId = _.keys(filter.term)[0]; var fieldId = _.keys(filter.term)[0];
return (doc['_source'][fieldId] == filter.term[fieldId]); return (doc[fieldId] == filter.term[fieldId]);
}); });
}); });
return results; return results;
}, };
// we OR across fields but AND across terms in query string // we OR across fields but AND across terms in query string
_applyFreeTextQuery: function(results, queryObj) { _applyFreeTextQuery = function(results, queryObj) {
if (queryObj.q) { if (queryObj.q) {
var terms = queryObj.q.split(' '); var terms = queryObj.q.split(' ');
results = _.filter(results, function(rawdoc) { results = _.filter(results, function(rawdoc) {
rawdoc = rawdoc['_source'];
var matches = true; var matches = true;
_.each(terms, function(term) { _.each(terms, function(term) {
var foundmatch = false; var foundmatch = false;
//_.each(self.fields, function(field) {
_.each(_.keys(rawdoc), function(field) { _.each(_.keys(rawdoc), function(field) {
var value = rawdoc[field]; var value = rawdoc[field];
if (value !== null) { value = value.toString(); } if (value !== null) { value = value.toString(); }
@@ -418,9 +366,9 @@ this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
}); });
} }
return results; return results;
}, };
computeFacets: function(records, queryObj) { _computeFacets = function(records, queryObj) {
var facetResults = {}; var facetResults = {};
if (!queryObj.facets) { if (!queryObj.facets) {
return facetResults; return facetResults;
@@ -434,7 +382,7 @@ this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
_.each(records, function(doc) { _.each(records, function(doc) {
_.each(queryObj.facets, function(query, facetId) { _.each(queryObj.facets, function(query, facetId) {
var fieldId = query.terms.field; var fieldId = query.terms.field;
var val = doc['_source'][fieldId]; var val = doc[fieldId];
var tmp = facetResults[facetId]; var tmp = facetResults[facetId];
if (val) { if (val) {
tmp.termsall[val] = tmp.termsall[val] ? tmp.termsall[val] + 1 : 1; tmp.termsall[val] = tmp.termsall[val] ? tmp.termsall[val] + 1 : 1;
@@ -455,9 +403,97 @@ this.recline.Backend.CouchDB = this.recline.Backend.CouchDB || {};
tmp.terms = tmp.terms.slice(0, 10); tmp.terms = tmp.terms.slice(0, 10);
}); });
return facetResults; return facetResults;
}, };
_createDocument = function (new_doc, dataset) {
var dfd = $.Deferred();
var db_url = dataset.db_url;
var view_url = dataset.view_url;
var _id = new_doc['id'];
var cdb = new my.CouchDBWrapper(db_url, view_url);
delete new_doc['id'];
if (view_url.search('_all_docs') !== -1) {
jqxhr = cdb.get(_id);
}
else {
_id = new_doc['_id'].split('__')[0];
jqxhr = cdb.get(_id);
}
jqxhr.done(function(old_doc){
if (dataset.record_create)
new_doc = dataset.record_create(new_doc, old_doc);
new_doc = _.extend(old_doc, new_doc);
new_doc['_id'] = _id;
dfd.resolve(cdb.upsert(new_doc));
}).fail(function(args){
dfd.reject(args);
}); });
return dfd.promise();
};
_updateDocument = function (new_doc, dataset) {
var dfd = $.Deferred();
var db_url = dataset.db_url;
var view_url = dataset.view_url;
var _id = new_doc['id'];
var cdb = new my.CouchDBWrapper(db_url, view_url);
delete new_doc['id'];
if (view_url.search('_all_docs') !== -1) {
jqxhr = cdb.get(_id);
}
else {
_id = new_doc['_id'].split('__')[0];
jqxhr = cdb.get(_id);
}
jqxhr.done(function(old_doc){
if (dataset.record_update)
new_doc = dataset.record_update(new_doc, old_doc);
new_doc = _.extend(old_doc, new_doc);
new_doc['_id'] = _id;
dfd.resolve(cdb.upsert(new_doc));
}).fail(function(args){
dfd.reject(args);
});
return dfd.promise();
};
_deleteDocument = function (del_doc, dataset) {
var dfd = $.Deferred();
var db_url = dataset.db_url;
var view_url = dataset.view_url;
var _id = del_doc['id'];
var cdb = new my.CouchDBWrapper(db_url, view_url);
if (view_url.search('_all_docs') !== -1)
return cdb.delete(_id);
else {
_id = model.get('_id').split('__')[0];
var jqxhr = cdb.get(_id);
jqxhr.done(function(old_doc){
if (dataset.record_delete)
old_doc = dataset.record_delete(del_doc, old_doc);
if (_.isNull(del_doc))
dfd.resolve(cdb.delete(_id)); // XXX is this the right thing to do?
else {
// couchdb uses _id to identify documents, Backbone models use id.
// we should remove it before sending it to the server.
old_doc['_id'] = _id;
delete old_doc['id'];
dfd.resolve(cdb.upsert(old_doc));
}
}).fail(function(args){
dfd.reject(args);
});
return dfd.promise();
};
}(jQuery, this.recline.Backend.CouchDB)); }(jQuery, this.recline.Backend.CouchDB));