knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#314,backend][s]: remove gdocs backend - now in separate repo.

Browse Source
This commit is contained in:
Rufus Pollock 2013-06-30 22:08:06 +01:00
parent e4adc0c34a
commit d1b04a4d10
7 changed files with 8 additions and 661 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -37,6 +37,7 @@ See CONTRIBUTING.md.
Possible breaking changes
* Many backends moved to their own repositories #314
* Updated Leaflet to latest version 0.4.4 #220
* Added marker clustering in map view to handle a large number of markers
* Dataset.restore method removed (not used internally except from Multiview.restore)

2
_includes/recline-deps.html
View File

@ -53,7 +53,7 @@
<script type="text/javascript" src="{{page.root}}src/model.js"></script>
<script type="text/javascript" src="{{page.root}}src/backend.memory.js"></script>
<script type="text/javascript" src="{{page.root}}src/backend.dataproxy.js"></script>
<script type="text/javascript" src="{{page.root}}src/backend.gdocs.js"></script>
<script type="text/javascript" src="http://okfnlabs.org/recline.backend.gdocs/backend.gdocs.js"></script>
<script type="text/javascript" src="{{page.root}}src/backend.csv.js"></script>
<!-- views -->

168
dist/recline.js vendored
View File

@ -383,174 +383,6 @@ this.recline.Backend.DataProxy = this.recline.Backend.DataProxy || {};
}(this.recline.Backend.DataProxy));
this.recline = this.recline || {};
this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
(function(my) {
"use strict";
my.__type__ = 'gdocs';
// use either jQuery or Underscore Deferred depending on what is available
var Deferred = (typeof jQuery !== "undefined" && jQuery.Deferred) || _.Deferred;
// ## Google spreadsheet backend
//
// Fetch data from a Google Docs spreadsheet.
//
// Dataset must have a url attribute pointing to the Gdocs or its JSON feed e.g.
// <pre>
// var dataset = new recline.Model.Dataset({
// url: 'https://docs.google.com/spreadsheet/ccc?key=0Aon3JiuouxLUdGlQVDJnbjZRSU1tUUJWOUZXRG53VkE#gid=0'
// },
// 'gdocs'
// );
//
// var dataset = new recline.Model.Dataset({
// url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
// },
// 'gdocs'
// );
// </pre>
//
// @return object with two attributes
//
// * fields: array of Field objects
// * records: array of objects for each row
my.fetch = function(dataset) {
var dfd = new Deferred();
var urls = my.getGDocsAPIUrls(dataset.url);
// TODO cover it with tests
// get the spreadsheet title
(function () {
var titleDfd = new Deferred();
jQuery.getJSON(urls.spreadsheet, function (d) {
titleDfd.resolve({
spreadsheetTitle: d.feed.title.$t
});
});
return titleDfd.promise();
}()).then(function (response) {
// get the actual worksheet data
jQuery.getJSON(urls.worksheet, function(d) {
var result = my.parseData(d);
var fields = _.map(result.fields, function(fieldId) {
return {id: fieldId};
});
dfd.resolve({
metadata: {
title: response.spreadsheetTitle +" :: "+ result.worksheetTitle,
spreadsheetTitle: response.spreadsheetTitle,
worksheetTitle : result.worksheetTitle
},
records : result.records,
fields : fields,
useMemoryStore: true
});
});
});
return dfd.promise();
};
// ## parseData
//
// Parse data from Google Docs API into a reasonable form
//
// :options: (optional) optional argument dictionary:
// columnsToUse: list of columns to use (specified by field names)
// colTypes: dictionary (with column names as keys) specifying types (e.g. range, percent for use in conversion).
// :return: tabular data object (hash with keys: field and data).
//
// Issues: seems google docs return columns in rows in random order and not even sure whether consistent across rows.
my.parseData = function(gdocsSpreadsheet, options) {
var options = options || {};
var colTypes = options.colTypes || {};
var results = {
fields : [],
records: []
};
var entries = gdocsSpreadsheet.feed.entry || [];
var key;
var colName;
// percentage values (e.g. 23.3%)
var rep = /^([\d\.\-]+)\%$/;
for(key in entries[0]) {
// it's barely possible it has inherited keys starting with 'gsx$'
if(/^gsx/.test(key)) {
colName = key.substr(4);
results.fields.push(colName);
}
}
// converts non numberical values that should be numerical (22.3%[string] -> 0.223[float])
results.records = _.map(entries, function(entry) {
var row = {};
_.each(results.fields, function(col) {
var _keyname = 'gsx$' + col;
var value = entry[_keyname].$t;
var num;
// TODO cover this part of code with test
// TODO use the regexp only once
// if labelled as % and value contains %, convert
if(colTypes[col] === 'percent' && rep.test(value)) {
num = rep.exec(value)[1];
value = parseFloat(num) / 100;
}
row[col] = value;
});
return row;
});
results.worksheetTitle = gdocsSpreadsheet.feed.title.$t;
return results;
};
// Convenience function to get GDocs JSON API Url from standard URL
my.getGDocsAPIUrls = function(url) {
// https://docs.google.com/spreadsheet/ccc?key=XXXX#gid=YYY
var regex = /.*spreadsheet\/ccc?.*key=([^#?&+]+)[^#]*(#gid=([\d]+).*)?/;
var matches = url.match(regex);
var key;
var worksheet;
var urls;
if(!!matches) {
key = matches[1];
// the gid in url is 0-based and feed url is 1-based
worksheet = parseInt(matches[3], 10) + 1;
if (isNaN(worksheet)) {
worksheet = 1;
}
urls = {
worksheet : 'https://spreadsheets.google.com/feeds/list/'+ key +'/'+ worksheet +'/public/values?alt=json',
spreadsheet: 'https://spreadsheets.google.com/feeds/worksheets/'+ key +'/public/basic?alt=json'
};
}
else {
// we assume that it's one of the feeds urls
key = url.split('/')[5];
// by default then, take first worksheet
worksheet = 1;
urls = {
worksheet : 'https://spreadsheets.google.com/feeds/list/'+ key +'/'+ worksheet +'/public/values?alt=json',
spreadsheet: 'https://spreadsheets.google.com/feeds/worksheets/'+ key +'/public/basic?alt=json'
};
}
return urls;
};
}(this.recline.Backend.GDocs));
this.recline = this.recline || {};
this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.Memory = this.recline.Backend.Memory || {};
(function(my) {

8
docs/tutorial-backends.markdown
View File

@ -76,8 +76,8 @@ much more limited if you are just using a Backend. Specifically:
<script type="text/javascript" src="vendor/jquery/1.7.1/jquery.js"></script>
<script type="text/javascript" src="vendor/underscore/1.1.6/underscore.js"></script>
<script type="text/javascript" src="vendor/backbone/0.5.1/backbone.js"></script>
<!-- include the backend code you need e.g. here for gdocs -->
<script type="text/javascript" src="src/backend.gdocs.js"></script>
<!-- include the backend code you need e.g. here for csv -->
<script type="text/javascript" src="src/backend.csv.js"></script>
<!-- Or you can just include all of recline. -->
<script type="text/javascript" src="dist/recline.js"></script>
@ -99,6 +99,8 @@ a bespoke chooser and a Kartograph (svg-only) map.
</div>
{% highlight javascript %}
// include the Recline backend for Google Docs
<script type="text/javascript" src="http://okfnlabs.org/recline.backend.gdocs/backend.gdocs.js"></script>
{% include example-backends-gdocs.js %}
{% endhighlight %}
@ -106,6 +108,8 @@ a bespoke chooser and a Kartograph (svg-only) map.
<div id="my-gdocs" class="doc-ex-rendered">&nbsp;</div>
<script type="text/javascript" src="http://okfnlabs.org/recline.backend.gdocs/backend.gdocs.js">&nbsp;</script>
<script type="text/javascript">
{% include example-backends-gdocs.js %}
</script>

168
src/backend.gdocs.js
View File

@ -1,168 +0,0 @@
this.recline = this.recline || {};
this.recline.Backend = this.recline.Backend || {};
this.recline.Backend.GDocs = this.recline.Backend.GDocs || {};
(function(my) {
"use strict";
my.__type__ = 'gdocs';
// use either jQuery or Underscore Deferred depending on what is available
var Deferred = (typeof jQuery !== "undefined" && jQuery.Deferred) || _.Deferred;
// ## Google spreadsheet backend
//
// Fetch data from a Google Docs spreadsheet.
//
// Dataset must have a url attribute pointing to the Gdocs or its JSON feed e.g.
// <pre>
// var dataset = new recline.Model.Dataset({
// url: 'https://docs.google.com/spreadsheet/ccc?key=0Aon3JiuouxLUdGlQVDJnbjZRSU1tUUJWOUZXRG53VkE#gid=0'
// },
// 'gdocs'
// );
//
// var dataset = new recline.Model.Dataset({
// url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json'
// },
// 'gdocs'
// );
// </pre>
//
// @return object with two attributes
//
// * fields: array of Field objects
// * records: array of objects for each row
my.fetch = function(dataset) {
var dfd = new Deferred();
var urls = my.getGDocsAPIUrls(dataset.url);
// TODO cover it with tests
// get the spreadsheet title
(function () {
var titleDfd = new Deferred();
jQuery.getJSON(urls.spreadsheet, function (d) {
titleDfd.resolve({
spreadsheetTitle: d.feed.title.$t
});
});
return titleDfd.promise();
}()).then(function (response) {
// get the actual worksheet data
jQuery.getJSON(urls.worksheet, function(d) {
var result = my.parseData(d);
var fields = _.map(result.fields, function(fieldId) {
return {id: fieldId};
});
dfd.resolve({
metadata: {
title: response.spreadsheetTitle +" :: "+ result.worksheetTitle,
spreadsheetTitle: response.spreadsheetTitle,
worksheetTitle : result.worksheetTitle
},
records : result.records,
fields : fields,
useMemoryStore: true
});
});
});
return dfd.promise();
};
// ## parseData
//
// Parse data from Google Docs API into a reasonable form
//
// :options: (optional) optional argument dictionary:
// columnsToUse: list of columns to use (specified by field names)
// colTypes: dictionary (with column names as keys) specifying types (e.g. range, percent for use in conversion).
// :return: tabular data object (hash with keys: field and data).
//
// Issues: seems google docs return columns in rows in random order and not even sure whether consistent across rows.
my.parseData = function(gdocsSpreadsheet, options) {
var options = options || {};
var colTypes = options.colTypes || {};
var results = {
fields : [],
records: []
};
var entries = gdocsSpreadsheet.feed.entry || [];
var key;
var colName;
// percentage values (e.g. 23.3%)
var rep = /^([\d\.\-]+)\%$/;
for(key in entries[0]) {
// it's barely possible it has inherited keys starting with 'gsx$'
if(/^gsx/.test(key)) {
colName = key.substr(4);
results.fields.push(colName);
}
}
// converts non numberical values that should be numerical (22.3%[string] -> 0.223[float])
results.records = _.map(entries, function(entry) {
var row = {};
_.each(results.fields, function(col) {
var _keyname = 'gsx$' + col;
var value = entry[_keyname].$t;
var num;
// TODO cover this part of code with test
// TODO use the regexp only once
// if labelled as % and value contains %, convert
if(colTypes[col] === 'percent' && rep.test(value)) {
num = rep.exec(value)[1];
value = parseFloat(num) / 100;
}
row[col] = value;
});
return row;
});
results.worksheetTitle = gdocsSpreadsheet.feed.title.$t;
return results;
};
// Convenience function to get GDocs JSON API Url from standard URL
my.getGDocsAPIUrls = function(url) {
// https://docs.google.com/spreadsheet/ccc?key=XXXX#gid=YYY
var regex = /.*spreadsheet\/ccc?.*key=([^#?&+]+)[^#]*(#gid=([\d]+).*)?/;
var matches = url.match(regex);
var key;
var worksheet;
var urls;
if(!!matches) {
key = matches[1];
// the gid in url is 0-based and feed url is 1-based
worksheet = parseInt(matches[3], 10) + 1;
if (isNaN(worksheet)) {
worksheet = 1;
}
urls = {
worksheet : 'https://spreadsheets.google.com/feeds/list/'+ key +'/'+ worksheet +'/public/values?alt=json',
spreadsheet: 'https://spreadsheets.google.com/feeds/worksheets/'+ key +'/public/basic?alt=json'
};
}
else {
// we assume that it's one of the feeds urls
key = url.split('/')[5];
// by default then, take first worksheet
worksheet = 1;
urls = {
worksheet : 'https://spreadsheets.google.com/feeds/list/'+ key +'/'+ worksheet +'/public/values?alt=json',
spreadsheet: 'https://spreadsheets.google.com/feeds/worksheets/'+ key +'/public/basic?alt=json'
};
}
return urls;
};
}(this.recline.Backend.GDocs));

320
test/backend.gdocs.test.js
View File

@ -1,320 +0,0 @@
(function ($) {
module("Backend GDocs");
var sampleGDocsSpreadsheetMetadata = {
feed: {
category: [
{
term: "http://schemas.google.com/spreadsheets/2006#worksheet",
scheme: "http://schemas.google.com/spreadsheets/2006"
}
],
updated: {
$t: "2010-07-13T09:57:28.408Z"
},
xmlns: "http://www.w3.org/2005/Atom",
title: {
$t: "javascript-test",
type: "text"
},
author: [
{
name: {
$t: "okfn.rufus.pollock"
},
email: {
$t: "okfn.rufus.pollock@gmail.com"
}
}
],
openSearch$startIndex: {
$t: "1"
},
xmlns$gs: "http://schemas.google.com/spreadsheets/2006",
xmlns$openSearch: "http://a9.com/-/spec/opensearchrss/1.0/",
entry: [
{
category: [
{
term: "http://schemas.google.com/spreadsheets/2006#worksheet",
scheme: "http://schemas.google.com/spreadsheets/2006"
}
],
updated: {
$t: "2010-07-13T09:57:28.408Z"
},
title: {
$t: "Sheet1",
type: "text"
},
content: {
$t: "Sheet1",
type: "text"
},
link: [
{
href: "https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/basic",
type: "application/atom+xml",
rel: "http://schemas.google.com/spreadsheets/2006#listfeed"
},
{
href: "https://spreadsheets.google.com/feeds/cells/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/basic",
type: "application/atom+xml",
rel: "http://schemas.google.com/spreadsheets/2006#cellsfeed"
},
{
href: "https://spreadsheets.google.com/tq?key=0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc&sheet=od6&pub=1",
type: "application/atom+xml",
rel: "http://schemas.google.com/visualization/2008#visualizationApi"
},
{
href: "https://spreadsheets.google.com/feeds/worksheets/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/public/basic/od6",
type: "application/atom+xml",
rel: "self"
}
],
id: {
$t: "https://spreadsheets.google.com/feeds/worksheets/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/public/basic/od6"
}
}
],
link: [
{
href: "https://spreadsheets.google.com/pub?key=0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc",
type: "text/html",
rel: "alternate"
},
{
href: "https://spreadsheets.google.com/feeds/worksheets/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/public/basic",
type: "application/atom+xml",
rel: "http://schemas.google.com/g/2005#feed"
},
{
href: "https://spreadsheets.google.com/feeds/worksheets/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/public/basic?alt=json",
type: "application/atom+xml",
rel: "self"
}
],
openSearch$totalResults: {
$t: "1"
},
id: {
$t: "https://spreadsheets.google.com/feeds/worksheets/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/public/basic"
}
},
version: "1.0",
encoding: "UTF-8"
}
var sampleGDocsSpreadsheetData = {
feed: {
category: [
{
term: "http://schemas.google.com/spreadsheets/2006#list",
scheme: "http://schemas.google.com/spreadsheets/2006"
}
],
updated: {
$t: "2010-07-12T18:32:16.200Z"
},
xmlns: "http://www.w3.org/2005/Atom",
xmlns$gsx: "http://schemas.google.com/spreadsheets/2006/extended",
title: {
$t: "Sheet1",
type: "text"
},
author: [
{
name: {
$t: "okfn.rufus.pollock"
},
email: {
$t: "okfn.rufus.pollock@gmail.com"
}
}
],
openSearch$startIndex: {
$t: "1"
},
link: [
{
href: "http://spreadsheets.google.com/pub?key=0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc",
type: "text/html",
rel: "alternate"
},
{
href: "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values",
type: "application/atom+xml",
rel: "http://schemas.google.com/g/2005#feed"
},
{
href: "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json-in-script",
type: "application/atom+xml",
rel: "self"
}
],
xmlns$openSearch: "http://a9.com/-/spec/opensearchrss/1.0/",
entry: [
{
category: [
{
term: "http://schemas.google.com/spreadsheets/2006#list",
scheme: "http://schemas.google.com/spreadsheets/2006"
}
],
updated: {
$t: "2010-07-12T18:32:16.200Z"
},
'gsx$column-2': {
$t: "1"
},
'gsx$column-1': {
$t: "A"
},
title: {
$t: "A",
type: "text"
},
content: {
$t: "column-2: 1",
type: "text"
},
link: [
{
href: "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cokwr",
type: "application/atom+xml",
rel: "self"
}
],
id: {
$t: "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cokwr"
}
},
{
category: [
{
term: "http://schemas.google.com/spreadsheets/2006#list",
scheme: "http://schemas.google.com/spreadsheets/2006"
}
],
updated: {
$t: "2010-07-12T18:32:16.200Z"
},
'gsx$column-2': {
$t: "2"
},
'gsx$column-1': {
$t: "b"
},
title: {
$t: "b",
type: "text"
},
content: {
$t: "column-2: 2",
type: "text"
},
link: [
{
href: "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cpzh4",
type: "application/atom+xml",
rel: "self"
}
],
id: {
$t: "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cpzh4"
}
},
{
category: [
{
term: "http://schemas.google.com/spreadsheets/2006#list",
scheme: "http://schemas.google.com/spreadsheets/2006"
}
],
updated: {
$t: "2010-07-12T18:32:16.200Z"
},
'gsx$column-2': {
$t: "3"
},
'gsx$column-1': {
$t: "c"
},
title: {
$t: "c",
type: "text"
},
content: {
$t: "column-2: 3",
type: "text"
},
link: [
{
href: "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cre1l",
type: "application/atom+xml",
rel: "self"
}
],
id: {
$t: "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values/cre1l"
}
}
],
openSearch$totalResults: {
$t: "3"
},
id: {
$t: "http://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values"
}
},
version: "1.0",
encoding: "UTF-8"
}
test("GDocs Backend", function() {
var dataset = new recline.Model.Dataset({
url: 'https://spreadsheets.google.com/feeds/list/0Aon3JiuouxLUdDQwZE1JdV94cUd6NWtuZ0IyWTBjLWc/od6/public/values?alt=json',
backend: 'gdocs'
});
var stub = sinon.stub($, 'getJSON', function(options, cb) {
var spreadsheetUrl = 'spreadsheets.google.com/feeds/worksheets/';
var worksheetUrl = 'spreadsheets.google.com/feeds/list/';
if (options.indexOf(spreadsheetUrl) !== -1) {
cb(sampleGDocsSpreadsheetMetadata)
}
else if(options.indexOf(worksheetUrl) !== -1) {
cb(sampleGDocsSpreadsheetData)
}
});
dataset.fetch().then(function() {
var docList = dataset.records;
deepEqual(['column-2', 'column-1'], _.pluck(dataset.fields.toJSON(), 'id'));
equal(3, docList.length);
equal("A", docList.models[0].get('column-1'));
equal('javascript-test :: Sheet1', dataset.get('title'));
});
$.getJSON.restore();
});
test("GDocs Backend.getUrl", function() {
var key = 'Abc_dajkdkjdafkj';
var gid = 0;
var worksheet = 1;
var url = 'https://docs.google.com/spreadsheet/ccc?key=' + key + '#gid=' + gid
var out = recline.Backend.GDocs.getGDocsAPIUrls(url);
var exp1 = 'https://spreadsheets.google.com/feeds/list/' + key + '/' + worksheet + '/public/values?alt=json'
var exp2 = 'https://spreadsheets.google.com/feeds/worksheets/' + key + '/public/basic?alt=json'
equal(exp1, out.worksheet);
equal(exp2, out.spreadsheet);
var url = 'https://docs.google.com/spreadsheet/ccc?key=' + key;
var out = recline.Backend.GDocs.getGDocsAPIUrls(url);
equal(out.worksheet, exp1);
});
})(this.jQuery);

2
test/index.html
View File

@ -40,13 +40,11 @@
<script type="text/javascript" src="../src/model.js"></script>
<script type="text/javascript" src="../src/backend.memory.js"></script>
<script type="text/javascript" src="../src/backend.dataproxy.js"></script>
<script type="text/javascript" src="../src/backend.gdocs.js"></script>
<script type="text/javascript" src="../src/backend.csv.js"></script>
<script type="text/javascript" src="model.test.js"></script>
<script type="text/javascript" src="backend.memory.test.js"></script>
<script type="text/javascript" src="backend.dataproxy.test.js"></script>
<script type="text/javascript" src="backend.gdocs.test.js"></script>
<script type="text/javascript" src="backend.csv.test.js"></script>
<!-- views and view tests -->