knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#154,model/query][m]: refactor to new filter structure (see ticket) updating FilterEditor widget and backends.

Browse Source 
* ElasticSearch changes represents a significant refactor and now support filters and query via constant_score (did not support this before!)
This commit is contained in:
Rufus Pollock 2012-06-16 13:04:03 +01:00
parent 617d3440f0
commit f14dcdcaaf
8 changed files with 124 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
src/backend/elasticsearch.js
View File

@ -87,44 +87,57 @@ this.recline.Backend.ElasticSearch = this.recline.Backend.ElasticSearch || {};
};
this._normalizeQuery = function(queryObj) {
var out = queryObj && queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj);
if (out.q !== undefined && out.q.trim() === '') {
delete out.q;
}
if (!out.q) {
out.query = {
var self = this;
var queryInfo = (queryObj && queryObj.toJSON) ? queryObj.toJSON() : _.extend({}, queryObj);
var out = {
constant_score: {
query: {}
}
};
if (!queryInfo.q) {
out.constant_score.query = {
match_all: {}
};
} else {
out.query = {
out.constant_score.query = {
query_string: {
query: out.q
query: queryInfo.q
}
};
delete out.q;
}
// now do filters (note the *plural*)
if (out.filters && out.filters.length) {
if (!out.filter) {
out.filter = {};
}
if (!out.filter.and) {
out.filter.and = [];
}
out.filter.and = out.filter.and.concat(out.filters);
}
if (out.filters !== undefined) {
delete out.filters;
if (queryInfo.filters && queryInfo.filters.length) {
out.constant_score.filter = {
and: []
};
_.each(queryInfo.filters, function(filter) {
out.constant_score.filter.and.push(self._convertFilter(filter));
});
}
return out;
};
},
this._convertFilter = function(filter) {
var out = {};
out[filter.type] = {}
if (filter.type === 'term') {
out.term[filter.field] = filter.term;
} else if (filter.type === 'geo_point') {
out.geo_point[filter.field] = filter.point;
out.geo_point[filter.distance] = filter.distance;
}
return out;
},
// ### query
//
// @return deferred supporting promise API
this.query = function(queryObj) {
var esQuery = (queryObj && queryObj.toJSON) ? queryObj.toJSON() : _.extend({}, queryObj);
var queryNormalized = this._normalizeQuery(queryObj);
var data = {source: JSON.stringify(queryNormalized)};
delete esQuery.q;
delete esQuery.filters;
esQuery.query = queryNormalized;
var data = {source: JSON.stringify(esQuery)};
var url = this.endpoint + '/_search';
var jqxhr = recline.Backend.makeRequest({
url: url,

5
src/backend/memory.js
View File

@ -91,10 +91,9 @@ this.recline.Backend.Memory = this.recline.Backend.Memory || {};
this._applyFilters = function(results, queryObj) {
_.each(queryObj.filters, function(filter) {
// if a term filter ...
if (filter.term) {
if (filter.type === 'term') {
results = _.filter(results, function(doc) {
var fieldId = _.keys(filter.term)[0];
return (doc[fieldId] == filter.term[fieldId]);
return (doc[filter.field] == filter.term);
});
}
});

32
src/model.js
View File

@ -411,39 +411,39 @@ my.Query = Backbone.Model.extend({
return {
size: 100,
from: 0,
q: '',
facets: {},
// <http://www.elasticsearch.org/guide/reference/query-dsl/and-filter.html>
// , filter: {}
filters: []
};
},
_filterTemplates: {
term: {
'{{fieldId}}': ''
type: 'term',
field: '',
term: ''
},
geo_distance: {
distance: '10km',
'{{fieldId}}': {
point: {
lon: 0,
lat: 0
}
}
},
},
// ### addFilter
//
// Add a new filter (appended to the list of filters) with default
// value. To set value for the filter use updateFilter.
// Add a new filter (appended to the list of filters)
//
// @param type type of this filter e.g. term, geo_distance etc
// @param fieldId the field to add this filter on
addFilter: function(type, fieldId) {
var tmpl = JSON.stringify(this._filterTemplates[type]);
// @param filter an object specifying the filter - see _filterTemplates for examples. If only type is provided will generate a filter by cloning _filterTemplates
addFilter: function(filter) {
// crude deep copy
var ourfilter = JSON.parse(JSON.stringify(filter));
// not full specified so use template and over-write
if (_.keys(filter).length <= 2) {
ourfilter = _.extend(this._filterTemplates[filter.type], ourfilter);
}
var filters = this.get('filters');
var filter = {};
filter[type] = JSON.parse(Mustache.render(tmpl, {type: type, fieldId: fieldId}));
filter[type]._type = type;
filter[type]._field = fieldId;
filters.push(filter);
filters.push(ourfilter);
this.trigger('change:filters:new-blank');
},
updateFilter: function(index, value) {

31
src/widget.filtereditor.js
View File

@ -39,22 +39,22 @@ my.FilterEditor = Backbone.View.extend({
',
filterTemplates: {
term: ' \
<div class="control-group filter-{{_type}} filter"> \
<label class="control-label" for="">{{_field}}</label> \
<div class="control-group filter-{{type}} filter"> \
<label class="control-label" for="">{{field}}</label> \
<div class="controls"> \
<input type="text" value="{{_value}}" name="term" data-filter-field="{{_field}}" data-filter-id="{{id}}" data-filter-type="{{_type}}" /> \
<input type="text" value="{{term}}" name="term" data-filter-field="{{field}}" data-filter-id="{{id}}" data-filter-type="{{type}}" /> \
<a class="js-remove-filter" href="#">&times;</a> \
</div> \
</div> \
',
geo_distance: ' \
<div class="control-group filter-{{_type}} filter"> \
<label class="control-label" for="">{{_field}}</label> \
<div class="control-group filter-{{type}} filter"> \
<label class="control-label" for="">{{field}}</label> \
<a class="js-remove-filter" href="#">&times;</a> \
<div class="controls"> \
<input type="text" value="{{_value.lon}}" name="lon" data-filter-field="{{_field}}" data-filter-id="{{id}}" data-filter-type="{{_type}}" /> \
<input type="text" value="{{_value.lat}}" name="lat" data-filter-field="{{_field}}" data-filter-id="{{id}}" data-filter-type="{{_type}}" /> \
<input type="text" value="{{distance}}" name="distance" data-filter-field="{{_field}}" data-filter-id="{{id}}" data-filter-type="{{_type}}" /> \
<input type="text" value="{{point.lon}}" name="lon" data-filter-field="{{field}}" data-filter-id="{{id}}" data-filter-type="{{type}}" /> \
<input type="text" value="{{point.lat}}" name="lat" data-filter-field="{{field}}" data-filter-id="{{id}}" data-filter-type="{{type}}" /> \
<input type="text" value="{{distance}}" name="distance" data-filter-field="{{field}}" data-filter-id="{{id}}" data-filter-type="{{type}}" /> \
</div> \
</div> \
'
@ -83,12 +83,7 @@ my.FilterEditor = Backbone.View.extend({
});
tmplData.fields = this.model.fields.toJSON();
tmplData.filterRender = function() {
var filterType = _.keys(this)[0];
var _data = this[filterType];
_data.id = this.id;
_data._type = filterType;
_data._value = _data[_data._field];
return Mustache.render(self.filterTemplates[filterType], _data);
return Mustache.render(self.filterTemplates[this.type], this);
};
var out = Mustache.render(this.template, tmplData);
this.el.html(out);
@ -105,7 +100,7 @@ my.FilterEditor = Backbone.View.extend({
$target.hide();
var filterType = $target.find('select.filterType').val();
var field = $target.find('select.fields').val();
this.model.queryState.addFilter(filterType, field);
this.model.queryState.addFilter({type: filterType, field: field});
// trigger render explicitly as queryState change will not be triggered (as blank value for filter)
this.render();
},
@ -128,12 +123,12 @@ my.FilterEditor = Backbone.View.extend({
var name = $input.attr('name');
var value = $input.val();
if (filterType === 'term') {
filters[filterIndex].term[fieldId] = value;
filters[filterIndex].term = value;
} else if (filterType === 'geo_distance') {
if (name === 'distance') {
filters[filterIndex].geo_distance.distance = parseInt(value);
filters[filterIndex].distance = parseInt(value);
} else {
filters[filterIndex].geo_distance[fieldId][name] = parseFloat(value);
filters[filterIndex].point[name] = parseFloat(value);
}
}
});

54
test/backend/elasticsearch.test.js
View File

@ -3,32 +3,52 @@ module("Backend ElasticSearch - Wrapper");
test("queryNormalize", function() {
var backend = new recline.Backend.ElasticSearch.Wrapper();
var in_ = new recline.Model.Query();
var out = backend._normalizeQuery(in_);
equal(out.size, 100);
var exp = {
constant_score: {
query: {
match_all: {}
}
}
};
deepEqual(out, exp);
var in_ = new recline.Model.Query();
in_.set({q: ''});
var out = backend._normalizeQuery(in_);
equal(out.q, undefined);
deepEqual(out.query.match_all, {});
var in_ = new recline.Model.Query().toJSON();
in_.q = '';
var out = backend._normalizeQuery(in_);
equal(out.q, undefined);
deepEqual(out.query.match_all, {});
var in_ = new recline.Model.Query().toJSON();
in_.q = 'abc';
var out = backend._normalizeQuery(in_);
equal(out.query.query_string.query, 'abc');
deepEqual(out, exp);
var in_ = new recline.Model.Query();
in_.addTermFilter('xyz', 'XXX');
in_ = in_.toJSON();
in_.attributes.q = 'abc';
var out = backend._normalizeQuery(in_);
deepEqual(out.filter.and[0], {term: { xyz: 'XXX'}});
equal(out.constant_score.query.query_string.query, 'abc');
var in_ = new recline.Model.Query();
in_.addFilter({
type: 'term',
field: 'xyz',
term: 'XXX'
});
var out = backend._normalizeQuery(in_);
var exp = {
constant_score: {
query: {
match_all: {}
},
filter: {
and: [
{
term: {
xyz: 'XXX'
}
}
]
}
}
};
deepEqual(out, exp);
});
var mapping_data = {

4
test/backend/memory.test.js
View File

@ -60,7 +60,7 @@ test('query string', function () {
test('filters', function () {
var data = _wrapData();
var query = new recline.Model.Query();
query.addTermFilter('country', 'UK');
query.addFilter({type: 'term', field: 'country', term: 'UK'});
var out = data.query(query.toJSON());
equal(out.total, 3);
deepEqual(_.pluck(out.records, 'country'), ['UK', 'UK', 'UK']);
@ -198,7 +198,7 @@ test('query string', function () {
test('filters', function () {
var dataset = makeBackendDataset();
dataset.queryState.addTermFilter('country', 'UK');
dataset.queryState.addFilter({type: 'term', field: 'country', term: 'UK'});
dataset.query().then(function() {
equal(dataset.currentRecords.length, 3);
deepEqual(dataset.currentRecords.pluck('country'), ['UK', 'UK', 'UK']);

30
test/model.test.js
View File

@ -151,27 +151,23 @@ test('Query', function () {
test('Query.addFilter', function () {
var query = new recline.Model.Query();
query.addFilter('term', 'xyz');
query.addFilter({type: 'term', field: 'xyz'});
var exp = {
term: {
xyz: '',
_field: 'xyz',
_type: 'term'
}
field: 'xyz',
type: 'term',
term: ''
};
deepEqual(exp, query.get('filters')[0]);
deepEqual(query.get('filters')[0], exp);
query.addFilter('geo_distance', 'xyz');
query.addFilter({type: 'geo_distance', field: 'xyz'});
var exp = {
geo_distance: {
distance: '10km',
xyz: {
lon: 0,
lat: 0
},
_field: 'xyz',
_type: 'geo_distance'
}
distance: '10km',
point: {
lon: 0,
lat: 0
},
field: 'xyz',
type: 'geo_distance'
};
deepEqual(exp, query.get('filters')[1]);
});

9
test/widget.filtereditor.test.js
View File

@ -21,12 +21,12 @@ test('basics', function () {
ok(!$addForm.is(":visible"));
$editForm = view.el.find('form.js-edit');
equal($editForm.find('.filter-term').length, 1)
equal(_.keys(dataset.queryState.attributes.filters[0].term)[0], 'country');
equal(dataset.queryState.attributes.filters[0].field, 'country');
// now set filter value and apply
$editForm.find('input').val('UK');
$editForm.submit();
equal(dataset.queryState.attributes.filters[0].term.country, 'UK');
equal(dataset.queryState.attributes.filters[0].term, 'UK');
equal(dataset.currentRecords.length, 3);
// now remove filter
@ -55,12 +55,13 @@ test('geo_distance', function () {
// now check we have new filter
$editForm = view.el.find('form.js-edit');
equal($editForm.find('.filter-geo_distance').length, 1)
deepEqual(_.keys(dataset.queryState.attributes.filters[0].geo_distance), ['distance', 'lon', '_type', '_field']);
deepEqual(_.keys(dataset.queryState.attributes.filters[0]), ['distance',
'point', 'type', 'field']);
// now set filter value and apply
$editForm.find('input[name="lat"]').val(10);
$editForm.submit();
equal(dataset.queryState.attributes.filters[0].geo_distance.lon.lat, 10);
equal(dataset.queryState.attributes.filters[0].point.lat, 10);
view.remove();
});