From 03578d9a11d3e5effda49452c11d5475beac27c6 Mon Sep 17 00:00:00 2001 From: anuveyatsu Date: Fri, 12 Jun 2020 14:46:03 +0600 Subject: [PATCH] [utils][xl]: copied over utils from frontend-v2. --- package.json | 6 +- utils/index.ts | 509 +++++++++++++++++++++++++++++++++++++++++++++++++ yarn.lock | 37 +++- 3 files changed, 549 insertions(+), 3 deletions(-) create mode 100644 utils/index.ts diff --git a/package.json b/package.json index b5163205..c71b615c 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,13 @@ "test:coverage": "jest --coverage" }, "dependencies": { + "bytes": "^3.1.0", + "markdown-it": "^11.0.0", "next": "9.4.2", + "querystring": "^0.2.0", "react": "16.13.1", - "react-dom": "16.13.1" + "react-dom": "16.13.1", + "slugify": "^1.4.0" }, "devDependencies": { "@testing-library/jest-dom": "^5.8.0", diff --git a/utils/index.ts b/utils/index.ts new file mode 100644 index 00000000..b3ab96d2 --- /dev/null +++ b/utils/index.ts @@ -0,0 +1,509 @@ +const { URL } = require('url') +const bytes = require('bytes') +const slugify = require('slugify') +const config = require('../config') + +module.exports.ckanToDataPackage = function (descriptor) { + // Make a copy + const datapackage = JSON.parse(JSON.stringify(descriptor)) + + // Lowercase name + datapackage.name = datapackage.name.toLowerCase() + + // Rename notes => description + if (datapackage.notes) { + datapackage.description = datapackage.notes + delete datapackage.notes + } + + // Rename ckan_url => homepage + if (datapackage.ckan_url) { + datapackage.homepage = datapackage.ckan_url + delete datapackage.ckan_url + } + + // Parse license + const license = {} + if (datapackage.license_id) { + license.type = datapackage.license_id + delete datapackage.license_id + } + if (datapackage.license_title) { + license.title = datapackage.license_title + delete datapackage.license_title + } + if (datapackage.license_url) { + license.url = datapackage.license_url + delete datapackage.license_url + } + if (Object.keys(license).length > 0) { + datapackage.license = license + } + + // Parse author and sources + const source = {} + if (datapackage.author) { + source.name = datapackage.author + delete datapackage.author + } + if (datapackage.author_email) { + source.email = datapackage.author_email + delete datapackage.author_email + } + if (datapackage.url) { + source.web = datapackage.url + delete datapackage.url + } + if (Object.keys(source).length > 0) { + datapackage.sources = [source] + } + + // Parse maintainer + const author = {} + if (datapackage.maintainer) { + author.name = datapackage.maintainer + delete datapackage.maintainer + } + if (datapackage.maintainer_email) { + author.email = datapackage.maintainer_email + delete datapackage.maintainer_email + } + if (Object.keys(author).length > 0) { + datapackage.author = author + } + + // Parse tags + if (datapackage.tags) { + datapackage.keywords = [] + datapackage.tags.forEach(tag => { + datapackage.keywords.push(tag.name) + }) + delete datapackage.tags + } + + // Parse extras + // TODO + + // Resources + datapackage.resources = datapackage.resources.map(resource => { + if (resource.name) { + resource.title = resource.title || resource.name + resource.name = resource.name.toLowerCase().replace(/ /g, '_') + } else { + resource.name = resource.id + } + + if (resource.url) { + resource.path = resource.url + delete resource.url + } + + if (!resource.schema) { + // If 'fields' property exists use it as schema fields + if (resource.fields) { + if (typeof(resource.fields) === 'string') { + try { + resource.fields = JSON.parse(resource.fields) + } catch (e) { + console.log('Could not parse resource.fields') + } + } + resource.schema = {fields: resource.fields} + delete resource.fields + } + } + + return resource + }) + + return datapackage +} + +/* + At the moment, we're considering only following examples of CKAN view: + 1. recline_view => Data Explorer with Table view, Chart Builder, Map Builder + and Query Builder. + 2. geojson_view => Leaflet map + 3. pdf_view => our PDF viewer + 4. recline_grid_view => our Table viewer + 5. recline_graph_view => our Simple graph + 6. recline_map_view => our Leaflet map + 7. image_view => not supported at the moment + 8. text_view => not supported at the moment + 9. webpage_view => not supported at the moment +*/ +module.exports.ckanViewToDataPackageView = (ckanView) => { + const viewTypeToSpecType = { + recline_view: 'dataExplorer', // from datastore data + recline_grid_view: 'table', + recline_graph_view: 'simple', + recline_map_view: 'tabularmap', + geojson_view: 'map', + pdf_view: 'document', + image_view: 'web', + webpage_view: 'web' + } + const dataPackageView = JSON.parse(JSON.stringify(ckanView)) + dataPackageView.specType = viewTypeToSpecType[ckanView.view_type] + || dataPackageView.specType + || 'unsupported' + + if (dataPackageView.specType === 'dataExplorer') { + dataPackageView.spec = { + widgets: [ + {specType: 'table'}, + {specType: 'simple'}, + {specType: 'tabularmap'} + ] + } + } else if (dataPackageView.specType === 'simple') { + const graphTypeConvert = { + lines: 'line', + 'lines-and-points': 'lines-and-points', + points: 'points', + bars: 'horizontal-bar', + columns: 'bar' + } + dataPackageView.spec = { + group: ckanView.group, + series: Array.isArray(ckanView.series) ? ckanView.series : [ckanView.series], + type: graphTypeConvert[ckanView.graph_type] || 'line' + } + } else if (dataPackageView.specType === 'tabularmap') { + if (ckanView.map_field_type === 'geojson') { + dataPackageView.spec = { + geomField: ckanView.geojson_field + } + } else { + dataPackageView.spec = { + lonField: ckanView.longitude_field, + latField: ckanView.latitude_field + } + } + } + + return dataPackageView +} + +/* +Takes single field descriptor from datastore data dictionary and coverts into +tableschema field descriptor. +*/ +module.exports.dataStoreDataDictionaryToTableSchema = (dataDictionary) => { + const internalDataStoreFields = ['_id', '_full_text', '_count'] + if (internalDataStoreFields.includes(dataDictionary.id)) { + return null + } + const dataDictionaryType2TableSchemaType = { + 'text': 'string', + 'int': 'integer', + 'float': 'number', + 'date': 'date', + 'time': 'time', + 'timestamp': 'datetime', + 'bool': 'boolean', + 'json': 'object' + } + const field = { + name: dataDictionary.id, + type: dataDictionaryType2TableSchemaType[dataDictionary.type] || 'any' + } + if (dataDictionary.info) { + const constraintsAttributes = ['required', 'unique', 'minLength', 'maxLength', 'minimum', 'maximum', 'pattern', 'enum'] + field.constraints = {} + Object.keys(dataDictionary.info).forEach(key => { + if (constraintsAttributes.includes(key)) { + field.constraints[key] = dataDictionary.info[key] + } else { + field[key] = dataDictionary.info[key] + } + }) + } + return field +} + +module.exports.convertToStandardCollection = (descriptor) => { + const standard = { + name: '', + title: '', + summary: '', + image: '', + count: null + } + + standard.name = descriptor.name + standard.title = descriptor.title || descriptor.display_name + standard.summary = descriptor.description || '' + standard.image = descriptor.image_display_url || descriptor.image_url + standard.count = descriptor.package_count || 0 + standard.extras = descriptor.extras || [] + standard.groups = descriptor.groups || [] + + return standard +} + + +module.exports.convertToCkanSearchQuery = (query) => { + const ckanQuery = { + q: '', + fq: '', + rows: '', + start: '', + sort: '', + 'facet.field': ['organization', 'groups', 'tags', 'res_format', 'license_id'], + 'facet.limit': 5, + 'facet.mincount': 0 + } + // Split by space but ignore spaces within double quotes: + if (query.q) { + query.q.match(/(?:[^\s"]+|"[^"]*")+/g).forEach(part => { + if (part.includes(':')) { + ckanQuery.fq += part + ' ' + } else { + ckanQuery.q += part + ' ' + } + }) + ckanQuery.fq = ckanQuery.fq.trim() + ckanQuery.q = ckanQuery.q.trim() + } + + if (query.fq) { + ckanQuery.fq = ckanQuery.fq ? ckanQuery.fq + ' ' + query.fq : query.fq + } + + // standard 'size' => ckan 'rows' + ckanQuery.rows = query.size || '' + + // standard 'from' => ckan 'start' + ckanQuery.start = query.from || '' + + // standard 'sort' => ckan 'sort' + const sortQueries = [] + if (query.sort && query.sort.constructor == Object) { + for (let [key, value] of Object.entries(query.sort)) { + sortQueries.push(`${key} ${value}`) + } + ckanQuery.sort = sortQueries.join(',') + } else if (query.sort && query.sort.constructor == String) { + ckanQuery.sort = query.sort.replace(':', ' ') + } else if (query.sort && query.sort.constructor == Array) { + query.sort.forEach(sort => { + sortQueries.push(sort.replace(':', ' ')) + }) + ckanQuery.sort = sortQueries.join(',') + } + + // Facets + ckanQuery['facet.field'] = query['facet.field'] || ckanQuery['facet.field'] + ckanQuery['facet.limit'] = query['facet.limit'] || ckanQuery['facet.limit'] + ckanQuery['facet.mincount'] = query['facet.mincount'] || ckanQuery['facet.mincount'] + ckanQuery['facet.field'] = query['facet.field'] || ckanQuery['facet.field'] + + // Remove attributes with empty string, null or undefined values + Object.keys(ckanQuery).forEach((key) => (!ckanQuery[key]) && delete ckanQuery[key]) + + return ckanQuery +} + + +module.exports.pagination = (c, m) => { + let current = c, + last = m, + delta = 2, + left = current - delta, + right = current + delta + 1, + range = [], + rangeWithDots = [], + l; + + range.push(1) + for (let i = c - delta; i <= c + delta; i++) { + if (i >= left && i < right && i < m && i > 1) { + range.push(i); + } + } + range.push(m) + + for (let i of range) { + if (l) { + if (i - l === 2) { + rangeWithDots.push(l + 1); + } else if (i - l !== 1) { + rangeWithDots.push('...'); + } + } + rangeWithDots.push(i); + l = i; + } + return rangeWithDots; +} + + +module.exports.processMarkdown = require('markdown-it')({ + html: true, + linkify: true, + typographer: true +}) + + +/** + * Process data package attributes prior to display to users. + * Process markdown + * Convert bytes to human readable format + * etc. + **/ +module.exports.processDataPackage = function (datapackage) { + const newDatapackage = JSON.parse(JSON.stringify(datapackage)) + if (newDatapackage.description) { + newDatapackage.descriptionHtml = module.exports.processMarkdown + .render(newDatapackage.description) + } + + if (newDatapackage.readme) { + newDatapackage.readmeHtml = module.exports.processMarkdown + .render(newDatapackage.readme) + } + + newDatapackage.formats = newDatapackage.formats || [] + // Per each resource: + newDatapackage.resources.forEach(resource => { + if (resource.description) { + resource.descriptionHtml = module.exports.processMarkdown + .render(resource.description) + } + // Normalize format (lowercase) + if (resource.format) { + resource.format = resource.format.toLowerCase() + newDatapackage.formats.push(resource.format) + } + + // Convert bytes into human-readable format: + if (resource.size) { + resource.sizeFormatted = bytes(resource.size, {decimalPlaces: 0}) + } + }) + + return newDatapackage +} + + +/** + * Create 'displayResources' property which has: + * resource: Object containing resource descriptor + * api: API URL for the resource if available, e.g., Datastore + * proxy: path via proxy for the resource if available + * cc_proxy: path via CKAN Classic proxy if available + * slug: slugified name of a resource + **/ +module.exports.prepareResourcesForDisplay = function (datapackage) { + const newDatapackage = JSON.parse(JSON.stringify(datapackage)) + newDatapackage.displayResources = [] + newDatapackage.resources.forEach((resource, index) => { + const api = resource.datastore_active + ? config.get('API_URL') + 'datastore_search?resource_id=' + resource.id + '&sort=_id asc' + : null + // Use proxy path if datastore/filestore proxies are given: + let proxy, cc_proxy + try { + const resourceUrl = new URL(resource.path) + if (resourceUrl.host === config.get('PROXY_DATASTORE') && resource.format !== 'pdf') { + proxy = '/proxy/datastore' + resourceUrl.pathname + resourceUrl.search + } + if (resourceUrl.host === config.get('PROXY_FILESTORE') && resource.format !== 'pdf') { + proxy = '/proxy/filestore' + resourceUrl.pathname + resourceUrl.search + } + // Store a CKAN Classic proxy path + // https://github.com/ckan/ckan/blob/master/ckanext/resourceproxy/plugin.py#L59 + const apiUrlObject = new URL(config.get('API_URL')) + cc_proxy = apiUrlObject.origin + `/dataset/${datapackage.id}/resource/${resource.id}/proxy` + } catch (e) { + console.warn(e) + } + const displayResource = { + resource, + api, // URI for getting the resource via API, e.g., Datastore. Useful when you want to fetch only 100 rows or similar. + proxy, // alternative for path in case there is CORS issue + cc_proxy, + slug: slugify(resource.name) + '-' + index // Used for anchor links + } + newDatapackage.displayResources.push(displayResource) + }) + return newDatapackage +} + + +/** + * Prepare 'views' property which is used by 'datapackage-views-js' library to + * render visualizations such as tables, graphs and maps. + **/ +module.exports.prepareViews = function (datapackage) { + const newDatapackage = JSON.parse(JSON.stringify(datapackage)) + newDatapackage.views = newDatapackage.views || [] + newDatapackage.resources.forEach(resource => { + const resourceViews = resource.views && resource.views.map(view => { + view.resources = [resource.name] + return view + }) + + newDatapackage.views = newDatapackage.views.concat(resourceViews) + }) + + return newDatapackage +} + + +/** + * Create 'dataExplorers' property which is used by 'data-explorer' library to + * render data explorer widgets. + **/ +module.exports.prepareDataExplorers = function (datapackage) { + const newDatapackage = JSON.parse(JSON.stringify(datapackage)) + newDatapackage.displayResources.forEach((displayResource, idx) => { + newDatapackage.displayResources[idx].dataExplorers = [] + displayResource.resource.views && displayResource.resource.views.forEach(view => { + const widgets = [] + if (view.specType === 'dataExplorer') { + view.spec.widgets.forEach((widget, index) => { + const widgetNames = { + table: 'Table', + simple: 'Chart', + tabularmap: 'Map' + } + widget = { + name: widgetNames[widget.specType] || 'Widget-' + index, + active: index === 0 ? true : false, + datapackage: { + views: [ + { + id: view.id, + specType: widget.specType + } + ] + } + } + widgets.push(widget) + }) + } else { + const widget = { + name: view.title || '', + active: true, + datapackage: { + views: [view] + } + } + widgets.push(widget) + } + + displayResource.resource.api = displayResource.resource.api || displayResource.api + const dataExplorer = JSON.stringify({ + widgets, + datapackage: { + resources: [displayResource.resource] + } + }).replace(/'/g, "'") + newDatapackage.displayResources[idx].dataExplorers.push(dataExplorer) + }) + }) + + return newDatapackage +} diff --git a/yarn.lock b/yarn.lock index cb78ff91..a9a3802f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2202,7 +2202,7 @@ builtin-status-codes@^3.0.0: resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz#85982878e21b98e1c66425e03d0174788f569ee8" integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug= -bytes@^3.0.0: +bytes@^3.0.0, bytes@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== @@ -3278,7 +3278,7 @@ enhanced-resolve@^4.1.0: memory-fs "^0.5.0" tapable "^1.0.0" -entities@^2.0.0: +entities@^2.0.0, entities@~2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.3.tgz#5c487e5742ab93c15abb5da22759b8590ec03b7f" integrity sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ== @@ -4982,6 +4982,13 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA= +linkify-it@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-3.0.2.tgz#f55eeb8bc1d3ae754049e124ab3bb56d97797fb8" + integrity sha512-gDBO4aHNZS6coiZCKVhSNh43F9ioIL4JwRjLZPkoLIY4yZFwg264Y5lu2x6rb1Js42Gh6Yqm2f6L2AJcnkzinQ== + dependencies: + uc.micro "^1.0.1" + loader-runner@^2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357" @@ -5130,6 +5137,17 @@ map-visit@^1.0.0: dependencies: object-visit "^1.0.0" +markdown-it@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-11.0.0.tgz#dbfc30363e43d756ebc52c38586b91b90046b876" + integrity sha512-+CvOnmbSubmQFSA9dKz1BRiaSMV7rhexl3sngKqFyXSagoA3fBdJQ8oZWtRy2knXdpDXaBw44euz37DeJQ9asg== + dependencies: + argparse "^1.0.7" + entities "~2.0.0" + linkify-it "^3.0.1" + mdurl "^1.0.1" + uc.micro "^1.0.5" + md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" @@ -5149,6 +5167,11 @@ mdn-data@2.0.6: resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.6.tgz#852dc60fcaa5daa2e8cf6c9189c440ed3e042978" integrity sha512-rQvjv71olwNHgiTbfPZFkJtjNMciWgswYeciZhtvWLO8bmX3TnhyA62I6sTWOyZssWHJJjY6/KiWwqQsWWsqOA== +mdurl@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e" + integrity sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4= + memory-fs@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552" @@ -7417,6 +7440,11 @@ slash@^3.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== +slugify@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/slugify/-/slugify-1.4.0.tgz#c9557c653c54b0c7f7a8e786ef3431add676d2cb" + integrity sha512-FtLNsMGBSRB/0JOE2A0fxlqjI6fJsgHGS13iTuVT28kViI4JjUiNqp/vyis0ZXYcMnpR3fzGNkv+6vRlI2GwdQ== + snapdragon-node@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" @@ -8130,6 +8158,11 @@ typescript@^3.9.3: resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.5.tgz#586f0dba300cde8be52dd1ac4f7e1009c1b13f36" integrity sha512-hSAifV3k+i6lEoCJ2k6R2Z/rp/H3+8sdmcn5NrS3/3kE7+RyZXm9aqvxWqjEXHAd8b0pShatpcdMTvEdvAJltQ== +uc.micro@^1.0.1, uc.micro@^1.0.5: + version "1.0.6" + resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac" + integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA== + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818"