knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[utils][xl]: copied over utils from frontend-v2.

Browse Source
This commit is contained in:
anuveyatsu 2020-06-12 14:46:03 +06:00
parent 56aaed8f87
commit 03578d9a11
3 changed files with 549 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
package.json
View File

@ -11,9 +11,13 @@
"test:coverage": "jest --coverage"
},
"dependencies": {
"bytes": "^3.1.0",
"markdown-it": "^11.0.0",
"next": "9.4.2",
"querystring": "^0.2.0",
"react": "16.13.1",
"react-dom": "16.13.1"
"react-dom": "16.13.1",
"slugify": "^1.4.0"
},
"devDependencies": {
"@testing-library/jest-dom": "^5.8.0",

509
utils/index.ts Normal file
View File

@ -0,0 +1,509 @@
const { URL } = require('url')
const bytes = require('bytes')
const slugify = require('slugify')
const config = require('../config')
module.exports.ckanToDataPackage = function (descriptor) {
// Make a copy
const datapackage = JSON.parse(JSON.stringify(descriptor))
// Lowercase name
datapackage.name = datapackage.name.toLowerCase()
// Rename notes => description
if (datapackage.notes) {
datapackage.description = datapackage.notes
delete datapackage.notes
}
// Rename ckan_url => homepage
if (datapackage.ckan_url) {
datapackage.homepage = datapackage.ckan_url
delete datapackage.ckan_url
}
// Parse license
const license = {}
if (datapackage.license_id) {
license.type = datapackage.license_id
delete datapackage.license_id
}
if (datapackage.license_title) {
license.title = datapackage.license_title
delete datapackage.license_title
}
if (datapackage.license_url) {
license.url = datapackage.license_url
delete datapackage.license_url
}
if (Object.keys(license).length > 0) {
datapackage.license = license
}
// Parse author and sources
const source = {}
if (datapackage.author) {
source.name = datapackage.author
delete datapackage.author
}
if (datapackage.author_email) {
source.email = datapackage.author_email
delete datapackage.author_email
}
if (datapackage.url) {
source.web = datapackage.url
delete datapackage.url
}
if (Object.keys(source).length > 0) {
datapackage.sources = [source]
}
// Parse maintainer
const author = {}
if (datapackage.maintainer) {
author.name = datapackage.maintainer
delete datapackage.maintainer
}
if (datapackage.maintainer_email) {
author.email = datapackage.maintainer_email
delete datapackage.maintainer_email
}
if (Object.keys(author).length > 0) {
datapackage.author = author
}
// Parse tags
if (datapackage.tags) {
datapackage.keywords = []
datapackage.tags.forEach(tag => {
datapackage.keywords.push(tag.name)
})
delete datapackage.tags
}
// Parse extras
// TODO
// Resources
datapackage.resources = datapackage.resources.map(resource => {
if (resource.name) {
resource.title = resource.title || resource.name
resource.name = resource.name.toLowerCase().replace(/ /g, '_')
} else {
resource.name = resource.id
}
if (resource.url) {
resource.path = resource.url
delete resource.url
}
if (!resource.schema) {
// If 'fields' property exists use it as schema fields
if (resource.fields) {
if (typeof(resource.fields) === 'string') {
try {
resource.fields = JSON.parse(resource.fields)
} catch (e) {
console.log('Could not parse resource.fields')
}
}
resource.schema = {fields: resource.fields}
delete resource.fields
}
}
return resource
})
return datapackage
}
/*
At the moment, we're considering only following examples of CKAN view:
1. recline_view => Data Explorer with Table view, Chart Builder, Map Builder
and Query Builder.
2. geojson_view => Leaflet map
3. pdf_view => our PDF viewer
4. recline_grid_view => our Table viewer
5. recline_graph_view => our Simple graph
6. recline_map_view => our Leaflet map
7. image_view => not supported at the moment
8. text_view => not supported at the moment
9. webpage_view => not supported at the moment
*/
module.exports.ckanViewToDataPackageView = (ckanView) => {
const viewTypeToSpecType = {
recline_view: 'dataExplorer', // from datastore data
recline_grid_view: 'table',
recline_graph_view: 'simple',
recline_map_view: 'tabularmap',
geojson_view: 'map',
pdf_view: 'document',
image_view: 'web',
webpage_view: 'web'
}
const dataPackageView = JSON.parse(JSON.stringify(ckanView))
dataPackageView.specType = viewTypeToSpecType[ckanView.view_type]
|| dataPackageView.specType
|| 'unsupported'
if (dataPackageView.specType === 'dataExplorer') {
dataPackageView.spec = {
widgets: [
{specType: 'table'},
{specType: 'simple'},
{specType: 'tabularmap'}
]
}
} else if (dataPackageView.specType === 'simple') {
const graphTypeConvert = {
lines: 'line',
'lines-and-points': 'lines-and-points',
points: 'points',
bars: 'horizontal-bar',
columns: 'bar'
}
dataPackageView.spec = {
group: ckanView.group,
series: Array.isArray(ckanView.series) ? ckanView.series : [ckanView.series],
type: graphTypeConvert[ckanView.graph_type] || 'line'
}
} else if (dataPackageView.specType === 'tabularmap') {
if (ckanView.map_field_type === 'geojson') {
dataPackageView.spec = {
geomField: ckanView.geojson_field
}
} else {
dataPackageView.spec = {
lonField: ckanView.longitude_field,
latField: ckanView.latitude_field
}
}
}
return dataPackageView
}
/*
Takes single field descriptor from datastore data dictionary and coverts into
tableschema field descriptor.
*/
module.exports.dataStoreDataDictionaryToTableSchema = (dataDictionary) => {
const internalDataStoreFields = ['_id', '_full_text', '_count']
if (internalDataStoreFields.includes(dataDictionary.id)) {
return null
}
const dataDictionaryType2TableSchemaType = {
'text': 'string',
'int': 'integer',
'float': 'number',
'date': 'date',
'time': 'time',
'timestamp': 'datetime',
'bool': 'boolean',
'json': 'object'
}
const field = {
name: dataDictionary.id,
type: dataDictionaryType2TableSchemaType[dataDictionary.type] || 'any'
}
if (dataDictionary.info) {
const constraintsAttributes = ['required', 'unique', 'minLength', 'maxLength', 'minimum', 'maximum', 'pattern', 'enum']
field.constraints = {}
Object.keys(dataDictionary.info).forEach(key => {
if (constraintsAttributes.includes(key)) {
field.constraints[key] = dataDictionary.info[key]
} else {
field[key] = dataDictionary.info[key]
}
})
}
return field
}
module.exports.convertToStandardCollection = (descriptor) => {
const standard = {
name: '',
title: '',
summary: '',
image: '',
count: null
}
standard.name = descriptor.name
standard.title = descriptor.title || descriptor.display_name
standard.summary = descriptor.description || ''
standard.image = descriptor.image_display_url || descriptor.image_url
standard.count = descriptor.package_count || 0
standard.extras = descriptor.extras || []
standard.groups = descriptor.groups || []
return standard
}
module.exports.convertToCkanSearchQuery = (query) => {
const ckanQuery = {
q: '',
fq: '',
rows: '',
start: '',
sort: '',
'facet.field': ['organization', 'groups', 'tags', 'res_format', 'license_id'],
'facet.limit': 5,
'facet.mincount': 0
}
// Split by space but ignore spaces within double quotes:
if (query.q) {
query.q.match(/(?:[^\s"]+|"[^"]*")+/g).forEach(part => {
if (part.includes(':')) {
ckanQuery.fq += part + ' '
} else {
ckanQuery.q += part + ' '
}
})
ckanQuery.fq = ckanQuery.fq.trim()
ckanQuery.q = ckanQuery.q.trim()
}
if (query.fq) {
ckanQuery.fq = ckanQuery.fq ? ckanQuery.fq + ' ' + query.fq : query.fq
}
// standard 'size' => ckan 'rows'
ckanQuery.rows = query.size || ''
// standard 'from' => ckan 'start'
ckanQuery.start = query.from || ''
// standard 'sort' => ckan 'sort'
const sortQueries = []
if (query.sort && query.sort.constructor == Object) {
for (let [key, value] of Object.entries(query.sort)) {
sortQueries.push(`${key} ${value}`)
}
ckanQuery.sort = sortQueries.join(',')
} else if (query.sort && query.sort.constructor == String) {
ckanQuery.sort = query.sort.replace(':', ' ')
} else if (query.sort && query.sort.constructor == Array) {
query.sort.forEach(sort => {
sortQueries.push(sort.replace(':', ' '))
})
ckanQuery.sort = sortQueries.join(',')
}
// Facets
ckanQuery['facet.field'] = query['facet.field'] || ckanQuery['facet.field']
ckanQuery['facet.limit'] = query['facet.limit'] || ckanQuery['facet.limit']
ckanQuery['facet.mincount'] = query['facet.mincount'] || ckanQuery['facet.mincount']
ckanQuery['facet.field'] = query['facet.field'] || ckanQuery['facet.field']
// Remove attributes with empty string, null or undefined values
Object.keys(ckanQuery).forEach((key) => (!ckanQuery[key]) && delete ckanQuery[key])
return ckanQuery
}
module.exports.pagination = (c, m) => {
let current = c,
last = m,
delta = 2,
left = current - delta,
right = current + delta + 1,
range = [],
rangeWithDots = [],
l;
range.push(1)
for (let i = c - delta; i <= c + delta; i++) {
if (i >= left && i < right && i < m && i > 1) {
range.push(i);
}
}
range.push(m)
for (let i of range) {
if (l) {
if (i - l === 2) {
rangeWithDots.push(l + 1);
} else if (i - l !== 1) {
rangeWithDots.push('...');
}
}
rangeWithDots.push(i);
l = i;
}
return rangeWithDots;
}
module.exports.processMarkdown = require('markdown-it')({
html: true,
linkify: true,
typographer: true
})
/**
* Process data package attributes prior to display to users.
* Process markdown
* Convert bytes to human readable format
* etc.
**/
module.exports.processDataPackage = function (datapackage) {
const newDatapackage = JSON.parse(JSON.stringify(datapackage))
if (newDatapackage.description) {
newDatapackage.descriptionHtml = module.exports.processMarkdown
.render(newDatapackage.description)
}
if (newDatapackage.readme) {
newDatapackage.readmeHtml = module.exports.processMarkdown
.render(newDatapackage.readme)
}
newDatapackage.formats = newDatapackage.formats || []
// Per each resource:
newDatapackage.resources.forEach(resource => {
if (resource.description) {
resource.descriptionHtml = module.exports.processMarkdown
.render(resource.description)
}
// Normalize format (lowercase)
if (resource.format) {
resource.format = resource.format.toLowerCase()
newDatapackage.formats.push(resource.format)
}
// Convert bytes into human-readable format:
if (resource.size) {
resource.sizeFormatted = bytes(resource.size, {decimalPlaces: 0})
}
})
return newDatapackage
}
/**
* Create 'displayResources' property which has:
* resource: Object containing resource descriptor
* api: API URL for the resource if available, e.g., Datastore
* proxy: path via proxy for the resource if available
* cc_proxy: path via CKAN Classic proxy if available
* slug: slugified name of a resource
**/
module.exports.prepareResourcesForDisplay = function (datapackage) {
const newDatapackage = JSON.parse(JSON.stringify(datapackage))
newDatapackage.displayResources = []
newDatapackage.resources.forEach((resource, index) => {
const api = resource.datastore_active
? config.get('API_URL') + 'datastore_search?resource_id=' + resource.id + '&sort=_id asc'
: null
// Use proxy path if datastore/filestore proxies are given:
let proxy, cc_proxy
try {
const resourceUrl = new URL(resource.path)
if (resourceUrl.host === config.get('PROXY_DATASTORE') && resource.format !== 'pdf') {
proxy = '/proxy/datastore' + resourceUrl.pathname + resourceUrl.search
}
if (resourceUrl.host === config.get('PROXY_FILESTORE') && resource.format !== 'pdf') {
proxy = '/proxy/filestore' + resourceUrl.pathname + resourceUrl.search
}
// Store a CKAN Classic proxy path
// https://github.com/ckan/ckan/blob/master/ckanext/resourceproxy/plugin.py#L59
const apiUrlObject = new URL(config.get('API_URL'))
cc_proxy = apiUrlObject.origin + `/dataset/${datapackage.id}/resource/${resource.id}/proxy`
} catch (e) {
console.warn(e)
}
const displayResource = {
resource,
api, // URI for getting the resource via API, e.g., Datastore. Useful when you want to fetch only 100 rows or similar.
proxy, // alternative for path in case there is CORS issue
cc_proxy,
slug: slugify(resource.name) + '-' + index // Used for anchor links
}
newDatapackage.displayResources.push(displayResource)
})
return newDatapackage
}
/**
* Prepare 'views' property which is used by 'datapackage-views-js' library to
* render visualizations such as tables, graphs and maps.
**/
module.exports.prepareViews = function (datapackage) {
const newDatapackage = JSON.parse(JSON.stringify(datapackage))
newDatapackage.views = newDatapackage.views || []
newDatapackage.resources.forEach(resource => {
const resourceViews = resource.views && resource.views.map(view => {
view.resources = [resource.name]
return view
})
newDatapackage.views = newDatapackage.views.concat(resourceViews)
})
return newDatapackage
}
/**
* Create 'dataExplorers' property which is used by 'data-explorer' library to
* render data explorer widgets.
**/
module.exports.prepareDataExplorers = function (datapackage) {
const newDatapackage = JSON.parse(JSON.stringify(datapackage))
newDatapackage.displayResources.forEach((displayResource, idx) => {
newDatapackage.displayResources[idx].dataExplorers = []
displayResource.resource.views && displayResource.resource.views.forEach(view => {
const widgets = []
if (view.specType === 'dataExplorer') {
view.spec.widgets.forEach((widget, index) => {
const widgetNames = {
table: 'Table',
simple: 'Chart',
tabularmap: 'Map'
}
widget = {
name: widgetNames[widget.specType] || 'Widget-' + index,
active: index === 0 ? true : false,
datapackage: {
views: [
{
id: view.id,
specType: widget.specType
}
]
}
}
widgets.push(widget)
})
} else {
const widget = {
name: view.title || '',
active: true,
datapackage: {
views: [view]
}
}
widgets.push(widget)
}
displayResource.resource.api = displayResource.resource.api || displayResource.api
const dataExplorer = JSON.stringify({
widgets,
datapackage: {
resources: [displayResource.resource]
}
}).replace(/'/g, "&#x27;")
newDatapackage.displayResources[idx].dataExplorers.push(dataExplorer)
})
})
return newDatapackage
}

37
yarn.lock
View File

@ -2202,7 +2202,7 @@ builtin-status-codes@^3.0.0:
resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz#85982878e21b98e1c66425e03d0174788f569ee8"
integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=
bytes@^3.0.0:
bytes@^3.0.0, bytes@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==
@ -3278,7 +3278,7 @@ enhanced-resolve@^4.1.0:
memory-fs "^0.5.0"
tapable "^1.0.0"
entities@^2.0.0:
entities@^2.0.0, entities@~2.0.0:
version "2.0.3"
resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.3.tgz#5c487e5742ab93c15abb5da22759b8590ec03b7f"
integrity sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ==
@ -4982,6 +4982,13 @@ lines-and-columns@^1.1.6:
resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=
linkify-it@^3.0.1:
version "3.0.2"
resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-3.0.2.tgz#f55eeb8bc1d3ae754049e124ab3bb56d97797fb8"
integrity sha512-gDBO4aHNZS6coiZCKVhSNh43F9ioIL4JwRjLZPkoLIY4yZFwg264Y5lu2x6rb1Js42Gh6Yqm2f6L2AJcnkzinQ==
dependencies:
uc.micro "^1.0.1"
loader-runner@^2.4.0:
version "2.4.0"
resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357"
@ -5130,6 +5137,17 @@ map-visit@^1.0.0:
dependencies:
object-visit "^1.0.0"
markdown-it@^11.0.0:
version "11.0.0"
resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-11.0.0.tgz#dbfc30363e43d756ebc52c38586b91b90046b876"
integrity sha512-+CvOnmbSubmQFSA9dKz1BRiaSMV7rhexl3sngKqFyXSagoA3fBdJQ8oZWtRy2knXdpDXaBw44euz37DeJQ9asg==
dependencies:
argparse "^1.0.7"
entities "~2.0.0"
linkify-it "^3.0.1"
mdurl "^1.0.1"
uc.micro "^1.0.5"
md5.js@^1.3.4:
version "1.3.5"
resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f"
@ -5149,6 +5167,11 @@ mdn-data@2.0.6:
resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.6.tgz#852dc60fcaa5daa2e8cf6c9189c440ed3e042978"
integrity sha512-rQvjv71olwNHgiTbfPZFkJtjNMciWgswYeciZhtvWLO8bmX3TnhyA62I6sTWOyZssWHJJjY6/KiWwqQsWWsqOA==
mdurl@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e"
integrity sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=
memory-fs@^0.4.1:
version "0.4.1"
resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552"
@ -7417,6 +7440,11 @@ slash@^3.0.0:
resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
slugify@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/slugify/-/slugify-1.4.0.tgz#c9557c653c54b0c7f7a8e786ef3431add676d2cb"
integrity sha512-FtLNsMGBSRB/0JOE2A0fxlqjI6fJsgHGS13iTuVT28kViI4JjUiNqp/vyis0ZXYcMnpR3fzGNkv+6vRlI2GwdQ==
snapdragon-node@^2.0.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b"
@ -8130,6 +8158,11 @@ typescript@^3.9.3:
resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.5.tgz#586f0dba300cde8be52dd1ac4f7e1009c1b13f36"
integrity sha512-hSAifV3k+i6lEoCJ2k6R2Z/rp/H3+8sdmcn5NrS3/3kE7+RyZXm9aqvxWqjEXHAd8b0pShatpcdMTvEdvAJltQ==
uc.micro@^1.0.1, uc.micro@^1.0.5:
version "1.0.6"
resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==
unicode-canonical-property-names-ecmascript@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818"