knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[packages/portal][l]: Moved portal folder to root dir

Browse Source
This commit is contained in:
Rising Odegua
2021-03-08 11:33:21 +01:00
parent 6aec97a728
commit c905b59f3e
41 changed files with 502 additions and 4002 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/portal/.babelrc
View File

@@ -1,3 +0,0 @@
{
"presets": ["next/babel"]
}

22
packages/portal/README.md
View File

@@ -1,22 +0,0 @@
Present a (Frictionless) dataset for viewing and exploration.
## Install
Git clone then:
```
yarn install
```
## Usage
In this directory:
```bash
export PORTAL_DATASET_PATH=/path/to/my/dataset
yarn dev
```
And you will get a nice dataset page at `http://localhost:3000`
![](https://i.imgur.com/KSEtNF1.png)

130
packages/portal/bin/portal.js
View File

@@ -1,130 +0,0 @@
#!/usr/bin/env node
const Listr = require('listr')
const { program } = require('commander')
const chalk = require('chalk')
const path = require('path')
const figlet = require('figlet')
const { exec } = require('child_process');
const package = require('../package.json')
const fs = require('fs')
console.log(
chalk.yellow(figlet.textSync('Portal App', { horizontalLayout: 'full' }))
)
function directoryExists(filePath) {
return fs.existsSync(filePath);
}
/**
* Executes a shell command and return it as a Promise.
* @param cmd {string}
* @return {Promise<string>}
*/
function execShellCommand(cmd) {
return new Promise((resolve, reject) => {
exec(cmd, (error, stdout, stderr) => {
if (error) {
console.warn(error);
reject(error)
}
resolve(stdout ? stdout : stderr);
});
});
}
function processArgs(args) {
const userArgs = { npm: false, override: false, port: 3000, path: process.cwd() }
args.forEach((arg) => {
if (arg.includes("=")) {
let temp = arg.split("=")
userArgs[temp[0]] = temp[1]
}
})
return userArgs
}
// Output path to create new portal app
// Commander parameters to specify CLI behavior
program
.version(package.version)
.usage('show [ path=/some/path | npm=true | port=3000 ]')
.description('Creates a portal application from specified dataset',
)
.option('npm', '[true || false] Install dependencies using npm instead yarn, defaults to false (yarn)')
.option('port', 'Server port, defaults to 3000')
.parse(process.argv,)
const userArgs = processArgs(program.args)
/**
* Main method to start CLI and validate inputs
*/
async function run() {
const datasetPath = userArgs.path.trim()
if (directoryExists(datasetPath)) {
console.log(
`${chalk.yellow(`Using dataset found in: ${chalk.cyan(datasetPath)}`)}`
)
} else {
console.log(
`${chalk.red(`Directory: ${chalk.cyan(datasetPath)} does not exist!`)}`
)
process.exit(1)
}
const portalGithubRepo = "https://github.com/datopian/portal-experiment.git"
const portalLocalRepoDirectory = path.join(datasetPath, 'portal-experiment')
const cloneRepoCmd = `cd ${datasetPath} &&
export PORTAL_DATASET_PATH=${datasetPath} &&
git clone ${portalGithubRepo}`
const buildNextAppCmd = userArgs.npm ? `cd ${portalLocalRepoDirectory} && npm install && npm run build` :
`cd ${portalLocalRepoDirectory} && yarn && yarn build`
const startNextAppCmd = userArgs.npm ?
`cd ${portalLocalRepoDirectory} && npm run start -p ${userArgs.port}` :
`cd ${portalLocalRepoDirectory} && yarn start -p ${userArgs.port}`
//Tasks workflow
const tasks = new Listr([
{
title: 'Getting portal tools...',
task: async () => {
try {
if (directoryExists(portalLocalRepoDirectory)) {
console.log(
chalk.cyan(`${package.name} ${chalk.yellow('already exists! Skipping this step')}`))
} else {
await execShellCommand(cloneRepoCmd)
}
} catch (error) {
throw error
}
},
},
{
title: 'Preparing your app...',
task: async () => { await execShellCommand(buildNextAppCmd) }
},
{
title: `Displaying dataset at http://localhost:${userArgs.port}`,
task: () => execShellCommand(startNextAppCmd),
}
])
tasks.run()
}
run().catch((error) => {
console.log(error)
console.log()
process.exit(1)
})

31
packages/portal/components/Chart.js vendored
View File

@@ -1,31 +0,0 @@
import React, { useState, useEffect } from 'react';
import createPlotlyComponent from "react-plotly.js/factory";
let Plot;
const Chart = (props) => {
const [plotCreated, setPlotCreated] = useState(0) //0: false, 1: true
useEffect(() => {
import(`plotly.js-basic-dist`).then(Plotly => { //import Plotly dist when Page has been generated
Plot = createPlotlyComponent(Plotly);
setPlotCreated(1)
});
}, [])
if (!plotCreated) {
return (<div>Loading...</div>)
}
return (
<div data-testid="plotlyChart">
<Plot {...props.spec}
layout={{ autosize: true }}
style={{ width: "100%", height: "100%" }}
useResizeHandler={true}
/>
</div>
)
}
export default Chart

28
packages/portal/components/Table.js
View File

@@ -1,28 +0,0 @@
/* eslint-disable max-len */
import React from 'react';
import { DataGrid } from '@material-ui/data-grid';
/*
* @param schema: Frictionless Table Schmea
* @param data: an array of data objects e.g. [ {a: 1, b: 2}, {a: 5, b: 7} ]
*/
const Table = ({ schema, data }) => {
const columns = schema.fields.map((field) => (
{
field: field.title || field.name,
headerName: field.name,
width: 300
}
))
data = data.map((item, index)=>{
item.id = index //Datagrid requires every row to have an ID
return item
})
return (
<div data-testid="tableGrid" style={{ height: 400, width: '100%' }}>
<DataGrid rows={data} columns={columns} pageSize={5} />
</div>
);
}
export default Table

51
packages/portal/fixtures/datasetsDoubleView/README.md
View File

@@ -1,51 +0,0 @@
CBOE Volatility Index (VIX) time-series dataset including daily open, close,
high and low. The CBOE Volatility Index (VIX) is a key measure of market
expectations of near-term volatility conveyed by S&P 500 stock index option
prices introduced in 1993.
## Data
From the [VIX FAQ][faq]:
> In 1993, the Chicago Board Options Exchange® (CBOE®) introduced the CBOE
> Volatility Index®, VIX®, and it quickly became the benchmark for stock market
> volatility. It is widely followed and has been cited in hundreds of news
> articles in the Wall Street Journal, Barron's and other leading financial
> publications. Since volatility often signifies financial turmoil, VIX is
> often referred to as the "investor fear gauge".
>
> VIX measures market expectation of near term volatility conveyed by stock
> index option prices. The original VIX was constructed using the implied
> volatilities of eight different OEX option series so that, at any given time,
> it represented the implied volatility of a hypothetical at-the-money OEX
> option with exactly 30 days to expiration.
>
> The New VIX still measures the market's expectation of 30-day volatility, but
> in a way that conforms to the latest thinking and research among industry
> practitioners. The New VIX is based on S&P 500 index option prices and
> incorporates information from the volatility "skew" by using a wider range of
> strike prices rather than just at-the-money series.
[faq]: http://www.cboe.com/micro/vix/faq.aspx
## Preparation
Run the shell script:
. scripts/process.sh
Output data is in `data/`.
### TODO
* Incorporate computed historical data (1990-2003)
* Consider incorporating VOX data
## License
No obvious statement on [historical data page][historical]. Given size and
factual nature of the data and its source from a US company would imagine this
was public domain and as such have licensed the Data Package under the Public
Domain Dedication and License (PDDL).
[historical]: http://www.cboe.com/micro/vix/historical.aspx

112
packages/portal/fixtures/datasetsDoubleView/datapackage.json
View File

@@ -1,112 +0,0 @@
{
"name": "finance-vix",
"title": "VIX - CBOE Volatility Index",
"homepage": "http://www.cboe.com/micro/VIX/",
"version": "0.1.0",
"license": "PDDL-1.0",
"sources": [
{
"title": "CBOE VIX Page",
"name": "CBOE VIX Page",
"web": "http://www.cboe.com/micro/vix/historical.aspx"
}
],
"resources": [
{
"name": "vix-daily",
"path": "vix-daily.csv",
"format": "csv",
"mediatype": "text/csv",
"schema": {
"fields": [
{
"name": "Date",
"type": "date",
"description": ""
},
{
"name": "VIXOpen",
"type": "number",
"description": ""
},
{
"name": "VIXHigh",
"type": "number",
"description": ""
},
{
"name": "VIXLow",
"type": "number",
"description": ""
},
{
"name": "VIXClose",
"type": "number",
"description": ""
}
],
"primaryKey": "Date"
}
}
],
"views": [
{
"name": "simple graph",
"id": 1,
"title": "title1",
"specType": "simple",
"spec": {
"type": "line",
"group": "VIXClose",
"series": [
"VIXOpen",
"VIXHigh"
]
}
},
{
"name": "plotly graph",
"id": 2,
"specType": "plotly",
"resources": [
"vix-daily"
],
"spec": {
"group": "VIXClose",
"series": [
"VIXOpen",
"VIXHigh",
"VIXLow"
],
"data": [
{
"type": "bar"
}
],
"layout": {
"title": "Plotly Layout Title",
"height": 450,
"xaxis": {
"title": "X Axis Title"
},
"yaxis": {
"title": "Y Axis Title"
},
"font": {
"family": "\"Open Sans\", verdana, arial, sans-serif",
"size": 12,
"color": "rgb(169, 169, 169)"
},
"titlefont": {
"family": "\"Open Sans\", verdana, arial, sans-serif",
"size": 17,
"color": "rgb(76, 76, 76)"
}
},
"config": {
"displayModeBar": false
}
}
}
]
}

20
packages/portal/fixtures/datasetsDoubleView/vix-daily.csv
View File

@@ -1,20 +0,0 @@
Date,VIXOpen,VIXHigh,VIXLow,VIXClose
2004-01-02,17.96,18.68,17.54,18.22
2004-01-05,18.45,18.49,17.44,17.49
2004-01-06,17.66,17.67,16.19,16.73
2004-01-07,16.72,16.75,15.05,15.05
2004-01-08,15.42,15.68,15.32,15.61
2004-01-09,16.15,16.88,15.57,16.75
2004-01-12,17.32,17.46,16.79,16.82
2004-01-13,16.06,18.33,16.53,18.04
2004-01-14,17.29,17.03,16.04,16.75
2004-01-15,17.07,17.31,15.49,15.56
2004-01-16,15.04,15.44,14.09,15
2004-01-20,15.77,16.13,15.09,15.21
2004-01-21,15.63,15.63,14.24,14.34
2004-01-22,14.02,14.87,14.01,14.71
2004-01-23,14.73,15.05,14.56,14.84
2004-01-26,15.78,15.78,14.52,14.55
2004-01-27,15.28,15.44,14.74,15.35
2004-01-28,15.37,17.06,15.29,16.78
2004-01-29,16.88,17.66,16.79,17.14
1 Date VIXOpen VIXHigh VIXLow VIXClose
2 2004-01-02 17.96 18.68 17.54 18.22
3 2004-01-05 18.45 18.49 17.44 17.49
4 2004-01-06 17.66 17.67 16.19 16.73
5 2004-01-07 16.72 16.75 15.05 15.05
6 2004-01-08 15.42 15.68 15.32 15.61
7 2004-01-09 16.15 16.88 15.57 16.75
8 2004-01-12 17.32 17.46 16.79 16.82
9 2004-01-13 16.06 18.33 16.53 18.04
10 2004-01-14 17.29 17.03 16.04 16.75
11 2004-01-15 17.07 17.31 15.49 15.56
12 2004-01-16 15.04 15.44 14.09 15
13 2004-01-20 15.77 16.13 15.09 15.21
14 2004-01-21 15.63 15.63 14.24 14.34
15 2004-01-22 14.02 14.87 14.01 14.71
16 2004-01-23 14.73 15.05 14.56 14.84
17 2004-01-26 15.78 15.78 14.52 14.55
18 2004-01-27 15.28 15.44 14.74 15.35
19 2004-01-28 15.37 17.06 15.29 16.78
20 2004-01-29 16.88 17.66 16.79 17.14

51
packages/portal/fixtures/datasetsPlotlyView/README.md
View File

@@ -1,51 +0,0 @@
CBOE Volatility Index (VIX) time-series dataset including daily open, close,
high and low. The CBOE Volatility Index (VIX) is a key measure of market
expectations of near-term volatility conveyed by S&P 500 stock index option
prices introduced in 1993.
## Data
From the [VIX FAQ][faq]:
> In 1993, the Chicago Board Options Exchange® (CBOE®) introduced the CBOE
> Volatility Index®, VIX®, and it quickly became the benchmark for stock market
> volatility. It is widely followed and has been cited in hundreds of news
> articles in the Wall Street Journal, Barron's and other leading financial
> publications. Since volatility often signifies financial turmoil, VIX is
> often referred to as the "investor fear gauge".
>
> VIX measures market expectation of near term volatility conveyed by stock
> index option prices. The original VIX was constructed using the implied
> volatilities of eight different OEX option series so that, at any given time,
> it represented the implied volatility of a hypothetical at-the-money OEX
> option with exactly 30 days to expiration.
>
> The New VIX still measures the market's expectation of 30-day volatility, but
> in a way that conforms to the latest thinking and research among industry
> practitioners. The New VIX is based on S&P 500 index option prices and
> incorporates information from the volatility "skew" by using a wider range of
> strike prices rather than just at-the-money series.
[faq]: http://www.cboe.com/micro/vix/faq.aspx
## Preparation
Run the shell script:
. scripts/process.sh
Output data is in `data/`.
### TODO
* Incorporate computed historical data (1990-2003)
* Consider incorporating VOX data
## License
No obvious statement on [historical data page][historical]. Given size and
factual nature of the data and its source from a US company would imagine this
was public domain and as such have licensed the Data Package under the Public
Domain Dedication and License (PDDL).
[historical]: http://www.cboe.com/micro/vix/historical.aspx

98
packages/portal/fixtures/datasetsPlotlyView/datapackage.json
View File

@@ -1,98 +0,0 @@
{
"name": "finance-vix",
"title": "VIX - CBOE Volatility Index",
"homepage": "http://www.cboe.com/micro/VIX/",
"version": "0.1.0",
"license": "PDDL-1.0",
"sources": [
{
"title": "CBOE VIX Page",
"name": "CBOE VIX Page",
"web": "http://www.cboe.com/micro/vix/historical.aspx"
}
],
"resources": [
{
"name": "vix-daily",
"path": "vix-daily.csv",
"format": "csv",
"mediatype": "text/csv",
"schema": {
"fields": [
{
"name": "Date",
"type": "date",
"description": ""
},
{
"name": "VIXOpen",
"type": "number",
"description": ""
},
{
"name": "VIXHigh",
"type": "number",
"description": ""
},
{
"name": "VIXLow",
"type": "number",
"description": ""
},
{
"name": "VIXClose",
"type": "number",
"description": ""
}
],
"primaryKey": "Date"
}
}
],
"views": [
{
"name": "plotly graph",
"id": 2,
"specType": "plotly",
"resources": [
"vix-daily"
],
"spec": {
"group": "VIXClose",
"series": [
"VIXOpen",
"VIXHigh",
"VIXLow"
],
"data": [
{
"type": "bar"
}
],
"layout": {
"title": "Plotly Layout Title",
"height": 450,
"xaxis": {
"title": "X Axis Title"
},
"yaxis": {
"title": "Y Axis Title"
},
"font": {
"family": "\"Open Sans\", verdana, arial, sans-serif",
"size": 12,
"color": "rgb(169, 169, 169)"
},
"titlefont": {
"family": "\"Open Sans\", verdana, arial, sans-serif",
"size": 17,
"color": "rgb(76, 76, 76)"
}
},
"config": {
"displayModeBar": false
}
}
}
]
}

20
packages/portal/fixtures/datasetsPlotlyView/vix-daily.csv
View File

@@ -1,20 +0,0 @@
Date,VIXOpen,VIXHigh,VIXLow,VIXClose
2004-01-02,17.96,18.68,17.54,18.22
2004-01-05,18.45,18.49,17.44,17.49
2004-01-06,17.66,17.67,16.19,16.73
2004-01-07,16.72,16.75,15.05,15.05
2004-01-08,15.42,15.68,15.32,15.61
2004-01-09,16.15,16.88,15.57,16.75
2004-01-12,17.32,17.46,16.79,16.82
2004-01-13,16.06,18.33,16.53,18.04
2004-01-14,17.29,17.03,16.04,16.75
2004-01-15,17.07,17.31,15.49,15.56
2004-01-16,15.04,15.44,14.09,15
2004-01-20,15.77,16.13,15.09,15.21
2004-01-21,15.63,15.63,14.24,14.34
2004-01-22,14.02,14.87,14.01,14.71
2004-01-23,14.73,15.05,14.56,14.84
2004-01-26,15.78,15.78,14.52,14.55
2004-01-27,15.28,15.44,14.74,15.35
2004-01-28,15.37,17.06,15.29,16.78
2004-01-29,16.88,17.66,16.79,17.14
1 Date VIXOpen VIXHigh VIXLow VIXClose
2 2004-01-02 17.96 18.68 17.54 18.22
3 2004-01-05 18.45 18.49 17.44 17.49
4 2004-01-06 17.66 17.67 16.19 16.73
5 2004-01-07 16.72 16.75 15.05 15.05
6 2004-01-08 15.42 15.68 15.32 15.61
7 2004-01-09 16.15 16.88 15.57 16.75
8 2004-01-12 17.32 17.46 16.79 16.82
9 2004-01-13 16.06 18.33 16.53 18.04
10 2004-01-14 17.29 17.03 16.04 16.75
11 2004-01-15 17.07 17.31 15.49 15.56
12 2004-01-16 15.04 15.44 14.09 15
13 2004-01-20 15.77 16.13 15.09 15.21
14 2004-01-21 15.63 15.63 14.24 14.34
15 2004-01-22 14.02 14.87 14.01 14.71
16 2004-01-23 14.73 15.05 14.56 14.84
17 2004-01-26 15.78 15.78 14.52 14.55
18 2004-01-27 15.28 15.44 14.74 15.35
19 2004-01-28 15.37 17.06 15.29 16.78
20 2004-01-29 16.88 17.66 16.79 17.14

51
packages/portal/fixtures/datasetsVegaView/README.md
View File

@@ -1,51 +0,0 @@
CBOE Volatility Index (VIX) time-series dataset including daily open, close,
high and low. The CBOE Volatility Index (VIX) is a key measure of market
expectations of near-term volatility conveyed by S&P 500 stock index option
prices introduced in 1993.
## Data
From the [VIX FAQ][faq]:
> In 1993, the Chicago Board Options Exchange® (CBOE®) introduced the CBOE
> Volatility Index®, VIX®, and it quickly became the benchmark for stock market
> volatility. It is widely followed and has been cited in hundreds of news
> articles in the Wall Street Journal, Barron's and other leading financial
> publications. Since volatility often signifies financial turmoil, VIX is
> often referred to as the "investor fear gauge".
>
> VIX measures market expectation of near term volatility conveyed by stock
> index option prices. The original VIX was constructed using the implied
> volatilities of eight different OEX option series so that, at any given time,
> it represented the implied volatility of a hypothetical at-the-money OEX
> option with exactly 30 days to expiration.
>
> The New VIX still measures the market's expectation of 30-day volatility, but
> in a way that conforms to the latest thinking and research among industry
> practitioners. The New VIX is based on S&P 500 index option prices and
> incorporates information from the volatility "skew" by using a wider range of
> strike prices rather than just at-the-money series.
[faq]: http://www.cboe.com/micro/vix/faq.aspx
## Preparation
Run the shell script:
. scripts/process.sh
Output data is in `data/`.
### TODO
* Incorporate computed historical data (1990-2003)
* Consider incorporating VOX data
## License
No obvious statement on [historical data page][historical]. Given size and
factual nature of the data and its source from a US company would imagine this
was public domain and as such have licensed the Data Package under the Public
Domain Dedication and License (PDDL).
[historical]: http://www.cboe.com/micro/vix/historical.aspx

131
packages/portal/fixtures/datasetsVegaView/datapackage.json
View File

@@ -1,131 +0,0 @@
{
"name": "finance-vix",
"title": "VIX - CBOE Volatility Index",
"homepage": "http://www.cboe.com/micro/VIX/",
"version": "0.1.0",
"license": "PDDL-1.0",
"sources": [
{
"title": "CBOE VIX Page",
"name": "CBOE VIX Page",
"web": "http://www.cboe.com/micro/vix/historical.aspx"
}
],
"resources": [
{
"name": "vix-daily",
"path": "vix-daily.csv",
"format": "csv",
"mediatype": "text/csv",
"schema": {
"fields": [
{
"name": "Date",
"type": "date",
"description": ""
},
{
"name": "VIXOpen",
"type": "number",
"description": ""
},
{
"name": "VIXHigh",
"type": "number",
"description": ""
},
{
"name": "VIXLow",
"type": "number",
"description": ""
},
{
"name": "VIXClose",
"type": "number",
"description": ""
}
],
"primaryKey": "Date"
}
}
],
"views": [
{
"name": "vega4",
"resources": [
0
],
"specType": "vega",
"spec": {
"width": 600,
"height": 300,
"data": [
{
"name": "vix-daily"
}
],
"scales": [
{
"name": "VIXOpen",
"type": "point",
"range": "width",
"domain": {
"data": "vix-daily",
"field": "VIXOpen"
}
},
{
"name": "VIXHigh",
"type": "linear",
"range": "height",
"domain": {
"data": "vix-daily",
"field": "VIXHigh"
}
}
],
"axes": [
{
"orient": "bottom",
"scale": "VIXOpen"
},
{
"orient": "left",
"scale": "VIXHigh"
}
],
"marks": [
{
"type": "line",
"from": {
"data": "vix-daily"
},
"encode": {
"enter": {
"x": {
"scale": "VIXOpen",
"field": "VIXOpen"
},
"y": {
"scale": "VIXHigh",
"field": "VIXHigh"
},
"strokeWidth": {
"value": 2
}
},
"strokeOpacity": {
"value": 1
}
},
"hover": {
"strokeOpacity": {
"value": 0.5
}
}
}
]
}
}
]
}

20
packages/portal/fixtures/datasetsVegaView/vix-daily.csv
View File

@@ -1,20 +0,0 @@
Date,VIXOpen,VIXHigh,VIXLow,VIXClose
2004-01-02,17.96,18.68,17.54,18.22
2004-01-05,18.45,18.49,17.44,17.49
2004-01-06,17.66,17.67,16.19,16.73
2004-01-07,16.72,16.75,15.05,15.05
2004-01-08,15.42,15.68,15.32,15.61
2004-01-09,16.15,16.88,15.57,16.75
2004-01-12,17.32,17.46,16.79,16.82
2004-01-13,16.06,18.33,16.53,18.04
2004-01-14,17.29,17.03,16.04,16.75
2004-01-15,17.07,17.31,15.49,15.56
2004-01-16,15.04,15.44,14.09,15
2004-01-20,15.77,16.13,15.09,15.21
2004-01-21,15.63,15.63,14.24,14.34
2004-01-22,14.02,14.87,14.01,14.71
2004-01-23,14.73,15.05,14.56,14.84
2004-01-26,15.78,15.78,14.52,14.55
2004-01-27,15.28,15.44,14.74,15.35
2004-01-28,15.37,17.06,15.29,16.78
2004-01-29,16.88,17.66,16.79,17.14
1 Date VIXOpen VIXHigh VIXLow VIXClose
2 2004-01-02 17.96 18.68 17.54 18.22
3 2004-01-05 18.45 18.49 17.44 17.49
4 2004-01-06 17.66 17.67 16.19 16.73
5 2004-01-07 16.72 16.75 15.05 15.05
6 2004-01-08 15.42 15.68 15.32 15.61
7 2004-01-09 16.15 16.88 15.57 16.75
8 2004-01-12 17.32 17.46 16.79 16.82
9 2004-01-13 16.06 18.33 16.53 18.04
10 2004-01-14 17.29 17.03 16.04 16.75
11 2004-01-15 17.07 17.31 15.49 15.56
12 2004-01-16 15.04 15.44 14.09 15
13 2004-01-20 15.77 16.13 15.09 15.21
14 2004-01-21 15.63 15.63 14.24 14.34
15 2004-01-22 14.02 14.87 14.01 14.71
16 2004-01-23 14.73 15.05 14.56 14.84
17 2004-01-26 15.78 15.78 14.52 14.55
18 2004-01-27 15.28 15.44 14.74 15.35
19 2004-01-28 15.37 17.06 15.29 16.78
20 2004-01-29 16.88 17.66 16.79 17.14

9
packages/portal/jest.config.js
View File

@@ -1,9 +0,0 @@
module.exports = {
testPathIgnorePatterns: ["<rootDir>/.next/", "<rootDir>/node_modules/"],
setupFilesAfterEnv: ["<rootDir>/tests/setupTests.js"],
setupFiles: ["<rootDir>/tests/setupTests.js"],
transform: {
"^.+\\.(js|jsx|ts|tsx)$": "<rootDir>/node_modules/babel-jest",
"\\.(css|less|scss|sass)$": "identity-obj-proxy"
}
};

32
packages/portal/lib/dataset.js
View File

@@ -1,32 +0,0 @@
import remark from 'remark'
import html from 'remark-html'
import { Dataset } from 'frictionless.js'
import toArray from 'stream-to-array'
export async function getDataset(directory) {
// get dataset descriptor and resources
const f11sDataset = await Dataset.load(directory)
const descriptor = f11sDataset.descriptor
const resources = await Promise.all(f11sDataset.resources.map(async (resource) => {
let _tmp = resource.descriptor
let rowStream = await resource.rows({ keyed: true })
_tmp.sample = await toArray(rowStream)
_tmp.size = resource.size
return _tmp
}))
const readme = descriptor.readme
const processed = await remark()
.use(html)
.process(readme)
const readmeHtml = processed.toString()
const dataset = {
readme: readme,
readmeHtml: readmeHtml,
descriptor: descriptor,
resources: resources
}
return dataset
}

110
packages/portal/lib/utils.js
View File

@@ -1,110 +0,0 @@
import { simpleToPlotly, plotlyToPlotly, vegaToVega } from 'datapackage-render'
/**
* Prepare views for dataset
* @params {object} dataset object of the form:
* { readme: readme,
readmeHtml: readmeHtml,
descriptor: descriptor,
resources: resources
}
*/
export function addView(dataset) {
const views = dataset.descriptor.views
const countViews = views ? views.length : 0
if (countViews === 0) {
return {
props: {
dataset,
error: true
}
}
}
const specs = {} //hold list of view specs
for (let i = 0; i < countViews; i++) {
const view = views[i]
if (("resources" in view) && Array.isArray(view.resources)) {
let resource;
const resourceKey = view.resources[0]
if (typeof resourceKey == 'number') {
resource = dataset.resources[resourceKey]
view.resources[0] = resource
} else {
resource = dataset.resources.filter((resource) => {
return resource.name == resourceKey
})
view.resources[0] = resource[0]
}
} else {
view.resources = [dataset.resources[0]] //take the first resources in datapackage
}
view.resources[0].data = getDataForViewSpec(view.resources[0], view.specType)
view.resources[0]._values = view.resources[0].data
if (view.specType === 'simple') {
try {
const spec = simpleToPlotly(view)
if (spec) {
spec.specType = 'simple'
specs[i] = spec
}
} catch (err) {
console.log(err);
}
} else if (view.specType === 'plotly') {
try {
const spec = plotlyToPlotly(view)
if (spec) {
spec.specType = 'plotly'
specs[i] = spec
}
} catch (err) {
console.log(err);
}
} else if (view.specType === 'vega') {
try {
const spec = vegaToVega(view)
if (spec) {
spec.specType = 'vega'
specs[i] = spec
}
} catch (err) {
console.log(err);
}
}
}
return {
props: {
dataset,
specs: JSON.stringify(specs),
error: false
}
}
}
/**
* Generates the data for each view spec. Plotly and Vega accept different data
* formats.
* @param {*} resource
* @param {*} specType
*/
export function getDataForViewSpec(resource, specType) {
if (specType == "vega") {
return resource.sample
} else if (["simple", 'plotly'].includes(specType)) {
const sample = resource.sample
let data = []
data.push(Object.keys(sample[0])) //add the column names
for (let i = 0; i < sample.length; i++) {
const item = sample[i];
data.push(Object.values(item)) //add the rows
}
return data
}
}

57
packages/portal/package.json
View File

@@ -1,57 +0,0 @@
{
"name": "portal",
"description": "The data presentation framework",
"author": "Datopian",
"license": "MIT",
"version": "0.1.0",
"bin": {
"portal": "./bin/portal.js"
},
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"test": "jest --coverage"
},
"dependencies": {
"@material-ui/core": "^4.11.3",
"@material-ui/data-grid": "^4.0.0-alpha.20",
"@tailwindcss/typography": "^0.4.0",
"autoprefixer": "^10.0.4",
"datapackage-render": "git+https://github.com/frictionlessdata/datapackage-render-js.git",
"filesize": "^6.1.0",
"frictionless.js": "^0.13.4",
"next": "latest",
"plotly.js-basic-dist": "^1.58.4",
"postcss": "^8.1.10",
"react": "^17.0.1",
"react-dom": "^17.0.1",
"react-plotly.js": "^2.5.1",
"react-table": "^7.6.3",
"react-vega": "^7.4.2",
"remark": "^13.0.0",
"remark-html": "^13.0.1",
"tailwindcss": "^2.0.2",
"vega": "^5.19.1",
"vega-lite": "^5.0.0",
"chalk": "^4.1.0",
"commander": "^6.2.0",
"cpy": "^8.1.1",
"cross-spawn": "^7.0.3",
"figlet": "^1.5.0",
"listr": "^0.14.3",
"open": "^8.0.2",
"ora": "^5.1.0",
"prompts": "^2.4.0"
},
"devDependencies": {
"@testing-library/dom": "^7.29.6",
"@testing-library/jest-dom": "^5.11.9",
"@testing-library/react": "^11.2.5",
"babel-jest": "^26.6.3",
"identity-obj-proxy": "^3.0.0",
"jest": "^26.6.3",
"jest-canvas-mock": "^2.3.1",
"jest-dom": "^4.0.0"
}
}

8
packages/portal/pages/_app.js
View File

@@ -1,8 +0,0 @@
import '../styles/globals.css'
import '../styles/tailwind.css'
function MyApp({ Component, pageProps }) {
return <Component {...pageProps} />
}
export default MyApp

214
packages/portal/pages/index.js
View File

@@ -1,214 +0,0 @@
import path from 'path'
import Head from 'next/head'
import Table from '../components/Table'
import filesize from 'filesize'
import { Vega } from 'react-vega';
import { getDataset } from '../lib/dataset'
import Chart from '../components/Chart'
import { addView } from '../lib/utils'
const datasetsDirectory = process.env.PORTAL_DATASET_PATH
export default function Home({ dataset, specs }) {
if (!dataset && !specs) {
return (
<div className="container">
<Head>
<title>Dataset</title>
<link rel="icon" href="/favicon.ico" />
<link rel="preconnect" href="https://fonts.gstatic.com" />
<link href="https://fonts.googleapis.com/css2?family=Inconsolata&display=swap" rel="stylesheet" />
</Head>
<h1 data-testid="datasetTitle" className="text-3xl font-bold mb-8">
No dataset found in path
</h1>
</div>
)
}
const descriptor = dataset['descriptor']
const resources = dataset['resources']
let datasetSize = 0
if (resources) {
datasetSize = resources.length == 1 ?
resources[0].size :
resources.reduce((accumulator, currentValue) => {
return accumulator.size + currentValue.size
})
}
return (
<div className="container">
<Head>
<title>Dataset</title>
<link rel="icon" href="/favicon.ico" />
<link rel="preconnect" href="https://fonts.gstatic.com" />
<link href="https://fonts.googleapis.com/css2?family=Inconsolata&display=swap" rel="stylesheet" />
</Head>
<section className="m-8" name="key-info">
<h1 data-testid="datasetTitle" className="text-3xl font-bold mb-8">
{descriptor.title}
</h1>
<h1 className="text-2xl font-bold mb-4">Key info</h1>
<div className="grid grid-cols-7 gap-4">
<div>
<h3 className="text-1xl font-bold mb-2">Files</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Size</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Format</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Created</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Updated</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Licence</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Source</h3>
</div>
</div>
<div className="grid grid-cols-7 gap-4">
<div>
<h3 className="text-1xl">{resources.length}</h3>
</div>
<div>
<h3 className="text-1xl">{filesize(datasetSize, { bits: true })}</h3>
</div>
<div>
<h3 className="text-1xl">{resources[0].format} zip</h3>
</div>
<div>
<h3 className="text-1xl">{descriptor.created}</h3>
</div>
<div>
<h3 className="text-1xl">{descriptor.updated}</h3>
</div>
<div>
<h3 className="text-1xl">{descriptor.license}</h3>
</div>
<div>
<h3 className="text-1xl">
<a className="text-yellow-600"
href={descriptor.sources[0].web}>
{descriptor.sources[0].title}
</a>
</h3>
</div>
</div>
</section>
<section className="m-8" name="file-list">
<h1 className="text-2xl font-bold mb-4">Data Files</h1>
<div className="grid grid-cols-7 gap-4">
<div>
<h3 className="text-1xl font-bold mb-2">File</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Description</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Size</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Last Changed</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Download</h3>
</div>
</div>
{resources.map((resource, index) => {
return (
<div key={`${index}_${resource.name}`} className="grid grid-cols-7 gap-4">
<div>
<h3 className="text-1xl">{resource.name}</h3>
</div>
<div>
<h3 className="text-1xl">{resource.description || "No description"}</h3>
</div>
<div>
<h3 className="text-1xl">{filesize(resource.size, { bits: true })}</h3>
</div>
<div>
<h3 className="text-1xl">{resource.updated}</h3>
</div>
<div>
<h3 className="text-1xl">
<a className="text-yellow-600" href={resource.path}>
{resource.format} ({filesize(resource.size, { bits: true })})
</a>
</h3>
</div>
</div>
)
})}
</section>
<section className="m-8" name="graph">
<h1 className="text-2xl font-bold mb-4">Graph</h1>
{!specs || Object.keys(specs).length == 0 ? (<div>
<h1>No graph to display</h1>
</div>) :
(
Object.values(JSON.parse(specs)).map((spec, i) => {
if (spec.specType == "vega") {
return (
<div key={`${i}_views`} className="ml-14">
<Vega spec={spec} />
</div>
)
} else if (["simple", "plotly"].includes(spec.specType)) {
return (
<div key={`${i}_views`}>
<Chart spec={spec} />
</div>)
} else {
return <h1 key={`${i}_views`}>Cannot display view</h1>
}
})
)}
</section>
<section className="m-8" name="sample-table" >
<h1 className="text-2xl font-bold mb-4">Data Preview</h1>
<h2 className="text-1xl">{descriptor.title}</h2>
{resources[0].sample ? (
<Table data={resources[0].sample} schema={resources[0].schema} />
) : (
'No preview is available for this dataset'
)}
</section>
<section className="m-8" name="sample-table">
<h1 className="text-2xl font-bold mb-4">README</h1>
<div className="prose">
<div dangerouslySetInnerHTML={{ __html: dataset.readmeHtml }} />
</div>
</section>
</div>
)
}
export async function getStaticProps() {
if (!datasetsDirectory) {
return { props: {} }
}
const dataset = await getDataset(datasetsDirectory)
const datasetWithViews = addView(dataset)
return datasetWithViews
}

8
packages/portal/postcss.config.js
View File

@@ -1,8 +0,0 @@
// If you want to use other PostCSS plugins, see the following:
// https://tailwindcss.com/docs/using-with-preprocessors
module.exports = {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
}

BIN
packages/portal/public/favicon.ico
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 15 KiB

1
packages/portal/setupTests.js
View File

@@ -1 +0,0 @@
import "@testing-library/jest-dom/extend-expect";

3
packages/portal/styles/globals.css
View File

@@ -1,3 +0,0 @@
.MuiTableCell-root {
@apply font-mono
}

3
packages/portal/styles/tailwind.css
View File

@@ -1,3 +0,0 @@
@tailwind base;
@tailwind components;
@tailwind utilities;

22
packages/portal/tailwind.config.js
View File

@@ -1,22 +0,0 @@
const defaultTheme = require("tailwindcss/defaultTheme");
module.exports = {
purge: ['./pages/**/*.{js,ts,jsx,tsx}', './components/**/*.{js,ts,jsx,tsx}'],
darkMode: false, // or 'media' or 'class'
theme: {
container: {
center: true,
},
extend: {
fontFamily: {
mono: ["Inconsolata", ...defaultTheme.fontFamily.mono]
}
},
},
variants: {
extend: {},
},
plugins: [
require('@tailwindcss/typography'),
],
}

26
packages/portal/tests/components/Chart.test.js
View File

@@ -1,26 +0,0 @@
import React from 'react'
import {render } from '@testing-library/react';
import path from 'path'
import Chart from '../../components/Chart';
import { getDataset } from "../../lib/dataset"
import { addView } from '../../lib/utils'
let dataset
let datasetWithView
beforeAll(async () => {
const datasetsDirectory = path.join(process.cwd(), 'fixtures', 'datasetsPlotlyView')
dataset = await getDataset(datasetsDirectory)
datasetWithView = addView(dataset)
});
/** @test {Chart Component} */
describe('Chart Component', () => {
it('should render without crashing', async () => {
const spec = JSON.parse(datasetWithView.props.specs)[0]
const { findByTestId } = render(<Chart spec={spec} />)
expect(await findByTestId("plotlyChart"))
});
});

27
packages/portal/tests/components/Table.test.js
View File

@@ -1,27 +0,0 @@
import React from 'react'
import { render } from '@testing-library/react';
import path from 'path'
import Table from '../../components/Table';
import { getDataset } from "../../lib/dataset"
let dataset
beforeAll(async () => {
const datasetsDirectory = path.join(process.cwd(), 'fixtures', 'datasetsPlotlyView')
dataset = await getDataset(datasetsDirectory)
});
/** @test {Table Component} */
describe('Table Component', () => {
it('should render without crashing', async () => {
const resource = dataset.resources[0]
render(<Table data={resource.sample} schema={resource.schema} />)
});
it('tableGrid div is found', async () => {
const resource = dataset.resources[0]
const { findByTestId } = render(<Table data={resource.sample} schema={resource.schema} />)
expect(await findByTestId('tableGrid'))
});
});

33
packages/portal/tests/lib/dataset.test.js
View File

@@ -1,33 +0,0 @@
import { getDataset } from "../../lib/dataset"
import path from 'path'
let directory
let dataset
beforeAll(async () => {
directory = path.join(process.cwd(), 'fixtures', 'datasetsDoubleView')
dataset = await getDataset(directory)
})
describe("Dataset", () => {
it("loads a dataset from a local folder", async () => {
expect(dataset).toStrictEqual(
expect.objectContaining({
readme: expect.any(String),
readmeHtml: expect.any(String),
descriptor: expect.any(Object),
resources: expect.any(Object),
})
)
})
it("returns a resource with required fields", () => {
const resource = dataset.resources[0]
const expectedFields = ["path", "pathType", "name", "format", "mediatype",
"schema", "encoding", "sample", "size"]
expect(expectedFields).toStrictEqual(
Object.keys(resource)
)
})
});

78
packages/portal/tests/lib/utils.test.js
View File

@@ -1,78 +0,0 @@
import path from 'path'
import { getDataset } from "../../lib/dataset"
import { addView, getDataForViewSpec } from '../../lib/utils'
const plotlyDatasetsDirectory = path.join(process.cwd(), 'fixtures', 'datasetsPlotlyView')
const vegaDatasetsDirectory = path.join(process.cwd(), 'fixtures', 'datasetsVegaView')
const doubleDatasetsDirectory = path.join(process.cwd(), 'fixtures', 'datasetsDoubleView')
let plotlyDataset
let vegaDataset
let doubleDataset
let plotlyDatasetWithView
let vegaDatasetWithView
let doubleDatasetWithView
beforeAll(async () => {
plotlyDataset = await getDataset(plotlyDatasetsDirectory)
vegaDataset = await getDataset(vegaDatasetsDirectory)
doubleDataset = await getDataset(doubleDatasetsDirectory)
plotlyDatasetWithView = addView(plotlyDataset)
vegaDatasetWithView = addView(vegaDataset)
doubleDatasetWithView = addView(doubleDataset)
});
describe("AddView", () => {
it("_value field is added to Plotly datapackage", () => {
const resource = plotlyDatasetWithView.props.dataset.resources[0]
expect("_values" in resource).toBe(true)
expect(resource["_values"].length > 0).toBe(true)
});
it("Plotly spec is added to datapackage", () => {
const spec = JSON.parse(plotlyDatasetWithView.props.specs)[0]
expect(spec.specType).toBe("plotly")
expect(spec.layout.title).toBe("Plotly Layout Title")
expect(spec.data[0].x.length).toBeGreaterThan(0)
expect(spec.data[0].y.length).toBeGreaterThan(0)
});
it("_value field is added to datapackage with double views", () => {
const resources = doubleDatasetWithView.props.dataset.resources
resources.map((resource) => {
expect("_values" in resource).toBe(true)
expect(resource["_values"].length > 0).toBe(true)
})
});
it("view spec is created for each view in a datapackage", () => {
const specs = JSON.parse(doubleDatasetWithView.props.specs)
const simpleSpec = specs[0]
const plotlySpec = specs[1]
expect(simpleSpec.specType).toBe("simple")
expect(simpleSpec.layout.title).toBe("title1")
expect(simpleSpec.data[0].x.length).toBeGreaterThan(0)
expect(simpleSpec.data[0].y.length).toBeGreaterThan(0)
expect(plotlySpec.specType).toBe("plotly")
expect(plotlySpec.layout.title).toBe("Plotly Layout Title")
expect(plotlySpec.data[0].x.length).toBeGreaterThan(0)
expect(plotlySpec.data[0].y.length).toBeGreaterThan(0)
});
});
describe("getDataForViewSpec", () => {
it("Generates right data for vega spec", ()=>{
const resource = vegaDataset.resources[0]
const data = getDataForViewSpec(resource, "vega")
expect(data).toStrictEqual(resource.sample)
})
it("Generates right data for plotly spec", ()=>{
const resource = plotlyDataset.resources[0]
const data = getDataForViewSpec(resource, "plotly")
expect(data).not.toStrictEqual(resource.sample[0])
expect(data[0]).toStrictEqual(Object.keys(resource.sample[0]))
})
})

39
packages/portal/tests/pages/index.test.js
View File

@@ -1,39 +0,0 @@
import React from 'react'
import { render } from '@testing-library/react';
import path from 'path'
import Home from '../../pages/index';
import { getDataset } from "../../lib/dataset"
import { addView } from '../../lib/utils'
let plotlyDatasetWithView
beforeAll(async () => {
const plotlyDatasetsDirectory = path.join(process.cwd(), 'fixtures', 'datasetsPlotlyView')
const plotlyDataset = await getDataset(plotlyDatasetsDirectory)
plotlyDatasetWithView = addView(plotlyDataset)
});
/** @test {Home Component} */
describe('Home Component', () => {
it('should render without crashing', async () => {
const dataset = plotlyDatasetWithView.props.dataset
const specs = plotlyDatasetWithView.props.specs
const { findAllByText } = render(<Home dataset={dataset} specs={specs} />)
expect(await findAllByText('README'))
});
it('Sections are found in home page', async () => {
const dataset = plotlyDatasetWithView.props.dataset
const specs = plotlyDatasetWithView.props.specs
const { findByTestId, findAllByText } = render(<Home dataset={dataset} specs={specs} />)
expect(await findAllByText('Key info'))
expect(await findAllByText('Data Files'))
expect(await findAllByText('Graph'))
expect(await findAllByText('Data Preview'))
expect(await findByTestId('datasetTitle'))
});
});

1
packages/portal/tests/setupTests.js
View File

@@ -1 +0,0 @@
import 'jest-canvas-mock';