Merge branch 'master' of https://github.com/datopian/portal.js into add/new-portaljs

This commit is contained in:
Rising Odegua 2021-03-15 14:10:14 +01:00
commit b50b7e7a04
107 changed files with 47710 additions and 0 deletions

3
.babelrc Normal file
View File

@ -0,0 +1,3 @@
{
"presets": ["next/babel"]
}

View File

@ -0,0 +1,17 @@
name: Cypress Integration Tests
on: [push]
jobs:
cypress-run:
runs-on: ubuntu-16.04
steps:
- name: Checkout
uses: actions/checkout@v1
- name: Cypress run
uses: cypress-io/github-action@v2
with:
working-directory: examples/catalog
browser: chrome
build: yarn run build
start: yarn start
wait-on: "http://localhost:3000"

View File

@ -0,0 +1,30 @@
name: Portal-Catalog-Tests
on: push
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install modules
run: |
cd examples/catalog
yarn
- name: Run tests
run: |
cd examples/catalog
yarn test -u
cypress-run:
runs-on: ubuntu-16.04
steps:
- name: Checkout
uses: actions/checkout@v1
- name: Cypress run
uses: cypress-io/github-action@v2
with:
working-directory: examples/catalog
browser: chrome
build: yarn run build
start: yarn start
wait-on: "http://localhost:3000"

17
.github/workflows/portal-test.yml vendored Normal file
View File

@ -0,0 +1,17 @@
name: portal
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
jest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install modules
run: yarn
- name: Run tests
run: yarn test

30
.gitignore vendored Normal file
View File

@ -0,0 +1,30 @@
# dependencies
node_modules/
.pnp
.pnp.js
# testing
coverage
cypress/videos
# next.js
.next/
/out/
# production
/build
# misc
.DS_Store
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# local env files
.env
.env.local
.env.development.local
.env.test.local
.env.production.local

50
README.md Normal file
View File

@ -0,0 +1,50 @@
<h1 align="center">
🌀 Portal.JS<br/>
A gateway to your data
</h1>
🌀 `Portal` is the data presentation framework. `Portal` can be used to showcase a single dataset or build a full-scale data catalog/portal. `Portal` is built in Javascript and React on top of the popular Next.js framework.
## Status
`Portal` is currently focused on presenting a single (Frictionless) dataset for viewing and exploration.
## Install
Git clone then:
```
yarn install
```
## Usage
In this directory:
```bash
export PORTAL_DATASET_PATH=/path/to/my/dataset
yarn dev
```
And you will get a nice dataset page at `http://localhost:3000`
![](https://i.imgur.com/KSEtNF1.png)
## Design Notes
Portal.js is a React and NextJS based framework for building dataset/resources pages and catalogs. It consists of:
* React components for data portal functionality e.g. data tables, graphs, dataset pages etc
* Tooling to load data (based on Frictionless)
* Template sites you can reuse using `create-next-app`
* Single dataset micro-site
* Github backed catalog
* CKAN backed catalog
* ...
* Local development environment
* Deployment integration with DataHub.io
In summary, technically PortalJS is: NextJS + data specific react components + data loading glue (mostly using frictionless-js).

130
bin/portal.js Executable file
View File

@ -0,0 +1,130 @@
#!/usr/bin/env node
const Listr = require('listr')
const { program } = require('commander')
const chalk = require('chalk')
const path = require('path')
const figlet = require('figlet')
const { exec } = require('child_process');
const package = require('../package.json')
const fs = require('fs')
console.log(
chalk.yellow(figlet.textSync('Portal App', { horizontalLayout: 'full' }))
)
function directoryExists(filePath) {
return fs.existsSync(filePath);
}
/**
* Executes a shell command and return it as a Promise.
* @param cmd {string}
* @return {Promise<string>}
*/
function execShellCommand(cmd) {
return new Promise((resolve, reject) => {
exec(cmd, (error, stdout, stderr) => {
if (error) {
console.warn(error);
reject(error)
}
resolve(stdout ? stdout : stderr);
});
});
}
function processArgs(args) {
const userArgs = { npm: false, override: false, port: 3000, path: process.cwd() }
args.forEach((arg) => {
if (arg.includes("=")) {
let temp = arg.split("=")
userArgs[temp[0]] = temp[1]
}
})
return userArgs
}
// Output path to create new portal app
// Commander parameters to specify CLI behavior
program
.version(package.version)
.usage('show [ path=/some/path | npm=true | port=3000 ]')
.description('Creates a portal application from specified dataset',
)
.option('npm', '[true || false] Install dependencies using npm instead yarn, defaults to false (yarn)')
.option('port', 'Server port, defaults to 3000')
.parse(process.argv,)
const userArgs = processArgs(program.args)
/**
* Main method to start CLI and validate inputs
*/
async function run() {
const datasetPath = userArgs.path.trim()
if (directoryExists(datasetPath)) {
console.log(
`${chalk.yellow(`Using dataset found in: ${chalk.cyan(datasetPath)}`)}`
)
} else {
console.log(
`${chalk.red(`Directory: ${chalk.cyan(datasetPath)} does not exist!`)}`
)
process.exit(1)
}
const portalGithubRepo = "https://github.com/datopian/portal-experiment.git"
const portalLocalRepoDirectory = path.join(datasetPath, 'portal-experiment')
const cloneRepoCmd = `cd ${datasetPath} &&
export PORTAL_DATASET_PATH=${datasetPath} &&
git clone ${portalGithubRepo}`
const buildNextAppCmd = userArgs.npm ? `cd ${portalLocalRepoDirectory} && npm install && npm run build` :
`cd ${portalLocalRepoDirectory} && yarn && yarn build`
const startNextAppCmd = userArgs.npm ?
`cd ${portalLocalRepoDirectory} && npm run start -p ${userArgs.port}` :
`cd ${portalLocalRepoDirectory} && yarn start -p ${userArgs.port}`
//Tasks workflow
const tasks = new Listr([
{
title: 'Getting portal tools...',
task: async () => {
try {
if (directoryExists(portalLocalRepoDirectory)) {
console.log(
chalk.cyan(`${package.name} ${chalk.yellow('already exists! Skipping this step')}`))
} else {
await execShellCommand(cloneRepoCmd)
}
} catch (error) {
throw error
}
},
},
{
title: 'Preparing your app...',
task: async () => { await execShellCommand(buildNextAppCmd) }
},
{
title: `Displaying dataset at http://localhost:${userArgs.port}`,
task: () => execShellCommand(startNextAppCmd),
}
])
tasks.run()
}
run().catch((error) => {
console.log(error)
console.log()
process.exit(1)
})

31
components/Chart.js vendored Normal file
View File

@ -0,0 +1,31 @@
import React, { useState, useEffect } from 'react';
import createPlotlyComponent from "react-plotly.js/factory";
let Plot;
const Chart = (props) => {
const [plotCreated, setPlotCreated] = useState(0) //0: false, 1: true
useEffect(() => {
import(`plotly.js-basic-dist`).then(Plotly => { //import Plotly dist when Page has been generated
Plot = createPlotlyComponent(Plotly);
setPlotCreated(1)
});
}, [])
if (!plotCreated) {
return (<div>Loading...</div>)
}
return (
<div data-testid="plotlyChart">
<Plot {...props.spec}
layout={{ autosize: true }}
style={{ width: "100%", height: "100%" }}
useResizeHandler={true}
/>
</div>
)
}
export default Chart

28
components/Table.js Normal file
View File

@ -0,0 +1,28 @@
/* eslint-disable max-len */
import React from 'react';
import { DataGrid } from '@material-ui/data-grid';
/*
* @param schema: Frictionless Table Schmea
* @param data: an array of data objects e.g. [ {a: 1, b: 2}, {a: 5, b: 7} ]
*/
const Table = ({ schema, data }) => {
const columns = schema.fields.map((field) => (
{
field: field.title || field.name,
headerName: field.name,
width: 300
}
))
data = data.map((item, index)=>{
item.id = index //Datagrid requires every row to have an ID
return item
})
return (
<div data-testid="tableGrid" style={{ height: 400, width: '100%' }}>
<DataGrid rows={data} columns={columns} pageSize={5} />
</div>
);
}
export default Table

View File

@ -0,0 +1,3 @@
{
"presets": ["next/babel"]
}

View File

@ -0,0 +1,46 @@
module.exports = {
root: true,
parser: '@typescript-eslint/parser',
parserOptions: {
ecmaFeatures: { jsx: true },
},
env: {
browser: true,
node: true,
},
extends: [
'eslint:recommended',
'plugin:@typescript-eslint/eslint-recommended',
'plugin:@typescript-eslint/recommended',
'plugin:react/recommended',
'plugin:jsx-a11y/recommended',
// Prettier plugin and recommended rules
'prettier/@typescript-eslint',
'plugin:prettier/recommended',
],
rules: {
// Include .prettierrc.js rules
'prettier/prettier': ['error', {}, { usePrettierrc: true }],
'react/prop-types': 'off',
'react/react-in-jsx-scope': 'off',
'@typescript-eslint/explicit-function-return-type': 'off',
'@typescript-eslint/ban-ts-ignore': 'off',
'@typescript-eslint/no-var-requires': 'off',
'jsx-a11y/label-has-associated-control': [
'error',
{
labelComponents: [],
labelAttributes: [],
controlComponents: [],
assert: 'either',
depth: 25,
},
],
'@typescript-eslint/no-explicit-any': 'off',
},
settings: {
react: {
version: 'detect',
},
},
};

View File

@ -0,0 +1,24 @@
node_modules/
# testing
/coverage
# next.js
.next/
/out/
# yarn
yarn-error.log
yarn.lock
# sass
.sass-cache
# misc
sandbox/*
.env
.staging.env
.nyc_output/*
.DS_Store
public/

View File

@ -0,0 +1,8 @@
module.exports = {
semi: true,
trailingComma: 'es5',
singleQuote: true,
printWidth: 79,
tabWidth: 2,
useTabs: false,
};

306
examples/catalog/README.md Normal file
View File

@ -0,0 +1,306 @@
<h1 align="center">
🌀 Portal.JS<br/>
The javascript framework for<br/>
data portals
</h1>
🌀 `Portal` is a framework for rapidly building rich data portal frontends using a modern frontend approach (javascript, React, SSR).
`Portal` assumes a "decoupled" approach where the frontend is a separate service from the backend and interacts with backend(s) via an API. It can be used with any backend and has out of the box support for [CKAN][]. `portal` is built in Javascript and React on top of the popular [Next.js][] framework.
[ckan]: https://ckan.org/
[next.js]: https://nextjs.com/
Live DEMO: https://portal.datopian1.now.sh
## Features
- 🗺️ Unified sites: present data and content in one seamless site, pulling datasets from a DMS (e.g. CKAN) and content from a CMS (e.g. wordpress) with a common internal API.
- 👩‍💻 Developer friendly: built with familiar frontend tech Javascript, React etc
- 🔋 Batteries included: Full set of portal components out of the box e.g. catalog search, dataset showcase, blog etc.
- 🎨 Easy to theme and customize: installable themes, use standard CSS and React+CSS tooling. Add new routes quickly.
- 🧱 Extensible: quickly extend and develop/import your own React components
- 📝 Well documented: full set of documentation plus the documentation of NextJS and Apollo.
### For developers
- 🏗 Build with modern, familiar frontend tech such as Javascript and React.
- 🚀 NextJS framework: so everything in NextJS for free React, SSR, static site generation, huge number of examples and integrations etc.
- SSR => unlimited number of pages, SEO etc whilst still using React.
- Static Site Generation (SSG) (good for small sites) => ultra-simple deployment, great performance and lighthouse scores etc
- 📋 Typescript support
## Getting Started
### Setup
Install a recent version of Node. You'll need Node 10.13 or later.
### Create a Portal app
To create a Portal app, open your terminal, cd into the directory you'd like to create the app in, and run the following command:
```console
npm init portal-app my-data-portal
```
> NB: Under the hood, this uses the tool called create-next-app, which bootstraps a Next.js app for you. It uses this template through the --example flag.
>
> If it doesnt work, please open an issue.
## Guide
### Styling 🎨
We use Tailwind as a CSS framework. Take a look at `/styles/index.css` to see what we're importing from Tailwind bundle. You can also configure Tailwind using `tailwind.config.js` file.
Have a look at Next.js support of CSS and ways of writing CSS:
https://nextjs.org/docs/basic-features/built-in-css-support
### Backend
So far the app is running with mocked data behind. You can connect CMS and DMS backends easily via environment variables:
```console
$ export DMS=http://ckan:5000
$ export CMS=http://myblog.wordpress.com
```
> Note that we don't yet have implementations for the following CKAN features:
>
> - Activities
> - Auth
> - Groups
> - Facets
### Routes
These are the default routes set up in the "starter" app.
- Home `/`
- Search `/search`
- Dataset `/@org/dataset`
- Resource `/@org/dataset/r/resource`
- Organization `/@org`
- Collection (aka group in CKAN) (?) - suggest to merge into org
- Static pages, eg, `/about` etc. from CMS or can do it without external CMS, e.g., in Next.js
### New Routes
TODO
### Data fetching
We use Apollo client which allows us to query data with GraphQL. We have setup CKAN API for the demo (it uses demo.ckan.org as DMS):
http://portal.datopian1.now.sh/
Note that we don't have Apollo Server but we connect CKAN API using [`apollo-link-rest`](https://www.apollographql.com/docs/link/links/rest/) module. You can see how it works in [lib/apolloClient.ts](https://github.com/datopian/portal/blob/master/lib/apolloClient.ts) and then have a look at [pages/\_app.tsx](https://github.com/datopian/portal/blob/master/pages/_app.tsx).
For development/debugging purposes, we suggest installing the Chrome extension - https://chrome.google.com/webstore/detail/apollo-client-developer-t/jdkknkkbebbapilgoeccciglkfbmbnfm.
#### i18n configuration
Portal.js is configured by default to support both `English` and `French` subpath for language translation. But for subsequent users, this following steps can be used to configure i18n for other languages;
1. Update `next.config.js`, to add more languages to the i18n locales
```js
i18n: {
locales: ['en', 'fr', 'nl-NL'], // add more language to the list
defaultLocale: 'en', // set the default language to use
},
```
2. Create a folder for the language in `locales` --> `locales/en-Us`
3. In the language folder, different namespace files (json) can be created for each translation. For the `index.js` use-case, I named it `common.json`
```json
// locales/en/common.json
{
"title" : "Portal js in English",
}
// locales/fr/common.json
{
"title" : "Portal js in French",
}
```
4. To use on pages using Server-side Props.
```js
import { loadNamespaces } from './_app';
import useTranslation from 'next-translate/useTranslation';
const Home: React.FC = ()=> {
const { t } = useTranslation();
return (
<div>{t(`common:title`)}</div> // we use common and title base on the common.json data
);
};
export const getServerSideProps: GetServerSideProps = async ({ locale }) => {
........ ........
return {
props : {
_ns: await loadNamespaces(['common'], locale),
}
};
};
```
5. Go to the browser and view the changes using language subpath like this `http://localhost:3000` and `http://localhost:3000/fr`. **Note** The subpath also activate chrome language Translator
#### Pre-fetch data in the server-side
When visiting a dataset page, you may want to fetch the dataset metadata in the server-side. To do so, you can use `getServerSideProps` function from NextJS:
```javascript
import { GetServerSideProps } from 'next';
import { initializeApollo } from '../lib/apolloClient';
import gql from 'graphql-tag';
const QUERY = gql`
query dataset($id: String) {
dataset(id: $id) @rest(type: "Response", path: "package_show?{args}") {
result
}
}
`;
...
export const getServerSideProps: GetServerSideProps = async (context) => {
const apolloClient = initializeApollo();
await apolloClient.query({
query: QUERY,
variables: {
id: 'my-dataset'
},
});
return {
props: {
initialApolloState: apolloClient.cache.extract(),
},
};
};
```
This would fetch the data from DMS and save it in the Apollo cache so that we can query it again from the components.
#### Access data from a component
Consider situation when rendering a component for org info on the dataset page. We already have pre-fetched dataset metadata that includes `organization` property with attributes such as `name`, `title` etc. We can now query only organization part for our `Org` component:
```javascript
import { useQuery } from '@apollo/react-hooks';
import gql from 'graphql-tag';
export const GET_ORG_QUERY = gql`
query dataset($id: String) {
dataset(id: $id) @rest(type: "Response", path: "package_show?{args}") {
result {
organization {
name
title
image_url
}
}
}
}
`;
export default function Org({ variables }) {
const { loading, error, data } = useQuery(
GET_ORG_QUERY,
{
variables: { id: 'my-dataset' }
}
);
...
const { organization } = data.dataset.result;
return (
<>
{organization ? (
<>
<img
src={
organization.image_url
}
className="h-5 w-5 mr-2 inline-block"
/>
<Link href={`/@${organization.name}`}>
<a className="font-semibold text-primary underline">
{organization.title || organization.name}
</a>
</Link>
</>
) : (
''
)}
</>
);
}
```
#### Add a new data source
TODO
## Developers
### Boot the local instance
Install the dependencies:
```bash
yarn # or npm i
```
Boot the demo portal:
```console
$ yarn dev # or npm run dev
```
Open [http://localhost:3000](http://localhost:3000) to see the home page 🎉
You can start editing the page by modifying `/pages/index.tsx`. The page auto-updates as you edit the file.
### Tests
We use Jest for running tests:
```bash
yarn test # or npm run test
# turn on watching
yarn test --watch
```
We use Cypress tests as well
```
yarn run e2e
```
### Architecture
- Language: Javascript
- Framework: NextJS - https://nextjs.org/
- Data layer API: GraphQL using Apollo. So controllers access data using GraphQL “gatsby like”
### Key Pages
See https://tech.datopian.com/frontend/

View File

@ -0,0 +1,23 @@
import React from 'react';
import { render } from '@testing-library/react';
import Form from '../../../components/search/Form';
const useRouter = jest.spyOn(require('next/router'), 'useRouter');
test('📸 of Form component with empty', () => {
useRouter.mockImplementationOnce(() => ({
query: { search: '', sort: '' },
}));
const { container } = render(<Form />);
expect(container).toMatchSnapshot();
});
test('📸 of Form component with query', () => {
useRouter.mockImplementationOnce(() => ({
query: { search: 'gdp', sort: '' },
}));
const { container } = render(<Form />);
expect(container).toMatchSnapshot();
});

View File

@ -0,0 +1,16 @@
import React from 'react';
import renderer from 'react-test-renderer';
import Item from '../../../components/search/Item';
test('📸 of Input component with empty', () => {
const fixture = {
name: 'qw',
title: '12',
organization: null,
__typename: 'Package',
};
const tree = renderer
.create(<Item datapackage={fixture} key={0} />)
.toJSON();
expect(tree).toMatchSnapshot();
});

View File

@ -0,0 +1,133 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`📸 of Form component with empty 1`] = `
<div>
<form
class="items-center"
>
<div
class="flex"
>
<input
aria-label="Search"
class="bg-white focus:outline-none focus:shadow-outline border border-gray-300 w-1/2 rounded-lg py-2 px-4 block appearance-none leading-normal"
name="q"
placeholder="Search"
type="text"
value=""
/>
<button
class="inline-block text-sm px-4 py-3 mx-3 leading-none border rounded text-white bg-black border-black lg:mt-0"
>
Search
</button>
</div>
<div
class="inline-block my-6 float-right"
>
<label
for="field-order-by"
>
Order by:
</label>
<select
class="bg-white"
id="field-order-by"
name="sort"
>
<option
value="score:desc"
>
Relevance
</option>
<option
value="title_string:asc"
>
Name Ascending
</option>
<option
value="title_string:desc"
>
Name Descending
</option>
<option
value="metadata_modified:desc"
>
Last Modified
</option>
<option
value="views_recent:desc"
>
Popular
</option>
</select>
</div>
</form>
</div>
`;
exports[`📸 of Form component with query 1`] = `
<div>
<form
class="items-center"
>
<div
class="flex"
>
<input
aria-label="Search"
class="bg-white focus:outline-none focus:shadow-outline border border-gray-300 w-1/2 rounded-lg py-2 px-4 block appearance-none leading-normal"
name="q"
placeholder="Search"
type="text"
value=""
/>
<button
class="inline-block text-sm px-4 py-3 mx-3 leading-none border rounded text-white bg-black border-black lg:mt-0"
>
Search
</button>
</div>
<div
class="inline-block my-6 float-right"
>
<label
for="field-order-by"
>
Order by:
</label>
<select
class="bg-white"
id="field-order-by"
name="sort"
>
<option
value="score:desc"
>
Relevance
</option>
<option
value="title_string:asc"
>
Name Ascending
</option>
<option
value="title_string:desc"
>
Name Descending
</option>
<option
value="metadata_modified:desc"
>
Last Modified
</option>
<option
value="views_recent:desc"
>
Popular
</option>
</select>
</div>
</form>
</div>
`;

View File

@ -0,0 +1,31 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`📸 of Input component with empty 1`] = `
<div
className="mb-6"
>
<h3
className="text-xl font-semibold"
>
<a
className="text-primary"
href="/@dataset/qw"
onClick={[Function]}
onMouseEnter={[Function]}
>
12
</a>
</h3>
<a
className="text-gray-500 block mt-1"
href="/@dataset"
onClick={[Function]}
onMouseEnter={[Function]}
>
dataset
</a>
<div
className="leading-relaxed mt-2"
/>
</div>
`;

View File

@ -0,0 +1,15 @@
type LinkProps = {
url: string;
format: any;
};
const CustomLink: React.FC<LinkProps> = ({ url, format }: LinkProps) => (
<a
href={url}
className="bg-white hover:bg-gray-200 border text-black font-semibold py-2 px-4 rounded"
>
{format}
</a>
);
export default CustomLink;

View File

@ -0,0 +1,17 @@
const ErrorMessage: React.FC<{ message: any }> = ({ message }) => {
return (
<aside>
{message}
<style jsx>{`
aside {
padding: 1.5em;
font-size: 14px;
color: white;
background-color: red;
}
`}</style>
</aside>
);
};
export default ErrorMessage;

View File

@ -0,0 +1,36 @@
interface TableProps {
columns: Array<any>;
data: Array<any>;
className?: string;
}
const Table: React.FC<TableProps> = ({ columns, data, className }) => {
return (
<table className={`table-auto w-full text-sm text-left my-6 ${className}`}>
<thead>
<tr>
{columns.map(({ key, name }) => (
<th key={key} className="px-4 py-2">
{name}
</th>
))}
</tr>
</thead>
<tbody>
{data.map((item) => (
<tr key={item.id}>
{columns.map(({ key, render }) => (
<td key={key} className="px-4 py-2">
{(render && typeof render === 'function' && render(item)) ||
item[key] ||
''}
</td>
))}
</tr>
))}
</tbody>
</table>
);
};
export default Table;

View File

@ -0,0 +1,5 @@
import Table from './Table';
import ErrorMessage from './Error';
import CustomLink from './CustomLink';
export { Table, ErrorMessage, CustomLink };

View File

@ -0,0 +1,53 @@
import { useQuery } from '@apollo/react-hooks';
import { Table, ErrorMessage } from '../_shared';
import { GET_DATAPACKAGE_QUERY } from '../../graphql/queries';
const columns = [
{
name: 'Files',
key: 'files',
render: ({ resources }) => (resources && resources.length) || 0,
},
{
name: 'Size',
key: 'size',
},
{
name: 'Format',
key: 'format',
},
{
name: 'Created',
key: 'metadata_created',
},
{
name: 'Updated',
key: 'metadata_modified',
},
{
name: 'License',
key: 'license',
},
{
name: 'Source',
key: 'source',
},
];
const About: React.FC<{ variables: any }> = ({ variables }) => {
const { loading, error, data } = useQuery(GET_DATAPACKAGE_QUERY, {
variables,
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading dataset." />;
if (loading) return <div>Loading</div>;
const { result } = data.dataset;
return <Table columns={columns} data={[result]} />;
};
export default About;

View File

@ -0,0 +1,45 @@
import Link from 'next/link';
import { useQuery } from '@apollo/react-hooks';
import { ErrorMessage } from '../_shared';
import { GET_ORG_QUERY } from '../../graphql/queries';
const Org: React.FC<{ variables: any }> = ({ variables }) => {
const { loading, error, data } = useQuery(GET_ORG_QUERY, {
variables,
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading dataset." />;
if (loading) return <div>Loading</div>;
const { organization } = data.dataset.result;
return (
<>
{organization ? (
<>
<img
src={
organization.image_url ||
'https://datahub.io/static/img/datahub-cube-edited.svg'
}
className="h-5 w-5 mr-2 inline-block"
alt="org_img"
/>
<Link href={`/@${organization.name}`}>
{/* eslint-disable-next-line jsx-a11y/anchor-is-valid */}
<a className="font-semibold text-primary underline">
{organization.title || organization.name}
</a>
</Link>
</>
) : (
''
)}
</>
);
};
export default Org;

View File

@ -0,0 +1,69 @@
/* eslint-disable jsx-a11y/anchor-is-valid */
/* eslint-disable react/display-name */
import Link from 'next/link';
import { useQuery } from '@apollo/react-hooks';
import { Table, ErrorMessage } from '../_shared';
import { GET_RESOURCES_QUERY } from '../../graphql/queries';
const columns = [
{
name: 'File',
key: 'file',
render: ({ name: resName, title, parentName }) => (
<Link href={`${parentName}/r/${resName}`}>
<a className="underline">{title || resName}</a>
</Link>
),
},
{
name: 'Format',
key: 'format',
},
{
name: 'Created',
key: 'created',
},
{
name: 'Updated',
key: 'last_modified',
},
{
name: 'Link',
key: 'link',
render: ({ name: resName, parentName }) => (
<Link href={`${parentName}/r/${resName}`}>
<a className="underline">Preview</a>
</Link>
),
},
];
const Resources: React.FC<{ variables: any }> = ({ variables }) => {
const { loading, error, data } = useQuery(GET_RESOURCES_QUERY, {
variables,
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading dataset." />;
if (loading) return <div>Loading</div>;
const { result } = data.dataset;
return (
<>
<h3 className="text-xl font-semibold">Data Files</h3>
<Table
columns={columns}
data={result.resources.map((resource) => ({
...resource,
parentName: result.name,
}))}
/>
</>
);
};
export default Resources;

View File

@ -0,0 +1,65 @@
/* eslint-disable jsx-a11y/anchor-is-valid */
import Link from 'next/link';
import { useState } from 'react';
const Nav: React.FC = () => {
const [open, setOpen] = useState(false);
const handleClick = (event) => {
event.preventDefault();
setOpen(!open);
};
return (
<nav className="flex items-center justify-between flex-wrap bg-white p-4 border-b border-gray-200">
<div className="flex items-center flex-shrink-0 text-gray-700 mr-6">
<img src="/images/logo.svg" alt="portal logo" width="40" />
</div>
<div className="block lg:hidden mx-4">
<button
onClick={handleClick}
className="flex items-center px-3 py-2 border rounded text-gray-700 border-orange-400 hover:text-black hover:border-black"
>
<svg
className="fill-current h-3 w-3"
viewBox="0 0 20 20"
xmlns="http://www.w3.org/2000/svg"
>
<title>Menu</title>
<path d="M0 3h20v2H0V3zm0 6h20v2H0V9zm0 6h20v2H0v-2z" />
</svg>
</button>
</div>
<div className={`${open ? `block` : `hidden`} lg:block`}>
<Link href="/blog">
<a className="block mt-4 lg:inline-block lg:mt-0 active:bg-primary-background text-gray-700 hover:text-black mr-6">
Blog
</a>
</Link>
<Link href="/search">
<a className="block mt-4 lg:inline-block lg:mt-0 text-gray-700 hover:text-black mr-6">
Search
</a>
</Link>
<a
href="http://tech.datopian.com/frontend/"
className="block mt-4 lg:inline-block lg:mt-0 text-gray-700 hover:text-black mr-6"
target="_blank"
rel="noreferrer"
>
Docs
</a>
<a
href="https://github.com/datopian/portal"
className="inline-block text-tiny px-4 py-2 leading-none border rounded text-primary bg-primary-background border-black hover:border-gray-700 hover:text-gray-700 hover:bg-white mt-4 lg:mt-0"
target="_blank"
rel="noreferrer"
>
GitHub
</a>
</div>
</nav>
);
};
export default Nav;

View File

@ -0,0 +1,53 @@
/* eslint-disable jsx-a11y/anchor-is-valid */
import Link from 'next/link';
import { useQuery } from '@apollo/react-hooks';
import { ErrorMessage } from '../_shared';
import { SEARCH_QUERY } from '../../graphql/queries';
const Recent: React.FC = () => {
const { loading, error, data } = useQuery(SEARCH_QUERY, {
variables: {
sort: 'metadata_created desc',
rows: 3,
},
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading search results." />;
if (loading) return <div>Loading</div>;
const { result } = data.search;
return (
<section className="my-10 mx-4 lg:my-20">
<h1 className="text-2xl font-thin mb-4">Recent Datasets</h1>
<div className="recent flex flex-col lg:flex-row">
{result.results.map((dataset, index) => (
<div
key={index}
className="border px-4 mb-4 mr-3 border-gray-100 w-5/6 shadow-sm"
>
<h1 className="text-2xl font-thin">{dataset.title}</h1>
<p className="text-gray-500">
{dataset.organization && dataset.organization.description}
</p>
<Link
href={`/@${
dataset.organization ? dataset.organization.name : 'dataset'
}/${dataset.name}`}
>
<a className="pt-3 flex justify-end text-orange-500">
View Dataset
</a>
</Link>
</div>
))}
</div>
</section>
);
};
export default Recent;

View File

@ -0,0 +1,69 @@
/* eslint-disable react/display-name */
import { useQuery } from '@apollo/react-hooks';
import { Table, ErrorMessage } from '../_shared';
import { GET_RESOURCES_QUERY } from '../../graphql/queries';
const columns = [
{
name: 'Name',
key: 'name',
render: ({ name, id }) => name || id,
},
{
name: 'Title',
key: 'title',
},
{
name: 'Description',
key: 'description',
},
{
name: 'Format',
key: 'format',
},
{
name: 'Size',
key: 'size',
},
{
name: 'Created',
key: 'created',
},
{
name: 'Updated',
key: 'last_modified',
},
{
name: 'Download',
key: 'download',
render: ({ url, format }) => (
<a
href={url}
className="bg-white hover:bg-gray-200 border text-black font-semibold py-2 px-4 rounded"
>
{format}
</a>
),
},
];
const About: React.FC<{ variables: any }> = ({ variables }) => {
const { loading, error, data } = useQuery(GET_RESOURCES_QUERY, {
variables,
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading dataset." />;
if (loading) return <div>Loading</div>;
const { result } = data.dataset;
const resource = result.resources.find(
(item) => item.name === variables.resource
);
return <Table columns={columns} data={[resource]} />;
};
export default About;

View File

@ -0,0 +1,25 @@
import { useQuery } from '@apollo/react-hooks';
import { ErrorMessage } from '../_shared';
import { GET_RESOURCES_QUERY } from '../../graphql/queries';
const DataExplorer: React.FC<{ variables: any }> = ({ variables }) => {
const { loading, error, data } = useQuery(GET_RESOURCES_QUERY, {
variables,
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading dataset." />;
if (loading) return <div>Loading</div>;
const { result } = data.dataset;
const resource = result.resources.find(
(item) => item.name === variables.resource
);
return <>{JSON.stringify(resource)}</>;
};
export default DataExplorer;

View File

@ -0,0 +1,65 @@
import { useState } from 'react';
import { useRouter } from 'next/router';
const Form: React.FC = () => {
const router = useRouter();
const [q, setQ] = useState(router.query.q);
const [sort, setSort] = useState(router.query.sort);
const handleChange = (event) => {
if (event.target.name === 'q') {
setQ(event.target.value);
} else if (event.target.name === 'sort') {
setSort(event.target.value);
}
};
const handleSubmit = (event) => {
event.preventDefault();
router.push({
pathname: '/search',
query: { q, sort },
});
};
return (
<form onSubmit={handleSubmit} className="items-center">
<div className="flex">
<input
type="text"
name="q"
value={q}
onChange={handleChange}
placeholder="Search"
aria-label="Search"
className="bg-white focus:outline-none focus:shadow-outline border border-gray-300 w-1/2 rounded-lg py-2 px-4 block appearance-none leading-normal"
/>
<button
onClick={handleSubmit}
className="inline-block text-sm px-4 py-3 mx-3 leading-none border rounded text-white bg-black border-black lg:mt-0"
>
Search
</button>
</div>
<div className="inline-block my-6 float-right">
<label htmlFor="field-order-by">Order by:</label>
<select
className="bg-white"
id="field-order-by"
name="sort"
onChange={handleChange}
onBlur={handleChange}
value={sort}
>
<option value="score:desc">Relevance</option>
<option value="title_string:asc">Name Ascending</option>
<option value="title_string:desc">Name Descending</option>
<option value="metadata_modified:desc">Last Modified</option>
<option value="views_recent:desc">Popular</option>
</select>
</div>
</form>
);
};
export default Form;

View File

@ -0,0 +1,38 @@
/* eslint-disable jsx-a11y/anchor-is-valid */
import Link from 'next/link';
const Item: React.FC<{ datapackage: any }> = ({ datapackage }) => {
return (
<div className="mb-6">
<h3 className="text-xl font-semibold">
<Link
href={`/@${
datapackage.organization
? datapackage.organization.name
: 'dataset'
}/${datapackage.name}`}
>
<a className="text-primary">
{datapackage.title || datapackage.name}
</a>
</Link>
</h3>
<Link
href={`/@${
datapackage.organization ? datapackage.organization.name : 'dataset'
}`}
>
<a className="text-gray-500 block mt-1">
{datapackage.organization
? datapackage.organization.title
: 'dataset'}
</a>
</Link>
<div className="leading-relaxed mt-2">
{datapackage.description || datapackage.notes}
</div>
</div>
);
};
export default Item;

View File

@ -0,0 +1,27 @@
import { useQuery } from '@apollo/react-hooks';
import Item from './Item';
import { ErrorMessage } from '../_shared';
import { SEARCH_QUERY } from '../../graphql/queries';
const List: React.FC<{ variables: any }> = ({ variables }) => {
const { loading, error, data } = useQuery(SEARCH_QUERY, {
variables,
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading search results." />;
if (loading) return <div>Loading</div>;
const { result } = data.search;
return (
<ul>
{result.results.map((pkg, index) => (
<Item datapackage={pkg} key={index} />
))}
</ul>
);
};
export default List;

View File

@ -0,0 +1,26 @@
import { useQuery } from '@apollo/react-hooks';
import { ErrorMessage } from '../_shared';
import { GET_TOTAL_COUNT_QUERY } from '../../graphql/queries';
const Total: React.FC<{ variables: any }> = ({ variables }) => {
const { loading, error, data } = useQuery(GET_TOTAL_COUNT_QUERY, {
variables,
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading search results." />;
if (loading) return <div>Loading</div>;
const { result } = data.search;
return (
<h1 className="text-3xl font-semibold text-primary my-6 inline-block">
{result.count} results found
</h1>
);
};
export default Total;

View File

@ -0,0 +1,39 @@
import parse from 'html-react-parser';
import { useQuery } from '@apollo/react-hooks';
import { ErrorMessage } from '../_shared';
import { GET_POSTS_QUERY } from '../../graphql/queries';
const List: React.FC = () => {
const { loading, error, data } = useQuery(GET_POSTS_QUERY, {
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading search results." />;
if (loading) return <div>Loading</div>;
const { posts, found } = data.posts;
return (
<>
<h1 className="text-3xl font-semibold text-primary my-6 inline-block">
{found} posts found
</h1>
{posts.map((post, index) => (
<div key={index}>
<a
href={`/blog/${post.slug}`}
className="text-2xl font-semibold text-primary my-6 inline-block"
>
{parse(post.title)}
</a>
<p>{parse(post.excerpt)}</p>
</div>
))}
</>
);
};
export default List;

View File

@ -0,0 +1,32 @@
import parse from 'html-react-parser';
import { useQuery } from '@apollo/react-hooks';
import { ErrorMessage } from '../_shared';
import { GET_PAGE_QUERY } from '../../graphql/queries';
const Page: React.FC<{ variables: any }> = ({ variables }) => {
const { loading, error, data } = useQuery(GET_PAGE_QUERY, {
variables,
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading search results." />;
if (loading) return <div>Loading</div>;
const { title, content, modified, featured_image } = data.page;
return (
<>
<h1 className="text-3xl font-semibold text-primary my-6 inline-block">
{title}
</h1>
<p className="mb-6">Edited: {modified}</p>
<img src={featured_image} className="mb-6" alt="featured_img" />
<div>{parse(content)}</div>
</>
);
};
export default Page;

View File

@ -0,0 +1,32 @@
import parse from 'html-react-parser';
import { useQuery } from '@apollo/react-hooks';
import { ErrorMessage } from '../_shared';
import { GET_PAGE_QUERY } from '../../graphql/queries';
const Post: React.FC<{ variables: any }> = ({ variables }) => {
const { loading, error, data } = useQuery(GET_PAGE_QUERY, {
variables,
// Setting this value to true will make the component rerender when
// the "networkStatus" changes, so we are able to know if it is fetching
// more data
notifyOnNetworkStatusChange: true,
});
if (error) return <ErrorMessage message="Error loading search results." />;
if (loading) return <div>Loading</div>;
const { title, content, modified, featured_image } = data.page;
return (
<>
<h1 className="text-3xl font-semibold text-primary my-6 inline-block">
{title}
</h1>
<p className="mb-6">Edited: {modified}</p>
<img src={featured_image} className="mb-6" alt="featured_img" />
<div>{parse(content)}</div>
</>
);
};
export default Post;

View File

@ -0,0 +1,8 @@
module.exports = {
process() {
return 'module.exports = {};';
},
getCacheKey() {
return 'cssTransform';
},
};

View File

@ -0,0 +1,3 @@
{
"baseUrl": "http://localhost:3000"
}

View File

@ -0,0 +1,5 @@
{
"name": "Using fixtures to represent data",
"email": "hello@cypress.io",
"body": "Fixtures are a great way to mock data for responses to routes"
}

View File

@ -0,0 +1,32 @@
describe('Test Home Page', () => {
beforeEach(() => {
cy.visit('/');
});
it('renders the hero title', () => {
cy.contains('Find, Share and Publish Quality Data with Datahub');
});
it('checks that a search form exists', () => {
cy.get('form').contains('Search');
});
// it('submits the search form', () => {
// cy.get('form').find('[type="text"]').type('gdp');
// cy.get('form').submit();
// cy.url().should('include', '/search?q=gdp&sort=');
// cy.get('.text-3xl').and('contain.text', '1 results found');
// });
it('shows the recent datasets', () => {
cy.contains('Recent Datasets');
});
it('returns the expected number of recent datasets', () => {
cy.get('.recent')
.find('div')
.should(($div) => {
expect($div).to.have.length.of.at.least(2);
});
});
});

View File

@ -0,0 +1,21 @@
describe('Test Search Page', () => {
beforeEach(() => {
cy.visit('/search');
});
it('has a search form', () => {
cy.contains('form');
cy.contains('Search');
});
// it('should return a search result', () => {
// cy.get('form').find('[type="text"]').type('gdp');
// cy.get('form').submit();
// cy.url().should('include', 'search?q=gdp&sort=');
// cy.get('.text-3xl').should('have.text', '1 results found');
// cy.get('.text-xl > .text-primary').should(
// 'have.text',
// 'Country, Regional and World GDP (Gross Domestic Product)'
// );
// });
});

View File

@ -0,0 +1,21 @@
/// <reference types="cypress" />
// ***********************************************************
// This example plugins/index.js can be used to load plugins
//
// You can change the location of this file or turn off loading
// the plugins file with the 'pluginsFile' configuration option.
//
// You can read more here:
// https://on.cypress.io/plugins-guide
// ***********************************************************
// This function is called when a project is opened or re-opened (e.g. due to
// the project's config changing)
/**
* @type {Cypress.PluginConfig}
*/
module.exports = (on, config) => {
// `on` is used to hook into various events Cypress emits
// `config` is the resolved Cypress config
};

View File

@ -0,0 +1,25 @@
// ***********************************************
// This example commands.js shows you how to
// create various custom commands and overwrite
// existing commands.
//
// For more comprehensive examples of custom
// commands please read more here:
// https://on.cypress.io/custom-commands
// ***********************************************
//
//
// -- This is a parent command --
// Cypress.Commands.add("login", (email, password) => { ... })
//
//
// -- This is a child command --
// Cypress.Commands.add("drag", { prevSubject: 'element'}, (subject, options) => { ... })
//
//
// -- This is a dual command --
// Cypress.Commands.add("dismiss", { prevSubject: 'optional'}, (subject, options) => { ... })
//
//
// -- This will overwrite an existing command --
// Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... })

View File

@ -0,0 +1,20 @@
// ***********************************************************
// This example support/index.js is processed and
// loaded automatically before your test files.
//
// This is a great place to put global configuration and
// behavior that modifies Cypress.
//
// You can change the location of this file or turn off
// automatically serving support files with the
// 'supportFile' configuration option.
//
// You can read more here:
// https://on.cypress.io/configuration
// ***********************************************************
// Import commands.js using ES2015 syntax:
import './commands';
// Alternatively you can use CommonJS syntax:
// require('./commands')

View File

@ -0,0 +1,12 @@
{
"compilerOptions": {
"strict": true,
"baseUrl": "../node_modules",
"target": "es5",
"lib": ["es5", "dom"],
"types": ["cypress"]
},
"include": [
"**/*.ts"
]
}

View File

@ -0,0 +1,145 @@
import gql from 'graphql-tag';
export const GET_ORG_QUERY = gql`
query dataset($id: String) {
dataset(id: $id) @rest(type: "Response", path: "package_show?{args}") {
result {
organization {
name
title
image_url
}
}
}
}
`;
export const GET_DATAPACKAGE_QUERY = gql`
query dataset($id: String) {
dataset(id: $id) @rest(type: "Response", path: "package_show?{args}") {
result {
name
title
size
metadata_created
metadata_modified
resources {
name
}
}
}
}
`;
export const GET_RESOURCES_QUERY = gql`
query dataset($id: String) {
dataset(id: $id) @rest(type: "Response", path: "package_show?{args}") {
result {
name
resources {
name
title
format
created
last_modified
}
}
}
}
`;
export const SEARCH_QUERY = gql`
query search($q: String, $sort: String, $rows: Int) {
search(q: $q, sort: $sort, rows: $rows)
@rest(type: "Search", path: "package_search?{args}") {
result {
count
results {
name
title
organization {
name
title
description
}
}
}
}
}
`;
export const GET_TOTAL_COUNT_QUERY = gql`
query search($q: String, $sort: String) {
search(q: $q, sort: $sort)
@rest(type: "Search", path: "package_search?{args}") {
result {
count
}
}
}
`;
export const GET_POSTS_QUERY = gql`
query posts {
posts @rest(type: "Posts", path: "", endpoint: "wordpress-posts") {
found
posts
meta
}
}
`;
export const GET_PAGE_QUERY = gql`
query page($slug: String) {
page(slug: $slug)
@rest(type: "Page", path: "{args.slug}", endpoint: "wordpress") {
title
content
excerpt
slug
date
modified
featured_image
}
}
`;
export const GET_DATASET_QUERY = gql`
query dataset($id: String) {
dataset(id: $id) @rest(type: "Response", path: "package_show?{args}") {
result {
name
title
size
metadata_created
metadata_modified
resources {
name
title
format
created
last_modified
}
organization {
name
title
image_url
}
}
}
}
`;
export const GET_POST_QUERY = gql`
query post($slug: String) {
post(slug: $slug)
@rest(type: "Post", path: "{args.slug}", endpoint: "wordpress") {
title
content
excerpt
slug
date
modified
}
}
`;

View File

@ -0,0 +1,29 @@
module.exports = {
collectCoverageFrom: [
'**/*.{js,jsx,ts,tsx}',
'!**/*.d.ts',
'!**/node_modules/**',
'!**/config/**',
'!**/coverage/**',
'!**/**.config.js**',
],
setupFilesAfterEnv: ['<rootDir>/setupTests.js'],
testPathIgnorePatterns: [
'/node_modules/',
'/.next/',
'/jest.config.js/',
'/tailwind.config.js/',
'<rootDir>/postcss.config.js',
],
transform: {
'^.+\\.(js|jsx|ts|tsx)$': 'babel-jest',
'^.+\\.css$': '<rootDir>/config/jest/cssTransform.js',
},
transformIgnorePatterns: [
'/node_modules/',
'^.+\\.module\\.(css|sass|scss)$',
],
moduleNameMapper: {
'^.+\\.module\\.(css|sass|scss)$': 'identity-obj-proxy',
},
};

View File

@ -0,0 +1,92 @@
import { useMemo } from 'react';
import getConfig from 'next/config';
import { ApolloClient } from 'apollo-client';
import {
InMemoryCache,
NormalizedCache,
NormalizedCacheObject,
} from 'apollo-cache-inmemory';
import { RestLink } from 'apollo-link-rest';
let apolloClient:
| ApolloClient<NormalizedCache>
| ApolloClient<NormalizedCacheObject>;
const restLink = new RestLink({
uri: getConfig().publicRuntimeConfig.DMS + '/api/3/action/',
endpoints: {
wordpress: `https://public-api.wordpress.com/rest/v1.1/sites/${
getConfig().publicRuntimeConfig.CMS
}/posts/slug:`,
'wordpress-posts': `https://public-api.wordpress.com/rest/v1.1/sites/${
getConfig().publicRuntimeConfig.CMS
}/posts/`,
},
typePatcher: {
Search: (data: any): any => {
if (data.result != null) {
data.result.__typename = 'SearchResponse';
if (data.result.results != null) {
data.result.results = data.result.results.map((item) => {
if (item.organization != null) {
item.organization.__typename = 'Organization';
}
return { __typename: 'Package', ...item };
});
}
}
return data;
},
Response: (data: any): any => {
if (data.result != null) {
data.result.__typename = 'Package';
if (data.result.organization != null) {
data.result.organization.__typename = 'Organization';
}
if (data.result.resources != null) {
data.result.resources = data.result.resources.map((item) => {
return { __typename: 'Resource', ...item };
});
}
}
return data;
},
},
});
function createApolloClient() {
return new ApolloClient({
link: restLink,
cache: new InMemoryCache(),
});
}
export function initializeApollo(
initialState = null
): ApolloClient<NormalizedCache> | ApolloClient<NormalizedCacheObject> {
const _apolloClient:
| ApolloClient<NormalizedCache>
| ApolloClient<NormalizedCacheObject> =
apolloClient ?? createApolloClient();
// If your page has Next.js data fetching methods that use Apollo Client, the initial state
// gets hydrated here
if (initialState) {
_apolloClient.cache.restore(initialState);
}
// For SSG and SSR always create a new Apollo Client
if (typeof window === 'undefined') return _apolloClient;
// Create the Apollo Client once in the client
if (!apolloClient) apolloClient = _apolloClient;
return _apolloClient;
}
export function useApollo(
initialState = null
): ApolloClient<NormalizedCache> | ApolloClient<NormalizedCacheObject> {
const store = useMemo(() => initializeApollo(initialState), [initialState]);
return store;
}

View File

@ -0,0 +1,4 @@
{
"title": "Portal in English",
"description": "At Datahub, we have over thousands of datasets for free and a Premium Data Service for additional or customised data with guaranteed updates."
}

View File

@ -0,0 +1,4 @@
{
"title": "Portal in french",
"description": "Chez Datahub, nous avons plus de milliers d'ensembles de données gratuitement et un service de données Premium pour des données supplémentaires ou personnalisées avec des mises à jour garanties."
}

View File

@ -0,0 +1,155 @@
const nock = require('nock');
const gdp = {
name: 'gdp',
title: 'Country, Regional and World GDP (Gross Domestic Product)',
notes:
'Country, regional and world GDP in current US Dollars ($). Regional means collections of countries e.g. Europe & Central Asia. Data is sourced from the World Bank and turned into a standard normalized CSV.',
resources: [
{
name: 'gdp',
id: 'gdp',
title: 'GDP data',
format: 'csv',
created: '2019-03-07T12:00:36.273495',
last_modified: '2020-05-07T12:00:36.273495',
datastore_active: false,
url: 'http://mock.filestore/gdp.csv',
},
],
organization: {
title: 'World Bank',
name: 'world-bank',
description:
'The World Bank is an international financial institution that provides loans and grants to the governments of poorer countries for the purpose of pursuing capital projects.',
created: '2019-03-07T11:51:13.758844',
image_url:
'https://github.com/datahq/frontend/raw/master/public/img/avatars/world-bank.jpg',
},
metadata_created: '2019-03-07T11:56:19.696257',
metadata_modified: '2019-03-07T12:03:58.817280',
size: '',
};
const population = {
name: 'population',
title: 'World population data',
notes:
'Population figures for countries, regions (e.g. Asia) and the world. Data comes originally from World Bank and has been converted into standard CSV.',
resources: [
{
name: 'population',
id: 'population',
title: 'Population data',
format: 'csv',
created: '2019-03-07T12:00:36.273495',
last_modified: '2020-05-07T12:00:36.273495',
datastore_active: true,
url: 'http://mock.filestore/population.csv',
},
],
organization: {
title: 'World Bank',
name: 'world-bank',
description:
'The World Bank is an international financial institution that provides loans and grants to the governments of poorer countries for the purpose of pursuing capital projects.',
created: '2019-03-07T11:51:13.758844',
image_url:
'https://github.com/datahq/frontend/raw/master/public/img/avatars/world-bank.jpg',
},
};
module.exports.initMocks = function () {
// Uncomment this line if you want to record API calls
// nock.recorder.rec()
// "package_search" mocks
nock('http://mock.ckan/api/3/action', { encodedQueryParams: true })
.persist()
// 1. Call without query.
.get('/package_search?')
.reply(200, {
success: true,
result: {
count: 2,
sort: 'score desc, metadata_modified desc',
facets: {},
results: [gdp, population],
search_facets: {},
},
})
// 2. Call with `q=gdp` query.
.get('/package_search?q=gdp')
.reply(200, {
success: true,
result: {
count: 1,
sort: 'score desc, metadata_modified desc',
facets: {},
results: [gdp],
search_facets: {},
},
})
// 3. Call for recent packages.
.get('/package_search?sort=metadata_created%20desc&rows=3')
.reply(200, {
success: true,
result: {
count: 2,
sort: 'metadata_created desc',
facets: {},
results: [gdp, population],
search_facets: {},
},
});
// "package_show" mocks
nock('http://mock.ckan/api/3/action', { encodedQueryParams: true })
.persist()
.get('/package_show?id=gdp')
.reply(200, {
success: true,
result: gdp,
})
.get('/package_show?id=population')
.reply(200, {
success: true,
result: population,
});
// "datastore_search" mocks
nock('http://mock.ckan/api/3/action', { encodedQueryParams: true })
.persist()
.get('/datastore_search?resource_id=population')
.reply(200, {
success: true,
result: {
records: [
{
'Country Code': 'ARB',
'Country Name': 'Arab World',
Value: 92197753,
Year: 1960,
},
{
'Country Code': 'ARB',
'Country Name': 'Arab World',
Value: 94724510,
Year: 1961,
},
{
'Country Code': 'ARB',
'Country Name': 'Arab World',
Value: 97334442,
Year: 1962,
},
],
},
});
// Filestore mocks
nock('http://mock.filestore', { encodedQueryParams: true })
.persist()
.get('/gdp.csv')
.reply(200, 'a,b,c\n1,2,3\n4,5,6\n');
};

2
examples/catalog/next-env.d.ts vendored Normal file
View File

@ -0,0 +1,2 @@
/// <reference types="next" />
/// <reference types="next/types/global" />

View File

@ -0,0 +1,48 @@
const { PHASE_DEVELOPMENT_SERVER } = require('next/constants');
module.exports = (phase, { defaultConfig }) => {
const dms = process.env.DMS;
const cms = process.env.CMS;
if (phase === PHASE_DEVELOPMENT_SERVER) {
if (dms) {
console.log('\nYou are running the app in dev mode 🌀');
console.log(
'Did you know that you can use mocked CKAN API? Just unset your `DMS` env var.'
);
console.log('Happy coding ☀️\n');
} else {
const mocks = require('./mocks');
mocks.initMocks();
console.log(
'\nYou have not defined any DMS API so we are activating the mocks ⚠️'
);
console.log(
'If you wish to run against your CKAN API, you can set `DMS` env var.'
);
console.log(
'For example, to run against demo ckan site: `DMS=https://demo.ckan.org`\n'
);
}
return {
i18n: {
locales: ['en', 'fr', 'nl-NL'],
defaultLocale: 'en',
},
publicRuntimeConfig: {
DMS: dms ? dms.replace(/\/?$/, '') : 'http://mock.ckan',
CMS: cms ? cms.replace(/\/?$/, '') : 'oddk.home.blog',
},
};
}
return {
i18n: {
locales: ['en', 'fr', 'nl-NL'],
defaultLocale: 'en',
},
publicRuntimeConfig: {
DMS: dms ? dms.replace(/\/?$/, '') : 'https://demo.ckan.org',
CMS: cms ? cms.replace(/\/?$/, '') : 'oddk.home.blog',
},
};
};

22523
examples/catalog/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,88 @@
{
"name": "portal-main",
"version": "0.1.0",
"private": true,
"homepage": "https://github.com/datopian/portal#readme",
"directories": {
"test": "__tests__"
},
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"test": "jest",
"test:watch": "jest --watch",
"test:coverage": "jest --coverage",
"cypress:open": "cypress open",
"cypress:ci": "cypress run --config video=false",
"e2e": "cypress run",
"format": "prettier --single-quote --write .",
"pre-commit": "yarn lint:fix && prettier --single-quote --write",
"lint": "eslint . --ext .ts,.tsx",
"lint:fix": "yarn lint --fix"
},
"dependencies": {
"@apollo/client": "^3.0.2",
"@apollo/react-hooks": "^3.1.5",
"@fullhuman/postcss-purgecss": "^2.3.0",
"apollo-cache-inmemory": "^1.6.6",
"apollo-client": "^2.6.10",
"apollo-link": "^1.2.14",
"apollo-link-rest": "0.7.3",
"apollo-server-testing": "^2.16.0",
"bytes": "^3.1.0",
"enzyme": "^3.11.0",
"enzyme-adapter-react-15": "^1.4.1",
"graphql": "^15.1.0",
"graphql-anywhere": "^4.2.7",
"graphql-tag": "^2.10.3",
"html-react-parser": "^0.13.0",
"markdown-it": "^11.0.0",
"next": "^10.0.3",
"next-translate": "^0.20.2",
"qs": "^6.9.4",
"react": "^16.13.1",
"react-dom": "^16.13.1",
"react-router": "^5.2.0",
"slugify": "^1.4.0"
},
"devDependencies": {
"@testing-library/jest-dom": "^5.8.0",
"@testing-library/react": "^10.0.4",
"@types/jest": "^25.2.3",
"@types/react": "^16.9.35",
"@typescript-eslint/eslint-plugin": "^3.8.0",
"@typescript-eslint/parser": "^3.8.0",
"autoprefixer": "^9.8.6",
"babel-jest": "^26.0.1",
"babel-plugin-graphql-tag": "^2.5.0",
"cypress": "^6.6.0",
"eslint": "^7.6.0",
"eslint-config-prettier": "^6.11.0",
"eslint-plugin-jsx-a11y": "^6.3.1",
"eslint-plugin-prettier": "^3.1.4",
"eslint-plugin-react": "^7.20.5",
"eslint-plugin-react-hooks": "^4.0.8",
"husky": ">=4",
"jest": "^26.1.0",
"lerna": "^3.22.1",
"lint-staged": ">=10",
"nock": "^12.0.3",
"npm-run-all": "^4.1.5",
"postcss-cli": "^7.1.1",
"postcss-import": "^12.0.1",
"postcss-preset-env": "6.7.0",
"prettier": "^2.0.5",
"react-test-renderer": "^16.13.1",
"tailwindcss": "^1.4.6",
"typescript": "^3.9.3"
},
"husky": {
"hooks": {
"pre-commit": "lint-staged"
}
},
"lint-staged": {
"*.{ts,tsx,js,jsx,css,html,md}": "yarn pre-commit"
}
}

View File

@ -0,0 +1,55 @@
import { GetServerSideProps } from 'next';
import { useQuery } from '@apollo/react-hooks';
import Head from 'next/head';
import { initializeApollo } from '../../../lib/apolloClient';
import Nav from '../../../components/home/Nav';
import About from '../../../components/dataset/About';
import Org from '../../../components/dataset/Org';
import Resources from '../../../components/dataset/Resources';
import { GET_DATASET_QUERY } from '../../../graphql/queries';
const Dataset: React.FC<{ variables: any }> = ({ variables }) => {
const { data, loading } = useQuery(GET_DATASET_QUERY, { variables });
if (loading) return <div>Loading</div>;
const { result } = data.dataset;
return (
<>
<Head>
<title>Portal | {result.title || result.name}</title>
<link rel="icon" href="/favicon.ico" />
</Head>
<Nav />
<main className="p-6">
<h1 className="text-3xl font-semibold text-primary mb-2">
{result.title || result.name}
</h1>
<Org variables={variables} />
<About variables={variables} />
<Resources variables={variables} />
</main>
</>
);
};
export const getServerSideProps: GetServerSideProps = async (context) => {
const apolloClient = initializeApollo();
const variables = {
id: context.query.dataset,
};
await apolloClient.query({
query: GET_DATASET_QUERY,
variables,
});
return {
props: {
initialApolloState: apolloClient.cache.extract(),
variables,
},
};
};
export default Dataset;

View File

@ -0,0 +1,57 @@
import { GetServerSideProps } from 'next';
import { useQuery } from '@apollo/react-hooks';
import Head from 'next/head';
import { initializeApollo } from '../../../../../lib/apolloClient';
import Nav from '../../../../../components/home/Nav';
import About from '../../../../../components/resource/About';
import DataExplorer from '../../../../../components/resource/DataExplorer';
import { GET_RESOURCES_QUERY } from '../../../../../graphql/queries';
const Resource: React.FC<{ variables: any }> = ({ variables }) => {
const { data, loading } = useQuery(GET_RESOURCES_QUERY, { variables });
if (loading) return <div>Loading</div>;
const result = data.dataset.result;
// Find right resource
const resource = result.resources.find(
(item) => item.name === variables.resource
);
return (
<>
<Head>
<title>Portal | {resource.title || resource.name}</title>
<link rel="icon" href="/favicon.ico" />
</Head>
<Nav />
<main className="p-6">
<h1 className="text-3xl font-semibold text-primary mb-2">
{resource.title || resource.name}
</h1>
<About variables={variables} />
<DataExplorer variables={variables} />
</main>
</>
);
};
export const getServerSideProps: GetServerSideProps = async (context) => {
const apolloClient = initializeApollo();
const variables = {
id: context.query.dataset,
resource: context.query.resource,
};
await apolloClient.query({
query: GET_RESOURCES_QUERY,
variables,
});
return {
props: {
initialApolloState: apolloClient.cache.extract(),
variables,
},
};
};
export default Resource;

View File

@ -0,0 +1,55 @@
import { useEffect, useState } from 'react';
import { ApolloProvider } from '@apollo/react-hooks';
import { useApollo } from '../lib/apolloClient';
import { DEFAULT_THEME } from '../themes';
import { applyTheme } from '../themes/utils';
import I18nProvider from 'next-translate/I18nProvider';
import { useRouter } from 'next/router';
import '../styles/app.css';
interface I8nObject {
[property: string]: any;
}
export async function loadNamespaces(
namespaces: string[],
lang: string
): Promise<I8nObject> {
const res = {};
for (const ns of namespaces) {
res[ns] = await import(`../locales/${lang}/${ns}.json`).then(
(m) => m.default
);
}
return res;
}
type Props = {
Component: any;
pageProps: any;
};
const MyApp: React.FC<Props> = ({ Component, pageProps }) => {
const apolloClient = useApollo(pageProps.initialApolloState);
const [theme] = useState(DEFAULT_THEME); // setTheme
const router = useRouter();
useEffect(() => {
/**
* We can switch theme.
* e.g. setTheme('primary');
* */
applyTheme(theme);
}, [theme]);
return (
<I18nProvider lang={router.locale} namespaces={pageProps._ns}>
<ApolloProvider client={apolloClient}>
<Component {...pageProps} />
</ApolloProvider>
</I18nProvider>
);
};
export default MyApp;

View File

@ -0,0 +1,34 @@
import Document, { Html, Head, Main, NextScript } from 'next/document';
const GA_TRACKING_ID = 'G-NX72GYFHFS';
export default class CustomDocument extends Document {
render() {
return (
<Html>
<Head>
<script
async
src="https://www.googletagmanager.com/gtag/js?id=G-NX72GYFHFS"
/>
<script
dangerouslySetInnerHTML={{
__html: `
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', '${GA_TRACKING_ID}',{
page_path: window.location.pathname,
});
`,
}}
/>
</Head>
<body>
<Main />
</body>
<NextScript />
</Html>
);
}
}

View File

@ -0,0 +1,45 @@
import { GetServerSideProps } from 'next';
import Head from 'next/head';
import { initializeApollo } from '../../../lib/apolloClient';
import Nav from '../../../components/home/Nav';
import Post from '../../../components/static/Post';
import { GET_POST_QUERY } from '../../../graphql/queries';
type Props = {
variables: any;
};
const PostItem: React.FC<Props> = ({ variables }) => (
<>
<Head>
<title>Portal | {variables.slug}</title>
<link rel="icon" href="/favicon.ico" />
</Head>
<Nav />
<main className="p-6">
<Post variables={variables} />
</main>
</>
);
export const getServerSideProps: GetServerSideProps = async (context) => {
const variables = {
slug: context.query.post,
};
const apolloClient = initializeApollo();
await apolloClient.query({
query: GET_POST_QUERY,
variables,
});
return {
props: {
initialApolloState: apolloClient.cache.extract(),
variables,
},
};
};
export default PostItem;

View File

@ -0,0 +1,35 @@
import { GetServerSideProps } from 'next';
import Head from 'next/head';
import { initializeApollo } from '../../lib/apolloClient';
import Nav from '../../components/home/Nav';
import List from '../../components/static/List';
import { GET_POSTS_QUERY } from '../../graphql/queries';
const PostList: React.FC = () => (
<>
<Head>
<title>Portal | Blog</title>
<link rel="icon" href="/favicon.ico" />
</Head>
<Nav />
<main className="p-6">
<List />
</main>
</>
);
export const getServerSideProps: GetServerSideProps = async () => {
const apolloClient = initializeApollo();
await apolloClient.query({
query: GET_POSTS_QUERY,
});
return {
props: {
initialApolloState: apolloClient.cache.extract(),
},
};
};
export default PostList;

View File

@ -0,0 +1,70 @@
import { GetServerSideProps } from 'next';
import Head from 'next/head';
import { initializeApollo } from '../lib/apolloClient';
import Nav from '../components/home/Nav';
import Recent from '../components/home/Recent';
import Form from '../components/search/Form';
import { SEARCH_QUERY } from '../graphql/queries';
import { loadNamespaces } from './_app';
import useTranslation from 'next-translate/useTranslation';
const Home: React.FC<{ locale: any; locales: any }> = ({
locale,
locales,
}) => {
const { t } = useTranslation();
return (
<>
<div className="container mx-auto">
<Head>
<title>{t(`common:title`)}</title>
<link rel="icon" href="/favicon.ico" />
</Head>
<Nav />
<section className="flex justify-center items-center flex-col mt-8 mx-4 lg:flex-row">
<div>
<h1 className="text-4xl mb-3 font-thin">
Find, Share and Publish <br /> Quality Data with{' '}
<span className="text-orange-500">Datahub</span>
</h1>
<p className="text-md font-light mb-3 w-4/5">
{t(`common:description`)}
</p>
<Form />
</div>
<div className="mt-4">
<img src="/images/banner.svg" className="w-4/5" alt="banner_img" />
</div>
</section>
<Recent />
</div>
</>
);
};
export const getServerSideProps: GetServerSideProps = async ({
locale,
locales,
}) => {
const apolloClient = initializeApollo();
await apolloClient.query({
query: SEARCH_QUERY,
variables: {
sort: 'metadata_created desc',
rows: 3,
},
});
return {
props: {
initialApolloState: apolloClient.cache.extract(),
_ns: await loadNamespaces(['common'], locale),
locale,
locales,
},
};
};
export default Home;

View File

@ -0,0 +1,45 @@
import { GetServerSideProps } from 'next';
import Head from 'next/head';
import { initializeApollo } from '../../../lib/apolloClient';
import Nav from '../../../components/home/Nav';
import Page from '../../../components/static/Page';
import { GET_PAGE_QUERY } from '../../../graphql/queries';
type Props = {
variables: any;
};
const PageItem: React.FC<Props> = ({ variables }) => (
<>
<Head>
<title>Portal | {variables.slug}</title>
<link rel="icon" href="/favicon.ico" />
</Head>
<Nav />
<main className="p-6">
<Page variables={variables} />
</main>
</>
);
export const getServerSideProps: GetServerSideProps = async (context) => {
const variables = {
slug: context.query.page,
};
const apolloClient = initializeApollo();
await apolloClient.query({
query: GET_PAGE_QUERY,
variables,
});
return {
props: {
initialApolloState: apolloClient.cache.extract(),
variables,
},
};
};
export default PageItem;

View File

@ -0,0 +1,49 @@
import { GetServerSideProps } from 'next';
import { initializeApollo } from '../lib/apolloClient';
import utils from '../utils';
import Head from 'next/head';
import Nav from '../components/home/Nav';
import Form from '../components/search/Form';
import Total from '../components/search/Total';
import List from '../components/search/List';
import { SEARCH_QUERY } from '../graphql/queries';
type Props = {
variables: any;
};
const Search: React.FC<Props> = ({ variables }) => (
<>
<Head>
<title>Portal | Search</title>
<link rel="icon" href="/favicon.ico" />
</Head>
<Nav />
<main className="p-6">
<Form />
<Total variables={variables} />
<List variables={variables} />
</main>
</>
);
export const getServerSideProps: GetServerSideProps = async (context) => {
const query = context.query || {};
const variables = utils.convertToCkanSearchQuery(query);
const apolloClient = initializeApollo();
await apolloClient.query({
query: SEARCH_QUERY,
variables,
});
return {
props: {
initialApolloState: apolloClient.cache.extract(),
variables,
},
};
};
export default Search;

View File

@ -0,0 +1,17 @@
const purgecss = [
'@fullhuman/postcss-purgecss',
{
content: ['./components/**/*.tsx', './pages/**/*.tsx'],
defaultExtractor: (content) => content.match(/[\w-/:]+(?<!:)/g) || [],
},
];
module.exports = {
plugins: [
'postcss-preset-env',
'postcss-import',
'tailwindcss',
'autoprefixer',
...(process.env.NODE_ENV === 'production' ? [purgecss] : []),
],
};

Binary file not shown.

After

Width:  |  Height:  |  Size: 318 B

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 85 KiB

View File

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64" viewBox="0 0 64 64"><g transform="translate(-395.882 247.118)"><rect width="64" height="64" rx="2" transform="translate(395.882 -247.118)" fill="#030303"/><text transform="translate(408.882 -221.118)" fill="#fff" font-size="20" font-family="AndaleMono, Andale Mono"><tspan x="0" y="0">POR</tspan><tspan x="0" y="22">TAL</tspan></text></g></svg>

After

Width:  |  Height:  |  Size: 408 B

View File

@ -0,0 +1 @@
import '@testing-library/jest-dom/extend-expect';

View File

@ -0,0 +1 @@
@import './tailwind.css';

View File

@ -0,0 +1,3 @@
@import 'tailwindcss/base';
@import 'tailwindcss/components';
@import 'tailwindcss/utilities';

View File

@ -0,0 +1,30 @@
module.exports = {
theme: {
fontSize: {
tiny: 'var(--font-size-small)',
md: 'var(--font-size-medium)',
lg: 'var(--font-size-large)',
},
extend: {
colors: {
primary: 'var(--color-primary)',
secondary: 'var(--color-secondary)',
negative: 'var(--color-negative)',
positive: 'var(--color-positive)',
'primary-background': 'var(--background-primary)',
'sec-background': 'var(--background-sec)',
'primary-text': 'var(--color-text-primary)',
},
},
backgroundColor: (theme) => ({
...theme('colors'),
}),
},
variants: {
backgroundColor: ['active'],
},
plugins: ['font-size'],
corePlugins: {
fontSize: true,
},
};

View File

@ -0,0 +1,13 @@
export default {
primary: '#896A00',
secondary: '#254E70',
black: '#0C0C0C',
positive: '#0C0C0C',
textPrimary: '#896A00',
backgroundPrimary: '#FAEEC5',
// Define font size variables
fontSmall: '18px',
fontMedium: '30px',
fontLarge: '45px',
};

View File

@ -0,0 +1,11 @@
import base from './base';
import { IThemes } from './utils';
/**
* The default theme to load
*/
export const DEFAULT_THEME = 'base';
export const themes: IThemes = {
base,
};

View File

@ -0,0 +1,6 @@
import { extend } from './utils';
import base from './base';
export default extend(base, {
// Custom styles for primary theme
});

View File

@ -0,0 +1,47 @@
import { themes } from './index';
export interface ITheme {
[key: string]: string;
}
export interface IThemes {
[key: string]: ITheme;
}
export interface IMappedTheme {
[key: string]: string | null;
}
export const mapTheme = (variables: ITheme): IMappedTheme => {
return {
'--color-primary': variables.primary || '',
'--color-secondary': variables.secondary || '',
'--color-positive': variables.positive || '',
'--color-negative': variables.negative || '',
'--color-text-primary': variables.textPrimary || '',
'--background-primary': variables.backgroundPrimary || '',
'--background-sec': variables.backgroundSecondary || '',
'--font-size-small': variables.fontSmall || '18px',
'--font-size-medium': variables.fontMedium || '30px',
'--font-size-large': variables.fontLarge || '45px',
};
};
export const applyTheme = (theme: string): void => {
const themeObject: IMappedTheme = mapTheme(themes[theme]);
if (!themeObject) return;
const root = document.documentElement;
Object.keys(themeObject).forEach((property) => {
if (property === 'name') {
return;
}
root.style.setProperty(property, themeObject[property]);
});
};
export const extend = (extending: ITheme, newTheme: ITheme): ITheme => {
return { ...extending, ...newTheme };
};

View File

@ -0,0 +1,26 @@
{
"compilerOptions": {
"target": "es5",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": false,
"forceConsistentCasingInFileNames": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "node",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve"
},
"exclude": ["node_modules"],
"include": [
"next-env.d.ts",
"**/*.ts",
"**/*.tsx",
"setupTests.js",
"jest.config.js",
"config/jest/cssTransform.js"
]
}

View File

@ -0,0 +1,541 @@
const { URL } = require('url');
const bytes = require('bytes');
const slugify = require('slugify');
const config = require('../next.config.js');
module.exports.ckanToDataPackage = function (descriptor) {
// Make a copy
const datapackage = JSON.parse(JSON.stringify(descriptor));
// Lowercase name
datapackage.name = datapackage.name.toLowerCase();
// Rename notes => description
if (datapackage.notes) {
datapackage.description = datapackage.notes;
delete datapackage.notes;
}
// Rename ckan_url => homepage
if (datapackage.ckan_url) {
datapackage.homepage = datapackage.ckan_url;
delete datapackage.ckan_url;
}
// Parse license
const license = {};
if (datapackage.license_id) {
license.type = datapackage.license_id;
delete datapackage.license_id;
}
if (datapackage.license_title) {
license.title = datapackage.license_title;
delete datapackage.license_title;
}
if (datapackage.license_url) {
license.url = datapackage.license_url;
delete datapackage.license_url;
}
if (Object.keys(license).length > 0) {
datapackage.license = license;
}
// Parse author and sources
const source = {};
if (datapackage.author) {
source.name = datapackage.author;
delete datapackage.author;
}
if (datapackage.author_email) {
source.email = datapackage.author_email;
delete datapackage.author_email;
}
if (datapackage.url) {
source.web = datapackage.url;
delete datapackage.url;
}
if (Object.keys(source).length > 0) {
datapackage.sources = [source];
}
// Parse maintainer
const author = {};
if (datapackage.maintainer) {
author.name = datapackage.maintainer;
delete datapackage.maintainer;
}
if (datapackage.maintainer_email) {
author.email = datapackage.maintainer_email;
delete datapackage.maintainer_email;
}
if (Object.keys(author).length > 0) {
datapackage.author = author;
}
// Parse tags
if (datapackage.tags) {
datapackage.keywords = [];
datapackage.tags.forEach((tag) => {
datapackage.keywords.push(tag.name);
});
delete datapackage.tags;
}
// Parse extras
// TODO
// Resources
datapackage.resources = datapackage.resources.map((resource) => {
if (resource.name) {
resource.title = resource.title || resource.name;
resource.name = resource.name.toLowerCase().replace(/ /g, '_');
} else {
resource.name = resource.id;
}
if (resource.url) {
resource.path = resource.url;
delete resource.url;
}
if (!resource.schema) {
// If 'fields' property exists use it as schema fields
if (resource.fields) {
if (typeof resource.fields === 'string') {
try {
resource.fields = JSON.parse(resource.fields);
} catch (e) {
console.log('Could not parse resource.fields');
}
}
resource.schema = { fields: resource.fields };
delete resource.fields;
}
}
return resource;
});
return datapackage;
};
/*
At the moment, we're considering only following examples of CKAN view:
1. recline_view => Data Explorer with Table view, Chart Builder, Map Builder
and Query Builder.
2. geojson_view => Leaflet map
3. pdf_view => our PDF viewer
4. recline_grid_view => our Table viewer
5. recline_graph_view => our Simple graph
6. recline_map_view => our Leaflet map
7. image_view => not supported at the moment
8. text_view => not supported at the moment
9. webpage_view => not supported at the moment
*/
module.exports.ckanViewToDataPackageView = (ckanView) => {
const viewTypeToSpecType = {
recline_view: 'dataExplorer', // from datastore data
recline_grid_view: 'table',
recline_graph_view: 'simple',
recline_map_view: 'tabularmap',
geojson_view: 'map',
pdf_view: 'document',
image_view: 'web',
webpage_view: 'web',
};
const dataPackageView = JSON.parse(JSON.stringify(ckanView));
dataPackageView.specType =
viewTypeToSpecType[ckanView.view_type] ||
dataPackageView.specType ||
'unsupported';
if (dataPackageView.specType === 'dataExplorer') {
dataPackageView.spec = {
widgets: [
{ specType: 'table' },
{ specType: 'simple' },
{ specType: 'tabularmap' },
],
};
} else if (dataPackageView.specType === 'simple') {
const graphTypeConvert = {
lines: 'line',
'lines-and-points': 'lines-and-points',
points: 'points',
bars: 'horizontal-bar',
columns: 'bar',
};
dataPackageView.spec = {
group: ckanView.group,
series: Array.isArray(ckanView.series)
? ckanView.series
: [ckanView.series],
type: graphTypeConvert[ckanView.graph_type] || 'line',
};
} else if (dataPackageView.specType === 'tabularmap') {
if (ckanView.map_field_type === 'geojson') {
dataPackageView.spec = {
geomField: ckanView.geojson_field,
};
} else {
dataPackageView.spec = {
lonField: ckanView.longitude_field,
latField: ckanView.latitude_field,
};
}
}
return dataPackageView;
};
/*
Takes single field descriptor from datastore data dictionary and coverts into
tableschema field descriptor.
*/
module.exports.dataStoreDataDictionaryToTableSchema = (dataDictionary) => {
const internalDataStoreFields = ['_id', '_full_text', '_count'];
if (internalDataStoreFields.includes(dataDictionary.id)) {
return null;
}
const dataDictionaryType2TableSchemaType = {
text: 'string',
int: 'integer',
float: 'number',
date: 'date',
time: 'time',
timestamp: 'datetime',
bool: 'boolean',
json: 'object',
};
const field = {
name: dataDictionary.id,
type: dataDictionaryType2TableSchemaType[dataDictionary.type] || 'any',
};
if (dataDictionary.info) {
const constraintsAttributes = [
'required',
'unique',
'minLength',
'maxLength',
'minimum',
'maximum',
'pattern',
'enum',
];
field.constraints = {};
Object.keys(dataDictionary.info).forEach((key) => {
if (constraintsAttributes.includes(key)) {
field.constraints[key] = dataDictionary.info[key];
} else {
field[key] = dataDictionary.info[key];
}
});
}
return field;
};
module.exports.convertToStandardCollection = (descriptor) => {
const standard = {
name: '',
title: '',
summary: '',
image: '',
count: null,
};
standard.name = descriptor.name;
standard.title = descriptor.title || descriptor.display_name;
standard.summary = descriptor.description || '';
standard.image = descriptor.image_display_url || descriptor.image_url;
standard.count = descriptor.package_count || 0;
standard.extras = descriptor.extras || [];
standard.groups = descriptor.groups || [];
return standard;
};
module.exports.convertToCkanSearchQuery = (query) => {
const ckanQuery = {
q: '',
fq: '',
rows: '',
start: '',
sort: '',
'facet.field': [
'organization',
'groups',
'tags',
'res_format',
'license_id',
],
'facet.limit': 5,
'facet.mincount': 0,
};
// Split by space but ignore spaces within double quotes:
if (query.q) {
query.q.match(/(?:[^\s"]+|"[^"]*")+/g).forEach((part) => {
if (part.includes(':')) {
ckanQuery.fq += part + ' ';
} else {
ckanQuery.q += part + ' ';
}
});
ckanQuery.fq = ckanQuery.fq.trim();
ckanQuery.q = ckanQuery.q.trim();
}
if (query.fq) {
ckanQuery.fq = ckanQuery.fq ? ckanQuery.fq + ' ' + query.fq : query.fq;
}
// standard 'size' => ckan 'rows'
ckanQuery.rows = query.size || '';
// standard 'from' => ckan 'start'
ckanQuery.start = query.from || '';
// standard 'sort' => ckan 'sort'
const sortQueries = [];
if (query.sort && query.sort.constructor == Object) {
for (let [key, value] of Object.entries(query.sort)) {
sortQueries.push(`${key} ${value}`);
}
ckanQuery.sort = sortQueries.join(',');
} else if (query.sort && query.sort.constructor == String) {
ckanQuery.sort = query.sort.replace(':', ' ');
} else if (query.sort && query.sort.constructor == Array) {
query.sort.forEach((sort) => {
sortQueries.push(sort.replace(':', ' '));
});
ckanQuery.sort = sortQueries.join(',');
}
// Facets
ckanQuery['facet.field'] = query['facet.field'] || ckanQuery['facet.field'];
ckanQuery['facet.limit'] = query['facet.limit'] || ckanQuery['facet.limit'];
ckanQuery['facet.mincount'] =
query['facet.mincount'] || ckanQuery['facet.mincount'];
ckanQuery['facet.field'] = query['facet.field'] || ckanQuery['facet.field'];
// Remove attributes with empty string, null or undefined values
Object.keys(ckanQuery).forEach(
(key) => !ckanQuery[key] && delete ckanQuery[key]
);
return ckanQuery;
};
module.exports.pagination = (c, m) => {
let current = c,
last = m,
delta = 2,
left = current - delta,
right = current + delta + 1,
range = [],
rangeWithDots = [],
l;
range.push(1);
for (let i = c - delta; i <= c + delta; i++) {
if (i >= left && i < right && i < m && i > 1) {
range.push(i);
}
}
range.push(m);
for (let i of range) {
if (l) {
if (i - l === 2) {
rangeWithDots.push(l + 1);
} else if (i - l !== 1) {
rangeWithDots.push('...');
}
}
rangeWithDots.push(i);
l = i;
}
return rangeWithDots;
};
module.exports.processMarkdown = require('markdown-it')({
html: true,
linkify: true,
typographer: true,
});
/**
* Process data package attributes prior to display to users.
* Process markdown
* Convert bytes to human readable format
* etc.
**/
module.exports.processDataPackage = function (datapackage) {
const newDatapackage = JSON.parse(JSON.stringify(datapackage));
if (newDatapackage.description) {
newDatapackage.descriptionHtml = module.exports.processMarkdown.render(
newDatapackage.description
);
}
if (newDatapackage.readme) {
newDatapackage.readmeHtml = module.exports.processMarkdown.render(
newDatapackage.readme
);
}
newDatapackage.formats = newDatapackage.formats || [];
// Per each resource:
newDatapackage.resources.forEach((resource) => {
if (resource.description) {
resource.descriptionHtml = module.exports.processMarkdown.render(
resource.description
);
}
// Normalize format (lowercase)
if (resource.format) {
resource.format = resource.format.toLowerCase();
newDatapackage.formats.push(resource.format);
}
// Convert bytes into human-readable format:
if (resource.size) {
resource.sizeFormatted = bytes(resource.size, { decimalPlaces: 0 });
}
});
return newDatapackage;
};
/**
* Create 'displayResources' property which has:
* resource: Object containing resource descriptor
* api: API URL for the resource if available, e.g., Datastore
* proxy: path via proxy for the resource if available
* cc_proxy: path via CKAN Classic proxy if available
* slug: slugified name of a resource
**/
module.exports.prepareResourcesForDisplay = function (datapackage) {
const newDatapackage = JSON.parse(JSON.stringify(datapackage));
newDatapackage.displayResources = [];
newDatapackage.resources.forEach((resource, index) => {
const api = resource.datastore_active
? config.get('API_URL') +
'datastore_search?resource_id=' +
resource.id +
'&sort=_id asc'
: null;
// Use proxy path if datastore/filestore proxies are given:
let proxy, cc_proxy;
try {
const resourceUrl = new URL(resource.path);
if (
resourceUrl.host === config.get('PROXY_DATASTORE') &&
resource.format !== 'pdf'
) {
proxy = '/proxy/datastore' + resourceUrl.pathname + resourceUrl.search;
}
if (
resourceUrl.host === config.get('PROXY_FILESTORE') &&
resource.format !== 'pdf'
) {
proxy = '/proxy/filestore' + resourceUrl.pathname + resourceUrl.search;
}
// Store a CKAN Classic proxy path
// https://github.com/ckan/ckan/blob/master/ckanext/resourceproxy/plugin.py#L59
const apiUrlObject = new URL(config.get('API_URL'));
cc_proxy =
apiUrlObject.origin +
`/dataset/${datapackage.id}/resource/${resource.id}/proxy`;
} catch (e) {
console.warn(e);
}
const displayResource = {
resource,
api, // URI for getting the resource via API, e.g., Datastore. Useful when you want to fetch only 100 rows or similar.
proxy, // alternative for path in case there is CORS issue
cc_proxy,
slug: slugify(resource.name) + '-' + index, // Used for anchor links
};
newDatapackage.displayResources.push(displayResource);
});
return newDatapackage;
};
/**
* Prepare 'views' property which is used by 'datapackage-views-js' library to
* render visualizations such as tables, graphs and maps.
**/
module.exports.prepareViews = function (datapackage) {
const newDatapackage = JSON.parse(JSON.stringify(datapackage));
newDatapackage.views = newDatapackage.views || [];
newDatapackage.resources.forEach((resource) => {
const resourceViews =
resource.views &&
resource.views.map((view) => {
view.resources = [resource.name];
return view;
});
newDatapackage.views = newDatapackage.views.concat(resourceViews);
});
return newDatapackage;
};
/**
* Create 'dataExplorers' property which is used by 'data-explorer' library to
* render data explorer widgets.
**/
module.exports.prepareDataExplorers = function (datapackage) {
const newDatapackage = JSON.parse(JSON.stringify(datapackage));
newDatapackage.displayResources.forEach((displayResource, idx) => {
newDatapackage.displayResources[idx].dataExplorers = [];
displayResource.resource.views &&
displayResource.resource.views.forEach((view) => {
const widgets = [];
if (view.specType === 'dataExplorer') {
view.spec.widgets.forEach((widget, index) => {
const widgetNames = {
table: 'Table',
simple: 'Chart',
tabularmap: 'Map',
};
widget = {
name: widgetNames[widget.specType] || 'Widget-' + index,
active: index === 0 ? true : false,
datapackage: {
views: [
{
id: view.id,
specType: widget.specType,
},
],
},
};
widgets.push(widget);
});
} else {
const widget = {
name: view.title || '',
active: true,
datapackage: {
views: [view],
},
};
widgets.push(widget);
}
displayResource.resource.api =
displayResource.resource.api || displayResource.api;
const dataExplorer = JSON.stringify({
widgets,
datapackage: {
resources: [displayResource.resource],
},
}).replace(/'/g, '&#x27;');
newDatapackage.displayResources[idx].dataExplorers.push(dataExplorer);
});
});
return newDatapackage;
};

12996
examples/catalog/yarn.lock Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,51 @@
CBOE Volatility Index (VIX) time-series dataset including daily open, close,
high and low. The CBOE Volatility Index (VIX) is a key measure of market
expectations of near-term volatility conveyed by S&P 500 stock index option
prices introduced in 1993.
## Data
From the [VIX FAQ][faq]:
> In 1993, the Chicago Board Options Exchange® (CBOE®) introduced the CBOE
> Volatility Index®, VIX®, and it quickly became the benchmark for stock market
> volatility. It is widely followed and has been cited in hundreds of news
> articles in the Wall Street Journal, Barron's and other leading financial
> publications. Since volatility often signifies financial turmoil, VIX is
> often referred to as the "investor fear gauge".
>
> VIX measures market expectation of near term volatility conveyed by stock
> index option prices. The original VIX was constructed using the implied
> volatilities of eight different OEX option series so that, at any given time,
> it represented the implied volatility of a hypothetical at-the-money OEX
> option with exactly 30 days to expiration.
>
> The New VIX still measures the market's expectation of 30-day volatility, but
> in a way that conforms to the latest thinking and research among industry
> practitioners. The New VIX is based on S&P 500 index option prices and
> incorporates information from the volatility "skew" by using a wider range of
> strike prices rather than just at-the-money series.
[faq]: http://www.cboe.com/micro/vix/faq.aspx
## Preparation
Run the shell script:
. scripts/process.sh
Output data is in `data/`.
### TODO
* Incorporate computed historical data (1990-2003)
* Consider incorporating VOX data
## License
No obvious statement on [historical data page][historical]. Given size and
factual nature of the data and its source from a US company would imagine this
was public domain and as such have licensed the Data Package under the Public
Domain Dedication and License (PDDL).
[historical]: http://www.cboe.com/micro/vix/historical.aspx

View File

@ -0,0 +1,112 @@
{
"name": "finance-vix",
"title": "VIX - CBOE Volatility Index",
"homepage": "http://www.cboe.com/micro/VIX/",
"version": "0.1.0",
"license": "PDDL-1.0",
"sources": [
{
"title": "CBOE VIX Page",
"name": "CBOE VIX Page",
"web": "http://www.cboe.com/micro/vix/historical.aspx"
}
],
"resources": [
{
"name": "vix-daily",
"path": "vix-daily.csv",
"format": "csv",
"mediatype": "text/csv",
"schema": {
"fields": [
{
"name": "Date",
"type": "date",
"description": ""
},
{
"name": "VIXOpen",
"type": "number",
"description": ""
},
{
"name": "VIXHigh",
"type": "number",
"description": ""
},
{
"name": "VIXLow",
"type": "number",
"description": ""
},
{
"name": "VIXClose",
"type": "number",
"description": ""
}
],
"primaryKey": "Date"
}
}
],
"views": [
{
"name": "simple graph",
"id": 1,
"title": "title1",
"specType": "simple",
"spec": {
"type": "line",
"group": "VIXClose",
"series": [
"VIXOpen",
"VIXHigh"
]
}
},
{
"name": "plotly graph",
"id": 2,
"specType": "plotly",
"resources": [
"vix-daily"
],
"spec": {
"group": "VIXClose",
"series": [
"VIXOpen",
"VIXHigh",
"VIXLow"
],
"data": [
{
"type": "bar"
}
],
"layout": {
"title": "Plotly Layout Title",
"height": 450,
"xaxis": {
"title": "X Axis Title"
},
"yaxis": {
"title": "Y Axis Title"
},
"font": {
"family": "\"Open Sans\", verdana, arial, sans-serif",
"size": 12,
"color": "rgb(169, 169, 169)"
},
"titlefont": {
"family": "\"Open Sans\", verdana, arial, sans-serif",
"size": 17,
"color": "rgb(76, 76, 76)"
}
},
"config": {
"displayModeBar": false
}
}
}
]
}

View File

@ -0,0 +1,20 @@
Date,VIXOpen,VIXHigh,VIXLow,VIXClose
2004-01-02,17.96,18.68,17.54,18.22
2004-01-05,18.45,18.49,17.44,17.49
2004-01-06,17.66,17.67,16.19,16.73
2004-01-07,16.72,16.75,15.05,15.05
2004-01-08,15.42,15.68,15.32,15.61
2004-01-09,16.15,16.88,15.57,16.75
2004-01-12,17.32,17.46,16.79,16.82
2004-01-13,16.06,18.33,16.53,18.04
2004-01-14,17.29,17.03,16.04,16.75
2004-01-15,17.07,17.31,15.49,15.56
2004-01-16,15.04,15.44,14.09,15
2004-01-20,15.77,16.13,15.09,15.21
2004-01-21,15.63,15.63,14.24,14.34
2004-01-22,14.02,14.87,14.01,14.71
2004-01-23,14.73,15.05,14.56,14.84
2004-01-26,15.78,15.78,14.52,14.55
2004-01-27,15.28,15.44,14.74,15.35
2004-01-28,15.37,17.06,15.29,16.78
2004-01-29,16.88,17.66,16.79,17.14
1 Date VIXOpen VIXHigh VIXLow VIXClose
2 2004-01-02 17.96 18.68 17.54 18.22
3 2004-01-05 18.45 18.49 17.44 17.49
4 2004-01-06 17.66 17.67 16.19 16.73
5 2004-01-07 16.72 16.75 15.05 15.05
6 2004-01-08 15.42 15.68 15.32 15.61
7 2004-01-09 16.15 16.88 15.57 16.75
8 2004-01-12 17.32 17.46 16.79 16.82
9 2004-01-13 16.06 18.33 16.53 18.04
10 2004-01-14 17.29 17.03 16.04 16.75
11 2004-01-15 17.07 17.31 15.49 15.56
12 2004-01-16 15.04 15.44 14.09 15
13 2004-01-20 15.77 16.13 15.09 15.21
14 2004-01-21 15.63 15.63 14.24 14.34
15 2004-01-22 14.02 14.87 14.01 14.71
16 2004-01-23 14.73 15.05 14.56 14.84
17 2004-01-26 15.78 15.78 14.52 14.55
18 2004-01-27 15.28 15.44 14.74 15.35
19 2004-01-28 15.37 17.06 15.29 16.78
20 2004-01-29 16.88 17.66 16.79 17.14

View File

@ -0,0 +1,51 @@
CBOE Volatility Index (VIX) time-series dataset including daily open, close,
high and low. The CBOE Volatility Index (VIX) is a key measure of market
expectations of near-term volatility conveyed by S&P 500 stock index option
prices introduced in 1993.
## Data
From the [VIX FAQ][faq]:
> In 1993, the Chicago Board Options Exchange® (CBOE®) introduced the CBOE
> Volatility Index®, VIX®, and it quickly became the benchmark for stock market
> volatility. It is widely followed and has been cited in hundreds of news
> articles in the Wall Street Journal, Barron's and other leading financial
> publications. Since volatility often signifies financial turmoil, VIX is
> often referred to as the "investor fear gauge".
>
> VIX measures market expectation of near term volatility conveyed by stock
> index option prices. The original VIX was constructed using the implied
> volatilities of eight different OEX option series so that, at any given time,
> it represented the implied volatility of a hypothetical at-the-money OEX
> option with exactly 30 days to expiration.
>
> The New VIX still measures the market's expectation of 30-day volatility, but
> in a way that conforms to the latest thinking and research among industry
> practitioners. The New VIX is based on S&P 500 index option prices and
> incorporates information from the volatility "skew" by using a wider range of
> strike prices rather than just at-the-money series.
[faq]: http://www.cboe.com/micro/vix/faq.aspx
## Preparation
Run the shell script:
. scripts/process.sh
Output data is in `data/`.
### TODO
* Incorporate computed historical data (1990-2003)
* Consider incorporating VOX data
## License
No obvious statement on [historical data page][historical]. Given size and
factual nature of the data and its source from a US company would imagine this
was public domain and as such have licensed the Data Package under the Public
Domain Dedication and License (PDDL).
[historical]: http://www.cboe.com/micro/vix/historical.aspx

View File

@ -0,0 +1,98 @@
{
"name": "finance-vix",
"title": "VIX - CBOE Volatility Index",
"homepage": "http://www.cboe.com/micro/VIX/",
"version": "0.1.0",
"license": "PDDL-1.0",
"sources": [
{
"title": "CBOE VIX Page",
"name": "CBOE VIX Page",
"web": "http://www.cboe.com/micro/vix/historical.aspx"
}
],
"resources": [
{
"name": "vix-daily",
"path": "vix-daily.csv",
"format": "csv",
"mediatype": "text/csv",
"schema": {
"fields": [
{
"name": "Date",
"type": "date",
"description": ""
},
{
"name": "VIXOpen",
"type": "number",
"description": ""
},
{
"name": "VIXHigh",
"type": "number",
"description": ""
},
{
"name": "VIXLow",
"type": "number",
"description": ""
},
{
"name": "VIXClose",
"type": "number",
"description": ""
}
],
"primaryKey": "Date"
}
}
],
"views": [
{
"name": "plotly graph",
"id": 2,
"specType": "plotly",
"resources": [
"vix-daily"
],
"spec": {
"group": "VIXClose",
"series": [
"VIXOpen",
"VIXHigh",
"VIXLow"
],
"data": [
{
"type": "bar"
}
],
"layout": {
"title": "Plotly Layout Title",
"height": 450,
"xaxis": {
"title": "X Axis Title"
},
"yaxis": {
"title": "Y Axis Title"
},
"font": {
"family": "\"Open Sans\", verdana, arial, sans-serif",
"size": 12,
"color": "rgb(169, 169, 169)"
},
"titlefont": {
"family": "\"Open Sans\", verdana, arial, sans-serif",
"size": 17,
"color": "rgb(76, 76, 76)"
}
},
"config": {
"displayModeBar": false
}
}
}
]
}

View File

@ -0,0 +1,20 @@
Date,VIXOpen,VIXHigh,VIXLow,VIXClose
2004-01-02,17.96,18.68,17.54,18.22
2004-01-05,18.45,18.49,17.44,17.49
2004-01-06,17.66,17.67,16.19,16.73
2004-01-07,16.72,16.75,15.05,15.05
2004-01-08,15.42,15.68,15.32,15.61
2004-01-09,16.15,16.88,15.57,16.75
2004-01-12,17.32,17.46,16.79,16.82
2004-01-13,16.06,18.33,16.53,18.04
2004-01-14,17.29,17.03,16.04,16.75
2004-01-15,17.07,17.31,15.49,15.56
2004-01-16,15.04,15.44,14.09,15
2004-01-20,15.77,16.13,15.09,15.21
2004-01-21,15.63,15.63,14.24,14.34
2004-01-22,14.02,14.87,14.01,14.71
2004-01-23,14.73,15.05,14.56,14.84
2004-01-26,15.78,15.78,14.52,14.55
2004-01-27,15.28,15.44,14.74,15.35
2004-01-28,15.37,17.06,15.29,16.78
2004-01-29,16.88,17.66,16.79,17.14
1 Date VIXOpen VIXHigh VIXLow VIXClose
2 2004-01-02 17.96 18.68 17.54 18.22
3 2004-01-05 18.45 18.49 17.44 17.49
4 2004-01-06 17.66 17.67 16.19 16.73
5 2004-01-07 16.72 16.75 15.05 15.05
6 2004-01-08 15.42 15.68 15.32 15.61
7 2004-01-09 16.15 16.88 15.57 16.75
8 2004-01-12 17.32 17.46 16.79 16.82
9 2004-01-13 16.06 18.33 16.53 18.04
10 2004-01-14 17.29 17.03 16.04 16.75
11 2004-01-15 17.07 17.31 15.49 15.56
12 2004-01-16 15.04 15.44 14.09 15
13 2004-01-20 15.77 16.13 15.09 15.21
14 2004-01-21 15.63 15.63 14.24 14.34
15 2004-01-22 14.02 14.87 14.01 14.71
16 2004-01-23 14.73 15.05 14.56 14.84
17 2004-01-26 15.78 15.78 14.52 14.55
18 2004-01-27 15.28 15.44 14.74 15.35
19 2004-01-28 15.37 17.06 15.29 16.78
20 2004-01-29 16.88 17.66 16.79 17.14

View File

@ -0,0 +1,51 @@
CBOE Volatility Index (VIX) time-series dataset including daily open, close,
high and low. The CBOE Volatility Index (VIX) is a key measure of market
expectations of near-term volatility conveyed by S&P 500 stock index option
prices introduced in 1993.
## Data
From the [VIX FAQ][faq]:
> In 1993, the Chicago Board Options Exchange® (CBOE®) introduced the CBOE
> Volatility Index®, VIX®, and it quickly became the benchmark for stock market
> volatility. It is widely followed and has been cited in hundreds of news
> articles in the Wall Street Journal, Barron's and other leading financial
> publications. Since volatility often signifies financial turmoil, VIX is
> often referred to as the "investor fear gauge".
>
> VIX measures market expectation of near term volatility conveyed by stock
> index option prices. The original VIX was constructed using the implied
> volatilities of eight different OEX option series so that, at any given time,
> it represented the implied volatility of a hypothetical at-the-money OEX
> option with exactly 30 days to expiration.
>
> The New VIX still measures the market's expectation of 30-day volatility, but
> in a way that conforms to the latest thinking and research among industry
> practitioners. The New VIX is based on S&P 500 index option prices and
> incorporates information from the volatility "skew" by using a wider range of
> strike prices rather than just at-the-money series.
[faq]: http://www.cboe.com/micro/vix/faq.aspx
## Preparation
Run the shell script:
. scripts/process.sh
Output data is in `data/`.
### TODO
* Incorporate computed historical data (1990-2003)
* Consider incorporating VOX data
## License
No obvious statement on [historical data page][historical]. Given size and
factual nature of the data and its source from a US company would imagine this
was public domain and as such have licensed the Data Package under the Public
Domain Dedication and License (PDDL).
[historical]: http://www.cboe.com/micro/vix/historical.aspx

View File

@ -0,0 +1,131 @@
{
"name": "finance-vix",
"title": "VIX - CBOE Volatility Index",
"homepage": "http://www.cboe.com/micro/VIX/",
"version": "0.1.0",
"license": "PDDL-1.0",
"sources": [
{
"title": "CBOE VIX Page",
"name": "CBOE VIX Page",
"web": "http://www.cboe.com/micro/vix/historical.aspx"
}
],
"resources": [
{
"name": "vix-daily",
"path": "vix-daily.csv",
"format": "csv",
"mediatype": "text/csv",
"schema": {
"fields": [
{
"name": "Date",
"type": "date",
"description": ""
},
{
"name": "VIXOpen",
"type": "number",
"description": ""
},
{
"name": "VIXHigh",
"type": "number",
"description": ""
},
{
"name": "VIXLow",
"type": "number",
"description": ""
},
{
"name": "VIXClose",
"type": "number",
"description": ""
}
],
"primaryKey": "Date"
}
}
],
"views": [
{
"name": "vega4",
"resources": [
0
],
"specType": "vega",
"spec": {
"width": 600,
"height": 300,
"data": [
{
"name": "vix-daily"
}
],
"scales": [
{
"name": "VIXOpen",
"type": "point",
"range": "width",
"domain": {
"data": "vix-daily",
"field": "VIXOpen"
}
},
{
"name": "VIXHigh",
"type": "linear",
"range": "height",
"domain": {
"data": "vix-daily",
"field": "VIXHigh"
}
}
],
"axes": [
{
"orient": "bottom",
"scale": "VIXOpen"
},
{
"orient": "left",
"scale": "VIXHigh"
}
],
"marks": [
{
"type": "line",
"from": {
"data": "vix-daily"
},
"encode": {
"enter": {
"x": {
"scale": "VIXOpen",
"field": "VIXOpen"
},
"y": {
"scale": "VIXHigh",
"field": "VIXHigh"
},
"strokeWidth": {
"value": 2
}
},
"strokeOpacity": {
"value": 1
}
},
"hover": {
"strokeOpacity": {
"value": 0.5
}
}
}
]
}
}
]
}

View File

@ -0,0 +1,20 @@
Date,VIXOpen,VIXHigh,VIXLow,VIXClose
2004-01-02,17.96,18.68,17.54,18.22
2004-01-05,18.45,18.49,17.44,17.49
2004-01-06,17.66,17.67,16.19,16.73
2004-01-07,16.72,16.75,15.05,15.05
2004-01-08,15.42,15.68,15.32,15.61
2004-01-09,16.15,16.88,15.57,16.75
2004-01-12,17.32,17.46,16.79,16.82
2004-01-13,16.06,18.33,16.53,18.04
2004-01-14,17.29,17.03,16.04,16.75
2004-01-15,17.07,17.31,15.49,15.56
2004-01-16,15.04,15.44,14.09,15
2004-01-20,15.77,16.13,15.09,15.21
2004-01-21,15.63,15.63,14.24,14.34
2004-01-22,14.02,14.87,14.01,14.71
2004-01-23,14.73,15.05,14.56,14.84
2004-01-26,15.78,15.78,14.52,14.55
2004-01-27,15.28,15.44,14.74,15.35
2004-01-28,15.37,17.06,15.29,16.78
2004-01-29,16.88,17.66,16.79,17.14
1 Date VIXOpen VIXHigh VIXLow VIXClose
2 2004-01-02 17.96 18.68 17.54 18.22
3 2004-01-05 18.45 18.49 17.44 17.49
4 2004-01-06 17.66 17.67 16.19 16.73
5 2004-01-07 16.72 16.75 15.05 15.05
6 2004-01-08 15.42 15.68 15.32 15.61
7 2004-01-09 16.15 16.88 15.57 16.75
8 2004-01-12 17.32 17.46 16.79 16.82
9 2004-01-13 16.06 18.33 16.53 18.04
10 2004-01-14 17.29 17.03 16.04 16.75
11 2004-01-15 17.07 17.31 15.49 15.56
12 2004-01-16 15.04 15.44 14.09 15
13 2004-01-20 15.77 16.13 15.09 15.21
14 2004-01-21 15.63 15.63 14.24 14.34
15 2004-01-22 14.02 14.87 14.01 14.71
16 2004-01-23 14.73 15.05 14.56 14.84
17 2004-01-26 15.78 15.78 14.52 14.55
18 2004-01-27 15.28 15.44 14.74 15.35
19 2004-01-28 15.37 17.06 15.29 16.78
20 2004-01-29 16.88 17.66 16.79 17.14

8
jest.config.js Normal file
View File

@ -0,0 +1,8 @@
module.exports = {
testPathIgnorePatterns: ["<rootDir>/.next/", "<rootDir>/node_modules/", "<rootDir>/examples/catalog/*"],
setupFilesAfterEnv: ["<rootDir>/tests/setupTests.js"],
transform: {
"^.+\\.(js|jsx|ts|tsx)$": "<rootDir>/node_modules/babel-jest",
"\\.(css|less|scss|sass)$": "identity-obj-proxy"
}
};

32
lib/dataset.js Normal file
View File

@ -0,0 +1,32 @@
import remark from 'remark'
import html from 'remark-html'
import { Dataset } from 'frictionless.js'
import toArray from 'stream-to-array'
export async function getDataset(directory) {
// get dataset descriptor and resources
const f11sDataset = await Dataset.load(directory)
const descriptor = f11sDataset.descriptor
const resources = await Promise.all(f11sDataset.resources.map(async (resource) => {
let _tmp = resource.descriptor
let rowStream = await resource.rows({ keyed: true })
_tmp.sample = await toArray(rowStream)
_tmp.size = resource.size
return _tmp
}))
const readme = descriptor.readme
const processed = await remark()
.use(html)
.process(readme)
const readmeHtml = processed.toString()
const dataset = {
readme: readme,
readmeHtml: readmeHtml,
descriptor: descriptor,
resources: resources
}
return dataset
}

110
lib/utils.js Normal file
View File

@ -0,0 +1,110 @@
import { simpleToPlotly, plotlyToPlotly, vegaToVega } from 'datapackage-render'
/**
* Prepare views for dataset
* @params {object} dataset object of the form:
* { readme: readme,
readmeHtml: readmeHtml,
descriptor: descriptor,
resources: resources
}
*/
export function addView(dataset) {
const views = dataset.descriptor.views
const countViews = views ? views.length : 0
if (countViews === 0) {
return {
props: {
dataset,
error: true
}
}
}
const specs = {} //hold list of view specs
for (let i = 0; i < countViews; i++) {
const view = views[i]
if (("resources" in view) && Array.isArray(view.resources)) {
let resource;
const resourceKey = view.resources[0]
if (typeof resourceKey == 'number') {
resource = dataset.resources[resourceKey]
view.resources[0] = resource
} else {
resource = dataset.resources.filter((resource) => {
return resource.name == resourceKey
})
view.resources[0] = resource[0]
}
} else {
view.resources = [dataset.resources[0]] //take the first resources in datapackage
}
view.resources[0].data = getDataForViewSpec(view.resources[0], view.specType)
view.resources[0]._values = view.resources[0].data
if (view.specType === 'simple') {
try {
const spec = simpleToPlotly(view)
if (spec) {
spec.specType = 'simple'
specs[i] = spec
}
} catch (err) {
console.log(err);
}
} else if (view.specType === 'plotly') {
try {
const spec = plotlyToPlotly(view)
if (spec) {
spec.specType = 'plotly'
specs[i] = spec
}
} catch (err) {
console.log(err);
}
} else if (view.specType === 'vega') {
try {
const spec = vegaToVega(view)
if (spec) {
spec.specType = 'vega'
specs[i] = spec
}
} catch (err) {
console.log(err);
}
}
}
return {
props: {
dataset,
specs: JSON.stringify(specs),
error: false
}
}
}
/**
* Generates the data for each view spec. Plotly and Vega accept different data
* formats.
* @param {*} resource
* @param {*} specType
*/
export function getDataForViewSpec(resource, specType) {
if (specType == "vega") {
return resource.sample
} else if (["simple", 'plotly'].includes(specType)) {
const sample = resource.sample
let data = []
data.push(Object.keys(sample[0])) //add the column names
for (let i = 0; i < sample.length; i++) {
const item = sample[i];
data.push(Object.values(item)) //add the rows
}
return data
}
}

21
license Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Datopian
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

57
package.json Normal file
View File

@ -0,0 +1,57 @@
{
"name": "portal",
"description": "The data presentation framework",
"author": "Datopian",
"license": "MIT",
"version": "0.1.0",
"bin": {
"portal": "./bin/portal.js"
},
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"test": "jest --coverage"
},
"dependencies": {
"@material-ui/core": "^4.11.3",
"@material-ui/data-grid": "^4.0.0-alpha.20",
"@tailwindcss/typography": "^0.4.0",
"autoprefixer": "^10.0.4",
"datapackage-render": "git+https://github.com/frictionlessdata/datapackage-render-js.git",
"filesize": "^6.1.0",
"frictionless.js": "^0.13.4",
"next": "latest",
"plotly.js-basic-dist": "^1.58.4",
"postcss": "^8.1.10",
"react": "^17.0.1",
"react-dom": "^17.0.1",
"react-plotly.js": "^2.5.1",
"react-table": "^7.6.3",
"react-vega": "^7.4.2",
"remark": "^13.0.0",
"remark-html": "^13.0.1",
"tailwindcss": "^2.0.2",
"vega": "^5.19.1",
"vega-lite": "^5.0.0",
"chalk": "^4.1.0",
"commander": "^6.2.0",
"cpy": "^8.1.1",
"cross-spawn": "^7.0.3",
"figlet": "^1.5.0",
"listr": "^0.14.3",
"open": "^8.0.2",
"ora": "^5.1.0",
"prompts": "^2.4.0"
},
"devDependencies": {
"@testing-library/dom": "^7.29.6",
"@testing-library/jest-dom": "^5.11.9",
"@testing-library/react": "^11.2.5",
"babel-jest": "^26.6.3",
"identity-obj-proxy": "^3.0.0",
"jest": "^26.6.3",
"jest-canvas-mock": "^2.3.1",
"jest-dom": "^4.0.0"
}
}

8
pages/_app.js Normal file
View File

@ -0,0 +1,8 @@
import '../styles/globals.css'
import '../styles/tailwind.css'
function MyApp({ Component, pageProps }) {
return <Component {...pageProps} />
}
export default MyApp

214
pages/index.js Normal file
View File

@ -0,0 +1,214 @@
import path from 'path'
import Head from 'next/head'
import Table from '../components/Table'
import filesize from 'filesize'
import { Vega } from 'react-vega';
import { getDataset } from '../lib/dataset'
import Chart from '../components/Chart'
import { addView } from '../lib/utils'
const datasetsDirectory = process.env.PORTAL_DATASET_PATH || path.join(process.cwd(), "fixtures", "datasetsDoubleView")
export default function Home({ dataset, specs }) {
if (!dataset && !specs) {
return (
<div className="container">
<Head>
<title>Dataset</title>
<link rel="icon" href="/favicon.ico" />
<link rel="preconnect" href="https://fonts.gstatic.com" />
<link href="https://fonts.googleapis.com/css2?family=Inconsolata&display=swap" rel="stylesheet" />
</Head>
<h1 data-testid="datasetTitle" className="text-3xl font-bold mb-8">
No dataset found in path
</h1>
</div>
)
}
const descriptor = dataset['descriptor']
const resources = dataset['resources']
let datasetSize = 0
if (resources) {
datasetSize = resources.length == 1 ?
resources[0].size :
resources.reduce((accumulator, currentValue) => {
return accumulator.size + currentValue.size
})
}
return (
<div className="container">
<Head>
<title>Dataset</title>
<link rel="icon" href="/favicon.ico" />
<link rel="preconnect" href="https://fonts.gstatic.com" />
<link href="https://fonts.googleapis.com/css2?family=Inconsolata&display=swap" rel="stylesheet" />
</Head>
<section className="m-8" name="key-info">
<h1 data-testid="datasetTitle" className="text-3xl font-bold mb-8">
{descriptor.title}
</h1>
<h1 className="text-2xl font-bold mb-4">Key info</h1>
<div className="grid grid-cols-7 gap-4">
<div>
<h3 className="text-1xl font-bold mb-2">Files</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Size</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Format</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Created</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Updated</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Licence</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Source</h3>
</div>
</div>
<div className="grid grid-cols-7 gap-4">
<div>
<h3 className="text-1xl">{resources.length}</h3>
</div>
<div>
<h3 className="text-1xl">{filesize(datasetSize, { bits: true })}</h3>
</div>
<div>
<h3 className="text-1xl">{resources[0].format} zip</h3>
</div>
<div>
<h3 className="text-1xl">{descriptor.created}</h3>
</div>
<div>
<h3 className="text-1xl">{descriptor.updated}</h3>
</div>
<div>
<h3 className="text-1xl">{descriptor.license}</h3>
</div>
<div>
<h3 className="text-1xl">
<a className="text-yellow-600"
href={descriptor.sources[0].web}>
{descriptor.sources[0].title}
</a>
</h3>
</div>
</div>
</section>
<section className="m-8" name="file-list">
<h1 className="text-2xl font-bold mb-4">Data Files</h1>
<div className="grid grid-cols-7 gap-4">
<div>
<h3 className="text-1xl font-bold mb-2">File</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Description</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Size</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Last Changed</h3>
</div>
<div>
<h3 className="text-1xl font-bold mb-2">Download</h3>
</div>
</div>
{resources.map((resource, index) => {
return (
<div key={`${index}_${resource.name}`} className="grid grid-cols-7 gap-4">
<div>
<h3 className="text-1xl">{resource.name}</h3>
</div>
<div>
<h3 className="text-1xl">{resource.description || "No description"}</h3>
</div>
<div>
<h3 className="text-1xl">{filesize(resource.size, { bits: true })}</h3>
</div>
<div>
<h3 className="text-1xl">{resource.updated}</h3>
</div>
<div>
<h3 className="text-1xl">
<a className="text-yellow-600" href={resource.path}>
{resource.format} ({filesize(resource.size, { bits: true })})
</a>
</h3>
</div>
</div>
)
})}
</section>
<section className="m-8" name="graph">
<h1 className="text-2xl font-bold mb-4">Graph</h1>
{!specs || Object.keys(specs).length == 0 ? (<div>
<h1>No graph to display</h1>
</div>) :
(
Object.values(JSON.parse(specs)).map((spec, i) => {
if (spec.specType == "vega") {
return (
<div key={`${i}_views`} className="ml-14">
<Vega spec={spec} />
</div>
)
} else if (["simple", "plotly"].includes(spec.specType)) {
return (
<div key={`${i}_views`}>
<Chart spec={spec} />
</div>)
} else {
return <h1 key={`${i}_views`}>Cannot display view</h1>
}
})
)}
</section>
<section className="m-8" name="sample-table" >
<h1 className="text-2xl font-bold mb-4">Data Preview</h1>
<h2 className="text-1xl">{descriptor.title}</h2>
{resources[0].sample ? (
<Table data={resources[0].sample} schema={resources[0].schema} />
) : (
'No preview is available for this dataset'
)}
</section>
<section className="m-8" name="sample-table">
<h1 className="text-2xl font-bold mb-4">README</h1>
<div className="prose">
<div dangerouslySetInnerHTML={{ __html: dataset.readmeHtml }} />
</div>
</section>
</div>
)
}
export async function getStaticProps() {
if (!datasetsDirectory) {
return { props: {} }
}
const dataset = await getDataset(datasetsDirectory)
const datasetWithViews = addView(dataset)
return datasetWithViews
}

8
postcss.config.js Normal file
View File

@ -0,0 +1,8 @@
// If you want to use other PostCSS plugins, see the following:
// https://tailwindcss.com/docs/using-with-preprocessors
module.exports = {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
}

BIN
public/favicon.ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

3
styles/globals.css Normal file
View File

@ -0,0 +1,3 @@
.MuiTableCell-root {
@apply font-mono
}

3
styles/tailwind.css Normal file
View File

@ -0,0 +1,3 @@
@tailwind base;
@tailwind components;
@tailwind utilities;

22
tailwind.config.js Normal file
View File

@ -0,0 +1,22 @@
const defaultTheme = require("tailwindcss/defaultTheme");
module.exports = {
purge: ['./pages/**/*.{js,ts,jsx,tsx}', './components/**/*.{js,ts,jsx,tsx}'],
darkMode: false, // or 'media' or 'class'
theme: {
container: {
center: true,
},
extend: {
fontFamily: {
mono: ["Inconsolata", ...defaultTheme.fontFamily.mono]
}
},
},
variants: {
extend: {},
},
plugins: [
require('@tailwindcss/typography'),
],
}

Some files were not shown because too many files have changed in this diff Show More