[site,#572][l]: merge in data literate material.
* Nav bar and layout * Main catch all route that loads material from content * Data Literate: demo plus all associated components * content/data-literate/demo.mdx * components/* (pretty much all are related to demo) * lib/markdown.js, lib/mdxUtils.js * api/proxy.js to proxy remote urls (to handle CORs) * content/data-literate.md (DL home page - old data literate home page converted to markdown) * excel-viewer.js: excel viewer demo from data literate * package.json / yarn.lock * Nav: @headlessui/react @heroicons/react/outline @heroicons/react * CSV support for table: papaparse * Excel support for tables etc: xlsx * Vega: react-vega vega vega-lite * MDX: next-mdx-remote (yarn rm @next/mdx)
This commit is contained in:
Rufus Pollock
parent
b81cf992de
commit
8c3b1bccd3
16
site/components/CustomLink.js
Normal file
16
site/components/CustomLink.js
Normal file
@ -0,0 +1,16 @@
|
||||
import Link from 'next/link'
|
||||
|
||||
export default function CustomLink({ as, href, ...otherProps }) {
|
||||
return (
|
||||
<>
|
||||
<Link as={as} href={href}>
|
||||
<a {...otherProps} />
|
||||
</Link>
|
||||
<style jsx>{`
|
||||
a {
|
||||
color: tomato;
|
||||
}
|
||||
`}</style>
|
||||
</>
|
||||
)
|
||||
}
|
||||
45
site/components/DataLiterate.js
Normal file
45
site/components/DataLiterate.js
Normal file
@ -0,0 +1,45 @@
|
||||
import Layout from '../components/Layout'
|
||||
|
||||
import { MDXRemote } from 'next-mdx-remote'
|
||||
import dynamic from 'next/dynamic'
|
||||
import Head from 'next/head'
|
||||
import Link from 'next/link'
|
||||
|
||||
import CustomLink from '../components/CustomLink'
|
||||
import { Vega, VegaLite } from 'react-vega'
|
||||
|
||||
// Custom components/renderers to pass to MDX.
|
||||
// Since the MDX files aren't loaded by webpack, they have no knowledge of how
|
||||
// to handle import statements. Instead, you must include components in scope
|
||||
// here.
|
||||
const components = {
|
||||
a: CustomLink,
|
||||
Table: dynamic(() => import('../components/Table')),
|
||||
Excel: dynamic(() => import('../components/Excel')),
|
||||
// TODO: try and make these dynamic ...
|
||||
Vega: Vega,
|
||||
VegaLite: VegaLite,
|
||||
LineChart: dynamic(() => import('../components/LineChart')),
|
||||
Head,
|
||||
}
|
||||
|
||||
export default function DataLiterate({ children, source, frontMatter }) {
|
||||
return (
|
||||
<Layout title={frontMatter.title}>
|
||||
<header>
|
||||
<div className="mb-6">
|
||||
<h1>{frontMatter.title}</h1>
|
||||
{frontMatter.author && (
|
||||
<div className="-mt-6"><p className="opacity-60 pl-1">{frontMatter.author}</p></div>
|
||||
)}
|
||||
{frontMatter.description && (
|
||||
<p className="description">{frontMatter.description}</p>
|
||||
)}
|
||||
</div>
|
||||
</header>
|
||||
<main>
|
||||
<MDXRemote {...source} components={components} />
|
||||
</main>
|
||||
</Layout>
|
||||
)
|
||||
}
|
||||
74
site/components/Excel.js
Normal file
74
site/components/Excel.js
Normal file
@ -0,0 +1,74 @@
|
||||
import axios from 'axios'
|
||||
import XLSX from 'xlsx'
|
||||
import React, { useEffect, useState } from 'react'
|
||||
|
||||
import Table from './Table'
|
||||
|
||||
export default function Excel ({ src='' }) {
|
||||
const [data, setData] = React.useState([])
|
||||
const [cols, setCols] = React.useState([])
|
||||
const [workbook, setWorkbook] = React.useState(null)
|
||||
const [error, setError] = React.useState('')
|
||||
const [hasMounted, setHasMounted] = React.useState(0)
|
||||
|
||||
// so this is here so we re-render this in the browser
|
||||
// and not just when we build the page statically in nextjs
|
||||
useEffect(() => {
|
||||
if (hasMounted==0) {
|
||||
handleUrl(src)
|
||||
}
|
||||
setHasMounted(1)
|
||||
})
|
||||
|
||||
function handleUrl(url) {
|
||||
// if url is external may have CORS issue so we proxy it ...
|
||||
if (url.startsWith('http')) {
|
||||
const PROXY_URL = window.location.origin + '/api/proxy'
|
||||
url = PROXY_URL + '?url=' + encodeURIComponent(url)
|
||||
}
|
||||
axios.get(url, {
|
||||
responseType: 'arraybuffer'
|
||||
}).then((res) => {
|
||||
let out = new Uint8Array(res.data)
|
||||
let workbook = XLSX.read(out, {type: "array"})
|
||||
// Get first worksheet
|
||||
const wsname = workbook.SheetNames[0]
|
||||
const ws = workbook.Sheets[wsname]
|
||||
// Convert array of arrays
|
||||
const datatmp = XLSX.utils.sheet_to_json(ws, {header:1})
|
||||
const colstmp = make_cols(ws['!ref'])
|
||||
setData(datatmp)
|
||||
setCols(colstmp)
|
||||
setWorkbook(workbook)
|
||||
}).catch((e) => {
|
||||
setError(e.message)
|
||||
})
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
{error &&
|
||||
<div>
|
||||
There was an error loading the excel file at {src}:
|
||||
<p>{error}</p>
|
||||
</div>
|
||||
}
|
||||
{workbook &&
|
||||
<ul>
|
||||
{workbook.SheetNames.map((value, index) => {
|
||||
return <li key={index}>{value}</li>
|
||||
})}
|
||||
</ul>
|
||||
}
|
||||
<Table data={data} cols={cols} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
/* generate an array of column objects */
|
||||
const make_cols = refstr => {
|
||||
let o = [], C = XLSX.utils.decode_range(refstr).e.c + 1
|
||||
for(var i = 0; i < C; ++i) o[i] = {name:XLSX.utils.encode_col(i), key:i}
|
||||
return o
|
||||
}
|
||||
|
||||
186
site/components/ExcelViewerApp.js
Normal file
186
site/components/ExcelViewerApp.js
Normal file
@ -0,0 +1,186 @@
|
||||
import XLSX from 'xlsx';
|
||||
import React from 'react';
|
||||
|
||||
function SheetJSApp() {
|
||||
const [data, setData] = React.useState([]);
|
||||
const [cols, setCols] = React.useState([]);
|
||||
|
||||
const handleFile = (file) => {
|
||||
const reader = new FileReader();
|
||||
const rABS = !!reader.readAsBinaryString;
|
||||
reader.onload = (e) => {
|
||||
/* Parse data */
|
||||
const bstr = e.target.result;
|
||||
const wb = XLSX.read(bstr, {type:rABS ? 'binary' : 'array'});
|
||||
displayWorkbook(wb);
|
||||
};
|
||||
if(rABS) reader.readAsBinaryString(file); else reader.readAsArrayBuffer(file);
|
||||
}
|
||||
|
||||
const handleUrl = (url) => {
|
||||
let oReq = new XMLHttpRequest();
|
||||
oReq.open("GET", url, true);
|
||||
oReq.responseType = "arraybuffer";
|
||||
oReq.onload = function (e) {
|
||||
let arraybuffer = oReq.response;
|
||||
/* not responseText!! */
|
||||
|
||||
/* convert data to binary string */
|
||||
let data = new Uint8Array(arraybuffer);
|
||||
let arr = new Array();
|
||||
for (let i = 0; i != data.length; ++i) arr[i] = String.fromCharCode(data[i]);
|
||||
let bstr = arr.join("");
|
||||
/* Call XLSX */
|
||||
let workbook = XLSX.read(bstr, {type: "binary"});
|
||||
displayWorkbook(workbook);
|
||||
};
|
||||
|
||||
oReq.send();
|
||||
}
|
||||
|
||||
const displayWorkbook = (wb) => {
|
||||
/* Get first worksheet */
|
||||
const wsname = wb.SheetNames[0];
|
||||
const ws = wb.Sheets[wsname];
|
||||
/t Convert array of arrays */
|
||||
const data = XLSX.utils.sheet_to_json(ws, {header:1});
|
||||
/* Update state */
|
||||
setData(data);
|
||||
setCols(make_cols(ws['!ref']));
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<DragDropFile handleFile={handleFile}>
|
||||
<h2>Drag or choose a spreadsheet file</h2>
|
||||
<div className="">
|
||||
<DataInput handleFile={handleFile} />
|
||||
</div>
|
||||
</DragDropFile>
|
||||
<div className="mb-6">
|
||||
<h3>Enter spreadsheet URL</h3>
|
||||
<UrlInput handleUrl={handleUrl} />
|
||||
</div>
|
||||
<div className="row">
|
||||
<OutTable data={data} cols={cols} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if(typeof module !== 'undefined') module.exports = SheetJSApp
|
||||
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
||||
/*
|
||||
Simple HTML5 file drag-and-drop wrapper
|
||||
usage: <DragDropFile handleFile={handleFile}>...</DragDropFile>
|
||||
handleFile(file:File):void;
|
||||
*/
|
||||
|
||||
function DragDropFile({ handleFile, children }) {
|
||||
const suppress = (e) => { e.stopPropagation(); e.preventDefault(); };
|
||||
const handleDrop = (e) => { e.stopPropagation(); e.preventDefault();
|
||||
const files = e.dataTransfer.files;
|
||||
if(files && files[0]) handleFile(files[0]);
|
||||
};
|
||||
|
||||
return (
|
||||
<div
|
||||
onDrop={handleDrop}
|
||||
onDragEnter={suppress}
|
||||
onDragOver={suppress}
|
||||
>
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function UrlInput({ handleUrl }) {
|
||||
const handleChange = (e) => {
|
||||
const url = e.target.value;
|
||||
if(url) handleUrl(url);
|
||||
};
|
||||
|
||||
return (
|
||||
<form className="form-inline">
|
||||
<div className="form-group">
|
||||
<label htmlFor="url">Spreadsheet URL (with CORS enabled!)</label>
|
||||
<br />
|
||||
<small>Here is one: http://localhost:3000/_files/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx</small>
|
||||
<br />
|
||||
<input
|
||||
type="text"
|
||||
id="url"
|
||||
className="border w-96"
|
||||
accept={SheetJSFT}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
</div>
|
||||
</form>
|
||||
)
|
||||
}
|
||||
|
||||
/*
|
||||
Simple HTML5 file input wrapper
|
||||
usage: <DataInput handleFile={callback} />
|
||||
handleFile(file:File):void;
|
||||
*/
|
||||
|
||||
function DataInput({ handleFile }) {
|
||||
const handleChange = (e) => {
|
||||
const files = e.target.files;
|
||||
if(files && files[0]) handleFile(files[0]);
|
||||
};
|
||||
|
||||
return (
|
||||
<form className="form-inline">
|
||||
<div className="form-group">
|
||||
<label htmlFor="file">Select spreadsheet file</label>
|
||||
<br />
|
||||
<input
|
||||
type="file"
|
||||
className="form-control"
|
||||
id="file"
|
||||
accept={SheetJSFT}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
</div>
|
||||
</form>
|
||||
)
|
||||
}
|
||||
|
||||
/*
|
||||
Simple HTML Table
|
||||
usage: <OutTable data={data} cols={cols} />
|
||||
data:Array<Array<any> >;
|
||||
cols:Array<{name:string, key:number|string}>;
|
||||
*/
|
||||
function OutTable({ data, cols }) {
|
||||
return (
|
||||
<div className="table-responsive">
|
||||
<table className="table table-striped">
|
||||
<thead>
|
||||
<tr>{cols.map((c) => <th key={c.key}>{c.name}</th>)}</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{data.map((r,i) => <tr key={i}>
|
||||
{cols.map(c => <td key={c.key}>{ r[c.key] }</td>)}
|
||||
</tr>)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* list of supported file types */
|
||||
const SheetJSFT = [
|
||||
"xlsx", "xlsb", "xlsm", "xls", "xml", "csv", "txt", "ods", "fods", "uos", "sylk", "dif", "dbf", "prn", "qpw", "123", "wb*", "wq*", "html", "htm"
|
||||
].map(x => `.${x}`).join(",");
|
||||
|
||||
/* generate an array of column objects */
|
||||
const make_cols = refstr => {
|
||||
let o = [], C = XLSX.utils.decode_range(refstr).e.c + 1;
|
||||
for(var i = 0; i < C; ++i) o[i] = {name:XLSX.utils.encode_col(i), key:i}
|
||||
return o;
|
||||
};
|
||||
33
site/components/LineChart.js
Normal file
33
site/components/LineChart.js
Normal file
@ -0,0 +1,33 @@
|
||||
import { Vega, VegaLite } from 'react-vega'
|
||||
|
||||
export default function LineChart( { data=[] }) {
|
||||
var tmp = data
|
||||
if (Array.isArray(data)) {
|
||||
tmp = data.map((r,i) => {
|
||||
return { x: r[0], y: r[1] }
|
||||
})
|
||||
}
|
||||
const vegaData = { "table": tmp }
|
||||
const spec = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
|
||||
"mark": "line",
|
||||
"data": {
|
||||
"name": "table"
|
||||
},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": "x",
|
||||
"timeUnit": "year",
|
||||
"type": "temporal"
|
||||
},
|
||||
"y": {
|
||||
"field": "y",
|
||||
"type": "quantitative"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<VegaLite data={ vegaData } spec={ spec } />
|
||||
)
|
||||
}
|
||||
92
site/components/Nav.js
Normal file
92
site/components/Nav.js
Normal file
@ -0,0 +1,92 @@
|
||||
import { Fragment } from 'react'
|
||||
import { Disclosure, Menu, Transition } from '@headlessui/react'
|
||||
import { BellIcon, MenuIcon, XIcon } from '@heroicons/react/outline'
|
||||
|
||||
import Link from 'next/link'
|
||||
|
||||
const navigation = [
|
||||
{ name: 'Docs', href: '/docs' },
|
||||
{ name: 'Components', href: '/docs/components' },
|
||||
{ name: 'Learn', href: '/learn' },
|
||||
{ name: 'Gallery', href: '/gallery' },
|
||||
{ name: 'Data Literate', href: '/data-literate/', current: false },
|
||||
{ name: 'DL Demo', href: '/data-literate/demo/', current: false },
|
||||
{ name: 'Excel Viewer', href: '/excel-viewer/', current: false },
|
||||
{ name: 'Github', href: 'https://github.com/datopian/portal.js', current: false },
|
||||
]
|
||||
|
||||
function classNames(...classes) {
|
||||
return classes.filter(Boolean).join(' ')
|
||||
}
|
||||
|
||||
export default function Nav() {
|
||||
return (
|
||||
<Disclosure as="nav" className="bg-gray-800">
|
||||
{({ open }) => (
|
||||
<>
|
||||
<div className="max-w-7xl mx-auto px-2 sm:px-6 lg:px-8">
|
||||
<div className="relative flex items-center justify-between h-16">
|
||||
<div className="absolute inset-y-0 left-0 flex items-center sm:hidden">
|
||||
{/* Mobile menu button*/}
|
||||
<Disclosure.Button className="inline-flex items-center justify-center p-2 rounded-md text-gray-400 hover:text-white hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-inset focus:ring-white">
|
||||
<span className="sr-only">Open main menu</span>
|
||||
{open ? (
|
||||
<XIcon className="block h-6 w-6" aria-hidden="true" />
|
||||
) : (
|
||||
<MenuIcon className="block h-6 w-6" aria-hidden="true" />
|
||||
)}
|
||||
</Disclosure.Button>
|
||||
</div>
|
||||
<div className="flex-1 flex items-center justify-center sm:items-stretch sm:justify-start">
|
||||
<div className="flex-shrink-0 flex items-center">
|
||||
<Link href="/">
|
||||
<a className="text-white">
|
||||
Portal.JS
|
||||
</a>
|
||||
</Link>
|
||||
</div>
|
||||
<div className="hidden sm:block sm:ml-6">
|
||||
<div className="flex space-x-4">
|
||||
{navigation.map((item) => (
|
||||
<Link href={item.href}>
|
||||
<a
|
||||
key={item.name}
|
||||
href={item.href}
|
||||
className={classNames(
|
||||
item.current ? 'bg-gray-900 text-white' : 'text-gray-300 hover:bg-gray-700 hover:text-white',
|
||||
'px-3 py-2 rounded-md text-sm font-medium'
|
||||
)}
|
||||
aria-current={item.current ? 'page' : undefined}
|
||||
>
|
||||
{item.name}
|
||||
</a>
|
||||
</Link>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<Disclosure.Panel className="sm:hidden">
|
||||
<div className="px-2 pt-2 pb-3 space-y-1">
|
||||
{navigation.map((item) => (
|
||||
<a
|
||||
key={item.name}
|
||||
href={item.href}
|
||||
className={classNames(
|
||||
item.current ? 'bg-gray-900 text-white' : 'text-gray-300 hover:bg-gray-700 hover:text-white',
|
||||
'block px-3 py-2 rounded-md text-base font-medium'
|
||||
)}
|
||||
aria-current={item.current ? 'page' : undefined}
|
||||
>
|
||||
{item.name}
|
||||
</a>
|
||||
))}
|
||||
</div>
|
||||
</Disclosure.Panel>
|
||||
</>
|
||||
)}
|
||||
</Disclosure>
|
||||
)
|
||||
}
|
||||
@ -1,52 +0,0 @@
|
||||
import { useState } from 'react'
|
||||
import Link from 'next/link'
|
||||
|
||||
export default function Nav() {
|
||||
const [open, setOpen] = useState(false);
|
||||
const handleClick = (event) => {
|
||||
event.preventDefault();
|
||||
setOpen(!open);
|
||||
};
|
||||
|
||||
const navMenu = [
|
||||
{ title: 'Docs', path: '/docs' },
|
||||
{ title: 'Components', path: '/docs/components' },
|
||||
{ title: 'Learn', path: '/learn' },
|
||||
{ title: 'Gallery', path: '/gallery' },
|
||||
{ title: 'Github', path: 'https://github.com/datopian/portal.js' }
|
||||
]
|
||||
|
||||
return (
|
||||
<nav className="flex items-center justify-between flex-wrap bg-white p-4 border-b border-gray-200">
|
||||
<div className="flex items-center flex-shrink-0 text-gray-700 mr-6">
|
||||
<Link href="/">
|
||||
<img src="/logo.svg" alt="portal logo" width="110" />
|
||||
</Link>
|
||||
</div>
|
||||
<div className="block lg:hidden mx-4">
|
||||
<button
|
||||
onClick={handleClick}
|
||||
className="flex items-center px-3 py-2 border rounded text-gray-700 border-orange-400 hover:text-black hover:border-black"
|
||||
>
|
||||
<svg
|
||||
className="fill-current h-3 w-3"
|
||||
viewBox="0 0 20 20"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<title>Menu</title>
|
||||
<path d="M0 3h20v2H0V3zm0 6h20v2H0V9zm0 6h20v2H0v-2z" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
<div className={`${open ? `block` : `hidden`} lg:block`}>
|
||||
{navMenu.map((menu, index) => {
|
||||
return (<Link href={menu.path} key={index}>
|
||||
<a className="block mt-4 lg:inline-block lg:mt-0 active:bg-primary-background text-gray-700 hover:text-black mr-6">
|
||||
{menu.title}
|
||||
</a>
|
||||
</Link>)
|
||||
})}
|
||||
</div>
|
||||
</nav>
|
||||
)
|
||||
}
|
||||
83
site/components/Table.js
Normal file
83
site/components/Table.js
Normal file
@ -0,0 +1,83 @@
|
||||
import axios from 'axios'
|
||||
import React, { useEffect, useState } from 'react'
|
||||
|
||||
const papa = require("papaparse")
|
||||
|
||||
/*
|
||||
Simple HTML Table
|
||||
usage: <OutTable data={data} cols={cols} />
|
||||
data:Array<Array<any> >;
|
||||
cols:Array<{name:string, key:number|string}>;
|
||||
*/
|
||||
export default function Table({ data=[], cols=[], csv='', url='' }) {
|
||||
if (csv) {
|
||||
const out = parseCsv(csv)
|
||||
data = out.rows
|
||||
cols = out.cols
|
||||
}
|
||||
|
||||
const [ourdata, setData] = React.useState(data)
|
||||
const [ourcols, setCols] = React.useState(cols)
|
||||
const [error, setError] = React.useState('')
|
||||
|
||||
useEffect(() => {
|
||||
if (url) {
|
||||
loadUrl(url)
|
||||
}
|
||||
}, [url])
|
||||
|
||||
function loadUrl(path) {
|
||||
// HACK: duplicate of Excel code - maybe refactor
|
||||
// if url is external may have CORS issue so we proxy it ...
|
||||
if (url.startsWith('http')) {
|
||||
const PROXY_URL = window.location.origin + '/api/proxy'
|
||||
url = PROXY_URL + '?url=' + encodeURIComponent(url)
|
||||
}
|
||||
axios.get(url).then((res) => {
|
||||
const { rows, fields } = parseCsv(res.data)
|
||||
setData(rows)
|
||||
setCols(fields)
|
||||
})
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<SimpleTable data={ourdata} cols={ourcols} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
/*
|
||||
Simple HTML Table
|
||||
usage: <OutTable data={data} cols={cols} />
|
||||
data:Array<Array<any> >;
|
||||
cols:Array<{name:string, key:number|string}>;
|
||||
*/
|
||||
function SimpleTable({ data=[], cols=[] }) {
|
||||
return (
|
||||
<div className="table-responsive">
|
||||
<table className="table table-striped">
|
||||
<thead>
|
||||
<tr>{cols.map((c) => <th key={c.key}>{c.name}</th>)}</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{data.map((r,i) => <tr key={i}>
|
||||
{cols.map(c => <td key={c.key}>{ r[c.key] }</td>)}
|
||||
</tr>)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function parseCsv(csv) {
|
||||
csv = csv.trim()
|
||||
const rawdata = papa.parse(csv, {header: true})
|
||||
const cols = rawdata.meta.fields.map((r,i) => {
|
||||
return { key: r, name: r }
|
||||
})
|
||||
return {
|
||||
rows: rawdata.data,
|
||||
fields: cols
|
||||
}
|
||||
}
|
||||
@ -3,22 +3,19 @@ import Head from 'next/head'
|
||||
|
||||
import Nav from '../components/Nav'
|
||||
|
||||
export default function Layout({
|
||||
children,
|
||||
title = 'Portal.JS',
|
||||
}) {
|
||||
export default function Layout({ children, title = 'Home' }) {
|
||||
return (
|
||||
<div>
|
||||
<>
|
||||
<Head>
|
||||
<title>{title}</title>
|
||||
<title>Portal.JS - {title}</title>
|
||||
<link rel="icon" href="/favicon.ico" />
|
||||
<meta charSet="utf-8" />
|
||||
<meta name="viewport" content="initial-scale=1.0, width=device-width" />
|
||||
</Head>
|
||||
<Nav />
|
||||
|
||||
{children}
|
||||
|
||||
<div className="prose mx-auto p-6">
|
||||
{children}
|
||||
</div>
|
||||
<footer className="flex items-center justify-center w-full h-24 border-t">
|
||||
<a
|
||||
className="flex items-center justify-center"
|
||||
@ -30,6 +27,6 @@ export default function Layout({
|
||||
<img src="/datopian-logo.png" alt="Datopian Logo" className="h-6 ml-2" />
|
||||
</a>
|
||||
</footer>
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
21
site/content/data-literate.md
Normal file
21
site/content/data-literate.md
Normal file
@ -0,0 +1,21 @@
|
||||
---
|
||||
title: Data Literate Documents
|
||||
author: Rufus Pollock and Friends
|
||||
---
|
||||
|
||||
**What?** An experiment in simple, lightweight approach to creating, displaying and sharing datasets and data-driven stories.
|
||||
|
||||
**Why?** a simple, fast, extensible way to present data(sets) and author data-driven content. I want to work with markdown for content and quickly add data in the simplest way possible e.g. dropping in links, pasting tables or adding links to the metadata.
|
||||
|
||||
**How?** Technically the essence is Markdown+React (MDX) + a curated toolkit of components for data-presentation + NextJS for framework and deployment.
|
||||
|
||||
Check out the [demo](/data-literate/demo).
|
||||
|
||||
## Background
|
||||
|
||||
I have observed two converging data-rich use cases:
|
||||
|
||||
* **Data Publishing**: quickly presenting data whether a single file or a full dataset.
|
||||
* **Data Stories**: creating data-driven content from the simplest of a blog post with a graph to high end there is sophisticated data journalism and visualization.
|
||||
|
||||
Both of these can now be well served by a simple markdown-plus approach. Taking data publishing first. I've long been a fan of ultra-simple `README + metadata + csv` datasets. With the evolution of frontmatter we can merge the metadata into the README. However, we still need to "present" the dataset and the key thing for a dataset is the data and this is not something markdown ever supported well ... But now with MDX and the richness of the javascript ecosystem it's quite easy to enhance our markdown and build a rendering pipeleine.
|
||||
268
site/content/data-literate/demo.mdx
Normal file
268
site/content/data-literate/demo.mdx
Normal file
@ -0,0 +1,268 @@
|
||||
---
|
||||
title: Demo
|
||||
---
|
||||
|
||||
This demos and documents Data Literate features live.
|
||||
|
||||
You can see the raw source of this page here: https://raw.githubusercontent.com/datopian/data-literate/main/content/demo.mdx
|
||||
|
||||
## Table of Contents
|
||||
|
||||
## GFM
|
||||
|
||||
We can have github-flavored markdown including markdown tables, auto-linked links and checklists:
|
||||
|
||||
```
|
||||
https://github.com/datopian/portal.js
|
||||
|
||||
| a | b |
|
||||
|---|---|
|
||||
| 1 | 2 |
|
||||
|
||||
* [x] one thing to do
|
||||
* [ ] a second thing to do
|
||||
```
|
||||
|
||||
https://github.com/datopian/portal.js
|
||||
|
||||
| a | b |
|
||||
|---|---|
|
||||
| 1 | 2 |
|
||||
|
||||
* [x] one thing to do
|
||||
* [ ] a second thing to do
|
||||
|
||||
## Footnotes
|
||||
|
||||
```
|
||||
here is a footnote reference[^1]
|
||||
|
||||
[^1]: a very interesting footnote.
|
||||
```
|
||||
|
||||
here is a footnote reference[^1]
|
||||
|
||||
[^1]: a very interesting footnote.
|
||||
|
||||
|
||||
## Frontmatter
|
||||
|
||||
Posts can have frontmatter like:
|
||||
|
||||
```
|
||||
---
|
||||
title: Hello World
|
||||
author: Rufus Pollock
|
||||
---
|
||||
```
|
||||
|
||||
The title and description are pulled from the MDX file and processed using `gray-matter`. Additionally, links are rendered using a custom component passed to `next-mdx-remote`.
|
||||
|
||||
## A Table of Contents
|
||||
|
||||
You can create a table of contents by having a markdown heading named `Table of Contents`. You can see an example at the start of this post.
|
||||
|
||||
|
||||
## A Table
|
||||
|
||||
You can create tables ...
|
||||
|
||||
```
|
||||
<Table cols={[
|
||||
{ key: 'id', name: 'ID' },
|
||||
{ key: 'firstName', name: 'First name' },
|
||||
{ key: 'lastName', name: 'Last name' },
|
||||
{ key: 'age', name: 'Age' }
|
||||
]} data={[
|
||||
{ id: 1, lastName: 'Snow', firstName: 'Jon', age: 35 },
|
||||
{ id: 2, lastName: 'Lannister', firstName: 'Cersei', age: 42 },
|
||||
{ id: 3, lastName: 'Lannister', firstName: 'Jaime', age: 45 },
|
||||
{ id: 4, lastName: 'Stark', firstName: 'Arya', age: 16 },
|
||||
{ id: 7, lastName: 'Clifford', firstName: 'Ferrara', age: 44 },
|
||||
{ id: 8, lastName: 'Frances', firstName: 'Rossini', age: 36 },
|
||||
{ id: 9, lastName: 'Roxie', firstName: 'Harvey', age: 65 },
|
||||
]}
|
||||
/>
|
||||
```
|
||||
|
||||
<Table cols={[
|
||||
{ key: 'id', name: 'ID' },
|
||||
{ key: 'firstName', name: 'First name' },
|
||||
{ key: 'lastName', name: 'Last name' },
|
||||
{ key: 'age', name: 'Age' }
|
||||
]} data={[
|
||||
{ id: 1, lastName: 'Snow', firstName: 'Jon', age: 35 },
|
||||
{ id: 2, lastName: 'Lannister', firstName: 'Cersei', age: 42 },
|
||||
{ id: 3, lastName: 'Lannister', firstName: 'Jaime', age: 45 },
|
||||
{ id: 4, lastName: 'Stark', firstName: 'Arya', age: 16 },
|
||||
{ id: 7, lastName: 'Clifford', firstName: 'Ferrara', age: 44 },
|
||||
{ id: 8, lastName: 'Frances', firstName: 'Rossini', age: 36 },
|
||||
{ id: 9, lastName: 'Roxie', firstName: 'Harvey', age: 65 },
|
||||
]}
|
||||
/>
|
||||
|
||||
### Table from Raw CSV
|
||||
|
||||
You can also pass raw CSV as the content ...
|
||||
|
||||
```
|
||||
<Table csv={`
|
||||
Year,Temp Anomaly
|
||||
1850,-0.418
|
||||
2020,0.923
|
||||
`} />
|
||||
```
|
||||
|
||||
<Table csv={`
|
||||
Year,Temp Anomaly,
|
||||
1850,-0.418
|
||||
2020,0.923
|
||||
`} />
|
||||
|
||||
### Table from a URL
|
||||
|
||||
<Table url='https://raw.githubusercontent.com/datopian/data-literate/main/public/_files/HadCRUT.5.0.1.0.analysis.summary_series.global.annual.csv' />
|
||||
|
||||
```
|
||||
<Table url='https://raw.githubusercontent.com/datopian/data-literate/main/public/_files/HadCRUT.5.0.1.0.analysis.summary_series.global.annual.csv' />
|
||||
```
|
||||
|
||||
## Charts
|
||||
|
||||
You can create charts using a simple syntax.
|
||||
|
||||
### Line Chart
|
||||
|
||||
<LineChart data={
|
||||
[
|
||||
["1850",-0.41765878],
|
||||
["1851",-0.2333498],
|
||||
["1852",-0.22939907],
|
||||
["1853",-0.27035445],
|
||||
["1854",-0.29163003]
|
||||
]
|
||||
}
|
||||
/>
|
||||
|
||||
```
|
||||
<LineChart data={
|
||||
[
|
||||
["1850",-0.41765878],
|
||||
["1851",-0.2333498],
|
||||
["1852",-0.22939907],
|
||||
["1853",-0.27035445],
|
||||
["1854",-0.29163003]
|
||||
]
|
||||
}
|
||||
/>
|
||||
```
|
||||
|
||||
NB: we have quoted years as otherwise not interpreted as dates but as integers ...
|
||||
|
||||
|
||||
### Vega and Vega Lite
|
||||
|
||||
You can using vega or vega-lite. Here's an example using vega-lite:
|
||||
|
||||
<VegaLite data={ { "table": [
|
||||
{
|
||||
"y": -0.418,
|
||||
"x": 1850
|
||||
},
|
||||
{
|
||||
"y": 0.923,
|
||||
"x": 2020
|
||||
}
|
||||
]
|
||||
}
|
||||
} spec={
|
||||
{
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"mark": "bar",
|
||||
"data": {
|
||||
"name": "table"
|
||||
},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": "x",
|
||||
"type": "ordinal"
|
||||
},
|
||||
"y": {
|
||||
"field": "y",
|
||||
"type": "quantitative"
|
||||
}
|
||||
}
|
||||
}
|
||||
} />
|
||||
|
||||
|
||||
```jsx
|
||||
<VegaLite data={ { "table": [
|
||||
{
|
||||
"y": -0.418,
|
||||
"x": 1850
|
||||
},
|
||||
{
|
||||
"y": 0.923,
|
||||
"x": 2020
|
||||
}
|
||||
]
|
||||
}
|
||||
} spec={
|
||||
{
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"mark": "bar",
|
||||
"data": {
|
||||
"name": "table"
|
||||
},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": "x",
|
||||
"type": "ordinal"
|
||||
},
|
||||
"y": {
|
||||
"field": "y",
|
||||
"type": "quantitative"
|
||||
}
|
||||
}
|
||||
}
|
||||
} />
|
||||
|
||||
```
|
||||
|
||||
#### Line Chart from URL with Tooltip
|
||||
|
||||
https://vega.github.io/vega-lite/examples/interactive_multi_line_pivot_tooltip.html
|
||||
|
||||
<VegaLite spec={
|
||||
{
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
|
||||
"data": {"url": "_files/HadCRUT.5.0.1.0.analysis.summary_series.global.annual.csv"},
|
||||
"width": 600,
|
||||
"height": 250,
|
||||
"mark": "line",
|
||||
"encoding": {
|
||||
"x": {"field": "Time", "type": "temporal"},
|
||||
"y": {"field": "Anomaly (deg C)", "type": "quantitative"},
|
||||
"tooltip": {"field": "Anomaly (deg C)", "type": "quantitative"}
|
||||
}
|
||||
}
|
||||
} />
|
||||
|
||||
## Display Excel Files
|
||||
|
||||
Local file ...
|
||||
|
||||
```
|
||||
<Excel src='/_files/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx' />
|
||||
```
|
||||
|
||||
<Excel src='/_files/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx' />
|
||||
|
||||
Remote files work too (even without CORS) thanks to proxying:
|
||||
|
||||
```
|
||||
<Excel src='https://github.com/datasets/awesome-data/files/6604635/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx' />
|
||||
```
|
||||
|
||||
<Excel src='https://github.com/datasets/awesome-data/files/6604635/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx' />
|
||||
33
site/lib/markdown.js
Normal file
33
site/lib/markdown.js
Normal file
@ -0,0 +1,33 @@
|
||||
import matter from 'gray-matter'
|
||||
import toc from 'remark-toc'
|
||||
import slug from 'remark-slug'
|
||||
import gfm from 'remark-gfm'
|
||||
import footnotes from 'remark-footnotes'
|
||||
|
||||
import { serialize } from 'next-mdx-remote/serialize'
|
||||
|
||||
/**
|
||||
* Parse a markdown or MDX file to an MDX source form + front matter data
|
||||
*
|
||||
* @source: the contents of a markdown or mdx file
|
||||
* @returns: { mdxSource: mdxSource, frontMatter: ...}
|
||||
*/
|
||||
const parse = async function(source) {
|
||||
const { content, data } = matter(source)
|
||||
|
||||
const mdxSource = await serialize(content, {
|
||||
// Optionally pass remark/rehype plugins
|
||||
mdxOptions: {
|
||||
remarkPlugins: [gfm, toc, slug, footnotes],
|
||||
rehypePlugins: [],
|
||||
},
|
||||
scope: data,
|
||||
})
|
||||
|
||||
return {
|
||||
mdxSource: mdxSource,
|
||||
frontMatter: data
|
||||
}
|
||||
}
|
||||
|
||||
export default parse
|
||||
23
site/lib/mdxUtils.js
Normal file
23
site/lib/mdxUtils.js
Normal file
@ -0,0 +1,23 @@
|
||||
import fs from 'fs'
|
||||
import glob from 'glob'
|
||||
import path from 'path'
|
||||
|
||||
// POSTS_PATH is useful when you want to get the path to a specific file
|
||||
export const POSTS_PATH = path.join(process.cwd(), 'content')
|
||||
|
||||
const walkSync = (dir, filelist = []) => {
|
||||
fs.readdirSync(dir).forEach(file => {
|
||||
|
||||
filelist = fs.statSync(path.join(dir, file)).isDirectory()
|
||||
? walkSync(path.join(dir, file), filelist)
|
||||
: filelist.concat(path.join(dir, file))
|
||||
|
||||
})
|
||||
return filelist
|
||||
}
|
||||
|
||||
// postFilePaths is the list of all mdx files inside the POSTS_PATH directory
|
||||
export const postFilePaths = walkSync(POSTS_PATH)
|
||||
.map((file) => { return file.slice(POSTS_PATH.length) })
|
||||
// Only include md(x) files
|
||||
.filter((path) => /\.mdx?$/.test(path))
|
||||
@ -8,17 +8,29 @@
|
||||
"start": "next start"
|
||||
},
|
||||
"dependencies": {
|
||||
"@headlessui/react": "^1.3.0",
|
||||
"@heroicons/react": "^1.0.3",
|
||||
"@mdx-js/loader": "^1.6.22",
|
||||
"@next/mdx": "^10.2.0",
|
||||
"@tailwindcss/typography": "^0.4.0",
|
||||
"autoprefixer": "^10.0.4",
|
||||
"frictionless.js": "^0.13.4",
|
||||
"gray-matter": "^4.0.3",
|
||||
"next": "10.2.0",
|
||||
"next-mdx-remote": "^3.0.4",
|
||||
"papaparse": "^5.3.1",
|
||||
"postcss": "^8.2.10",
|
||||
"react": "17.0.2",
|
||||
"react-dom": "17.0.2",
|
||||
"react-vega": "^7.4.3",
|
||||
"remark": "^13.0.0",
|
||||
"remark-footnotes": "^3.0.0",
|
||||
"remark-gfm": "^1.0.0",
|
||||
"remark-html": "^13.0.1",
|
||||
"tailwindcss": "^2.0.2"
|
||||
"remark-slug": "^6.1.0",
|
||||
"remark-toc": "^7.2.0",
|
||||
"tailwindcss": "^2.0.2",
|
||||
"vega": "^5.20.2",
|
||||
"vega-lite": "^5.1.0",
|
||||
"xlsx": "^0.17.0"
|
||||
}
|
||||
}
|
||||
|
||||
48
site/pages/[...slug].js
Normal file
48
site/pages/[...slug].js
Normal file
@ -0,0 +1,48 @@
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
|
||||
import parse from '../lib/markdown.js'
|
||||
|
||||
import DataLiterate from '../components/DataLiterate'
|
||||
|
||||
import { postFilePaths, POSTS_PATH } from '../lib/mdxUtils'
|
||||
|
||||
|
||||
export default function PostPage({ source, frontMatter }) {
|
||||
return (
|
||||
<DataLiterate source={source} frontMatter={frontMatter} />
|
||||
)
|
||||
}
|
||||
|
||||
export const getStaticProps = async ({ params }) => {
|
||||
const mdxPath = path.join(POSTS_PATH, `${params.slug.join('/')}.mdx`)
|
||||
const postFilePath = fs.existsSync(mdxPath) ? mdxPath : mdxPath.slice(0, -1)
|
||||
const source = fs.readFileSync(postFilePath)
|
||||
|
||||
const { mdxSource, frontMatter } = await parse(source)
|
||||
|
||||
return {
|
||||
props: {
|
||||
source: mdxSource,
|
||||
frontMatter: frontMatter,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const getStaticPaths = async () => {
|
||||
var paths = postFilePaths
|
||||
// Remove file extensions for page paths
|
||||
.map((path) => path.replace(/\.mdx?$/, ''))
|
||||
|
||||
// Map the path into the static paths object required by Next.js
|
||||
paths = paths.map((slug) => {
|
||||
// /demo => [demo]
|
||||
const parts = slug.slice(1).split('/')
|
||||
return { params: { slug: parts } }
|
||||
})
|
||||
|
||||
return {
|
||||
paths,
|
||||
fallback: false,
|
||||
}
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
|
||||
|
||||
export default (req, res) => {
|
||||
res.status(200).json({ name: 'John Doe' })
|
||||
}
|
||||
26
site/pages/api/proxy.js
Normal file
26
site/pages/api/proxy.js
Normal file
@ -0,0 +1,26 @@
|
||||
import axios from 'axios'
|
||||
|
||||
export default function handler(req, res) {
|
||||
if (!req.query.url) {
|
||||
res.status(200).send({
|
||||
error: true,
|
||||
info: 'No url to proxy in query string i.e. ?url=...'
|
||||
})
|
||||
return
|
||||
}
|
||||
axios({
|
||||
method: 'get',
|
||||
url: req.query.url,
|
||||
responseType:'stream'
|
||||
})
|
||||
.then(resp => {
|
||||
resp.data.pipe(res)
|
||||
})
|
||||
.catch(err => {
|
||||
res.status(400).send({
|
||||
error: true,
|
||||
info: err.message,
|
||||
detailed: err
|
||||
})
|
||||
})
|
||||
}
|
||||
12
site/pages/excel-viewer.js
Normal file
12
site/pages/excel-viewer.js
Normal file
@ -0,0 +1,12 @@
|
||||
import Head from 'next/head'
|
||||
import SheetJSApp from '../components/ExcelViewerApp.js'
|
||||
import Layout from '../components/Layout'
|
||||
|
||||
export default function Index() {
|
||||
return (
|
||||
<Layout title='Excel Viewer'>
|
||||
<h1>Excel Viewer</h1>
|
||||
<SheetJSApp />
|
||||
</Layout>
|
||||
)
|
||||
}
|
||||
Binary file not shown.
1047
site/yarn.lock
1047
site/yarn.lock
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user