+
)
}
diff --git a/site/content/data-literate.md b/site/content/data-literate.md
new file mode 100644
index 00000000..3901566e
--- /dev/null
+++ b/site/content/data-literate.md
@@ -0,0 +1,21 @@
+---
+title: Data Literate Documents
+author: Rufus Pollock and Friends
+---
+
+**What?** An experiment in simple, lightweight approach to creating, displaying and sharing datasets and data-driven stories.
+
+**Why?** a simple, fast, extensible way to present data(sets) and author data-driven content. I want to work with markdown for content and quickly add data in the simplest way possible e.g. dropping in links, pasting tables or adding links to the metadata.
+
+**How?** Technically the essence is Markdown+React (MDX) + a curated toolkit of components for data-presentation + NextJS for framework and deployment.
+
+Check out the [demo](/data-literate/demo).
+
+## Background
+
+I have observed two converging data-rich use cases:
+
+* **Data Publishing**: quickly presenting data whether a single file or a full dataset.
+* **Data Stories**: creating data-driven content from the simplest of a blog post with a graph to high end there is sophisticated data journalism and visualization.
+
+Both of these can now be well served by a simple markdown-plus approach. Taking data publishing first. I've long been a fan of ultra-simple `README + metadata + csv` datasets. With the evolution of frontmatter we can merge the metadata into the README. However, we still need to "present" the dataset and the key thing for a dataset is the data and this is not something markdown ever supported well ... But now with MDX and the richness of the javascript ecosystem it's quite easy to enhance our markdown and build a rendering pipeleine.
diff --git a/site/content/data-literate/demo.mdx b/site/content/data-literate/demo.mdx
new file mode 100644
index 00000000..0ae01850
--- /dev/null
+++ b/site/content/data-literate/demo.mdx
@@ -0,0 +1,268 @@
+---
+title: Demo
+---
+
+This demos and documents Data Literate features live.
+
+You can see the raw source of this page here: https://raw.githubusercontent.com/datopian/data-literate/main/content/demo.mdx
+
+## Table of Contents
+
+## GFM
+
+We can have github-flavored markdown including markdown tables, auto-linked links and checklists:
+
+```
+https://github.com/datopian/portal.js
+
+| a | b |
+|---|---|
+| 1 | 2 |
+
+* [x] one thing to do
+* [ ] a second thing to do
+```
+
+https://github.com/datopian/portal.js
+
+| a | b |
+|---|---|
+| 1 | 2 |
+
+* [x] one thing to do
+* [ ] a second thing to do
+
+## Footnotes
+
+```
+here is a footnote reference[^1]
+
+[^1]: a very interesting footnote.
+```
+
+here is a footnote reference[^1]
+
+[^1]: a very interesting footnote.
+
+
+## Frontmatter
+
+Posts can have frontmatter like:
+
+```
+---
+title: Hello World
+author: Rufus Pollock
+---
+```
+
+The title and description are pulled from the MDX file and processed using `gray-matter`. Additionally, links are rendered using a custom component passed to `next-mdx-remote`.
+
+## A Table of Contents
+
+You can create a table of contents by having a markdown heading named `Table of Contents`. You can see an example at the start of this post.
+
+
+## A Table
+
+You can create tables ...
+
+```
+
+```
+
+
+
+### Table from Raw CSV
+
+You can also pass raw CSV as the content ...
+
+```
+
+```
+
+
+
+### Table from a URL
+
+
+
+```
+
+```
+
+## Charts
+
+You can create charts using a simple syntax.
+
+### Line Chart
+
+
+
+```
+
+```
+
+NB: we have quoted years as otherwise not interpreted as dates but as integers ...
+
+
+### Vega and Vega Lite
+
+You can using vega or vega-lite. Here's an example using vega-lite:
+
+
+
+
+```jsx
+
+
+```
+
+#### Line Chart from URL with Tooltip
+
+https://vega.github.io/vega-lite/examples/interactive_multi_line_pivot_tooltip.html
+
+
+
+## Display Excel Files
+
+Local file ...
+
+```
+
+```
+
+
+
+Remote files work too (even without CORS) thanks to proxying:
+
+```
+
+```
+
+
diff --git a/site/lib/markdown.js b/site/lib/markdown.js
new file mode 100644
index 00000000..c4d3ceeb
--- /dev/null
+++ b/site/lib/markdown.js
@@ -0,0 +1,33 @@
+import matter from 'gray-matter'
+import toc from 'remark-toc'
+import slug from 'remark-slug'
+import gfm from 'remark-gfm'
+import footnotes from 'remark-footnotes'
+
+import { serialize } from 'next-mdx-remote/serialize'
+
+/**
+ * Parse a markdown or MDX file to an MDX source form + front matter data
+ *
+ * @source: the contents of a markdown or mdx file
+ * @returns: { mdxSource: mdxSource, frontMatter: ...}
+ */
+const parse = async function(source) {
+ const { content, data } = matter(source)
+
+ const mdxSource = await serialize(content, {
+ // Optionally pass remark/rehype plugins
+ mdxOptions: {
+ remarkPlugins: [gfm, toc, slug, footnotes],
+ rehypePlugins: [],
+ },
+ scope: data,
+ })
+
+ return {
+ mdxSource: mdxSource,
+ frontMatter: data
+ }
+}
+
+export default parse
diff --git a/site/lib/mdxUtils.js b/site/lib/mdxUtils.js
new file mode 100644
index 00000000..d89b09f4
--- /dev/null
+++ b/site/lib/mdxUtils.js
@@ -0,0 +1,23 @@
+import fs from 'fs'
+import glob from 'glob'
+import path from 'path'
+
+// POSTS_PATH is useful when you want to get the path to a specific file
+export const POSTS_PATH = path.join(process.cwd(), 'content')
+
+const walkSync = (dir, filelist = []) => {
+ fs.readdirSync(dir).forEach(file => {
+
+ filelist = fs.statSync(path.join(dir, file)).isDirectory()
+ ? walkSync(path.join(dir, file), filelist)
+ : filelist.concat(path.join(dir, file))
+
+ })
+ return filelist
+}
+
+// postFilePaths is the list of all mdx files inside the POSTS_PATH directory
+export const postFilePaths = walkSync(POSTS_PATH)
+ .map((file) => { return file.slice(POSTS_PATH.length) })
+ // Only include md(x) files
+ .filter((path) => /\.mdx?$/.test(path))
diff --git a/site/package.json b/site/package.json
index b253f36e..f4d935d7 100644
--- a/site/package.json
+++ b/site/package.json
@@ -8,17 +8,29 @@
"start": "next start"
},
"dependencies": {
+ "@headlessui/react": "^1.3.0",
+ "@heroicons/react": "^1.0.3",
"@mdx-js/loader": "^1.6.22",
- "@next/mdx": "^10.2.0",
"@tailwindcss/typography": "^0.4.0",
"autoprefixer": "^10.0.4",
"frictionless.js": "^0.13.4",
+ "gray-matter": "^4.0.3",
"next": "10.2.0",
+ "next-mdx-remote": "^3.0.4",
+ "papaparse": "^5.3.1",
"postcss": "^8.2.10",
"react": "17.0.2",
"react-dom": "17.0.2",
+ "react-vega": "^7.4.3",
"remark": "^13.0.0",
+ "remark-footnotes": "^3.0.0",
+ "remark-gfm": "^1.0.0",
"remark-html": "^13.0.1",
- "tailwindcss": "^2.0.2"
+ "remark-slug": "^6.1.0",
+ "remark-toc": "^7.2.0",
+ "tailwindcss": "^2.0.2",
+ "vega": "^5.20.2",
+ "vega-lite": "^5.1.0",
+ "xlsx": "^0.17.0"
}
}
diff --git a/site/pages/[...slug].js b/site/pages/[...slug].js
new file mode 100644
index 00000000..0370be0a
--- /dev/null
+++ b/site/pages/[...slug].js
@@ -0,0 +1,48 @@
+import fs from 'fs'
+import path from 'path'
+
+import parse from '../lib/markdown.js'
+
+import DataLiterate from '../components/DataLiterate'
+
+import { postFilePaths, POSTS_PATH } from '../lib/mdxUtils'
+
+
+export default function PostPage({ source, frontMatter }) {
+ return (
+
+ )
+}
+
+export const getStaticProps = async ({ params }) => {
+ const mdxPath = path.join(POSTS_PATH, `${params.slug.join('/')}.mdx`)
+ const postFilePath = fs.existsSync(mdxPath) ? mdxPath : mdxPath.slice(0, -1)
+ const source = fs.readFileSync(postFilePath)
+
+ const { mdxSource, frontMatter } = await parse(source)
+
+ return {
+ props: {
+ source: mdxSource,
+ frontMatter: frontMatter,
+ },
+ }
+}
+
+export const getStaticPaths = async () => {
+ var paths = postFilePaths
+ // Remove file extensions for page paths
+ .map((path) => path.replace(/\.mdx?$/, ''))
+
+ // Map the path into the static paths object required by Next.js
+ paths = paths.map((slug) => {
+ // /demo => [demo]
+ const parts = slug.slice(1).split('/')
+ return { params: { slug: parts } }
+ })
+
+ return {
+ paths,
+ fallback: false,
+ }
+}
diff --git a/site/pages/api/hello.js b/site/pages/api/hello.js
deleted file mode 100644
index 9987aff4..00000000
--- a/site/pages/api/hello.js
+++ /dev/null
@@ -1,5 +0,0 @@
-// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
-
-export default (req, res) => {
- res.status(200).json({ name: 'John Doe' })
-}
diff --git a/site/pages/api/proxy.js b/site/pages/api/proxy.js
new file mode 100644
index 00000000..5cd851dd
--- /dev/null
+++ b/site/pages/api/proxy.js
@@ -0,0 +1,26 @@
+import axios from 'axios'
+
+export default function handler(req, res) {
+ if (!req.query.url) {
+ res.status(200).send({
+ error: true,
+ info: 'No url to proxy in query string i.e. ?url=...'
+ })
+ return
+ }
+ axios({
+ method: 'get',
+ url: req.query.url,
+ responseType:'stream'
+ })
+ .then(resp => {
+ resp.data.pipe(res)
+ })
+ .catch(err => {
+ res.status(400).send({
+ error: true,
+ info: err.message,
+ detailed: err
+ })
+ })
+}
diff --git a/site/pages/excel-viewer.js b/site/pages/excel-viewer.js
new file mode 100644
index 00000000..886bd241
--- /dev/null
+++ b/site/pages/excel-viewer.js
@@ -0,0 +1,12 @@
+import Head from 'next/head'
+import SheetJSApp from '../components/ExcelViewerApp.js'
+import Layout from '../components/Layout'
+
+export default function Index() {
+ return (
+
+