From 996568c0f9ce838e2fc45bc66ef382c90ab534fa Mon Sep 17 00:00:00 2001 From: Luccas Mateus de Medeiros Gomes Date: Sat, 22 Apr 2023 14:05:21 -0300 Subject: [PATCH 1/4] [simple-example][lg] - multiple datasets per repo are now possible --- examples/ckan-example/package.json | 109 +++++++++++++ examples/simple-example/.env | 1 + examples/simple-example/README.md | 59 ++++++- examples/simple-example/datasets.json | 47 +++++- .../simple-example/lib/loadUrlProxied.tsx | 11 -- examples/simple-example/lib/markdown.js | 105 ------------- examples/simple-example/lib/octokit.ts | 147 ++++++++++++++++++ examples/simple-example/lib/parseCsv.ts | 16 -- examples/simple-example/lib/project.ts | 60 ------- .../simple-example/lib/viewSpecConversion.ts | 47 ------ examples/simple-example/next.config.js | 14 +- .../pages/@org/[org]/[...path].tsx | 122 +++++++++++++++ .../pages/@org/[org]/[project]/index.tsx | 110 ------------- examples/simple-example/pages/_app.tsx | 1 - examples/simple-example/pages/api/proxy.ts | 26 ---- examples/simple-example/pages/index.tsx | 88 ++++------- examples/simple-example/styles/global.css | 67 -------- package-lock.json | 65 ++++++++ package.json | 1 + site/content/blog/example-data-catalog.md | 6 +- 20 files changed, 577 insertions(+), 525 deletions(-) create mode 100644 examples/ckan-example/package.json create mode 100644 examples/simple-example/.env delete mode 100644 examples/simple-example/lib/loadUrlProxied.tsx delete mode 100644 examples/simple-example/lib/markdown.js create mode 100644 examples/simple-example/lib/octokit.ts delete mode 100644 examples/simple-example/lib/parseCsv.ts delete mode 100644 examples/simple-example/lib/project.ts delete mode 100644 examples/simple-example/lib/viewSpecConversion.ts create mode 100644 examples/simple-example/pages/@org/[org]/[...path].tsx delete mode 100644 examples/simple-example/pages/@org/[org]/[project]/index.tsx delete mode 100644 examples/simple-example/pages/api/proxy.ts delete mode 100644 examples/simple-example/styles/global.css diff --git a/examples/ckan-example/package.json b/examples/ckan-example/package.json new file mode 100644 index 00000000..4c421d3e --- /dev/null +++ b/examples/ckan-example/package.json @@ -0,0 +1,109 @@ +{ + "name": "portaljs", + "version": "0.0.0", + "license": "MIT", + "scripts": {}, + "private": true, + "dependencies": { + "@apollo/client": "^3.7.11", + "@apollo/react-hooks": "^4.0.0", + "@emotion/react": "^11.10.6", + "@emotion/styled": "^11.10.6", + "@flowershow/core": "^0.4.9", + "@flowershow/markdowndb": "^0.1.0", + "@flowershow/remark-callouts": "^1.0.0", + "@flowershow/remark-embed": "^1.0.0", + "@flowershow/remark-wiki-link": "^1.0.1", + "@headlessui/react": "^1.7.13", + "@heroicons/react": "^2.0.17", + "@mui/icons-material": "^5.11.16", + "@mui/material": "^5.11.16", + "@mui/x-data-grid": "^6.1.0", + "@opentelemetry/api": "^1.4.0", + "@tailwindcss/typography": "^0.5.9", + "@tanstack/react-table": "^8.8.5", + "apollo-cache-inmemory": "^1.6.6", + "apollo-link": "^1.2.14", + "apollo-link-rest": "^0.9.0", + "filesize": "^10.0.7", + "gray-matter": "^4.0.3", + "html-react-parser": "^3.0.15", + "markdown-it": "^13.0.1", + "next": "^13.2.1", + "next-mdx-remote": "^4.4.1", + "next-seo": "^6.0.0", + "next-translate": "^2.0.5", + "nock": "^13.3.0", + "octokit": "^2.0.14", + "papaparse": "^5.4.1", + "plotly.js-basic-dist": "^2.20.0", + "prop-types": "^15.8.1", + "react": "18.2.0", + "react-dom": "18.2.0", + "react-next-github-btn": "^1.2.1", + "react-plotly.js": "^2.6.0", + "react-plotlyjs": "^0.4.4", + "react-vega": "^7.6.0", + "rehype-autolink-headings": "^6.1.1", + "rehype-katex": "^6.0.2", + "rehype-prism-plus": "^1.5.1", + "rehype-slug": "^5.1.0", + "remark-footnotes": "^4.0.1", + "remark-gfm": "^3.0.1", + "remark-math": "^5.1.1", + "remark-slug": "^7.0.1", + "remark-smartypants": "^2.0.0", + "remark-toc": "^8.0.1", + "slugify": "^1.6.6", + "timeago.js": "^4.0.2", + "tslib": "^2.3.0", + "vega": "^5.24.0", + "xlsx": "^0.18.5" + }, + "devDependencies": { + "@babel/preset-react": "^7.14.5", + "@nrwl/cypress": "15.9.2", + "@nrwl/eslint-plugin-nx": "15.9.2", + "@nrwl/jest": "15.9.2", + "@nrwl/js": "15.9.2", + "@nrwl/linter": "15.9.2", + "@nrwl/next": "^15.9.2", + "@nrwl/react": "15.9.2", + "@nrwl/rollup": "15.9.2", + "@nrwl/workspace": "15.9.2", + "@rollup/plugin-url": "^7.0.0", + "@svgr/rollup": "^6.1.2", + "@swc/core": "^1.2.173", + "@swc/helpers": "~0.5.0", + "@swc/jest": "0.2.20", + "@testing-library/react": "14.0.0", + "@types/jest": "^29.4.0", + "@types/node": "18.14.2", + "@types/react": "18.0.28", + "@types/react-dom": "18.0.11", + "@typescript-eslint/eslint-plugin": "^5.36.1", + "@typescript-eslint/parser": "^5.36.1", + "autoprefixer": "10.4.13", + "babel-jest": "^29.4.1", + "cypress": "^12.2.0", + "eslint": "~8.15.0", + "eslint-config-next": "13.1.1", + "eslint-config-prettier": "8.1.0", + "eslint-plugin-cypress": "^2.10.3", + "eslint-plugin-import": "2.27.5", + "eslint-plugin-jsx-a11y": "6.7.1", + "eslint-plugin-react": "7.32.2", + "eslint-plugin-react-hooks": "4.6.0", + "jest": "^29.4.1", + "jest-environment-jsdom": "^29.4.1", + "nx": "15.9.2", + "postcss": "8.4.21", + "prettier": "^2.6.2", + "react-test-renderer": "18.2.0", + "swc-loader": "0.1.15", + "tailwindcss": "3.2.7", + "ts-jest": "^29.0.5", + "ts-node": "10.9.1", + "typescript": "~4.9.5" + } +} diff --git a/examples/simple-example/.env b/examples/simple-example/.env new file mode 100644 index 00000000..67f0db30 --- /dev/null +++ b/examples/simple-example/.env @@ -0,0 +1 @@ +GITHUB_PAT=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 diff --git a/examples/simple-example/README.md b/examples/simple-example/README.md index 733b798b..f198eee4 100644 --- a/examples/simple-example/README.md +++ b/examples/simple-example/README.md @@ -1,17 +1,66 @@ This is a repo intended to serve as a simple example of a data catalog that get its data from a series of github repos, you can init an example just like this one by. -- Creating a new file inside o `examples` with `create-next-app` like so: +- Cloning the PortalJS repo on your machine + +``` +git clone https://github.com/datopian/portaljs.git +``` + +- Creating a new file inside the `examples` folder with `create-next-app` like so: + ``` npx create-next-app --example https://github.com/datopian/portaljs/tree/main/ --example-path examples/simple-example ``` + - Inside `` go to the `project.json` file and replace all instances of `simple-example` with `` +- Create a `.env` file with the following content + +``` +PROJECT_NAME= +``` + +- This project uses the github api, which for anonymous users will cap at 50 requests per hour, so you might want to get a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and add it to your .env file like so + +``` +PAT_TOKEN= +``` + - Edit the file `datasets.json` to your liking, some examples can be found inside this [repo](https://github.com/datasets) - Run the app using: + ``` nx serve ``` -Congratulations, you now have something similar to this running on `http://localhost:4200` -![](https://i.imgur.com/JrDLycF.png) -If yo go to any one of those pages by clicking on `More info` you will see something similar to this -![](https://i.imgur.com/cpKMS80.png) +Congratulations, you now have something similar to this running on `http://localhost:4200` +![](https://i.imgur.com/jAljJ9C.png) +If yo go to any one of those pages by clicking on `More info` you will see something similar to this +![](https://i.imgur.com/AoJd4O0.png) + + +## Structure of `datasets.json` + +The `datasets.json` file is simply a list of datasets, below you can see a minimal example of a dataset + +```json +{ + "owner": "fivethirtyeight", + "repo": "data", + "branch": "master", + "files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"], + "readme": "nba-raptor/README.md" +} +``` + +It has + +- A `owner` which is going to be the github repo owner +- A `repo` which is going to be the github repo name +- A `branch` which is going to be the branch to which we need to get the files and the readme +- A list of `files` which is going to be a list of paths with files that you want to show to the world +- A `readme` which is going to be the path to your data description, it can also be a subpath eg: `example/README.md` + +You can also add + +- A `description` which is useful if you have more than one dataset for each repo, if not provided we are just going to use the repo description +- A `Name` which is useful if you want to give your dataset a nice name, if not provided we are going to use the junction of the `owner` the `repo` + the path of the README, in the exaple above it will be `fivethirtyeight/data/nba-raptor` diff --git a/examples/simple-example/datasets.json b/examples/simple-example/datasets.json index 6263dc59..97a73c5b 100644 --- a/examples/simple-example/datasets.json +++ b/examples/simple-example/datasets.json @@ -1,7 +1,44 @@ [ - { "owner": "datasets", "repo": "oil-prices"}, - { "owner": "datasets", "repo": "investor-flow-of-funds-us"}, - { "owner": "datasets", "repo": "browser-stats"}, - { "owner": "datasets", "repo": "glacier-mass-balance"}, - { "owner": "datasets", "repo": "bond-yields-us-10y"} + { + "owner": "datasets", + "branch": "main", + "repo": "oil-prices", + "files": [ + "data/brent-daily.csv", + "data/brent-monthly.csv", + "data/brent-weekly.csv", + "data/brent-year.csv", + "data/wti-daily.csv", + "data/wti-monthly.csv", + "data/wti-weekly.csv", + "data/wti-year.csv" + ], + "readme": "README.md" + }, + { + "owner": "datasets", + "branch": "main", + "repo": "investor-flow-of-funds-us", + "files": [ + "data/monthly.csv", + "data/weekly.csv" + ], + "readme": "README.md" + }, + { + "owner": "fivethirtyeight", + "repo": "data", + "branch": "master", + "description": "Data about bad drivers", + "name": "Bad Drivers", + "files": ["bad-drivers/bad-drivers.csv"], + "readme": "bad-drivers/README.md" + }, + { + "owner": "fivethirtyeight", + "repo": "data", + "branch": "master", + "files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"], + "readme": "nba-raptor/README.md" + } ] diff --git a/examples/simple-example/lib/loadUrlProxied.tsx b/examples/simple-example/lib/loadUrlProxied.tsx deleted file mode 100644 index 82e371b1..00000000 --- a/examples/simple-example/lib/loadUrlProxied.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import axios from "axios"; - -export default function loadUrlProxied(url: string) { - // HACK: duplicate of Excel code - maybe refactor - // if url is external may have CORS issue so we proxy it ... - if (url.startsWith("http")) { - const PROXY_URL = "/api/proxy"; - url = PROXY_URL + "?url=" + encodeURIComponent(url); - } - return axios.get(url).then((res) => res.data); -} diff --git a/examples/simple-example/lib/markdown.js b/examples/simple-example/lib/markdown.js deleted file mode 100644 index ada9b8ab..00000000 --- a/examples/simple-example/lib/markdown.js +++ /dev/null @@ -1,105 +0,0 @@ -import matter from "gray-matter"; -import mdxmermaid from "mdx-mermaid"; -import { h } from "hastscript"; -import remarkCallouts from "@flowershow/remark-callouts"; -import remarkEmbed from "@flowershow/remark-embed"; -import remarkGfm from "remark-gfm"; -import remarkMath from "remark-math"; -import remarkSmartypants from "remark-smartypants"; -import remarkToc from "remark-toc"; -import remarkWikiLink from "@flowershow/remark-wiki-link"; -import rehypeAutolinkHeadings from "rehype-autolink-headings"; -import rehypeKatex from "rehype-katex"; -import rehypeSlug from "rehype-slug"; -import rehypePrismPlus from "rehype-prism-plus"; - -import { serialize } from "next-mdx-remote/serialize"; - -/** - * Parse a markdown or MDX file to an MDX source form + front matter data - * - * @source: the contents of a markdown or mdx file - * @format: used to indicate to next-mdx-remote which format to use (md or mdx) - * @returns: { mdxSource: mdxSource, frontMatter: ...} - */ -const parse = async function (source, format) { - const { content, data, excerpt } = matter(source, { - excerpt: (file, options) => { - // Generate an excerpt for the file - file.excerpt = file.content.split("\n\n")[0]; - }, - }); - - const mdxSource = await serialize( - { value: content, path: format }, - { - // Optionally pass remark/rehype plugins - mdxOptions: { - remarkPlugins: [ - remarkEmbed, - remarkGfm, - [remarkSmartypants, { quotes: false, dashes: "oldschool" }], - remarkMath, - remarkCallouts, - remarkWikiLink, - [ - remarkToc, - { - heading: "Table of contents", - tight: true, - }, - ], - [mdxmermaid, {}], - ], - rehypePlugins: [ - rehypeSlug, - [ - rehypeAutolinkHeadings, - { - properties: { className: 'heading-link' }, - test(element) { - return ( - ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) && - element.properties?.id !== "table-of-contents" && - element.properties?.className !== "blockquote-heading" - ); - }, - content() { - return [ - h( - "svg", - { - xmlns: "http:www.w3.org/2000/svg", - fill: "#ab2b65", - viewBox: "0 0 20 20", - className: "w-5 h-5", - }, - [ - h("path", { - fillRule: "evenodd", - clipRule: "evenodd", - d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z", - }), - ] - ), - ]; - }, - }, - ], - [rehypeKatex, { output: "mathml" }], - [rehypePrismPlus, { ignoreMissing: true }], - ], - format, - }, - scope: data, - } - ); - - return { - mdxSource: mdxSource, - frontMatter: data, - excerpt, - }; -}; - -export default parse; diff --git a/examples/simple-example/lib/octokit.ts b/examples/simple-example/lib/octokit.ts new file mode 100644 index 00000000..2d580ffc --- /dev/null +++ b/examples/simple-example/lib/octokit.ts @@ -0,0 +1,147 @@ +import { Octokit } from 'octokit'; + +export interface GithubProject { + owner: string; + repo: string; + branch: string; + files: string[]; + readme: string; + description?: string; + name?: string; +} + +export async function getProjectReadme( + owner: string, + repo: string, + branch: string, + readme: string, + github_pat?: string +) { + const octokit = new Octokit({ auth: github_pat }); + try { + const response = await octokit.rest.repos.getContent({ + owner, + repo, + path: readme, + ref: branch, + }); + const data = response.data as { content?: string }; + const fileContent = data.content ? data.content : ''; + if (fileContent === '') { + return null; + } + const decodedContent = Buffer.from(fileContent, 'base64').toString(); + return decodedContent; + } catch (error) { + console.log(error); + return null; + } +} + +export async function getLastUpdated( + owner: string, + repo: string, + branch: string, + readme: string, + github_pat?: string +) { + const octokit = new Octokit({ auth: github_pat }); + try { + const response = await octokit.rest.repos.listCommits({ + owner, + repo, + path: readme, + ref: branch, + }); + return response.data[0].commit.committer.date; + } catch (error) { + console.log(error); + return null; + } +} +export async function getProjectMetadata( + owner: string, + repo: string, + github_pat?: string +) { + const octokit = new Octokit({ auth: github_pat }); + try { + const response = await octokit.rest.repos.get({ + owner, + repo, + }); + return response.data; + } catch (error) { + console.log(error); + return null; + } +} + +export async function getRepoContents( + owner: string, + repo: string, + branch: string, + files: string[], + github_pat?: string +) { + const octokit = new Octokit({ auth: github_pat }); + try { + const contents = []; + for (const path of files) { + const response = await octokit.rest.repos.getContent({ + owner, + repo, + ref: branch, + path: path, + }); + const data = response.data as { download_url?: string, name: string, size: number }; + contents.push({ download_url: data.download_url, name: data.name, size: data.size}); + } + return contents; + } catch (error) { + console.log(error); + return null; + } +} + +export async function getProject(project: GithubProject, github_pat?: string) { + const projectMetadata = await getProjectMetadata( + project.owner, + project.repo, + github_pat + ); + if (!projectMetadata) { + return null; + } + const projectReadme = await getProjectReadme( + project.owner, + project.repo, + project.branch, + project.readme, + github_pat + ); + if (!projectReadme) { + return null; + } + const projectData = await getRepoContents( + project.owner, + project.repo, + project.branch, + project.files, + github_pat + ); + if (!projectData) { + return null; + } + const projectBase = project.readme.split('/').length > 1 + ? project.readme.split('/').slice(0, -1).join('/') + : '/' + const last_updated = await getLastUpdated( + project.owner, + project.repo, + project.branch, + projectBase, + github_pat + ); + return { ...projectMetadata, files: projectData, readmeContent: projectReadme, last_updated, base_path: projectBase }; +} diff --git a/examples/simple-example/lib/parseCsv.ts b/examples/simple-example/lib/parseCsv.ts deleted file mode 100644 index 28a56996..00000000 --- a/examples/simple-example/lib/parseCsv.ts +++ /dev/null @@ -1,16 +0,0 @@ -import papa from "papaparse"; - -const parseCsv = (csv) => { - csv = csv.trim(); - const rawdata = papa.parse(csv, { header: true }); - const cols = rawdata.meta.fields.map((r, i) => { - return { key: r, name: r }; - }); - - return { - rows: rawdata.data, - fields: cols, - }; -}; - -export default parseCsv; diff --git a/examples/simple-example/lib/project.ts b/examples/simple-example/lib/project.ts deleted file mode 100644 index 77529a20..00000000 --- a/examples/simple-example/lib/project.ts +++ /dev/null @@ -1,60 +0,0 @@ -import * as crypto from "crypto"; -import axios from "axios"; -import { Octokit } from "octokit" - -export default class Project { - id: string; - name: string; - owner: string; - github_repo: string; - readme: string; - metadata: any; - repo_metadata: any; - - constructor(owner: string, name: string) { - this.name = name; - this.owner = owner; - this.github_repo = `https://github.com/${owner}/${name}`; - - // TODO: using the GitHub repo to set the id is not a good idea - // since repos can be renamed and then we are going to end up with - // a duplicate - const encodedGHRepo = Buffer.from(this.github_repo, "utf-8").toString(); - this.id = crypto.createHash("sha1").update(encodedGHRepo).digest("hex"); - } - - initFromGitHub = async () => { - const octokit = new Octokit() - // TODO: what if the repo doesn't exist? - await this.getFileContent("README.md") - .then((content) => (this.readme = content)) - .catch((e) => (this.readme = null)); - - await this.getFileContent("datapackage.json") - .then((content) => (this.metadata = content)) - .catch((e) => (this.metadata = {})); - - const github_metadata = await octokit.rest.repos.get({ owner: this.owner, repo: this.name }) - this.repo_metadata = github_metadata.data ? github_metadata.data : null - }; - - getFileContent = (path, branch = "main") => { - return axios - .get( - `https://raw.githubusercontent.com/${this.owner}/${this.name}/${branch}/${path}` - ) - .then((res) => res.data); - }; - - serialize() { - return JSON.parse(JSON.stringify(this)); - } - - static async getFromGitHub(owner: string, name: string) { - const project = new Project(owner, name); - await project.initFromGitHub(); - - return project; - } -} - diff --git a/examples/simple-example/lib/viewSpecConversion.ts b/examples/simple-example/lib/viewSpecConversion.ts deleted file mode 100644 index cd6d3f33..00000000 --- a/examples/simple-example/lib/viewSpecConversion.ts +++ /dev/null @@ -1,47 +0,0 @@ -export function convertSimpleToVegaLite(view, resource) { - const x = resource.schema.fields.find((f) => f.name === view.spec.group); - const y = resource.schema.fields.find((f) => f.name === view.spec.series[0]); - - const xType = inferVegaType(x.type); - const yType = inferVegaType(y.type); - - let vegaLiteSpec = { - $schema: "https://vega.github.io/schema/vega-lite/v5.json", - mark: { - type: view.spec.type, - color: "black", - strokeWidth: 1, - tooltip: true, - }, - title: view.title, - width: 500, - height: 300, - selection: { - grid: { - type: "interval", - bind: "scales", - }, - }, - encoding: { - x: { - field: x.name, - type: xType, - }, - y: { - field: y.name, - type: yType, - }, - }, - }; - - return vegaLiteSpec; -} - -const inferVegaType = (fieldType) => { - switch (fieldType) { - case "date": - return "Temporal"; - case "number": - return "Quantitative"; - } -}; diff --git a/examples/simple-example/next.config.js b/examples/simple-example/next.config.js index 05964864..ec8ae629 100644 --- a/examples/simple-example/next.config.js +++ b/examples/simple-example/next.config.js @@ -8,16 +8,6 @@ const nextConfig = { async rewrites() { return { beforeFiles: [ - { - source: "/@org/:org/:project/:file(\.\+\\\.\.\+\$)", - destination: - '/api/proxy?url=https://raw.githubusercontent.com/:org/:project/main/:file', - }, - { - source: "/@:org/:project/:file(\.\+\\\.\.\+\$)", - destination: - '/api/proxy?url=https://raw.githubusercontent.com/:org/:project/main/:file', - }, { source: '/@:org/:project*', destination: '/@org/:org/:project*', @@ -25,6 +15,10 @@ const nextConfig = { ], }; }, + serverRuntimeConfig: { + github_pat: process.env.GITHUB_PAT ? process.env.GITHUB_PAT : null, + project_name: process.env.PROJECT_NAME ? process.env.PROJECT_NAME : 'simple-example' + }, nx: { // Set this to true if you would like to use SVGR // See: https://github.com/gregberge/svgr diff --git a/examples/simple-example/pages/@org/[org]/[...path].tsx b/examples/simple-example/pages/@org/[org]/[...path].tsx new file mode 100644 index 00000000..62c9e9a8 --- /dev/null +++ b/examples/simple-example/pages/@org/[org]/[...path].tsx @@ -0,0 +1,122 @@ +import Head from 'next/head'; +import { useRouter } from 'next/router'; + +import { NextSeo } from 'next-seo'; +import { promises as fs } from 'fs'; +import path from 'path'; +import getConfig from 'next/config'; +import { getProject, GithubProject } from '../../../lib/octokit'; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; +import Link from 'next/link'; + +export default function ProjectPage({ project }) { + return ( + <> + +
+ Back to homepage +

Data

+
+ + + + + + + + + {project.files.map((file) => ( + + + + + ))} + +
+ Name + + Size +
+ {file.name} + + {file.size} Bytes +
+
+ +

Readme

+ + {project.readmeContent} + +
+ + ); +} + +// Generates `/posts/1` and `/posts/2` +export async function getStaticPaths() { + const project_name = getConfig().serverRuntimeConfig.project_name; + const jsonDirectory = path.join( + process.cwd(), + `/examples/${project_name}/datasets.json` + ); + const repos = await fs.readFile(jsonDirectory, 'utf8'); + + return { + paths: JSON.parse(repos).map((repo) => { + const projectPath = + repo.readme.split('/').length > 1 + ? repo.readme.split('/').slice(0, -1) + : null; + let path = [repo.repo]; + if (projectPath) { + projectPath.forEach((element) => { + path.push(element); + }); + } + return { + params: { org: repo.owner, path }, + }; + }), + fallback: false, // can also be true or 'blocking' + }; +} + +export async function getStaticProps({ params }) { + const project_name = getConfig().serverRuntimeConfig.project_name; + const jsonDirectory = path.join( + process.cwd(), + `/examples/${project_name}/datasets.json` + ); + const reposFile = await fs.readFile(jsonDirectory, 'utf8'); + const repos: GithubProject[] = JSON.parse(reposFile); + const repo = repos.find((_repo) => { + const projectPath = + _repo.readme.split('/').length > 1 + ? _repo.readme.split('/').slice(0, -1) + : null; + let path = [_repo.repo]; + if (projectPath) { + projectPath.forEach((element) => { + path.push(element); + }); + } + return ( + _repo.owner == params.org && + JSON.stringify(path) === JSON.stringify(params.path) + ); + }); + const github_pat = getConfig().serverRuntimeConfig.github_pat; + const project = await getProject(repo, github_pat); + return { + props: { + project: { ...project, repo_config: repo }, + }, + }; +} diff --git a/examples/simple-example/pages/@org/[org]/[project]/index.tsx b/examples/simple-example/pages/@org/[org]/[project]/index.tsx deleted file mode 100644 index dae4b4e1..00000000 --- a/examples/simple-example/pages/@org/[org]/[project]/index.tsx +++ /dev/null @@ -1,110 +0,0 @@ -import Head from 'next/head'; -import { useRouter } from 'next/router'; - -import DRD from '../../../../components/drd/DRD'; -import parse from '../../../../lib/markdown'; -import Project from '../../../../lib/project'; -import { NextSeo } from 'next-seo'; -import MDLayout from 'examples/simple-example/components/MDLayout'; -import { promises as fs } from 'fs'; -import path from 'path'; - -function CollectionsLayout({ children, ...frontMatter }) { - const { title, date, description } = frontMatter; - - return ( -
-
-
- {date && ( -

- -

- )} - {title &&

{title}

} - {description &&

{description}

} -
-
-
{children}
-
- ); -} - -export default function ProjectPage({ - mdxSource, - frontMatter, - excerpt, - project, -}) { - const router = useRouter(); - - return ( - <> - - - {/* - On index files, add trailling slash to the base path - see notes: https://github.com/datopian/datahub-next/issues/69 - */} - - -
- - - -
- - ); -} - -// Generates `/posts/1` and `/posts/2` -export async function getStaticPaths() { - const jsonDirectory = path.join(process.cwd(), '/examples/simple-example/datasets.json'); - const repos = await fs.readFile(jsonDirectory, 'utf8'); - - return { - paths: JSON.parse(repos).map(repo => ({ params: { org: repo.owner, project: repo.repo}})), - fallback: false, // can also be true or 'blocking' - } -} - -export async function getStaticProps({ params }) { - const { org: orgName, project: projectName } = params; - - const project = await Project.getFromGitHub(orgName, projectName); - - // Defaults to README - let content = project.readme; - - if (content === null) { - return { - notFound: true, - }; - } - - let { mdxSource, frontMatter, excerpt } = await parse(content, '.mdx'); - - if (project.metadata?.resources) { - frontMatter.layout = 'datapackage'; - } - - return { - props: { - mdxSource, - frontMatter, - excerpt, - project: project.serialize(), - }, - }; -} diff --git a/examples/simple-example/pages/_app.tsx b/examples/simple-example/pages/_app.tsx index 9b087458..e0bf2912 100644 --- a/examples/simple-example/pages/_app.tsx +++ b/examples/simple-example/pages/_app.tsx @@ -1,7 +1,6 @@ import { AppProps } from 'next/app'; import Head from 'next/head'; import './styles.css'; -import "../styles/global.css"; function CustomApp({ Component, pageProps }: AppProps) { return ( diff --git a/examples/simple-example/pages/api/proxy.ts b/examples/simple-example/pages/api/proxy.ts deleted file mode 100644 index 4de13fe8..00000000 --- a/examples/simple-example/pages/api/proxy.ts +++ /dev/null @@ -1,26 +0,0 @@ -import axios from "axios"; - -export default function handler(req, res) { - if (!req.query.url) { - res.status(200).send({ - error: true, - info: "No url to proxy in query string i.e. ?url=...", - }); - return; - } - axios({ - method: "get", - url: req.query.url, - responseType: "stream", - }) - .then((resp) => { - resp.data.pipe(res); - }) - .catch((err) => { - res.status(400).send({ - error: true, - info: err.message, - detailed: err, - }); - }); -} diff --git a/examples/simple-example/pages/index.tsx b/examples/simple-example/pages/index.tsx index b3e6774f..d48a8084 100644 --- a/examples/simple-example/pages/index.tsx +++ b/examples/simple-example/pages/index.tsx @@ -1,35 +1,21 @@ -import parse from '../lib/markdown'; -import Project from '../lib/project'; import { promises as fs } from 'fs'; import path from 'path'; -import Link from 'next/link'; +import { getProject } from '../lib/octokit'; +import getConfig from 'next/config'; export async function getStaticProps() { + const project_name = getConfig().serverRuntimeConfig.project_name; const jsonDirectory = path.join( process.cwd(), - '/examples/simple-example/datasets.json' + `/examples/${project_name}/datasets.json` ); const repos = await fs.readFile(jsonDirectory, 'utf8'); + const github_pat = getConfig().serverRuntimeConfig.github_pat; const projects = await Promise.all( - JSON.parse(repos).map(async (repo) => { - const project = await Project.getFromGitHub(repo.owner, repo.repo); - - // Defaults to README - const content = project.readme ? project.readme : ''; - - let { mdxSource, frontMatter, excerpt } = await parse(content, '.mdx'); - - if (project.metadata?.resources) { - frontMatter.layout = 'datapackage'; - } - - return { - mdxSource, - frontMatter, - excerpt, - project: project.serialize(), - }; + (JSON.parse(repos)).map(async (repo) => { + const project = await getProject(repo, github_pat); + return { ...project, repo_config: repo }; }) ); return { @@ -53,37 +39,13 @@ export function Datasets({ projects }) { return (
-

+

My Datasets

Here is a list of all my datasets for easy access and sharing

- {/* -
- {projects.map((project) => ( -
-
- - {project.project.owner}/{project.project.name} - -
-
- - Github repo - -
-
- {project.excerpt !== '' ? project.excerpt : 'No description'} -
-
- ))} -
*/}
@@ -93,7 +55,13 @@ export function Datasets({ projects }) { scope="col" className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900" > - Dataset name + Name + + - + +
+ Repo (
- - {project.project.owner}/{project.project.name} - - - {project.project.repo_metadata.description} + {project.repo_config.name + ? project.repo_config.name + : project.full_name + (project.base_path === '/' ? '' : '/' + project.base_path)} - {formatter.format( - new Date(project.project.repo_metadata.updated_at) - )} + {project.full_name} + + {project.repo_config.description + ? project.repo_config.description + : project.description} + + {formatter.format(new Date(project.last_updated))} More info diff --git a/examples/simple-example/styles/global.css b/examples/simple-example/styles/global.css deleted file mode 100644 index d2c4f7ae..00000000 --- a/examples/simple-example/styles/global.css +++ /dev/null @@ -1,67 +0,0 @@ -@import "@flowershow/remark-callouts/styles.css"; - -/* mathjax */ -.math-inline > mjx-container > svg { - display: inline; - align-items: center; -} - -/* smooth scrolling in modern browsers */ -html { - scroll-behavior: smooth !important; -} - -/* tooltip fade-out clip */ -.tooltip-body::after { - content: ""; - position: absolute; - right: 0; - top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */ - height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/ - width: 10rem; - background: linear-gradient( - to right, - rgba(255, 255, 255, 0), - rgba(255, 255, 255, 1) 100% - ); -} - -:is(h2, h3, h4, h5, h6):not(.blogitem-title) { - margin-left: -2rem !important; - padding-left: 2rem !important; - scroll-margin-top: 4.5rem; - position: relative; -} - -.heading-link { - padding: 1px; - position: absolute; - left: 0; - top: 50%; - transform: translateY(-50%); - margin: auto 0; - border-radius: 5px; - background: #1e293b; - opacity: 0; - transition: opacity 0.2s; -} - -.light .heading-link { - /* border: 1px solid #ab2b65; */ - /* background: none; */ - background: #e2e8f0; -} - -:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link { - opacity: 100; -} - -.heading-link svg { - transform: scale(0.75); -} - -@media screen and (max-width: 640px) { - .heading-link { - visibility: hidden; - } -} diff --git a/package-lock.json b/package-lock.json index 63d39f0f..f0a7c470 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "prop-types": "^15.8.1", "react": "18.2.0", "react-dom": "18.2.0", + "react-markdown": "^8.0.7", "react-next-github-btn": "^1.2.1", "react-plotly.js": "^2.6.0", "react-plotlyjs": "^0.4.4", @@ -29695,6 +29696,41 @@ "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true }, + "node_modules/react-markdown": { + "version": "8.0.7", + "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-8.0.7.tgz", + "integrity": "sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==", + "dependencies": { + "@types/hast": "^2.0.0", + "@types/prop-types": "^15.0.0", + "@types/unist": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^2.0.0", + "prop-types": "^15.0.0", + "property-information": "^6.0.0", + "react-is": "^18.0.0", + "remark-parse": "^10.0.0", + "remark-rehype": "^10.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^0.4.0", + "unified": "^10.0.0", + "unist-util-visit": "^4.0.0", + "vfile": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "@types/react": ">=16", + "react": ">=16" + } + }, + "node_modules/react-markdown/node_modules/react-is": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==" + }, "node_modules/react-next-github-btn": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-next-github-btn/-/react-next-github-btn-1.2.1.tgz", @@ -57219,6 +57255,35 @@ "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true }, + "react-markdown": { + "version": "8.0.7", + "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-8.0.7.tgz", + "integrity": "sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==", + "requires": { + "@types/hast": "^2.0.0", + "@types/prop-types": "^15.0.0", + "@types/unist": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^2.0.0", + "prop-types": "^15.0.0", + "property-information": "^6.0.0", + "react-is": "^18.0.0", + "remark-parse": "^10.0.0", + "remark-rehype": "^10.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^0.4.0", + "unified": "^10.0.0", + "unist-util-visit": "^4.0.0", + "vfile": "^5.0.0" + }, + "dependencies": { + "react-is": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==" + } + } + }, "react-next-github-btn": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-next-github-btn/-/react-next-github-btn-1.2.1.tgz", diff --git a/package.json b/package.json index 4c421d3e..d65f09b2 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "prop-types": "^15.8.1", "react": "18.2.0", "react-dom": "18.2.0", + "react-markdown": "^8.0.7", "react-next-github-btn": "^1.2.1", "react-plotly.js": "^2.6.0", "react-plotlyjs": "^0.4.4", diff --git a/site/content/blog/example-data-catalog.md b/site/content/blog/example-data-catalog.md index 2dd4f40d..a3e81c6f 100644 --- a/site/content/blog/example-data-catalog.md +++ b/site/content/blog/example-data-catalog.md @@ -14,15 +14,15 @@ Below are some screenshots: ### Front page -![](https://i.imgur.com/wMk3pGf.png) +![](https://i.imgur.com/jAljJ9C.png) ### Individual dataset page -![](https://i.imgur.com/UipBNEY.png) +![](https://i.imgur.com/AoJd4O0.png) ## Links - [Documentation](/docs/example-data-catalog) - [Repo](https://github.com/datopian/portaljs/tree/main/examples/simple-example) -- [Live Demo](https://example.portaljs.org) \ No newline at end of file +- [Live Demo](https://example.portaljs.org) From 07d903e454267ade7b88bbbcca09e6cdaa2c9811 Mon Sep 17 00:00:00 2001 From: Luccas Mateus de Medeiros Gomes Date: Sat, 22 Apr 2023 14:07:21 -0300 Subject: [PATCH 2/4] [simple-example][sm] - .gitignore --- .gitignore | 6 +++++- examples/simple-example/.env | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) delete mode 100644 examples/simple-example/.env diff --git a/.gitignore b/.gitignore index 794fab1f..cfed33d4 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,8 @@ testem.log Thumbs.db # Next.js -.next \ No newline at end of file +.next + +# Env +.env +**/.env diff --git a/examples/simple-example/.env b/examples/simple-example/.env deleted file mode 100644 index 67f0db30..00000000 --- a/examples/simple-example/.env +++ /dev/null @@ -1 +0,0 @@ -GITHUB_PAT=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 From a9940a41fefef0b569f57abea1db16cb003a5046 Mon Sep 17 00:00:00 2001 From: Luccas Mateus de Medeiros Gomes Date: Sat, 22 Apr 2023 14:09:23 -0300 Subject: [PATCH 3/4] [ckan-example][sm] - remove package.json --- examples/ckan-example/package.json | 109 ----------------------------- 1 file changed, 109 deletions(-) delete mode 100644 examples/ckan-example/package.json diff --git a/examples/ckan-example/package.json b/examples/ckan-example/package.json deleted file mode 100644 index 4c421d3e..00000000 --- a/examples/ckan-example/package.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "name": "portaljs", - "version": "0.0.0", - "license": "MIT", - "scripts": {}, - "private": true, - "dependencies": { - "@apollo/client": "^3.7.11", - "@apollo/react-hooks": "^4.0.0", - "@emotion/react": "^11.10.6", - "@emotion/styled": "^11.10.6", - "@flowershow/core": "^0.4.9", - "@flowershow/markdowndb": "^0.1.0", - "@flowershow/remark-callouts": "^1.0.0", - "@flowershow/remark-embed": "^1.0.0", - "@flowershow/remark-wiki-link": "^1.0.1", - "@headlessui/react": "^1.7.13", - "@heroicons/react": "^2.0.17", - "@mui/icons-material": "^5.11.16", - "@mui/material": "^5.11.16", - "@mui/x-data-grid": "^6.1.0", - "@opentelemetry/api": "^1.4.0", - "@tailwindcss/typography": "^0.5.9", - "@tanstack/react-table": "^8.8.5", - "apollo-cache-inmemory": "^1.6.6", - "apollo-link": "^1.2.14", - "apollo-link-rest": "^0.9.0", - "filesize": "^10.0.7", - "gray-matter": "^4.0.3", - "html-react-parser": "^3.0.15", - "markdown-it": "^13.0.1", - "next": "^13.2.1", - "next-mdx-remote": "^4.4.1", - "next-seo": "^6.0.0", - "next-translate": "^2.0.5", - "nock": "^13.3.0", - "octokit": "^2.0.14", - "papaparse": "^5.4.1", - "plotly.js-basic-dist": "^2.20.0", - "prop-types": "^15.8.1", - "react": "18.2.0", - "react-dom": "18.2.0", - "react-next-github-btn": "^1.2.1", - "react-plotly.js": "^2.6.0", - "react-plotlyjs": "^0.4.4", - "react-vega": "^7.6.0", - "rehype-autolink-headings": "^6.1.1", - "rehype-katex": "^6.0.2", - "rehype-prism-plus": "^1.5.1", - "rehype-slug": "^5.1.0", - "remark-footnotes": "^4.0.1", - "remark-gfm": "^3.0.1", - "remark-math": "^5.1.1", - "remark-slug": "^7.0.1", - "remark-smartypants": "^2.0.0", - "remark-toc": "^8.0.1", - "slugify": "^1.6.6", - "timeago.js": "^4.0.2", - "tslib": "^2.3.0", - "vega": "^5.24.0", - "xlsx": "^0.18.5" - }, - "devDependencies": { - "@babel/preset-react": "^7.14.5", - "@nrwl/cypress": "15.9.2", - "@nrwl/eslint-plugin-nx": "15.9.2", - "@nrwl/jest": "15.9.2", - "@nrwl/js": "15.9.2", - "@nrwl/linter": "15.9.2", - "@nrwl/next": "^15.9.2", - "@nrwl/react": "15.9.2", - "@nrwl/rollup": "15.9.2", - "@nrwl/workspace": "15.9.2", - "@rollup/plugin-url": "^7.0.0", - "@svgr/rollup": "^6.1.2", - "@swc/core": "^1.2.173", - "@swc/helpers": "~0.5.0", - "@swc/jest": "0.2.20", - "@testing-library/react": "14.0.0", - "@types/jest": "^29.4.0", - "@types/node": "18.14.2", - "@types/react": "18.0.28", - "@types/react-dom": "18.0.11", - "@typescript-eslint/eslint-plugin": "^5.36.1", - "@typescript-eslint/parser": "^5.36.1", - "autoprefixer": "10.4.13", - "babel-jest": "^29.4.1", - "cypress": "^12.2.0", - "eslint": "~8.15.0", - "eslint-config-next": "13.1.1", - "eslint-config-prettier": "8.1.0", - "eslint-plugin-cypress": "^2.10.3", - "eslint-plugin-import": "2.27.5", - "eslint-plugin-jsx-a11y": "6.7.1", - "eslint-plugin-react": "7.32.2", - "eslint-plugin-react-hooks": "4.6.0", - "jest": "^29.4.1", - "jest-environment-jsdom": "^29.4.1", - "nx": "15.9.2", - "postcss": "8.4.21", - "prettier": "^2.6.2", - "react-test-renderer": "18.2.0", - "swc-loader": "0.1.15", - "tailwindcss": "3.2.7", - "ts-jest": "^29.0.5", - "ts-node": "10.9.1", - "typescript": "~4.9.5" - } -} From a89dfaae389e0e5e3632d74f20e80dc5f98ede43 Mon Sep 17 00:00:00 2001 From: Luccas Mateus de Medeiros Gomes Date: Sat, 22 Apr 2023 14:28:13 -0300 Subject: [PATCH 4/4] [simple-example][m] - remove unused components --- examples/simple-example/README.md | 2 +- .../simple-example/components/MDLayout.tsx | 85 -------- .../simple-example/components/drd/DRD.tsx | 40 ---- .../components/drd/DebouncedInput.tsx | 33 --- .../components/drd/FrictionlessView.tsx | 55 ----- .../components/drd/LineChart.tsx | 49 ----- .../simple-example/components/drd/Table.tsx | 188 ------------------ .../simple-example/components/drd/Vega.tsx | 4 - .../components/drd/VegaLite.tsx | 4 - 9 files changed, 1 insertion(+), 459 deletions(-) delete mode 100644 examples/simple-example/components/MDLayout.tsx delete mode 100644 examples/simple-example/components/drd/DRD.tsx delete mode 100644 examples/simple-example/components/drd/DebouncedInput.tsx delete mode 100644 examples/simple-example/components/drd/FrictionlessView.tsx delete mode 100644 examples/simple-example/components/drd/LineChart.tsx delete mode 100644 examples/simple-example/components/drd/Table.tsx delete mode 100644 examples/simple-example/components/drd/Vega.tsx delete mode 100644 examples/simple-example/components/drd/VegaLite.tsx diff --git a/examples/simple-example/README.md b/examples/simple-example/README.md index f198eee4..79c9aa84 100644 --- a/examples/simple-example/README.md +++ b/examples/simple-example/README.md @@ -22,7 +22,7 @@ PROJECT_NAME= - This project uses the github api, which for anonymous users will cap at 50 requests per hour, so you might want to get a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and add it to your .env file like so ``` -PAT_TOKEN= +GITHUB_PAT= ``` - Edit the file `datasets.json` to your liking, some examples can be found inside this [repo](https://github.com/datasets) diff --git a/examples/simple-example/components/MDLayout.tsx b/examples/simple-example/components/MDLayout.tsx deleted file mode 100644 index b1041f68..00000000 --- a/examples/simple-example/components/MDLayout.tsx +++ /dev/null @@ -1,85 +0,0 @@ -import FrictionlessViewFactory from "./drd/FrictionlessView"; -import Table from "./drd/Table"; - -/* eslint import/no-default-export: off */ -function DatapackageLayout({ children, project, excerpt }) { - const { metadata } = project; - - const title = metadata.title; - const resources = metadata.resources; - const views = metadata.views; - - const FrictionlessView = FrictionlessViewFactory({ views, resources }); - - return ( -
-
- {title &&

{title}

} - - @{project.owner} / {project.name} - - {excerpt &&

{excerpt}

} -
-
- {views.map((view, i) => { - return ( -
- -
- ); - })} -
-
-

Data files

- - - - - - - - - - {resources.map((r) => { - return ( - - - - - - ); - })} - -
FileTitleFormat
- - {r.path} - - {r.title}{r.format.toUpperCase()}
- {resources.slice(0, 5).map((resource) => { - return ( -
-

{resource.title || resource.name || resource.path}

- - - ); - })} - -
-
-

Read me

- {children} -
- - ); -} - -export default function MDLayout({ children, layout, ...props }) { - return {children}; -} diff --git a/examples/simple-example/components/drd/DRD.tsx b/examples/simple-example/components/drd/DRD.tsx deleted file mode 100644 index 50d0f39e..00000000 --- a/examples/simple-example/components/drd/DRD.tsx +++ /dev/null @@ -1,40 +0,0 @@ -import { MDXRemote } from "next-mdx-remote"; -import dynamic from "next/dynamic"; -import { Mermaid } from "@flowershow/core"; - -import FrictionlessViewFactory from "./FrictionlessView"; - -// Custom components/renderers to pass to MDX. -// Since the MDX files aren't loaded by webpack, they have no knowledge of how -// to handle import statements. Instead, you must include components in scope -// here. -const components = { - Table: dynamic(() => import("./Table")), - mermaid: Mermaid, - // Excel: dynamic(() => import('../components/Excel')), - // TODO: try and make these dynamic ... - Vega: dynamic(() => import("./Vega")), - VegaLite: dynamic(() => import("./VegaLite")), - LineChart: dynamic(() => import("./LineChart")), -} as any; - -export default function DRD({ - source, - frictionless = { - views: [], - resources: [], - }, -}: { - source: any; - frictionless?: any; -}) { - // dynamic() can't be used inside of React rendering - // as it needs to be marked in the top level of the - // module for preloading to work - components.FrictionlessView = FrictionlessViewFactory({ - views: frictionless.views, - resources: frictionless.resources, - }); - - return ; -} diff --git a/examples/simple-example/components/drd/DebouncedInput.tsx b/examples/simple-example/components/drd/DebouncedInput.tsx deleted file mode 100644 index c5fa34e0..00000000 --- a/examples/simple-example/components/drd/DebouncedInput.tsx +++ /dev/null @@ -1,33 +0,0 @@ -import { useEffect, useState } from "react"; - -const DebouncedInput = ({ - value: initialValue, - onChange, - debounce = 500, - ...props -}) => { - const [value, setValue] = useState(initialValue); - - useEffect(() => { - setValue(initialValue); - }, [initialValue]); - - useEffect(() => { - const timeout = setTimeout(() => { - onChange(value); - }, debounce); - - return () => clearTimeout(timeout); - }, [value]); - - return ( - setValue(e.target.value)} - /> - ); -}; - -export default DebouncedInput; - diff --git a/examples/simple-example/components/drd/FrictionlessView.tsx b/examples/simple-example/components/drd/FrictionlessView.tsx deleted file mode 100644 index f2256de6..00000000 --- a/examples/simple-example/components/drd/FrictionlessView.tsx +++ /dev/null @@ -1,55 +0,0 @@ -// FrictionlessView is a factory because we have to -// set the views and resources lists before using it - -import { convertSimpleToVegaLite } from "../../lib/viewSpecConversion"; -import VegaLite from "./VegaLite"; - -export default function FrictionlessViewFactory({ - views = [], - resources = [], -}): ({ - viewId, - fullWidth, -}: { - viewId: number; - fullWidth?: boolean; -}) => JSX.Element { - return ({ viewId, fullWidth = false }) => { - if (!(viewId in views)) { - console.error(`View ${viewId} not found`); - return <>; - } - const view = views[viewId]; - - let resource; - if (resources.length > 1) { - resource = resources.find((r) => r.name === view.resourceName); - } else { - resource = resources[0]; - } - - if (!resource) { - console.error(`Resource not found for view id ${viewId}`); - return <>; - } - - let vegaSpec; - switch (view.specType) { - case "simple": - vegaSpec = convertSimpleToVegaLite(view, resource); - break; - // ... other conversions - } - - vegaSpec.data = { url: resource.path }; - - return ( - - ); - }; -} diff --git a/examples/simple-example/components/drd/LineChart.tsx b/examples/simple-example/components/drd/LineChart.tsx deleted file mode 100644 index e5ee69fb..00000000 --- a/examples/simple-example/components/drd/LineChart.tsx +++ /dev/null @@ -1,49 +0,0 @@ -import { VegaLite } from "react-vega"; - -export default function LineChart({ - data = [], - fullWidth = false, - title = "", -}) { - var tmp = data; - if (Array.isArray(data)) { - tmp = data.map((r, i) => { - return { x: r[0], y: r[1] }; - }); - } - const vegaData = { table: tmp }; - const spec = { - $schema: "https://vega.github.io/schema/vega-lite/v5.json", - title, - width: "container" as "container", - height: 300, - mark: { - type: "line" as "line", - color: "black", - strokeWidth: 1, - tooltip: true, - }, - data: { - name: "table", - }, - selection: { - grid: { - type: "interval" as "interval", - bind: "scales", - }, - }, - encoding: { - x: { - field: "x", - timeUnit: "year", - type: "temporal" as "temporal", - }, - y: { - field: "y", - type: "quantitative" as "temporal", - }, - }, - }; - - return ; -} diff --git a/examples/simple-example/components/drd/Table.tsx b/examples/simple-example/components/drd/Table.tsx deleted file mode 100644 index 4866b645..00000000 --- a/examples/simple-example/components/drd/Table.tsx +++ /dev/null @@ -1,188 +0,0 @@ -import { - createColumnHelper, - FilterFn, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table"; - -import { - ArrowDownIcon, - ArrowUpIcon, - ChevronDoubleLeftIcon, - ChevronDoubleRightIcon, - ChevronLeftIcon, - ChevronRightIcon, -} from "@heroicons/react/24/solid"; - -import React, { useEffect, useMemo, useState } from "react"; - -import loadUrlProxied from "../../lib/loadUrlProxied"; -import parseCsv from "../../lib/parseCsv"; -import DebouncedInput from "./DebouncedInput"; - -const Table = ({ - data: ogData = [], - cols: ogCols = [], - csv = "", - url = "", -}) => { - if (csv) { - const out = parseCsv(csv); - ogData = out.rows; - ogCols = out.fields; - } - - const [data, setData] = React.useState(ogData); - const [cols, setCols] = React.useState(ogCols); - const [error, setError] = React.useState(""); // TODO: add error handling - - const tableCols = useMemo(() => { - const columnHelper = createColumnHelper(); - return cols.map((c) => - columnHelper.accessor(c.key, { - header: () => c.name, - cell: (info) => info.getValue(), - }) - ); - }, [data, cols]); - - const [globalFilter, setGlobalFilter] = useState(""); - - const table = useReactTable({ - data, - columns: tableCols, - getCoreRowModel: getCoreRowModel(), - state: { - globalFilter, - }, - globalFilterFn: globalFilterFn, - onGlobalFilterChange: setGlobalFilter, - getFilteredRowModel: getFilteredRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - }); - - useEffect(() => { - if (url) { - loadUrlProxied(url).then((data) => { - const { rows, fields } = parseCsv(data); - setData(rows); - setCols(fields); - }); - } - }, [url]); - - return ( -
- setGlobalFilter(String(value))} - className="p-2 text-sm shadow border border-block" - placeholder="Search all columns..." - /> -
- - {table.getHeaderGroups().map((hg) => ( - - {hg.headers.map((h) => ( - - ))} - - ))} - - - {table.getRowModel().rows.map((r) => ( - - {r.getVisibleCells().map((c) => ( - - ))} - - ))} - -
-
- {flexRender(h.column.columnDef.header, h.getContext())} - {{ - asc: ( - - ), - desc: ( - - ), - }[h.column.getIsSorted() as string] ?? ( -
- )} -
-
- {flexRender(c.column.columnDef.cell, c.getContext())} -
-
- - - -
Page
- - {table.getState().pagination.pageIndex + 1} of{" "} - {table.getPageCount()} - - - - -
-
- ); -}; - -const globalFilterFn: FilterFn = (row, columnId, filterValue: string) => { - const search = filterValue.toLowerCase(); - - let value = row.getValue(columnId) as string; - if (typeof value === "number") value = String(value); - - return value?.toLowerCase().includes(search); -}; - -export default Table; diff --git a/examples/simple-example/components/drd/Vega.tsx b/examples/simple-example/components/drd/Vega.tsx deleted file mode 100644 index ed24606c..00000000 --- a/examples/simple-example/components/drd/Vega.tsx +++ /dev/null @@ -1,4 +0,0 @@ -import { Vega as VegaOg } from "react-vega"; -export default function Vega(props) { - return ; -} diff --git a/examples/simple-example/components/drd/VegaLite.tsx b/examples/simple-example/components/drd/VegaLite.tsx deleted file mode 100644 index 439a632d..00000000 --- a/examples/simple-example/components/drd/VegaLite.tsx +++ /dev/null @@ -1,4 +0,0 @@ -import { VegaLite as VegaOg } from "react-vega"; -export default function Vega(props) { - return ; -}