diff --git a/examples/ckan-example/package.json b/examples/ckan-example/package.json new file mode 100644 index 00000000..4c421d3e --- /dev/null +++ b/examples/ckan-example/package.json @@ -0,0 +1,109 @@ +{ + "name": "portaljs", + "version": "0.0.0", + "license": "MIT", + "scripts": {}, + "private": true, + "dependencies": { + "@apollo/client": "^3.7.11", + "@apollo/react-hooks": "^4.0.0", + "@emotion/react": "^11.10.6", + "@emotion/styled": "^11.10.6", + "@flowershow/core": "^0.4.9", + "@flowershow/markdowndb": "^0.1.0", + "@flowershow/remark-callouts": "^1.0.0", + "@flowershow/remark-embed": "^1.0.0", + "@flowershow/remark-wiki-link": "^1.0.1", + "@headlessui/react": "^1.7.13", + "@heroicons/react": "^2.0.17", + "@mui/icons-material": "^5.11.16", + "@mui/material": "^5.11.16", + "@mui/x-data-grid": "^6.1.0", + "@opentelemetry/api": "^1.4.0", + "@tailwindcss/typography": "^0.5.9", + "@tanstack/react-table": "^8.8.5", + "apollo-cache-inmemory": "^1.6.6", + "apollo-link": "^1.2.14", + "apollo-link-rest": "^0.9.0", + "filesize": "^10.0.7", + "gray-matter": "^4.0.3", + "html-react-parser": "^3.0.15", + "markdown-it": "^13.0.1", + "next": "^13.2.1", + "next-mdx-remote": "^4.4.1", + "next-seo": "^6.0.0", + "next-translate": "^2.0.5", + "nock": "^13.3.0", + "octokit": "^2.0.14", + "papaparse": "^5.4.1", + "plotly.js-basic-dist": "^2.20.0", + "prop-types": "^15.8.1", + "react": "18.2.0", + "react-dom": "18.2.0", + "react-next-github-btn": "^1.2.1", + "react-plotly.js": "^2.6.0", + "react-plotlyjs": "^0.4.4", + "react-vega": "^7.6.0", + "rehype-autolink-headings": "^6.1.1", + "rehype-katex": "^6.0.2", + "rehype-prism-plus": "^1.5.1", + "rehype-slug": "^5.1.0", + "remark-footnotes": "^4.0.1", + "remark-gfm": "^3.0.1", + "remark-math": "^5.1.1", + "remark-slug": "^7.0.1", + "remark-smartypants": "^2.0.0", + "remark-toc": "^8.0.1", + "slugify": "^1.6.6", + "timeago.js": "^4.0.2", + "tslib": "^2.3.0", + "vega": "^5.24.0", + "xlsx": "^0.18.5" + }, + "devDependencies": { + "@babel/preset-react": "^7.14.5", + "@nrwl/cypress": "15.9.2", + "@nrwl/eslint-plugin-nx": "15.9.2", + "@nrwl/jest": "15.9.2", + "@nrwl/js": "15.9.2", + "@nrwl/linter": "15.9.2", + "@nrwl/next": "^15.9.2", + "@nrwl/react": "15.9.2", + "@nrwl/rollup": "15.9.2", + "@nrwl/workspace": "15.9.2", + "@rollup/plugin-url": "^7.0.0", + "@svgr/rollup": "^6.1.2", + "@swc/core": "^1.2.173", + "@swc/helpers": "~0.5.0", + "@swc/jest": "0.2.20", + "@testing-library/react": "14.0.0", + "@types/jest": "^29.4.0", + "@types/node": "18.14.2", + "@types/react": "18.0.28", + "@types/react-dom": "18.0.11", + "@typescript-eslint/eslint-plugin": "^5.36.1", + "@typescript-eslint/parser": "^5.36.1", + "autoprefixer": "10.4.13", + "babel-jest": "^29.4.1", + "cypress": "^12.2.0", + "eslint": "~8.15.0", + "eslint-config-next": "13.1.1", + "eslint-config-prettier": "8.1.0", + "eslint-plugin-cypress": "^2.10.3", + "eslint-plugin-import": "2.27.5", + "eslint-plugin-jsx-a11y": "6.7.1", + "eslint-plugin-react": "7.32.2", + "eslint-plugin-react-hooks": "4.6.0", + "jest": "^29.4.1", + "jest-environment-jsdom": "^29.4.1", + "nx": "15.9.2", + "postcss": "8.4.21", + "prettier": "^2.6.2", + "react-test-renderer": "18.2.0", + "swc-loader": "0.1.15", + "tailwindcss": "3.2.7", + "ts-jest": "^29.0.5", + "ts-node": "10.9.1", + "typescript": "~4.9.5" + } +} diff --git a/examples/simple-example/.env b/examples/simple-example/.env new file mode 100644 index 00000000..67f0db30 --- /dev/null +++ b/examples/simple-example/.env @@ -0,0 +1 @@ +GITHUB_PAT=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 diff --git a/examples/simple-example/README.md b/examples/simple-example/README.md index 733b798b..f198eee4 100644 --- a/examples/simple-example/README.md +++ b/examples/simple-example/README.md @@ -1,17 +1,66 @@ This is a repo intended to serve as a simple example of a data catalog that get its data from a series of github repos, you can init an example just like this one by. -- Creating a new file inside o `examples` with `create-next-app` like so: +- Cloning the PortalJS repo on your machine + +``` +git clone https://github.com/datopian/portaljs.git +``` + +- Creating a new file inside the `examples` folder with `create-next-app` like so: + ``` npx create-next-app --example https://github.com/datopian/portaljs/tree/main/ --example-path examples/simple-example ``` + - Inside `` go to the `project.json` file and replace all instances of `simple-example` with `` +- Create a `.env` file with the following content + +``` +PROJECT_NAME= +``` + +- This project uses the github api, which for anonymous users will cap at 50 requests per hour, so you might want to get a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and add it to your .env file like so + +``` +PAT_TOKEN= +``` + - Edit the file `datasets.json` to your liking, some examples can be found inside this [repo](https://github.com/datasets) - Run the app using: + ``` nx serve ``` -Congratulations, you now have something similar to this running on `http://localhost:4200` -![](https://i.imgur.com/JrDLycF.png) -If yo go to any one of those pages by clicking on `More info` you will see something similar to this -![](https://i.imgur.com/cpKMS80.png) +Congratulations, you now have something similar to this running on `http://localhost:4200` +![](https://i.imgur.com/jAljJ9C.png) +If yo go to any one of those pages by clicking on `More info` you will see something similar to this +![](https://i.imgur.com/AoJd4O0.png) + + +## Structure of `datasets.json` + +The `datasets.json` file is simply a list of datasets, below you can see a minimal example of a dataset + +```json +{ + "owner": "fivethirtyeight", + "repo": "data", + "branch": "master", + "files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"], + "readme": "nba-raptor/README.md" +} +``` + +It has + +- A `owner` which is going to be the github repo owner +- A `repo` which is going to be the github repo name +- A `branch` which is going to be the branch to which we need to get the files and the readme +- A list of `files` which is going to be a list of paths with files that you want to show to the world +- A `readme` which is going to be the path to your data description, it can also be a subpath eg: `example/README.md` + +You can also add + +- A `description` which is useful if you have more than one dataset for each repo, if not provided we are just going to use the repo description +- A `Name` which is useful if you want to give your dataset a nice name, if not provided we are going to use the junction of the `owner` the `repo` + the path of the README, in the exaple above it will be `fivethirtyeight/data/nba-raptor` diff --git a/examples/simple-example/datasets.json b/examples/simple-example/datasets.json index 6263dc59..97a73c5b 100644 --- a/examples/simple-example/datasets.json +++ b/examples/simple-example/datasets.json @@ -1,7 +1,44 @@ [ - { "owner": "datasets", "repo": "oil-prices"}, - { "owner": "datasets", "repo": "investor-flow-of-funds-us"}, - { "owner": "datasets", "repo": "browser-stats"}, - { "owner": "datasets", "repo": "glacier-mass-balance"}, - { "owner": "datasets", "repo": "bond-yields-us-10y"} + { + "owner": "datasets", + "branch": "main", + "repo": "oil-prices", + "files": [ + "data/brent-daily.csv", + "data/brent-monthly.csv", + "data/brent-weekly.csv", + "data/brent-year.csv", + "data/wti-daily.csv", + "data/wti-monthly.csv", + "data/wti-weekly.csv", + "data/wti-year.csv" + ], + "readme": "README.md" + }, + { + "owner": "datasets", + "branch": "main", + "repo": "investor-flow-of-funds-us", + "files": [ + "data/monthly.csv", + "data/weekly.csv" + ], + "readme": "README.md" + }, + { + "owner": "fivethirtyeight", + "repo": "data", + "branch": "master", + "description": "Data about bad drivers", + "name": "Bad Drivers", + "files": ["bad-drivers/bad-drivers.csv"], + "readme": "bad-drivers/README.md" + }, + { + "owner": "fivethirtyeight", + "repo": "data", + "branch": "master", + "files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"], + "readme": "nba-raptor/README.md" + } ] diff --git a/examples/simple-example/lib/loadUrlProxied.tsx b/examples/simple-example/lib/loadUrlProxied.tsx deleted file mode 100644 index 82e371b1..00000000 --- a/examples/simple-example/lib/loadUrlProxied.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import axios from "axios"; - -export default function loadUrlProxied(url: string) { - // HACK: duplicate of Excel code - maybe refactor - // if url is external may have CORS issue so we proxy it ... - if (url.startsWith("http")) { - const PROXY_URL = "/api/proxy"; - url = PROXY_URL + "?url=" + encodeURIComponent(url); - } - return axios.get(url).then((res) => res.data); -} diff --git a/examples/simple-example/lib/markdown.js b/examples/simple-example/lib/markdown.js deleted file mode 100644 index ada9b8ab..00000000 --- a/examples/simple-example/lib/markdown.js +++ /dev/null @@ -1,105 +0,0 @@ -import matter from "gray-matter"; -import mdxmermaid from "mdx-mermaid"; -import { h } from "hastscript"; -import remarkCallouts from "@flowershow/remark-callouts"; -import remarkEmbed from "@flowershow/remark-embed"; -import remarkGfm from "remark-gfm"; -import remarkMath from "remark-math"; -import remarkSmartypants from "remark-smartypants"; -import remarkToc from "remark-toc"; -import remarkWikiLink from "@flowershow/remark-wiki-link"; -import rehypeAutolinkHeadings from "rehype-autolink-headings"; -import rehypeKatex from "rehype-katex"; -import rehypeSlug from "rehype-slug"; -import rehypePrismPlus from "rehype-prism-plus"; - -import { serialize } from "next-mdx-remote/serialize"; - -/** - * Parse a markdown or MDX file to an MDX source form + front matter data - * - * @source: the contents of a markdown or mdx file - * @format: used to indicate to next-mdx-remote which format to use (md or mdx) - * @returns: { mdxSource: mdxSource, frontMatter: ...} - */ -const parse = async function (source, format) { - const { content, data, excerpt } = matter(source, { - excerpt: (file, options) => { - // Generate an excerpt for the file - file.excerpt = file.content.split("\n\n")[0]; - }, - }); - - const mdxSource = await serialize( - { value: content, path: format }, - { - // Optionally pass remark/rehype plugins - mdxOptions: { - remarkPlugins: [ - remarkEmbed, - remarkGfm, - [remarkSmartypants, { quotes: false, dashes: "oldschool" }], - remarkMath, - remarkCallouts, - remarkWikiLink, - [ - remarkToc, - { - heading: "Table of contents", - tight: true, - }, - ], - [mdxmermaid, {}], - ], - rehypePlugins: [ - rehypeSlug, - [ - rehypeAutolinkHeadings, - { - properties: { className: 'heading-link' }, - test(element) { - return ( - ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) && - element.properties?.id !== "table-of-contents" && - element.properties?.className !== "blockquote-heading" - ); - }, - content() { - return [ - h( - "svg", - { - xmlns: "http:www.w3.org/2000/svg", - fill: "#ab2b65", - viewBox: "0 0 20 20", - className: "w-5 h-5", - }, - [ - h("path", { - fillRule: "evenodd", - clipRule: "evenodd", - d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z", - }), - ] - ), - ]; - }, - }, - ], - [rehypeKatex, { output: "mathml" }], - [rehypePrismPlus, { ignoreMissing: true }], - ], - format, - }, - scope: data, - } - ); - - return { - mdxSource: mdxSource, - frontMatter: data, - excerpt, - }; -}; - -export default parse; diff --git a/examples/simple-example/lib/octokit.ts b/examples/simple-example/lib/octokit.ts new file mode 100644 index 00000000..2d580ffc --- /dev/null +++ b/examples/simple-example/lib/octokit.ts @@ -0,0 +1,147 @@ +import { Octokit } from 'octokit'; + +export interface GithubProject { + owner: string; + repo: string; + branch: string; + files: string[]; + readme: string; + description?: string; + name?: string; +} + +export async function getProjectReadme( + owner: string, + repo: string, + branch: string, + readme: string, + github_pat?: string +) { + const octokit = new Octokit({ auth: github_pat }); + try { + const response = await octokit.rest.repos.getContent({ + owner, + repo, + path: readme, + ref: branch, + }); + const data = response.data as { content?: string }; + const fileContent = data.content ? data.content : ''; + if (fileContent === '') { + return null; + } + const decodedContent = Buffer.from(fileContent, 'base64').toString(); + return decodedContent; + } catch (error) { + console.log(error); + return null; + } +} + +export async function getLastUpdated( + owner: string, + repo: string, + branch: string, + readme: string, + github_pat?: string +) { + const octokit = new Octokit({ auth: github_pat }); + try { + const response = await octokit.rest.repos.listCommits({ + owner, + repo, + path: readme, + ref: branch, + }); + return response.data[0].commit.committer.date; + } catch (error) { + console.log(error); + return null; + } +} +export async function getProjectMetadata( + owner: string, + repo: string, + github_pat?: string +) { + const octokit = new Octokit({ auth: github_pat }); + try { + const response = await octokit.rest.repos.get({ + owner, + repo, + }); + return response.data; + } catch (error) { + console.log(error); + return null; + } +} + +export async function getRepoContents( + owner: string, + repo: string, + branch: string, + files: string[], + github_pat?: string +) { + const octokit = new Octokit({ auth: github_pat }); + try { + const contents = []; + for (const path of files) { + const response = await octokit.rest.repos.getContent({ + owner, + repo, + ref: branch, + path: path, + }); + const data = response.data as { download_url?: string, name: string, size: number }; + contents.push({ download_url: data.download_url, name: data.name, size: data.size}); + } + return contents; + } catch (error) { + console.log(error); + return null; + } +} + +export async function getProject(project: GithubProject, github_pat?: string) { + const projectMetadata = await getProjectMetadata( + project.owner, + project.repo, + github_pat + ); + if (!projectMetadata) { + return null; + } + const projectReadme = await getProjectReadme( + project.owner, + project.repo, + project.branch, + project.readme, + github_pat + ); + if (!projectReadme) { + return null; + } + const projectData = await getRepoContents( + project.owner, + project.repo, + project.branch, + project.files, + github_pat + ); + if (!projectData) { + return null; + } + const projectBase = project.readme.split('/').length > 1 + ? project.readme.split('/').slice(0, -1).join('/') + : '/' + const last_updated = await getLastUpdated( + project.owner, + project.repo, + project.branch, + projectBase, + github_pat + ); + return { ...projectMetadata, files: projectData, readmeContent: projectReadme, last_updated, base_path: projectBase }; +} diff --git a/examples/simple-example/lib/parseCsv.ts b/examples/simple-example/lib/parseCsv.ts deleted file mode 100644 index 28a56996..00000000 --- a/examples/simple-example/lib/parseCsv.ts +++ /dev/null @@ -1,16 +0,0 @@ -import papa from "papaparse"; - -const parseCsv = (csv) => { - csv = csv.trim(); - const rawdata = papa.parse(csv, { header: true }); - const cols = rawdata.meta.fields.map((r, i) => { - return { key: r, name: r }; - }); - - return { - rows: rawdata.data, - fields: cols, - }; -}; - -export default parseCsv; diff --git a/examples/simple-example/lib/project.ts b/examples/simple-example/lib/project.ts deleted file mode 100644 index 77529a20..00000000 --- a/examples/simple-example/lib/project.ts +++ /dev/null @@ -1,60 +0,0 @@ -import * as crypto from "crypto"; -import axios from "axios"; -import { Octokit } from "octokit" - -export default class Project { - id: string; - name: string; - owner: string; - github_repo: string; - readme: string; - metadata: any; - repo_metadata: any; - - constructor(owner: string, name: string) { - this.name = name; - this.owner = owner; - this.github_repo = `https://github.com/${owner}/${name}`; - - // TODO: using the GitHub repo to set the id is not a good idea - // since repos can be renamed and then we are going to end up with - // a duplicate - const encodedGHRepo = Buffer.from(this.github_repo, "utf-8").toString(); - this.id = crypto.createHash("sha1").update(encodedGHRepo).digest("hex"); - } - - initFromGitHub = async () => { - const octokit = new Octokit() - // TODO: what if the repo doesn't exist? - await this.getFileContent("README.md") - .then((content) => (this.readme = content)) - .catch((e) => (this.readme = null)); - - await this.getFileContent("datapackage.json") - .then((content) => (this.metadata = content)) - .catch((e) => (this.metadata = {})); - - const github_metadata = await octokit.rest.repos.get({ owner: this.owner, repo: this.name }) - this.repo_metadata = github_metadata.data ? github_metadata.data : null - }; - - getFileContent = (path, branch = "main") => { - return axios - .get( - `https://raw.githubusercontent.com/${this.owner}/${this.name}/${branch}/${path}` - ) - .then((res) => res.data); - }; - - serialize() { - return JSON.parse(JSON.stringify(this)); - } - - static async getFromGitHub(owner: string, name: string) { - const project = new Project(owner, name); - await project.initFromGitHub(); - - return project; - } -} - diff --git a/examples/simple-example/lib/viewSpecConversion.ts b/examples/simple-example/lib/viewSpecConversion.ts deleted file mode 100644 index cd6d3f33..00000000 --- a/examples/simple-example/lib/viewSpecConversion.ts +++ /dev/null @@ -1,47 +0,0 @@ -export function convertSimpleToVegaLite(view, resource) { - const x = resource.schema.fields.find((f) => f.name === view.spec.group); - const y = resource.schema.fields.find((f) => f.name === view.spec.series[0]); - - const xType = inferVegaType(x.type); - const yType = inferVegaType(y.type); - - let vegaLiteSpec = { - $schema: "https://vega.github.io/schema/vega-lite/v5.json", - mark: { - type: view.spec.type, - color: "black", - strokeWidth: 1, - tooltip: true, - }, - title: view.title, - width: 500, - height: 300, - selection: { - grid: { - type: "interval", - bind: "scales", - }, - }, - encoding: { - x: { - field: x.name, - type: xType, - }, - y: { - field: y.name, - type: yType, - }, - }, - }; - - return vegaLiteSpec; -} - -const inferVegaType = (fieldType) => { - switch (fieldType) { - case "date": - return "Temporal"; - case "number": - return "Quantitative"; - } -}; diff --git a/examples/simple-example/next.config.js b/examples/simple-example/next.config.js index 05964864..ec8ae629 100644 --- a/examples/simple-example/next.config.js +++ b/examples/simple-example/next.config.js @@ -8,16 +8,6 @@ const nextConfig = { async rewrites() { return { beforeFiles: [ - { - source: "/@org/:org/:project/:file(\.\+\\\.\.\+\$)", - destination: - '/api/proxy?url=https://raw.githubusercontent.com/:org/:project/main/:file', - }, - { - source: "/@:org/:project/:file(\.\+\\\.\.\+\$)", - destination: - '/api/proxy?url=https://raw.githubusercontent.com/:org/:project/main/:file', - }, { source: '/@:org/:project*', destination: '/@org/:org/:project*', @@ -25,6 +15,10 @@ const nextConfig = { ], }; }, + serverRuntimeConfig: { + github_pat: process.env.GITHUB_PAT ? process.env.GITHUB_PAT : null, + project_name: process.env.PROJECT_NAME ? process.env.PROJECT_NAME : 'simple-example' + }, nx: { // Set this to true if you would like to use SVGR // See: https://github.com/gregberge/svgr diff --git a/examples/simple-example/pages/@org/[org]/[...path].tsx b/examples/simple-example/pages/@org/[org]/[...path].tsx new file mode 100644 index 00000000..62c9e9a8 --- /dev/null +++ b/examples/simple-example/pages/@org/[org]/[...path].tsx @@ -0,0 +1,122 @@ +import Head from 'next/head'; +import { useRouter } from 'next/router'; + +import { NextSeo } from 'next-seo'; +import { promises as fs } from 'fs'; +import path from 'path'; +import getConfig from 'next/config'; +import { getProject, GithubProject } from '../../../lib/octokit'; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; +import Link from 'next/link'; + +export default function ProjectPage({ project }) { + return ( + <> + +
+ Back to homepage +

Data

+
+ + + + + + + + + {project.files.map((file) => ( + + + + + ))} + +
+ Name + + Size +
+ {file.name} + + {file.size} Bytes +
+
+ +

Readme

+ + {project.readmeContent} + +
+ + ); +} + +// Generates `/posts/1` and `/posts/2` +export async function getStaticPaths() { + const project_name = getConfig().serverRuntimeConfig.project_name; + const jsonDirectory = path.join( + process.cwd(), + `/examples/${project_name}/datasets.json` + ); + const repos = await fs.readFile(jsonDirectory, 'utf8'); + + return { + paths: JSON.parse(repos).map((repo) => { + const projectPath = + repo.readme.split('/').length > 1 + ? repo.readme.split('/').slice(0, -1) + : null; + let path = [repo.repo]; + if (projectPath) { + projectPath.forEach((element) => { + path.push(element); + }); + } + return { + params: { org: repo.owner, path }, + }; + }), + fallback: false, // can also be true or 'blocking' + }; +} + +export async function getStaticProps({ params }) { + const project_name = getConfig().serverRuntimeConfig.project_name; + const jsonDirectory = path.join( + process.cwd(), + `/examples/${project_name}/datasets.json` + ); + const reposFile = await fs.readFile(jsonDirectory, 'utf8'); + const repos: GithubProject[] = JSON.parse(reposFile); + const repo = repos.find((_repo) => { + const projectPath = + _repo.readme.split('/').length > 1 + ? _repo.readme.split('/').slice(0, -1) + : null; + let path = [_repo.repo]; + if (projectPath) { + projectPath.forEach((element) => { + path.push(element); + }); + } + return ( + _repo.owner == params.org && + JSON.stringify(path) === JSON.stringify(params.path) + ); + }); + const github_pat = getConfig().serverRuntimeConfig.github_pat; + const project = await getProject(repo, github_pat); + return { + props: { + project: { ...project, repo_config: repo }, + }, + }; +} diff --git a/examples/simple-example/pages/@org/[org]/[project]/index.tsx b/examples/simple-example/pages/@org/[org]/[project]/index.tsx deleted file mode 100644 index dae4b4e1..00000000 --- a/examples/simple-example/pages/@org/[org]/[project]/index.tsx +++ /dev/null @@ -1,110 +0,0 @@ -import Head from 'next/head'; -import { useRouter } from 'next/router'; - -import DRD from '../../../../components/drd/DRD'; -import parse from '../../../../lib/markdown'; -import Project from '../../../../lib/project'; -import { NextSeo } from 'next-seo'; -import MDLayout from 'examples/simple-example/components/MDLayout'; -import { promises as fs } from 'fs'; -import path from 'path'; - -function CollectionsLayout({ children, ...frontMatter }) { - const { title, date, description } = frontMatter; - - return ( -
-
-
- {date && ( -

- -

- )} - {title &&

{title}

} - {description &&

{description}

} -
-
-
{children}
-
- ); -} - -export default function ProjectPage({ - mdxSource, - frontMatter, - excerpt, - project, -}) { - const router = useRouter(); - - return ( - <> - - - {/* - On index files, add trailling slash to the base path - see notes: https://github.com/datopian/datahub-next/issues/69 - */} - - -
- - - -
- - ); -} - -// Generates `/posts/1` and `/posts/2` -export async function getStaticPaths() { - const jsonDirectory = path.join(process.cwd(), '/examples/simple-example/datasets.json'); - const repos = await fs.readFile(jsonDirectory, 'utf8'); - - return { - paths: JSON.parse(repos).map(repo => ({ params: { org: repo.owner, project: repo.repo}})), - fallback: false, // can also be true or 'blocking' - } -} - -export async function getStaticProps({ params }) { - const { org: orgName, project: projectName } = params; - - const project = await Project.getFromGitHub(orgName, projectName); - - // Defaults to README - let content = project.readme; - - if (content === null) { - return { - notFound: true, - }; - } - - let { mdxSource, frontMatter, excerpt } = await parse(content, '.mdx'); - - if (project.metadata?.resources) { - frontMatter.layout = 'datapackage'; - } - - return { - props: { - mdxSource, - frontMatter, - excerpt, - project: project.serialize(), - }, - }; -} diff --git a/examples/simple-example/pages/_app.tsx b/examples/simple-example/pages/_app.tsx index 9b087458..e0bf2912 100644 --- a/examples/simple-example/pages/_app.tsx +++ b/examples/simple-example/pages/_app.tsx @@ -1,7 +1,6 @@ import { AppProps } from 'next/app'; import Head from 'next/head'; import './styles.css'; -import "../styles/global.css"; function CustomApp({ Component, pageProps }: AppProps) { return ( diff --git a/examples/simple-example/pages/api/proxy.ts b/examples/simple-example/pages/api/proxy.ts deleted file mode 100644 index 4de13fe8..00000000 --- a/examples/simple-example/pages/api/proxy.ts +++ /dev/null @@ -1,26 +0,0 @@ -import axios from "axios"; - -export default function handler(req, res) { - if (!req.query.url) { - res.status(200).send({ - error: true, - info: "No url to proxy in query string i.e. ?url=...", - }); - return; - } - axios({ - method: "get", - url: req.query.url, - responseType: "stream", - }) - .then((resp) => { - resp.data.pipe(res); - }) - .catch((err) => { - res.status(400).send({ - error: true, - info: err.message, - detailed: err, - }); - }); -} diff --git a/examples/simple-example/pages/index.tsx b/examples/simple-example/pages/index.tsx index b3e6774f..d48a8084 100644 --- a/examples/simple-example/pages/index.tsx +++ b/examples/simple-example/pages/index.tsx @@ -1,35 +1,21 @@ -import parse from '../lib/markdown'; -import Project from '../lib/project'; import { promises as fs } from 'fs'; import path from 'path'; -import Link from 'next/link'; +import { getProject } from '../lib/octokit'; +import getConfig from 'next/config'; export async function getStaticProps() { + const project_name = getConfig().serverRuntimeConfig.project_name; const jsonDirectory = path.join( process.cwd(), - '/examples/simple-example/datasets.json' + `/examples/${project_name}/datasets.json` ); const repos = await fs.readFile(jsonDirectory, 'utf8'); + const github_pat = getConfig().serverRuntimeConfig.github_pat; const projects = await Promise.all( - JSON.parse(repos).map(async (repo) => { - const project = await Project.getFromGitHub(repo.owner, repo.repo); - - // Defaults to README - const content = project.readme ? project.readme : ''; - - let { mdxSource, frontMatter, excerpt } = await parse(content, '.mdx'); - - if (project.metadata?.resources) { - frontMatter.layout = 'datapackage'; - } - - return { - mdxSource, - frontMatter, - excerpt, - project: project.serialize(), - }; + (JSON.parse(repos)).map(async (repo) => { + const project = await getProject(repo, github_pat); + return { ...project, repo_config: repo }; }) ); return { @@ -53,37 +39,13 @@ export function Datasets({ projects }) { return (
-

+

My Datasets

Here is a list of all my datasets for easy access and sharing

- {/* -
- {projects.map((project) => ( -
-
- - {project.project.owner}/{project.project.name} - -
-
- - Github repo - -
-
- {project.excerpt !== '' ? project.excerpt : 'No description'} -
-
- ))} -
*/}
@@ -93,7 +55,13 @@ export function Datasets({ projects }) { scope="col" className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900" > - Dataset name + Name + + - + +
+ Repo (
- - {project.project.owner}/{project.project.name} - - - {project.project.repo_metadata.description} + {project.repo_config.name + ? project.repo_config.name + : project.full_name + (project.base_path === '/' ? '' : '/' + project.base_path)} - {formatter.format( - new Date(project.project.repo_metadata.updated_at) - )} + {project.full_name} + + {project.repo_config.description + ? project.repo_config.description + : project.description} + + {formatter.format(new Date(project.last_updated))} More info diff --git a/examples/simple-example/styles/global.css b/examples/simple-example/styles/global.css deleted file mode 100644 index d2c4f7ae..00000000 --- a/examples/simple-example/styles/global.css +++ /dev/null @@ -1,67 +0,0 @@ -@import "@flowershow/remark-callouts/styles.css"; - -/* mathjax */ -.math-inline > mjx-container > svg { - display: inline; - align-items: center; -} - -/* smooth scrolling in modern browsers */ -html { - scroll-behavior: smooth !important; -} - -/* tooltip fade-out clip */ -.tooltip-body::after { - content: ""; - position: absolute; - right: 0; - top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */ - height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/ - width: 10rem; - background: linear-gradient( - to right, - rgba(255, 255, 255, 0), - rgba(255, 255, 255, 1) 100% - ); -} - -:is(h2, h3, h4, h5, h6):not(.blogitem-title) { - margin-left: -2rem !important; - padding-left: 2rem !important; - scroll-margin-top: 4.5rem; - position: relative; -} - -.heading-link { - padding: 1px; - position: absolute; - left: 0; - top: 50%; - transform: translateY(-50%); - margin: auto 0; - border-radius: 5px; - background: #1e293b; - opacity: 0; - transition: opacity 0.2s; -} - -.light .heading-link { - /* border: 1px solid #ab2b65; */ - /* background: none; */ - background: #e2e8f0; -} - -:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link { - opacity: 100; -} - -.heading-link svg { - transform: scale(0.75); -} - -@media screen and (max-width: 640px) { - .heading-link { - visibility: hidden; - } -} diff --git a/package-lock.json b/package-lock.json index 63d39f0f..f0a7c470 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "prop-types": "^15.8.1", "react": "18.2.0", "react-dom": "18.2.0", + "react-markdown": "^8.0.7", "react-next-github-btn": "^1.2.1", "react-plotly.js": "^2.6.0", "react-plotlyjs": "^0.4.4", @@ -29695,6 +29696,41 @@ "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true }, + "node_modules/react-markdown": { + "version": "8.0.7", + "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-8.0.7.tgz", + "integrity": "sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==", + "dependencies": { + "@types/hast": "^2.0.0", + "@types/prop-types": "^15.0.0", + "@types/unist": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^2.0.0", + "prop-types": "^15.0.0", + "property-information": "^6.0.0", + "react-is": "^18.0.0", + "remark-parse": "^10.0.0", + "remark-rehype": "^10.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^0.4.0", + "unified": "^10.0.0", + "unist-util-visit": "^4.0.0", + "vfile": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "@types/react": ">=16", + "react": ">=16" + } + }, + "node_modules/react-markdown/node_modules/react-is": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==" + }, "node_modules/react-next-github-btn": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-next-github-btn/-/react-next-github-btn-1.2.1.tgz", @@ -57219,6 +57255,35 @@ "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true }, + "react-markdown": { + "version": "8.0.7", + "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-8.0.7.tgz", + "integrity": "sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==", + "requires": { + "@types/hast": "^2.0.0", + "@types/prop-types": "^15.0.0", + "@types/unist": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^2.0.0", + "prop-types": "^15.0.0", + "property-information": "^6.0.0", + "react-is": "^18.0.0", + "remark-parse": "^10.0.0", + "remark-rehype": "^10.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^0.4.0", + "unified": "^10.0.0", + "unist-util-visit": "^4.0.0", + "vfile": "^5.0.0" + }, + "dependencies": { + "react-is": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==" + } + } + }, "react-next-github-btn": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-next-github-btn/-/react-next-github-btn-1.2.1.tgz", diff --git a/package.json b/package.json index 4c421d3e..d65f09b2 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "prop-types": "^15.8.1", "react": "18.2.0", "react-dom": "18.2.0", + "react-markdown": "^8.0.7", "react-next-github-btn": "^1.2.1", "react-plotly.js": "^2.6.0", "react-plotlyjs": "^0.4.4", diff --git a/site/content/blog/example-data-catalog.md b/site/content/blog/example-data-catalog.md index 2dd4f40d..a3e81c6f 100644 --- a/site/content/blog/example-data-catalog.md +++ b/site/content/blog/example-data-catalog.md @@ -14,15 +14,15 @@ Below are some screenshots: ### Front page -![](https://i.imgur.com/wMk3pGf.png) +![](https://i.imgur.com/jAljJ9C.png) ### Individual dataset page -![](https://i.imgur.com/UipBNEY.png) +![](https://i.imgur.com/AoJd4O0.png) ## Links - [Documentation](/docs/example-data-catalog) - [Repo](https://github.com/datopian/portaljs/tree/main/examples/simple-example) -- [Live Demo](https://example.portaljs.org) \ No newline at end of file +- [Live Demo](https://example.portaljs.org)