[simple-example][lg] - multiple datasets per repo are now possible

This commit is contained in:
Luccas Mateus de Medeiros Gomes 2023-04-22 14:05:21 -03:00
parent 6b2b5f5e87
commit 996568c0f9
20 changed files with 577 additions and 525 deletions

View File

@ -0,0 +1,109 @@
{
"name": "portaljs",
"version": "0.0.0",
"license": "MIT",
"scripts": {},
"private": true,
"dependencies": {
"@apollo/client": "^3.7.11",
"@apollo/react-hooks": "^4.0.0",
"@emotion/react": "^11.10.6",
"@emotion/styled": "^11.10.6",
"@flowershow/core": "^0.4.9",
"@flowershow/markdowndb": "^0.1.0",
"@flowershow/remark-callouts": "^1.0.0",
"@flowershow/remark-embed": "^1.0.0",
"@flowershow/remark-wiki-link": "^1.0.1",
"@headlessui/react": "^1.7.13",
"@heroicons/react": "^2.0.17",
"@mui/icons-material": "^5.11.16",
"@mui/material": "^5.11.16",
"@mui/x-data-grid": "^6.1.0",
"@opentelemetry/api": "^1.4.0",
"@tailwindcss/typography": "^0.5.9",
"@tanstack/react-table": "^8.8.5",
"apollo-cache-inmemory": "^1.6.6",
"apollo-link": "^1.2.14",
"apollo-link-rest": "^0.9.0",
"filesize": "^10.0.7",
"gray-matter": "^4.0.3",
"html-react-parser": "^3.0.15",
"markdown-it": "^13.0.1",
"next": "^13.2.1",
"next-mdx-remote": "^4.4.1",
"next-seo": "^6.0.0",
"next-translate": "^2.0.5",
"nock": "^13.3.0",
"octokit": "^2.0.14",
"papaparse": "^5.4.1",
"plotly.js-basic-dist": "^2.20.0",
"prop-types": "^15.8.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-next-github-btn": "^1.2.1",
"react-plotly.js": "^2.6.0",
"react-plotlyjs": "^0.4.4",
"react-vega": "^7.6.0",
"rehype-autolink-headings": "^6.1.1",
"rehype-katex": "^6.0.2",
"rehype-prism-plus": "^1.5.1",
"rehype-slug": "^5.1.0",
"remark-footnotes": "^4.0.1",
"remark-gfm": "^3.0.1",
"remark-math": "^5.1.1",
"remark-slug": "^7.0.1",
"remark-smartypants": "^2.0.0",
"remark-toc": "^8.0.1",
"slugify": "^1.6.6",
"timeago.js": "^4.0.2",
"tslib": "^2.3.0",
"vega": "^5.24.0",
"xlsx": "^0.18.5"
},
"devDependencies": {
"@babel/preset-react": "^7.14.5",
"@nrwl/cypress": "15.9.2",
"@nrwl/eslint-plugin-nx": "15.9.2",
"@nrwl/jest": "15.9.2",
"@nrwl/js": "15.9.2",
"@nrwl/linter": "15.9.2",
"@nrwl/next": "^15.9.2",
"@nrwl/react": "15.9.2",
"@nrwl/rollup": "15.9.2",
"@nrwl/workspace": "15.9.2",
"@rollup/plugin-url": "^7.0.0",
"@svgr/rollup": "^6.1.2",
"@swc/core": "^1.2.173",
"@swc/helpers": "~0.5.0",
"@swc/jest": "0.2.20",
"@testing-library/react": "14.0.0",
"@types/jest": "^29.4.0",
"@types/node": "18.14.2",
"@types/react": "18.0.28",
"@types/react-dom": "18.0.11",
"@typescript-eslint/eslint-plugin": "^5.36.1",
"@typescript-eslint/parser": "^5.36.1",
"autoprefixer": "10.4.13",
"babel-jest": "^29.4.1",
"cypress": "^12.2.0",
"eslint": "~8.15.0",
"eslint-config-next": "13.1.1",
"eslint-config-prettier": "8.1.0",
"eslint-plugin-cypress": "^2.10.3",
"eslint-plugin-import": "2.27.5",
"eslint-plugin-jsx-a11y": "6.7.1",
"eslint-plugin-react": "7.32.2",
"eslint-plugin-react-hooks": "4.6.0",
"jest": "^29.4.1",
"jest-environment-jsdom": "^29.4.1",
"nx": "15.9.2",
"postcss": "8.4.21",
"prettier": "^2.6.2",
"react-test-renderer": "18.2.0",
"swc-loader": "0.1.15",
"tailwindcss": "3.2.7",
"ts-jest": "^29.0.5",
"ts-node": "10.9.1",
"typescript": "~4.9.5"
}
}

View File

@ -0,0 +1 @@
GITHUB_PAT=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

View File

@ -1,17 +1,66 @@
This is a repo intended to serve as a simple example of a data catalog that get its data from a series of github repos, you can init an example just like this one by.
- Creating a new file inside o `examples` with `create-next-app` like so:
- Cloning the PortalJS repo on your machine
```
git clone https://github.com/datopian/portaljs.git
```
- Creating a new file inside the `examples` folder with `create-next-app` like so:
```
npx create-next-app <app-name> --example https://github.com/datopian/portaljs/tree/main/ --example-path examples/simple-example
```
- Inside `<app-name>` go to the `project.json` file and replace all instances of `simple-example` with `<app-name>`
- Create a `.env` file with the following content
```
PROJECT_NAME=<app-name>
```
- This project uses the github api, which for anonymous users will cap at 50 requests per hour, so you might want to get a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and add it to your .env file like so
```
PAT_TOKEN=<github token>
```
- Edit the file `datasets.json` to your liking, some examples can be found inside this [repo](https://github.com/datasets)
- Run the app using:
```
nx serve <app-name>
```
Congratulations, you now have something similar to this running on `http://localhost:4200`
![](https://i.imgur.com/JrDLycF.png)
If yo go to any one of those pages by clicking on `More info` you will see something similar to this
![](https://i.imgur.com/cpKMS80.png)
Congratulations, you now have something similar to this running on `http://localhost:4200`
![](https://i.imgur.com/jAljJ9C.png)
If yo go to any one of those pages by clicking on `More info` you will see something similar to this
![](https://i.imgur.com/AoJd4O0.png)
## Structure of `datasets.json`
The `datasets.json` file is simply a list of datasets, below you can see a minimal example of a dataset
```json
{
"owner": "fivethirtyeight",
"repo": "data",
"branch": "master",
"files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"],
"readme": "nba-raptor/README.md"
}
```
It has
- A `owner` which is going to be the github repo owner
- A `repo` which is going to be the github repo name
- A `branch` which is going to be the branch to which we need to get the files and the readme
- A list of `files` which is going to be a list of paths with files that you want to show to the world
- A `readme` which is going to be the path to your data description, it can also be a subpath eg: `example/README.md`
You can also add
- A `description` which is useful if you have more than one dataset for each repo, if not provided we are just going to use the repo description
- A `Name` which is useful if you want to give your dataset a nice name, if not provided we are going to use the junction of the `owner` the `repo` + the path of the README, in the exaple above it will be `fivethirtyeight/data/nba-raptor`

View File

@ -1,7 +1,44 @@
[
{ "owner": "datasets", "repo": "oil-prices"},
{ "owner": "datasets", "repo": "investor-flow-of-funds-us"},
{ "owner": "datasets", "repo": "browser-stats"},
{ "owner": "datasets", "repo": "glacier-mass-balance"},
{ "owner": "datasets", "repo": "bond-yields-us-10y"}
{
"owner": "datasets",
"branch": "main",
"repo": "oil-prices",
"files": [
"data/brent-daily.csv",
"data/brent-monthly.csv",
"data/brent-weekly.csv",
"data/brent-year.csv",
"data/wti-daily.csv",
"data/wti-monthly.csv",
"data/wti-weekly.csv",
"data/wti-year.csv"
],
"readme": "README.md"
},
{
"owner": "datasets",
"branch": "main",
"repo": "investor-flow-of-funds-us",
"files": [
"data/monthly.csv",
"data/weekly.csv"
],
"readme": "README.md"
},
{
"owner": "fivethirtyeight",
"repo": "data",
"branch": "master",
"description": "Data about bad drivers",
"name": "Bad Drivers",
"files": ["bad-drivers/bad-drivers.csv"],
"readme": "bad-drivers/README.md"
},
{
"owner": "fivethirtyeight",
"repo": "data",
"branch": "master",
"files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"],
"readme": "nba-raptor/README.md"
}
]

View File

@ -1,11 +0,0 @@
import axios from "axios";
export default function loadUrlProxied(url: string) {
// HACK: duplicate of Excel code - maybe refactor
// if url is external may have CORS issue so we proxy it ...
if (url.startsWith("http")) {
const PROXY_URL = "/api/proxy";
url = PROXY_URL + "?url=" + encodeURIComponent(url);
}
return axios.get(url).then((res) => res.data);
}

View File

@ -1,105 +0,0 @@
import matter from "gray-matter";
import mdxmermaid from "mdx-mermaid";
import { h } from "hastscript";
import remarkCallouts from "@flowershow/remark-callouts";
import remarkEmbed from "@flowershow/remark-embed";
import remarkGfm from "remark-gfm";
import remarkMath from "remark-math";
import remarkSmartypants from "remark-smartypants";
import remarkToc from "remark-toc";
import remarkWikiLink from "@flowershow/remark-wiki-link";
import rehypeAutolinkHeadings from "rehype-autolink-headings";
import rehypeKatex from "rehype-katex";
import rehypeSlug from "rehype-slug";
import rehypePrismPlus from "rehype-prism-plus";
import { serialize } from "next-mdx-remote/serialize";
/**
* Parse a markdown or MDX file to an MDX source form + front matter data
*
* @source: the contents of a markdown or mdx file
* @format: used to indicate to next-mdx-remote which format to use (md or mdx)
* @returns: { mdxSource: mdxSource, frontMatter: ...}
*/
const parse = async function (source, format) {
const { content, data, excerpt } = matter(source, {
excerpt: (file, options) => {
// Generate an excerpt for the file
file.excerpt = file.content.split("\n\n")[0];
},
});
const mdxSource = await serialize(
{ value: content, path: format },
{
// Optionally pass remark/rehype plugins
mdxOptions: {
remarkPlugins: [
remarkEmbed,
remarkGfm,
[remarkSmartypants, { quotes: false, dashes: "oldschool" }],
remarkMath,
remarkCallouts,
remarkWikiLink,
[
remarkToc,
{
heading: "Table of contents",
tight: true,
},
],
[mdxmermaid, {}],
],
rehypePlugins: [
rehypeSlug,
[
rehypeAutolinkHeadings,
{
properties: { className: 'heading-link' },
test(element) {
return (
["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
element.properties?.id !== "table-of-contents" &&
element.properties?.className !== "blockquote-heading"
);
},
content() {
return [
h(
"svg",
{
xmlns: "http:www.w3.org/2000/svg",
fill: "#ab2b65",
viewBox: "0 0 20 20",
className: "w-5 h-5",
},
[
h("path", {
fillRule: "evenodd",
clipRule: "evenodd",
d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
}),
]
),
];
},
},
],
[rehypeKatex, { output: "mathml" }],
[rehypePrismPlus, { ignoreMissing: true }],
],
format,
},
scope: data,
}
);
return {
mdxSource: mdxSource,
frontMatter: data,
excerpt,
};
};
export default parse;

View File

@ -0,0 +1,147 @@
import { Octokit } from 'octokit';
export interface GithubProject {
owner: string;
repo: string;
branch: string;
files: string[];
readme: string;
description?: string;
name?: string;
}
export async function getProjectReadme(
owner: string,
repo: string,
branch: string,
readme: string,
github_pat?: string
) {
const octokit = new Octokit({ auth: github_pat });
try {
const response = await octokit.rest.repos.getContent({
owner,
repo,
path: readme,
ref: branch,
});
const data = response.data as { content?: string };
const fileContent = data.content ? data.content : '';
if (fileContent === '') {
return null;
}
const decodedContent = Buffer.from(fileContent, 'base64').toString();
return decodedContent;
} catch (error) {
console.log(error);
return null;
}
}
export async function getLastUpdated(
owner: string,
repo: string,
branch: string,
readme: string,
github_pat?: string
) {
const octokit = new Octokit({ auth: github_pat });
try {
const response = await octokit.rest.repos.listCommits({
owner,
repo,
path: readme,
ref: branch,
});
return response.data[0].commit.committer.date;
} catch (error) {
console.log(error);
return null;
}
}
export async function getProjectMetadata(
owner: string,
repo: string,
github_pat?: string
) {
const octokit = new Octokit({ auth: github_pat });
try {
const response = await octokit.rest.repos.get({
owner,
repo,
});
return response.data;
} catch (error) {
console.log(error);
return null;
}
}
export async function getRepoContents(
owner: string,
repo: string,
branch: string,
files: string[],
github_pat?: string
) {
const octokit = new Octokit({ auth: github_pat });
try {
const contents = [];
for (const path of files) {
const response = await octokit.rest.repos.getContent({
owner,
repo,
ref: branch,
path: path,
});
const data = response.data as { download_url?: string, name: string, size: number };
contents.push({ download_url: data.download_url, name: data.name, size: data.size});
}
return contents;
} catch (error) {
console.log(error);
return null;
}
}
export async function getProject(project: GithubProject, github_pat?: string) {
const projectMetadata = await getProjectMetadata(
project.owner,
project.repo,
github_pat
);
if (!projectMetadata) {
return null;
}
const projectReadme = await getProjectReadme(
project.owner,
project.repo,
project.branch,
project.readme,
github_pat
);
if (!projectReadme) {
return null;
}
const projectData = await getRepoContents(
project.owner,
project.repo,
project.branch,
project.files,
github_pat
);
if (!projectData) {
return null;
}
const projectBase = project.readme.split('/').length > 1
? project.readme.split('/').slice(0, -1).join('/')
: '/'
const last_updated = await getLastUpdated(
project.owner,
project.repo,
project.branch,
projectBase,
github_pat
);
return { ...projectMetadata, files: projectData, readmeContent: projectReadme, last_updated, base_path: projectBase };
}

View File

@ -1,16 +0,0 @@
import papa from "papaparse";
const parseCsv = (csv) => {
csv = csv.trim();
const rawdata = papa.parse(csv, { header: true });
const cols = rawdata.meta.fields.map((r, i) => {
return { key: r, name: r };
});
return {
rows: rawdata.data,
fields: cols,
};
};
export default parseCsv;

View File

@ -1,60 +0,0 @@
import * as crypto from "crypto";
import axios from "axios";
import { Octokit } from "octokit"
export default class Project {
id: string;
name: string;
owner: string;
github_repo: string;
readme: string;
metadata: any;
repo_metadata: any;
constructor(owner: string, name: string) {
this.name = name;
this.owner = owner;
this.github_repo = `https://github.com/${owner}/${name}`;
// TODO: using the GitHub repo to set the id is not a good idea
// since repos can be renamed and then we are going to end up with
// a duplicate
const encodedGHRepo = Buffer.from(this.github_repo, "utf-8").toString();
this.id = crypto.createHash("sha1").update(encodedGHRepo).digest("hex");
}
initFromGitHub = async () => {
const octokit = new Octokit()
// TODO: what if the repo doesn't exist?
await this.getFileContent("README.md")
.then((content) => (this.readme = content))
.catch((e) => (this.readme = null));
await this.getFileContent("datapackage.json")
.then((content) => (this.metadata = content))
.catch((e) => (this.metadata = {}));
const github_metadata = await octokit.rest.repos.get({ owner: this.owner, repo: this.name })
this.repo_metadata = github_metadata.data ? github_metadata.data : null
};
getFileContent = (path, branch = "main") => {
return axios
.get(
`https://raw.githubusercontent.com/${this.owner}/${this.name}/${branch}/${path}`
)
.then((res) => res.data);
};
serialize() {
return JSON.parse(JSON.stringify(this));
}
static async getFromGitHub(owner: string, name: string) {
const project = new Project(owner, name);
await project.initFromGitHub();
return project;
}
}

View File

@ -1,47 +0,0 @@
export function convertSimpleToVegaLite(view, resource) {
const x = resource.schema.fields.find((f) => f.name === view.spec.group);
const y = resource.schema.fields.find((f) => f.name === view.spec.series[0]);
const xType = inferVegaType(x.type);
const yType = inferVegaType(y.type);
let vegaLiteSpec = {
$schema: "https://vega.github.io/schema/vega-lite/v5.json",
mark: {
type: view.spec.type,
color: "black",
strokeWidth: 1,
tooltip: true,
},
title: view.title,
width: 500,
height: 300,
selection: {
grid: {
type: "interval",
bind: "scales",
},
},
encoding: {
x: {
field: x.name,
type: xType,
},
y: {
field: y.name,
type: yType,
},
},
};
return vegaLiteSpec;
}
const inferVegaType = (fieldType) => {
switch (fieldType) {
case "date":
return "Temporal";
case "number":
return "Quantitative";
}
};

View File

@ -8,16 +8,6 @@ const nextConfig = {
async rewrites() {
return {
beforeFiles: [
{
source: "/@org/:org/:project/:file(\.\+\\\.\.\+\$)",
destination:
'/api/proxy?url=https://raw.githubusercontent.com/:org/:project/main/:file',
},
{
source: "/@:org/:project/:file(\.\+\\\.\.\+\$)",
destination:
'/api/proxy?url=https://raw.githubusercontent.com/:org/:project/main/:file',
},
{
source: '/@:org/:project*',
destination: '/@org/:org/:project*',
@ -25,6 +15,10 @@ const nextConfig = {
],
};
},
serverRuntimeConfig: {
github_pat: process.env.GITHUB_PAT ? process.env.GITHUB_PAT : null,
project_name: process.env.PROJECT_NAME ? process.env.PROJECT_NAME : 'simple-example'
},
nx: {
// Set this to true if you would like to use SVGR
// See: https://github.com/gregberge/svgr

View File

@ -0,0 +1,122 @@
import Head from 'next/head';
import { useRouter } from 'next/router';
import { NextSeo } from 'next-seo';
import { promises as fs } from 'fs';
import path from 'path';
import getConfig from 'next/config';
import { getProject, GithubProject } from '../../../lib/octokit';
import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
import Link from 'next/link';
export default function ProjectPage({ project }) {
return (
<>
<NextSeo title={`PortalJS - @${project.repo_config.owner}/${project.repo_config.repo}${project.base_path !== '/' ? '/' + project.base_path : ''}`} />
<main className="prose mx-auto my-8">
<Link href='/'>Back to homepage</Link>
<h1 className="mb-0">Data</h1>
<div className="inline-block min-w-full py-2 align-middle">
<table className="min-w-full divide-y divide-gray-300">
<thead>
<tr>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Name
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Size
</th>
</tr>
</thead>
<tbody className="divide-y divide-gray-200">
{project.files.map((file) => (
<tr key={file.download_url}>
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
<a href={file.download_url}>{file.name}</a>
</td>
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
{file.size} Bytes
</td>
</tr>
))}
</tbody>
</table>
</div>
<h1>Readme</h1>
<ReactMarkdown remarkPlugins={[remarkGfm]}>
{project.readmeContent}
</ReactMarkdown>
</main>
</>
);
}
// Generates `/posts/1` and `/posts/2`
export async function getStaticPaths() {
const project_name = getConfig().serverRuntimeConfig.project_name;
const jsonDirectory = path.join(
process.cwd(),
`/examples/${project_name}/datasets.json`
);
const repos = await fs.readFile(jsonDirectory, 'utf8');
return {
paths: JSON.parse(repos).map((repo) => {
const projectPath =
repo.readme.split('/').length > 1
? repo.readme.split('/').slice(0, -1)
: null;
let path = [repo.repo];
if (projectPath) {
projectPath.forEach((element) => {
path.push(element);
});
}
return {
params: { org: repo.owner, path },
};
}),
fallback: false, // can also be true or 'blocking'
};
}
export async function getStaticProps({ params }) {
const project_name = getConfig().serverRuntimeConfig.project_name;
const jsonDirectory = path.join(
process.cwd(),
`/examples/${project_name}/datasets.json`
);
const reposFile = await fs.readFile(jsonDirectory, 'utf8');
const repos: GithubProject[] = JSON.parse(reposFile);
const repo = repos.find((_repo) => {
const projectPath =
_repo.readme.split('/').length > 1
? _repo.readme.split('/').slice(0, -1)
: null;
let path = [_repo.repo];
if (projectPath) {
projectPath.forEach((element) => {
path.push(element);
});
}
return (
_repo.owner == params.org &&
JSON.stringify(path) === JSON.stringify(params.path)
);
});
const github_pat = getConfig().serverRuntimeConfig.github_pat;
const project = await getProject(repo, github_pat);
return {
props: {
project: { ...project, repo_config: repo },
},
};
}

View File

@ -1,110 +0,0 @@
import Head from 'next/head';
import { useRouter } from 'next/router';
import DRD from '../../../../components/drd/DRD';
import parse from '../../../../lib/markdown';
import Project from '../../../../lib/project';
import { NextSeo } from 'next-seo';
import MDLayout from 'examples/simple-example/components/MDLayout';
import { promises as fs } from 'fs';
import path from 'path';
function CollectionsLayout({ children, ...frontMatter }) {
const { title, date, description } = frontMatter;
return (
<article className="docs prose text-primary dark:text-primary-dark dark:prose-invert prose-headings:font-headings prose-a:break-words mx-auto p-6">
<header>
<div className="mb-6">
{date && (
<p className="text-sm text-zinc-400 dark:text-zinc-500">
<time dateTime={date}>{date}</time>
</p>
)}
{title && <h1 className="mb-2">{title}</h1>}
{description && <p className="text-xl mt-0">{description}</p>}
</div>
</header>
<section>{children}</section>
</article>
);
}
export default function ProjectPage({
mdxSource,
frontMatter,
excerpt,
project,
}) {
const router = useRouter();
return (
<>
<NextSeo title={`PortalJS - @${project.owner}/${project.name}`} />
<Head>
{/*
On index files, add trailling slash to the base path
see notes: https://github.com/datopian/datahub-next/issues/69
*/}
<base href={router.asPath.split('#')[0] + '/'} />
</Head>
<main>
<MDLayout
layout={frontMatter.layout}
excerpt={excerpt}
project={project}
{...frontMatter}
>
<DRD
source={mdxSource}
frictionless={{
views: project.metadata?.views,
resources: project.metadata?.resources,
}}
/>
</MDLayout>
</main>
</>
);
}
// Generates `/posts/1` and `/posts/2`
export async function getStaticPaths() {
const jsonDirectory = path.join(process.cwd(), '/examples/simple-example/datasets.json');
const repos = await fs.readFile(jsonDirectory, 'utf8');
return {
paths: JSON.parse(repos).map(repo => ({ params: { org: repo.owner, project: repo.repo}})),
fallback: false, // can also be true or 'blocking'
}
}
export async function getStaticProps({ params }) {
const { org: orgName, project: projectName } = params;
const project = await Project.getFromGitHub(orgName, projectName);
// Defaults to README
let content = project.readme;
if (content === null) {
return {
notFound: true,
};
}
let { mdxSource, frontMatter, excerpt } = await parse(content, '.mdx');
if (project.metadata?.resources) {
frontMatter.layout = 'datapackage';
}
return {
props: {
mdxSource,
frontMatter,
excerpt,
project: project.serialize(),
},
};
}

View File

@ -1,7 +1,6 @@
import { AppProps } from 'next/app';
import Head from 'next/head';
import './styles.css';
import "../styles/global.css";
function CustomApp({ Component, pageProps }: AppProps) {
return (

View File

@ -1,26 +0,0 @@
import axios from "axios";
export default function handler(req, res) {
if (!req.query.url) {
res.status(200).send({
error: true,
info: "No url to proxy in query string i.e. ?url=...",
});
return;
}
axios({
method: "get",
url: req.query.url,
responseType: "stream",
})
.then((resp) => {
resp.data.pipe(res);
})
.catch((err) => {
res.status(400).send({
error: true,
info: err.message,
detailed: err,
});
});
}

View File

@ -1,35 +1,21 @@
import parse from '../lib/markdown';
import Project from '../lib/project';
import { promises as fs } from 'fs';
import path from 'path';
import Link from 'next/link';
import { getProject } from '../lib/octokit';
import getConfig from 'next/config';
export async function getStaticProps() {
const project_name = getConfig().serverRuntimeConfig.project_name;
const jsonDirectory = path.join(
process.cwd(),
'/examples/simple-example/datasets.json'
`/examples/${project_name}/datasets.json`
);
const repos = await fs.readFile(jsonDirectory, 'utf8');
const github_pat = getConfig().serverRuntimeConfig.github_pat;
const projects = await Promise.all(
JSON.parse(repos).map(async (repo) => {
const project = await Project.getFromGitHub(repo.owner, repo.repo);
// Defaults to README
const content = project.readme ? project.readme : '';
let { mdxSource, frontMatter, excerpt } = await parse(content, '.mdx');
if (project.metadata?.resources) {
frontMatter.layout = 'datapackage';
}
return {
mdxSource,
frontMatter,
excerpt,
project: project.serialize(),
};
(JSON.parse(repos)).map(async (repo) => {
const project = await getProject(repo, github_pat);
return { ...project, repo_config: repo };
})
);
return {
@ -53,37 +39,13 @@ export function Datasets({ projects }) {
return (
<div className="bg-white">
<div className="mx-auto max-w-7xl px-6 py-16 sm:py-24 lg:px-8">
<h2 className="text-2xl font-bold leading-10 tracking-tight text-indigo-500">
<h2 className="text-2xl font-bold leading-10 tracking-tight">
My Datasets
</h2>
<p className="mt-6 max-w-2xl text-base leading-7 text-gray-600">
Here is a list of all my datasets for easy access and sharing
</p>
<div className="mt-20">
{/*
<dl className="space-y-16 sm:grid sm:grid-cols-2 sm:gap-x-6 sm:gap-y-16 sm:space-y-0 lg:grid-cols-3 lg:gap-x-10">
{projects.map((project) => (
<div>
<dt className="text-base font-semibold leading-7 text-gray-900">
<Link
href={`@${project.project.owner}/${project.project.name}`}
>
{project.project.owner}/{project.project.name}
</Link>
</dt>
<dt className="text-base font-semibold leading-7 text-indigo-600">
<a
href={`https://github.com/${project.project.owner}/${project.project.name}`}
>
Github repo
</a>
</dt>
<dd className="mt-2 text-base leading-7 text-gray-600">
{project.excerpt !== '' ? project.excerpt : 'No description'}
</dd>
</div>
))}
</dl> */}
<div className="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
<div className="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
<table className="min-w-full divide-y divide-gray-300">
@ -93,7 +55,13 @@ export function Datasets({ projects }) {
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Dataset name
Name
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Repo
</th>
<th
scope="col"
@ -117,22 +85,24 @@ export function Datasets({ projects }) {
{projects.map((project) => (
<tr>
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
<a href={project.project.repo_metadata.html_url}>
{project.project.owner}/{project.project.name}
</a>
</td>
<td className="px-3 py-4 text-sm text-gray-500">
{project.project.repo_metadata.description}
{project.repo_config.name
? project.repo_config.name
: project.full_name + (project.base_path === '/' ? '' : '/' + project.base_path)}
</td>
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
{formatter.format(
new Date(project.project.repo_metadata.updated_at)
)}
<a href={project.html_url}>{project.full_name}</a>
</td>
<td className="px-3 py-4 text-sm text-gray-500">
{project.repo_config.description
? project.repo_config.description
: project.description}
</td>
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
{formatter.format(new Date(project.last_updated))}
</td>
<td className="relative whitespace-nowrap py-4 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
<a
href={`/@${project.project.owner}/${project.project.name}`}
className="text-indigo-600 hover:text-indigo-900"
href={`/@${project.repo_config.owner}/${project.repo_config.repo}/${project.base_path === '/' ? '' : project.base_path}`}
>
More info
</a>

View File

@ -1,67 +0,0 @@
@import "@flowershow/remark-callouts/styles.css";
/* mathjax */
.math-inline > mjx-container > svg {
display: inline;
align-items: center;
}
/* smooth scrolling in modern browsers */
html {
scroll-behavior: smooth !important;
}
/* tooltip fade-out clip */
.tooltip-body::after {
content: "";
position: absolute;
right: 0;
top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
width: 10rem;
background: linear-gradient(
to right,
rgba(255, 255, 255, 0),
rgba(255, 255, 255, 1) 100%
);
}
:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
margin-left: -2rem !important;
padding-left: 2rem !important;
scroll-margin-top: 4.5rem;
position: relative;
}
.heading-link {
padding: 1px;
position: absolute;
left: 0;
top: 50%;
transform: translateY(-50%);
margin: auto 0;
border-radius: 5px;
background: #1e293b;
opacity: 0;
transition: opacity 0.2s;
}
.light .heading-link {
/* border: 1px solid #ab2b65; */
/* background: none; */
background: #e2e8f0;
}
:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
opacity: 100;
}
.heading-link svg {
transform: scale(0.75);
}
@media screen and (max-width: 640px) {
.heading-link {
visibility: hidden;
}
}

65
package-lock.json generated
View File

@ -44,6 +44,7 @@
"prop-types": "^15.8.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-markdown": "^8.0.7",
"react-next-github-btn": "^1.2.1",
"react-plotly.js": "^2.6.0",
"react-plotlyjs": "^0.4.4",
@ -29695,6 +29696,41 @@
"integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==",
"dev": true
},
"node_modules/react-markdown": {
"version": "8.0.7",
"resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-8.0.7.tgz",
"integrity": "sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==",
"dependencies": {
"@types/hast": "^2.0.0",
"@types/prop-types": "^15.0.0",
"@types/unist": "^2.0.0",
"comma-separated-tokens": "^2.0.0",
"hast-util-whitespace": "^2.0.0",
"prop-types": "^15.0.0",
"property-information": "^6.0.0",
"react-is": "^18.0.0",
"remark-parse": "^10.0.0",
"remark-rehype": "^10.0.0",
"space-separated-tokens": "^2.0.0",
"style-to-object": "^0.4.0",
"unified": "^10.0.0",
"unist-util-visit": "^4.0.0",
"vfile": "^5.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
},
"peerDependencies": {
"@types/react": ">=16",
"react": ">=16"
}
},
"node_modules/react-markdown/node_modules/react-is": {
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
"integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w=="
},
"node_modules/react-next-github-btn": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/react-next-github-btn/-/react-next-github-btn-1.2.1.tgz",
@ -57219,6 +57255,35 @@
"integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==",
"dev": true
},
"react-markdown": {
"version": "8.0.7",
"resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-8.0.7.tgz",
"integrity": "sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==",
"requires": {
"@types/hast": "^2.0.0",
"@types/prop-types": "^15.0.0",
"@types/unist": "^2.0.0",
"comma-separated-tokens": "^2.0.0",
"hast-util-whitespace": "^2.0.0",
"prop-types": "^15.0.0",
"property-information": "^6.0.0",
"react-is": "^18.0.0",
"remark-parse": "^10.0.0",
"remark-rehype": "^10.0.0",
"space-separated-tokens": "^2.0.0",
"style-to-object": "^0.4.0",
"unified": "^10.0.0",
"unist-util-visit": "^4.0.0",
"vfile": "^5.0.0"
},
"dependencies": {
"react-is": {
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
"integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w=="
}
}
},
"react-next-github-btn": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/react-next-github-btn/-/react-next-github-btn-1.2.1.tgz",

View File

@ -40,6 +40,7 @@
"prop-types": "^15.8.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-markdown": "^8.0.7",
"react-next-github-btn": "^1.2.1",
"react-plotly.js": "^2.6.0",
"react-plotlyjs": "^0.4.4",

View File

@ -14,15 +14,15 @@ Below are some screenshots:
### Front page
![](https://i.imgur.com/wMk3pGf.png)
![](https://i.imgur.com/jAljJ9C.png)
### Individual dataset page
![](https://i.imgur.com/UipBNEY.png)
![](https://i.imgur.com/AoJd4O0.png)
## Links
- [Documentation](/docs/example-data-catalog)
- [Repo](https://github.com/datopian/portaljs/tree/main/examples/simple-example)
- [Live Demo](https://example.portaljs.org)
- [Live Demo](https://example.portaljs.org)