Merge pull request #792 from datopian/multiple_datasets_simple_example
Multiple datasets simple example
This commit is contained in:
Anuar Ustayev (aka Anu)
GitHub
commit
6e90f1897b
6
.gitignore
vendored
6
.gitignore
vendored
@ -39,4 +39,8 @@ testem.log
|
||||
Thumbs.db
|
||||
|
||||
# Next.js
|
||||
.next
|
||||
.next
|
||||
|
||||
# Env
|
||||
.env
|
||||
**/.env
|
||||
|
||||
@ -1,17 +1,66 @@
|
||||
This is a repo intended to serve as a simple example of a data catalog that get its data from a series of github repos, you can init an example just like this one by.
|
||||
|
||||
- Creating a new file inside o `examples` with `create-next-app` like so:
|
||||
- Cloning the PortalJS repo on your machine
|
||||
|
||||
```
|
||||
git clone https://github.com/datopian/portaljs.git
|
||||
```
|
||||
|
||||
- Creating a new file inside the `examples` folder with `create-next-app` like so:
|
||||
|
||||
```
|
||||
npx create-next-app <app-name> --example https://github.com/datopian/portaljs/tree/main/ --example-path examples/simple-example
|
||||
```
|
||||
|
||||
- Inside `<app-name>` go to the `project.json` file and replace all instances of `simple-example` with `<app-name>`
|
||||
- Create a `.env` file with the following content
|
||||
|
||||
```
|
||||
PROJECT_NAME=<app-name>
|
||||
```
|
||||
|
||||
- This project uses the github api, which for anonymous users will cap at 50 requests per hour, so you might want to get a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and add it to your .env file like so
|
||||
|
||||
```
|
||||
GITHUB_PAT=<github token>
|
||||
```
|
||||
|
||||
- Edit the file `datasets.json` to your liking, some examples can be found inside this [repo](https://github.com/datasets)
|
||||
- Run the app using:
|
||||
|
||||
```
|
||||
nx serve <app-name>
|
||||
```
|
||||
Congratulations, you now have something similar to this running on `http://localhost:4200`
|
||||
|
||||
If yo go to any one of those pages by clicking on `More info` you will see something similar to this
|
||||
|
||||
|
||||
Congratulations, you now have something similar to this running on `http://localhost:4200`
|
||||
|
||||
If yo go to any one of those pages by clicking on `More info` you will see something similar to this
|
||||
|
||||
|
||||
|
||||
## Structure of `datasets.json`
|
||||
|
||||
The `datasets.json` file is simply a list of datasets, below you can see a minimal example of a dataset
|
||||
|
||||
```json
|
||||
{
|
||||
"owner": "fivethirtyeight",
|
||||
"repo": "data",
|
||||
"branch": "master",
|
||||
"files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"],
|
||||
"readme": "nba-raptor/README.md"
|
||||
}
|
||||
```
|
||||
|
||||
It has
|
||||
|
||||
- A `owner` which is going to be the github repo owner
|
||||
- A `repo` which is going to be the github repo name
|
||||
- A `branch` which is going to be the branch to which we need to get the files and the readme
|
||||
- A list of `files` which is going to be a list of paths with files that you want to show to the world
|
||||
- A `readme` which is going to be the path to your data description, it can also be a subpath eg: `example/README.md`
|
||||
|
||||
You can also add
|
||||
|
||||
- A `description` which is useful if you have more than one dataset for each repo, if not provided we are just going to use the repo description
|
||||
- A `Name` which is useful if you want to give your dataset a nice name, if not provided we are going to use the junction of the `owner` the `repo` + the path of the README, in the exaple above it will be `fivethirtyeight/data/nba-raptor`
|
||||
|
||||
@ -1,85 +0,0 @@
|
||||
import FrictionlessViewFactory from "./drd/FrictionlessView";
|
||||
import Table from "./drd/Table";
|
||||
|
||||
/* eslint import/no-default-export: off */
|
||||
function DatapackageLayout({ children, project, excerpt }) {
|
||||
const { metadata } = project;
|
||||
|
||||
const title = metadata.title;
|
||||
const resources = metadata.resources;
|
||||
const views = metadata.views;
|
||||
|
||||
const FrictionlessView = FrictionlessViewFactory({ views, resources });
|
||||
|
||||
return (
|
||||
<article className="docs prose text-primary dark:text-primary-dark dark:prose-invert prose-headings:font-headings prose-a:break-words mx-auto p-6">
|
||||
<header>
|
||||
{title && <h1 className="mb-4">{title}</h1>}
|
||||
<a
|
||||
className="font-semibold mb-4"
|
||||
target="_blank"
|
||||
href={project.github_repo}
|
||||
>
|
||||
@{project.owner} / {project.name}
|
||||
</a>
|
||||
{excerpt && <p className="text-md">{excerpt}</p>}
|
||||
</header>
|
||||
<section className="mt-10">
|
||||
{views.map((view, i) => {
|
||||
return (
|
||||
<div key={`visualization-${i}`}>
|
||||
<FrictionlessView viewId={i} />
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</section>
|
||||
<section className="mt-10">
|
||||
<h2>Data files</h2>
|
||||
<table className="table-auto">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>File</th>
|
||||
<th>Title</th>
|
||||
<th>Format</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{resources.map((r) => {
|
||||
return (
|
||||
<tr key={`resources-list-${r.name}`}>
|
||||
<td>
|
||||
<a
|
||||
target="_blank"
|
||||
href={`https://github.com/${project.owner}/${project.name}/blob/main/${r.path}`}
|
||||
>
|
||||
{r.path}
|
||||
</a>
|
||||
</td>
|
||||
<td>{r.title}</td>
|
||||
<td>{r.format.toUpperCase()}</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
{resources.slice(0, 5).map((resource) => {
|
||||
return (
|
||||
<div key={`resource-preview-${resource.name}`} className="mt-10">
|
||||
<h3>{resource.title || resource.name || resource.path}</h3>
|
||||
<Table url={resource.path} />
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</section>
|
||||
<hr />
|
||||
<section>
|
||||
<h2>Read me</h2>
|
||||
{children}
|
||||
</section>
|
||||
</article>
|
||||
);
|
||||
}
|
||||
|
||||
export default function MDLayout({ children, layout, ...props }) {
|
||||
return <DatapackageLayout project={props.project} excerpt={props.excerpt}>{children}</DatapackageLayout>;
|
||||
}
|
||||
@ -1,40 +0,0 @@
|
||||
import { MDXRemote } from "next-mdx-remote";
|
||||
import dynamic from "next/dynamic";
|
||||
import { Mermaid } from "@flowershow/core";
|
||||
|
||||
import FrictionlessViewFactory from "./FrictionlessView";
|
||||
|
||||
// Custom components/renderers to pass to MDX.
|
||||
// Since the MDX files aren't loaded by webpack, they have no knowledge of how
|
||||
// to handle import statements. Instead, you must include components in scope
|
||||
// here.
|
||||
const components = {
|
||||
Table: dynamic(() => import("./Table")),
|
||||
mermaid: Mermaid,
|
||||
// Excel: dynamic(() => import('../components/Excel')),
|
||||
// TODO: try and make these dynamic ...
|
||||
Vega: dynamic(() => import("./Vega")),
|
||||
VegaLite: dynamic(() => import("./VegaLite")),
|
||||
LineChart: dynamic(() => import("./LineChart")),
|
||||
} as any;
|
||||
|
||||
export default function DRD({
|
||||
source,
|
||||
frictionless = {
|
||||
views: [],
|
||||
resources: [],
|
||||
},
|
||||
}: {
|
||||
source: any;
|
||||
frictionless?: any;
|
||||
}) {
|
||||
// dynamic() can't be used inside of React rendering
|
||||
// as it needs to be marked in the top level of the
|
||||
// module for preloading to work
|
||||
components.FrictionlessView = FrictionlessViewFactory({
|
||||
views: frictionless.views,
|
||||
resources: frictionless.resources,
|
||||
});
|
||||
|
||||
return <MDXRemote {...source} components={components} />;
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
const DebouncedInput = ({
|
||||
value: initialValue,
|
||||
onChange,
|
||||
debounce = 500,
|
||||
...props
|
||||
}) => {
|
||||
const [value, setValue] = useState(initialValue);
|
||||
|
||||
useEffect(() => {
|
||||
setValue(initialValue);
|
||||
}, [initialValue]);
|
||||
|
||||
useEffect(() => {
|
||||
const timeout = setTimeout(() => {
|
||||
onChange(value);
|
||||
}, debounce);
|
||||
|
||||
return () => clearTimeout(timeout);
|
||||
}, [value]);
|
||||
|
||||
return (
|
||||
<input
|
||||
{...props}
|
||||
value={value}
|
||||
onChange={(e) => setValue(e.target.value)}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
export default DebouncedInput;
|
||||
|
||||
@ -1,55 +0,0 @@
|
||||
// FrictionlessView is a factory because we have to
|
||||
// set the views and resources lists before using it
|
||||
|
||||
import { convertSimpleToVegaLite } from "../../lib/viewSpecConversion";
|
||||
import VegaLite from "./VegaLite";
|
||||
|
||||
export default function FrictionlessViewFactory({
|
||||
views = [],
|
||||
resources = [],
|
||||
}): ({
|
||||
viewId,
|
||||
fullWidth,
|
||||
}: {
|
||||
viewId: number;
|
||||
fullWidth?: boolean;
|
||||
}) => JSX.Element {
|
||||
return ({ viewId, fullWidth = false }) => {
|
||||
if (!(viewId in views)) {
|
||||
console.error(`View ${viewId} not found`);
|
||||
return <></>;
|
||||
}
|
||||
const view = views[viewId];
|
||||
|
||||
let resource;
|
||||
if (resources.length > 1) {
|
||||
resource = resources.find((r) => r.name === view.resourceName);
|
||||
} else {
|
||||
resource = resources[0];
|
||||
}
|
||||
|
||||
if (!resource) {
|
||||
console.error(`Resource not found for view id ${viewId}`);
|
||||
return <></>;
|
||||
}
|
||||
|
||||
let vegaSpec;
|
||||
switch (view.specType) {
|
||||
case "simple":
|
||||
vegaSpec = convertSimpleToVegaLite(view, resource);
|
||||
break;
|
||||
// ... other conversions
|
||||
}
|
||||
|
||||
vegaSpec.data = { url: resource.path };
|
||||
|
||||
return (
|
||||
<VegaLite
|
||||
fullWidth={fullWidth}
|
||||
spec={vegaSpec}
|
||||
actions={{ editor: false }}
|
||||
downloadFileName={resource.name}
|
||||
/>
|
||||
);
|
||||
};
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
import { VegaLite } from "react-vega";
|
||||
|
||||
export default function LineChart({
|
||||
data = [],
|
||||
fullWidth = false,
|
||||
title = "",
|
||||
}) {
|
||||
var tmp = data;
|
||||
if (Array.isArray(data)) {
|
||||
tmp = data.map((r, i) => {
|
||||
return { x: r[0], y: r[1] };
|
||||
});
|
||||
}
|
||||
const vegaData = { table: tmp };
|
||||
const spec = {
|
||||
$schema: "https://vega.github.io/schema/vega-lite/v5.json",
|
||||
title,
|
||||
width: "container" as "container",
|
||||
height: 300,
|
||||
mark: {
|
||||
type: "line" as "line",
|
||||
color: "black",
|
||||
strokeWidth: 1,
|
||||
tooltip: true,
|
||||
},
|
||||
data: {
|
||||
name: "table",
|
||||
},
|
||||
selection: {
|
||||
grid: {
|
||||
type: "interval" as "interval",
|
||||
bind: "scales",
|
||||
},
|
||||
},
|
||||
encoding: {
|
||||
x: {
|
||||
field: "x",
|
||||
timeUnit: "year",
|
||||
type: "temporal" as "temporal",
|
||||
},
|
||||
y: {
|
||||
field: "y",
|
||||
type: "quantitative" as "temporal",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return <VegaLite data={vegaData} spec={spec} />;
|
||||
}
|
||||
@ -1,188 +0,0 @@
|
||||
import {
|
||||
createColumnHelper,
|
||||
FilterFn,
|
||||
flexRender,
|
||||
getCoreRowModel,
|
||||
getFilteredRowModel,
|
||||
getPaginationRowModel,
|
||||
getSortedRowModel,
|
||||
useReactTable,
|
||||
} from "@tanstack/react-table";
|
||||
|
||||
import {
|
||||
ArrowDownIcon,
|
||||
ArrowUpIcon,
|
||||
ChevronDoubleLeftIcon,
|
||||
ChevronDoubleRightIcon,
|
||||
ChevronLeftIcon,
|
||||
ChevronRightIcon,
|
||||
} from "@heroicons/react/24/solid";
|
||||
|
||||
import React, { useEffect, useMemo, useState } from "react";
|
||||
|
||||
import loadUrlProxied from "../../lib/loadUrlProxied";
|
||||
import parseCsv from "../../lib/parseCsv";
|
||||
import DebouncedInput from "./DebouncedInput";
|
||||
|
||||
const Table = ({
|
||||
data: ogData = [],
|
||||
cols: ogCols = [],
|
||||
csv = "",
|
||||
url = "",
|
||||
}) => {
|
||||
if (csv) {
|
||||
const out = parseCsv(csv);
|
||||
ogData = out.rows;
|
||||
ogCols = out.fields;
|
||||
}
|
||||
|
||||
const [data, setData] = React.useState(ogData);
|
||||
const [cols, setCols] = React.useState(ogCols);
|
||||
const [error, setError] = React.useState(""); // TODO: add error handling
|
||||
|
||||
const tableCols = useMemo(() => {
|
||||
const columnHelper = createColumnHelper();
|
||||
return cols.map((c) =>
|
||||
columnHelper.accessor(c.key, {
|
||||
header: () => c.name,
|
||||
cell: (info) => info.getValue(),
|
||||
})
|
||||
);
|
||||
}, [data, cols]);
|
||||
|
||||
const [globalFilter, setGlobalFilter] = useState("");
|
||||
|
||||
const table = useReactTable({
|
||||
data,
|
||||
columns: tableCols,
|
||||
getCoreRowModel: getCoreRowModel(),
|
||||
state: {
|
||||
globalFilter,
|
||||
},
|
||||
globalFilterFn: globalFilterFn,
|
||||
onGlobalFilterChange: setGlobalFilter,
|
||||
getFilteredRowModel: getFilteredRowModel(),
|
||||
getPaginationRowModel: getPaginationRowModel(),
|
||||
getSortedRowModel: getSortedRowModel(),
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (url) {
|
||||
loadUrlProxied(url).then((data) => {
|
||||
const { rows, fields } = parseCsv(data);
|
||||
setData(rows);
|
||||
setCols(fields);
|
||||
});
|
||||
}
|
||||
}, [url]);
|
||||
|
||||
return (
|
||||
<div>
|
||||
<DebouncedInput
|
||||
value={globalFilter ?? ""}
|
||||
onChange={(value) => setGlobalFilter(String(value))}
|
||||
className="p-2 text-sm shadow border border-block"
|
||||
placeholder="Search all columns..."
|
||||
/>
|
||||
<table>
|
||||
<thead>
|
||||
{table.getHeaderGroups().map((hg) => (
|
||||
<tr key={hg.id}>
|
||||
{hg.headers.map((h) => (
|
||||
<th key={h.id}>
|
||||
<div
|
||||
{...{
|
||||
className: h.column.getCanSort()
|
||||
? "cursor-pointer select-none"
|
||||
: "",
|
||||
onClick: h.column.getToggleSortingHandler(),
|
||||
}}
|
||||
>
|
||||
{flexRender(h.column.columnDef.header, h.getContext())}
|
||||
{{
|
||||
asc: (
|
||||
<ArrowUpIcon className="inline-block ml-2 h-4 w-4" />
|
||||
),
|
||||
desc: (
|
||||
<ArrowDownIcon className="inline-block ml-2 h-4 w-4" />
|
||||
),
|
||||
}[h.column.getIsSorted() as string] ?? (
|
||||
<div className="inline-block ml-2 h-4 w-4" />
|
||||
)}
|
||||
</div>
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</thead>
|
||||
<tbody>
|
||||
{table.getRowModel().rows.map((r) => (
|
||||
<tr key={r.id}>
|
||||
{r.getVisibleCells().map((c) => (
|
||||
<td key={c.id}>
|
||||
{flexRender(c.column.columnDef.cell, c.getContext())}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
<div className="flex gap-2 items-center justify-center">
|
||||
<button
|
||||
className={`w-6 h-6 ${
|
||||
!table.getCanPreviousPage() ? "opacity-25" : "opacity-100"
|
||||
}`}
|
||||
onClick={() => table.setPageIndex(0)}
|
||||
disabled={!table.getCanPreviousPage()}
|
||||
>
|
||||
<ChevronDoubleLeftIcon />
|
||||
</button>
|
||||
<button
|
||||
className={`w-6 h-6 ${
|
||||
!table.getCanPreviousPage() ? "opacity-25" : "opacity-100"
|
||||
}`}
|
||||
onClick={() => table.previousPage()}
|
||||
disabled={!table.getCanPreviousPage()}
|
||||
>
|
||||
<ChevronLeftIcon />
|
||||
</button>
|
||||
<span className="flex items-center gap-1">
|
||||
<div>Page</div>
|
||||
<strong>
|
||||
{table.getState().pagination.pageIndex + 1} of{" "}
|
||||
{table.getPageCount()}
|
||||
</strong>
|
||||
</span>
|
||||
<button
|
||||
className={`w-6 h-6 ${
|
||||
!table.getCanNextPage() ? "opacity-25" : "opacity-100"
|
||||
}`}
|
||||
onClick={() => table.nextPage()}
|
||||
disabled={!table.getCanNextPage()}
|
||||
>
|
||||
<ChevronRightIcon />
|
||||
</button>
|
||||
<button
|
||||
className={`w-6 h-6 ${
|
||||
!table.getCanNextPage() ? "opacity-25" : "opacity-100"
|
||||
}`}
|
||||
onClick={() => table.setPageIndex(table.getPageCount() - 1)}
|
||||
disabled={!table.getCanNextPage()}
|
||||
>
|
||||
<ChevronDoubleRightIcon />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const globalFilterFn: FilterFn<any> = (row, columnId, filterValue: string) => {
|
||||
const search = filterValue.toLowerCase();
|
||||
|
||||
let value = row.getValue(columnId) as string;
|
||||
if (typeof value === "number") value = String(value);
|
||||
|
||||
return value?.toLowerCase().includes(search);
|
||||
};
|
||||
|
||||
export default Table;
|
||||
@ -1,4 +0,0 @@
|
||||
import { Vega as VegaOg } from "react-vega";
|
||||
export default function Vega(props) {
|
||||
return <VegaOg className="w-full" {...props} />;
|
||||
}
|
||||
@ -1,4 +0,0 @@
|
||||
import { VegaLite as VegaOg } from "react-vega";
|
||||
export default function Vega(props) {
|
||||
return <VegaOg className="w-full" {...props} />;
|
||||
}
|
||||
@ -1,7 +1,44 @@
|
||||
[
|
||||
{ "owner": "datasets", "repo": "oil-prices"},
|
||||
{ "owner": "datasets", "repo": "investor-flow-of-funds-us"},
|
||||
{ "owner": "datasets", "repo": "browser-stats"},
|
||||
{ "owner": "datasets", "repo": "glacier-mass-balance"},
|
||||
{ "owner": "datasets", "repo": "bond-yields-us-10y"}
|
||||
{
|
||||
"owner": "datasets",
|
||||
"branch": "main",
|
||||
"repo": "oil-prices",
|
||||
"files": [
|
||||
"data/brent-daily.csv",
|
||||
"data/brent-monthly.csv",
|
||||
"data/brent-weekly.csv",
|
||||
"data/brent-year.csv",
|
||||
"data/wti-daily.csv",
|
||||
"data/wti-monthly.csv",
|
||||
"data/wti-weekly.csv",
|
||||
"data/wti-year.csv"
|
||||
],
|
||||
"readme": "README.md"
|
||||
},
|
||||
{
|
||||
"owner": "datasets",
|
||||
"branch": "main",
|
||||
"repo": "investor-flow-of-funds-us",
|
||||
"files": [
|
||||
"data/monthly.csv",
|
||||
"data/weekly.csv"
|
||||
],
|
||||
"readme": "README.md"
|
||||
},
|
||||
{
|
||||
"owner": "fivethirtyeight",
|
||||
"repo": "data",
|
||||
"branch": "master",
|
||||
"description": "Data about bad drivers",
|
||||
"name": "Bad Drivers",
|
||||
"files": ["bad-drivers/bad-drivers.csv"],
|
||||
"readme": "bad-drivers/README.md"
|
||||
},
|
||||
{
|
||||
"owner": "fivethirtyeight",
|
||||
"repo": "data",
|
||||
"branch": "master",
|
||||
"files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"],
|
||||
"readme": "nba-raptor/README.md"
|
||||
}
|
||||
]
|
||||
|
||||
@ -1,11 +0,0 @@
|
||||
import axios from "axios";
|
||||
|
||||
export default function loadUrlProxied(url: string) {
|
||||
// HACK: duplicate of Excel code - maybe refactor
|
||||
// if url is external may have CORS issue so we proxy it ...
|
||||
if (url.startsWith("http")) {
|
||||
const PROXY_URL = "/api/proxy";
|
||||
url = PROXY_URL + "?url=" + encodeURIComponent(url);
|
||||
}
|
||||
return axios.get(url).then((res) => res.data);
|
||||
}
|
||||
@ -1,105 +0,0 @@
|
||||
import matter from "gray-matter";
|
||||
import mdxmermaid from "mdx-mermaid";
|
||||
import { h } from "hastscript";
|
||||
import remarkCallouts from "@flowershow/remark-callouts";
|
||||
import remarkEmbed from "@flowershow/remark-embed";
|
||||
import remarkGfm from "remark-gfm";
|
||||
import remarkMath from "remark-math";
|
||||
import remarkSmartypants from "remark-smartypants";
|
||||
import remarkToc from "remark-toc";
|
||||
import remarkWikiLink from "@flowershow/remark-wiki-link";
|
||||
import rehypeAutolinkHeadings from "rehype-autolink-headings";
|
||||
import rehypeKatex from "rehype-katex";
|
||||
import rehypeSlug from "rehype-slug";
|
||||
import rehypePrismPlus from "rehype-prism-plus";
|
||||
|
||||
import { serialize } from "next-mdx-remote/serialize";
|
||||
|
||||
/**
|
||||
* Parse a markdown or MDX file to an MDX source form + front matter data
|
||||
*
|
||||
* @source: the contents of a markdown or mdx file
|
||||
* @format: used to indicate to next-mdx-remote which format to use (md or mdx)
|
||||
* @returns: { mdxSource: mdxSource, frontMatter: ...}
|
||||
*/
|
||||
const parse = async function (source, format) {
|
||||
const { content, data, excerpt } = matter(source, {
|
||||
excerpt: (file, options) => {
|
||||
// Generate an excerpt for the file
|
||||
file.excerpt = file.content.split("\n\n")[0];
|
||||
},
|
||||
});
|
||||
|
||||
const mdxSource = await serialize(
|
||||
{ value: content, path: format },
|
||||
{
|
||||
// Optionally pass remark/rehype plugins
|
||||
mdxOptions: {
|
||||
remarkPlugins: [
|
||||
remarkEmbed,
|
||||
remarkGfm,
|
||||
[remarkSmartypants, { quotes: false, dashes: "oldschool" }],
|
||||
remarkMath,
|
||||
remarkCallouts,
|
||||
remarkWikiLink,
|
||||
[
|
||||
remarkToc,
|
||||
{
|
||||
heading: "Table of contents",
|
||||
tight: true,
|
||||
},
|
||||
],
|
||||
[mdxmermaid, {}],
|
||||
],
|
||||
rehypePlugins: [
|
||||
rehypeSlug,
|
||||
[
|
||||
rehypeAutolinkHeadings,
|
||||
{
|
||||
properties: { className: 'heading-link' },
|
||||
test(element) {
|
||||
return (
|
||||
["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
|
||||
element.properties?.id !== "table-of-contents" &&
|
||||
element.properties?.className !== "blockquote-heading"
|
||||
);
|
||||
},
|
||||
content() {
|
||||
return [
|
||||
h(
|
||||
"svg",
|
||||
{
|
||||
xmlns: "http:www.w3.org/2000/svg",
|
||||
fill: "#ab2b65",
|
||||
viewBox: "0 0 20 20",
|
||||
className: "w-5 h-5",
|
||||
},
|
||||
[
|
||||
h("path", {
|
||||
fillRule: "evenodd",
|
||||
clipRule: "evenodd",
|
||||
d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
|
||||
}),
|
||||
]
|
||||
),
|
||||
];
|
||||
},
|
||||
},
|
||||
],
|
||||
[rehypeKatex, { output: "mathml" }],
|
||||
[rehypePrismPlus, { ignoreMissing: true }],
|
||||
],
|
||||
format,
|
||||
},
|
||||
scope: data,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
mdxSource: mdxSource,
|
||||
frontMatter: data,
|
||||
excerpt,
|
||||
};
|
||||
};
|
||||
|
||||
export default parse;
|
||||
147
examples/simple-example/lib/octokit.ts
Normal file
147
examples/simple-example/lib/octokit.ts
Normal file
@ -0,0 +1,147 @@
|
||||
import { Octokit } from 'octokit';
|
||||
|
||||
export interface GithubProject {
|
||||
owner: string;
|
||||
repo: string;
|
||||
branch: string;
|
||||
files: string[];
|
||||
readme: string;
|
||||
description?: string;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
export async function getProjectReadme(
|
||||
owner: string,
|
||||
repo: string,
|
||||
branch: string,
|
||||
readme: string,
|
||||
github_pat?: string
|
||||
) {
|
||||
const octokit = new Octokit({ auth: github_pat });
|
||||
try {
|
||||
const response = await octokit.rest.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: readme,
|
||||
ref: branch,
|
||||
});
|
||||
const data = response.data as { content?: string };
|
||||
const fileContent = data.content ? data.content : '';
|
||||
if (fileContent === '') {
|
||||
return null;
|
||||
}
|
||||
const decodedContent = Buffer.from(fileContent, 'base64').toString();
|
||||
return decodedContent;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getLastUpdated(
|
||||
owner: string,
|
||||
repo: string,
|
||||
branch: string,
|
||||
readme: string,
|
||||
github_pat?: string
|
||||
) {
|
||||
const octokit = new Octokit({ auth: github_pat });
|
||||
try {
|
||||
const response = await octokit.rest.repos.listCommits({
|
||||
owner,
|
||||
repo,
|
||||
path: readme,
|
||||
ref: branch,
|
||||
});
|
||||
return response.data[0].commit.committer.date;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
export async function getProjectMetadata(
|
||||
owner: string,
|
||||
repo: string,
|
||||
github_pat?: string
|
||||
) {
|
||||
const octokit = new Octokit({ auth: github_pat });
|
||||
try {
|
||||
const response = await octokit.rest.repos.get({
|
||||
owner,
|
||||
repo,
|
||||
});
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getRepoContents(
|
||||
owner: string,
|
||||
repo: string,
|
||||
branch: string,
|
||||
files: string[],
|
||||
github_pat?: string
|
||||
) {
|
||||
const octokit = new Octokit({ auth: github_pat });
|
||||
try {
|
||||
const contents = [];
|
||||
for (const path of files) {
|
||||
const response = await octokit.rest.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
ref: branch,
|
||||
path: path,
|
||||
});
|
||||
const data = response.data as { download_url?: string, name: string, size: number };
|
||||
contents.push({ download_url: data.download_url, name: data.name, size: data.size});
|
||||
}
|
||||
return contents;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getProject(project: GithubProject, github_pat?: string) {
|
||||
const projectMetadata = await getProjectMetadata(
|
||||
project.owner,
|
||||
project.repo,
|
||||
github_pat
|
||||
);
|
||||
if (!projectMetadata) {
|
||||
return null;
|
||||
}
|
||||
const projectReadme = await getProjectReadme(
|
||||
project.owner,
|
||||
project.repo,
|
||||
project.branch,
|
||||
project.readme,
|
||||
github_pat
|
||||
);
|
||||
if (!projectReadme) {
|
||||
return null;
|
||||
}
|
||||
const projectData = await getRepoContents(
|
||||
project.owner,
|
||||
project.repo,
|
||||
project.branch,
|
||||
project.files,
|
||||
github_pat
|
||||
);
|
||||
if (!projectData) {
|
||||
return null;
|
||||
}
|
||||
const projectBase = project.readme.split('/').length > 1
|
||||
? project.readme.split('/').slice(0, -1).join('/')
|
||||
: '/'
|
||||
const last_updated = await getLastUpdated(
|
||||
project.owner,
|
||||
project.repo,
|
||||
project.branch,
|
||||
projectBase,
|
||||
github_pat
|
||||
);
|
||||
return { ...projectMetadata, files: projectData, readmeContent: projectReadme, last_updated, base_path: projectBase };
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
import papa from "papaparse";
|
||||
|
||||
const parseCsv = (csv) => {
|
||||
csv = csv.trim();
|
||||
const rawdata = papa.parse(csv, { header: true });
|
||||
const cols = rawdata.meta.fields.map((r, i) => {
|
||||
return { key: r, name: r };
|
||||
});
|
||||
|
||||
return {
|
||||
rows: rawdata.data,
|
||||
fields: cols,
|
||||
};
|
||||
};
|
||||
|
||||
export default parseCsv;
|
||||
@ -1,60 +0,0 @@
|
||||
import * as crypto from "crypto";
|
||||
import axios from "axios";
|
||||
import { Octokit } from "octokit"
|
||||
|
||||
export default class Project {
|
||||
id: string;
|
||||
name: string;
|
||||
owner: string;
|
||||
github_repo: string;
|
||||
readme: string;
|
||||
metadata: any;
|
||||
repo_metadata: any;
|
||||
|
||||
constructor(owner: string, name: string) {
|
||||
this.name = name;
|
||||
this.owner = owner;
|
||||
this.github_repo = `https://github.com/${owner}/${name}`;
|
||||
|
||||
// TODO: using the GitHub repo to set the id is not a good idea
|
||||
// since repos can be renamed and then we are going to end up with
|
||||
// a duplicate
|
||||
const encodedGHRepo = Buffer.from(this.github_repo, "utf-8").toString();
|
||||
this.id = crypto.createHash("sha1").update(encodedGHRepo).digest("hex");
|
||||
}
|
||||
|
||||
initFromGitHub = async () => {
|
||||
const octokit = new Octokit()
|
||||
// TODO: what if the repo doesn't exist?
|
||||
await this.getFileContent("README.md")
|
||||
.then((content) => (this.readme = content))
|
||||
.catch((e) => (this.readme = null));
|
||||
|
||||
await this.getFileContent("datapackage.json")
|
||||
.then((content) => (this.metadata = content))
|
||||
.catch((e) => (this.metadata = {}));
|
||||
|
||||
const github_metadata = await octokit.rest.repos.get({ owner: this.owner, repo: this.name })
|
||||
this.repo_metadata = github_metadata.data ? github_metadata.data : null
|
||||
};
|
||||
|
||||
getFileContent = (path, branch = "main") => {
|
||||
return axios
|
||||
.get(
|
||||
`https://raw.githubusercontent.com/${this.owner}/${this.name}/${branch}/${path}`
|
||||
)
|
||||
.then((res) => res.data);
|
||||
};
|
||||
|
||||
serialize() {
|
||||
return JSON.parse(JSON.stringify(this));
|
||||
}
|
||||
|
||||
static async getFromGitHub(owner: string, name: string) {
|
||||
const project = new Project(owner, name);
|
||||
await project.initFromGitHub();
|
||||
|
||||
return project;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,47 +0,0 @@
|
||||
export function convertSimpleToVegaLite(view, resource) {
|
||||
const x = resource.schema.fields.find((f) => f.name === view.spec.group);
|
||||
const y = resource.schema.fields.find((f) => f.name === view.spec.series[0]);
|
||||
|
||||
const xType = inferVegaType(x.type);
|
||||
const yType = inferVegaType(y.type);
|
||||
|
||||
let vegaLiteSpec = {
|
||||
$schema: "https://vega.github.io/schema/vega-lite/v5.json",
|
||||
mark: {
|
||||
type: view.spec.type,
|
||||
color: "black",
|
||||
strokeWidth: 1,
|
||||
tooltip: true,
|
||||
},
|
||||
title: view.title,
|
||||
width: 500,
|
||||
height: 300,
|
||||
selection: {
|
||||
grid: {
|
||||
type: "interval",
|
||||
bind: "scales",
|
||||
},
|
||||
},
|
||||
encoding: {
|
||||
x: {
|
||||
field: x.name,
|
||||
type: xType,
|
||||
},
|
||||
y: {
|
||||
field: y.name,
|
||||
type: yType,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return vegaLiteSpec;
|
||||
}
|
||||
|
||||
const inferVegaType = (fieldType) => {
|
||||
switch (fieldType) {
|
||||
case "date":
|
||||
return "Temporal";
|
||||
case "number":
|
||||
return "Quantitative";
|
||||
}
|
||||
};
|
||||
@ -8,16 +8,6 @@ const nextConfig = {
|
||||
async rewrites() {
|
||||
return {
|
||||
beforeFiles: [
|
||||
{
|
||||
source: "/@org/:org/:project/:file(\.\+\\\.\.\+\$)",
|
||||
destination:
|
||||
'/api/proxy?url=https://raw.githubusercontent.com/:org/:project/main/:file',
|
||||
},
|
||||
{
|
||||
source: "/@:org/:project/:file(\.\+\\\.\.\+\$)",
|
||||
destination:
|
||||
'/api/proxy?url=https://raw.githubusercontent.com/:org/:project/main/:file',
|
||||
},
|
||||
{
|
||||
source: '/@:org/:project*',
|
||||
destination: '/@org/:org/:project*',
|
||||
@ -25,6 +15,10 @@ const nextConfig = {
|
||||
],
|
||||
};
|
||||
},
|
||||
serverRuntimeConfig: {
|
||||
github_pat: process.env.GITHUB_PAT ? process.env.GITHUB_PAT : null,
|
||||
project_name: process.env.PROJECT_NAME ? process.env.PROJECT_NAME : 'simple-example'
|
||||
},
|
||||
nx: {
|
||||
// Set this to true if you would like to use SVGR
|
||||
// See: https://github.com/gregberge/svgr
|
||||
|
||||
122
examples/simple-example/pages/@org/[org]/[...path].tsx
Normal file
122
examples/simple-example/pages/@org/[org]/[...path].tsx
Normal file
@ -0,0 +1,122 @@
|
||||
import Head from 'next/head';
|
||||
import { useRouter } from 'next/router';
|
||||
|
||||
import { NextSeo } from 'next-seo';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import getConfig from 'next/config';
|
||||
import { getProject, GithubProject } from '../../../lib/octokit';
|
||||
import ReactMarkdown from 'react-markdown';
|
||||
import remarkGfm from 'remark-gfm';
|
||||
import Link from 'next/link';
|
||||
|
||||
export default function ProjectPage({ project }) {
|
||||
return (
|
||||
<>
|
||||
<NextSeo title={`PortalJS - @${project.repo_config.owner}/${project.repo_config.repo}${project.base_path !== '/' ? '/' + project.base_path : ''}`} />
|
||||
<main className="prose mx-auto my-8">
|
||||
<Link href='/'>Back to homepage</Link>
|
||||
<h1 className="mb-0">Data</h1>
|
||||
<div className="inline-block min-w-full py-2 align-middle">
|
||||
<table className="min-w-full divide-y divide-gray-300">
|
||||
<thead>
|
||||
<tr>
|
||||
<th
|
||||
scope="col"
|
||||
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
|
||||
>
|
||||
Name
|
||||
</th>
|
||||
<th
|
||||
scope="col"
|
||||
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
|
||||
>
|
||||
Size
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{project.files.map((file) => (
|
||||
<tr key={file.download_url}>
|
||||
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
|
||||
<a href={file.download_url}>{file.name}</a>
|
||||
</td>
|
||||
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
|
||||
{file.size} Bytes
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h1>Readme</h1>
|
||||
<ReactMarkdown remarkPlugins={[remarkGfm]}>
|
||||
{project.readmeContent}
|
||||
</ReactMarkdown>
|
||||
</main>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// Generates `/posts/1` and `/posts/2`
|
||||
export async function getStaticPaths() {
|
||||
const project_name = getConfig().serverRuntimeConfig.project_name;
|
||||
const jsonDirectory = path.join(
|
||||
process.cwd(),
|
||||
`/examples/${project_name}/datasets.json`
|
||||
);
|
||||
const repos = await fs.readFile(jsonDirectory, 'utf8');
|
||||
|
||||
return {
|
||||
paths: JSON.parse(repos).map((repo) => {
|
||||
const projectPath =
|
||||
repo.readme.split('/').length > 1
|
||||
? repo.readme.split('/').slice(0, -1)
|
||||
: null;
|
||||
let path = [repo.repo];
|
||||
if (projectPath) {
|
||||
projectPath.forEach((element) => {
|
||||
path.push(element);
|
||||
});
|
||||
}
|
||||
return {
|
||||
params: { org: repo.owner, path },
|
||||
};
|
||||
}),
|
||||
fallback: false, // can also be true or 'blocking'
|
||||
};
|
||||
}
|
||||
|
||||
export async function getStaticProps({ params }) {
|
||||
const project_name = getConfig().serverRuntimeConfig.project_name;
|
||||
const jsonDirectory = path.join(
|
||||
process.cwd(),
|
||||
`/examples/${project_name}/datasets.json`
|
||||
);
|
||||
const reposFile = await fs.readFile(jsonDirectory, 'utf8');
|
||||
const repos: GithubProject[] = JSON.parse(reposFile);
|
||||
const repo = repos.find((_repo) => {
|
||||
const projectPath =
|
||||
_repo.readme.split('/').length > 1
|
||||
? _repo.readme.split('/').slice(0, -1)
|
||||
: null;
|
||||
let path = [_repo.repo];
|
||||
if (projectPath) {
|
||||
projectPath.forEach((element) => {
|
||||
path.push(element);
|
||||
});
|
||||
}
|
||||
return (
|
||||
_repo.owner == params.org &&
|
||||
JSON.stringify(path) === JSON.stringify(params.path)
|
||||
);
|
||||
});
|
||||
const github_pat = getConfig().serverRuntimeConfig.github_pat;
|
||||
const project = await getProject(repo, github_pat);
|
||||
return {
|
||||
props: {
|
||||
project: { ...project, repo_config: repo },
|
||||
},
|
||||
};
|
||||
}
|
||||
@ -1,110 +0,0 @@
|
||||
import Head from 'next/head';
|
||||
import { useRouter } from 'next/router';
|
||||
|
||||
import DRD from '../../../../components/drd/DRD';
|
||||
import parse from '../../../../lib/markdown';
|
||||
import Project from '../../../../lib/project';
|
||||
import { NextSeo } from 'next-seo';
|
||||
import MDLayout from 'examples/simple-example/components/MDLayout';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
function CollectionsLayout({ children, ...frontMatter }) {
|
||||
const { title, date, description } = frontMatter;
|
||||
|
||||
return (
|
||||
<article className="docs prose text-primary dark:text-primary-dark dark:prose-invert prose-headings:font-headings prose-a:break-words mx-auto p-6">
|
||||
<header>
|
||||
<div className="mb-6">
|
||||
{date && (
|
||||
<p className="text-sm text-zinc-400 dark:text-zinc-500">
|
||||
<time dateTime={date}>{date}</time>
|
||||
</p>
|
||||
)}
|
||||
{title && <h1 className="mb-2">{title}</h1>}
|
||||
{description && <p className="text-xl mt-0">{description}</p>}
|
||||
</div>
|
||||
</header>
|
||||
<section>{children}</section>
|
||||
</article>
|
||||
);
|
||||
}
|
||||
|
||||
export default function ProjectPage({
|
||||
mdxSource,
|
||||
frontMatter,
|
||||
excerpt,
|
||||
project,
|
||||
}) {
|
||||
const router = useRouter();
|
||||
|
||||
return (
|
||||
<>
|
||||
<NextSeo title={`PortalJS - @${project.owner}/${project.name}`} />
|
||||
<Head>
|
||||
{/*
|
||||
On index files, add trailling slash to the base path
|
||||
see notes: https://github.com/datopian/datahub-next/issues/69
|
||||
*/}
|
||||
<base href={router.asPath.split('#')[0] + '/'} />
|
||||
</Head>
|
||||
<main>
|
||||
<MDLayout
|
||||
layout={frontMatter.layout}
|
||||
excerpt={excerpt}
|
||||
project={project}
|
||||
{...frontMatter}
|
||||
>
|
||||
<DRD
|
||||
source={mdxSource}
|
||||
frictionless={{
|
||||
views: project.metadata?.views,
|
||||
resources: project.metadata?.resources,
|
||||
}}
|
||||
/>
|
||||
</MDLayout>
|
||||
</main>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// Generates `/posts/1` and `/posts/2`
|
||||
export async function getStaticPaths() {
|
||||
const jsonDirectory = path.join(process.cwd(), '/examples/simple-example/datasets.json');
|
||||
const repos = await fs.readFile(jsonDirectory, 'utf8');
|
||||
|
||||
return {
|
||||
paths: JSON.parse(repos).map(repo => ({ params: { org: repo.owner, project: repo.repo}})),
|
||||
fallback: false, // can also be true or 'blocking'
|
||||
}
|
||||
}
|
||||
|
||||
export async function getStaticProps({ params }) {
|
||||
const { org: orgName, project: projectName } = params;
|
||||
|
||||
const project = await Project.getFromGitHub(orgName, projectName);
|
||||
|
||||
// Defaults to README
|
||||
let content = project.readme;
|
||||
|
||||
if (content === null) {
|
||||
return {
|
||||
notFound: true,
|
||||
};
|
||||
}
|
||||
|
||||
let { mdxSource, frontMatter, excerpt } = await parse(content, '.mdx');
|
||||
|
||||
if (project.metadata?.resources) {
|
||||
frontMatter.layout = 'datapackage';
|
||||
}
|
||||
|
||||
return {
|
||||
props: {
|
||||
mdxSource,
|
||||
frontMatter,
|
||||
excerpt,
|
||||
project: project.serialize(),
|
||||
},
|
||||
};
|
||||
}
|
||||
@ -1,7 +1,6 @@
|
||||
import { AppProps } from 'next/app';
|
||||
import Head from 'next/head';
|
||||
import './styles.css';
|
||||
import "../styles/global.css";
|
||||
|
||||
function CustomApp({ Component, pageProps }: AppProps) {
|
||||
return (
|
||||
|
||||
@ -1,26 +0,0 @@
|
||||
import axios from "axios";
|
||||
|
||||
export default function handler(req, res) {
|
||||
if (!req.query.url) {
|
||||
res.status(200).send({
|
||||
error: true,
|
||||
info: "No url to proxy in query string i.e. ?url=...",
|
||||
});
|
||||
return;
|
||||
}
|
||||
axios({
|
||||
method: "get",
|
||||
url: req.query.url,
|
||||
responseType: "stream",
|
||||
})
|
||||
.then((resp) => {
|
||||
resp.data.pipe(res);
|
||||
})
|
||||
.catch((err) => {
|
||||
res.status(400).send({
|
||||
error: true,
|
||||
info: err.message,
|
||||
detailed: err,
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -1,35 +1,21 @@
|
||||
import parse from '../lib/markdown';
|
||||
import Project from '../lib/project';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import Link from 'next/link';
|
||||
import { getProject } from '../lib/octokit';
|
||||
import getConfig from 'next/config';
|
||||
|
||||
export async function getStaticProps() {
|
||||
const project_name = getConfig().serverRuntimeConfig.project_name;
|
||||
const jsonDirectory = path.join(
|
||||
process.cwd(),
|
||||
'/examples/simple-example/datasets.json'
|
||||
`/examples/${project_name}/datasets.json`
|
||||
);
|
||||
const repos = await fs.readFile(jsonDirectory, 'utf8');
|
||||
const github_pat = getConfig().serverRuntimeConfig.github_pat;
|
||||
|
||||
const projects = await Promise.all(
|
||||
JSON.parse(repos).map(async (repo) => {
|
||||
const project = await Project.getFromGitHub(repo.owner, repo.repo);
|
||||
|
||||
// Defaults to README
|
||||
const content = project.readme ? project.readme : '';
|
||||
|
||||
let { mdxSource, frontMatter, excerpt } = await parse(content, '.mdx');
|
||||
|
||||
if (project.metadata?.resources) {
|
||||
frontMatter.layout = 'datapackage';
|
||||
}
|
||||
|
||||
return {
|
||||
mdxSource,
|
||||
frontMatter,
|
||||
excerpt,
|
||||
project: project.serialize(),
|
||||
};
|
||||
(JSON.parse(repos)).map(async (repo) => {
|
||||
const project = await getProject(repo, github_pat);
|
||||
return { ...project, repo_config: repo };
|
||||
})
|
||||
);
|
||||
return {
|
||||
@ -53,37 +39,13 @@ export function Datasets({ projects }) {
|
||||
return (
|
||||
<div className="bg-white">
|
||||
<div className="mx-auto max-w-7xl px-6 py-16 sm:py-24 lg:px-8">
|
||||
<h2 className="text-2xl font-bold leading-10 tracking-tight text-indigo-500">
|
||||
<h2 className="text-2xl font-bold leading-10 tracking-tight">
|
||||
My Datasets
|
||||
</h2>
|
||||
<p className="mt-6 max-w-2xl text-base leading-7 text-gray-600">
|
||||
Here is a list of all my datasets for easy access and sharing
|
||||
</p>
|
||||
<div className="mt-20">
|
||||
{/*
|
||||
<dl className="space-y-16 sm:grid sm:grid-cols-2 sm:gap-x-6 sm:gap-y-16 sm:space-y-0 lg:grid-cols-3 lg:gap-x-10">
|
||||
{projects.map((project) => (
|
||||
<div>
|
||||
<dt className="text-base font-semibold leading-7 text-gray-900">
|
||||
<Link
|
||||
href={`@${project.project.owner}/${project.project.name}`}
|
||||
>
|
||||
{project.project.owner}/{project.project.name}
|
||||
</Link>
|
||||
</dt>
|
||||
<dt className="text-base font-semibold leading-7 text-indigo-600">
|
||||
<a
|
||||
href={`https://github.com/${project.project.owner}/${project.project.name}`}
|
||||
>
|
||||
Github repo
|
||||
</a>
|
||||
</dt>
|
||||
<dd className="mt-2 text-base leading-7 text-gray-600">
|
||||
{project.excerpt !== '' ? project.excerpt : 'No description'}
|
||||
</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl> */}
|
||||
<div className="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
|
||||
<div className="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
|
||||
<table className="min-w-full divide-y divide-gray-300">
|
||||
@ -93,7 +55,13 @@ export function Datasets({ projects }) {
|
||||
scope="col"
|
||||
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
|
||||
>
|
||||
Dataset name
|
||||
Name
|
||||
</th>
|
||||
<th
|
||||
scope="col"
|
||||
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
|
||||
>
|
||||
Repo
|
||||
</th>
|
||||
<th
|
||||
scope="col"
|
||||
@ -117,22 +85,24 @@ export function Datasets({ projects }) {
|
||||
{projects.map((project) => (
|
||||
<tr>
|
||||
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
|
||||
<a href={project.project.repo_metadata.html_url}>
|
||||
{project.project.owner}/{project.project.name}
|
||||
</a>
|
||||
</td>
|
||||
<td className="px-3 py-4 text-sm text-gray-500">
|
||||
{project.project.repo_metadata.description}
|
||||
{project.repo_config.name
|
||||
? project.repo_config.name
|
||||
: project.full_name + (project.base_path === '/' ? '' : '/' + project.base_path)}
|
||||
</td>
|
||||
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
|
||||
{formatter.format(
|
||||
new Date(project.project.repo_metadata.updated_at)
|
||||
)}
|
||||
<a href={project.html_url}>{project.full_name}</a>
|
||||
</td>
|
||||
<td className="px-3 py-4 text-sm text-gray-500">
|
||||
{project.repo_config.description
|
||||
? project.repo_config.description
|
||||
: project.description}
|
||||
</td>
|
||||
<td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
|
||||
{formatter.format(new Date(project.last_updated))}
|
||||
</td>
|
||||
<td className="relative whitespace-nowrap py-4 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
|
||||
<a
|
||||
href={`/@${project.project.owner}/${project.project.name}`}
|
||||
className="text-indigo-600 hover:text-indigo-900"
|
||||
href={`/@${project.repo_config.owner}/${project.repo_config.repo}/${project.base_path === '/' ? '' : project.base_path}`}
|
||||
>
|
||||
More info
|
||||
</a>
|
||||
|
||||
@ -1,67 +0,0 @@
|
||||
@import "@flowershow/remark-callouts/styles.css";
|
||||
|
||||
/* mathjax */
|
||||
.math-inline > mjx-container > svg {
|
||||
display: inline;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
/* smooth scrolling in modern browsers */
|
||||
html {
|
||||
scroll-behavior: smooth !important;
|
||||
}
|
||||
|
||||
/* tooltip fade-out clip */
|
||||
.tooltip-body::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
right: 0;
|
||||
top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
|
||||
height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
|
||||
width: 10rem;
|
||||
background: linear-gradient(
|
||||
to right,
|
||||
rgba(255, 255, 255, 0),
|
||||
rgba(255, 255, 255, 1) 100%
|
||||
);
|
||||
}
|
||||
|
||||
:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
|
||||
margin-left: -2rem !important;
|
||||
padding-left: 2rem !important;
|
||||
scroll-margin-top: 4.5rem;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.heading-link {
|
||||
padding: 1px;
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
margin: auto 0;
|
||||
border-radius: 5px;
|
||||
background: #1e293b;
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
|
||||
.light .heading-link {
|
||||
/* border: 1px solid #ab2b65; */
|
||||
/* background: none; */
|
||||
background: #e2e8f0;
|
||||
}
|
||||
|
||||
:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
|
||||
opacity: 100;
|
||||
}
|
||||
|
||||
.heading-link svg {
|
||||
transform: scale(0.75);
|
||||
}
|
||||
|
||||
@media screen and (max-width: 640px) {
|
||||
.heading-link {
|
||||
visibility: hidden;
|
||||
}
|
||||
}
|
||||
65
package-lock.json
generated
65
package-lock.json
generated
@ -44,6 +44,7 @@
|
||||
"prop-types": "^15.8.1",
|
||||
"react": "18.2.0",
|
||||
"react-dom": "18.2.0",
|
||||
"react-markdown": "^8.0.7",
|
||||
"react-next-github-btn": "^1.2.1",
|
||||
"react-plotly.js": "^2.6.0",
|
||||
"react-plotlyjs": "^0.4.4",
|
||||
@ -29695,6 +29696,41 @@
|
||||
"integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/react-markdown": {
|
||||
"version": "8.0.7",
|
||||
"resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-8.0.7.tgz",
|
||||
"integrity": "sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==",
|
||||
"dependencies": {
|
||||
"@types/hast": "^2.0.0",
|
||||
"@types/prop-types": "^15.0.0",
|
||||
"@types/unist": "^2.0.0",
|
||||
"comma-separated-tokens": "^2.0.0",
|
||||
"hast-util-whitespace": "^2.0.0",
|
||||
"prop-types": "^15.0.0",
|
||||
"property-information": "^6.0.0",
|
||||
"react-is": "^18.0.0",
|
||||
"remark-parse": "^10.0.0",
|
||||
"remark-rehype": "^10.0.0",
|
||||
"space-separated-tokens": "^2.0.0",
|
||||
"style-to-object": "^0.4.0",
|
||||
"unified": "^10.0.0",
|
||||
"unist-util-visit": "^4.0.0",
|
||||
"vfile": "^5.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/react": ">=16",
|
||||
"react": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/react-markdown/node_modules/react-is": {
|
||||
"version": "18.2.0",
|
||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
|
||||
"integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w=="
|
||||
},
|
||||
"node_modules/react-next-github-btn": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/react-next-github-btn/-/react-next-github-btn-1.2.1.tgz",
|
||||
@ -57219,6 +57255,35 @@
|
||||
"integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==",
|
||||
"dev": true
|
||||
},
|
||||
"react-markdown": {
|
||||
"version": "8.0.7",
|
||||
"resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-8.0.7.tgz",
|
||||
"integrity": "sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==",
|
||||
"requires": {
|
||||
"@types/hast": "^2.0.0",
|
||||
"@types/prop-types": "^15.0.0",
|
||||
"@types/unist": "^2.0.0",
|
||||
"comma-separated-tokens": "^2.0.0",
|
||||
"hast-util-whitespace": "^2.0.0",
|
||||
"prop-types": "^15.0.0",
|
||||
"property-information": "^6.0.0",
|
||||
"react-is": "^18.0.0",
|
||||
"remark-parse": "^10.0.0",
|
||||
"remark-rehype": "^10.0.0",
|
||||
"space-separated-tokens": "^2.0.0",
|
||||
"style-to-object": "^0.4.0",
|
||||
"unified": "^10.0.0",
|
||||
"unist-util-visit": "^4.0.0",
|
||||
"vfile": "^5.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"react-is": {
|
||||
"version": "18.2.0",
|
||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
|
||||
"integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"react-next-github-btn": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/react-next-github-btn/-/react-next-github-btn-1.2.1.tgz",
|
||||
|
||||
@ -40,6 +40,7 @@
|
||||
"prop-types": "^15.8.1",
|
||||
"react": "18.2.0",
|
||||
"react-dom": "18.2.0",
|
||||
"react-markdown": "^8.0.7",
|
||||
"react-next-github-btn": "^1.2.1",
|
||||
"react-plotly.js": "^2.6.0",
|
||||
"react-plotlyjs": "^0.4.4",
|
||||
|
||||
@ -14,15 +14,15 @@ Below are some screenshots:
|
||||
|
||||
### Front page
|
||||
|
||||
|
||||
|
||||
|
||||
### Individual dataset page
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Links
|
||||
|
||||
- [Documentation](/docs/example-data-catalog)
|
||||
- [Repo](https://github.com/datopian/portaljs/tree/main/examples/simple-example)
|
||||
- [Live Demo](https://example.portaljs.org)
|
||||
- [Live Demo](https://example.portaljs.org)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user