Compare commits

..

1 Commits

Author SHA1 Message Date
Luccas Mateus de Medeiros Gomes
5a29959d14 [alan-turing][m] - small tweaks 2023-05-02 12:53:44 -03:00
77 changed files with 389 additions and 27909 deletions

View File

@@ -23,7 +23,12 @@ import { serialize } from "next-mdx-remote/serialize";
* @returns: { mdxSource: mdxSource, frontMatter: ...}
*/
const parse = async function (source, format) {
const { content, data } = matter(source);
const { content, data, excerpt } = matter(source, {
excerpt: (file, options) => {
// Generate an excerpt for the file
file.excerpt = file.content.split("\n\n")[0];
},
});
const mdxSource = await serialize(
{ value: content, path: format },
@@ -51,7 +56,7 @@ const parse = async function (source, format) {
[
rehypeAutolinkHeadings,
{
properties: { className: "heading-link" },
properties: { className: 'heading-link' },
test(element) {
return (
["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
@@ -86,12 +91,14 @@ const parse = async function (source, format) {
],
format,
},
scope: data,
}
);
return {
mdxSource: mdxSource,
frontMatter: data,
excerpt,
};
};

View File

@@ -36,8 +36,7 @@
"focus-visible": "^5.2.0",
"gray-matter": "^4.0.3",
"hastscript": "^7.2.0",
"mdx-mermaid": "^2.0.0-rc7",
"mermaid": "^10.1.0",
"mdx-mermaid": "2.0.0-rc7",
"next": "13.2.1",
"next-mdx-remote": "^4.4.1",
"next-router-mock": "^0.9.3",
@@ -3339,6 +3338,7 @@
"version": "7.0.10",
"resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.10.tgz",
"integrity": "sha512-qTCQmEhcynucuaZgY5/+ti3X/rnszKZhEQH/ZdWdtP1tA/y3VoHJzcVrO9pjjJCNpigfscAtoUB5ONcd2wNn0A==",
"peer": true,
"dependencies": {
"d3": "^7.8.2",
"lodash-es": "^4.17.21"
@@ -3353,7 +3353,8 @@
"node_modules/dayjs": {
"version": "1.11.7",
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz",
"integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ=="
"integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==",
"peer": true
},
"node_modules/debug": {
"version": "4.3.4",
@@ -3543,7 +3544,8 @@
"node_modules/dompurify": {
"version": "2.4.5",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.4.5.tgz",
"integrity": "sha512-jggCCd+8Iqp4Tsz0nIvpcb22InKEBrGz5dw3EQJMs8HPJDsKbFIO3STYtAvCfDx26Muevn1MHVI0XxjgFfmiSA=="
"integrity": "sha512-jggCCd+8Iqp4Tsz0nIvpcb22InKEBrGz5dw3EQJMs8HPJDsKbFIO3STYtAvCfDx26Muevn1MHVI0XxjgFfmiSA==",
"peer": true
},
"node_modules/dot-prop": {
"version": "5.3.0",
@@ -7531,6 +7533,7 @@
"version": "10.1.0",
"resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.1.0.tgz",
"integrity": "sha512-LYekSMNJygI1VnMizAPUddY95hZxOjwZxr7pODczILInO0dhQKuhXeu4sargtnuTwCilSuLS7Uiq/Qn7HTVrmA==",
"peer": true,
"dependencies": {
"@braintree/sanitize-url": "^6.0.0",
"@khanacademy/simple-markdown": "^0.8.6",
@@ -7555,6 +7558,7 @@
"version": "0.8.6",
"resolved": "https://registry.npmjs.org/@khanacademy/simple-markdown/-/simple-markdown-0.8.6.tgz",
"integrity": "sha512-mAUlR9lchzfqunR89pFvNI51jQKsMpJeWYsYWw0DQcUXczn/T/V6510utgvm7X0N3zN87j1SvuKk8cMbl9IAFw==",
"peer": true,
"dependencies": {
"@types/react": ">=16.0.0"
},
@@ -15735,6 +15739,7 @@
"version": "7.0.10",
"resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.10.tgz",
"integrity": "sha512-qTCQmEhcynucuaZgY5/+ti3X/rnszKZhEQH/ZdWdtP1tA/y3VoHJzcVrO9pjjJCNpigfscAtoUB5ONcd2wNn0A==",
"peer": true,
"requires": {
"d3": "^7.8.2",
"lodash-es": "^4.17.21"
@@ -15749,7 +15754,8 @@
"dayjs": {
"version": "1.11.7",
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz",
"integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ=="
"integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==",
"peer": true
},
"debug": {
"version": "4.3.4",
@@ -15888,7 +15894,8 @@
"dompurify": {
"version": "2.4.5",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.4.5.tgz",
"integrity": "sha512-jggCCd+8Iqp4Tsz0nIvpcb22InKEBrGz5dw3EQJMs8HPJDsKbFIO3STYtAvCfDx26Muevn1MHVI0XxjgFfmiSA=="
"integrity": "sha512-jggCCd+8Iqp4Tsz0nIvpcb22InKEBrGz5dw3EQJMs8HPJDsKbFIO3STYtAvCfDx26Muevn1MHVI0XxjgFfmiSA==",
"peer": true
},
"dot-prop": {
"version": "5.3.0",
@@ -18840,6 +18847,7 @@
"version": "10.1.0",
"resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.1.0.tgz",
"integrity": "sha512-LYekSMNJygI1VnMizAPUddY95hZxOjwZxr7pODczILInO0dhQKuhXeu4sargtnuTwCilSuLS7Uiq/Qn7HTVrmA==",
"peer": true,
"requires": {
"@braintree/sanitize-url": "^6.0.0",
"@khanacademy/simple-markdown": "^0.8.6",
@@ -18864,6 +18872,7 @@
"version": "0.8.6",
"resolved": "https://registry.npmjs.org/@khanacademy/simple-markdown/-/simple-markdown-0.8.6.tgz",
"integrity": "sha512-mAUlR9lchzfqunR89pFvNI51jQKsMpJeWYsYWw0DQcUXczn/T/V6510utgvm7X0N3zN87j1SvuKk8cMbl9IAFw==",
"peer": true,
"requires": {
"@types/react": ">=16.0.0"
}

View File

@@ -12,46 +12,47 @@
},
"browserslist": "defaults, not ie <= 11",
"dependencies": {
"@flowershow/core": "^0.4.10",
"@flowershow/markdowndb": "^0.1.1",
"@flowershow/remark-callouts": "^1.0.0",
"@flowershow/remark-embed": "^1.0.0",
"@flowershow/remark-wiki-link": "^1.1.2",
"@headlessui/react": "^1.7.13",
"@heroicons/react": "^2.0.17",
"@mapbox/rehype-prism": "^0.8.0",
"@mdx-js/loader": "^2.1.5",
"@mdx-js/react": "^2.1.5",
"@next/mdx": "^13.0.2",
"@opentelemetry/api": "^1.4.0",
"@tailwindcss/forms": "^0.5.3",
"@tailwindcss/typography": "^0.5.4",
"@tanstack/react-table": "^8.8.5",
"@types/node": "18.16.0",
"@types/react": "18.2.0",
"@types/react-dom": "18.2.0",
"autoprefixer": "^10.4.12",
"clsx": "^1.2.1",
"eslint": "8.39.0",
"eslint-config-next": "13.3.1",
"fast-glob": "^3.2.11",
"feed": "^4.2.2",
"flexsearch": "^0.7.31",
"focus-visible": "^5.2.0",
"gray-matter": "^4.0.3",
"hastscript": "^7.2.0",
"mdx-mermaid": "^2.0.0-rc7",
"mermaid": "^10.1.0",
"next": "13.2.1",
"next-mdx-remote": "^4.4.1",
"next-router-mock": "^0.9.3",
"next-superjson-plugin": "^0.5.7",
"papaparse": "^5.4.1",
"postcss-focus-visible": "^6.0.4",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-hook-form": "^7.43.9",
"react-markdown": "^8.0.7",
"superjson": "^1.12.3",
"tailwindcss": "^3.3.0",
"@flowershow/core": "^0.4.10",
"@flowershow/remark-callouts": "^1.0.0",
"@flowershow/remark-embed": "^1.0.0",
"@flowershow/remark-wiki-link": "^1.1.2",
"@heroicons/react": "^2.0.17",
"@opentelemetry/api": "^1.4.0",
"@tanstack/react-table": "^8.8.5",
"@types/node": "18.16.0",
"@types/react": "18.2.0",
"@types/react-dom": "18.2.0",
"eslint": "8.39.0",
"eslint-config-next": "13.3.1",
"gray-matter": "^4.0.3",
"hastscript": "^7.2.0",
"mdx-mermaid": "2.0.0-rc7",
"next": "13.2.1",
"next-mdx-remote": "^4.4.1",
"papaparse": "^5.4.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-vega": "^7.6.0",
"rehype-autolink-headings": "^6.1.1",
"rehype-katex": "^6.0.3",
@@ -60,9 +61,7 @@
"remark-gfm": "^3.0.1",
"remark-math": "^5.1.1",
"remark-smartypants": "^2.0.0",
"remark-toc": "^8.0.1",
"superjson": "^1.12.3",
"tailwindcss": "^3.3.0"
"remark-toc": "^8.0.1"
},
"devDependencies": {
"eslint": "8.26.0",

View File

@@ -1,12 +1,10 @@
import { Container } from '../components/Container'
import clientPromise from '../lib/mddb'
import { promises as fs } from 'fs';
import fs from 'fs'
import { MDXRemote } from 'next-mdx-remote'
import { serialize } from 'next-mdx-remote/serialize'
import { Card } from '../components/Card'
import Head from 'next/head'
import parse from '../lib/markdown'
import { Mermaid } from '@flowershow/core';
export const getStaticProps = async ({ params }) => {
const urlPath = params.slug ? params.slug.join('/') : ''
@@ -14,8 +12,8 @@ export const getStaticProps = async ({ params }) => {
const mddb = await clientPromise
const dbFile = await mddb.getFileByUrl(urlPath)
const source = await fs.readFile(dbFile.file_path,'utf-8')
let mdxSource = await parse(source, '.mdx')
const source = fs.readFileSync(dbFile.file_path, { encoding: 'utf-8' })
const mdxSource = await serialize(source, { parseFrontmatter: true })
return {
props: {
@@ -76,7 +74,7 @@ const Meta = ({keyValuePairs}) => {
}
export default function DRDPage({ mdxSource }) {
const meta = mdxSource.frontMatter
const meta = mdxSource.frontmatter
const keyValuePairs = Object.entries(meta).filter(
(entry) => entry[0] !== 'title'
)
@@ -96,7 +94,7 @@ export default function DRDPage({ mdxSource }) {
</Card>
</header>
<div className="prose dark:prose-invert">
<MDXRemote {...mdxSource.mdxSource} components={{mermaid: Mermaid}} />
<MDXRemote {...mdxSource} />
</div>
</article>
</Container>

View File

@@ -1,119 +0,0 @@
import { Index } from 'flexsearch';
import { useState } from 'react';
import DebouncedInput from './DebouncedInput';
import { useForm } from 'react-hook-form';
export default function Catalog({
datasets,
facets,
}: {
datasets: any[];
facets: string[];
}) {
const [indexFilter, setIndexFilter] = useState('');
const index = new Index({ tokenize: 'full' });
datasets.forEach((dataset) =>
index.add(
dataset._id,
//This will join every metadata value + the url_path into one big string and index that
Object.entries(dataset.metadata).reduce(
(acc, curr) => acc + ' ' + curr[1].toString(),
''
) +
' ' +
dataset.url_path
)
);
const facetValues = facets
? facets.reduce((acc, facet) => {
const possibleValues = datasets.reduce((acc, curr) => {
const facetValue = curr.metadata[facet];
if (facetValue) {
return Array.isArray(facetValue)
? acc.concat(facetValue)
: acc.concat([facetValue]);
}
return acc;
}, []);
acc[facet] = {
possibleValues: [...new Set(possibleValues)],
selectedValue: null,
};
return acc;
}, {})
: [];
const { register, watch } = useForm(facetValues);
const filteredDatasets = datasets
// First filter by flex search
.filter((dataset) =>
indexFilter !== ''
? index.search(indexFilter).includes(dataset._id)
: true
)
//Then check if the selectedValue for the given facet is included in the dataset metadata
.filter((dataset) => {
//Avoids a server rendering breakage
if (!watch() || Object.keys(watch()).length === 0) return true
//This will filter only the key pairs of the metadata values that were selected as facets
const datasetFacets = Object.entries(dataset.metadata).filter((entry) =>
facets.includes(entry[0])
);
//Check if the value present is included in the selected value in the form
return datasetFacets.every((elem) =>
watch()[elem[0]].selectedValue
? (elem[1] as string | string[]).includes(
watch()[elem[0]].selectedValue
)
: true
);
});
return (
<>
<DebouncedInput
value={indexFilter ?? ''}
onChange={(value) => setIndexFilter(String(value))}
className="p-2 text-sm shadow border border-block mr-1"
placeholder="Search all datasets..."
/>
{Object.entries(facetValues).map((elem) => (
<select
key={elem[0]}
defaultValue=""
className="p-2 ml-1 text-sm shadow border border-block"
{...register(elem[0] + '.selectedValue')}
>
<option value="">
Filter by {elem[0]}
</option>
{(elem[1] as { possibleValues: string[] }).possibleValues.map(
(val) => (
<option
key={val}
className="dark:bg-white dark:text-black"
value={val}
>
{val}
</option>
)
)}
</select>
))}
<ul>
{filteredDatasets.map((dataset) => (
<li key={dataset._id}>
<a href={dataset.url_path}>
{dataset.metadata.title
? dataset.metadata.title
: dataset.url_path}
</a>
</li>
))}
</ul>
</>
);
}

View File

@@ -7,12 +7,13 @@ import { Mermaid } from '@flowershow/core';
// to handle import statements. Instead, you must include components in scope
// here.
const components = {
Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
Catalog: dynamic(() => import('./Catalog')),
Table: dynamic(() => import('./Table')),
mermaid: Mermaid,
Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),
LineChart: dynamic(() => import('@portaljs/components').then(mod => mod.LineChart)),
// Excel: dynamic(() => import('../components/Excel')),
// TODO: try and make these dynamic ...
Vega: dynamic(() => import('./Vega')),
VegaLite: dynamic(() => import('./VegaLite')),
LineChart: dynamic(() => import('./LineChart')),
} as any;
export default function DRD({ source }: { source: any }) {

View File

@@ -0,0 +1,55 @@
import VegaLite from "./VegaLite";
export default function LineChart({
data = [],
fullWidth = false,
title = "",
xAxis = "x",
yAxis = "y",
}) {
var tmp = data;
if (Array.isArray(data)) {
tmp = data.map((r, i) => {
return { x: r[0], y: r[1] };
});
}
const vegaData = { table: tmp };
const spec = {
$schema: "https://vega.github.io/schema/vega-lite/v5.json",
title,
width: "container",
height: 300,
mark: {
type: "line",
color: "black",
strokeWidth: 1,
tooltip: true,
},
data: {
name: "table",
},
selection: {
grid: {
type: "interval",
bind: "scales",
},
},
encoding: {
x: {
field: xAxis,
timeUnit: "year",
type: "temporal",
},
y: {
field: yAxis,
type: "quantitative",
},
},
};
if (typeof data === 'string') {
spec.data = { "url": data } as any
return <VegaLite fullWidth={fullWidth} spec={spec} />;
}
return <VegaLite fullWidth={fullWidth} data={vegaData} spec={spec} />;
}

View File

@@ -7,7 +7,7 @@ import {
getPaginationRowModel,
getSortedRowModel,
useReactTable,
} from '@tanstack/react-table';
} from "@tanstack/react-table";
import {
ArrowDownIcon,
@@ -16,29 +16,21 @@ import {
ChevronDoubleRightIcon,
ChevronLeftIcon,
ChevronRightIcon,
} from '@heroicons/react/24/solid';
} from "@heroicons/react/24/solid";
import React, { useEffect, useMemo, useState } from 'react';
import React, { useEffect, useMemo, useState } from "react";
import parseCsv from '../lib/parseCsv';
import DebouncedInput from './DebouncedInput';
import loadData from '../lib/loadData';
import parseCsv from "../lib/parseCsv";
import DebouncedInput from "./DebouncedInput";
import loadData from "../lib/loadData";
export type TableProps = {
data?: Array<{ [key: string]: number | string }>;
cols?: Array<{ [key: string]: string }>;
csv?: string;
url?: string;
fullWidth?: boolean;
};
export const Table = ({
const Table = ({
data: ogData = [],
cols: ogCols = [],
csv = '',
url = '',
csv = "",
url = "",
fullWidth = false,
}: TableProps) => {
}) => {
if (csv) {
const out = parseCsv(csv);
ogData = out.rows;
@@ -47,19 +39,19 @@ export const Table = ({
const [data, setData] = React.useState(ogData);
const [cols, setCols] = React.useState(ogCols);
// const [error, setError] = React.useState(""); // TODO: add error handling
const [error, setError] = React.useState(""); // TODO: add error handling
const tableCols = useMemo(() => {
const columnHelper = createColumnHelper();
return cols.map((c) =>
columnHelper.accessor<any, string>(c.key, {
columnHelper.accessor(c.key, {
header: () => c.name,
cell: (info) => info.getValue(),
})
);
}, [data, cols]);
const [globalFilter, setGlobalFilter] = useState('');
const [globalFilter, setGlobalFilter] = useState("");
const table = useReactTable({
data,
@@ -86,24 +78,24 @@ export const Table = ({
}, [url]);
return (
<div className={`${fullWidth ? 'w-[90vw] ml-[calc(50%-45vw)]' : 'w-full'}`}>
<div className={`${fullWidth ? "w-[90vw] ml-[calc(50%-45vw)]" : "w-full"}`}>
<DebouncedInput
value={globalFilter ?? ''}
onChange={(value: any) => setGlobalFilter(String(value))}
value={globalFilter ?? ""}
onChange={(value) => setGlobalFilter(String(value))}
className="p-2 text-sm shadow border border-block"
placeholder="Search all columns..."
/>
<table className="w-full mt-10">
<thead className="text-left border-b border-b-slate-300">
<table>
<thead>
{table.getHeaderGroups().map((hg) => (
<tr key={hg.id}>
{hg.headers.map((h) => (
<th key={h.id} className="pr-2 pb-2">
<th key={h.id}>
<div
{...{
className: h.column.getCanSort()
? 'cursor-pointer select-none'
: '',
? "cursor-pointer select-none"
: "",
onClick: h.column.getToggleSortingHandler(),
}}
>
@@ -126,9 +118,9 @@ export const Table = ({
</thead>
<tbody>
{table.getRowModel().rows.map((r) => (
<tr key={r.id} className="border-b border-b-slate-200">
<tr key={r.id}>
{r.getVisibleCells().map((c) => (
<td key={c.id} className="py-2">
<td key={c.id}>
{flexRender(c.column.columnDef.cell, c.getContext())}
</td>
))}
@@ -136,10 +128,10 @@ export const Table = ({
))}
</tbody>
</table>
<div className="flex gap-2 items-center justify-center mt-10">
<div className="flex gap-2 items-center justify-center">
<button
className={`w-6 h-6 ${
!table.getCanPreviousPage() ? 'opacity-25' : 'opacity-100'
!table.getCanPreviousPage() ? "opacity-25" : "opacity-100"
}`}
onClick={() => table.setPageIndex(0)}
disabled={!table.getCanPreviousPage()}
@@ -148,7 +140,7 @@ export const Table = ({
</button>
<button
className={`w-6 h-6 ${
!table.getCanPreviousPage() ? 'opacity-25' : 'opacity-100'
!table.getCanPreviousPage() ? "opacity-25" : "opacity-100"
}`}
onClick={() => table.previousPage()}
disabled={!table.getCanPreviousPage()}
@@ -158,13 +150,13 @@ export const Table = ({
<span className="flex items-center gap-1">
<div>Page</div>
<strong>
{table.getState().pagination.pageIndex + 1} of{' '}
{table.getState().pagination.pageIndex + 1} of{" "}
{table.getPageCount()}
</strong>
</span>
<button
className={`w-6 h-6 ${
!table.getCanNextPage() ? 'opacity-25' : 'opacity-100'
!table.getCanNextPage() ? "opacity-25" : "opacity-100"
}`}
onClick={() => table.nextPage()}
disabled={!table.getCanNextPage()}
@@ -173,7 +165,7 @@ export const Table = ({
</button>
<button
className={`w-6 h-6 ${
!table.getCanNextPage() ? 'opacity-25' : 'opacity-100'
!table.getCanNextPage() ? "opacity-25" : "opacity-100"
}`}
onClick={() => table.setPageIndex(table.getPageCount() - 1)}
disabled={!table.getCanNextPage()}
@@ -189,7 +181,9 @@ const globalFilterFn: FilterFn<any> = (row, columnId, filterValue: string) => {
const search = filterValue.toLowerCase();
let value = row.getValue(columnId) as string;
if (typeof value === 'number') value = String(value);
if (typeof value === "number") value = String(value);
return value?.toLowerCase().includes(search);
};
export default Table;

View File

@@ -1,6 +1,6 @@
// Wrapper for the Vega component
import { Vega as VegaOg } from "react-vega";
export function Vega(props) {
export default function Vega(props) {
return <VegaOg {...props} />;
}

View File

@@ -2,7 +2,7 @@
import { VegaLite as VegaLiteOg } from "react-vega";
import applyFullWidthDirective from "../lib/applyFullWidthDirective";
export function VegaLite(props) {
export default function VegaLite(props) {
const Component = applyFullWidthDirective({ Component: VegaLiteOg });
return <Component {...props} />;

View File

@@ -22,7 +22,7 @@ import { serialize } from "next-mdx-remote/serialize";
* @format: used to indicate to next-mdx-remote which format to use (md or mdx)
* @returns: { mdxSource: mdxSource, frontMatter: ...}
*/
const parse = async function (source, format, scope) {
const parse = async function (source, format) {
const { content, data, excerpt } = matter(source, {
excerpt: (file, options) => {
// Generate an excerpt for the file
@@ -91,7 +91,7 @@ const parse = async function (source, format, scope) {
],
format,
},
scope,
scope: data,
}
);

View File

@@ -1,14 +0,0 @@
import { MarkdownDB } from "@flowershow/markdowndb";
const dbPath = "markdown.db";
const client = new MarkdownDB({
client: "sqlite3",
connection: {
filename: dbPath,
},
});
const clientPromise = client.init();
export default clientPromise;

View File

@@ -0,0 +1,16 @@
import papa from "papaparse";
const parseCsv = (csv) => {
csv = csv.trim();
const rawdata = papa.parse(csv, { header: true });
const cols = rawdata.meta.fields.map((r, i) => {
return { key: r, name: r };
});
return {
rows: rawdata.data,
fields: cols,
};
};
export default parseCsv;

File diff suppressed because it is too large Load Diff

View File

@@ -7,26 +7,21 @@
"build": "next build",
"start": "next start",
"lint": "next lint",
"export": "npm run build && next export -o out",
"prebuild": "npm run mddb",
"mddb": "mddb ./content"
"export": "npm run build && next export -o out"
},
"dependencies": {
"@flowershow/core": "^0.4.10",
"@flowershow/markdowndb": "^0.1.1",
"@flowershow/remark-callouts": "^1.0.0",
"@flowershow/remark-embed": "^1.0.0",
"@flowershow/remark-wiki-link": "^1.1.2",
"@heroicons/react": "^2.0.17",
"@opentelemetry/api": "^1.4.0",
"@portaljs/components": "^0.0.3",
"@tanstack/react-table": "^8.8.5",
"@types/node": "18.16.0",
"@types/react": "18.2.0",
"@types/react-dom": "18.2.0",
"eslint": "8.39.0",
"eslint-config-next": "13.3.1",
"flexsearch": "0.7.21",
"gray-matter": "^4.0.3",
"hastscript": "^7.2.0",
"mdx-mermaid": "2.0.0-rc7",
@@ -35,7 +30,6 @@
"papaparse": "^5.4.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-hook-form": "^7.43.9",
"react-vega": "^7.6.0",
"rehype-autolink-headings": "^6.1.1",
"rehype-katex": "^6.0.3",
@@ -49,7 +43,6 @@
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.9",
"@types/flexsearch": "^0.7.3",
"autoprefixer": "^10.4.14",
"postcss": "^8.4.23",
"tailwindcss": "^3.3.1"

View File

@@ -1,9 +1,7 @@
import { existsSync, promises as fs } from 'fs';
import { promises as fs } from 'fs';
import path from 'path';
import parse from '../lib/markdown';
import DataRichDocument from '../components/DataRichDocument';
import clientPromise from '../lib/mddb';
import DRD from '../components/DRD';
export const getStaticPaths = async () => {
const contentDir = path.join(process.cwd(), '/content/');
@@ -25,28 +23,9 @@ export const getStaticProps = async (context) => {
pathToFile = context.params.path.join('/') + '/index.md';
}
let datasets = [];
const mddbFileExists = existsSync('markdown.db');
if (mddbFileExists) {
const mddb = await clientPromise;
const datasetsFiles = await mddb.getFiles({
extensions: ['md', 'mdx'],
});
datasets = datasetsFiles
.filter((dataset) => dataset.url_path !== '/')
.map((dataset) => ({
_id: dataset._id,
url_path: dataset.url_path,
file_path: dataset.file_path,
metadata: dataset.metadata,
}));
}
const indexFile = path.join(process.cwd(), '/content/' + pathToFile);
const readme = await fs.readFile(indexFile, 'utf8');
let { mdxSource, frontMatter } = await parse(readme, '.mdx', { datasets });
let { mdxSource, frontMatter } = await parse(readme, '.mdx');
return {
props: {
mdxSource,
@@ -74,7 +53,7 @@ export default function DatasetPage({ mdxSource, frontMatter }) {
</div>
</header>
<main>
<DataRichDocument source={mdxSource} />
<DRD source={mdxSource} />
</main>
</div>
);

View File

@@ -1,6 +1,4 @@
import '../styles/globals.css'
import '@portaljs/components/styles.css'
import type { AppProps } from 'next/app'
export default function App({ Component, pageProps }: AppProps) {

View File

@@ -1,6 +1,6 @@
{
"compilerOptions": {
"target": "es6",
"target": "es5",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,

1424
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -4,10 +4,11 @@
"license": "MIT",
"scripts": {},
"private": true,
"dependencies": {},
"devDependencies": {
"@babel/preset-react": "^7.14.5",
"@nrwl/cypress": "15.9.2",
"@nrwl/eslint-plugin-nx": "^16.0.2",
"@nrwl/eslint-plugin-nx": "15.9.2",
"@nrwl/jest": "15.9.2",
"@nrwl/js": "15.9.2",
"@nrwl/linter": "15.9.2",

View File

@@ -1,16 +0,0 @@
module.exports = {
env: {
browser: true,
es2020: true
},
extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'plugin:react-hooks/recommended', 'plugin:storybook/recommended'],
parser: '@typescript-eslint/parser',
parserOptions: {
ecmaVersion: 'latest',
sourceType: 'module'
},
plugins: ['react-refresh'],
rules: {
'react-refresh/only-export-components': 'warn'
}
};

View File

@@ -1,24 +0,0 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
dist
dist-ssr
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?

View File

@@ -1,17 +0,0 @@
import type { StorybookConfig } from '@storybook/react-vite';
const config: StorybookConfig = {
stories: ['../stories/**/*.mdx', '../stories/**/*.stories.@(js|jsx|ts|tsx)'],
addons: [
'@storybook/addon-links',
'@storybook/addon-essentials',
'@storybook/addon-interactions',
],
framework: {
name: '@storybook/react-vite',
options: {},
},
docs: {
autodocs: 'tag',
},
};
export default config;

View File

@@ -1,17 +0,0 @@
import 'tailwindcss/tailwind.css'
import type { Preview } from '@storybook/react';
const preview: Preview = {
parameters: {
actions: { argTypesRegex: '^on[A-Z].*' },
controls: {
matchers: {
color: /(background|color)$/i,
date: /Date$/,
},
},
},
};
export default preview;

View File

@@ -1,29 +0,0 @@
# PortalJS React Components
**Storybook:** https://storybook.portaljs.org
**Docs**: https://portaljs.org/docs
## Usage
To install this package on your project:
```bash
npm i @portaljs/components
```
> Note: React 18 is required.
You'll also have to import the styles CSS file in your project:
```ts
// E.g.: Next.js => pages/_app.tsx
import '@portaljs/components/styles.css'
```
## Dev
Use Storybook to work on components by running:
```bash
npm run storybook
```

File diff suppressed because it is too large Load Diff

View File

@@ -1,83 +0,0 @@
{
"name": "@portaljs/components",
"version": "0.0.3",
"type": "module",
"description": "https://portaljs.org",
"keywords": [
"data portal",
"data catalog",
"table",
"charts",
"visualization"
],
"scripts": {
"dev": "npm run storybook",
"build": "tsc && vite build && npm run build-tailwind",
"lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
"prepack": "json -f package.json -I -e \"delete this.devDependencies; delete this.dependencies\"",
"storybook": "storybook dev -p 6006",
"build-storybook": "storybook build",
"build-tailwind": "NODE_ENV=production npx tailwindcss -o ./dist/styles.css --minify"
},
"peerDependencies": {
"react": "^18.2.0",
"react-dom": "^18.2.0"
},
"dependencies": {
"@heroicons/react": "^2.0.17",
"next-mdx-remote": "^4.4.1",
"papaparse": "^5.4.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-vega": "^7.6.0",
"vega": "5.20.2",
"vega-lite": "5.1.0",
"@tanstack/react-table": "^8.8.5"
},
"devDependencies": {
"@storybook/addon-essentials": "^7.0.7",
"@storybook/addon-interactions": "^7.0.7",
"@storybook/addon-links": "^7.0.7",
"@storybook/blocks": "^7.0.7",
"@storybook/react": "^7.0.7",
"@storybook/react-vite": "^7.0.7",
"@storybook/testing-library": "^0.0.14-next.2",
"@types/papaparse": "^5.3.7",
"@types/react": "^18.0.28",
"@types/react-dom": "^18.0.11",
"@typescript-eslint/eslint-plugin": "^5.57.1",
"@typescript-eslint/parser": "^5.57.1",
"@vitejs/plugin-react": "^4.0.0",
"autoprefixer": "^10.4.14",
"eslint": "^8.38.0",
"eslint-plugin-react-hooks": "^4.6.0",
"eslint-plugin-react-refresh": "^0.3.4",
"eslint-plugin-storybook": "^0.6.11",
"json": "^11.0.0",
"postcss": "^8.4.23",
"prop-types": "^15.8.1",
"storybook": "^7.0.7",
"tailwindcss": "^3.3.2",
"typescript": "^5.0.2",
"vite": "^4.3.2",
"vite-plugin-dts": "^2.3.0"
},
"files": [
"dist"
],
"main": "./dist/components.umd.js",
"module": "./dist/components.es.js",
"types": "./dist/index.d.ts",
"exports": {
".": {
"import": "./dist/components.es.js",
"require": "./dist/components.umd.js"
},
"./styles.css": {
"import": "./dist/styles.css"
}
},
"publishConfig": {
"access": "public"
}
}

View File

@@ -1,6 +0,0 @@
export default {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
}

View File

@@ -1,32 +0,0 @@
import { useEffect, useState } from "react";
const DebouncedInput = ({
value: initialValue,
onChange,
debounce = 500,
...props
}) => {
const [value, setValue] = useState(initialValue);
useEffect(() => {
setValue(initialValue);
}, [initialValue]);
useEffect(() => {
const timeout = setTimeout(() => {
onChange(value);
}, debounce);
return () => clearTimeout(timeout);
}, [value]);
return (
<input
{...props}
value={value}
onChange={(e) => setValue(e.target.value)}
/>
);
};
export default DebouncedInput;

View File

@@ -1,63 +0,0 @@
import { VegaLite } from './VegaLite';
export type LineChartProps = {
data: Array<Array<string | number>> | string | { x: string; y: number }[];
title?: string;
xAxis?: string;
yAxis?: string;
fullWidth?: boolean;
};
export function LineChart({
data = [],
fullWidth = false,
title = '',
xAxis = 'x',
yAxis = 'y',
}: LineChartProps) {
var tmp = data;
if (Array.isArray(data)) {
tmp = data.map((r) => {
return { x: r[0], y: r[1] };
});
}
const vegaData = { table: tmp };
const spec = {
$schema: 'https://vega.github.io/schema/vega-lite/v5.json',
title,
width: 'container',
height: 300,
mark: {
type: 'line',
color: 'black',
strokeWidth: 1,
tooltip: true,
},
data: {
name: 'table',
},
selection: {
grid: {
type: 'interval',
bind: 'scales',
},
},
encoding: {
x: {
field: xAxis,
timeUnit: 'year',
type: 'temporal',
},
y: {
field: yAxis,
type: 'quantitative',
},
},
};
if (typeof data === 'string') {
spec.data = { url: data } as any;
return <VegaLite fullWidth={fullWidth} spec={spec} />;
}
return <VegaLite fullWidth={fullWidth} data={vegaData} spec={spec} />;
}

View File

@@ -1,3 +0,0 @@
@tailwind base;
@tailwind components;
@tailwind utilities;

View File

@@ -1,4 +0,0 @@
export * from "./components/Table";
export * from "./components/LineChart";
export * from "./components/Vega";
export * from "./components/VegaLite";

View File

@@ -1,20 +0,0 @@
import papa from "papaparse";
const parseCsv = (csv: string) => {
csv = csv.trim();
const rawdata = papa.parse(csv, { header: true });
let cols: any[] = [];
if(rawdata.meta.fields) {
cols = rawdata.meta.fields.map((r: string) => {
return { key: r, name: r };
});
}
return {
rows: rawdata.data as any,
fields: cols,
};
};
export default parseCsv;

View File

@@ -1 +0,0 @@
/// <reference types="vite/client" />

View File

@@ -1,9 +0,0 @@
import { Meta } from '@storybook/blocks';
<Meta title="Components/Introduction" />
# Welcome to the PortalJS components guide
**Official Website:** [portaljs.org](https://portaljs.org)
**Docs:** [portaljs.org/docs](https://portaljs.org/docs)
**GitHub:** [github.com/datopian/portaljs](https://github.com/datopian/portaljs)

View File

@@ -1,59 +0,0 @@
import type { Meta, StoryObj } from '@storybook/react';
import { LineChart, LineChartProps } from '../src/components/LineChart';
// More on how to set up stories at: https://storybook.js.org/docs/react/writing-stories/introduction
const meta: Meta = {
title: 'Components/LineChart',
component: LineChart,
tags: ['autodocs'],
argTypes: {
data: {
description:
'Data to be displayed.\n\n E.g.: [["1990", 1], ["1991", 2]] \n\nOR\n\n "https://url.to/data.csv"',
},
title: {
description: 'Title to display on the chart. Optional.',
},
xAxis: {
description:
'Name of the X axis on the data. Required when the "data" parameter is an URL.',
},
yAxis: {
description:
'Name of the Y axis on the data. Required when the "data" parameter is an URL.',
},
fullWidth: {
description:
'Whether the component should be rendered as full bleed or not',
},
},
};
export default meta;
type Story = StoryObj<LineChartProps>;
// More on writing stories with args: https://storybook.js.org/docs/react/writing-stories/args
export const FromDataPoints: Story = {
name: 'Line chart from array of data points',
args: {
data: [
['1850', -0.41765878],
['1851', -0.2333498],
['1852', -0.22939907],
['1853', -0.27035445],
['1854', -0.29163003],
],
},
};
export const FromURL: Story = {
name: 'Line chart from URL',
args: {
title: 'Oil Price x Year',
data: 'https://raw.githubusercontent.com/datasets/oil-prices/main/data/wti-year.csv',
xAxis: 'Date',
yAxis: 'Price',
},
};

View File

@@ -1,69 +0,0 @@
import type { Meta, StoryObj } from '@storybook/react';
import { Table, TableProps } from '../src/components/Table';
// More on how to set up stories at: https://storybook.js.org/docs/react/writing-stories/introduction
const meta: Meta = {
title: 'Components/Table',
component: Table,
tags: ['autodocs'],
argTypes: {
data: {
description: "Data to be displayed in the table, must also set \"cols\" to work."
},
cols: {
description: "Columns to be displayed in the table, must also set \"data\" to work."
},
csv: {
description: "CSV data as string.",
},
url: {
description: "Fetch the data from a CSV file remotely."
}
},
};
export default meta;
type Story = StoryObj<TableProps>;
// More on writing stories with args: https://storybook.js.org/docs/react/writing-stories/args
export const FromColumnsAndData: Story = {
name: "Table from columns and data",
args: {
data: [
{ id: 1, lastName: 'Snow', firstName: 'Jon', age: 35 },
{ id: 2, lastName: 'Lannister', firstName: 'Cersei', age: 42 },
{ id: 3, lastName: 'Lannister', firstName: 'Jaime', age: 45 },
{ id: 4, lastName: 'Stark', firstName: 'Arya', age: 16 },
{ id: 7, lastName: 'Clifford', firstName: 'Ferrara', age: 44 },
{ id: 8, lastName: 'Frances', firstName: 'Rossini', age: 36 },
{ id: 9, lastName: 'Roxie', firstName: 'Harvey', age: 65 },
],
cols: [
{ key: 'id', name: 'ID' },
{ key: 'firstName', name: 'First name' },
{ key: 'lastName', name: 'Last name' },
{ key: 'age', name: 'Age' },
],
},
};
export const FromRawCSV: Story = {
name: "Table from raw CSV",
args: {
csv: `
Year,Temp Anomaly
1850,-0.418
2020,0.923
`
}
};
export const FromURL: Story = {
name: "Table from URL",
args: {
url: "https://raw.githubusercontent.com/datasets/finance-vix/main/data/vix-daily.csv"
}
};

View File

@@ -1,50 +0,0 @@
import type { Meta, StoryObj } from '@storybook/react';
import { Vega } from '../src/components/Vega';
// More on how to set up stories at: https://storybook.js.org/docs/react/writing-stories/introduction
const meta: Meta = {
title: 'Components/Vega',
component: Vega,
tags: ['autodocs'],
};
export default meta;
type Story = StoryObj<any>;
// More on writing stories with args: https://storybook.js.org/docs/react/writing-stories/args
export const Primary: Story = {
name: 'Chart built with Vega',
args: {
data: {
table: [
{
y: -0.418,
x: 1850,
},
{
y: 0.923,
x: 2020,
},
],
},
spec: {
$schema: 'https://vega.github.io/schema/vega-lite/v4.json',
mark: 'bar',
data: {
name: 'table',
},
encoding: {
x: {
field: 'x',
type: 'ordinal',
},
y: {
field: 'y',
type: 'quantitative',
},
},
},
},
};

View File

@@ -1,60 +0,0 @@
import type { Meta, StoryObj } from '@storybook/react';
import { VegaLite } from '../src/components/VegaLite';
// More on how to set up stories at: https://storybook.js.org/docs/react/writing-stories/introduction
const meta: Meta = {
title: 'Components/VegaLite',
component: VegaLite,
tags: ['autodocs'],
argTypes: {
data: {
description:
'Data to be used by Vega Lite. See the Vega Lite docs: https://vega.github.io/vega-lite/docs/data.html.',
},
spec: {
description:
'Spec to be used by Vega Lite. See the Vega Lite docs: https://vega.github.io/vega-lite/docs/spec.html.',
},
},
};
export default meta;
type Story = StoryObj<any>;
// More on writing stories with args: https://storybook.js.org/docs/react/writing-stories/args
export const Primary: Story = {
name: 'Chart built with Vega Lite',
args: {
data: {
table: [
{
y: -0.418,
x: 1850,
},
{
y: 0.923,
x: 2020,
},
],
},
spec: {
$schema: 'https://vega.github.io/schema/vega-lite/v4.json',
mark: 'bar',
data: {
name: 'table',
},
encoding: {
x: {
field: 'x',
type: 'ordinal',
},
y: {
field: 'y',
type: 'quantitative',
},
},
},
},
};

View File

@@ -1,8 +0,0 @@
/** @type {import('tailwindcss').Config} */
export default {
content: ['./index.html', './src/**/*.{js,ts,jsx,tsx}'],
theme: {
extend: {},
},
plugins: [],
};

View File

@@ -1,28 +0,0 @@
{
"compilerOptions": {
"target": "ESNext",
"lib": ["DOM", "DOM.Iterable", "ESNext"],
"module": "ESNext",
"skipLibCheck": true,
/* Bundler mode */
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "react-jsx",
/* Linting */
// "strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"paths": {
"@/*": ["./*"]
}
},
"include": ["src"],
"references": [{ "path": "./tsconfig.node.json" }],
}

View File

@@ -1,10 +0,0 @@
{
"compilerOptions": {
"composite": true,
"skipLibCheck": true,
"module": "ESNext",
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true
},
"include": ["vite.config.ts"]
}

View File

@@ -1,30 +0,0 @@
import react from '@vitejs/plugin-react';
import path from 'node:path';
import { defineConfig } from 'vite';
import dts from 'vite-plugin-dts';
export default defineConfig({
plugins: [
react(),
dts({
insertTypesEntry: true,
}),
],
build: {
lib: {
entry: path.resolve(__dirname, 'src/index.ts'),
name: 'components',
formats: ['es', 'umd'],
fileName: (format) => `components.${format}.js`,
},
rollupOptions: {
external: ['react', 'react-dom', 'styled-components'],
output: {
globals: {
react: 'React',
'react-dom': 'ReactDOM'
},
},
},
},
});

View File

Before

Width:  |  Height:  |  Size: 1.8 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

View File

Before

Width:  |  Height:  |  Size: 627 B

After

Width:  |  Height:  |  Size: 627 B

View File

Before

Width:  |  Height:  |  Size: 1.0 KiB

After

Width:  |  Height:  |  Size: 1.0 KiB

View File

@@ -116,52 +116,6 @@ From the browser, access http://localhost:3000. You should see the following:
<img src="/assets/docs/datasets-index-page.png" />
### Creating a search page
Typing out every link in the index page will get cumbersome eventually, and as the portal grows, finding the datasets you are looking for on the index page will become harder and harder. Luckily we have a component for that. Change your `content/index.md` file to this:
```
# Welcome to my data portal!
List of available datasets:
<Catalog datasets={datasets} />
```
Before you refresh the page, however, you will need to run the following command:
```
npm run mddb
```
This example makes use of the [markdowndb](https://github.com/datopian/markdowndb) library. For now the only thing you need to know is that you should run the command above everytime you make some change to `/content`.
From the browser, access http://localhost:3000. You should see the following, you now have a searchable automatic list of your datasets:
![](https://i.imgur.com/9HfSPIx.png)
To make this catalog look even better, we can change the text that is being displayed for each dataset to a title. Let's do that by adding the "title" [frontmatter field](https://daily-dev-tips.com/posts/what-exactly-is-frontmatter/) to the first dataset in the list. Change `content/my-awesome-dataset/index.md` to the following:
```
---
title: 'My awesome dataset'
---
# My Awesome Dataset
Built with PortalJS
## Table
<Table url="data.csv" />
```
Rerun `npm run mddb` and, from the browser, access http://localhost:3000. You should see the title appearing instead of the folder name:
![](https://i.imgur.com/nvmSnJ5.png)
Any frontmatter attribute that you add will automatically get indexed and be usable in the search box.
## Deploying your PortalJS app
Finally, let's learn how to deploy PortalJS apps to Vercel or Cloudflare Pages.