[.github/workflows][m]: add release workflow

The workflow creates a PR with version bumps and related changelog
files' updates, each time a commit containing a changeset
file is added on the main branch.

.changeset/README.md

@@ -0,0 +1,8 @@
+# Changesets
+
+Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
+with multi-package repos, or single-package repos to help you version and publish your code. You can
+find the full documentation for it [in our repository](https://github.com/changesets/changesets)
+
+We have a quick list of common questions to get you started engaging with this project in
+[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)

.changeset/config.json

@@ -0,0 +1,14 @@
+{
+  "$schema": "https://unpkg.com/@changesets/config@2.3.0/schema.json",
+  "changelog": [
+    "@changesets/changelog-github",
+    { "repo": "datopian/portaljs" }
+  ],
+  "commit": false,
+  "fixed": [],
+  "linked": [],
+  "access": "restricted",
+  "baseBranch": "main",
+  "updateInternalDependencies": "patch",
+  "ignore": []
+}

.github/workflows/release.yml

@@ -0,0 +1,39 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+
+concurrency: release-${{ github.ref }}
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Setup Node.js 16.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Create Release Pull Request or Publish to npm
+        id: changesets
+        uses: changesets/action@v1
+        with:
+          publish: npm run release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      # - name: Send a Discord notification if a publish happens
+      #   if: steps.changesets.outputs.published == 'true'
+      #   uses: Ilshidur/action-discord@0.3.2
+      #   with:
+      #     args: 'The project {{ EVENT_PAYLOAD.repository.full_name }} has been deployed.'

CONTRIBUTING.md

@@ -13,7 +13,7 @@ Structure:
   - **dataset-frictionless**: Example utilizing a frictionless dataset as an example
 - **site**: the website for the project, with a landing page and the docs
 - **packages**:
-    - **portaljs-components**: the library of components for creating a data portal
+  - **portaljs-components**: the library of components for creating a data portal
 
 ## How to contribute
 
@@ -26,10 +26,11 @@ If you'd like to work on one of the issues you can:
 3. Clone the forked repository to your machine.
 4. Create a feature branch (e.g. `50-update-readme`, where `50` is the number of the related issue).
 5. Commit your changes to the feature branch.
-6. Push the feature branch to your forked repository.
+6. Add changeset file describing the changes. (See section below)
-7. Create a Pull Request against the original repository.
+7. Push the feature branch to your forked repository.
+8. Create a Pull Request against the original repository.
    - add a short description of the changes included in the PR
-8. Address review comments if requested by our demanding reviewers 😜.
+9. Address review comments if requested by our demanding reviewers 😜.
 
 If you have an idea for improvement, and it doesn't have a corresponding issue yet, simply submit a new one.
 
@@ -62,6 +63,7 @@ or you can use just:
 nx <target> <project>
 # e.g. npx nx serve ckan
 ```
+
 if you have the `nx` binary installed globally in your machine
 
 #### Running multiple tasks
@@ -174,3 +176,23 @@ To learn more see this [offical docs page](https://nx.dev/reference/nx-json).
 Each project also has it's own configuration file - `project.json`, where you can define and configure it's targets (and more).
 
 To learn more see this [offical docs page](https://nx.dev/reference/project-configuration).
+
+## Changesets and publishing packages
+
+> This monorepo is set up with changesets versioning tool. See their [github repository](https://github.com/changesets/changesets) to learn more.
+
+### What are Changesets?
+
+Changesets are files that describe the intention of a contributor to bump a version of the package according to their changes. Changeset file holds two key bits of information: a version type (following semver), and change information to be added to a changelog.
+
+### Adding changesets
+
+In the root directory of the repo, run:
+
+```
+npx changeset
+```
+
+Select the package that has been changed, the semver version that should be bumped with it and a description of your changes. Please make sure to add the most accurate but also concise information.
+
+To learn about semantic versioning standards see [this semver doc page](https://semver.org/).

examples/ckan-example/.env

@@ -1 +0,0 @@
-DMS=https://demo.dev.datopian.com

examples/ckan-ssg/package.json

@@ -10,6 +10,7 @@
   },
   "dependencies": {
     "@heroicons/react": "^2.0.17",
+    "@portaljs/ckan": "^0.0.2",
     "next": "13.3.1",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
@@ -20,14 +21,14 @@
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",
-    "autoprefixer": "^10.4.14",
-    "postcss": "^8.4.23",
-    "tailwindcss": "^3.3.1",
-    "eslint": "8.39.0",
-    "eslint-config-next": "13.3.1",
-    "typescript": "5.0.4",
     "@types/node": "18.16.0",
     "@types/react": "18.0.38",
-    "@types/react-dom": "18.0.11"
+    "@types/react-dom": "18.0.11",
+    "autoprefixer": "^10.4.14",
+    "eslint": "8.39.0",
+    "eslint-config-next": "13.3.1",
+    "postcss": "^8.4.23",
+    "tailwindcss": "^3.3.1",
+    "typescript": "5.0.4"
   }
 }

examples/ckan-ssg/pages/@org/[org]/[dataset]/index.tsx

@@ -11,8 +11,9 @@ import {
   ServerIcon,
   UserIcon,
 } from '@heroicons/react/20/solid';
+import { CKAN } from '@portaljs/ckan';
 
-const dms = getConfig().publicRuntimeConfig.DMS;
+const backend_url = getConfig().publicRuntimeConfig.DMS;
 
 const formatter = new Intl.DateTimeFormat('en-US', {
   year: 'numeric',
@@ -25,14 +26,12 @@ const formatter = new Intl.DateTimeFormat('en-US', {
 });
 
 export const getServerSideProps: GetServerSideProps = async (context) => {
+  const ckan = new CKAN(backend_url)
   const { dataset } = context.query;
-  const response = await fetch(
+  const _dataset = await ckan.getDatasetDetails(dataset as string)
-    `${dms}/api/3/action/package_show?id=${dataset}`
-  );
-  const _dataset = await response.json();
   return {
     props: {
-      dataset: _dataset.result,
+      dataset: _dataset,
     },
   };
 };

examples/ckan-ssg/pages/index.tsx

@@ -1,7 +1,8 @@
 import getConfig from 'next/config';
 import styles from './index.module.css';
+import { CKAN } from '@portaljs/ckan';
 
-const dms = getConfig().publicRuntimeConfig.DMS
+const backend_url = getConfig().publicRuntimeConfig.DMS
 
 const formatter = new Intl.DateTimeFormat('en-US', {
   year: 'numeric',
@@ -15,12 +16,11 @@ const formatter = new Intl.DateTimeFormat('en-US', {
 
 
 export async function getServerSideProps() {
-  const response = await fetch(`${dms}/api/3/action/package_search`)
+  const ckan = new CKAN(backend_url)
-  const datasets = await response.json()
+  const { datasets } = await ckan.packageSearch({ limit: 1000, offset: 0, groups:[], orgs: [], tags: []})
-  const datasetsWithDetails = await Promise.all(datasets.result.results.map(async (dataset) => {
+  const datasetsWithDetails = await Promise.all(datasets.map(async (dataset) => {
-    const response = await fetch(`${dms}/api/3/action/package_show?id=` + dataset.name)
+    const _dataset = await ckan.getDatasetDetails(dataset.name)
-    const json = await response.json()
+    return _dataset
-    return json.result
   }))
 
   return {
@@ -79,7 +79,7 @@ export function Index({ datasets }) {
                 </thead>
                 <tbody className="divide-y divide-gray-200">
                   {datasets.map((dataset) => (
-                    <tr>
+                    <tr key={dataset.name}>
                       <td className="px-3 py-4 text-sm text-gray-500">
                           {dataset.title}
                       </td>

examples/ckan/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

examples/ckan/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/ckan/README.md

@@ -0,0 +1,38 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file.
+
+[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
+
+The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

examples/ckan/components/DataRichDocument.tsx

@@ -9,6 +9,7 @@ import { Mermaid } from '@flowershow/core';
 const components = {
   Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
   Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
+  FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
   mermaid: Mermaid,
   Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
   VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),

examples/ckan/content/test.md

@@ -0,0 +1,3 @@
+# Test
+
+Test Data Rich Stories

examples/ckan/next.config.js

@@ -0,0 +1,11 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  reactStrictMode: true,
+  publicRuntimeConfig: {
+    DMS: process.env.DMS
+      ? process.env.DMS.replace(/\/?$/, '')
+      : 'https://demo.dev.datopian.com/',
+  },
+};
+
+module.exports = nextConfig;

examples/ckan/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "ckan",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "prebuild": "npm run mddb",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint",
+    "mddb": "mddb ./content"
+  },
+  "dependencies": {
+    "@flowershow/core": "^0.4.13",
+    "@flowershow/markdowndb": "^0.1.5",
+    "@flowershow/remark-callouts": "^1.0.0",
+    "@flowershow/remark-embed": "^1.0.0",
+    "@githubocto/flat-ui": "^0.14.1",
+    "@heroicons/react": "^2.0.18",
+    "@portaljs/ckan": "^0.0.2",
+    "@portaljs/components": "0.1.6",
+    "@tailwindcss/typography": "^0.5.9",
+    "@types/node": "20.2.3",
+    "@types/react": "18.2.6",
+    "@types/react-dom": "18.2.4",
+    "autoprefixer": "10.4.14",
+    "eslint": "8.41.0",
+    "eslint-config-next": "13.4.3",
+    "isomorphic-unfetch": "^4.0.2",
+    "next": "13.4.3",
+    "next-mdx-remote": "^4.4.1",
+    "papaparse": "^5.4.1",
+    "postcss": "8.4.23",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-query": "^3.39.3",
+    "rehype-autolink-headings": "^6.1.1",
+    "rehype-katex": "^6.0.3",
+    "rehype-prism-plus": "^1.5.1",
+    "rehype-slug": "^5.1.0",
+    "remark-math": "^5.1.1",
+    "remark-smartypants": "^2.0.0",
+    "remark-toc": "^8.0.1",
+    "tailwindcss": "3.3.2",
+    "typescript": "5.0.4"
+  }
+}

examples/ckan/pages/[org]/[dataset]/index.tsx

@@ -0,0 +1,179 @@
+import Head from "next/head";
+import { CKAN, Dataset } from "@portaljs/ckan";
+import {
+  ChevronRightIcon,
+  HomeIcon,
+  PaperClipIcon,
+} from "@heroicons/react/20/solid";
+import Link from "next/link";
+import getConfig from "next/config";
+
+const backend_url = getConfig().publicRuntimeConfig.DMS
+
+export const getServerSideProps = async (context: any) => {
+  try {
+    const datasetName = context.params?.dataset;
+    if (!datasetName) {
+      return {
+        notFound: true,
+      };
+    }
+    const ckan = new CKAN(backend_url);
+    const dataset = await ckan.getDatasetDetails(datasetName as string);
+    if (!dataset) {
+      return {
+        notFound: true,
+      };
+    }
+    return {
+      props: { dataset },
+    };
+  } catch {
+    return {
+      notFound: true,
+    };
+  }
+};
+
+export default function DatasetPage({
+  dataset,
+}: {
+  dataset: Dataset;
+}): JSX.Element {
+  return (
+    <>
+      <Head>
+        <title>{`${dataset.title || dataset.name} - Dataset`}</title>
+        <meta name="description" content="Generated by create next app" />
+        <link rel="icon" href="/favicon.ico" />
+      </Head>
+      <main className="flex min-h-screen flex-col items-center justify-between p-24 bg-zinc-900">
+        <div className="bg-white p-8 my-4 rounded-lg">
+          <nav className="flex px-4 py-8" aria-label="Breadcrumb">
+            <ol role="list" className="flex items-center space-x-4">
+              <li>
+                <div>
+                  <Link href="/" className="text-gray-400 hover:text-gray-500">
+                    <HomeIcon
+                      className="h-5 w-5 flex-shrink-0"
+                      aria-hidden="true"
+                    />
+                    <span className="sr-only">Home</span>
+                  </Link>
+                </div>
+              </li>
+              <li>
+                <div className="flex items-center">
+                  <ChevronRightIcon
+                    className="h-5 w-5 flex-shrink-0 text-gray-400"
+                    aria-hidden="true"
+                  />
+                  <span
+                    className="ml-4 text-sm font-medium text-gray-500 hover:text-gray-700"
+                    aria-current={"page"}
+                  >
+                    {dataset.name}
+                  </span>
+                </div>
+              </li>
+            </ol>
+          </nav>
+          {dataset && (
+            <div>
+              <div className="px-4 sm:px-0">
+                <h3 className="text-base font-semibold leading-7 text-gray-900">
+                  {dataset.title || dataset.name}
+                </h3>
+                <p className="mt-1 max-w-2xl text-sm leading-6 text-gray-500">
+                  Dataset details
+                </p>
+              </div>
+              <div className="mt-6 border-t border-gray-100">
+                <dl className="divide-y divide-gray-100">
+                  <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                    <dt className="text-sm font-medium leading-6 text-gray-900">
+                      Title
+                    </dt>
+                    <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                      {dataset.title}
+                    </dd>
+                  </div>
+                  {dataset.tags && dataset.tags.length > 0 && (
+                    <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                      <dt className="text-sm font-medium leading-6 text-gray-900">
+                        Tags
+                      </dt>
+                      <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                        {dataset.tags.map((tag) => tag.display_name).join(", ")}
+                      </dd>
+                    </div>
+                  )}
+                  {dataset.tags && dataset.tags.length > 0 && (
+                    <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                      <dt className="text-sm font-medium leading-6 text-gray-900">
+                        URL
+                      </dt>
+                      <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                        {dataset.url}
+                      </dd>
+                    </div>
+                  )}
+                  <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                    {dataset.notes && (
+                      <>
+                        <dt className="text-sm font-medium leading-6 text-gray-900">
+                          Description
+                        </dt>
+                        <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                          {dataset.notes}
+                        </dd>
+                      </>
+                    )}
+                  </div>
+                  <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                    <dt className="text-sm font-medium leading-6 text-gray-900">
+                      Files
+                    </dt>
+                    <dd className="mt-2 text-sm text-gray-900 sm:col-span-2 sm:mt-0">
+                      <ul
+                        role="list"
+                        className="divide-y divide-gray-100 rounded-md border border-gray-200"
+                      >
+                        {dataset.resources.map((resource) => (
+                          <li key={resource.id} className="flex items-center justify-between py-4 pl-4 pr-5 text-sm leading-6">
+                            <div className="flex w-0 flex-1 items-center">
+                              <PaperClipIcon
+                                className="h-5 w-5 flex-shrink-0 text-gray-400"
+                                aria-hidden="true"
+                              />
+                              <div className="ml-4 flex min-w-0 flex-1 gap-2">
+                                <span className="truncate font-medium">
+                                  {resource.name || resource.id}
+                                </span>
+                                <span className="flex-shrink-0 text-gray-400">
+                                  {resource.size}
+                                </span>
+                              </div>
+                            </div>
+                            <div className="ml-4 flex-shrink-0">
+                              <a
+                                href={resource.url}
+                                className="font-medium hover:text-indigo-500 mr-4"
+                              >
+                                Download
+                              </a>
+                            </div>
+                          </li>
+                        ))}
+                      </ul>
+                    </dd>
+                  </div>
+                </dl>
+              </div>
+            </div>
+          )}
+        </div>
+      </main>
+    </>
+  );
+}

examples/ckan/pages/_app.tsx

@@ -0,0 +1,7 @@
+import '@/styles/globals.css'
+import '@portaljs/ckan/styles.css'
+import type { AppProps } from 'next/app'
+
+export default function App({ Component, pageProps }: AppProps) {
+  return <Component {...pageProps} />
+}

examples/ckan/pages/_document.tsx

@@ -0,0 +1,13 @@
+import { Html, Head, Main, NextScript } from 'next/document'
+
+export default function Document() {
+  return (
+    <Html lang="en">
+      <Head />
+      <body>
+        <Main />
+        <NextScript />
+      </body>
+    </Html>
+  )
+}

examples/ckan/pages/api/cors.ts

@@ -0,0 +1,18 @@
+import fetch from 'isomorphic-unfetch';
+
+const Cors = async (req: any, res: any) => {
+  const { url } = req.query;
+  try {
+    const resProxy = await fetch(url, {
+      headers: {
+        Range: 'bytes=0-5132288',
+      },
+    });
+    const data = await resProxy.text();
+    return res.status(200).send(data);
+  } catch (error: any) {
+    res.status(400).send(error.toString());
+  }
+};
+
+export default Cors;

examples/ckan/pages/index.tsx

@@ -0,0 +1,55 @@
+import {
+  CKAN,
+  DatasetSearchForm,
+  ListOfDatasets,
+  PackageSearchOptions,
+  Organization,
+  Group,
+} from '@portaljs/ckan';
+import getConfig from 'next/config';
+import { useState } from 'react';
+
+const backend_url = getConfig().publicRuntimeConfig.DMS;
+
+export async function getServerSideProps() {
+  const ckan = new CKAN(backend_url);
+  const groups = await ckan.getGroupsWithDetails();
+  const orgs = await ckan.getOrgsWithDetails();
+  return {
+    props: {
+      groups,
+      orgs,
+    },
+  };
+}
+
+export default function Home({
+  orgs,
+  groups,
+}: {
+  orgs: Organization[];
+  groups: Group[];
+}) {
+  const ckan = new CKAN(backend_url);
+  const [options, setOptions] = useState<PackageSearchOptions>({
+    offset: 0,
+    limit: 5,
+    tags: [],
+    groups: [],
+    orgs: [],
+  });
+
+  return (
+    <main className="flex min-h-screen flex-col items-center p-24 bg-zinc-900">
+      <DatasetSearchForm
+        options={options}
+        setOptions={setOptions}
+        groups={groups}
+        orgs={orgs}
+      />
+      <div className="bg-white p-8 my-4 rounded-lg">
+        <ListOfDatasets options={options} setOptions={setOptions} ckan={ckan} />{' '}
+      </div>
+    </main>
+  );
+}

examples/ckan/pages/stories/[[...path]].tsx

@@ -0,0 +1,129 @@
+import { existsSync, promises as fs } from 'fs';
+import path from 'path';
+import parse from '../../lib/markdown';
+
+import DataRichDocument from '../../components/DataRichDocument';
+import clientPromise from '../../lib/mddb';
+import getConfig from 'next/config';
+import { CKAN } from '@portaljs/ckan';
+
+export const getStaticPaths = async () => {
+  const contentDir = path.join(process.cwd(), '/content/');
+  const contentFolders = await fs.readdir(contentDir, 'utf8');
+  const paths = contentFolders.map((folder: string) => ({
+    params: { path: [folder.split('.')[0]] },
+  }));
+  return {
+    paths,
+    fallback: false,
+  };
+};
+
+const backend_url = getConfig().publicRuntimeConfig.DMS;
+
+export const getStaticProps = async (context) => {
+  const mddb = await clientPromise;
+  const storyFile = await mddb.getFileByUrl(context.params.path);
+  const md = await fs.readFile(
+    `${process.cwd()}/${storyFile.file_path}`,
+    'utf8'
+  );
+
+  const ckan = new CKAN(backend_url);
+  const datasets = storyFile.metadata.datasets ? await Promise.all(
+    storyFile.metadata.datasets.map(
+      async (datasetName: string) => await ckan.getDatasetDetails(datasetName)
+    )
+  ) : [];
+  const orgs = storyFile.metadata.orgs ? await Promise.all(
+    storyFile.metadata.orgs.map(
+      async (orgName: string) => await ckan.getOrgDetails(orgName)
+    )
+  ) : [];
+
+  let { mdxSource, frontMatter } = await parse(md, '.mdx', { datasets, orgs });
+
+  return {
+    props: {
+      mdxSource,
+      frontMatter: JSON.stringify(frontMatter),
+    },
+  };
+};
+
+export default function DatasetPage({ mdxSource, frontMatter }) {
+  frontMatter = JSON.parse(frontMatter);
+  return (
+    <main className="flex min-h-screen flex-col justify-between p-16 bg-zinc-900">
+      <div className="bg-white p-8 my-4 rounded-lg">
+        <div className="prose mx-auto py-8">
+          <header>
+            <div className="mb-6">
+              <>
+                <h1 className="mb-2">{frontMatter.title}</h1>
+                {frontMatter.author && (
+                  <p className="my-0">
+                    <span className="font-semibold">Author: </span>
+                    <span className="my-0">{frontMatter.author}</span>
+                  </p>
+                )}
+                {frontMatter.description && (
+                  <p className="my-0">
+                    <span className="font-semibold">Description: </span>
+                    <span className="description my-0">
+                      {frontMatter.description}
+                    </span>
+                  </p>
+                )}
+                {frontMatter.modified && (
+                  <p className="my-0">
+                    <span className="font-semibold">Modified: </span>
+                    <span className="description my-0">
+                      {new Date(frontMatter.modified).toLocaleDateString()}
+                    </span>
+                  </p>
+                )}
+                {frontMatter.files && (
+                  <section className="py-6">
+                    <h2 className="mt-0">Data files</h2>
+                    <table className="table-auto">
+                      <thead>
+                        <tr>
+                          <th>File</th>
+                          <th>Format</th>
+                        </tr>
+                      </thead>
+                      <tbody>
+                        {frontMatter.files.map((f) => {
+                          const fileName = f.split('/').slice(-1);
+                          return (
+                            <tr key={`resources-list-${f}`}>
+                              <td>
+                                <a target="_blank" href={f}>
+                                  {fileName}
+                                </a>
+                              </td>
+                              <td>
+                                {fileName[0]
+                                  .split('.')
+                                  .slice(-1)[0]
+                                  .toUpperCase()}
+                              </td>
+                            </tr>
+                          );
+                        })}
+                      </tbody>
+                    </table>
+                  </section>
+                )}
+              </>
+            </div>
+          </header>
+          <main>
+            <DataRichDocument source={mdxSource} />
+          </main>
+        </div>
+      </div>
+    </main>
+  );
+}

examples/ckan/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

examples/ckan/public/vercel.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

examples/ckan/styles/globals.css

@@ -0,0 +1,71 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+@import "@flowershow/remark-callouts/styles.css";
+
+/* mathjax */
+.math-inline > mjx-container > svg {
+  display: inline;
+  align-items: center;
+}
+
+/* smooth scrolling in modern browsers */
+html {
+  scroll-behavior: smooth !important;
+}
+
+/* tooltip fade-out clip */
+.tooltip-body::after {
+  content: "";
+  position: absolute;
+  right: 0;
+  top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
+  height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
+  width: 10rem;
+  background: linear-gradient(
+    to right,
+    rgba(255, 255, 255, 0),
+    rgba(255, 255, 255, 1) 100%
+  );
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
+  margin-left: -2rem !important;
+  padding-left: 2rem !important;
+  scroll-margin-top: 4.5rem;
+  position: relative;
+}
+
+.heading-link {
+  padding: 1px;
+  position: absolute;
+  left: 0;
+  top: 50%;
+  transform: translateY(-50%);
+  margin: auto 0;
+  border-radius: 5px;
+  background: #1e293b;
+  opacity: 0;
+  transition: opacity 0.2s;
+}
+
+.light .heading-link {
+  /* border: 1px solid #ab2b65; */
+  /* background: none; */
+  background: #e2e8f0;
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
+  opacity: 100;
+}
+
+.heading-link svg {
+  transform: scale(0.75);
+}
+
+@media screen and (max-width: 640px) {
+  .heading-link {
+    visibility: hidden;
+  }
+}

examples/ckan/tailwind.config.js

@@ -0,0 +1,18 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    './pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './components/**/*.{js,ts,jsx,tsx,mdx}',
+    './app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      backgroundImage: {
+        'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
+        'gradient-conic':
+          'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
+      },
+    },
+  },
+  plugins: [require('@tailwindcss/typography')],
+}

examples/ckan/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": false,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+  "exclude": ["node_modules"]
+}

examples/github-backed-catalog/components/DataRichDocument.tsx

@@ -0,0 +1,21 @@
+import { MDXRemote } from 'next-mdx-remote';
+import dynamic from 'next/dynamic';
+import { Mermaid } from '@flowershow/core';
+
+// Custom components/renderers to pass to MDX.
+// Since the MDX files aren't loaded by webpack, they have no knowledge of how
+// to handle import statements. Instead, you must include components in scope
+// here.
+const components = {
+  Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
+  Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
+  FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
+  mermaid: Mermaid,
+  Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
+  VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),
+  LineChart: dynamic(() => import('@portaljs/components').then(mod => mod.LineChart)),
+} as any;
+
+export default function DRD({ source }: { source: any }) {
+  return <MDXRemote {...source} components={components} />;
+}

examples/github-backed-catalog/datasets.json

@@ -15,6 +15,13 @@
     ],
     "readme": "README.md"
   },
+  {
+    "owner": "luccasmmg",
+    "branch": "main",
+    "repo": "test-data-repo-1",
+    "files": ["data_1.csv", "data_2.csv"],
+    "readme": "README.md"
+  },
   {
     "owner": "datasets",
     "branch": "main",

examples/github-backed-catalog/lib/markdown.js

@@ -0,0 +1,105 @@
+import matter from "gray-matter";
+import mdxmermaid from "mdx-mermaid";
+import { h } from "hastscript";
+import remarkCallouts from "@flowershow/remark-callouts";
+import remarkEmbed from "@flowershow/remark-embed";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import remarkSmartypants from "remark-smartypants";
+import remarkToc from "remark-toc";
+import remarkWikiLink from "@flowershow/remark-wiki-link";
+import rehypeAutolinkHeadings from "rehype-autolink-headings";
+import rehypeKatex from "rehype-katex";
+import rehypeSlug from "rehype-slug";
+import rehypePrismPlus from "rehype-prism-plus";
+
+import { serialize } from "next-mdx-remote/serialize";
+
+/**
+ * Parse a markdown or MDX file to an MDX source form + front matter data
+ *
+ * @source: the contents of a markdown or mdx file
+ * @format: used to indicate to next-mdx-remote which format to use (md or mdx)
+ * @returns: { mdxSource: mdxSource, frontMatter: ...}
+ */
+const parse = async function (source, format, scope) {
+  const { content, data, excerpt } = matter(source, {
+    excerpt: (file, options) => {
+      // Generate an excerpt for the file
+      file.excerpt = file.content.split("\n\n")[0];
+    },
+  });
+
+  const mdxSource = await serialize(
+    { value: content, path: format },
+    {
+      // Optionally pass remark/rehype plugins
+      mdxOptions: {
+        remarkPlugins: [
+          remarkEmbed,
+          remarkGfm,
+          [remarkSmartypants, { quotes: false, dashes: "oldschool" }],
+          remarkMath,
+          remarkCallouts,
+          remarkWikiLink,
+          [
+            remarkToc,
+            {
+              heading: "Table of contents",
+              tight: true,
+            },
+          ],
+          [mdxmermaid, {}],
+        ],
+        rehypePlugins: [
+          rehypeSlug,
+          [
+            rehypeAutolinkHeadings,
+            {
+              properties: { className: 'heading-link' },
+              test(element) {
+                return (
+                  ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
+                  element.properties?.id !== "table-of-contents" &&
+                  element.properties?.className !== "blockquote-heading"
+                );
+              },
+              content() {
+                return [
+                  h(
+                    "svg",
+                    {
+                      xmlns: "http:www.w3.org/2000/svg",
+                      fill: "#ab2b65",
+                      viewBox: "0 0 20 20",
+                      className: "w-5 h-5",
+                    },
+                    [
+                      h("path", {
+                        fillRule: "evenodd",
+                        clipRule: "evenodd",
+                        d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
+                      }),
+                    ]
+                  ),
+                ];
+              },
+            },
+          ],
+          [rehypeKatex, { output: "mathml" }],
+          [rehypePrismPlus, { ignoreMissing: true }],
+        ],
+        format,
+      },
+      scope,
+    }
+  );
+
+  return {
+    mdxSource: mdxSource,
+    frontMatter: data,
+    excerpt,
+  };
+};
+
+export default parse;

examples/github-backed-catalog/lib/mddb.ts

@@ -0,0 +1,14 @@
+import { MarkdownDB } from "@flowershow/markdowndb";
+
+const dbPath = "markdown.db";
+
+const client = new MarkdownDB({
+  client: "sqlite3",
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const clientPromise = client.init();
+
+export default clientPromise;

examples/github-backed-catalog/lib/octokit.ts

@@ -39,6 +39,32 @@ export async function getProjectReadme(
   }
 }
 
+export async function getProjectDatapackage(
+  owner: string,
+  repo: string,
+  branch: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.getContent({
+      owner,
+      repo,
+      path: "datapackage.json",
+      ref: branch,
+    });
+    const data = response.data as { content?: string };
+    const fileContent = data.content ? data.content : "";
+    if (fileContent === "") {
+      return null;
+    }
+    const decodedContent = Buffer.from(fileContent, "base64").toString();
+    return JSON.parse(decodedContent);
+  } catch (error) {
+    return null
+  }
+}
+
 export async function getLastUpdated(
   owner: string,
   repo: string,
@@ -162,11 +188,20 @@ export async function getProject(project: GithubProject, github_pat?: string) {
     projectBase,
     github_pat
   );
+
+  const projectDatapackage = await getProjectDatapackage(
+    project.owner,
+    project.repo,
+    project.branch,
+    github_pat
+  );
+
   return {
     ...projectMetadata,
     files: projectData,
     readmeContent: projectReadme,
     last_updated,
     base_path: projectBase,
+    datapackage: projectDatapackage
   };
 }

examples/github-backed-catalog/package.json

@@ -10,19 +10,32 @@
     "prettier": "prettier --write ."
   },
   "dependencies": {
+    "@flowershow/core": "^0.4.13",
+    "@flowershow/markdowndb": "^0.1.5",
+    "@flowershow/remark-callouts": "^1.0.0",
+    "@flowershow/remark-embed": "^1.0.0",
+    "@portaljs/components": "^0.1.6",
     "@types/node": "18.16.0",
     "@types/react": "18.0.38",
     "@types/react-dom": "18.0.11",
     "eslint": "8.39.0",
     "eslint-config-next": "13.3.1",
-    "next": "13.3.1",
+    "next": "13.4.3",
+    "next-mdx-remote": "^4.4.1",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-markdown": "^8.0.7",
     "react-timeago": "^7.1.0",
+    "rehype-autolink-headings": "^6.1.1",
+    "rehype-katex": "^6.0.3",
+    "rehype-prism-plus": "^1.5.1",
+    "rehype-slug": "^5.1.0",
     "remark-gfm": "^3.0.1",
+    "remark-math": "^5.1.1",
+    "remark-smartypants": "^2.0.0",
+    "remark-toc": "^8.0.1",
     "typescript": "5.0.4"
   },
   "devDependencies": {

examples/github-backed-catalog/pages/@org/[org]/[...path].tsx

@@ -6,6 +6,8 @@ import { getProject, GithubProject } from "../../../lib/octokit";
 import ReactMarkdown from "react-markdown";
 import remarkGfm from "remark-gfm";
 import Breadcrumbs from "../../../components/_shared/Breadcrumbs";
+import parse from '../../../lib/markdown';
+import DataRichDocument from '../../../components/DataRichDocument'
 
 export default function ProjectPage({ project }) {
   const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`;
@@ -64,9 +66,7 @@ export default function ProjectPage({ project }) {
         <hr />
 
         <h2 className="uppercase font-black">Readme</h2>
-        <ReactMarkdown remarkPlugins={[remarkGfm]}>
+        <DataRichDocument source={project.mdxSource} />
-          {project.readmeContent}
-        </ReactMarkdown>
       </main>
     </>
   );
@@ -119,9 +119,10 @@ export async function getStaticProps({ params }) {
   });
   const github_pat = getConfig().serverRuntimeConfig.github_pat;
   const project = await getProject(repo, github_pat);
+  let { mdxSource, frontMatter } = await parse(project.readmeContent, '.mdx', { project });
   return {
     props: {
-      project: { ...project, repo_config: repo },
+      project: { ...project, repo_config: repo, mdxSource },
     },
   };
 }

examples/github-backed-catalog/pages/styles.css

@@ -78,3 +78,72 @@ pre {
   color: rgba(55, 65, 81, 1);
   width: 100%;
 }
+
+@import "@flowershow/remark-callouts/styles.css";
+
+/* mathjax */
+.math-inline > mjx-container > svg {
+  display: inline;
+  align-items: center;
+}
+
+/* smooth scrolling in modern browsers */
+html {
+  scroll-behavior: smooth !important;
+}
+
+/* tooltip fade-out clip */
+.tooltip-body::after {
+  content: "";
+  position: absolute;
+  right: 0;
+  top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
+  height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
+  width: 10rem;
+  background: linear-gradient(
+    to right,
+    rgba(255, 255, 255, 0),
+    rgba(255, 255, 255, 1) 100%
+  );
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
+  margin-left: -2rem !important;
+  padding-left: 2rem !important;
+  scroll-margin-top: 4.5rem;
+  position: relative;
+}
+
+.heading-link {
+  padding: 1px;
+  position: absolute;
+  left: 0;
+  top: 50%;
+  transform: translateY(-50%);
+  margin: auto 0;
+  border-radius: 5px;
+  background: #1e293b;
+  opacity: 0;
+  transition: opacity 0.2s;
+}
+
+.light .heading-link {
+  /* border: 1px solid #ab2b65; */
+  /* background: none; */
+  background: #e2e8f0;
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
+  opacity: 100;
+}
+
+.heading-link svg {
+  transform: scale(0.75);
+}
+
+@media screen and (max-width: 640px) {
+  .heading-link {
+    visibility: hidden;
+  }
+}
+

examples/learn/components/DataRichDocument.tsx

@@ -0,0 +1,21 @@
+import { MDXRemote } from 'next-mdx-remote';
+import dynamic from 'next/dynamic';
+import { Mermaid } from '@flowershow/core';
+
+// Custom components/renderers to pass to MDX.
+// Since the MDX files aren't loaded by webpack, they have no knowledge of how
+// to handle import statements. Instead, you must include components in scope
+// here.
+const components = {
+  Table: dynamic(() => import('../portaljs/components/Table').then(mod => mod.Table)),
+  Catalog: dynamic(() => import('../portaljs/components/Catalog').then(mod => mod.Catalog)),
+  mermaid: Mermaid,
+  Vega: dynamic(() => import('../portaljs/components/Vega').then(mod => mod.Vega)),
+  VegaLite: dynamic(() => import('../portaljs/components/VegaLite').then(mod => mod.VegaLite)),
+  LineChart: dynamic(() => import('../portaljs/components/LineChart').then(mod => mod.LineChart)),
+  FlatUiTable: dynamic(() => import('../portaljs/components/FlatUiTable').then(mod => mod.FlatUiTable)),
+} as any;
+
+export default function DRD({ source }: { source: any }) {
+  return <MDXRemote {...source} components={components} />;
+}

examples/learn/lib/markdown.js

@@ -0,0 +1,105 @@
+import matter from "gray-matter";
+import mdxmermaid from "mdx-mermaid";
+import { h } from "hastscript";
+import remarkCallouts from "@flowershow/remark-callouts";
+import remarkEmbed from "@flowershow/remark-embed";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import remarkSmartypants from "remark-smartypants";
+import remarkToc from "remark-toc";
+import remarkWikiLink from "@flowershow/remark-wiki-link";
+import rehypeAutolinkHeadings from "rehype-autolink-headings";
+import rehypeKatex from "rehype-katex";
+import rehypeSlug from "rehype-slug";
+import rehypePrismPlus from "rehype-prism-plus";
+
+import { serialize } from "next-mdx-remote/serialize";
+
+/**
+ * Parse a markdown or MDX file to an MDX source form + front matter data
+ *
+ * @source: the contents of a markdown or mdx file
+ * @format: used to indicate to next-mdx-remote which format to use (md or mdx)
+ * @returns: { mdxSource: mdxSource, frontMatter: ...}
+ */
+const parse = async function (source, format, scope) {
+  const { content, data, excerpt } = matter(source, {
+    excerpt: (file, options) => {
+      // Generate an excerpt for the file
+      file.excerpt = file.content.split("\n\n")[0];
+    },
+  });
+
+  const mdxSource = await serialize(
+    { value: content, path: format },
+    {
+      // Optionally pass remark/rehype plugins
+      mdxOptions: {
+        remarkPlugins: [
+          remarkEmbed,
+          remarkGfm,
+          [remarkSmartypants, { quotes: false, dashes: "oldschool" }],
+          remarkMath,
+          remarkCallouts,
+          remarkWikiLink,
+          [
+            remarkToc,
+            {
+              heading: "Table of contents",
+              tight: true,
+            },
+          ],
+          [mdxmermaid, {}],
+        ],
+        rehypePlugins: [
+          rehypeSlug,
+          [
+            rehypeAutolinkHeadings,
+            {
+              properties: { className: 'heading-link' },
+              test(element) {
+                return (
+                  ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
+                  element.properties?.id !== "table-of-contents" &&
+                  element.properties?.className !== "blockquote-heading"
+                );
+              },
+              content() {
+                return [
+                  h(
+                    "svg",
+                    {
+                      xmlns: "http:www.w3.org/2000/svg",
+                      fill: "#ab2b65",
+                      viewBox: "0 0 20 20",
+                      className: "w-5 h-5",
+                    },
+                    [
+                      h("path", {
+                        fillRule: "evenodd",
+                        clipRule: "evenodd",
+                        d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
+                      }),
+                    ]
+                  ),
+                ];
+              },
+            },
+          ],
+          [rehypeKatex, { output: "mathml" }],
+          [rehypePrismPlus, { ignoreMissing: true }],
+        ],
+        format,
+      },
+      scope,
+    }
+  );
+
+  return {
+    mdxSource: mdxSource,
+    frontMatter: data,
+    excerpt,
+  };
+};
+
+export default parse;

examples/learn/lib/mddb.ts

@@ -0,0 +1,14 @@
+import { MarkdownDB } from "@flowershow/markdowndb";
+
+const dbPath = "markdown.db";
+
+const client = new MarkdownDB({
+  client: "sqlite3",
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const clientPromise = client.init();
+
+export default clientPromise;

examples/learn/package.json

@@ -17,21 +17,12 @@
     "@flowershow/remark-callouts": "^1.0.0",
     "@flowershow/remark-embed": "^1.0.0",
     "@flowershow/remark-wiki-link": "^1.1.2",
-    "@heroicons/react": "^2.0.17",
     "@opentelemetry/api": "^1.4.0",
     "@portaljs/components": "^0.1.0",
-    "@tanstack/react-table": "^8.8.5",
-    "flexsearch": "0.7.21",
     "gray-matter": "^4.0.3",
     "hastscript": "^7.2.0",
     "mdx-mermaid": "2.0.0-rc7",
     "next": "13.2.1",
-    "next-mdx-remote": "^4.4.1",
-    "papaparse": "^5.4.1",
-    "react": "18.2.0",
-    "react-dom": "18.2.0",
-    "react-hook-form": "^7.43.9",
-    "react-vega": "^7.6.0",
     "rehype-autolink-headings": "^6.1.1",
     "rehype-katex": "^6.0.3",
     "rehype-prism-plus": "^1.5.1",
@@ -40,7 +31,20 @@
     "remark-math": "^5.1.1",
     "remark-smartypants": "^2.0.0",
     "remark-toc": "^8.0.1",
-    "typescript": "5.0.4"
+    "typescript": "5.0.4",
+    "@githubocto/flat-ui": "^0.14.1",
+    "@heroicons/react": "^2.0.17",
+    "@tanstack/react-table": "^8.8.5",
+    "flexsearch": "0.7.21",
+    "next-mdx-remote": "^4.4.1",
+    "papaparse": "^5.4.1",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-hook-form": "^7.43.9",
+    "react-query": "^3.39.3",
+    "react-vega": "^7.6.0",
+    "vega": "5.25.0",
+    "vega-lite": "5.1.0"
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",

examples/learn/portaljs/components/Catalog.tsx

@@ -0,0 +1,119 @@
+import { Index } from 'flexsearch';
+import { useState } from 'react';
+import DebouncedInput from './DebouncedInput';
+import { useForm } from 'react-hook-form';
+
+export function Catalog({
+  datasets,
+  facets,
+}: {
+  datasets: any[];
+  facets: string[];
+}) {
+  const [indexFilter, setIndexFilter] = useState('');
+  const index = new Index({ tokenize: 'full' });
+  datasets.forEach((dataset) =>
+    index.add(
+      dataset._id,
+      //This will join every metadata value + the url_path into one big string and index that
+      Object.entries(dataset.metadata).reduce(
+        (acc, curr) => acc + ' ' + curr[1].toString(),
+        ''
+      ) +
+        ' ' +
+        dataset.url_path
+    )
+  );
+
+  const facetValues = facets
+    ? facets.reduce((acc, facet) => {
+        const possibleValues = datasets.reduce((acc, curr) => {
+          const facetValue = curr.metadata[facet];
+          if (facetValue) {
+            return Array.isArray(facetValue)
+              ? acc.concat(facetValue)
+              : acc.concat([facetValue]);
+          }
+          return acc;
+        }, []);
+        acc[facet] = {
+          possibleValues: [...new Set(possibleValues)],
+          selectedValue: null,
+        };
+        return acc;
+      }, {})
+    : [];
+
+  const { register, watch } = useForm(facetValues);
+
+  const filteredDatasets = datasets
+    // First filter by flex search
+    .filter((dataset) =>
+      indexFilter !== ''
+        ? index.search(indexFilter).includes(dataset._id)
+        : true
+    )
+    //Then check if the selectedValue for the given facet is included in the dataset metadata
+    .filter((dataset) => {
+      //Avoids a server rendering breakage
+      if (!watch() || Object.keys(watch()).length === 0) return true
+      //This will filter only the key pairs of the metadata values that were selected as facets
+      const datasetFacets = Object.entries(dataset.metadata).filter((entry) =>
+        facets.includes(entry[0])
+      );
+      //Check if the value present is included in the selected value in the form
+      return datasetFacets.every((elem) =>
+        watch()[elem[0]].selectedValue
+          ? (elem[1] as string | string[]).includes(
+              watch()[elem[0]].selectedValue
+            )
+          : true
+      );
+    });
+
+  return (
+    <>
+      <DebouncedInput
+        value={indexFilter ?? ''}
+        onChange={(value) => setIndexFilter(String(value))}
+        className="p-2 text-sm shadow border border-block mr-1"
+        placeholder="Search all datasets..."
+      />
+      {Object.entries(facetValues).map((elem) => (
+        <select
+          key={elem[0]}
+          defaultValue=""
+          className="p-2 ml-1 text-sm shadow border border-block"
+          {...register(elem[0] + '.selectedValue')}
+        >
+          <option value="">
+            Filter by {elem[0]}
+          </option>
+          {(elem[1] as { possibleValues: string[] }).possibleValues.map(
+            (val) => (
+              <option
+                key={val}
+                className="dark:bg-white dark:text-black"
+                value={val}
+              >
+                {val}
+              </option>
+            )
+          )}
+        </select>
+      ))}
+      <ul className='mb-5 pl-6 mt-5 list-disc'>
+        {filteredDatasets.map((dataset) => (
+          <li className='py-2' key={dataset._id}>
+            <a className='font-medium underline' href={dataset.url_path}>
+              {dataset.metadata.title
+                ? dataset.metadata.title
+                : dataset.url_path}
+            </a>
+          </li>
+        ))}
+      </ul>
+    </>
+  );
+}
+

examples/learn/portaljs/components/DebouncedInput.tsx

@@ -0,0 +1,32 @@
+import { useEffect, useState } from "react";
+
+const DebouncedInput = ({
+  value: initialValue,
+  onChange,
+  debounce = 500,
+  ...props
+}) => {
+  const [value, setValue] = useState(initialValue);
+
+  useEffect(() => {
+    setValue(initialValue);
+  }, [initialValue]);
+
+  useEffect(() => {
+    const timeout = setTimeout(() => {
+      onChange(value);
+    }, debounce);
+
+    return () => clearTimeout(timeout);
+  }, [value]);
+
+  return (
+    <input
+      {...props}
+      value={value}
+      onChange={(e) => setValue(e.target.value)}
+    />
+  );
+};
+
+export default DebouncedInput;

examples/learn/portaljs/components/FlatUiTable.tsx

@@ -0,0 +1,113 @@
+import { QueryClient, QueryClientProvider, useQuery } from 'react-query';
+import Papa from 'papaparse';
+import { Grid } from '@githubocto/flat-ui';
+
+const queryClient = new QueryClient();
+
+export async function getCsv(url: string, corsProxy?: string) {
+  if (corsProxy) {
+    url = corsProxy + url
+  }
+  const response = await fetch(url, {
+    headers: {
+      Range: 'bytes=0-5132288',
+    },
+  });
+  const data = await response.text();
+  return data;
+}
+
+export async function parseCsv(file: string): Promise<any> {
+  return new Promise((resolve, reject) => {
+    Papa.parse(file, {
+      header: true,
+      dynamicTyping: true,
+      skipEmptyLines: true,
+      transform: (value: string): string => {
+        return value.trim();
+      },
+      complete: (results: any) => {
+        return resolve(results);
+      },
+      error: (error: any) => {
+        return reject(error);
+      },
+    });
+  });
+}
+
+const Spinning = () => {
+  return (
+    <div role="status w-fit mx-auto">
+      <svg
+        aria-hidden="true"
+        className="w-8 h-8 mr-2 text-gray-200 animate-spin dark:text-gray-600 fill-emerald-600"
+        viewBox="0 0 100 101"
+        fill="none"
+        xmlns="http://www.w3.org/2000/svg"
+      >
+        <path
+          d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
+          fill="currentColor"
+        />
+        <path
+          d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
+          fill="currentFill"
+        />
+      </svg>
+      <span className="sr-only">Loading...</span>
+    </div>
+  );
+};
+
+export interface FlatUiTableProps {
+  url?: string;
+  data?: { [key: string]: number | string }[];
+  rawCsv?: string;
+  corsProxy?: string;
+}
+export const FlatUiTable: React.FC<FlatUiTableProps> = ({
+  url,
+  data,
+  rawCsv,
+  corsProxy,
+}) => {
+  return (
+    // Provide the client to your App
+    <QueryClientProvider client={queryClient}>
+      <TableInner corsProxy={corsProxy} url={url} data={data} rawCsv={rawCsv} />
+    </QueryClientProvider>
+  );
+};
+
+const TableInner: React.FC<FlatUiTableProps> = ({ url, data, rawCsv, corsProxy }) => {
+  if (data) {
+    return (
+      <div className="w-full" style={{height: '500px'}}>
+        <Grid data={data} />
+      </div>
+    );
+  }
+  const { data: csvString, isLoading: isDownloadingCSV } = useQuery(
+    ['dataCsv', url],
+    () => getCsv(url as string, corsProxy),
+    { enabled: !!url }
+  );
+  const { data: parsedData, isLoading: isParsing } = useQuery(
+    ['dataPreview', csvString],
+    () => parseCsv(rawCsv ? rawCsv as string : csvString as string),
+    { enabled: rawCsv ? true : !!csvString }
+  );
+  if (isParsing || isDownloadingCSV)
+    <div className="w-full">
+      <Spinning />
+    </div>;
+  if (parsedData)
+    return (
+      <div className="w-full" style={{height: '500px'}}>
+        <Grid data={parsedData.data} />
+      </div>
+    );
+  return <Spinning />
+};
+

examples/learn/portaljs/components/LineChart.tsx

@@ -0,0 +1,63 @@
+import { VegaLite } from './VegaLite';
+
+export type LineChartProps = {
+  data: Array<Array<string | number>> | string | { x: string; y: number }[];
+  title?: string;
+  xAxis?: string;
+  yAxis?: string;
+  fullWidth?: boolean;
+};
+
+export function LineChart({
+  data = [],
+  fullWidth = false,
+  title = '',
+  xAxis = 'x',
+  yAxis = 'y',
+}: LineChartProps) {
+  var tmp = data;
+  if (Array.isArray(data)) {
+    tmp = data.map((r) => {
+      return { x: r[0], y: r[1] };
+    });
+  }
+  const vegaData = { table: tmp };
+  const spec = {
+    $schema: 'https://vega.github.io/schema/vega-lite/v5.json',
+    title,
+    width: 600,
+    height: 300,
+    mark: {
+      type: 'line',
+      color: 'black',
+      strokeWidth: 1,
+      tooltip: true,
+    },
+    data: {
+      name: 'table',
+    },
+    selection: {
+      grid: {
+        type: 'interval',
+        bind: 'scales',
+      },
+    },
+    encoding: {
+      x: {
+        field: xAxis,
+        timeUnit: 'year',
+        type: 'temporal',
+      },
+      y: {
+        field: yAxis,
+        type: 'quantitative',
+      },
+    },
+  };
+  if (typeof data === 'string') {
+    spec.data = { url: data } as any;
+    return <VegaLite fullWidth={fullWidth} spec={spec} />;
+  }
+
+  return <VegaLite fullWidth={fullWidth} data={vegaData} spec={spec} />;
+}

examples/learn/portaljs/components/Table.tsx

@@ -0,0 +1,195 @@
+import {
+  createColumnHelper,
+  FilterFn,
+  flexRender,
+  getCoreRowModel,
+  getFilteredRowModel,
+  getPaginationRowModel,
+  getSortedRowModel,
+  useReactTable,
+} from '@tanstack/react-table';
+
+import {
+  ArrowDownIcon,
+  ArrowUpIcon,
+  ChevronDoubleLeftIcon,
+  ChevronDoubleRightIcon,
+  ChevronLeftIcon,
+  ChevronRightIcon,
+} from '@heroicons/react/24/solid';
+
+import React, { useEffect, useMemo, useState } from 'react';
+
+import parseCsv from '../lib/parseCsv';
+import DebouncedInput from './DebouncedInput';
+import loadData from '../lib/loadData';
+
+export type TableProps = {
+  data?: Array<{ [key: string]: number | string }>;
+  cols?: Array<{ [key: string]: string }>;
+  csv?: string;
+  url?: string;
+  fullWidth?: boolean;
+};
+
+export const Table = ({
+  data: ogData = [],
+  cols: ogCols = [],
+  csv = '',
+  url = '',
+  fullWidth = false,
+}: TableProps) => {
+  if (csv) {
+    const out = parseCsv(csv);
+    ogData = out.rows;
+    ogCols = out.fields;
+  }
+
+  const [data, setData] = React.useState(ogData);
+  const [cols, setCols] = React.useState(ogCols);
+  // const [error, setError] = React.useState(""); //  TODO: add error handling
+
+  const tableCols = useMemo(() => {
+    const columnHelper = createColumnHelper();
+    return cols.map((c) =>
+      columnHelper.accessor<any, string>(c.key, {
+        header: () => c.name,
+        cell: (info) => info.getValue(),
+      })
+    );
+  }, [data, cols]);
+
+  const [globalFilter, setGlobalFilter] = useState('');
+
+  const table = useReactTable({
+    data,
+    columns: tableCols,
+    getCoreRowModel: getCoreRowModel(),
+    state: {
+      globalFilter,
+    },
+    globalFilterFn: globalFilterFn,
+    onGlobalFilterChange: setGlobalFilter,
+    getFilteredRowModel: getFilteredRowModel(),
+    getPaginationRowModel: getPaginationRowModel(),
+    getSortedRowModel: getSortedRowModel(),
+  });
+
+  useEffect(() => {
+    if (url) {
+      loadData(url).then((data) => {
+        const { rows, fields } = parseCsv(data);
+        setData(rows);
+        setCols(fields);
+      });
+    }
+  }, [url]);
+
+  return (
+    <div className={`${fullWidth ? 'w-[90vw] ml-[calc(50%-45vw)]' : 'w-full'}`}>
+      <DebouncedInput
+        value={globalFilter ?? ''}
+        onChange={(value: any) => setGlobalFilter(String(value))}
+        className="p-2 text-sm shadow border border-block"
+        placeholder="Search all columns..."
+      />
+      <table className="w-full mt-10">
+        <thead className="text-left border-b border-b-slate-300">
+          {table.getHeaderGroups().map((hg) => (
+            <tr key={hg.id}>
+              {hg.headers.map((h) => (
+                <th key={h.id} className="pr-2 pb-2">
+                  <div
+                    {...{
+                      className: h.column.getCanSort()
+                        ? 'cursor-pointer select-none'
+                        : '',
+                      onClick: h.column.getToggleSortingHandler(),
+                    }}
+                  >
+                    {flexRender(h.column.columnDef.header, h.getContext())}
+                    {{
+                      asc: (
+                        <ArrowUpIcon className="inline-block ml-2 h-4 w-4" />
+                      ),
+                      desc: (
+                        <ArrowDownIcon className="inline-block ml-2 h-4 w-4" />
+                      ),
+                    }[h.column.getIsSorted() as string] ?? (
+                      <div className="inline-block ml-2 h-4 w-4" />
+                    )}
+                  </div>
+                </th>
+              ))}
+            </tr>
+          ))}
+        </thead>
+        <tbody>
+          {table.getRowModel().rows.map((r) => (
+            <tr key={r.id} className="border-b border-b-slate-200">
+              {r.getVisibleCells().map((c) => (
+                <td key={c.id} className="py-2">
+                  {flexRender(c.column.columnDef.cell, c.getContext())}
+                </td>
+              ))}
+            </tr>
+          ))}
+        </tbody>
+      </table>
+      <div className="flex gap-2 items-center justify-center mt-10">
+        <button
+          className={`w-6 h-6 ${
+            !table.getCanPreviousPage() ? 'opacity-25' : 'opacity-100'
+          }`}
+          onClick={() => table.setPageIndex(0)}
+          disabled={!table.getCanPreviousPage()}
+        >
+          <ChevronDoubleLeftIcon />
+        </button>
+        <button
+          className={`w-6 h-6 ${
+            !table.getCanPreviousPage() ? 'opacity-25' : 'opacity-100'
+          }`}
+          onClick={() => table.previousPage()}
+          disabled={!table.getCanPreviousPage()}
+        >
+          <ChevronLeftIcon />
+        </button>
+        <span className="flex items-center gap-1">
+          <div>Page</div>
+          <strong>
+            {table.getState().pagination.pageIndex + 1} of{' '}
+            {table.getPageCount()}
+          </strong>
+        </span>
+        <button
+          className={`w-6 h-6 ${
+            !table.getCanNextPage() ? 'opacity-25' : 'opacity-100'
+          }`}
+          onClick={() => table.nextPage()}
+          disabled={!table.getCanNextPage()}
+        >
+          <ChevronRightIcon />
+        </button>
+        <button
+          className={`w-6 h-6 ${
+            !table.getCanNextPage() ? 'opacity-25' : 'opacity-100'
+          }`}
+          onClick={() => table.setPageIndex(table.getPageCount() - 1)}
+          disabled={!table.getCanNextPage()}
+        >
+          <ChevronDoubleRightIcon />
+        </button>
+      </div>
+    </div>
+  );
+};
+
+const globalFilterFn: FilterFn<any> = (row, columnId, filterValue: string) => {
+  const search = filterValue.toLowerCase();
+
+  let value = row.getValue(columnId) as string;
+  if (typeof value === 'number') value = String(value);
+
+  return value?.toLowerCase().includes(search);
+};

examples/learn/portaljs/components/Vega.tsx

@@ -0,0 +1,6 @@
+//  Wrapper for the Vega component
+import { Vega as VegaOg } from "react-vega";
+
+export function Vega(props) {
+  return <VegaOg {...props} />;
+}

examples/learn/portaljs/components/VegaLite.tsx

@@ -0,0 +1,9 @@
+//  Wrapper for the Vega Lite component
+import { VegaLite as VegaLiteOg } from "react-vega";
+import applyFullWidthDirective from "../lib/applyFullWidthDirective";
+
+export function VegaLite(props) {
+  const Component = applyFullWidthDirective({ Component: VegaLiteOg });
+
+  return <Component {...props} />;
+}

examples/learn/portaljs/lib/applyFullWidthDirective.tsx

@@ -0,0 +1,21 @@
+export default function applyFullWidthDirective({
+  Component,
+  defaultWFull = true,
+}) {
+  return (props) => {
+    const newProps = { ...props };
+
+    let newClassName = newProps.className || "";
+    if (newProps.fullWidth === true) {
+      newClassName += " w-[90vw] ml-[calc(50%-45vw)] max-w-none";
+    } else if (defaultWFull) {
+      //  So that charts and tables will have the
+      //  same width as the text content, but images
+      //  can have its width set using the width prop
+      newClassName += " w-full";
+    }
+    newProps.className = newClassName;
+
+    return <Component {...newProps} />;
+  };
+}

examples/learn/portaljs/lib/loadData.tsx

@@ -0,0 +1,5 @@
+export default async function loadData(url: string) {
+  const response = await fetch(url)
+  const data = await response.text()
+  return data
+}

examples/learn/portaljs/lib/parseCsv.ts

@@ -0,0 +1,20 @@
+import papa from "papaparse";
+
+const parseCsv = (csv: string) => {
+  csv = csv.trim();
+  const rawdata = papa.parse(csv, { header: true });
+
+  let cols: any[] = [];
+  if(rawdata.meta.fields) {
+    cols = rawdata.meta.fields.map((r: string) => {
+      return { key: r, name: r };
+    });
+  }
+
+  return {
+    rows: rawdata.data as any,
+    fields: cols,
+  };
+};
+
+export default parseCsv;

examples/learn/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

examples/openspending/components/FlatUiTable.tsx

@@ -1,9 +1,4 @@
-import {
+import { QueryClient, QueryClientProvider, useQuery } from 'react-query';
-  QueryClient,
-  QueryClientProvider,
-  useQuery,
-  useQueryClient,
-} from 'react-query';
 import Papa from 'papaparse';
 import { Grid } from '@githubocto/flat-ui';
 
@@ -62,24 +57,41 @@ const Spinning = () => {
   );
 };
 
-export const FlatUiTable: React.FC<{ url: string }> = ({ url }) => {
+export interface FlatUiTableProps {
+  url?: string;
+  data?: { [key: string]: number | string }[];
+  rawCsv?: string;
+}
+export const FlatUiTable: React.FC<FlatUiTableProps> = ({
+  url,
+  data,
+  rawCsv,
+}) => {
   return (
     // Provide the client to your App
     <QueryClientProvider client={queryClient}>
-      <TableInner url={url} />
+      <TableInner url={url} data={data} rawCsv={rawCsv} />
     </QueryClientProvider>
   );
 };
 
-const TableInner: React.FC<{ url: string }> = ({ url }) => {
+const TableInner: React.FC<FlatUiTableProps> = ({ url, data, rawCsv }) => {
+  if (data) {
+    return (
+      <div className="w-full" style={{height: '500px'}}>
+        <Grid data={data} />
+      </div>
+    );
+  }
   const { data: csvString, isLoading: isDownloadingCSV } = useQuery(
     ['dataCsv', url],
-    () => getCsv(url)
+    () => getCsv(url),
+    { enabled: !!url }
   );
   const { data: parsedData, isLoading: isParsing } = useQuery(
     ['dataPreview', csvString],
-    () => parseCsv(csvString),
+    () => parseCsv(rawCsv ? rawCsv : csvString),
-    { enabled: !!csvString }
+    { enabled: rawCsv ? true : !!csvString }
   );
   if (isParsing || isDownloadingCSV)
     <div className="w-full">
@@ -87,7 +99,7 @@ const TableInner: React.FC<{ url: string }> = ({ url }) => {
     </div>;
   if (parsedData)
     return (
-      <div className="h-[500px] w-full">
+      <div className="w-full" style={{height: '500px'}}>
         <Grid data={parsedData.data} />
       </div>
     );

examples/openspending/package.json

@@ -11,9 +11,9 @@
   },
   "dependencies": {
     "@flowershow/core": "^0.4.13",
-    "@githubocto/flat-ui": "^0.14.1",
     "@heroicons/react": "^2.0.18",
     "@octokit/plugin-throttling": "^5.2.2",
+    "@portaljs/components": "0.1.4",
     "@types/flexsearch": "^0.7.3",
     "@types/node": "18.16.0",
     "@types/react": "18.0.38",
@@ -26,13 +26,11 @@
     "next": "13.3.0",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
-    "papaparse": "^5.4.1",
     "prettier": "^2.8.8",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-hook-form": "^7.43.9",
     "react-markdown": "^8.0.7",
-    "react-query": "^3.39.3",
     "react-timeago": "^7.1.0",
     "remark-gfm": "^3.0.1",
     "typescript": "5.0.4"

examples/openspending/pages/@org/[org]/[project].tsx

@@ -13,7 +13,7 @@ import Layout from '../../../components/_shared/Layout';
 import Link from 'next/link';
 import { Project } from '../../../lib/project.interface';
 import ExternalLinkIcon from '../../../components/icons/ExternalLinkIcon';
-import { FlatUiTable } from '@/components/FlatUiTable';
+import { FlatUiTable } from '../../../components/FlatUiTable';
 
 export default function ProjectPage({
   project,

package.json

@@ -1,11 +1,18 @@
 {
   "name": "portaljs",
+  "workspaces": ["./packages/*"],
   "version": "0.0.0",
   "license": "MIT",
-  "scripts": {},
+  "scripts": {
+    "changeset": "changeset",
+    "prerelease": "nx affected --targets=lint,test",
+    "release": "changeset publish"
+  },
   "private": true,
   "devDependencies": {
     "@babel/preset-react": "^7.14.5",
+    "@changesets/changelog-github": "^0.4.8",
+    "@changesets/cli": "^2.26.1",
     "@nrwl/cypress": "15.9.2",
     "@nrwl/eslint-plugin-nx": "^16.0.2",
     "@nrwl/jest": "15.9.2",

packages/ckan/.eslintrc.cjs

@@ -0,0 +1,16 @@
+module.exports = {
+  env: {
+    browser: true,
+    es2020: true
+  },
+  extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'plugin:react-hooks/recommended', 'plugin:storybook/recommended'],
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    ecmaVersion: 'latest',
+    sourceType: 'module'
+  },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': 'warn'
+  }
+};

packages/ckan/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

packages/ckan/README.md

@@ -0,0 +1,24 @@
+# PortalJS CKAN Library
+
+This is a library intended for the use with a CKAN Backend, it is composed of:
+
+- Types mapping the CKAN API Objects
+- A CKAN object that makes API calls
+- Components that are usually needed in a data portal(searching, pagination, listing etc)
+
+## Usage
+
+To install this package on your project:
+
+```bash
+npm i @portaljs/ckan
+```
+
+> Note: React 18 is required.
+
+You'll also have to import the styles CSS file in your project:
+
+```ts
+//  E.g.: Next.js => pages/_app.tsx
+import '@portaljs/ckan/styles.css'
+```

packages/ckan/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "@portaljs/ckan",
+  "version": "0.0.2",
+  "type": "module",
+  "description": "https://portaljs.org",
+  "keywords": [
+    "data portal",
+    "data catalog",
+    "table",
+    "charts",
+    "visualization"
+  ],
+  "scripts": {
+    "build": "tsc && vite build && npm run build-tailwind",
+    "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "build-tailwind": "NODE_ENV=production npx tailwindcss -o ./dist/styles.css --minify"
+  },
+  "peerDependencies": {
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0"
+  },
+  "files": [
+    "dist"
+  ],
+  "devDependencies": {
+    "@storybook/addon-essentials": "^7.0.7",
+    "@storybook/addon-interactions": "^7.0.7",
+    "@storybook/addon-links": "^7.0.7",
+    "@storybook/blocks": "^7.0.7",
+    "@storybook/react": "^7.0.7",
+    "@storybook/react-vite": "^7.0.7",
+    "@storybook/testing-library": "^0.0.14-next.2",
+    "@types/react": "^18.0.28",
+    "@types/react-dom": "^18.0.11",
+    "@typescript-eslint/eslint-plugin": "^5.57.1",
+    "@typescript-eslint/parser": "^5.57.1",
+    "@vitejs/plugin-react": "^4.0.0",
+    "autoprefixer": "^10.4.14",
+    "eslint": "^8.38.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.3.4",
+    "json": "^11.0.0",
+    "postcss": "^8.4.23",
+    "prop-types": "^15.8.1",
+    "tailwindcss": "^3.3.2",
+    "typescript": "^5.0.2",
+    "vite": "^4.3.2",
+    "vite-plugin-dts": "^2.3.0"
+  },
+  "main": "./dist/components.umd.js",
+  "module": "./dist/components.es.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/components.es.js",
+      "require": "./dist/components.umd.js"
+    },
+    "./styles.css": {
+      "import": "./dist/styles.css"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "formik": "^2.2.9",
+    "swr": "^2.1.5",
+    "timeago.js": "^4.0.2"
+  }
+}

packages/ckan/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

packages/ckan/src/components/DatasetCard.tsx

@@ -0,0 +1,98 @@
+import Link from "next/link";
+import { format } from "timeago.js";
+import { Dataset } from "../interfaces/dataset.interface";
+import ResourceCard from "./ResourceCard";
+
+export default function DatasetCard({
+  dataset,
+  showOrg = true,
+  urlPrefix = ""
+}: {
+  dataset: Dataset;
+  showOrg: boolean;
+  urlPrefix?: string;
+}) {
+  const resourceBgColors = {
+    PDF: "bg-cyan-300",
+    CSV: "bg-emerald-300",
+    JSON: "bg-yellow-300",
+    ODS: "bg-amber-400",
+    XLS: "bg-orange-300",
+    DOC: "bg-red-300",
+    SHP: "bg-purple-400",
+    HTML: "bg-pink-300",
+  };
+
+  const resourceBgColorsProxy = new Proxy(resourceBgColors, {
+    get: (obj, prop) => {
+      if (prop in obj) {
+        return obj[prop]
+      }
+      return "bg-amber-400"
+    }
+  })
+
+  function DatasetInformations() {
+    return (
+      <div className="flex align-center gap-2">
+        {(dataset.resources.length > 0 && dataset.resources[0].format && (
+          <>
+            {showOrg !== false && (
+              <span
+                className={`${
+                  resourceBgColorsProxy[
+                    dataset.resources[0].format as keyof typeof resourceBgColors
+                  ]
+                } px-4 py-1 rounded-full text-xs`}
+              >
+                {dataset.organization
+                  ? dataset.organization.title
+                  : "No organization"}
+              </span>
+            )}
+            <span
+              className={`${
+                resourceBgColorsProxy[
+                  dataset.resources[0].format as keyof typeof resourceBgColors
+                ]
+              } px-4 py-1 rounded-full text-xs`}
+            >
+              {dataset.metadata_created && format(dataset.metadata_created)}
+            </span>
+          </>
+        )) || (
+          <>
+            {showOrg !== false && (
+              <span className="bg-gray-200 px-4 py-1 rounded-full text-xs">
+                {dataset.organization
+                  ? dataset.organization.title
+                  : "No organization"}
+              </span>
+            )}
+            <span className="bg-gray-200 px-4 py-1 rounded-full text-xs">
+              {dataset.metadata_created && format(dataset.metadata_created)}
+            </span>
+          </>
+        )}
+      </div>
+    );
+  }
+  return (
+    <article className="grid grid-cols-1 md:grid-cols-7 gap-x-2">
+      <ResourceCard
+        resource={dataset?.resources.find((resource) => resource.format)}
+      />
+      <div className="col-span-6 place-content-start flex flex-col gap-1">
+        <Link href={`${urlPrefix}/@${dataset.organization?.name}/${dataset.name}`}>
+            <h1 className="m-auto md:m-0 font-semibold text-lg text-zinc-900">
+              {dataset.title || "No title"}
+            </h1>
+        </Link>
+        <p className="text-sm font-normal text-stone-500  line-clamp-2 h-[44px] overflow-y-hidden ">
+          {dataset.notes?.replace(/<\/?[^>]+(>|$)/g, "") || "No description"}
+        </p>
+        <DatasetInformations />
+      </div>
+    </article>
+  );
+}

packages/ckan/src/components/DatasetSearchFilters.tsx

@@ -0,0 +1,146 @@
+import { Field, Form, Formik, useFormikContext } from "formik";
+import { Dispatch, SetStateAction, useEffect, useState } from "react";
+import { PackageSearchOptions, Tag, Group, Organization, FilterObj } from "../interfaces";
+
+function AutoSubmit({
+  setOptions,
+  options,
+}: {
+  options: PackageSearchOptions;
+  setOptions: Dispatch<SetStateAction<PackageSearchOptions>>;
+}) {
+  const { values } = useFormikContext<{
+    tags: string[];
+    orgs: string[];
+    groups: string[];
+  }>();
+  useEffect(() => {
+    setOptions({
+      ...options,
+      groups: values.groups,
+      tags: values.tags,
+      orgs: values.orgs,
+    });
+  }, [values]);
+  return null;
+}
+
+export default function DatasetSearchFilters({
+  tags,
+  orgs,
+  groups,
+  setOptions,
+  options,
+  filtersName,
+}: {
+  tags: Array<Tag>;
+  orgs: Array<Organization>;
+  groups: Array<Group>;
+  options: PackageSearchOptions;
+  setOptions: Dispatch<SetStateAction<PackageSearchOptions>>;
+  filtersName?: FilterObj | undefined;
+}) {
+  const [seeMoreOrgs, setSeeMoreOrgs] = useState(false);
+  const [seeMoreTags, setSeeMoreTags] = useState(false);
+  const [seeMoreGroups, setSeeMoreGroups] = useState(false);
+  return (
+    <Formik
+      initialValues={{
+        tags: [],
+        orgs: [],
+        groups: [],
+      }}
+      onSubmit={async (values) => {
+        alert(JSON.stringify(values, null, 2));
+      }}
+    >
+      <Form>
+        <section className="bg-white rounded-lg xl:p-8 p-4 mb-4 max-h-[400px] overflow-y-auto">
+          <h1 className="font-bold pb-4">Refine by {`${filtersName?.org || "Organization"}`}</h1>
+          {orgs
+            .filter((org) => org.title || org.id)
+            .slice(0, seeMoreOrgs ? orgs.length : 5)
+            .map((org) => (
+              <div key={org.id}>
+                <Field
+                  type="checkbox"
+                  id={org.id}
+                  name="orgs"
+                  value={org.name}
+                ></Field>
+                <label className="ml-1.5" htmlFor={org.id}>
+                  {org.title || org.display_name}
+                </label>
+              </div>
+            ))}
+          {orgs.length > 5 && (
+            <button
+              className="bg-gray-300 px-2 rounded text-gray-600 mt-2"
+              type="button"
+              onClick={() => setSeeMoreOrgs(!seeMoreOrgs)}
+            >
+              Show {seeMoreOrgs ? "less" : "more"}
+            </button>
+          )}
+        </section>
+        <section className="bg-white rounded-lg xl:p-8 p-4 mb-4 max-h-[400px] overflow-y-auto">
+          <h1 className="font-bold pb-4">Refine by {`${filtersName?.group || "Theme"}`}</h1>
+          {groups.slice(0, seeMoreGroups ? groups.length : 5).map((group) => (
+            <div key={group.id}>
+              <Field
+                type="checkbox"
+                id={group.id}
+                name="groups"
+                value={group.name}
+              ></Field>
+              <label className="ml-1.5" htmlFor={group.id}>
+                {group.display_name}
+              </label>
+            </div>
+          ))}
+          {groups.length > 5 && (
+            <button
+              onClick={() => setSeeMoreGroups(!seeMoreGroups)}
+              type="button"
+              className="bg-gray-300 px-2 rounded text-gray-600 mt-2"
+            >
+              See {seeMoreGroups ? "less" : "more..."}
+            </button>
+          )}
+        </section>
+        <section className="bg-white rounded-lg xl:p-8 p-4 mb-4 max-h-[400px] overflow-y-auto">
+          <h1 className="font-bold pb-4">Refine by Keyword</h1>
+          <div className="flex gap-2 flex-wrap">
+            {tags.slice(0, seeMoreTags ? tags.length : 5).map((tag) => (
+              <div key={tag.id}>
+                <Field
+                  type="checkbox"
+                  className="hidden tag-checkbox"
+                  id={tag.id}
+                  name="tags"
+                  value={tag.name}
+                ></Field>
+                <label
+                  className="bg-gray-200 px-4 py-1 rounded-full text-xs block"
+                  htmlFor={tag.id}
+                >
+                  {tag.display_name}
+                </label>
+              </div>
+            ))}
+          </div>
+          {tags.length > 5 && (
+            <button
+              onClick={() => setSeeMoreTags(!seeMoreTags)}
+              type="button"
+              className="bg-gray-300 px-2 rounded text-gray-600 mt-2"
+            >
+              See {seeMoreTags ? "less" : "more..."}
+            </button>
+          )}
+        </section>
+        <AutoSubmit options={options} setOptions={setOptions} />
+      </Form>
+    </Formik>
+  );
+}

packages/ckan/src/components/DatasetSearchForm.tsx

@@ -0,0 +1,91 @@
+import { Field, Form, Formik } from 'formik';
+import { Dispatch, SetStateAction } from 'react';
+import {
+  PackageSearchOptions,
+  Group,
+  Organization,
+  FilterObj,
+} from '../interfaces';
+
+export default function DatasetSearchForm({
+  orgs,
+  groups,
+  setOptions,
+  options,
+  filtersName,
+}: {
+  orgs: Array<Organization>;
+  groups: Array<Group>;
+  options: PackageSearchOptions;
+  setOptions: Dispatch<SetStateAction<PackageSearchOptions>>;
+  filtersName?: FilterObj | undefined;
+}) {
+  return (
+    <Formik
+      initialValues={{
+        org: '',
+        group: '',
+        query: '',
+      }}
+      enableReinitialize={true}
+      onSubmit={async (values) => {
+        const org = orgs.find(
+          (org) => (org.title || org.display_name) === values.org
+        );
+        const group = groups.find(
+          (group) => group.display_name === values.group
+        );
+        setOptions({
+          ...options,
+          groups: group ? [group.name] : [],
+          orgs: org ? [org.name] : [],
+          query: values.query,
+        });
+      }}
+    >
+      <div className="mx-auto" style={{ width: 'min(1100px, 95vw)' }}>
+        <Form className="min-h-[80px] flex flex-col lg:flex-row bg-white inline-block px-5 py-3 rounded-xl">
+          <Field
+            type="text"
+            placeholder="Search Datasets"
+            className="mx-4 grow py-4 border-0 placeholder:text-neutral-400"
+            name="query"
+          />
+          <Field
+            list="groups"
+            name="group"
+            placeholder={`${filtersName?.group || 'Theme'}`}
+            className="lg:border-l p-4 mx-2 placeholder:text-neutral-400"
+          ></Field>
+
+          <datalist aria-label="Formats" id="groups">
+            <option value="">{`${filtersName?.group || 'Theme'}`}</option>
+            {groups.map((group, index) => (
+              <option key={index}>{group.display_name}</option>
+            ))}
+          </datalist>
+          <Field
+            list="orgs"
+            name="org"
+            placeholder="Organization"
+            className="lg:border-l p-4 mx-2 placeholder:text-neutral-400"
+            autoComplete="off"
+          ></Field>
+
+          <datalist aria-label="Formats" id="orgs">
+            <option value="">Organization</option>
+            {orgs.map((org, index) => (
+              <option key={index}>{org.title || org.display_name}</option>
+            ))}
+          </datalist>
+          <button
+            className="font-bold text-black px-12 py-4 rounded-lg bg-accent hover:bg-cyan-500 duration-150"
+            type="submit"
+          >
+            SEARCH
+          </button>
+        </Form>
+      </div>
+    </Formik>
+  );
+}

packages/ckan/src/components/ListOfDatasets.tsx

@@ -0,0 +1,109 @@
+import { Dispatch, SetStateAction, useState } from 'react';
+import useSWR from 'swr';
+import { SWRConfig } from 'swr';
+import { PackageSearchOptions } from '../interfaces';
+import DatasetCard from './DatasetCard';
+import Pagination from './Pagination';
+import CKAN from '../lib/ckanapi';
+
+export default function ListOfDatasets({
+  ckan,
+  options,
+  setOptions,
+  urlPrefix = '',
+}: {
+  ckan: CKAN;
+  options: PackageSearchOptions;
+  setOptions: Dispatch<SetStateAction<PackageSearchOptions>>;
+  urlPrefix?: string;
+}) {
+  return (
+    <SWRConfig>
+      <ListOfDatasetsInner
+        ckan={ckan}
+        options={options}
+        setOptions={setOptions}
+        urlPrefix={urlPrefix}
+      />
+    </SWRConfig>
+  );
+}
+
+function ListOfDatasetsInner({
+  ckan,
+  options,
+  setOptions,
+  urlPrefix = '',
+}: {
+  ckan: CKAN;
+  options: PackageSearchOptions;
+  setOptions: Dispatch<SetStateAction<PackageSearchOptions>>;
+  urlPrefix?: string;
+}) {
+  return (
+    <div className="grid grid-cols-1 gap-4 homepage-padding">
+      <ListItems
+        ckan={ckan}
+        setOptions={setOptions}
+        options={options}
+        urlPrefix={urlPrefix}
+      />
+      <div style={{ display: 'none' }}>
+        <ListItems
+          ckan={ckan}
+          setOptions={setOptions}
+          options={{ ...options, offset: options.offset + 5 }}
+        />
+      </div>
+    </div>
+  );
+}
+
+function ListItems({
+  ckan,
+  options,
+  setOptions,
+  urlPrefix = '',
+}: {
+  ckan: CKAN;
+  options: PackageSearchOptions;
+  setOptions: Dispatch<SetStateAction<PackageSearchOptions>>;
+  urlPrefix?: string;
+}) {
+  const { data, isLoading } = useSWR(['package_search', options], async () =>
+    ckan.packageSearch({ ...options })
+  );
+  //Define which page buttons are going to be displayed in the pagination list
+  const [subsetOfPages, setSubsetOfPages] = useState(0);
+
+  return (
+    <>
+      {isLoading ? (
+        <div className="loader mb-4 h-12 w-12 rounded-full border-4 border-t-4 border-gray-200 ease-linear"></div>
+      ) : (
+        <>
+          <h2 className="text-4xl capitalize font-bold text-zinc-900">
+            {data?.count} Datasets
+          </h2>
+          {data?.datasets?.map((dataset) => (
+            <DatasetCard
+              key={dataset.id}
+              dataset={dataset}
+              showOrg={true}
+              urlPrefix={urlPrefix}
+            />
+          ))}
+          {data?.count && (
+            <Pagination
+              options={options}
+              subsetOfPages={subsetOfPages}
+              setSubsetOfPages={setSubsetOfPages}
+              setOptions={setOptions}
+              count={data.count}
+            />
+          )}
+        </>
+      )}
+    </>
+  );
+}

packages/ckan/src/components/Pagination.tsx

@@ -0,0 +1,80 @@
+import { Dispatch, SetStateAction } from "react";
+import { PackageSearchOptions } from "../interfaces";
+
+export default function Pagination({
+  options,
+  setOptions,
+  subsetOfPages,
+  setSubsetOfPages,
+  count,
+}: {
+  options: PackageSearchOptions;
+  setOptions: Dispatch<SetStateAction<PackageSearchOptions>>;
+  subsetOfPages: number;
+  setSubsetOfPages: Dispatch<SetStateAction<number>>;
+  count: number;
+}) {
+  return (
+    <div className="flex gap-2 align-center">
+      {subsetOfPages !== 0 && (
+        <button
+          className="font-semibold flex items-center gap-2"
+          onClick={() => setSubsetOfPages(subsetOfPages - 5)}
+        >
+          <svg
+            xmlns="http://www.w3.org/2000/svg"
+            fill="none"
+            viewBox="0 0 24 24"
+            strokeWidth={1.5}
+            stroke="currentColor"
+            className="w-6 h-6"
+          >
+            <path
+              strokeLinecap="round"
+              strokeLinejoin="round"
+              d="M10.5 19.5L3 12m0 0l7.5-7.5M3 12h18"
+            />
+          </svg>
+          Prev
+        </button>
+      )}
+      {Array.from(Array(Math.ceil(count / 5)).keys()).map((x) => (
+        <button
+          key={x}
+          className={`${
+            x == options.offset / 5 ? "bg-orange-500 text-white" : ""
+          } px-2 rounded font-semibold text-zinc-900`}
+          onClick={() => setOptions({ ...options, offset: x * 5 })}
+          style={{
+            display:
+              x >= subsetOfPages && x < subsetOfPages + 5 ? "block" : "none",
+          }}
+        >
+          {x + 1}
+        </button>
+      ))}
+      {subsetOfPages !== Math.ceil(count / 5) && count > 25 && (
+        <button
+          className="font-semibold flex items-center gap-2"
+          onClick={() => setSubsetOfPages(subsetOfPages + 5)}
+        >
+          Next
+          <svg
+            xmlns="http://www.w3.org/2000/svg"
+            fill="none"
+            viewBox="0 0 24 24"
+            strokeWidth={1.5}
+            stroke="currentColor"
+            className="w-6 h-6"
+          >
+            <path
+              strokeLinecap="round"
+              strokeLinejoin="round"
+              d="M13.5 4.5L21 12m0 0l-7.5 7.5M21 12H3"
+            />
+          </svg>
+        </button>
+      )}
+    </div>
+  );
+}