[examples/fivethirtyeight][lg] - first commmit of 538 Example

**Issue:** https://github.com/datopian/portaljs/issues/839

## Changes

- Update info about required Node version
- Remove mention to automatic reload from docs

examples/alan-turing-portal/README.md

@@ -1,6 +1,16 @@
+This demo data portal is designed for https://hatespeechdata.com. It catalogs datasets annotated for hate speech, online abuse, and offensive language which are useful for training a natural language processing system to detect this online abuse.
+
+The site is built on top of [PortalJS](https://portaljs.org/). It catalogs datasets and lists of offensive keywords. It also includes static pages. All of these are stored as markdown files inside the `content` folder.
+
+- .md files inside `content/datasets/` will appear on the dataset list section of the homepage and be searchable as well as having a individual page in `datasets/<file name>`
+- .md files inside `content/keywords/` will appear on the list of offensive keywords section of the homepage as well as having a individual page in `keywords/<file name>`
+- .md files inside `content/` will be converted to static pages in the url `/<file name>` eg: `content/about.md` becomes `/about`
+
+This is also a Next.JS project so you can use the following steps to run the website locally.
+
 ## Getting started
 
-To get started with this template, first install the npm dependencies:
+To get started first install the npm dependencies:
 
 ```bash
 npm install
@@ -13,7 +23,3 @@ npm run dev
 ```
 
 Finally, open [http://localhost:3000](http://localhost:3000) in your browser to view the website.
-
-## License
-
-This site template is a commercial product and is licensed under the [Tailwind UI license](https://tailwindui.com/license).

examples/alan-turing-portal/components/Footer.jsx

@@ -21,7 +21,7 @@ export function Footer() {
           <Container.Inner>
             <div className="flex flex-col items-center justify-between gap-6 sm:flex-row">
               <p className="text-sm font-medium text-zinc-800 dark:text-zinc-200">
-                hatespeechdata maintained by <a href='https://github.com/leondz'>leondz</a>
+                Built with <a href='https://portaljs.org'>PortalJS 🌀</a>
               </p>
               <p className="text-sm text-zinc-400 dark:text-zinc-500">
                 &copy; {new Date().getFullYear()} Leon Derczynski. All rights

examples/alan-turing-portal/components/Header.jsx

@@ -1,8 +1,41 @@
-import { useRef } from 'react'
+import { Fragment, useEffect, useRef } from 'react'
+import Image from 'next/image'
+import Link from 'next/link'
 import { useRouter } from 'next/router'
+import { Popover, Transition } from '@headlessui/react'
+import clsx from 'clsx'
 
 import { Container } from '../components/Container'
 
+function CloseIcon(props) {
+  return (
+    <svg viewBox="0 0 24 24" aria-hidden="true" {...props}>
+      <path
+        d="m17.25 6.75-10.5 10.5M6.75 6.75l10.5 10.5"
+        fill="none"
+        stroke="currentColor"
+        strokeWidth="1.5"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  )
+}
+
+function ChevronDownIcon(props) {
+  return (
+    <svg viewBox="0 0 8 6" aria-hidden="true" {...props}>
+      <path
+        d="M1.75 1.75 4 4.25l2.25-2.5"
+        fill="none"
+        strokeWidth="1.5"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  )
+}
+
 function SunIcon(props) {
   return (
     <svg
@@ -35,6 +68,125 @@ function MoonIcon(props) {
   )
 }
 
+function GithubIcon(props) {
+  return (
+    <svg
+      viewBox="0 0 24 24"
+      aria-hidden="true"
+      className="h-6 w-6 fill-slate-900 dark:fill-zinc-200"
+    >
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M12 2C6.477 2 2 6.463 2 11.97c0 4.404 2.865 8.14 6.839 9.458.5.092.682-.216.682-.48 0-.236-.008-.864-.013-1.695-2.782.602-3.369-1.337-3.369-1.337-.454-1.151-1.11-1.458-1.11-1.458-.908-.618.069-.606.069-.606 1.003.07 1.531 1.027 1.531 1.027.892 1.524 2.341 1.084 2.91.828.092-.643.35-1.083.636-1.332-2.22-.251-4.555-1.107-4.555-4.927 0-1.088.39-1.979 1.029-2.675-.103-.252-.446-1.266.098-2.638 0 0 .84-.268 2.75 1.022A9.607 9.607 0 0 1 12 6.82c.85.004 1.705.114 2.504.336 1.909-1.29 2.747-1.022 2.747-1.022.546 1.372.202 2.386.1 2.638.64.696 1.028 1.587 1.028 2.675 0 3.83-2.339 4.673-4.566 4.92.359.307.678.915.678 1.846 0 1.332-.012 2.407-.012 2.734 0 .267.18.577.688.48 3.97-1.32 6.833-5.054 6.833-9.458C22 6.463 17.522 2 12 2Z"
+      ></path>
+    </svg>
+  )
+}
+
+function MobileNavItem({ href, children }) {
+  return (
+    <li>
+      <Popover.Button
+        as={Link}
+        href={href}
+        className="flex items-center gap-x-2 py-2"
+      >
+        {children}
+      </Popover.Button>
+    </li>
+  )
+}
+
+function MobileNavigation(props) {
+  return (
+    <Popover {...props}>
+      <Popover.Button className="group flex items-center rounded-full bg-white/90 px-4 py-2 text-sm font-medium text-zinc-800 shadow-lg shadow-zinc-800/5 ring-1 ring-zinc-900/5 backdrop-blur dark:bg-zinc-800/90 dark:text-zinc-200 dark:ring-white/10 dark:hover:ring-white/20">
+        Menu
+        <ChevronDownIcon className="ml-3 h-auto w-2 stroke-zinc-500 group-hover:stroke-zinc-700 dark:group-hover:stroke-zinc-400" />
+      </Popover.Button>
+      <Transition.Root>
+        <Transition.Child
+          as={Fragment}
+          enter="duration-150 ease-out"
+          enterFrom="opacity-0"
+          enterTo="opacity-100"
+          leave="duration-150 ease-in"
+          leaveFrom="opacity-100"
+          leaveTo="opacity-0"
+        >
+          <Popover.Overlay className="fixed inset-0 z-50 bg-zinc-800/40 backdrop-blur-sm dark:bg-black/80" />
+        </Transition.Child>
+        <Transition.Child
+          as={Fragment}
+          enter="duration-150 ease-out"
+          enterFrom="opacity-0 scale-95"
+          enterTo="opacity-100 scale-100"
+          leave="duration-150 ease-in"
+          leaveFrom="opacity-100 scale-100"
+          leaveTo="opacity-0 scale-95"
+        >
+          <Popover.Panel
+            focus
+            className="fixed inset-x-4 top-8 z-50 origin-top rounded-3xl bg-white p-8 ring-1 ring-zinc-900/5 dark:bg-zinc-900 dark:ring-zinc-800"
+          >
+            <div className="flex flex-row-reverse items-center justify-between">
+              <Popover.Button aria-label="Close menu" className="-m-1 p-1">
+                <CloseIcon className="h-6 w-6 text-zinc-500 dark:text-zinc-400" />
+              </Popover.Button>
+              <h2 className="text-sm font-medium text-zinc-600 dark:text-zinc-400">
+                Navigation
+              </h2>
+            </div>
+            <nav className="mt-6">
+              <ul className="-my-2 divide-y divide-zinc-100 text-base text-zinc-800 dark:divide-zinc-100/5 dark:text-zinc-300">
+                <MobileNavItem href="https://github.com/leondz/hatespeechdata">
+                  View on Github <GithubIcon />
+                </MobileNavItem>
+              </ul>
+            </nav>
+          </Popover.Panel>
+        </Transition.Child>
+      </Transition.Root>
+    </Popover>
+  )
+}
+
+function NavItem({ href, children }) {
+  let isActive = useRouter().pathname === href
+
+  return (
+    <li>
+      <Link
+        href={href}
+        className={clsx(
+          'relative flex items-center gap-x-2 px-3 py-2 transition',
+          isActive
+            ? 'text-teal-500 dark:text-teal-400'
+            : 'hover:text-teal-500 dark:hover:text-teal-400'
+        )}
+      >
+        {children}
+        {isActive && (
+          <span className="absolute inset-x-1 -bottom-px h-px bg-gradient-to-r from-teal-500/0 via-teal-500/40 to-teal-500/0 dark:from-teal-400/0 dark:via-teal-400/40 dark:to-teal-400/0" />
+        )}
+      </Link>
+    </li>
+  )
+}
+
+function DesktopNavigation(props) {
+  return (
+    <nav {...props}>
+      <ul className="flex rounded-full bg-white/90 px-3 text-sm font-medium text-zinc-800 shadow-lg shadow-zinc-800/5 ring-1 ring-zinc-900/5 backdrop-blur dark:bg-zinc-800/90 dark:text-zinc-200 dark:ring-white/10">
+        <NavItem href="https://github.com/leondz/hatespeechdata">
+          View on Github <GithubIcon />
+        </NavItem>
+      </ul>
+    </nav>
+  )
+}
+
 function ModeToggle() {
   function disableTransitionsTemporarily() {
     document.documentElement.classList.add('[&_*]:!transition-none')
@@ -70,11 +222,13 @@ function ModeToggle() {
   )
 }
 
+function clamp(number, a, b) {
+  let min = Math.min(a, b)
+  let max = Math.max(a, b)
+  return Math.min(Math.max(number, min), max)
+}
+
 export function Header() {
-  let isHomePage = useRouter().pathname === '/'
-
-  let headerRef = useRef()
-
   return (
     <>
       <header
@@ -85,7 +239,6 @@ export function Header() {
         }}
       >
         <div
-          ref={headerRef}
           className="top-0 z-10 h-16 pt-6"
           style={{ position: 'var(--header-position)' }}
         >
@@ -94,6 +247,10 @@ export function Header() {
             style={{ position: 'var(--header-inner-position)' }}
           >
             <div className="relative flex gap-4">
+              <div className="flex flex-1">
+                <MobileNavigation className="pointer-events-auto md:hidden" />
+                <DesktopNavigation className="pointer-events-auto hidden md:block" />
+              </div>
               <div className="flex justify-end md:flex-1">
                 <div className="pointer-events-auto">
                   <ModeToggle />
@@ -103,7 +260,6 @@ export function Header() {
           </Container>
         </div>
       </header>
-      {isHomePage && <div style={{ height: 'var(--content-offset)' }} />}
     </>
   )
 }

examples/alan-turing-portal/content/about.md

@@ -0,0 +1,5 @@
+---
+title: About
+---
+
+This is an about page, left here as an example

examples/alan-turing-portal/content/contributing.md

@@ -1,22 +0,0 @@
----
-title: Contributing
----
-
-We accept entries to our catalogue based on pull requests to the content folder. The dataset must be avaliable for download to be included in the list. If you want to add an entry, follow these steps!
-
-Please send just one dataset addition/edit at a time - edit it in, then save. This will make everyone’s life easier (including yours!)
-
-- Go to the repo url file and click the "Add file" dropdown and then click on "Create new file".
-![](https://i.imgur.com/2PR0ZgL.png)
-   
-- In the following page type `content/datasets/<name-of-the-file>.md`. if you want to add an entry to the datasets catalog or `content/keywords/<name-of-the-file>.md` if you want to add an entry to the lists of abusive keywords.
-![](https://i.imgur.com/rr3uSYu.png)
-   
-- Copy the contents of `templates/dataset.md` or `templates/keywords.md` respectively to the camp below, filling out the fields with the correct data format
-![](https://i.imgur.com/x6JIjhz.png)
-   
-- Click on "Commit changes", on the popup make sure you give some brief detail on the proposed change.  and then click on Propose changes
-![](https://i.imgur.com/BxuxKEJ.png)
-   
-- Submit the pull request on the next page when prompted.
-

examples/alan-turing-portal/content/datasets/abusive-eval-v1-0.md

@@ -0,0 +1,14 @@
+---
+title: AbuseEval v1.0
+link-to-publication: http://www.lrec-conf.org/proceedings/lrec2020/pdf/2020.lrec-1.760.pdf
+link-to-data: https://github.com/tommasoc80/AbuseEval
+task-description: Explicitness annotation of offensive and abusive content
+details-of-task: "Enriched versions of the OffensEval/OLID dataset with the distinction of explicit/implicit offensive messages and the new dimension for abusive messages. Labels for offensive language: EXPLICIT, IMPLICT, NOT; Labels for abusive language: EXPLICIT, IMPLICT, NOTABU"
+size-of-dataset: 14100
+percentage-abusive: 20.75
+language: English
+level-of-annotation: ["Tweets"]
+platform: ["Twitter"]
+medium: ["Text"]
+reference: "Caselli, T., Basile, V., Jelena, M., Inga, K., and Michael, G. 2020. \"I feel offended, don’t be abusive! implicit/explicit messages in offensive and abusive language\". The 12th Language Resources and Evaluation Conference (pp. 6193-6202). European Language Resources Association."
+---

examples/alan-turing-portal/content/datasets/coral-a-context-aware-croatian-abusive-language-dataset.md

@@ -0,0 +1,14 @@
+---
+title: "CoRAL: a Context-aware Croatian Abusive Language Dataset"
+link-to-publication: https://aclanthology.org/2022.findings-aacl.21/
+link-to-data: https://github.com/shekharRavi/CoRAL-dataset-Findings-of-the-ACL-AACL-IJCNLP-2022
+task-description: Multi-class based on context dependency categories (CDC)
+details-of-task: Detectioning CDC from abusive comments
+size-of-dataset: 2240
+percentage-abusive: 100
+language: "Croatian"
+level-of-annotation: ["Posts"]
+platform: ["Posts"]
+medium: ["Newspaper Comments"]
+reference: "Ravi Shekhar, Mladen Karan and Matthew Purver (2022). CoRAL: a Context-aware Croatian Abusive Language Dataset. Findings of the ACL: AACL-IJCNLP."
+---

examples/alan-turing-portal/content/datasets/large-scale-hate-speech-detection-with-cross-domain-tranfer.md

@@ -0,0 +1,14 @@
+---
+title: Large-Scale Hate Speech Detection with Cross-Domain Transfer
+link-to-publication: https://aclanthology.org/2022.lrec-1.238/
+link-to-data: https://github.com/avaapm/hatespeech
+task-description: Three-class (Hate speech, Offensive language, None)
+details-of-task: Hate speech detection on social media (Twitter) including 5 target groups (gender, race, religion, politics, sports)
+size-of-dataset: "100k English (27593 hate, 30747 offensive, 41660 none)"
+percentage-abusive: 58.3
+language: English
+level-of-annotation: ["Posts"]
+platform: ["Twitter"]
+medium: ["Text", "Image"]
+reference: "Cagri Toraman, Furkan Şahinuç, Eyup Yilmaz. 2022. Large-Scale Hate Speech Detection with Cross-Domain Transfer. In Proceedings of the Thirteenth Language Resources and Evaluation Conference, pages 2215–2225, Marseille, France. European Language Resources Association."
+---

examples/alan-turing-portal/content/datasets/measuring-hate-speech.md

@@ -0,0 +1,14 @@
+---
+title: Measuring Hate Speech
+link-to-publication: https://arxiv.org/abs/2009.10277
+link-to-data: https://huggingface.co/datasets/ucberkeley-dlab/measuring-hate-speech
+task-description: 10 ordinal labels (sentiment, (dis)respect, insult, humiliation, inferior status, violence, dehumanization, genocide, attack/defense, hate speech), which are debiased and aggregated into a continuous hate speech severity score (hate_speech_score) that includes a region for counterspeech & supportive speeech. Includes 8 target identity groups (race/ethnicity, religion, national origin/citizenship, gender, sexual orientation, age, disability, political ideology) and 42 identity subgroups.
+details-of-task: Hate speech measurement on social media in English
+size-of-dataset: "39,565 comments annotated by 7,912 annotators on 10 ordinal labels, for 1,355,560 total labels."
+percentage-abusive: 25
+language: English
+level-of-annotation: ["Social media comment"]
+platform: ["Twitter", "Reddit", "Youtube"]
+medium: ["Text"]
+reference: "Kennedy, C. J., Bacon, G., Sahn, A., & von Vacano, C. (2020). Constructing interval variables via faceted Rasch measurement and multitask deep learning: a hate speech application. arXiv preprint arXiv:2009.10277."
+---

examples/alan-turing-portal/content/datasets/offensive-language-and-hate-speech-detection-for-danish.md

@@ -0,0 +1,14 @@
+---
+title: Offensive Language and Hate Speech Detection for Danish
+link-to-publication: http://www.derczynski.com/papers/danish_hsd.pdf
+link-to-data: https://figshare.com/articles/Danish_Hate_Speech_Abusive_Language_data/12220805
+task-description: "Branching structure of tasks: Binary (Offensive, Not), Within Offensive (Target, Not), Within Target (Individual, Group, Other)"
+details-of-task: Group-directed + Person-directed
+size-of-dataset: 3600
+percentage-abusive: 0.12
+language: Danish
+level-of-annotation: ["Posts"]
+platform: ["Twitter", "Reddit", "Newspaper comments"]
+medium: ["Text"]
+reference: "Sigurbergsson, G. and Derczynski, L., 2019. Offensive Language and Hate Speech Detection for Danish. ArXiv."
+---

examples/alan-turing-portal/content/index.md

@@ -1,13 +0,0 @@
----
-title: Hate Speech Dataset Catalogue
----
-
-This page catalogues datasets annotated for hate speech, online abuse, and offensive language. They may be useful for e.g. training a natural language processing system to detect this language.
-
-The list is maintained by Leon Derczynski, Bertie Vidgen, Hannah Rose Kirk, Pica Johansson, Yi-Ling Chung, Mads Guldborg Kjeldgaard Kongsbak, Laila Sprejer, and Philine Zeinert.
-
-We provide a list of datasets and keywords. If you would like to contribute to our catalogue or add your dataset, please see the instructions for contributing.
-
-If you use these resources, please cite (and read!) our paper: Directions in Abusive Language Training Data: Garbage In, Garbage Out. And if you would like to find other resources for researching online hate, visit The Alan Turing Institute’s Online Hate Research Hub or read The Alan Turing Institute’s Reading List on Online Hate and Abuse Research.
-
-If you’re looking for a good paper on online hate training datasets (beyond our paper, of course!) then have a look at ‘Resources and benchmark corpora for hate speech detection: a systematic review’ by Poletto et al. in Language Resources and Evaluation.

examples/alan-turing-portal/content/index.mdx

@@ -0,0 +1,52 @@
+---
+title: Hate Speech Dataset Catalogue
+---
+
+This page catalogues datasets annotated for hate speech, online abuse, and offensive language. They may be useful for e.g. training a natural language processing system to detect this language.
+
+The list is maintained by [Leon Derczynski](https://www.derczynski.com/), [Bertie Vidgen](https://www.turing.ac.uk/people/researchers/bertie-vidgen), [Hannah Rose Kirk](https://www.hannahrosekirk.com/), Pica Johansson, [Yi-Ling Chung](https://yilingchung.github.io/), Mads Guldborg Kjeldgaard Kongsbak, [Laila Sprejer](https://www.turing.ac.uk/people/researchers/laila-sprejer), and Philine Zeinert.
+
+We provide a list of [datasets](#Datasets-header) and [keywords](#Keywords-header). If you would like to contribute to our catalogue or add your dataset, please see the [instructions for contributing](#Contributing-header).
+
+If you use these resources, please cite (and read!) our paper: [Directions in Abusive Language Training Data: Garbage In, Garbage Out](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0243300). And if you would like to find other resources for researching online hate, visit The Alan Turing Institute's [Online Hate Research Hub](https://www.turing.ac.uk/research/research-programmes/public-policy/online-hate-research-hub) or read The Alan Turing Institute's [Reading List on Online Hate and Abuse Research](https://docs.google.com/document/d/1WVkVGp29Jt6d-4fBnZ5OWVYuFn_03rzz-KBqPsu6gTM/edit?usp=sharing).
+
+If you're looking for a good paper on online hate training datasets (beyond our paper, of course!) then have a look at ['Resources and benchmark corpora for hate speech detection: a systematic review'](https://link.springer.com/article/10.1007/s10579-020-09502-8) by Poletto et al. in *Language Resources and Evaluation*.
+
+Accompanying [data statements](https://www.mitpressjournals.org/doi/abs/10.1162/tacl_a_00041) preferred for all corpora.
+
+<a href="#Datasets-header" className="w-fit mx-auto no-underline rounded-md py-3 px-6 outline-offset-2 transition !active:transition-none bg-zinc-800 !font-semibold !text-zinc-100 hover:bg-zinc-700 active:bg-zinc-800 active:text-zinc-100/70 dark:bg-zinc-700 dark:hover:bg-zinc-600 !dark:active:bg-zinc-700 dark:active:text-zinc-100/70">See datasets</a>
+
+<h2 id="Contributing-header">How to contribute</h2>
+
+We accept entries to our catalogue based on pull requests to the content folder. The dataset must be avaliable for download to be included in the list. If you want to add an entry, follow these steps!
+
+Please send just one dataset addition/edit at a time - edit it in, then save. This will make everyone’s life easier (including yours!)
+
+### Create file
+
+Go to the repo url file and click the "Add file" dropdown and then click on "Create new file".
+
+![](https://i.imgur.com/2PR0ZgL.png)
+
+### Choose location
+
+In the following page type `content/datasets/<name-of-the-file>.md`. if you want to add an entry to the datasets catalog or `content/keywords/<name-of-the-file>.md` if you want to add an entry to the lists of abusive keywords, if you want to just add an static page you can leave in the root of `content` it will automatically get assigned an url eg: `/content/about.md` becomes the `/about` page
+
+![](https://i.imgur.com/rr3uSYu.png)
+
+### Fill in content
+
+Copy the contents of `templates/dataset.md` or `templates/keywords.md` respectively to the camp below, filling out the fields with the correct data format
+
+![](https://i.imgur.com/x6JIjhz.png)
+
+### Commit changes
+
+Click on "Commit changes", on the popup make sure you give some brief detail on the proposed change.  and then click on Propose changes
+
+<img src='https://i.imgur.com/BxuxKEJ.png' style={{ maxWidth: '50%', margin: '0 auto' }}/>
+
+### Submit PR
+
+Submit the pull request on the next page when prompted.
+

examples/alan-turing-portal/lib/markdown.js

@@ -23,12 +23,7 @@ import { serialize } from "next-mdx-remote/serialize";
  * @returns: { mdxSource: mdxSource, frontMatter: ...}
  */
 const parse = async function (source, format) {
-  const { content, data, excerpt } = matter(source, {
+  const { content, data } = matter(source);
-    excerpt: (file, options) => {
-      // Generate an excerpt for the file
-      file.excerpt = file.content.split("\n\n")[0];
-    },
-  });
 
   const mdxSource = await serialize(
     { value: content, path: format },
@@ -56,7 +51,7 @@ const parse = async function (source, format) {
           [
             rehypeAutolinkHeadings,
             {
-              properties: { className: 'heading-link' },
+              properties: { className: "heading-link" },
               test(element) {
                 return (
                   ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
@@ -91,14 +86,12 @@ const parse = async function (source, format) {
         ],
         format,
       },
-      scope: data,
     }
   );
 
   return {
     mdxSource: mdxSource,
     frontMatter: data,
-    excerpt,
   };
 };
 

examples/alan-turing-portal/package-lock.json

@@ -36,7 +36,8 @@
         "focus-visible": "^5.2.0",
         "gray-matter": "^4.0.3",
         "hastscript": "^7.2.0",
-        "mdx-mermaid": "2.0.0-rc7",
+        "mdx-mermaid": "^2.0.0-rc7",
+        "mermaid": "^10.1.0",
         "next": "13.2.1",
         "next-mdx-remote": "^4.4.1",
         "next-router-mock": "^0.9.3",
@@ -3338,7 +3339,6 @@
       "version": "7.0.10",
       "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.10.tgz",
       "integrity": "sha512-qTCQmEhcynucuaZgY5/+ti3X/rnszKZhEQH/ZdWdtP1tA/y3VoHJzcVrO9pjjJCNpigfscAtoUB5ONcd2wNn0A==",
-      "peer": true,
       "dependencies": {
         "d3": "^7.8.2",
         "lodash-es": "^4.17.21"
@@ -3353,8 +3353,7 @@
     "node_modules/dayjs": {
       "version": "1.11.7",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz",
-      "integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==",
+      "integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ=="
-      "peer": true
     },
     "node_modules/debug": {
       "version": "4.3.4",
@@ -3544,8 +3543,7 @@
     "node_modules/dompurify": {
       "version": "2.4.5",
       "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.4.5.tgz",
-      "integrity": "sha512-jggCCd+8Iqp4Tsz0nIvpcb22InKEBrGz5dw3EQJMs8HPJDsKbFIO3STYtAvCfDx26Muevn1MHVI0XxjgFfmiSA==",
+      "integrity": "sha512-jggCCd+8Iqp4Tsz0nIvpcb22InKEBrGz5dw3EQJMs8HPJDsKbFIO3STYtAvCfDx26Muevn1MHVI0XxjgFfmiSA=="
-      "peer": true
     },
     "node_modules/dot-prop": {
       "version": "5.3.0",
@@ -7533,7 +7531,6 @@
       "version": "10.1.0",
       "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.1.0.tgz",
       "integrity": "sha512-LYekSMNJygI1VnMizAPUddY95hZxOjwZxr7pODczILInO0dhQKuhXeu4sargtnuTwCilSuLS7Uiq/Qn7HTVrmA==",
-      "peer": true,
       "dependencies": {
         "@braintree/sanitize-url": "^6.0.0",
         "@khanacademy/simple-markdown": "^0.8.6",
@@ -7558,7 +7555,6 @@
       "version": "0.8.6",
       "resolved": "https://registry.npmjs.org/@khanacademy/simple-markdown/-/simple-markdown-0.8.6.tgz",
       "integrity": "sha512-mAUlR9lchzfqunR89pFvNI51jQKsMpJeWYsYWw0DQcUXczn/T/V6510utgvm7X0N3zN87j1SvuKk8cMbl9IAFw==",
-      "peer": true,
       "dependencies": {
         "@types/react": ">=16.0.0"
       },
@@ -15739,7 +15735,6 @@
       "version": "7.0.10",
       "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.10.tgz",
       "integrity": "sha512-qTCQmEhcynucuaZgY5/+ti3X/rnszKZhEQH/ZdWdtP1tA/y3VoHJzcVrO9pjjJCNpigfscAtoUB5ONcd2wNn0A==",
-      "peer": true,
       "requires": {
         "d3": "^7.8.2",
         "lodash-es": "^4.17.21"
@@ -15754,8 +15749,7 @@
     "dayjs": {
       "version": "1.11.7",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz",
-      "integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==",
+      "integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ=="
-      "peer": true
     },
     "debug": {
       "version": "4.3.4",
@@ -15894,8 +15888,7 @@
     "dompurify": {
       "version": "2.4.5",
       "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.4.5.tgz",
-      "integrity": "sha512-jggCCd+8Iqp4Tsz0nIvpcb22InKEBrGz5dw3EQJMs8HPJDsKbFIO3STYtAvCfDx26Muevn1MHVI0XxjgFfmiSA==",
+      "integrity": "sha512-jggCCd+8Iqp4Tsz0nIvpcb22InKEBrGz5dw3EQJMs8HPJDsKbFIO3STYtAvCfDx26Muevn1MHVI0XxjgFfmiSA=="
-      "peer": true
     },
     "dot-prop": {
       "version": "5.3.0",
@@ -18847,7 +18840,6 @@
       "version": "10.1.0",
       "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.1.0.tgz",
       "integrity": "sha512-LYekSMNJygI1VnMizAPUddY95hZxOjwZxr7pODczILInO0dhQKuhXeu4sargtnuTwCilSuLS7Uiq/Qn7HTVrmA==",
-      "peer": true,
       "requires": {
         "@braintree/sanitize-url": "^6.0.0",
         "@khanacademy/simple-markdown": "^0.8.6",
@@ -18872,7 +18864,6 @@
           "version": "0.8.6",
           "resolved": "https://registry.npmjs.org/@khanacademy/simple-markdown/-/simple-markdown-0.8.6.tgz",
           "integrity": "sha512-mAUlR9lchzfqunR89pFvNI51jQKsMpJeWYsYWw0DQcUXczn/T/V6510utgvm7X0N3zN87j1SvuKk8cMbl9IAFw==",
-          "peer": true,
           "requires": {
             "@types/react": ">=16.0.0"
           }

examples/alan-turing-portal/package.json

@@ -12,47 +12,41 @@
   },
   "browserslist": "defaults, not ie <= 11",
   "dependencies": {
+    "@flowershow/core": "^0.4.10",
     "@flowershow/markdowndb": "^0.1.1",
+    "@flowershow/remark-callouts": "^1.0.0",
+    "@flowershow/remark-embed": "^1.0.0",
+    "@flowershow/remark-wiki-link": "^1.1.2",
     "@headlessui/react": "^1.7.13",
+    "@heroicons/react": "^2.0.17",
     "@mapbox/rehype-prism": "^0.8.0",
     "@mdx-js/loader": "^2.1.5",
     "@mdx-js/react": "^2.1.5",
     "@next/mdx": "^13.0.2",
+    "@opentelemetry/api": "^1.4.0",
     "@tailwindcss/forms": "^0.5.3",
     "@tailwindcss/typography": "^0.5.4",
+    "@tanstack/react-table": "^8.8.5",
     "autoprefixer": "^10.4.12",
     "clsx": "^1.2.1",
     "fast-glob": "^3.2.11",
     "feed": "^4.2.2",
     "flexsearch": "^0.7.31",
     "focus-visible": "^5.2.0",
-    "next-router-mock": "^0.9.3",
-    "next-superjson-plugin": "^0.5.7",
-    "postcss-focus-visible": "^6.0.4",
-    "react-hook-form": "^7.43.9",
-    "react-markdown": "^8.0.7",
-    "superjson": "^1.12.3",
-    "tailwindcss": "^3.3.0",
-    "@flowershow/core": "^0.4.10",
-    "@flowershow/remark-callouts": "^1.0.0",
-    "@flowershow/remark-embed": "^1.0.0",
-    "@flowershow/remark-wiki-link": "^1.1.2",
-    "@heroicons/react": "^2.0.17",
-    "@opentelemetry/api": "^1.4.0",
-    "@tanstack/react-table": "^8.8.5",
-    "@types/node": "18.16.0",
-    "@types/react": "18.2.0",
-    "@types/react-dom": "18.2.0",
-    "eslint": "8.39.0",
-    "eslint-config-next": "13.3.1",
     "gray-matter": "^4.0.3",
     "hastscript": "^7.2.0",
-    "mdx-mermaid": "2.0.0-rc7",
+    "mdx-mermaid": "^2.0.0-rc7",
+    "mermaid": "^10.1.0",
     "next": "13.2.1",
     "next-mdx-remote": "^4.4.1",
+    "next-router-mock": "^0.9.3",
+    "next-superjson-plugin": "^0.5.7",
     "papaparse": "^5.4.1",
+    "postcss-focus-visible": "^6.0.4",
     "react": "18.2.0",
     "react-dom": "18.2.0",
+    "react-hook-form": "^7.43.9",
+    "react-markdown": "^8.0.7",
     "react-vega": "^7.6.0",
     "rehype-autolink-headings": "^6.1.1",
     "rehype-katex": "^6.0.3",
@@ -61,12 +55,17 @@
     "remark-gfm": "^3.0.1",
     "remark-math": "^5.1.1",
     "remark-smartypants": "^2.0.0",
-    "remark-toc": "^8.0.1"
+    "remark-toc": "^8.0.1",
+    "superjson": "^1.12.3",
+    "tailwindcss": "^3.3.0"
   },
   "devDependencies": {
     "eslint": "8.26.0",
     "eslint-config-next": "13.0.2",
     "prettier": "^2.8.7",
-    "prettier-plugin-tailwindcss": "^0.2.6"
+    "prettier-plugin-tailwindcss": "^0.2.6",
+    "@types/node": "18.16.0",
+    "@types/react": "18.2.0",
+    "@types/react-dom": "18.2.0"
   }
 }

examples/alan-turing-portal/pages/[...slug].jsx

@@ -1,9 +1,12 @@
 import { Container } from '../components/Container'
 import clientPromise from '../lib/mddb'
-import fs from 'fs'
+import { promises as fs } from 'fs';
 import { MDXRemote } from 'next-mdx-remote'
-import { serialize } from 'next-mdx-remote/serialize'
 import { Card } from '../components/Card'
+import Head from 'next/head'
+import parse from '../lib/markdown'
+import { Mermaid } from '@flowershow/core';
+import { Header } from '../components/Header';
 
 export const getStaticProps = async ({ params }) => {
   const urlPath = params.slug ? params.slug.join('/') : ''
@@ -11,8 +14,8 @@ export const getStaticProps = async ({ params }) => {
   const mddb = await clientPromise
   const dbFile = await mddb.getFileByUrl(urlPath)
 
-  const source = fs.readFileSync(dbFile.file_path, { encoding: 'utf-8' })
+  const source = await fs.readFile(dbFile.file_path,'utf-8')
-  const mdxSource = await serialize(source, { parseFrontmatter: true })
+  let mdxSource = await parse(source, '.mdx')
 
   return {
     props: {
@@ -73,13 +76,18 @@ const Meta = ({keyValuePairs}) => {
 }
 
 export default function DRDPage({ mdxSource }) {
-  const meta = mdxSource.frontmatter
+  const meta = mdxSource.frontMatter
   const keyValuePairs = Object.entries(meta).filter(
     (entry) => entry[0] !== 'title'
   )
   return (
     <>
-      <Container className="mt-16 lg:mt-32">
+      <Header />
+      <Head>
+        <title>{meta.title}</title>
+      </Head>
+      <Container className="mt-16 lg:mt-32 relative">
+        <Header />
         <article>
           <header className="flex flex-col">
             <h1 className="mt-6 text-4xl font-bold tracking-tight text-zinc-800 dark:text-zinc-100 sm:text-5xl">
@@ -90,7 +98,7 @@ export default function DRDPage({ mdxSource }) {
             </Card>
           </header>
           <div className="prose dark:prose-invert">
-            <MDXRemote {...mdxSource} />
+            <MDXRemote {...mdxSource.mdxSource} components={{mermaid: Mermaid}} />
           </div>
         </article>
       </Container>

examples/alan-turing-portal/pages/_document.jsx

@@ -1,25 +1,17 @@
 import { Head, Html, Main, NextScript } from 'next/document'
 
 const modeScript = `
-  let darkModeMediaQuery = window.matchMedia('(prefers-color-scheme: dark)')
-
   updateMode()
-  darkModeMediaQuery.addEventListener('change', updateModeWithoutTransitions)
   window.addEventListener('storage', updateModeWithoutTransitions)
 
   function updateMode() {
-    let isSystemDarkMode = darkModeMediaQuery.matches
+    let isDarkMode = window.localStorage.isDarkMode === 'true'
-    let isDarkMode = window.localStorage.isDarkMode === 'true' || (!('isDarkMode' in window.localStorage) && isSystemDarkMode)
 
     if (isDarkMode) {
       document.documentElement.classList.add('dark')
     } else {
       document.documentElement.classList.remove('dark')
     }
-
-    if (isDarkMode === isSystemDarkMode) {
-      delete window.localStorage.isDarkMode
-    }
   }
 
   function disableTransitionsTemporarily() {

examples/alan-turing-portal/pages/index.jsx

@@ -4,7 +4,6 @@ import fs from 'fs'
 import { Card } from '../components/Card'
 import { Container } from '../components/Container'
 import clientPromise from '../lib/mddb'
-import ReactMarkdown from 'react-markdown'
 import { Index } from 'flexsearch'
 import { useForm } from 'react-hook-form'
 import Link from 'next/link'
@@ -14,7 +13,9 @@ import { MDXRemote } from 'next-mdx-remote'
 function DatasetCard({ dataset }) {
   return (
     <Card as="article">
-      <Card.Title><Link href={dataset.url}>{dataset.title}</Link></Card.Title>
+      <Card.Title>
+        <Link href={dataset.url}>{dataset.title}</Link>
+      </Card.Title>
       <Card.Description>
         <span className="font-semibold">Link to publication: </span>{' '}
         <a
@@ -76,7 +77,9 @@ function DatasetCard({ dataset }) {
 function ListOfAbusiveKeywordsCard({ list }) {
   return (
     <Card as="article">
-      <Card.Title><Link href={list.url}>{list.title}</Link></Card.Title>
+      <Card.Title>
+        <Link href={list.url}>{list.title}</Link>
+      </Card.Title>
       {list.description && (
         <Card.Description>
           <span className="font-semibold">List Description: </span>{' '}
@@ -109,7 +112,6 @@ export default function Home({
   datasets,
   indexText,
   listsOfKeywords,
-  contributingText,
   availableLanguages,
   availablePlatforms,
 }) {
@@ -141,17 +143,23 @@ export default function Home({
           <h1 className="text-4xl font-bold tracking-tight text-zinc-800 dark:text-zinc-100 sm:text-5xl">
             {indexText.frontmatter.title}
           </h1>
-          <article className="mt-6 flex flex-col gap-y-2 text-base text-zinc-600 dark:text-zinc-400">
+          <article className="index-text prose mt-6 flex flex-col gap-y-2 text-base text-zinc-600 dark:prose-invert prose-h3:mt-4 prose-a:font-normal prose-a:text-zinc-600 prose-a:decoration-inherit prose-img:rounded-none dark:text-zinc-400 prose-a:dark:text-zinc-400 hover:prose-a:text-teal-600 hover:prose-a:dark:text-teal-900">
             <MDXRemote {...indexText} />
           </article>
         </div>
       </Container>
-      <Container className="mt-24 md:mt-28">
+      <Container className="mt-12 md:mt-14">
         <div className="mx-auto grid max-w-7xl grid-cols-1 gap-y-8 lg:max-w-none">
-          <h2 className="text-xl font-bold tracking-tight text-zinc-800 dark:text-zinc-100 sm:text-5xl">
+          <h2
+            id="Datasets-header"
+            className="text-xl font-bold tracking-tight text-zinc-800 dark:text-zinc-100 sm:text-5xl"
+          >
             Datasets
           </h2>
-          <form onSubmit={handleSubmit(() => reset())} className="rounded-2xl border border-zinc-100 px-4 py-6 dark:border-zinc-700/40 sm:p-6">
+          <form
+            onSubmit={handleSubmit(() => reset())}
+            className="rounded-2xl border border-zinc-100 px-4 py-6 dark:border-zinc-700/40 sm:p-6"
+          >
             <p className="mt-2 text-lg font-semibold text-zinc-600 dark:text-zinc-100">
               Search for datasets
             </p>
@@ -200,7 +208,12 @@ export default function Home({
                   </option>
                 ))}
               </select>
-              <button type='submit' className='inline-flex items-center gap-2 justify-center rounded-md py-2 px-3 text-sm outline-offset-2 transition active:transition-none bg-zinc-800 font-semibold text-zinc-100 hover:bg-zinc-700 active:bg-zinc-800 active:text-zinc-100/70 dark:bg-zinc-700 dark:hover:bg-zinc-600 dark:active:bg-zinc-700 dark:active:text-zinc-100/70 flex-none'>Clear filters</button>
+              <button
+                type="submit"
+                className="inline-flex flex-none items-center justify-center gap-2 rounded-md bg-zinc-800 px-3 py-2 text-sm font-semibold text-zinc-100 outline-offset-2 transition hover:bg-zinc-700 active:bg-zinc-800 active:text-zinc-100/70 active:transition-none dark:bg-zinc-700 dark:hover:bg-zinc-600 dark:active:bg-zinc-700 dark:active:text-zinc-100/70"
+              >
+                Clear filters
+              </button>
             </div>
           </form>
           <div className="flex flex-col gap-16">
@@ -227,7 +240,7 @@ export default function Home({
         </div>
       </Container>
       <Container className="mt-16">
-        <h2 className="text-xl font-bold tracking-tight text-zinc-800 dark:text-zinc-100 sm:text-5xl">
+        <h2 id="Keywords-header" className="text-xl font-bold tracking-tight text-zinc-800 dark:text-zinc-100 sm:text-5xl">
           Lists of Abusive Keywords
         </h2>
         <div className="mt-3 flex flex-col gap-16">
@@ -236,14 +249,6 @@ export default function Home({
           ))}
         </div>
       </Container>
-      <Container className="mt-16">
-        <h2 className="text-xl font-bold tracking-tight text-zinc-800 dark:text-zinc-100 sm:text-5xl">
-          How to contribute
-        </h2>
-          <article className="mt-6 flex flex-col gap-y-8 text-base text-zinc-600 dark:text-zinc-400 contributing">
-            <MDXRemote {...contributingText} />
-        </article>
-        </Container>
     </>
   )
 }
@@ -270,12 +275,7 @@ export async function getStaticProps() {
   }))
 
   const index = await mddb.getFileByUrl('/')
-  const contributing = await mddb.getFileByUrl('contributing')
   let indexSource = fs.readFileSync(index.file_path, { encoding: 'utf-8' })
-  let contributingSource = fs.readFileSync(contributing.file_path, {
-    encoding: 'utf-8',
-  })
-  contributingSource = await serialize(contributingSource, { parseFrontmatter: true })
   indexSource = await serialize(indexSource, { parseFrontmatter: true })
 
   const availableLanguages = [
@@ -289,7 +289,6 @@ export async function getStaticProps() {
       datasets,
       listsOfKeywords,
       indexText: indexSource,
-      contributingText: contributingSource,
       availableLanguages,
       availablePlatforms,
     },

examples/alan-turing-portal/styles/tailwind.css

@@ -3,6 +3,11 @@
 @import './prism.css';
 @import 'tailwindcss/utilities';
 
-.contributing li {
+.index-text ul,
-  margin-bottom: 1.75rem;
+.index-text p {
+  margin: 0;
+}
+
+.index-text h2 {
+  margin-top: 1rem;
 }

examples/alan-turing-portal/tailwind.config.js

@@ -2,6 +2,7 @@
 module.exports = {
   content: [
     "./app/**/*.{js,ts,jsx,tsx,mdx}",
+    "./content/**/*.{js,ts,jsx,tsx,mdx}",
     "./pages/**/*.{js,ts,jsx,tsx,mdx}",
     "./components/**/*.{js,ts,jsx,tsx,mdx}",
   ],

examples/ckan-example/package.json

@@ -10,24 +10,24 @@
   },
   "dependencies": {
     "@heroicons/react": "^2.0.17",
-    "@types/node": "18.16.0",
-    "@types/react": "18.0.38",
-    "@types/react-dom": "18.0.11",
-    "eslint": "8.39.0",
-    "eslint-config-next": "13.3.1",
     "next": "13.3.1",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-markdown": "^8.0.7",
-    "remark-gfm": "^3.0.1",
+    "remark-gfm": "^3.0.1"
-    "typescript": "5.0.4"
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",
     "autoprefixer": "^10.4.14",
     "postcss": "^8.4.23",
-    "tailwindcss": "^3.3.1"
+    "tailwindcss": "^3.3.1",
+    "eslint": "8.39.0",
+    "eslint-config-next": "13.3.1",
+    "typescript": "5.0.4",
+    "@types/node": "18.16.0",
+    "@types/react": "18.0.38",
+    "@types/react-dom": "18.0.11"
   }
 }

examples/fiverthirtyeight-example/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

examples/fiverthirtyeight-example/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/fiverthirtyeight-example/README.md

@@ -0,0 +1,38 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file.
+
+[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
+
+The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

examples/fiverthirtyeight-example/next.config.js

@@ -0,0 +1,6 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  reactStrictMode: true,
+}
+
+module.exports = nextConfig

examples/fiverthirtyeight-example/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "fiverthirtyeight-example",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@types/node": "20.1.1",
+    "@types/react": "18.2.6",
+    "@types/react-dom": "18.2.4",
+    "autoprefixer": "10.4.14",
+    "eslint": "8.40.0",
+    "eslint-config-next": "13.4.1",
+    "next": "13.4.1",
+    "postcss": "8.4.23",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "tailwindcss": "3.3.2",
+    "timeago.js": "^4.0.2",
+    "typescript": "5.0.4"
+  }
+}

examples/fiverthirtyeight-example/pages/_app.tsx

@@ -0,0 +1,6 @@
+import '@/styles/globals.css'
+import type { AppProps } from 'next/app'
+
+export default function App({ Component, pageProps }: AppProps) {
+  return <Component {...pageProps} />
+}

examples/fiverthirtyeight-example/pages/_document.tsx

@@ -0,0 +1,13 @@
+import { Html, Head, Main, NextScript } from 'next/document'
+
+export default function Document() {
+  return (
+    <Html lang="en">
+      <Head />
+      <body>
+        <Main />
+        <NextScript />
+      </body>
+    </Html>
+  )
+}

examples/fiverthirtyeight-example/pages/api/hello.ts

@@ -0,0 +1,13 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next'
+
+type Data = {
+  name: string
+}
+
+export default function handler(
+  req: NextApiRequest,
+  res: NextApiResponse<Data>
+) {
+  res.status(200).json({ name: 'John Doe' })
+}

examples/fiverthirtyeight-example/pages/index.tsx

@@ -0,0 +1,195 @@
+import Image from 'next/image';
+import { Inter } from 'next/font/google';
+import { format } from 'timeago.js'
+import { promises as fs } from 'fs';
+import path from 'path';
+
+const inter = Inter({ subsets: ['latin'] });
+
+interface Article {
+  date: string;
+  title: string;
+  url: string;
+}
+
+interface Dataset {
+  url: string;
+  name: string;
+  displayName: string;
+  articles: Article[];
+}
+
+export function MobileItem({dataset} : { dataset: Dataset}) {
+  return (
+    <div className="flex gap-x-2 pb-2 py-4 items-center justify-between border-b border-zinc-600">
+      <div className="flex flex-col">
+        <span className="font-light">{dataset.name}</span>
+        {dataset.articles.map((article) => (
+          <div className='py-1 flex flex-col'>
+            <span className="font-bold hover:underline">{article.title}</span>
+            <span className="font-light text-base">{format(article.date)}</span>{' '}
+          </div>
+        ))}
+      </div>
+      <div className="flex flex-col justify-start">
+        <a
+          className="border border-zinc-900 font-light px-4 py-1 text-sm transition hover:bg-zinc-900 hover:text-white"
+          href={dataset.url}
+          target="_blank"
+        >
+          info
+        </a>
+        {/*
+                <button>
+                  <svg
+                    xmlns="http://www.w3.org/2000/svg"
+                    viewBox="0 0 24 24"
+                    fill="currentColor"
+                    className="w-12 h-12 text-blue-400 hover:text-blue-300 transition mt-1"
+                  >
+                    <path
+                      fillRule="evenodd"
+                      d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
+                      clipRule="evenodd"
+                    />
+                  </svg>
+                </button> */}
+      </div>
+    </div>
+  );
+}
+
+export function DesktopItem({dataset} : { dataset: Dataset}) {
+  return (
+    <>
+      {dataset.articles.map((article, index) => (
+        <tr className={`${index === (dataset.articles.length - 1) ? 'border-b' : ''} border-zinc-400`}>
+          <td className="py-8 font-light">{index === 0 ? dataset.name : ''}</td>
+          <td>
+            <a className="py-8 font-bold hover:underline" href={article.url}>
+              {article.title}
+            </a>
+          </td>
+          <td className="py-8 font-light text-base min-w-[120px]">{format(article.date)}</td>
+          <td className="py-8">
+            {index === 0 && (
+              <a
+                className="border border-zinc-900 font-light px-[25px] py-2.5 text-sm transition hover:bg-zinc-900 hover:text-white"
+                href={dataset.url}
+                target="_blank"
+              >
+                info
+              </a>
+            )}
+          </td>
+          {/*
+              <td>
+                <button>
+                  <svg
+                    xmlns="http://www.w3.org/2000/svg"
+                    viewBox="0 0 24 24"
+                    fill="currentColor"
+                    className="w-12 h-12 text-blue-400 hover:text-blue-300 transition mt-1"
+                  >
+                    <path
+                      fillRule="evenodd"
+                      d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
+                      clipRule="evenodd"
+                    />
+                  </svg>
+                </button>
+              </td>*/}
+        </tr>
+      ))}
+    </>
+  );
+}
+
+export async function getStaticProps() {
+  const jsonDirectory = path.join(
+    process.cwd(),
+    '/datasets.json'
+  );
+  const datasetString = await fs.readFile(jsonDirectory, 'utf8');
+  const datasets = JSON.parse(datasetString)
+  return {
+    props: { datasets },
+  };
+}
+
+export default function Home( { datasets }: { datasets: Dataset[] }) {
+  return (
+    <>
+      <header className="max-w-5xl mx-auto mt-8 w-full">
+        <div className="border-b-2 pb-2.5 mx-2 border-zinc-800">
+          <h1>
+            <span className="sr-only">FiveThirtyEight</span>
+            <a className='flex gap-x-2 items-center' href="http://fivethirtyeight.com">
+              <img
+                width="197"
+                height="25"
+                alt="FiveThirtyEight"
+                src="data:image/svg+xml;base64,PHN2ZyBpZD0iTGF5ZXJfMSIgZGF0YS1uYW1lPSJMYXllciAxIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjEgNTMuNzYiPjxkZWZzPjxzdHlsZT4uY2xzLTF7ZmlsbDojMDEwMTAxO308L3N0eWxlPjwvZGVmcz48dGl0bGU+QXJ0Ym9hcmQgOTU8L3RpdGxlPjxwYXRoIGNsYXNzPSJjbHMtMSIgZD0iTTAgMGgyNXY4SDl2MTBoMTV2OEg5djE3SDBWMHpNMzEgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdIMzF6bTUtMzZoOHY4aC04ek0xNzkgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdoLTE3em01LTM2aDh2OGgtOHpNMzE2IDM2aDVWMThoLTV2LThoMTN2MjZoNHY3aC0xN3ptNS0zNmg4djhoLTh6TTU0IDI3VjEwaDh2MTVsNCA5Ljk4aDFMNzEgMjVWMTBoOHYxN2wtNyAxNkg2MWwtNy0xNnpNMTExIDQzSDk3LjQyQzg5LjIzIDQzIDg1IDM5LjE5IDg1IDMxLjE3VjIyYzAtNy41NyA0LjMtMTMgMTMtMTMgOS4zMyAwIDEzIDUuMDcgMTMgMTR2N0g5NHYxLjc0YzAgMi42MiAxIDQuMjYgMy40MiA0LjI2SDExMXpNOTQgMjNoOHYtMS41NWMwLTIuNjItMS4wNi01LjQ1LTQuMTMtNS40NS0yLjc5IDAtMy44NyAyLjItMy44NyA1LjQ1ek0xMjUgOGgtMTBWMGgyOXY4aC0xMHYzNWgtOVY4ek0yMDIgNDNWMTBoOHY0YzEuMTQtMi40NSAzLjc1LTQgNy4yMi00SDIyMHY4aC02Yy0yLjg0IDAtNCAuOTQtNCAzLjlWNDN6TTI0NSA0M2gtNC44NEMyMzMuMDUgNDMgMjMwIDM5LjMxIDIzMCAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0gyNDV6TTQyMSA0M2gtNC44NEM0MDkuMDUgNDMgNDA2IDM5LjMxIDQwNiAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0g0MjF6TTI1NC4yNiA1My43Nmw0LjYxLTkuNUwyNTEgMjdWMTBoOHYxNWw0IDEwaDFsNC0xMFYxMGg4djE3bC0xMi4zIDI2Ljc2aC05LjQ0ek0yODQgMGgyNXY4aC0xNnY5aDE1djhoLTE1djEwaDE2djhoLTI1VjB6TTMzNyA0OHYtMmgxNi4xYzIgMCAyLjktLjE4IDIuOS0xLjI3di0uMzRjMC0xLjA4LS45MS0xLjM5LTIuOS0xLjM5SDM0MHYtNWw1LTVjLTUuMjktMS40OC04LTUuNDMtOC0xMXYtMWMwLTcuNTYgNC40NC0xMiAxNC0xMmEyMS45MyAyMS45MyAwIDAgMSA1Ljk1IDFMMzYxIDRsNSAzLTQgNmMxLjM3IDEuOTMgMyA0LjkzIDMgOHYxYzAgNy0zLjMgMTAuNjYtMTIgMTFsLTMgNGg2YzUuOTIgMCA5IDIuNjIgOSA3LjY4di4xMWMwIDUuMDYtMi43MSA4LjIxLTguNjIgOC4yMWgtMTNjLTQuMjkgMC02LjM4LTEuODQtNi4zOC01em0xOS0yNXYtM2MwLTMuMy0xLjMzLTQtNS00cy01IC43LTUgNHYzYzAgMy4zIDEuMzkgNCA1IDRzNS0uNyA1LTR6TTM4MCA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuNC00IDctNCA2LjI2IDAgOSAzLjA4IDkgMTAuNzZWNDNoLThWMjJjMC0zLjEzLTEuMDctNS00LTVzLTQgMS44Ny00IDV6TTE1NyA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuOTEtNCA3LjQ5LTQgNi4yNiAwIDguNTEgMy4xMyA4LjUxIDEwLjgxVjQzaC04VjIxYzAtMy4xMy0xLjA3LTQuNDQtNC00LjQ0cy00IDIuMjYtNCA1LjM5eiIvPjwvc3ZnPg=="
+              /> by PortalJS
+            </a>
+          </h1>
+        </div>
+      </header>
+      <main
+        className={`flex min-h-screen flex-col items-center max-w-5xl mx-auto pt-20 px-2.5 ${inter.className}`}
+      >
+        <div>
+          <h1 className="text-[40px] font-bold text-zinc-800 text-center">
+            Our Data
+          </h1>
+          <p className="max-w-2xl text-lg text-center text-zinc-700">
+            We’re sharing the data and code behind some of our articles and
+            graphics. We hope you’ll use it to check our work and to create
+            stories and visualizations of&nbsp;your&nbsp;own.
+          </p>
+        </div>
+        <article className="w-full px-2 md:hidden py-4">{datasets.map(dataset => <MobileItem dataset={dataset} />)}</article>
+        <table className="w-full mt-10 mb-4 hidden md:table">
+          <thead className="border-b-4 pb-2 border-zinc-900">
+            <tr>
+              <th className="uppercase text-left font-light text-xs pb-3">
+                data set
+              </th>
+              <th className="uppercase text-left font-light text-xs pb-3">
+                related content
+              </th>
+              <th className="uppercase text-left font-light text-xs pb-3">
+                last updated
+              </th>
+            </tr>
+          </thead>
+          <tbody>{datasets.map(dataset => <DesktopItem dataset={dataset} />)}</tbody>
+        </table>
+        <p className="text-[13px] py-8">
+          Unless otherwise noted, our data sets are available under the{' '}
+          <a
+            className="text-blue-400 hover:underline"
+            href="http://creativecommons.org/licenses/by/4.0/"
+          >
+            Creative Commons Attribution 4.0 International license
+          </a>
+          , and the code is available under the{' '}
+          <a
+            className="text-blue-400 hover:underline"
+            href="http://opensource.org/licenses/MIT"
+          >
+            MIT license
+          </a>
+          . If you find this information useful, please{' '}
+          <a
+            className="text-blue-400 hover:underline"
+            href="mailto:data@fivethirtyeight.com"
+          >
+            let us know
+          </a>
+          .
+        </p>
+      </main>
+    </>
+  );
+}

examples/fiverthirtyeight-example/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

examples/fiverthirtyeight-example/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

examples/fiverthirtyeight-example/public/vercel.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

examples/fiverthirtyeight-example/styles/globals.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

examples/fiverthirtyeight-example/tailwind.config.js

@@ -0,0 +1,18 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    './pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './components/**/*.{js,ts,jsx,tsx,mdx}',
+    './app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      backgroundImage: {
+        'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
+        'gradient-conic':
+          'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
+      },
+    },
+  },
+  plugins: [],
+}

examples/fiverthirtyeight-example/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+  "exclude": ["node_modules"]
+}

examples/learn-example/components/DataRichDocument.tsx

@@ -7,13 +7,12 @@ import { Mermaid } from '@flowershow/core';
 // to handle import statements. Instead, you must include components in scope
 // here.
 const components = {
-  Table: dynamic(() => import('./Table')),
+  Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
+  Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
   mermaid: Mermaid,
-  // Excel: dynamic(() => import('../components/Excel')),
+  Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
-  // TODO: try and make these dynamic ...
+  VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),
-  Vega: dynamic(() => import('./Vega')),
+  LineChart: dynamic(() => import('@portaljs/components').then(mod => mod.LineChart)),
-  VegaLite: dynamic(() => import('./VegaLite')),
-  LineChart: dynamic(() => import('./LineChart')),
 } as any;
 
 export default function DRD({ source }: { source: any }) {

examples/learn-example/components/LineChart.tsx

@@ -1,55 +0,0 @@
-import VegaLite from "./VegaLite";
-
-export default function LineChart({
-  data = [],
-  fullWidth = false,
-  title = "",
-  xAxis = "x",
-  yAxis = "y",
-}) {
-  var tmp = data;
-  if (Array.isArray(data)) {
-    tmp = data.map((r, i) => {
-      return { x: r[0], y: r[1] };
-    });
-  }
-  const vegaData = { table: tmp };
-  const spec = {
-    $schema: "https://vega.github.io/schema/vega-lite/v5.json",
-    title,
-    width: "container",
-    height: 300,
-    mark: {
-      type: "line",
-      color: "black",
-      strokeWidth: 1,
-      tooltip: true,
-    },
-    data: {
-      name: "table",
-    },
-    selection: {
-      grid: {
-        type: "interval",
-        bind: "scales",
-      },
-    },
-    encoding: {
-      x: {
-        field: xAxis,
-        timeUnit: "year",
-        type: "temporal",
-      },
-      y: {
-        field: yAxis,
-        type: "quantitative",
-      },
-    },
-  };
-  if (typeof data === 'string') {
-    spec.data = { "url": data } as any
-    return <VegaLite fullWidth={fullWidth} spec={spec} />;
-  }
-
-  return <VegaLite fullWidth={fullWidth} data={vegaData} spec={spec} />;
-}

examples/learn-example/lib/markdown.js

@@ -22,7 +22,7 @@ import { serialize } from "next-mdx-remote/serialize";
  * @format: used to indicate to next-mdx-remote which format to use (md or mdx)
  * @returns: { mdxSource: mdxSource, frontMatter: ...}
  */
-const parse = async function (source, format) {
+const parse = async function (source, format, scope) {
   const { content, data, excerpt } = matter(source, {
     excerpt: (file, options) => {
       // Generate an excerpt for the file
@@ -91,7 +91,7 @@ const parse = async function (source, format) {
         ],
         format,
       },
-      scope: data,
+      scope,
     }
   );
 

examples/learn-example/lib/mddb.ts

@@ -0,0 +1,14 @@
+import { MarkdownDB } from "@flowershow/markdowndb";
+
+const dbPath = "markdown.db";
+
+const client = new MarkdownDB({
+  client: "sqlite3",
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const clientPromise = client.init();
+
+export default clientPromise;

examples/learn-example/lib/parseCsv.ts

@@ -1,16 +0,0 @@
-import papa from "papaparse";
-
-const parseCsv = (csv) => {
-  csv = csv.trim();
-  const rawdata = papa.parse(csv, { header: true });
-  const cols = rawdata.meta.fields.map((r, i) => {
-    return { key: r, name: r };
-  });
-
-  return {
-    rows: rawdata.data,
-    fields: cols,
-  };
-};
-
-export default parseCsv;

examples/learn-example/package.json

@@ -7,21 +7,21 @@
     "build": "next build",
     "start": "next start",
     "lint": "next lint",
-    "export": "npm run build && next export -o out"
+    "export": "npm run build && next export -o out",
+    "prebuild": "npm run mddb",
+    "mddb": "mddb ./content"
   },
   "dependencies": {
     "@flowershow/core": "^0.4.10",
+    "@flowershow/markdowndb": "^0.1.1",
     "@flowershow/remark-callouts": "^1.0.0",
     "@flowershow/remark-embed": "^1.0.0",
     "@flowershow/remark-wiki-link": "^1.1.2",
     "@heroicons/react": "^2.0.17",
     "@opentelemetry/api": "^1.4.0",
+    "@portaljs/components": "^0.1.0",
     "@tanstack/react-table": "^8.8.5",
-    "@types/node": "18.16.0",
+    "flexsearch": "0.7.21",
-    "@types/react": "18.2.0",
-    "@types/react-dom": "18.2.0",
-    "eslint": "8.39.0",
-    "eslint-config-next": "13.3.1",
     "gray-matter": "^4.0.3",
     "hastscript": "^7.2.0",
     "mdx-mermaid": "2.0.0-rc7",
@@ -30,6 +30,7 @@
     "papaparse": "^5.4.1",
     "react": "18.2.0",
     "react-dom": "18.2.0",
+    "react-hook-form": "^7.43.9",
     "react-vega": "^7.6.0",
     "rehype-autolink-headings": "^6.1.1",
     "rehype-katex": "^6.0.3",
@@ -43,7 +44,13 @@
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",
+    "@types/flexsearch": "^0.7.3",
+    "@types/node": "18.16.0",
+    "@types/react": "18.2.0",
+    "@types/react-dom": "18.2.0",
     "autoprefixer": "^10.4.14",
+    "eslint": "8.39.0",
+    "eslint-config-next": "13.3.1",
     "postcss": "^8.4.23",
     "tailwindcss": "^3.3.1"
   }

examples/learn-example/pages/[[...path]].tsx

@@ -1,7 +1,9 @@
-import { promises as fs } from 'fs';
+import { existsSync, promises as fs } from 'fs';
 import path from 'path';
 import parse from '../lib/markdown';
-import DRD from '../components/DRD';
+
+import DataRichDocument from '../components/DataRichDocument';
+import clientPromise from '../lib/mddb';
 
 export const getStaticPaths = async () => {
   const contentDir = path.join(process.cwd(), '/content/');
@@ -23,37 +25,101 @@ export const getStaticProps = async (context) => {
     pathToFile = context.params.path.join('/') + '/index.md';
   }
 
+  let datasets = [];
+  const mddbFileExists = existsSync('markdown.db');
+  if (mddbFileExists) {
+    const mddb = await clientPromise;
+    const datasetsFiles = await mddb.getFiles({
+      extensions: ['md', 'mdx'],
+    });
+    datasets = datasetsFiles
+      .filter((dataset) => dataset.url_path !== '/')
+      .map((dataset) => ({
+        _id: dataset._id,
+        url_path: dataset.url_path,
+        file_path: dataset.file_path,
+        metadata: dataset.metadata,
+      }));
+  }
+
   const indexFile = path.join(process.cwd(), '/content/' + pathToFile);
   const readme = await fs.readFile(indexFile, 'utf8');
-  let { mdxSource, frontMatter } = await parse(readme, '.mdx');
+
+  let { mdxSource, frontMatter } = await parse(readme, '.mdx', { datasets });
+
   return {
     props: {
       mdxSource,
-      frontMatter,
+      frontMatter: JSON.stringify(frontMatter),
     },
   };
 };
 
 export default function DatasetPage({ mdxSource, frontMatter }) {
+  frontMatter = JSON.parse(frontMatter);
   return (
-    <div className="prose dark:prose-invert mx-auto">
+    <div className="prose dark:prose-invert mx-auto py-8">
       <header>
         <div className="mb-6">
           <>
-            <h1>{frontMatter.title}</h1>
+            <h1 className="mb-2">{frontMatter.title}</h1>
             {frontMatter.author && (
-              <div className="-mt-6">
+              <p className="my-0">
-                <p className="opacity-60 pl-1">{frontMatter.author}</p>
+                <span className="font-semibold">Author: </span>
-              </div>
+                <span className="my-0">{frontMatter.author}</span>
+              </p>
             )}
             {frontMatter.description && (
-              <p className="description">{frontMatter.description}</p>
+              <p className="my-0">
+                <span className="font-semibold">Description: </span>
+                <span className="description my-0">
+                  {frontMatter.description}
+                </span>
+              </p>
+            )}
+            {frontMatter.modified && (
+              <p className="my-0">
+                <span className="font-semibold">Modified: </span>
+                <span className="description my-0">
+                  {new Date(frontMatter.modified).toLocaleDateString()}
+                </span>
+              </p>
+            )}
+            {frontMatter.files && (
+              <section className="py-6">
+                <h2 className="mt-0">Data files</h2>
+                <table className="table-auto">
+                  <thead>
+                    <tr>
+                      <th>File</th>
+                      <th>Format</th>
+                    </tr>
+                  </thead>
+                  <tbody>
+                    {frontMatter.files.map((f) => {
+                      const fileName = f.split('/').slice(-1);
+                      return (
+                        <tr key={`resources-list-${f}`}>
+                          <td>
+                            <a target="_blank" href={f}>
+                              {fileName}
+                            </a>
+                          </td>
+                          <td>
+                            {fileName[0].split('.').slice(-1)[0].toUpperCase()}
+                          </td>
+                        </tr>
+                      );
+                    })}
+                  </tbody>
+                </table>
+              </section>
             )}
           </>
         </div>
       </header>
       <main>
-        <DRD source={mdxSource} />
+        <DataRichDocument source={mdxSource} />
       </main>
     </div>
   );

examples/learn-example/pages/_app.tsx

@@ -1,4 +1,6 @@
 import '../styles/globals.css'
+import '@portaljs/components/styles.css'
+
 import type { AppProps } from 'next/app'
 
 export default function App({ Component, pageProps }: AppProps) {

examples/learn-example/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "es5",
+    "target": "es6",
     "lib": ["dom", "dom.iterable", "esnext"],
     "allowJs": true,
     "skipLibCheck": true,

package.json

@@ -4,11 +4,10 @@
   "license": "MIT",
   "scripts": {},
   "private": true,
-  "dependencies": {},
   "devDependencies": {
     "@babel/preset-react": "^7.14.5",
     "@nrwl/cypress": "15.9.2",
-    "@nrwl/eslint-plugin-nx": "15.9.2",
+    "@nrwl/eslint-plugin-nx": "^16.0.2",
     "@nrwl/jest": "15.9.2",
     "@nrwl/js": "15.9.2",
     "@nrwl/linter": "15.9.2",

packages/components/.eslintrc.cjs

@@ -0,0 +1,16 @@
+module.exports = {
+  env: {
+    browser: true,
+    es2020: true
+  },
+  extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'plugin:react-hooks/recommended', 'plugin:storybook/recommended'],
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    ecmaVersion: 'latest',
+    sourceType: 'module'
+  },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': 'warn'
+  }
+};

packages/components/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

packages/components/.storybook/main.ts

@@ -0,0 +1,17 @@
+import type { StorybookConfig } from '@storybook/react-vite';
+const config: StorybookConfig = {
+  stories: ['../stories/**/*.mdx', '../stories/**/*.stories.@(js|jsx|ts|tsx)'],
+  addons: [
+    '@storybook/addon-links',
+    '@storybook/addon-essentials',
+    '@storybook/addon-interactions',
+  ],
+  framework: {
+    name: '@storybook/react-vite',
+    options: {},
+  },
+  docs: {
+    autodocs: 'tag',
+  },
+};
+export default config;

packages/components/.storybook/preview.ts

@@ -0,0 +1,17 @@
+import 'tailwindcss/tailwind.css'
+
+import type { Preview } from '@storybook/react';
+
+const preview: Preview = {
+  parameters: {
+    actions: { argTypesRegex: '^on[A-Z].*' },
+    controls: {
+      matchers: {
+        color: /(background|color)$/i,
+        date: /Date$/,
+      },
+    },
+  },
+};
+
+export default preview;

packages/components/README.md

@@ -0,0 +1,29 @@
+# PortalJS React Components
+
+**Storybook:** https://storybook.portaljs.org  
+**Docs**: https://portaljs.org/docs  
+
+## Usage
+
+To install this package on your project:
+
+```bash
+npm i @portaljs/components
+```
+
+> Note: React 18 is required.
+
+You'll also have to import the styles CSS file in your project:
+
+```ts
+//  E.g.: Next.js => pages/_app.tsx
+import '@portaljs/components/styles.css'
+```
+
+## Dev
+
+Use Storybook to work on components by running:
+
+```bash
+npm run storybook
+```

packages/components/package.json

@@ -0,0 +1,86 @@
+{
+  "name": "@portaljs/components",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "https://portaljs.org",
+  "keywords": [
+    "data portal",
+    "data catalog",
+    "table",
+    "charts",
+    "visualization"
+  ],
+  "scripts": {
+    "dev": "npm run storybook",
+    "build": "tsc && vite build && npm run build-tailwind",
+    "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "prepack": "json -f package.json -I -e \"delete this.devDependencies; delete this.dependencies\"",
+    "storybook": "storybook dev -p 6006",
+    "build-storybook": "storybook build",
+    "build-tailwind": "NODE_ENV=production npx tailwindcss -o ./dist/styles.css --minify"
+  },
+  "peerDependencies": {
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0"
+  },
+  "dependencies": {
+    "@heroicons/react": "^2.0.17",
+    "@tanstack/react-table": "^8.8.5",
+    "flexsearch": "0.7.21",
+    "next-mdx-remote": "^4.4.1",
+    "papaparse": "^5.4.1",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-hook-form": "^7.43.9",
+    "react-vega": "^7.6.0",
+    "vega": "5.20.2",
+    "vega-lite": "5.1.0"
+  },
+  "devDependencies": {
+    "@types/flexsearch": "^0.7.3",
+    "@storybook/addon-essentials": "^7.0.7",
+    "@storybook/addon-interactions": "^7.0.7",
+    "@storybook/addon-links": "^7.0.7",
+    "@storybook/blocks": "^7.0.7",
+    "@storybook/react": "^7.0.7",
+    "@storybook/react-vite": "^7.0.7",
+    "@storybook/testing-library": "^0.0.14-next.2",
+    "@types/papaparse": "^5.3.7",
+    "@types/react": "^18.0.28",
+    "@types/react-dom": "^18.0.11",
+    "@typescript-eslint/eslint-plugin": "^5.57.1",
+    "@typescript-eslint/parser": "^5.57.1",
+    "@vitejs/plugin-react": "^4.0.0",
+    "autoprefixer": "^10.4.14",
+    "eslint": "^8.38.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.3.4",
+    "eslint-plugin-storybook": "^0.6.11",
+    "json": "^11.0.0",
+    "postcss": "^8.4.23",
+    "prop-types": "^15.8.1",
+    "storybook": "^7.0.7",
+    "tailwindcss": "^3.3.2",
+    "typescript": "^5.0.2",
+    "vite": "^4.3.2",
+    "vite-plugin-dts": "^2.3.0"
+  },
+  "files": [
+    "dist"
+  ],
+  "main": "./dist/components.umd.js",
+  "module": "./dist/components.es.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/components.es.js",
+      "require": "./dist/components.umd.js"
+    },
+    "./styles.css": {
+      "import": "./dist/styles.css"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/components/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

packages/components/src/components/Catalog.tsx

@@ -0,0 +1,119 @@
+import { Index } from 'flexsearch';
+import { useState } from 'react';
+import DebouncedInput from './DebouncedInput';
+import { useForm } from 'react-hook-form';
+
+export function Catalog({
+  datasets,
+  facets,
+}: {
+  datasets: any[];
+  facets: string[];
+}) {
+  const [indexFilter, setIndexFilter] = useState('');
+  const index = new Index({ tokenize: 'full' });
+  datasets.forEach((dataset) =>
+    index.add(
+      dataset._id,
+      //This will join every metadata value + the url_path into one big string and index that
+      Object.entries(dataset.metadata).reduce(
+        (acc, curr) => acc + ' ' + curr[1].toString(),
+        ''
+      ) +
+        ' ' +
+        dataset.url_path
+    )
+  );
+
+  const facetValues = facets
+    ? facets.reduce((acc, facet) => {
+        const possibleValues = datasets.reduce((acc, curr) => {
+          const facetValue = curr.metadata[facet];
+          if (facetValue) {
+            return Array.isArray(facetValue)
+              ? acc.concat(facetValue)
+              : acc.concat([facetValue]);
+          }
+          return acc;
+        }, []);
+        acc[facet] = {
+          possibleValues: [...new Set(possibleValues)],
+          selectedValue: null,
+        };
+        return acc;
+      }, {})
+    : [];
+
+  const { register, watch } = useForm(facetValues);
+
+  const filteredDatasets = datasets
+    // First filter by flex search
+    .filter((dataset) =>
+      indexFilter !== ''
+        ? index.search(indexFilter).includes(dataset._id)
+        : true
+    )
+    //Then check if the selectedValue for the given facet is included in the dataset metadata
+    .filter((dataset) => {
+      //Avoids a server rendering breakage
+      if (!watch() || Object.keys(watch()).length === 0) return true
+      //This will filter only the key pairs of the metadata values that were selected as facets
+      const datasetFacets = Object.entries(dataset.metadata).filter((entry) =>
+        facets.includes(entry[0])
+      );
+      //Check if the value present is included in the selected value in the form
+      return datasetFacets.every((elem) =>
+        watch()[elem[0]].selectedValue
+          ? (elem[1] as string | string[]).includes(
+              watch()[elem[0]].selectedValue
+            )
+          : true
+      );
+    });
+
+  return (
+    <>
+      <DebouncedInput
+        value={indexFilter ?? ''}
+        onChange={(value) => setIndexFilter(String(value))}
+        className="p-2 text-sm shadow border border-block mr-1"
+        placeholder="Search all datasets..."
+      />
+      {Object.entries(facetValues).map((elem) => (
+        <select
+          key={elem[0]}
+          defaultValue=""
+          className="p-2 ml-1 text-sm shadow border border-block"
+          {...register(elem[0] + '.selectedValue')}
+        >
+          <option value="">
+            Filter by {elem[0]}
+          </option>
+          {(elem[1] as { possibleValues: string[] }).possibleValues.map(
+            (val) => (
+              <option
+                key={val}
+                className="dark:bg-white dark:text-black"
+                value={val}
+              >
+                {val}
+              </option>
+            )
+          )}
+        </select>
+      ))}
+      <ul className='mb-5 pl-6 mt-5 list-disc'>
+        {filteredDatasets.map((dataset) => (
+          <li className='py-2' key={dataset._id}>
+            <a className='font-medium underline' href={dataset.url_path}>
+              {dataset.metadata.title
+                ? dataset.metadata.title
+                : dataset.url_path}
+            </a>
+          </li>
+        ))}
+      </ul>
+    </>
+  );
+}
+

packages/components/src/components/LineChart.tsx

@@ -0,0 +1,63 @@
+import { VegaLite } from './VegaLite';
+
+export type LineChartProps = {
+  data: Array<Array<string | number>> | string | { x: string; y: number }[];
+  title?: string;
+  xAxis?: string;
+  yAxis?: string;
+  fullWidth?: boolean;
+};
+
+export function LineChart({
+  data = [],
+  fullWidth = false,
+  title = '',
+  xAxis = 'x',
+  yAxis = 'y',
+}: LineChartProps) {
+  var tmp = data;
+  if (Array.isArray(data)) {
+    tmp = data.map((r) => {
+      return { x: r[0], y: r[1] };
+    });
+  }
+  const vegaData = { table: tmp };
+  const spec = {
+    $schema: 'https://vega.github.io/schema/vega-lite/v5.json',
+    title,
+    width: 'container',
+    height: 300,
+    mark: {
+      type: 'line',
+      color: 'black',
+      strokeWidth: 1,
+      tooltip: true,
+    },
+    data: {
+      name: 'table',
+    },
+    selection: {
+      grid: {
+        type: 'interval',
+        bind: 'scales',
+      },
+    },
+    encoding: {
+      x: {
+        field: xAxis,
+        timeUnit: 'year',
+        type: 'temporal',
+      },
+      y: {
+        field: yAxis,
+        type: 'quantitative',
+      },
+    },
+  };
+  if (typeof data === 'string') {
+    spec.data = { url: data } as any;
+    return <VegaLite fullWidth={fullWidth} spec={spec} />;
+  }
+
+  return <VegaLite fullWidth={fullWidth} data={vegaData} spec={spec} />;
+}

packages/components/src/components/Table.tsx

@@ -7,7 +7,7 @@ import {
   getPaginationRowModel,
   getSortedRowModel,
   useReactTable,
-} from "@tanstack/react-table";
+} from '@tanstack/react-table';
 
 import {
   ArrowDownIcon,
@@ -16,21 +16,29 @@ import {
   ChevronDoubleRightIcon,
   ChevronLeftIcon,
   ChevronRightIcon,
-} from "@heroicons/react/24/solid";
+} from '@heroicons/react/24/solid';
 
-import React, { useEffect, useMemo, useState } from "react";
+import React, { useEffect, useMemo, useState } from 'react';
 
-import parseCsv from "../lib/parseCsv";
+import parseCsv from '../lib/parseCsv';
-import DebouncedInput from "./DebouncedInput";
+import DebouncedInput from './DebouncedInput';
-import loadData from "../lib/loadData";
+import loadData from '../lib/loadData';
 
-const Table = ({
+export type TableProps = {
+  data?: Array<{ [key: string]: number | string }>;
+  cols?: Array<{ [key: string]: string }>;
+  csv?: string;
+  url?: string;
+  fullWidth?: boolean;
+};
+
+export const Table = ({
   data: ogData = [],
   cols: ogCols = [],
-  csv = "",
+  csv = '',
-  url = "",
+  url = '',
   fullWidth = false,
-}) => {
+}: TableProps) => {
   if (csv) {
     const out = parseCsv(csv);
     ogData = out.rows;
@@ -39,19 +47,19 @@ const Table = ({
 
   const [data, setData] = React.useState(ogData);
   const [cols, setCols] = React.useState(ogCols);
-  const [error, setError] = React.useState(""); //  TODO: add error handling
+  // const [error, setError] = React.useState(""); //  TODO: add error handling
 
   const tableCols = useMemo(() => {
     const columnHelper = createColumnHelper();
     return cols.map((c) =>
-      columnHelper.accessor(c.key, {
+      columnHelper.accessor<any, string>(c.key, {
         header: () => c.name,
         cell: (info) => info.getValue(),
       })
     );
   }, [data, cols]);
 
-  const [globalFilter, setGlobalFilter] = useState("");
+  const [globalFilter, setGlobalFilter] = useState('');
 
   const table = useReactTable({
     data,
@@ -78,24 +86,24 @@ const Table = ({
   }, [url]);
 
   return (
-    <div className={`${fullWidth ? "w-[90vw] ml-[calc(50%-45vw)]" : "w-full"}`}>
+    <div className={`${fullWidth ? 'w-[90vw] ml-[calc(50%-45vw)]' : 'w-full'}`}>
       <DebouncedInput
-        value={globalFilter ?? ""}
+        value={globalFilter ?? ''}
-        onChange={(value) => setGlobalFilter(String(value))}
+        onChange={(value: any) => setGlobalFilter(String(value))}
         className="p-2 text-sm shadow border border-block"
         placeholder="Search all columns..."
       />
-      <table>
+      <table className="w-full mt-10">
-        <thead>
+        <thead className="text-left border-b border-b-slate-300">
           {table.getHeaderGroups().map((hg) => (
             <tr key={hg.id}>
               {hg.headers.map((h) => (
-                <th key={h.id}>
+                <th key={h.id} className="pr-2 pb-2">
                   <div
                     {...{
                       className: h.column.getCanSort()
-                        ? "cursor-pointer select-none"
+                        ? 'cursor-pointer select-none'
-                        : "",
+                        : '',
                       onClick: h.column.getToggleSortingHandler(),
                     }}
                   >
@@ -118,9 +126,9 @@ const Table = ({
         </thead>
         <tbody>
           {table.getRowModel().rows.map((r) => (
-            <tr key={r.id}>
+            <tr key={r.id} className="border-b border-b-slate-200">
               {r.getVisibleCells().map((c) => (
-                <td key={c.id}>
+                <td key={c.id} className="py-2">
                   {flexRender(c.column.columnDef.cell, c.getContext())}
                 </td>
               ))}
@@ -128,10 +136,10 @@ const Table = ({
           ))}
         </tbody>
       </table>
-      <div className="flex gap-2 items-center justify-center">
+      <div className="flex gap-2 items-center justify-center mt-10">
         <button
           className={`w-6 h-6 ${
-            !table.getCanPreviousPage() ? "opacity-25" : "opacity-100"
+            !table.getCanPreviousPage() ? 'opacity-25' : 'opacity-100'
           }`}
           onClick={() => table.setPageIndex(0)}
           disabled={!table.getCanPreviousPage()}
@@ -140,7 +148,7 @@ const Table = ({
         </button>
         <button
           className={`w-6 h-6 ${
-            !table.getCanPreviousPage() ? "opacity-25" : "opacity-100"
+            !table.getCanPreviousPage() ? 'opacity-25' : 'opacity-100'
           }`}
           onClick={() => table.previousPage()}
           disabled={!table.getCanPreviousPage()}
@@ -150,13 +158,13 @@ const Table = ({
         <span className="flex items-center gap-1">
           <div>Page</div>
           <strong>
-            {table.getState().pagination.pageIndex + 1} of{" "}
+            {table.getState().pagination.pageIndex + 1} of{' '}
             {table.getPageCount()}
           </strong>
         </span>
         <button
           className={`w-6 h-6 ${
-            !table.getCanNextPage() ? "opacity-25" : "opacity-100"
+            !table.getCanNextPage() ? 'opacity-25' : 'opacity-100'
           }`}
           onClick={() => table.nextPage()}
           disabled={!table.getCanNextPage()}
@@ -165,7 +173,7 @@ const Table = ({
         </button>
         <button
           className={`w-6 h-6 ${
-            !table.getCanNextPage() ? "opacity-25" : "opacity-100"
+            !table.getCanNextPage() ? 'opacity-25' : 'opacity-100'
           }`}
           onClick={() => table.setPageIndex(table.getPageCount() - 1)}
           disabled={!table.getCanNextPage()}
@@ -181,9 +189,7 @@ const globalFilterFn: FilterFn<any> = (row, columnId, filterValue: string) => {
   const search = filterValue.toLowerCase();
 
   let value = row.getValue(columnId) as string;
-  if (typeof value === "number") value = String(value);
+  if (typeof value === 'number') value = String(value);
 
   return value?.toLowerCase().includes(search);
 };
-
-export default Table;

packages/components/src/components/Vega.tsx

@@ -1,6 +1,6 @@
 //  Wrapper for the Vega component
 import { Vega as VegaOg } from "react-vega";
 
-export default function Vega(props) {
+export function Vega(props) {
   return <VegaOg {...props} />;
 }

packages/components/src/components/VegaLite.tsx

@@ -2,7 +2,7 @@
 import { VegaLite as VegaLiteOg } from "react-vega";
 import applyFullWidthDirective from "../lib/applyFullWidthDirective";
 
-export default function VegaLite(props) {
+export function VegaLite(props) {
   const Component = applyFullWidthDirective({ Component: VegaLiteOg });
 
   return <Component {...props} />;

packages/components/src/index.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

packages/components/src/index.ts

@@ -0,0 +1,5 @@
+export * from "./components/Table";
+export * from "./components/Catalog";
+export * from "./components/LineChart";
+export * from "./components/Vega";
+export * from "./components/VegaLite";