[site/content/guide][m]: tutorial 3 overview

Site/minor fixes

.changeset/README.md

@@ -0,0 +1,8 @@
+# Changesets
+
+Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
+with multi-package repos, or single-package repos to help you version and publish your code. You can
+find the full documentation for it [in our repository](https://github.com/changesets/changesets)
+
+We have a quick list of common questions to get you started engaging with this project in
+[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)

.changeset/config.json

@@ -0,0 +1,14 @@
+{
+  "$schema": "https://unpkg.com/@changesets/config@2.3.0/schema.json",
+  "changelog": [
+    "@changesets/changelog-github",
+    { "repo": "datopian/portaljs" }
+  ],
+  "commit": false,
+  "fixed": [],
+  "linked": [],
+  "access": "restricted",
+  "baseBranch": "main",
+  "updateInternalDependencies": "patch",
+  "ignore": []
+}

.github/workflows/release.yml

@@ -0,0 +1,39 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+
+concurrency: release-${{ github.ref }}
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Setup Node.js 16.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Create Release Pull Request or Publish to npm
+        id: changesets
+        uses: changesets/action@v1
+        with:
+          publish: npm run release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      # - name: Send a Discord notification if a publish happens
+      #   if: steps.changesets.outputs.published == 'true'
+      #   uses: Ilshidur/action-discord@0.3.2
+      #   with:
+      #     args: 'The project {{ EVENT_PAYLOAD.repository.full_name }} has been deployed.'

.gitignore

@@ -4,6 +4,7 @@
 dist
 tmp
 /out-tsc
+**/*.tgz
 
 # dependencies
 node_modules
@@ -16,6 +17,7 @@ node_modules
 *.launch
 .settings/
 *.sublime-workspace
+.obsidian
 
 # IDE - VSCode
 .vscode/*

CONTRIBUTING.md

@@ -26,10 +26,11 @@ If you'd like to work on one of the issues you can:
 3. Clone the forked repository to your machine.
 4. Create a feature branch (e.g. `50-update-readme`, where `50` is the number of the related issue).
 5. Commit your changes to the feature branch.
-6. Push the feature branch to your forked repository.
-7. Create a Pull Request against the original repository.
+6. Add changeset file describing the changes. (See section below)
+7. Push the feature branch to your forked repository.
+8. Create a Pull Request against the original repository.
    - add a short description of the changes included in the PR
-8. Address review comments if requested by our demanding reviewers 😜.
+9. Address review comments if requested by our demanding reviewers 😜.
 
 If you have an idea for improvement, and it doesn't have a corresponding issue yet, simply submit a new one.
 
@@ -62,6 +63,7 @@ or you can use just:
 nx <target> <project>
 # e.g. npx nx serve ckan
 ```
+
 if you have the `nx` binary installed globally in your machine
 
 #### Running multiple tasks
@@ -174,3 +176,23 @@ To learn more see this [offical docs page](https://nx.dev/reference/nx-json).
 Each project also has it's own configuration file - `project.json`, where you can define and configure it's targets (and more).
 
 To learn more see this [offical docs page](https://nx.dev/reference/project-configuration).
+
+## Changesets and publishing packages
+
+> This monorepo is set up with changesets versioning tool. See their [github repository](https://github.com/changesets/changesets) to learn more.
+
+### What are Changesets?
+
+Changesets are files that describe the intention of a contributor to bump a version of the package according to their changes. Changeset file holds two key bits of information: a version type (following semver), and change information to be added to a changelog.
+
+### Adding changesets
+
+In the root directory of the repo, run:
+
+```
+npx changeset
+```
+
+Select the package that has been changed, the semver version that should be bumped with it and a description of your changes. Please make sure to add the most accurate but also concise information.
+
+To learn about semantic versioning standards see [this semver doc page](https://semver.org/).

examples/ckan-example/.env

@@ -1 +0,0 @@
-DMS=https://demo.dev.datopian.com

examples/ckan-ssg/package.json

@@ -10,6 +10,8 @@
   },
   "dependencies": {
     "@heroicons/react": "^2.0.17",
+    "@portaljs/ckan": "^0.0.2",
+    "@portaljs/remark-wiki-link": "^1.0.4",
     "next": "13.3.1",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
@@ -20,14 +22,14 @@
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",
-    "autoprefixer": "^10.4.14",
-    "postcss": "^8.4.23",
-    "tailwindcss": "^3.3.1",
-    "eslint": "8.39.0",
-    "eslint-config-next": "13.3.1",
-    "typescript": "5.0.4",
     "@types/node": "18.16.0",
     "@types/react": "18.0.38",
-    "@types/react-dom": "18.0.11"
+    "@types/react-dom": "18.0.11",
+    "autoprefixer": "^10.4.14",
+    "eslint": "8.39.0",
+    "eslint-config-next": "13.3.1",
+    "postcss": "^8.4.23",
+    "tailwindcss": "^3.3.1",
+    "typescript": "5.0.4"
   }
 }

examples/ckan-ssg/pages/@org/[org]/[dataset]/index.tsx

@@ -11,8 +11,9 @@ import {
   ServerIcon,
   UserIcon,
 } from '@heroicons/react/20/solid';
+import { CKAN } from '@portaljs/ckan';
 
-const dms = getConfig().publicRuntimeConfig.DMS;
+const backend_url = getConfig().publicRuntimeConfig.DMS;
 
 const formatter = new Intl.DateTimeFormat('en-US', {
   year: 'numeric',
@@ -25,14 +26,12 @@ const formatter = new Intl.DateTimeFormat('en-US', {
 });
 
 export const getServerSideProps: GetServerSideProps = async (context) => {
+  const ckan = new CKAN(backend_url)
   const { dataset } = context.query;
-  const response = await fetch(
-    `${dms}/api/3/action/package_show?id=${dataset}`
-  );
-  const _dataset = await response.json();
+  const _dataset = await ckan.getDatasetDetails(dataset as string)
   return {
     props: {
-      dataset: _dataset.result,
+      dataset: _dataset,
     },
   };
 };

examples/ckan-ssg/pages/index.tsx

@@ -1,7 +1,8 @@
 import getConfig from 'next/config';
 import styles from './index.module.css';
+import { CKAN } from '@portaljs/ckan';
 
-const dms = getConfig().publicRuntimeConfig.DMS
+const backend_url = getConfig().publicRuntimeConfig.DMS
 
 const formatter = new Intl.DateTimeFormat('en-US', {
   year: 'numeric',
@@ -15,12 +16,11 @@ const formatter = new Intl.DateTimeFormat('en-US', {
 
 
 export async function getServerSideProps() {
-  const response = await fetch(`${dms}/api/3/action/package_search`)
-  const datasets = await response.json()
-  const datasetsWithDetails = await Promise.all(datasets.result.results.map(async (dataset) => {
-    const response = await fetch(`${dms}/api/3/action/package_show?id=` + dataset.name)
-    const json = await response.json()
-    return json.result
+  const ckan = new CKAN(backend_url)
+  const { datasets } = await ckan.packageSearch({ limit: 1000, offset: 0, groups:[], orgs: [], tags: []})
+  const datasetsWithDetails = await Promise.all(datasets.map(async (dataset) => {
+    const _dataset = await ckan.getDatasetDetails(dataset.name)
+    return _dataset
   }))
 
   return {
@@ -79,7 +79,7 @@ export function Index({ datasets }) {
                 </thead>
                 <tbody className="divide-y divide-gray-200">
                   {datasets.map((dataset) => (
-                    <tr>
+                    <tr key={dataset.name}>
                       <td className="px-3 py-4 text-sm text-gray-500">
                           {dataset.title}
                       </td>

examples/ckan/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

examples/ckan/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/ckan/README.md

@@ -0,0 +1,38 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file.
+
+[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
+
+The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

examples/ckan/components/DataRichDocument.tsx

@@ -1,6 +1,6 @@
 import { MDXRemote } from 'next-mdx-remote';
 import dynamic from 'next/dynamic';
-import { Mermaid } from '@flowershow/core';
+import { Mermaid } from '@portaljs/core';
 
 // Custom components/renderers to pass to MDX.
 // Since the MDX files aren't loaded by webpack, they have no knowledge of how
@@ -9,6 +9,7 @@ import { Mermaid } from '@flowershow/core';
 const components = {
   Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
   Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
+  FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
   mermaid: Mermaid,
   Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
   VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),

examples/ckan/content/test.md

@@ -0,0 +1,3 @@
+# Test
+
+Test Data Rich Stories

examples/ckan/lib/markdown.js

@@ -1,13 +1,13 @@
 import matter from "gray-matter";
 import mdxmermaid from "mdx-mermaid";
 import { h } from "hastscript";
-import remarkCallouts from "@flowershow/remark-callouts";
-import remarkEmbed from "@flowershow/remark-embed";
+import remarkCallouts from "@portaljs/remark-callouts";
+import remarkEmbed from "@portaljs/remark-embed";
 import remarkGfm from "remark-gfm";
 import remarkMath from "remark-math";
 import remarkSmartypants from "remark-smartypants";
 import remarkToc from "remark-toc";
-import remarkWikiLink from "@flowershow/remark-wiki-link";
+import remarkWikiLink from "@portaljs/remark-wiki-link";
 import rehypeAutolinkHeadings from "rehype-autolink-headings";
 import rehypeKatex from "rehype-katex";
 import rehypeSlug from "rehype-slug";

examples/ckan/lib/mddb.ts

@@ -1,4 +1,4 @@
-import { MarkdownDB } from "@flowershow/markdowndb";
+import { MarkdownDB } from "mddb";
 
 const dbPath = "markdown.db";
 

examples/ckan/next.config.js

@@ -0,0 +1,11 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  reactStrictMode: true,
+  publicRuntimeConfig: {
+    DMS: process.env.DMS
+      ? process.env.DMS.replace(/\/?$/, '')
+      : 'https://demo.dev.datopian.com/',
+  },
+};
+
+module.exports = nextConfig;

examples/ckan/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "ckan",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "prebuild": "npm run mddb",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint",
+    "mddb": "mddb ./content"
+  },
+  "dependencies": {
+    "@githubocto/flat-ui": "^0.14.1",
+    "@heroicons/react": "^2.0.18",
+    "@portaljs/ckan": "^0.0.2",
+    "@portaljs/components": "0.1.6",
+    "@portaljs/core": "^1.0.5",
+    "@portaljs/remark-callouts": "^1.0.5",
+    "@portaljs/remark-embed": "^1.0.4",
+    "@portaljs/remark-wiki-link": "^1.0.4",
+    "@tailwindcss/typography": "^0.5.9",
+    "@types/node": "20.2.3",
+    "@types/react": "18.2.6",
+    "@types/react-dom": "18.2.4",
+    "autoprefixer": "10.4.14",
+    "eslint": "8.41.0",
+    "eslint-config-next": "13.4.3",
+    "isomorphic-unfetch": "^4.0.2",
+    "mddb": "^0.1.9",
+    "next": "13.4.3",
+    "next-mdx-remote": "^4.4.1",
+    "papaparse": "^5.4.1",
+    "postcss": "8.4.23",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-query": "^3.39.3",
+    "rehype-autolink-headings": "^6.1.1",
+    "rehype-katex": "^6.0.3",
+    "rehype-prism-plus": "^1.5.1",
+    "rehype-slug": "^5.1.0",
+    "remark-math": "^5.1.1",
+    "remark-smartypants": "^2.0.0",
+    "remark-toc": "^8.0.1",
+    "tailwindcss": "3.3.2",
+    "typescript": "5.0.4"
+  }
+}

examples/ckan/pages/[org]/[dataset]/index.tsx

@@ -0,0 +1,179 @@
+import Head from "next/head";
+import { CKAN, Dataset } from "@portaljs/ckan";
+import {
+  ChevronRightIcon,
+  HomeIcon,
+  PaperClipIcon,
+} from "@heroicons/react/20/solid";
+import Link from "next/link";
+import getConfig from "next/config";
+
+const backend_url = getConfig().publicRuntimeConfig.DMS
+
+export const getServerSideProps = async (context: any) => {
+  try {
+    const datasetName = context.params?.dataset;
+    if (!datasetName) {
+      return {
+        notFound: true,
+      };
+    }
+    const ckan = new CKAN(backend_url);
+    const dataset = await ckan.getDatasetDetails(datasetName as string);
+    if (!dataset) {
+      return {
+        notFound: true,
+      };
+    }
+    return {
+      props: { dataset },
+    };
+  } catch {
+    return {
+      notFound: true,
+    };
+  }
+};
+
+export default function DatasetPage({
+  dataset,
+}: {
+  dataset: Dataset;
+}): JSX.Element {
+  return (
+    <>
+      <Head>
+        <title>{`${dataset.title || dataset.name} - Dataset`}</title>
+        <meta name="description" content="Generated by create next app" />
+        <link rel="icon" href="/favicon.ico" />
+      </Head>
+      <main className="flex min-h-screen flex-col items-center justify-between p-24 bg-zinc-900">
+        <div className="bg-white p-8 my-4 rounded-lg">
+          <nav className="flex px-4 py-8" aria-label="Breadcrumb">
+            <ol role="list" className="flex items-center space-x-4">
+              <li>
+                <div>
+                  <Link href="/" className="text-gray-400 hover:text-gray-500">
+                    <HomeIcon
+                      className="h-5 w-5 flex-shrink-0"
+                      aria-hidden="true"
+                    />
+                    <span className="sr-only">Home</span>
+                  </Link>
+                </div>
+              </li>
+              <li>
+                <div className="flex items-center">
+                  <ChevronRightIcon
+                    className="h-5 w-5 flex-shrink-0 text-gray-400"
+                    aria-hidden="true"
+                  />
+                  <span
+                    className="ml-4 text-sm font-medium text-gray-500 hover:text-gray-700"
+                    aria-current={"page"}
+                  >
+                    {dataset.name}
+                  </span>
+                </div>
+              </li>
+            </ol>
+          </nav>
+          {dataset && (
+            <div>
+              <div className="px-4 sm:px-0">
+                <h3 className="text-base font-semibold leading-7 text-gray-900">
+                  {dataset.title || dataset.name}
+                </h3>
+                <p className="mt-1 max-w-2xl text-sm leading-6 text-gray-500">
+                  Dataset details
+                </p>
+              </div>
+              <div className="mt-6 border-t border-gray-100">
+                <dl className="divide-y divide-gray-100">
+                  <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                    <dt className="text-sm font-medium leading-6 text-gray-900">
+                      Title
+                    </dt>
+                    <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                      {dataset.title}
+                    </dd>
+                  </div>
+                  {dataset.tags && dataset.tags.length > 0 && (
+                    <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                      <dt className="text-sm font-medium leading-6 text-gray-900">
+                        Tags
+                      </dt>
+                      <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                        {dataset.tags.map((tag) => tag.display_name).join(", ")}
+                      </dd>
+                    </div>
+                  )}
+                  {dataset.tags && dataset.tags.length > 0 && (
+                    <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                      <dt className="text-sm font-medium leading-6 text-gray-900">
+                        URL
+                      </dt>
+                      <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                        {dataset.url}
+                      </dd>
+                    </div>
+                  )}
+                  <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                    {dataset.notes && (
+                      <>
+                        <dt className="text-sm font-medium leading-6 text-gray-900">
+                          Description
+                        </dt>
+                        <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                          {dataset.notes}
+                        </dd>
+                      </>
+                    )}
+                  </div>
+                  <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                    <dt className="text-sm font-medium leading-6 text-gray-900">
+                      Files
+                    </dt>
+                    <dd className="mt-2 text-sm text-gray-900 sm:col-span-2 sm:mt-0">
+                      <ul
+                        role="list"
+                        className="divide-y divide-gray-100 rounded-md border border-gray-200"
+                      >
+                        {dataset.resources.map((resource) => (
+                          <li key={resource.id} className="flex items-center justify-between py-4 pl-4 pr-5 text-sm leading-6">
+                            <div className="flex w-0 flex-1 items-center">
+                              <PaperClipIcon
+                                className="h-5 w-5 flex-shrink-0 text-gray-400"
+                                aria-hidden="true"
+                              />
+                              <div className="ml-4 flex min-w-0 flex-1 gap-2">
+                                <span className="truncate font-medium">
+                                  {resource.name || resource.id}
+                                </span>
+                                <span className="flex-shrink-0 text-gray-400">
+                                  {resource.size}
+                                </span>
+                              </div>
+                            </div>
+                            <div className="ml-4 flex-shrink-0">
+                              <a
+                                href={resource.url}
+                                className="font-medium hover:text-indigo-500 mr-4"
+                              >
+                                Download
+                              </a>
+                            </div>
+                          </li>
+                        ))}
+                      </ul>
+                    </dd>
+                  </div>
+                </dl>
+              </div>
+            </div>
+          )}
+        </div>
+      </main>
+    </>
+  );
+}

examples/ckan/pages/_app.tsx

@@ -0,0 +1,7 @@
+import '@/styles/globals.css'
+import '@portaljs/ckan/styles.css'
+import type { AppProps } from 'next/app'
+
+export default function App({ Component, pageProps }: AppProps) {
+  return <Component {...pageProps} />
+}

examples/ckan/pages/_document.tsx

@@ -0,0 +1,13 @@
+import { Html, Head, Main, NextScript } from 'next/document'
+
+export default function Document() {
+  return (
+    <Html lang="en">
+      <Head />
+      <body>
+        <Main />
+        <NextScript />
+      </body>
+    </Html>
+  )
+}

examples/ckan/pages/api/cors.ts

@@ -0,0 +1,18 @@
+import fetch from 'isomorphic-unfetch';
+
+const Cors = async (req: any, res: any) => {
+  const { url } = req.query;
+  try {
+    const resProxy = await fetch(url, {
+      headers: {
+        Range: 'bytes=0-5132288',
+      },
+    });
+    const data = await resProxy.text();
+    return res.status(200).send(data);
+  } catch (error: any) {
+    res.status(400).send(error.toString());
+  }
+};
+
+export default Cors;

examples/ckan/pages/index.tsx

@@ -0,0 +1,64 @@
+import {
+  CKAN,
+  DatasetSearchForm,
+  ListOfDatasets,
+  PackageSearchOptions,
+  Organization,
+  Group,
+} from '@portaljs/ckan';
+import getConfig from 'next/config';
+import { useState } from 'react';
+
+const backend_url = getConfig().publicRuntimeConfig.DMS;
+
+export async function getServerSideProps() {
+  const ckan = new CKAN(backend_url);
+  const groups = await ckan.getGroupsWithDetails();
+  const orgs = await ckan.getOrgsWithDetails();
+  return {
+    props: {
+      groups,
+      orgs,
+    },
+  };
+}
+
+export default function Home({
+  orgs,
+  groups,
+}: {
+  orgs: Organization[];
+  groups: Group[];
+}) {
+  const ckan = new CKAN(backend_url);
+  const [options, setOptions] = useState<PackageSearchOptions>({
+    offset: 0,
+    limit: 5,
+    tags: [],
+    groups: [],
+    orgs: [],
+  });
+
+  return (
+    <div>
+      <main className="py-12 bg-zinc-900">
+        <DatasetSearchForm
+          options={options}
+          setOptions={setOptions}
+          groups={groups}
+          orgs={orgs}
+        />
+        <div
+          className="bg-white p-8 mx-auto my-4 rounded-lg"
+          style={{ width: 'min(1100px, 95vw)' }}
+        >
+          <ListOfDatasets
+            options={options}
+            setOptions={setOptions}
+            ckan={ckan}
+          />{' '}
+        </div>
+      </main>
+    </div>
+  );
+}

examples/ckan/pages/stories/[[...path]].tsx

@@ -0,0 +1,129 @@
+import { existsSync, promises as fs } from 'fs';
+import path from 'path';
+import parse from '../../lib/markdown';
+
+import DataRichDocument from '../../components/DataRichDocument';
+import clientPromise from '../../lib/mddb';
+import getConfig from 'next/config';
+import { CKAN } from '@portaljs/ckan';
+
+export const getStaticPaths = async () => {
+  const contentDir = path.join(process.cwd(), '/content/');
+  const contentFolders = await fs.readdir(contentDir, 'utf8');
+  const paths = contentFolders.map((folder: string) => ({
+    params: { path: [folder.split('.')[0]] },
+  }));
+  return {
+    paths,
+    fallback: false,
+  };
+};
+
+const backend_url = getConfig().publicRuntimeConfig.DMS;
+
+export const getStaticProps = async (context) => {
+  const mddb = await clientPromise;
+  const storyFile = await mddb.getFileByUrl(context.params.path);
+  const md = await fs.readFile(
+    `${process.cwd()}/${storyFile.file_path}`,
+    'utf8'
+  );
+
+  const ckan = new CKAN(backend_url);
+  const datasets = storyFile.metadata.datasets ? await Promise.all(
+    storyFile.metadata.datasets.map(
+      async (datasetName: string) => await ckan.getDatasetDetails(datasetName)
+    )
+  ) : [];
+  const orgs = storyFile.metadata.orgs ? await Promise.all(
+    storyFile.metadata.orgs.map(
+      async (orgName: string) => await ckan.getOrgDetails(orgName)
+    )
+  ) : [];
+
+  let { mdxSource, frontMatter } = await parse(md, '.mdx', { datasets, orgs });
+
+  return {
+    props: {
+      mdxSource,
+      frontMatter: JSON.stringify(frontMatter),
+    },
+  };
+};
+
+export default function DatasetPage({ mdxSource, frontMatter }) {
+  frontMatter = JSON.parse(frontMatter);
+  return (
+    <main className="flex min-h-screen flex-col justify-between p-16 bg-zinc-900">
+      <div className="bg-white p-8 my-4 rounded-lg">
+        <div className="prose mx-auto py-8">
+          <header>
+            <div className="mb-6">
+              <>
+                <h1 className="mb-2">{frontMatter.title}</h1>
+                {frontMatter.author && (
+                  <p className="my-0">
+                    <span className="font-semibold">Author: </span>
+                    <span className="my-0">{frontMatter.author}</span>
+                  </p>
+                )}
+                {frontMatter.description && (
+                  <p className="my-0">
+                    <span className="font-semibold">Description: </span>
+                    <span className="description my-0">
+                      {frontMatter.description}
+                    </span>
+                  </p>
+                )}
+                {frontMatter.modified && (
+                  <p className="my-0">
+                    <span className="font-semibold">Modified: </span>
+                    <span className="description my-0">
+                      {new Date(frontMatter.modified).toLocaleDateString()}
+                    </span>
+                  </p>
+                )}
+                {frontMatter.files && (
+                  <section className="py-6">
+                    <h2 className="mt-0">Data files</h2>
+                    <table className="table-auto">
+                      <thead>
+                        <tr>
+                          <th>File</th>
+                          <th>Format</th>
+                        </tr>
+                      </thead>
+                      <tbody>
+                        {frontMatter.files.map((f) => {
+                          const fileName = f.split('/').slice(-1);
+                          return (
+                            <tr key={`resources-list-${f}`}>
+                              <td>
+                                <a target="_blank" href={f}>
+                                  {fileName}
+                                </a>
+                              </td>
+                              <td>
+                                {fileName[0]
+                                  .split('.')
+                                  .slice(-1)[0]
+                                  .toUpperCase()}
+                              </td>
+                            </tr>
+                          );
+                        })}
+                      </tbody>
+                    </table>
+                  </section>
+                )}
+              </>
+            </div>
+          </header>
+          <main>
+            <DataRichDocument source={mdxSource} />
+          </main>
+        </div>
+      </div>
+    </main>
+  );
+}

examples/ckan/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

examples/ckan/public/vercel.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

examples/ckan/styles/globals.css

@@ -0,0 +1,71 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+@import "@portaljs/remark-callouts/styles.css";
+
+/* mathjax */
+.math-inline > mjx-container > svg {
+  display: inline;
+  align-items: center;
+}
+
+/* smooth scrolling in modern browsers */
+html {
+  scroll-behavior: smooth !important;
+}
+
+/* tooltip fade-out clip */
+.tooltip-body::after {
+  content: "";
+  position: absolute;
+  right: 0;
+  top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
+  height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
+  width: 10rem;
+  background: linear-gradient(
+    to right,
+    rgba(255, 255, 255, 0),
+    rgba(255, 255, 255, 1) 100%
+  );
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
+  margin-left: -2rem !important;
+  padding-left: 2rem !important;
+  scroll-margin-top: 4.5rem;
+  position: relative;
+}
+
+.heading-link {
+  padding: 1px;
+  position: absolute;
+  left: 0;
+  top: 50%;
+  transform: translateY(-50%);
+  margin: auto 0;
+  border-radius: 5px;
+  background: #1e293b;
+  opacity: 0;
+  transition: opacity 0.2s;
+}
+
+.light .heading-link {
+  /* border: 1px solid #ab2b65; */
+  /* background: none; */
+  background: #e2e8f0;
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
+  opacity: 100;
+}
+
+.heading-link svg {
+  transform: scale(0.75);
+}
+
+@media screen and (max-width: 640px) {
+  .heading-link {
+    visibility: hidden;
+  }
+}

examples/ckan/tailwind.config.js

@@ -0,0 +1,18 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    './pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './components/**/*.{js,ts,jsx,tsx,mdx}',
+    './app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      backgroundImage: {
+        'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
+        'gradient-conic':
+          'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
+      },
+    },
+  },
+  plugins: [require('@tailwindcss/typography')],
+}

examples/ckan/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": false,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+  "exclude": ["node_modules"]
+}

examples/fivethirtyeight/components/Layout.tsx

@@ -0,0 +1,99 @@
+import { useState } from 'react';
+import { XMarkIcon } from '@heroicons/react/20/solid';
+import { Transition } from '@headlessui/react';
+
+export default function Layout({ children }: { children: React.ReactNode }) {
+  const [isShowing, setShow] = useState(true);
+  return (
+    <>
+      <Transition
+        show={isShowing}
+        enter="transition-opacity duration-75"
+        enterFrom="opacity-0"
+        enterTo="opacity-100"
+        leave="transition-opacity duration-150"
+        leaveFrom="opacity-100"
+        leaveTo="opacity-0"
+      >
+        <div className="flex items-center gap-x-6 bg-[#3c3c3c] px-6 py-2.5 sm:px-3.5 sm:before:flex-1">
+          <p className="text-sm leading-6 text-white">
+            This is a replica to the awesome{' '}
+            <a
+              className="hover:underline font-bold"
+              href="https://data.fivethirtyeight.com"
+            >
+              data.fivethirtyeight.com
+            </a>{' '}
+            website.{' '}
+            <a
+              className="hover:underline font-bold"
+              href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight#readme"
+            >
+              Read more here
+            </a>{' '}
+          </p>
+          <div className="flex flex-1 justify-end">
+            <button
+              type="button"
+              onClick={() => setShow(false)}
+              className="-m-3 p-3 focus-visible:outline-offset-[-4px]"
+            >
+              <span className="sr-only">Dismiss</span>
+              <XMarkIcon className="h-5 w-5 text-white" aria-hidden="true" />
+            </button>
+          </div>
+        </div>
+      </Transition>
+      <header className="max-w-5xl mx-auto mt-8 w-full">
+        <div className="border-b-2 pb-2.5 mx-2 border-zinc-800 flex justify-between">
+          <h1 className="flex gap-x-1 items-end">
+            <span className="sr-only">FiveThirtyEight</span>
+            <img
+              width="197"
+              height="25"
+              alt="FiveThirtyEight"
+              src="data:image/svg+xml;base64,PHN2ZyBpZD0iTGF5ZXJfMSIgZGF0YS1uYW1lPSJMYXllciAxIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjEgNTMuNzYiPjxkZWZzPjxzdHlsZT4uY2xzLTF7ZmlsbDojMDEwMTAxO308L3N0eWxlPjwvZGVmcz48dGl0bGU+QXJ0Ym9hcmQgOTU8L3RpdGxlPjxwYXRoIGNsYXNzPSJjbHMtMSIgZD0iTTAgMGgyNXY4SDl2MTBoMTV2OEg5djE3SDBWMHpNMzEgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdIMzF6bTUtMzZoOHY4aC04ek0xNzkgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdoLTE3em01LTM2aDh2OGgtOHpNMzE2IDM2aDVWMThoLTV2LThoMTN2MjZoNHY3aC0xN3ptNS0zNmg4djhoLTh6TTU0IDI3VjEwaDh2MTVsNCA5Ljk4aDFMNzEgMjVWMTBoOHYxN2wtNyAxNkg2MWwtNy0xNnpNMTExIDQzSDk3LjQyQzg5LjIzIDQzIDg1IDM5LjE5IDg1IDMxLjE3VjIyYzAtNy41NyA0LjMtMTMgMTMtMTMgOS4zMyAwIDEzIDUuMDcgMTMgMTR2N0g5NHYxLjc0YzAgMi42MiAxIDQuMjYgMy40MiA0LjI2SDExMXpNOTQgMjNoOHYtMS41NWMwLTIuNjItMS4wNi01LjQ1LTQuMTMtNS40NS0yLjc5IDAtMy44NyAyLjItMy44NyA1LjQ1ek0xMjUgOGgtMTBWMGgyOXY4aC0xMHYzNWgtOVY4ek0yMDIgNDNWMTBoOHY0YzEuMTQtMi40NSAzLjc1LTQgNy4yMi00SDIyMHY4aC02Yy0yLjg0IDAtNCAuOTQtNCAzLjlWNDN6TTI0NSA0M2gtNC44NEMyMzMuMDUgNDMgMjMwIDM5LjMxIDIzMCAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0gyNDV6TTQyMSA0M2gtNC44NEM0MDkuMDUgNDMgNDA2IDM5LjMxIDQwNiAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0g0MjF6TTI1NC4yNiA1My43Nmw0LjYxLTkuNUwyNTEgMjdWMTBoOHYxNWw0IDEwaDFsNC0xMFYxMGg4djE3bC0xMi4zIDI2Ljc2aC05LjQ0ek0yODQgMGgyNXY4aC0xNnY5aDE1djhoLTE1djEwaDE2djhoLTI1VjB6TTMzNyA0OHYtMmgxNi4xYzIgMCAyLjktLjE4IDIuOS0xLjI3di0uMzRjMC0xLjA4LS45MS0xLjM5LTIuOS0xLjM5SDM0MHYtNWw1LTVjLTUuMjktMS40OC04LTUuNDMtOC0xMXYtMWMwLTcuNTYgNC40NC0xMiAxNC0xMmEyMS45MyAyMS45MyAwIDAgMSA1Ljk1IDFMMzYxIDRsNSAzLTQgNmMxLjM3IDEuOTMgMyA0LjkzIDMgOHYxYzAgNy0zLjMgMTAuNjYtMTIgMTFsLTMgNGg2YzUuOTIgMCA5IDIuNjIgOSA3LjY4di4xMWMwIDUuMDYtMi43MSA4LjIxLTguNjIgOC4yMWgtMTNjLTQuMjkgMC02LjM4LTEuODQtNi4zOC01em0xOS0yNXYtM2MwLTMuMy0xLjMzLTQtNS00cy01IC43LTUgNHYzYzAgMy4zIDEuMzkgNCA1IDRzNS0uNyA1LTR6TTM4MCA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuNC00IDctNCA2LjI2IDAgOSAzLjA4IDkgMTAuNzZWNDNoLThWMjJjMC0zLjEzLTEuMDctNS00LTVzLTQgMS44Ny00IDV6TTE1NyA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuOTEtNCA3LjQ5LTQgNi4yNiAwIDguNTEgMy4xMyA4LjUxIDEwLjgxVjQzaC04VjIxYzAtMy4xMy0xLjA3LTQuNDQtNC00LjQ0cy00IDIuMjYtNCA1LjM5eiIvPjwvc3ZnPg=="
+            />{' '}
+            <span className="-mb-0.5 text-[#3c3c3c]">replica</span>
+          </h1>
+          <div className="md:flex items-center gap-x-3 text-[#3c3c3c] -mb-1 hidden">
+            <a
+              className="hover:opacity-75 transition"
+              href="https://portaljs.org"
+            >
+              Built with 🌀PortalJS
+            </a>
+            <hr className="h-[80%] border border-[#3c3c3c] opacity-75 my-2"></hr>
+            <a
+              className="hover:opacity-75 transition"
+              href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
+            >
+              Github
+            </a>
+          </div>
+        </div>
+        <div className="mx-2 py-1.5 text-[14px] text-[#3c3c3c] md:hidden">
+          <ul className="flex gap-x-4">
+            <li>
+              <a
+                className="hover:opacity-75 transition"
+                href="https://portaljs.org"
+              >
+                PortalJS
+              </a>
+            </li>
+            <li>
+              <a
+                className="hover:opacity-75 transition"
+                href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
+              >
+                View on Github
+              </a>
+            </li>
+          </ul>
+        </div>
+      </header>
+      {children}
+    </>
+  );
+}

examples/fivethirtyeight/datasets.json

@@ -1,33 +1,4 @@
 [
-  {
-    "url": "https://github.com/fivethirtyeight/data/tree/master/polls",
-    "name": "polls",
-    "displayName": "<span class=\"lastword\">polls</span>",
-    "articles": [
-      {
-        "date": "2023-05-11T14:35:40.000Z",
-        "title": "Latest Polls",
-        "url": "https://projects.fivethirtyeight.com/polls/"
-      }
-    ],
-    "files": [
-      "https://projects.fivethirtyeight.com/polls-page/data/president_primary_polls.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/president_primary_polls_historical.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/president_polls.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/president_polls_historical.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/senate_polls.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/senate_polls_historical.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/house_polls.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/house_polls_historical.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/governor_polls.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/governor_polls_historical.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/president_approval_polls.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/generic_ballot_polls.csv",
-      "https://projects.fivethirtyeight.com/polls-page/data/generic_ballot_polls_historical.csv",
-      "https://projects.fivethirtyeight.com/2020-primary-data/pres_primary_avgs_2020.csv",
-      "https://projects.fivethirtyeight.com/2020-general-data/presidential_poll_averages_2020.csv"
-    ]
-  },
   {
     "url": "https://github.com/fivethirtyeight/data/tree/master/congress-generic-ballot",
     "name": "congress-generic-ballot",
@@ -195,6 +166,35 @@
       }
     ]
   },
+  {
+    "url": "https://github.com/fivethirtyeight/data/tree/master/polls",
+    "name": "polls",
+    "displayName": "<span class=\"lastword\">polls</span>",
+    "articles": [
+      {
+        "date": "2023-05-11T14:35:40.000Z",
+        "title": "Latest Polls",
+        "url": "https://projects.fivethirtyeight.com/polls/"
+      }
+    ],
+    "files": [
+      "https://projects.fivethirtyeight.com/polls-page/data/president_primary_polls.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/president_primary_polls_historical.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/president_polls.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/president_polls_historical.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/senate_polls.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/senate_polls_historical.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/house_polls.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/house_polls_historical.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/governor_polls.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/governor_polls_historical.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/president_approval_polls.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/generic_ballot_polls.csv",
+      "https://projects.fivethirtyeight.com/polls-page/data/generic_ballot_polls_historical.csv",
+      "https://projects.fivethirtyeight.com/2020-primary-data/pres_primary_avgs_2020.csv",
+      "https://projects.fivethirtyeight.com/2020-general-data/presidential_poll_averages_2020.csv"
+    ]
+  },
   {
     "url": "https://github.com/fivethirtyeight/data/tree/master/nfl-elo",
     "name": "nfl-elo",
@@ -1169,18 +1169,6 @@
       }
     ]
   },
-  {
-    "url": "https://github.com/fivethirtyeight/data/tree/master/undefeated-boxers",
-    "name": "undefeated-boxers",
-    "displayName": "undefeated-<span class=\"lastword\">boxers</span>",
-    "articles": [
-      {
-        "date": "2017-08-18T18:47:32.000Z",
-        "title": "Mayweather Is Defined By The Zero Next To His Name",
-        "url": "https://fivethirtyeight.com/features/mayweather-is-defined-by-the-zero-next-to-his-name/"
-      }
-    ]
-  },
   {
     "url": "https://github.com/fivethirtyeight/data/tree/master/chess-transfers",
     "name": "chess-transfers",
@@ -2139,6 +2127,18 @@
       }
     ]
   },
+  {
+    "url": "https://github.com/fivethirtyeight/data/tree/master/undefeated-boxers",
+    "name": "undefeated-boxers",
+    "displayName": "undefeated-<span class=\"lastword\">boxers</span>",
+    "articles": [
+      {
+        "date": "2017-08-18T18:47:32.000Z",
+        "title": "Mayweather Is Defined By The Zero Next To His Name",
+        "url": "https://fivethirtyeight.com/features/mayweather-is-defined-by-the-zero-next-to-his-name/"
+      }
+    ]
+  },
   {
     "url": "https://github.com/fivethirtyeight/data/tree/master/march-madness-predictions",
     "name": "march-madness-predictions",

examples/fivethirtyeight/package.json

@@ -9,7 +9,11 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@portaljs/components": "^0.1.0",
+    "@headlessui/react": "^1.7.14",
+    "@heroicons/react": "^2.0.18",
+    "@portaljs/components": "^0.1.8",
+    "@portaljs/core": "^1.0.5",
+    "@portaljs/remark-wiki-link": "^1.0.4",
     "@tailwindcss/typography": "^0.5.9",
     "@types/node": "20.1.1",
     "@types/react": "18.2.6",
@@ -26,12 +30,15 @@
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-markdown": "^8.0.7",
+    "remark": "^14.0.3",
     "remark-code-frontmatter": "^1.0.0",
+    "remark-excerpt": "^1.0.0-beta.1",
     "remark-extract-frontmatter": "^3.2.0",
     "remark-frontmatter": "^4.0.1",
     "remark-gfm": "^3.0.1",
     "tailwindcss": "3.3.2",
     "timeago.js": "^4.0.2",
+    "to-vfile": "^7.2.4",
     "typescript": "5.0.4"
   }
 }

examples/fivethirtyeight/pages/_app.tsx

@@ -1,8 +1,49 @@
-import '@/styles/globals.css'
-import '@portaljs/components/styles.css'
+import '@/styles/globals.css';
+import '@portaljs/components/styles.css';
+import { useEffect } from 'react';
+import { pageview } from '@portaljs/core';
+import Script from 'next/script';
+import Head from 'next/head';
+import { useRouter } from 'next/router';
 
-import type { AppProps } from 'next/app'
+import type { AppProps } from 'next/app';
 
 export default function App({ Component, pageProps }: AppProps) {
-  return <Component {...pageProps} />
+  const router = useRouter();
+
+  useEffect(() => {
+    const handleRouteChange = (url: any) => {
+      pageview(url);
+    };
+    router.events.on('routeChangeComplete', handleRouteChange);
+    return () => {
+      router.events.off('routeChangeComplete', handleRouteChange);
+    };
+  }, [router.events]);
+  return (
+    <>
+      <Head>
+        <link rel="shortcut icon" href="/squared_logo.png" />
+      </Head>
+      <Script
+        strategy="afterInteractive"
+        src="https://www.googletagmanager.com/gtag/js?id=G-3N9SXTC7GS"
+      />
+      <Script
+        id="gtag-init"
+        strategy="afterInteractive"
+        dangerouslySetInnerHTML={{
+          __html: `
+          window.dataLayer = window.dataLayer || [];
+          function gtag(){dataLayer.push(arguments);}
+          gtag('js', new Date());
+          gtag('config', 'G-3N9SXTC7GS', {
+            page_path: window.location.pathname,
+          });
+        `,
+        }}
+      />
+      <Component {...pageProps} />
+    </>
+  );
 }

examples/fivethirtyeight/pages/_document.tsx

@@ -19,78 +19,9 @@ export default function Document() {
         />
       </Head>
       <body>
-        <div className="px-2 max-w-5xl mx-auto pb-2">
-          <div className="mt-2 px-2 bg-[#3c3c3c] text-white">
-            <div className="p-2 text-center">
-              This is a replica to the awesome{' '}
-              <a
-                className="hover:underline font-bold"
-                href="https://data.fivethirtyeight.com"
-              >
-                data.fivethirtyeight.com
-              </a>{' '}
-              website.{' '}
-              <a
-                className="hover:underline font-bold"
-                href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
-              >
-                Read more here
-              </a>{' '}
-            </div>
-          </div>
-        </div>
-        <header className="max-w-5xl mx-auto mt-8 w-full">
-          <div className="border-b-2 pb-2.5 mx-2 border-zinc-800 flex justify-between">
-            <h1 className="flex gap-x-1 items-end">
-              <span className="sr-only">FiveThirtyEight</span>
-              <img
-                width="197"
-                height="25"
-                alt="FiveThirtyEight"
-                src="data:image/svg+xml;base64,PHN2ZyBpZD0iTGF5ZXJfMSIgZGF0YS1uYW1lPSJMYXllciAxIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjEgNTMuNzYiPjxkZWZzPjxzdHlsZT4uY2xzLTF7ZmlsbDojMDEwMTAxO308L3N0eWxlPjwvZGVmcz48dGl0bGU+QXJ0Ym9hcmQgOTU8L3RpdGxlPjxwYXRoIGNsYXNzPSJjbHMtMSIgZD0iTTAgMGgyNXY4SDl2MTBoMTV2OEg5djE3SDBWMHpNMzEgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdIMzF6bTUtMzZoOHY4aC04ek0xNzkgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdoLTE3em01LTM2aDh2OGgtOHpNMzE2IDM2aDVWMThoLTV2LThoMTN2MjZoNHY3aC0xN3ptNS0zNmg4djhoLTh6TTU0IDI3VjEwaDh2MTVsNCA5Ljk4aDFMNzEgMjVWMTBoOHYxN2wtNyAxNkg2MWwtNy0xNnpNMTExIDQzSDk3LjQyQzg5LjIzIDQzIDg1IDM5LjE5IDg1IDMxLjE3VjIyYzAtNy41NyA0LjMtMTMgMTMtMTMgOS4zMyAwIDEzIDUuMDcgMTMgMTR2N0g5NHYxLjc0YzAgMi42MiAxIDQuMjYgMy40MiA0LjI2SDExMXpNOTQgMjNoOHYtMS41NWMwLTIuNjItMS4wNi01LjQ1LTQuMTMtNS40NS0yLjc5IDAtMy44NyAyLjItMy44NyA1LjQ1ek0xMjUgOGgtMTBWMGgyOXY4aC0xMHYzNWgtOVY4ek0yMDIgNDNWMTBoOHY0YzEuMTQtMi40NSAzLjc1LTQgNy4yMi00SDIyMHY4aC02Yy0yLjg0IDAtNCAuOTQtNCAzLjlWNDN6TTI0NSA0M2gtNC44NEMyMzMuMDUgNDMgMjMwIDM5LjMxIDIzMCAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0gyNDV6TTQyMSA0M2gtNC44NEM0MDkuMDUgNDMgNDA2IDM5LjMxIDQwNiAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0g0MjF6TTI1NC4yNiA1My43Nmw0LjYxLTkuNUwyNTEgMjdWMTBoOHYxNWw0IDEwaDFsNC0xMFYxMGg4djE3bC0xMi4zIDI2Ljc2aC05LjQ0ek0yODQgMGgyNXY4aC0xNnY5aDE1djhoLTE1djEwaDE2djhoLTI1VjB6TTMzNyA0OHYtMmgxNi4xYzIgMCAyLjktLjE4IDIuOS0xLjI3di0uMzRjMC0xLjA4LS45MS0xLjM5LTIuOS0xLjM5SDM0MHYtNWw1LTVjLTUuMjktMS40OC04LTUuNDMtOC0xMXYtMWMwLTcuNTYgNC40NC0xMiAxNC0xMmEyMS45MyAyMS45MyAwIDAgMSA1Ljk1IDFMMzYxIDRsNSAzLTQgNmMxLjM3IDEuOTMgMyA0LjkzIDMgOHYxYzAgNy0zLjMgMTAuNjYtMTIgMTFsLTMgNGg2YzUuOTIgMCA5IDIuNjIgOSA3LjY4di4xMWMwIDUuMDYtMi43MSA4LjIxLTguNjIgOC4yMWgtMTNjLTQuMjkgMC02LjM4LTEuODQtNi4zOC01em0xOS0yNXYtM2MwLTMuMy0xLjMzLTQtNS00cy01IC43LTUgNHYzYzAgMy4zIDEuMzkgNCA1IDRzNS0uNyA1LTR6TTM4MCA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuNC00IDctNCA2LjI2IDAgOSAzLjA4IDkgMTAuNzZWNDNoLThWMjJjMC0zLjEzLTEuMDctNS00LTVzLTQgMS44Ny00IDV6TTE1NyA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuOTEtNCA3LjQ5LTQgNi4yNiAwIDguNTEgMy4xMyA4LjUxIDEwLjgxVjQzaC04VjIxYzAtMy4xMy0xLjA3LTQuNDQtNC00LjQ0cy00IDIuMjYtNCA1LjM5eiIvPjwvc3ZnPg=="
-              />{' '}
-              <span className="-mb-0.5 text-[#3c3c3c]">replica</span>
-            </h1>
-            <div className="md:flex items-center gap-x-3 text-[#3c3c3c] -mb-1 hidden">
-              <a
-                className="hover:opacity-75 transition"
-                href="https://portaljs.org"
-              >
-                Built with 🌀PortalJS
-              </a>
-              <hr className="h-[80%] border border-[#3c3c3c] opacity-75 my-2"></hr>
-              <a
-                className="hover:opacity-75 transition"
-                href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
-              >
-                Github
-              </a>
-            </div>
-          </div>
-          <div className="mx-2 py-1.5 text-[14px] text-[#3c3c3c] md:hidden">
-            <ul className="flex gap-x-4">
-              <li>
-                <a
-                  className="hover:opacity-75 transition"
-                  href="https://portaljs.org"
-                >
-                  PortalJS
-                </a>
-              </li>
-              <li>
-                <a
-                  className="hover:opacity-75 transition"
-                  href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
-                >
-                  View on Github
-                </a>
-              </li>
-            </ul>
-          </div>
-        </header>
         <Main />
-        <NextScript />
       </body>
+      <NextScript />
     </Html>
   );
 }

examples/fivethirtyeight/pages/datasets/[datasetName].tsx

@@ -7,10 +7,19 @@ import remarkGfm from 'remark-gfm';
 import extract from 'remark-extract-frontmatter';
 import { Dataset } from '..';
 import { GetStaticProps } from 'next';
-import { Table } from '@portaljs/components';
+import { FlatUiTable } from '@portaljs/components';
 import Breadcrumbs from '@/components/Breadcrumbs';
 import { ReactMarkdown } from 'react-markdown/lib/react-markdown';
 import remarkFrontmatter from 'remark-frontmatter';
+import Layout from '@/components/Layout';
+import { format } from 'timeago.js';
+
+// Request a weekday along with a long date
+const options = {
+  year: 'numeric',
+  month: 'long',
+  day: 'numeric',
+} as const;
 
 export default function DatasetPage({
   dataset,
@@ -22,54 +31,28 @@ export default function DatasetPage({
   return (
     <>
       <NextSeo title={`${dataset.name} page`} />
-      <main className="max-w-5xl px-2 prose mx-auto my-8 prose-thead:border-b-4 prose-table:max-w-5xl prose-table:overflow-scroll prose-thead:overflow-scroll prose-tbody:overflow-scroll prose-thead:pb-2 prose-thead:border-zinc-900 prose-th:uppercase prose-th:text-left prose-th:font-light prose-th:text-xs">
+      <Layout>
+        <main className="max-w-5xl px-2 prose mx-auto my-8 pb-8 prose-thead:border-b-4 prose-table:max-w-5xl prose-table:overflow-scroll prose-thead:overflow-scroll prose-tbody:overflow-scroll prose-thead:pb-2 prose-thead:border-zinc-900 prose-th:uppercase prose-th:text-left prose-th:font-light prose-th:text-xs prose-a:no-underline">
           <Breadcrumbs links={[{ title: dataset.name, href: '' }]} />
           <h1 className="uppercase mb-0 mt-16">{dataset.name}</h1>
-        <p className="mb-8">
-          <span className="font-semibold">Repository:</span>{' '}
-          <a target="_blank" href={dataset.url}>
-            {dataset.url}
-          </a>
-        </p>
-
-        <h2 className="mb-0 mt-10">FILES</h2>
-        <div className="inline-block min-w-full py-2 align-middle">
-          <table className="min-w-full divide-y divide-gray-300">
+          <table className="w-full my-10 mb-8 hidden md:table">
             <thead className="border-b-4 pb-2 border-zinc-900">
               <tr>
-                <th
-                  className="uppercase text-left font-light text-xs pb-3"
-                  scope="col"
-                >
-                  Name
+                <th className="uppercase text-left font-normal text-xs pb-3">
+                  related content
+                </th>
+                <th className="uppercase text-left font-normal text-xs pb-3">
+                  last updated
                 </th>
               </tr>
             </thead>
-            <tbody className="divide-y divide-gray-200">
-              {dataset.files?.map((file) => (
-                <tr key={file}>
-                  <td className="whitespace-nowrap text-left py-4 text-sm text-gray-500">
-                    <a href={file}>{file.split('/').slice(-1)}</a>
-                  </td>
-                </tr>
-              ))}
+            <tbody>
+              <DesktopItem key={dataset.name} dataset={dataset} />
             </tbody>
           </table>
-        </div>
-        {dataset.files && dataset.files.length > 0 && (
-          <>
-            <h2 className="mb-0 mt-10">DATA PREVIEWS</h2>
-            {dataset.files?.map((file) => (
-              <div key={file} className="preview-table my-8">
-                <h3>{file.split('/').slice(-1)}</h3>
-                <Table url={file} />
-              </div>
-            ))}
-          </>
-        )}
+
           {dataset.readme && (
             <>
-            <h2 className="uppercase font-black">Readme</h2>
               {dataset.readme && (
                 <ReactMarkdown
                   remarkPlugins={[
@@ -83,7 +66,111 @@ export default function DatasetPage({
               )}
             </>
           )}
+
+          <h2 className="mb-0 mt-10">Files</h2>
+          <div className="inline-block min-w-full py-2 align-middle">
+            <table className="min-w-full divide-y divide-gray-300">
+              <thead className="border-b-4 pb-2 border-zinc-900">
+                <tr>
+                  <th
+                    className="uppercase text-left font-light text-xs pb-3"
+                    scope="col"
+                  >
+                    Name
+                  </th>
+                  <th
+                    className="uppercase text-left font-light text-xs pb-3"
+                    scope="col"
+                  >
+                    Download
+                  </th>
+                </tr>
+              </thead>
+              <tbody className="divide-y divide-gray-200">
+                {dataset.files?.map((file) => (
+                  <tr key={file}>
+                    <td className="whitespace-nowrap text-left py-4 text-sm text-gray-500">
+                      <a href={`#${file.split('/').slice(-1)}`}>
+                        {file.split('/').slice(-1)}
+                      </a>
+                    </td>
+                    <td className="whitespace-nowrap py-4 text-sm text-gray-500">
+                      <a href={file}>
+                        <svg
+                          xmlns="http://www.w3.org/2000/svg"
+                          viewBox="0 0 24 24"
+                          fill="currentColor"
+                          className="w-8 h-8 text-blue-400 hover:text-blue-300 transition mt-1 ml-3"
+                        >
+                          <path
+                            fillRule="evenodd"
+                            d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
+                            clipRule="evenodd"
+                          />
+                        </svg>
+                      </a>
+                    </td>
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          </div>
+          {dataset.files && dataset.files.length > 0 && (
+            <>
+              <h2 className="mb-0 mt-8">Data Previews</h2>
+              {dataset.files?.map((file) => (
+                <div
+                  key={file}
+                  id={file.split('/').slice(-1).join('')}
+                  className="preview-table my-8"
+                >
+                  <h3>{file.split('/').slice(-1)}</h3>
+                  <FlatUiTable url={file} />
+                </div>
+              ))}
+            </>
+          )}
         </main>
+      </Layout>
+    </>
+  );
+}
+
+export function DesktopItem({ dataset }: { dataset: Dataset }) {
+  return (
+    <>
+      {dataset.articles.map((article, index) => (
+        <tr
+          key={article.url}
+          className={`${
+            index === dataset.articles.length - 1 ? 'border-b' : ''
+          } border-zinc-400`}
+        >
+          <td>
+            <a
+              className="py-8 font-bold hover:underline pr-2"
+              href={article.url}
+            >
+              {article.title}
+            </a>
+          </td>
+          <td className="py-8 font-light text-[14px] min-w-[138px] font-mono text-[#999]">
+            {format(article.date).includes('years')
+              ? new Date(article.date).toLocaleString('en-US', options)
+              : format(article.date)}
+          </td>
+          <td className="py-8 text-end">
+            {index === 0 && (
+              <a
+                className="ml-auto border border-zinc-900 font-light px-[25px] py-2.5 text-sm transition hover:bg-zinc-900 hover:text-white"
+                href={dataset.url}
+              >
+                info
+              </a>
+            )}
+          </td>
+        </tr>
+      ))}
     </>
   );
 }
@@ -101,6 +188,7 @@ export async function getStaticPaths() {
     fallback: false, // can also be true or 'blocking'
   };
 }
+// change href base check datahub-next
 
 export const getStaticProps: GetStaticProps = async ({ params }) => {
   const datasetsFile = path.join(process.cwd(), 'datasets.json');
@@ -110,15 +198,20 @@ export const getStaticProps: GetStaticProps = async ({ params }) => {
     (_dataset) => _dataset.name === params?.datasetName
   );
   const github_pat = getConfig().serverRuntimeConfig.github_pat;
-  const readmes = await Promise.all(['/README.md', '/readme.md', '/Readme.md'].map(async (readme) => await getProjectReadme(
+  const readmes = await Promise.all(
+    ['/README.md', '/readme.md', '/Readme.md'].map(
+      async (readme) =>
+        await getProjectReadme(
           'fivethirtyeight',
           'data',
           'master',
           dataset?.name + readme,
           github_pat
-  )));
-  const readme = readmes.find(item => item !== null)
-  if (!readme) console.log('Readme not found for ' + dataset?.name)
+        )
+    )
+  );
+  const readme = readmes.find((item) => item !== null);
+  if (!readme) console.log('Readme not found for ' + dataset?.name);
   return {
     props: {
       dataset: {

examples/fivethirtyeight/pages/index.tsx

@@ -4,6 +4,7 @@ import { format } from 'timeago.js';
 import { promises as fs } from 'fs';
 import path from 'path';
 import { NextSeo } from 'next-seo';
+import Layout from '@/components/Layout';
 
 const inter = Inter({ subsets: ['latin'] });
 
@@ -51,21 +52,12 @@ export function MobileItem({ dataset }: { dataset: Dataset }) {
         >
           info
         </a>
-        {/*
-                <button>
-                  <svg
-                    xmlns="http://www.w3.org/2000/svg"
-                    viewBox="0 0 24 24"
-                    fill="currentColor"
-                    className="w-12 h-12 text-blue-400 hover:text-blue-300 transition mt-1"
+        <a
+          className="ml-2 border border-[#3c3c3c] px-[25px] py-2.5 text-sm transition bg-[#3c3c3c] text-white hover:bg-zinc-900"
+          href={`/datasets/${dataset.name}`}
         >
-                    <path
-                      fillRule="evenodd"
-                      d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
-                      clipRule="evenodd"
-                    />
-                  </svg>
-                </button> */}
+          explore
+        </a>
       </div>
     </div>
   );
@@ -97,6 +89,16 @@ export function DesktopItem({ dataset }: { dataset: Dataset }) {
               ? new Date(article.date).toLocaleString('en-US', options)
               : format(article.date)}
           </td>
+          <td>
+            {index === 0 && (
+            <a
+              className="ml-2 border border-[#3c3c3c] px-[25px] py-2.5 text-sm transition bg-[#3c3c3c] text-white hover:bg-zinc-900"
+              href={`/datasets/${dataset.name}`}
+            >
+              explore
+            </a>
+            )}
+          </td>
           <td className="py-8">
             {index === 0 && (
               <a
@@ -107,23 +109,6 @@ export function DesktopItem({ dataset }: { dataset: Dataset }) {
               </a>
             )}
           </td>
-          {/*
-              <td>
-                <button>
-                  <svg
-                    xmlns="http://www.w3.org/2000/svg"
-                    viewBox="0 0 24 24"
-                    fill="currentColor"
-                    className="w-12 h-12 text-blue-400 hover:text-blue-300 transition mt-1"
-                  >
-                    <path
-                      fillRule="evenodd"
-                      d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
-                      clipRule="evenodd"
-                    />
-                  </svg>
-                </button>
-              </td>*/}
         </tr>
       ))}
     </>
@@ -143,6 +128,7 @@ export default function Home({ datasets }: { datasets: Dataset[] }) {
   return (
     <>
       <NextSeo title="FiveThirtyEight tribute by PortalJS" />
+      <Layout>
       <main
         className={`flex min-h-screen flex-col items-center max-w-5xl mx-auto pt-20 px-2.5 ${inter.className}`}
       >
@@ -206,6 +192,7 @@ export default function Home({ datasets }: { datasets: Dataset[] }) {
           .
         </p>
       </main>
+      </Layout>
     </>
   );
 }

examples/fivethirtyeight/styles/globals.css

@@ -3,6 +3,9 @@
 @tailwind utilities;
 
 .preview-table > div {
-  overflow-x: scroll;
   overflow-y: hidden;
 }
+
+.prose h1 {
+  font-size: 1.5em !important;
+}

examples/github-backed-catalog/components/DataRichDocument.tsx

@@ -0,0 +1,21 @@
+import { MDXRemote } from 'next-mdx-remote';
+import dynamic from 'next/dynamic';
+import { Mermaid } from '@portaljs/core';
+
+// Custom components/renderers to pass to MDX.
+// Since the MDX files aren't loaded by webpack, they have no knowledge of how
+// to handle import statements. Instead, you must include components in scope
+// here.
+const components = {
+  Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
+  Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
+  FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
+  mermaid: Mermaid,
+  Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
+  VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),
+  LineChart: dynamic(() => import('@portaljs/components').then(mod => mod.LineChart)),
+} as any;
+
+export default function DRD({ source }: { source: any }) {
+  return <MDXRemote {...source} components={components} />;
+}

examples/github-backed-catalog/datasets.json

@@ -15,6 +15,13 @@
     ],
     "readme": "README.md"
   },
+  {
+    "owner": "luccasmmg",
+    "branch": "main",
+    "repo": "test-data-repo-1",
+    "files": ["data_1.csv", "data_2.csv"],
+    "readme": "README.md"
+  },
   {
     "owner": "datasets",
     "branch": "main",

examples/github-backed-catalog/lib/markdown.js

@@ -0,0 +1,105 @@
+import matter from "gray-matter";
+import mdxmermaid from "mdx-mermaid";
+import { h } from "hastscript";
+import remarkCallouts from "@portaljs/remark-callouts";
+import remarkEmbed from "@portaljs/remark-embed";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import remarkSmartypants from "remark-smartypants";
+import remarkToc from "remark-toc";
+import remarkWikiLink from "@portaljs/remark-wiki-link";
+import rehypeAutolinkHeadings from "rehype-autolink-headings";
+import rehypeKatex from "rehype-katex";
+import rehypeSlug from "rehype-slug";
+import rehypePrismPlus from "rehype-prism-plus";
+
+import { serialize } from "next-mdx-remote/serialize";
+
+/**
+ * Parse a markdown or MDX file to an MDX source form + front matter data
+ *
+ * @source: the contents of a markdown or mdx file
+ * @format: used to indicate to next-mdx-remote which format to use (md or mdx)
+ * @returns: { mdxSource: mdxSource, frontMatter: ...}
+ */
+const parse = async function (source, format, scope) {
+  const { content, data, excerpt } = matter(source, {
+    excerpt: (file, options) => {
+      // Generate an excerpt for the file
+      file.excerpt = file.content.split("\n\n")[0];
+    },
+  });
+
+  const mdxSource = await serialize(
+    { value: content, path: format },
+    {
+      // Optionally pass remark/rehype plugins
+      mdxOptions: {
+        remarkPlugins: [
+          remarkEmbed,
+          remarkGfm,
+          [remarkSmartypants, { quotes: false, dashes: "oldschool" }],
+          remarkMath,
+          remarkCallouts,
+          remarkWikiLink,
+          [
+            remarkToc,
+            {
+              heading: "Table of contents",
+              tight: true,
+            },
+          ],
+          [mdxmermaid, {}],
+        ],
+        rehypePlugins: [
+          rehypeSlug,
+          [
+            rehypeAutolinkHeadings,
+            {
+              properties: { className: 'heading-link' },
+              test(element) {
+                return (
+                  ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
+                  element.properties?.id !== "table-of-contents" &&
+                  element.properties?.className !== "blockquote-heading"
+                );
+              },
+              content() {
+                return [
+                  h(
+                    "svg",
+                    {
+                      xmlns: "http:www.w3.org/2000/svg",
+                      fill: "#ab2b65",
+                      viewBox: "0 0 20 20",
+                      className: "w-5 h-5",
+                    },
+                    [
+                      h("path", {
+                        fillRule: "evenodd",
+                        clipRule: "evenodd",
+                        d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
+                      }),
+                    ]
+                  ),
+                ];
+              },
+            },
+          ],
+          [rehypeKatex, { output: "mathml" }],
+          [rehypePrismPlus, { ignoreMissing: true }],
+        ],
+        format,
+      },
+      scope,
+    }
+  );
+
+  return {
+    mdxSource: mdxSource,
+    frontMatter: data,
+    excerpt,
+  };
+};
+
+export default parse;

examples/github-backed-catalog/lib/mddb.ts

@@ -0,0 +1,14 @@
+import { MarkdownDB } from "mddb";
+
+const dbPath = "markdown.db";
+
+const client = new MarkdownDB({
+  client: "sqlite3",
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const clientPromise = client.init();
+
+export default clientPromise;

examples/github-backed-catalog/lib/octokit.ts

@@ -39,6 +39,32 @@ export async function getProjectReadme(
   }
 }
 
+export async function getProjectDatapackage(
+  owner: string,
+  repo: string,
+  branch: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.getContent({
+      owner,
+      repo,
+      path: "datapackage.json",
+      ref: branch,
+    });
+    const data = response.data as { content?: string };
+    const fileContent = data.content ? data.content : "";
+    if (fileContent === "") {
+      return null;
+    }
+    const decodedContent = Buffer.from(fileContent, "base64").toString();
+    return JSON.parse(decodedContent);
+  } catch (error) {
+    return null
+  }
+}
+
 export async function getLastUpdated(
   owner: string,
   repo: string,
@@ -162,11 +188,20 @@ export async function getProject(project: GithubProject, github_pat?: string) {
     projectBase,
     github_pat
   );
+
+  const projectDatapackage = await getProjectDatapackage(
+    project.owner,
+    project.repo,
+    project.branch,
+    github_pat
+  );
+
   return {
     ...projectMetadata,
     files: projectData,
     readmeContent: projectReadme,
     last_updated,
     base_path: projectBase,
+    datapackage: projectDatapackage
   };
 }

examples/github-backed-catalog/package.json

@@ -10,19 +10,33 @@
     "prettier": "prettier --write ."
   },
   "dependencies": {
+    "@portaljs/components": "^0.1.6",
+    "@portaljs/core": "^1.0.5",
+    "@portaljs/remark-callouts": "^1.0.5",
+    "@portaljs/remark-embed": "^1.0.4",
+    "@portaljs/remark-wiki-link": "^1.0.4",
     "@types/node": "18.16.0",
     "@types/react": "18.0.38",
     "@types/react-dom": "18.0.11",
     "eslint": "8.39.0",
     "eslint-config-next": "13.3.1",
-    "next": "13.3.1",
+    "mddb": "^0.1.9",
+    "next": "13.4.3",
+    "next-mdx-remote": "^4.4.1",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-markdown": "^8.0.7",
     "react-timeago": "^7.1.0",
+    "rehype-autolink-headings": "^6.1.1",
+    "rehype-katex": "^6.0.3",
+    "rehype-prism-plus": "^1.5.1",
+    "rehype-slug": "^5.1.0",
     "remark-gfm": "^3.0.1",
+    "remark-math": "^5.1.1",
+    "remark-smartypants": "^2.0.0",
+    "remark-toc": "^8.0.1",
     "typescript": "5.0.4"
   },
   "devDependencies": {

examples/github-backed-catalog/pages/@org/[org]/[...path].tsx

@@ -6,6 +6,8 @@ import { getProject, GithubProject } from "../../../lib/octokit";
 import ReactMarkdown from "react-markdown";
 import remarkGfm from "remark-gfm";
 import Breadcrumbs from "../../../components/_shared/Breadcrumbs";
+import parse from '../../../lib/markdown';
+import DataRichDocument from '../../../components/DataRichDocument'
 
 export default function ProjectPage({ project }) {
   const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`;
@@ -64,9 +66,7 @@ export default function ProjectPage({ project }) {
         <hr />
 
         <h2 className="uppercase font-black">Readme</h2>
-        <ReactMarkdown remarkPlugins={[remarkGfm]}>
-          {project.readmeContent}
-        </ReactMarkdown>
+        <DataRichDocument source={project.mdxSource} />
       </main>
     </>
   );
@@ -119,9 +119,10 @@ export async function getStaticProps({ params }) {
   });
   const github_pat = getConfig().serverRuntimeConfig.github_pat;
   const project = await getProject(repo, github_pat);
+  let { mdxSource, frontMatter } = await parse(project.readmeContent, '.mdx', { project });
   return {
     props: {
-      project: { ...project, repo_config: repo },
+      project: { ...project, repo_config: repo, mdxSource },
     },
   };
 }

examples/github-backed-catalog/pages/styles.css

@@ -78,3 +78,72 @@ pre {
   color: rgba(55, 65, 81, 1);
   width: 100%;
 }
+
+@import "@portaljs/remark-callouts/styles.css";
+
+/* mathjax */
+.math-inline > mjx-container > svg {
+  display: inline;
+  align-items: center;
+}
+
+/* smooth scrolling in modern browsers */
+html {
+  scroll-behavior: smooth !important;
+}
+
+/* tooltip fade-out clip */
+.tooltip-body::after {
+  content: "";
+  position: absolute;
+  right: 0;
+  top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
+  height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
+  width: 10rem;
+  background: linear-gradient(
+    to right,
+    rgba(255, 255, 255, 0),
+    rgba(255, 255, 255, 1) 100%
+  );
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
+  margin-left: -2rem !important;
+  padding-left: 2rem !important;
+  scroll-margin-top: 4.5rem;
+  position: relative;
+}
+
+.heading-link {
+  padding: 1px;
+  position: absolute;
+  left: 0;
+  top: 50%;
+  transform: translateY(-50%);
+  margin: auto 0;
+  border-radius: 5px;
+  background: #1e293b;
+  opacity: 0;
+  transition: opacity 0.2s;
+}
+
+.light .heading-link {
+  /* border: 1px solid #ab2b65; */
+  /* background: none; */
+  background: #e2e8f0;
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
+  opacity: 100;
+}
+
+.heading-link svg {
+  transform: scale(0.75);
+}
+
+@media screen and (max-width: 640px) {
+  .heading-link {
+    visibility: hidden;
+  }
+}
+

examples/learn/components/DataRichDocument.tsx

@@ -0,0 +1,21 @@
+import { MDXRemote } from 'next-mdx-remote';
+import dynamic from 'next/dynamic';
+import { Mermaid } from '@portaljs/core';
+
+// Custom components/renderers to pass to MDX.
+// Since the MDX files aren't loaded by webpack, they have no knowledge of how
+// to handle import statements. Instead, you must include components in scope
+// here.
+const components = {
+  Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
+  Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
+  mermaid: Mermaid,
+  Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
+  VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),
+  LineChart: dynamic(() => import('@portaljs/components').then(mod => mod.LineChart)),
+  FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
+} as any;
+
+export default function DRD({ source }: { source: any }) {
+  return <MDXRemote {...source} components={components} />;
+}

examples/learn/lib/markdown.js

@@ -0,0 +1,105 @@
+import matter from "gray-matter";
+import mdxmermaid from "mdx-mermaid";
+import { h } from "hastscript";
+import remarkCallouts from "@portaljs/remark-callouts";
+import remarkEmbed from "@portaljs/remark-embed";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import remarkSmartypants from "remark-smartypants";
+import remarkToc from "remark-toc";
+import remarkWikiLink from "@portaljs/remark-wiki-link";
+import rehypeAutolinkHeadings from "rehype-autolink-headings";
+import rehypeKatex from "rehype-katex";
+import rehypeSlug from "rehype-slug";
+import rehypePrismPlus from "rehype-prism-plus";
+
+import { serialize } from "next-mdx-remote/serialize";
+
+/**
+ * Parse a markdown or MDX file to an MDX source form + front matter data
+ *
+ * @source: the contents of a markdown or mdx file
+ * @format: used to indicate to next-mdx-remote which format to use (md or mdx)
+ * @returns: { mdxSource: mdxSource, frontMatter: ...}
+ */
+const parse = async function (source, format, scope) {
+  const { content, data, excerpt } = matter(source, {
+    excerpt: (file, options) => {
+      // Generate an excerpt for the file
+      file.excerpt = file.content.split("\n\n")[0];
+    },
+  });
+
+  const mdxSource = await serialize(
+    { value: content, path: format },
+    {
+      // Optionally pass remark/rehype plugins
+      mdxOptions: {
+        remarkPlugins: [
+          remarkEmbed,
+          remarkGfm,
+          [remarkSmartypants, { quotes: false, dashes: "oldschool" }],
+          remarkMath,
+          remarkCallouts,
+          remarkWikiLink,
+          [
+            remarkToc,
+            {
+              heading: "Table of contents",
+              tight: true,
+            },
+          ],
+          [mdxmermaid, {}],
+        ],
+        rehypePlugins: [
+          rehypeSlug,
+          [
+            rehypeAutolinkHeadings,
+            {
+              properties: { className: 'heading-link' },
+              test(element) {
+                return (
+                  ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
+                  element.properties?.id !== "table-of-contents" &&
+                  element.properties?.className !== "blockquote-heading"
+                );
+              },
+              content() {
+                return [
+                  h(
+                    "svg",
+                    {
+                      xmlns: "http:www.w3.org/2000/svg",
+                      fill: "#ab2b65",
+                      viewBox: "0 0 20 20",
+                      className: "w-5 h-5",
+                    },
+                    [
+                      h("path", {
+                        fillRule: "evenodd",
+                        clipRule: "evenodd",
+                        d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
+                      }),
+                    ]
+                  ),
+                ];
+              },
+            },
+          ],
+          [rehypeKatex, { output: "mathml" }],
+          [rehypePrismPlus, { ignoreMissing: true }],
+        ],
+        format,
+      },
+      scope,
+    }
+  );
+
+  return {
+    mdxSource: mdxSource,
+    frontMatter: data,
+    excerpt,
+  };
+};
+
+export default parse;

examples/learn/lib/mddb.ts

@@ -0,0 +1,14 @@
+import { MarkdownDB } from "mddb";
+
+const dbPath = "markdown.db";
+
+const client = new MarkdownDB({
+  client: "sqlite3",
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const clientPromise = client.init();
+
+export default clientPromise;

examples/learn/next.config.js

@@ -0,0 +1,5 @@
+const nextConfig = {
+  swcMinify: false
+}
+
+module.exports = nextConfig

examples/learn/package.json

@@ -12,25 +12,27 @@
     "mddb": "mddb ./content"
   },
   "dependencies": {
-    "@flowershow/core": "^0.4.10",
-    "@flowershow/markdowndb": "^0.1.1",
-    "@flowershow/remark-callouts": "^1.0.0",
-    "@flowershow/remark-embed": "^1.0.0",
-    "@flowershow/remark-wiki-link": "^1.1.2",
+    "@githubocto/flat-ui": "^0.14.1",
     "@heroicons/react": "^2.0.17",
     "@opentelemetry/api": "^1.4.0",
-    "@portaljs/components": "^0.1.0",
+    "@portaljs/components": "^0.1.8",
+    "@portaljs/core": "^1.0.5",
+    "@portaljs/remark-callouts": "^1.0.5",
+    "@portaljs/remark-embed": "^1.0.4",
+    "@portaljs/remark-wiki-link": "^1.0.4",
     "@tanstack/react-table": "^8.8.5",
     "flexsearch": "0.7.21",
     "gray-matter": "^4.0.3",
     "hastscript": "^7.2.0",
+    "mddb": "^0.1.9",
     "mdx-mermaid": "2.0.0-rc7",
     "next": "13.2.1",
     "next-mdx-remote": "^4.4.1",
     "papaparse": "^5.4.1",
-    "react": "18.2.0",
-    "react-dom": "18.2.0",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
     "react-hook-form": "^7.43.9",
+    "react-query": "^3.39.3",
     "react-vega": "^7.6.0",
     "rehype-autolink-headings": "^6.1.1",
     "rehype-katex": "^6.0.3",
@@ -40,7 +42,9 @@
     "remark-math": "^5.1.1",
     "remark-smartypants": "^2.0.0",
     "remark-toc": "^8.0.1",
-    "typescript": "5.0.4"
+    "typescript": "5.0.4",
+    "vega": "5.25.0",
+    "vega-lite": "5.1.0"
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",

examples/learn/pages/[[...path]].tsx

@@ -81,7 +81,7 @@ export default function DatasetPage({ mdxSource, frontMatter }) {
               <p className="my-0">
                 <span className="font-semibold">Modified: </span>
                 <span className="description my-0">
-                  {new Date(frontMatter.modified).toLocaleDateString()}
+                  {new Date(frontMatter.modified).toLocaleDateString("en-US")}
                 </span>
               </p>
             )}

examples/learn/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

examples/learn/styles/globals.css

@@ -1,7 +1,7 @@
 @tailwind base;
 @tailwind components;
 @tailwind utilities;
-@import "@flowershow/remark-callouts/styles.css";
+@import "@portaljs/remark-callouts/styles.css";
 
 .w-5 {
   width: 1.25rem

examples/openspending/__tests__/os-data.test.ts

@@ -1,45 +1,116 @@
-import { Octokit } from 'octokit';
-import { assert, expect, test } from 'vitest'
-import { getProjectDataPackage } from '../lib/octokit';
+import { expect, test } from 'vitest';
+import { getAllProjectsFromOrg, getProjectDataPackage } from '../lib/project';
+import { loadDataPackage } from '../lib/loader';
+import { getProjectMetadata } from '../lib/project';
+import { validate } from 'datapackage';
 
-export async function getAllDataPackagesFromOrg(
-  org: string,
-  branch?: string,
-  github_pat?: string
-) {
-  const octokit = new Octokit({ auth: github_pat });
-  const repos = await octokit.rest.repos.listForOrg({ org, type: 'public', per_page: 100 });
-  let failedDataPackages = [];
-  const datapackages = await Promise.all(
-    repos.data.map(async (_repo) => {
-      const datapackage = await getProjectDataPackage(
-        org,
-        _repo.name,
-        branch ? branch : 'main',
-        github_pat
+test(
+  'Test OS-Data',
+  async () => {
+    const repos = await getAllProjectsFromOrg(
+      'os-data',
+      'main',
+      process.env.VITE_GITHUB_PAT
     );
-      if (!datapackage) {
-        failedDataPackages.push(_repo.name)
-        return null
-      };
-      return {...datapackage, repo: _repo.name};
+    if (repos.failed.length > 0)
+      console.log('Failed to get datapackage on', repos.failed);
+    let failedDatapackages = await Promise.all(
+      repos.results.map(async (item) => {
+        try {
+          const { valid, errors } = await validate(item.datapackage);
+          return errors.length > 0 ? item.repo.name : null;
+        } catch {
+          return item.repo.name;
+        }
       })
     );
-  return {
-    datapackages: datapackages.filter((item) => item !== null),
-    failedDataPackages,
-  };
-}
+    failedDatapackages = failedDatapackages.filter((item) => item !== null);
+    if (failedDatapackages.length > 0) {
+      console.log('Failed to validate datapackage on ', failedDatapackages);
+    } else {
+      console.log('No invalid packages');
+    }
+  },
+  { timeout: 100000 }
+);
 
-test('Test OS-Data', async () => {
-  const repos = await getAllDataPackagesFromOrg('os-data', 'main', process.env.VITE_GITHUB_PAT)
-  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
-  expect(repos.failedDataPackages.length).toBe(0)
-}, {timeout: 100000})
-
-test('Test Gift-Data', async () => {
-  const repos = await getAllDataPackagesFromOrg('gift-data', 'main', process.env.VITE_GITHUB_PAT)
-  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
-  expect(repos.failedDataPackages.length).toBe(0)
-}, {timeout: 100000})
+test(
+  'Test Gift-Data',
+  async () => {
+    const repos = await getAllProjectsFromOrg(
+      'gift-data',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    if (repos.failed.length > 0) console.log(repos.failed);
+    let failedDatapackages = await Promise.all(
+      repos.results.map(async (item) => {
+        try {
+          const { valid, errors } = await validate(item.datapackage);
+          return errors.length > 0 ? item.repo.name : null;
+        } catch {
+          return item.repo.name;
+        }
+      })
+    );
+    failedDatapackages = failedDatapackages.filter((item) => item !== null);
+    if (failedDatapackages.length > 0) {
+      console.log('Failed to validate datapackage on ', failedDatapackages);
+    } else {
+      console.log('No invalid packages');
+    }
+  },
+  { timeout: 100000 }
+);
 
+test(
+  'Test getting one dataset from github',
+  async () => {
+    const datapackage = await getProjectDataPackage(
+      'os-data',
+      'berlin-berlin',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    const repo = await getProjectMetadata(
+      'os-data',
+      'berlin-berlin',
+      process.env.VITE_GITHUB_PAT
+    );
+    const project = loadDataPackage(datapackage, repo);
+    delete project['datapackage'];
+    delete project.files[0]['dialect'];
+    delete project.files[0]['schema'];
+    expect(project).toStrictEqual({
+      name: 'berlin-berlin',
+      title: 'Berlin-Berlin',
+      description: null,
+      owner: {
+        name: 'os-data',
+        logo: 'https://avatars.githubusercontent.com/u/13695166?v=4',
+        title: 'os-data',
+      },
+      repo: {
+        name: 'berlin-berlin',
+        full_name: 'os-data/berlin-berlin',
+        url: 'https://github.com/os-data/berlin-berlin',
+      },
+      files: [
+        {
+          name: 'berlin-gesamt',
+          format: 'csv',
+          path: 'https://storage.openspending.org/berlin-berlin/berlin-gesamt.csv',
+          mediatype: 'text/csv',
+          bytes: 81128743,
+          encoding: 'utf-8',
+        },
+      ],
+      author: 'Michael Peters <michael.peters@okfn.de>',
+      cityCode: 'Berlin',
+      countryCode: 'DE',
+      fiscalPeriod: { start: '2014-01-01', end: '2019-12-31' },
+      readme: '',
+    });
+  },
+  { timeout: 100000 }
+);

examples/openspending/components/DataRichDocument.tsx

@@ -0,0 +1,29 @@
+import { MDXRemote } from 'next-mdx-remote';
+import dynamic from 'next/dynamic';
+import { Mermaid } from '@portaljs/core';
+
+// Custom components/renderers to pass to MDX.
+// Since the MDX files aren't loaded by webpack, they have no knowledge of how
+// to handle import statements. Instead, you must include components in scope
+// here.
+const components = {
+  Table: dynamic(() => import('@portaljs/components').then((mod) => mod.Table)),
+  Catalog: dynamic(() =>
+    import('@portaljs/components').then((mod) => mod.Catalog)
+  ),
+  mermaid: Mermaid,
+  Vega: dynamic(() => import('@portaljs/components').then((mod) => mod.Vega)),
+  VegaLite: dynamic(() =>
+    import('@portaljs/components').then((mod) => mod.VegaLite)
+  ),
+  LineChart: dynamic(() =>
+    import('@portaljs/components').then((mod) => mod.LineChart)
+  ),
+  FlatUiTable: dynamic(() =>
+    import('@portaljs/components').then((mod) => mod.FlatUiTable)
+  ),
+} as any;
+
+export default function DRD({ source }: { source: any }) {
+  return <MDXRemote {...source} components={components} />;
+}

examples/openspending/components/DatasetCard.tsx

@@ -9,7 +9,7 @@ export default function DatasetCard({ dataset }: { dataset: Project }) {
       className="overflow-hidden rounded-xl border border-gray-200"
     >
       <Link
-        href=""
+        href={`/@${dataset.owner.name}/${dataset.repo.name}`}
         className="flex items-center gap-x-4 border-b border-gray-900/5 bg-gray-50 p-6"
       >
         <img
@@ -45,12 +45,12 @@ export default function DatasetCard({ dataset }: { dataset: Project }) {
           <dt className="text-gray-500">Fiscal Period</dt>
           <dd className="text-gray-700">
             {dataset.fiscalPeriod?.start &&
-              new Date(dataset.fiscalPeriod.start).getFullYear()}
+              new Date(dataset.fiscalPeriod.start).getUTCFullYear()}
             {dataset.fiscalPeriod?.end &&
               dataset.fiscalPeriod?.start !== dataset.fiscalPeriod?.end && (
                 <>
                   {' - '}
-                  {new Date(dataset.fiscalPeriod.end).getFullYear()}
+                  {new Date(dataset.fiscalPeriod.end).getUTCFullYear()}
                 </>
               )}
           </dd>
@@ -60,8 +60,8 @@ export default function DatasetCard({ dataset }: { dataset: Project }) {
           <dd className="flex items-start gap-x-2">
             <div className="font-medium text-gray-900">
               <Link
-                // TODO: where do we get the info needed for this link?
-                href=""
+                // TODO: this link may be incorrect for some datasets
+                href={`https://github.com/${dataset.owner.name}/${dataset.repo.name}/blob/main/datapackage.json`}
                 target="_blank"
                 className="flex items-center hover:text-gray-700"
               >

examples/openspending/components/DatasetsSearch.tsx

@@ -2,9 +2,30 @@ import { useForm } from 'react-hook-form';
 import DatasetsGrid from './DatasetsGrid';
 import { Project } from '../lib/project.interface';
 import { Index } from 'flexsearch';
+import {
+  ChevronDoubleLeftIcon,
+  ChevronDoubleRightIcon,
+  ChevronLeftIcon,
+  ChevronRightIcon,
+} from '@heroicons/react/24/solid';
+import { useState } from 'react';
+
+export default function DatasetsSearch({
+  datasets,
+  availableCountries,
+  minPeriod,
+  maxPeriod,
+}: {
+  datasets: Project[];
+  availableCountries;
+  minPeriod: string;
+  maxPeriod: string;
+}) {
+  const itemsPerPage = 6;
+  const [page, setPage] = useState(1);
 
-export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
   const index = new Index({ tokenize: 'full' });
+
   datasets.forEach((dataset: Project) =>
     index.add(
       dataset.name,
@@ -21,12 +42,53 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
     },
   });
 
-  const allCountries = datasets
-    .map((item) => item.countryCode)
-    .filter((v) => v) //  Filters false values
-    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
-    //  TODO: title should be the full name
-    .map((code) => ({ code, title: code }));
+  const filteredDatasets = datasets
+    .filter((dataset: Project) =>
+      watch().searchTerm && watch().searchTerm !== ''
+        ? index.search(watch().searchTerm).includes(dataset.name)
+        : true
+    )
+    .filter((dataset) =>
+      watch().country && watch().country !== ''
+        ? dataset.countryCode === watch().country
+        : true
+    )
+    .filter((dataset) => {
+      const filterMinDate = watch().minDate;
+      const filterMaxDate = watch().maxDate;
+
+      const datasetMinDate = dataset.fiscalPeriod?.start;
+      const datasetMaxDate = dataset.fiscalPeriod?.end;
+
+      let datasetStartOverlaps = false;
+      if (datasetMinDate) {
+        datasetStartOverlaps =
+          datasetMinDate >= filterMinDate && datasetMinDate <= filterMaxDate;
+      }
+
+      let datasetEndOverlaps = false;
+      if (datasetMaxDate) {
+        datasetEndOverlaps =
+          datasetMaxDate >= filterMinDate && datasetMaxDate <= filterMaxDate;
+      }
+
+      if (filterMinDate && filterMaxDate) {
+        return datasetStartOverlaps || datasetEndOverlaps;
+      } else if (filterMinDate) {
+        return datasetMinDate >= filterMinDate;
+      } else if (filterMaxDate) {
+        return datasetMinDate <= filterMaxDate;
+      }
+
+      return true;
+    });
+
+  const paginatedDatasets = filteredDatasets.slice(
+    (page - 1) * itemsPerPage,
+    (page - 1) * itemsPerPage + itemsPerPage
+  );
+
+  const pageCount = Math.ceil(filteredDatasets.length / itemsPerPage) || 1;
 
   return (
     <>
@@ -37,7 +99,7 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
             <input
               placeholder="Search datasets"
               aria-label="Search datasets"
-              {...register('searchTerm')}
+              {...register('searchTerm', { onChange: () => setPage(1) })}
               className="h-[3em] relative w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
             />
             {watch().searchTerm !== '' && (
@@ -55,10 +117,10 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
           <label className="text-sm text-gray-600 font-medium">Country</label>
           <select
             className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
-            {...register('country')}
+            {...register('country', { onChange: () => setPage(1) })}
           >
             <option value="">All</option>
-            {allCountries.map((country) => {
+            {availableCountries.map((country) => {
               return (
                 <option key={country.code} value={country.code}>
                   {country.title}
@@ -68,73 +130,83 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
           </select>
         </div>
         <div className="sm:basis-1/6">
-          <label className="text-sm text-gray-600 font-medium">Min. date</label>
+          <label className="text-sm text-gray-600 font-medium">
+            Fiscal Period Start
+          </label>
           <div className="relative">
             <input
               aria-label="Min. date"
-              type="date"
-              {...register('minDate')}
+              type="text"
+              placeholder={minPeriod}
+              onFocus={(e) => (e.target.type = 'date')}
+              onBlur={(e) => (e.target.type = 'text')}
+              {...register('minDate', { onChange: () => setPage(1) })}
               className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
             />
-            {watch().minDate !== '' && (
-              <button
-                onClick={() => resetField('minDate')}
-                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
-              >
-                <CloseIcon />
-              </button>
-            )}
           </div>
         </div>
         <div className="sm:basis-1/6">
-          <label className="text-sm text-gray-600 font-medium">Max. date</label>
+          <label className="text-sm text-gray-600 font-medium">
+            Fiscal Period End
+          </label>
           <div className="relative">
             <input
               aria-label="Max. date"
-              type="date"
-              {...register('maxDate')}
+              type="text"
+              placeholder={maxPeriod}
+              onFocus={(e) => (e.target.type = 'date')}
+              onBlur={(e) => (e.target.type = 'text')}
+              {...register('maxDate', { onChange: () => setPage(1) })}
               className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
             />
-            {watch().maxDate !== '' && (
+          </div>
+        </div>
+      </div>
+      <div className="mt-5 mb-5">
+        <span className="text-lg font-medium">
+          {filteredDatasets.length} datasets found
+        </span>
+      </div>
+      <div className="min-w-full align-middle">
+        <DatasetsGrid datasets={paginatedDatasets} />
+        <div className="w-full flex justify-center mt-10">
           <button
-                onClick={() => resetField('maxDate')}
-                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
+            onClick={() => setPage(1)}
+            disabled={page <= 1}
+            className="disabled:text-gray-400"
           >
-                <CloseIcon />
+            <ChevronDoubleLeftIcon className="w-6 h-6" />
+          </button>
+          <button
+            onClick={() => {
+              if (page > 1) setPage((prev) => --prev);
+            }}
+            disabled={page <= 1}
+            className="disabled:text-gray-400"
+          >
+            <ChevronLeftIcon className="w-6 h-6" />
+          </button>
+          <span className="mx-5">
+            Page {page} of {pageCount}
+          </span>
+          <button
+            onClick={() => {
+              if (page < pageCount) setPage((prev) => ++prev);
+            }}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronRightIcon className="w-6 h-6" />
+          </button>
+          <button
+            onClick={() => setPage(pageCount)}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronDoubleRightIcon className="w-6 h-6" />
           </button>
-            )}
         </div>
       </div>
-      </div>
-      <div className="min-w-full mt-10 align-middle">
-        <DatasetsGrid
-          datasets={datasets
-            .filter((dataset: Project) =>
-              watch().searchTerm && watch().searchTerm !== ''
-                ? index.search(watch().searchTerm).includes(dataset.name)
-                : true
-            )
-            .filter((dataset) =>
-              watch().country && watch().country !== ''
-                ? dataset.countryCode === watch().country
-                : true
-            )
-            // TODO: Does that really makes sense?
-            // What if the fiscalPeriod is 2015-2017 and inputs are
-            // set to 2015-2016. It's going to be filtered out but
-            // it shouldn't.
-            .filter((dataset) =>
-              watch().minDate && watch().minDate !== ''
-                ? dataset.fiscalPeriod?.start >= watch().minDate
-                : true
-            )
-            .filter((dataset) =>
-              watch().maxDate && watch().maxDate !== ''
-                ? dataset.fiscalPeriod?.end <= watch().maxDate
-                : true
-            )}
-        />
-      </div>
     </>
   );
 }
@@ -153,9 +225,9 @@ const CloseIcon = () => {
           id="Vector"
           d="M18 18L12 12M12 12L6 6M12 12L18 6M12 12L6 18"
           stroke="currentColor"
-          stroke-width="2"
-          stroke-linecap="round"
-          stroke-linejoin="round"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
         />
       </g>
     </svg>

examples/openspending/components/FlatUiTable.tsx

@@ -0,0 +1,115 @@
+import { QueryClient, QueryClientProvider, useQuery } from 'react-query';
+import Papa from 'papaparse';
+import { Grid } from '@githubocto/flat-ui';
+
+const queryClient = new QueryClient();
+
+export async function getCsv(url: string, corsProxy?: string, range?: string) {
+  if (corsProxy) {
+    url = corsProxy + url
+  }
+  const response = await fetch(url, {
+    headers: {
+      Range: range ? `bytes=0-${range}` : 'bytes=0-512000',
+    },
+  });
+  const data = await response.text();
+  return data;
+}
+
+export async function parseCsv(file: string): Promise<any> {
+  return new Promise((resolve, reject) => {
+    Papa.parse(file, {
+      header: true,
+      dynamicTyping: true,
+      skipEmptyLines: true,
+      transform: (value: string): string => {
+        return value.trim();
+      },
+      complete: (results: any) => {
+        return resolve(results);
+      },
+      error: (error: any) => {
+        return reject(error);
+      },
+    });
+  });
+}
+
+const Spinning = () => {
+  return (
+    <div role="status w-fit mx-auto">
+      <svg
+        aria-hidden="true"
+        className="w-8 h-8 mr-2 text-gray-200 animate-spin dark:text-gray-600 fill-emerald-600"
+        viewBox="0 0 100 101"
+        fill="none"
+        xmlns="http://www.w3.org/2000/svg"
+      >
+        <path
+          d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
+          fill="currentColor"
+        />
+        <path
+          d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
+          fill="currentFill"
+        />
+      </svg>
+      <span className="sr-only">Loading...</span>
+    </div>
+  );
+};
+
+export interface FlatUiTableProps {
+  url?: string;
+  data?: { [key: string]: number | string }[];
+  rawCsv?: string;
+  range?: string;
+  corsProxy?: string;
+}
+export const FlatUiTable: React.FC<FlatUiTableProps> = ({
+  url,
+  data,
+  rawCsv,
+  corsProxy,
+  range
+}) => {
+  return (
+    // Provide the client to your App
+    <QueryClientProvider client={queryClient}>
+      <TableInner range={range} corsProxy={corsProxy} url={url} data={data} rawCsv={rawCsv} />
+    </QueryClientProvider>
+  );
+};
+
+const TableInner: React.FC<FlatUiTableProps> = ({ url, data, rawCsv, corsProxy, range }) => {
+  if (data) {
+    return (
+      <div className="w-full" style={{height: '500px'}}>
+        <Grid data={data} />
+      </div>
+    );
+  }
+  const { data: csvString, isLoading: isDownloadingCSV } = useQuery(
+    ['dataCsv', url],
+    () => getCsv(url as string, corsProxy, range),
+    { enabled: !!url }
+  );
+  const { data: parsedData, isLoading: isParsing } = useQuery(
+    ['dataPreview', csvString],
+    () => parseCsv(rawCsv ? rawCsv as string : csvString as string),
+    { enabled: rawCsv ? true : !!csvString }
+  );
+  if (isParsing || isDownloadingCSV)
+    <div className="w-full">
+      <Spinning />
+    </div>;
+  if (parsedData)
+    return (
+      <div className="w-full" style={{height: '500px'}}>
+        <Grid data={parsedData.data} />
+      </div>
+    );
+  return <Spinning />
+};
+

examples/openspending/components/Header.tsx

@@ -1,53 +1,194 @@
-import Image from 'next/image'
-import { Button } from './Button'
-import { Container } from './Container'
-import logo from "../public/logo.svg"
-import Link from 'next/link'
-import { useRouter } from 'next/router'
+import Image from 'next/image';
+import { Container } from './Container';
+import logo from '../public/logo.svg';
+import Link from 'next/link';
+import { useRouter } from 'next/router';
+import { Bars3Icon } from '@heroicons/react/24/outline';
+import { useState } from 'react';
+import { Fragment } from 'react';
+import { Menu, Transition } from '@headlessui/react';
+import { ChevronDownIcon } from '@heroicons/react/20/solid';
 
 export function Header() {
+  const [menuOpen, setMenuOpen] = useState<boolean>(false);
+
   const router = useRouter();
 
   const isActive = (navLink) => {
-    return router.asPath.split("?")[0] == navLink.href;
-  }
+    return router.asPath.split('?')[0] == navLink.href;
+  };
 
   const navLinks = [
     {
-      title: "Home",
-      href: "/#header"
+      title: 'Datasets',
+      href: '/#datasets',
     },
     {
-      title: "Datasets",
-      href: "/#datasets"
+      title: 'Data Stories',
+      href: '/stories',
     },
     {
-      title: "Community",
-      href: "https://community.openspending.org/"
-    }
-  ]
+      title: 'Blog',
+      href: '/blog',
+    },
+    {
+      title: 'About',
+      href: '/about',
+      children: [
+        {
+          title: 'Fiscal Data Package',
+          href: '/about/fiscaldatapackage/',
+        },
+        {
+          title: 'Tools',
+          href: '/about/tools/',
+        },
+        {
+          title: 'Funders',
+          href: '/about/funders/',
+        },
+        {
+          title: 'Presentations',
+          href: '/about/presentations/',
+        },
+      ],
+    },
+    {
+      title: 'Contributing',
+      href: '/contributing',
+    },
+    {
+      title: 'Resources',
+      href: '/resources',
+      children: [
+        {
+          title: 'Follow the money',
+          href: '/resources/journo',
+        },
+        {
+          title: 'Map of Spending Projects',
+          href: '/resources/map-of-spending-projects/',
+        },
+        {
+          title: 'Working Group On Open Spending Data',
+          href: '/resources/wg/',
+        },
+      ],
+    },
+  ];
 
   return (
-    <header className="z-50 pb-5 lg:pt-11 sticky top-0 backdrop-blur" id="header">
-      <Container className="flex flex-wrap items-center justify-center sm:justify-between lg:flex-nowrap">
-        <div className="mt-10 lg:mt-0 lg:grow lg:basis-0 flex items-center">
+    <header className="relative z-50 pb-11 lg:pt-11">
+      <Container className="flex flex-wrap justify-between lg:flex-nowrap mt-10 lg:mt-0">
+        <Link href="/" className="lg:mt-0 lg:grow lg:basis-0 flex items-center">
           <Image src={logo} alt="OpenSpending" className="h-12 w-auto" />
+        </Link>
+        <ul className="hidden list-none sm:flex gap-x-5 text-base font-medium">
+          {navLinks.map((link, i) => (
+            <li key={`nav-link-${i}`}>
+              <Dropdown navItem={link} />
+            </li>
+          ))}
+        </ul>
+        <div className="sm:hidden sm:mt-10 lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
+          <button onClick={() => setMenuOpen(!menuOpen)}>
+            <Bars3Icon className="w-8 h-8" />
+          </button>
         </div>
-        <ul className='list-none flex gap-x-5 text-base font-medium'>
+        {menuOpen && (
+          <div className={`sm:hidden basis-full mt-5 text-center`}>
+            <ul className="gap-x-5 text-base font-medium">
               {navLinks.map((link, i) => (
                 <li key={`nav-link-${i}`}>
                   <Link
-                className={`text-emerald-900 hover:text-emerald-600 ${isActive(link) ? "text-emerald-600" : ""}`}
+                    className={`text-emerald-900 hover:text-emerald-600 ${
+                      isActive(link) ? 'text-emerald-600' : ''
+                    }`}
                     href={link.href}
                     scroll={false}
                   >
                     {link.title}
                   </Link>
-            </li>))}
+                </li>
+              ))}
             </ul>
-        <div className="hidden sm:mt-10 sm:flex lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
           </div>
+        )}
       </Container>
-    </header >
-  )
+    </header>
+  );
+}
+
+function classNames(...classes) {
+  return classes.filter(Boolean).join(' ');
+}
+
+function Dropdown({ navItem }: { navItem: any }) {
+  const [showDropDown, setShowDropDown] = useState(false);
+  return (
+    <Menu as="div" className="relative inline-block text-left">
+      {({ open }) => (
+        <>
+          <div>
+            <Menu.Button
+              onMouseEnter={() => setShowDropDown(true)}
+              onMouseLeave={() => setShowDropDown(false)}
+              className="text-emerald-900 hover:text-emerald-600 inline-flex w-full justify-center gap-x-1.5 px-3 py-2 text-sm font-semibold"
+            >
+              <Link href={navItem.href}>{navItem.title}</Link>
+              {navItem.children && (
+                <ChevronDownIcon
+                  className="-mr-1 h-5 w-5 text-gray-400"
+                  aria-hidden="true"
+                />
+              )}
+            </Menu.Button>
+          </div>
+
+          {navItem.children && (
+            <Transition
+              as={Fragment}
+              show={showDropDown}
+              enter="transition ease-out duration-100"
+              enterFrom="transform opacity-0 scale-95"
+              enterTo="transform opacity-100 scale-100"
+              leave="transition ease-in duration-75"
+              leaveFrom="transform opacity-100 scale-100"
+              leaveTo="transform opacity-0 scale-95"
+            >
+              <div>
+                <Menu.Items
+                  static
+                  onMouseEnter={() => setShowDropDown(true)}
+                  onMouseLeave={() => setShowDropDown(false)}
+                  className="absolute right-0 z-10 w-56 origin-top-right rounded-md bg-white shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none"
+                >
+                  <div className="py-1">
+                    {navItem.children.map((item) => (
+                      <Menu.Item key={item.href}>
+                        {({ active }) => (
+                          <a
+                            key={item.href}
+                            href={item.href}
+                            className={classNames(
+                              active
+                                ? 'bg-gray-100 text-emerald-900 hover:text-emerald-600'
+                                : 'text-gray-700',
+                              'block px-4 py-2 text-sm'
+                            )}
+                          >
+                            {item.title}
+                          </a>
+                        )}
+                      </Menu.Item>
+                    ))}
+                  </div>
+                </Menu.Items>
+              </div>
+            </Transition>
+          )}
+        </>
+      )}
+    </Menu>
+  );
 }

examples/openspending/components/Hero.tsx

@@ -1,9 +1,9 @@
-import { Button } from './Button'
-import { Container } from './Container'
+import { Button } from './Button';
+import { Container } from './Container';
 
-export function Hero() {
+export function Hero({ countriesCount, datasetsCount, filesCount }) {
   return (
-    <div className="relative pb-20 pt-10 sm:py-40">
+    <div className="relative pb-20 pt-10 sm:py-40" id="hero">
       <div className="absolute inset-x-0 -bottom-14 -top-48 overflow-hidden bg-green-50 bg-opacity-50">
         <div className="absolute inset-x-0 top-0 h-40 bg-gradient-to-b from-white" />
         <div className="absolute inset-x-0 bottom-0 h-40 bg-gradient-to-t from-white" />
@@ -15,33 +15,44 @@ export function Hero() {
           </h1>
           <div className="mt-6 space-y-6 font-display text-2xl tracking-tight text-emerald-900">
             <p>
-              By understanding how governments spend money in our name can we have a say
-              in how that money will affect our own lives. The journey starts here.
+              By understanding how governments spend money in our name can we
+              have a say in how that money will affect our own lives. The
+              journey starts here.
             </p>
             <p>
-              OpenSpending is a free, open and global platform to search, visualise and analyse
-              fiscal data in the public sphere.
+              OpenSpending is a free, open and global platform to search,
+              visualise and analyse fiscal data in the public sphere.
             </p>
           </div>
           <Button href="#datasets" className="mt-10">
             Search datasets
           </Button>
-          <dl className="mt-10 grid grid-cols-2 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
+          <dl className="mt-10 grid grid-cols-1 sm:grid-cols-3 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
             {[
-              ['Countries', '75'],
-              ['Datasets', '2091'],
-              ['Files', '9230'],
+              //  Added the plus sign because some datasets do not
+              //  contain defined countries
+              ['Countries', '+' + countriesCount],
+              ['Datasets', datasetsCount],
+              ['Files', filesCount],
             ].map(([name, value]) => (
               <div key={name}>
+                <div className='flex gap-x-2 items-center sm:hidden' key={name}>
+                  <dd className="mt-0.5 text-2xl font-semibold tracking-tight text-emerald-900">
+                    {value}
+                  </dd>
+                  <dt className="font-mono text-sm text-emerald-600">{name}</dt>
+                </div>
+                <div className='hidden sm:block' key={name}>
                   <dt className="font-mono text-sm text-emerald-600">{name}</dt>
                   <dd className="mt-0.5 text-2xl font-semibold tracking-tight text-emerald-900">
                     {value}
                   </dd>
                 </div>
+              </div>
             ))}
           </dl>
         </div>
       </Container>
     </div>
-  )
+  );
 }

examples/openspending/components/_shared/Footer.tsx

@@ -0,0 +1,25 @@
+import Image from 'next/image';
+import Link from 'next/link';
+export default function Footer() {
+  return (
+    <footer className="bg-slate-50 border-t border-slate-100 py-10">
+      <div className="mx-auto max-w-7xl px-4 sm:px-6 lg:px-8 flex flex-col items-center justify-between md:flex-row">
+        <div className="flex gap-x-2 items-center justify-center mx-auto">
+          <p className="text-base text-slate-500">Maintained by</p>
+          <a href="https://www.datopian.com/" target="_blank">
+            <Image
+              alt="Datopian logo"
+              className="mb-1"
+              src="/datopian-logotype.png"
+              width={120}
+              height={30}
+            />
+          </a>
+        </div>
+        <div className="flex gap-x-2 items-center mx-auto h-20">
+          <p className="mt-8 text-base text-slate-500 md:mt-0">Built with <a href="https://portaljs.org" target="_blank" className='text-xl font-medium'>🌀 PortalJS</a></p>
+        </div>
+      </div>
+    </footer>
+  );
+}

examples/openspending/components/_shared/Layout.tsx

@@ -0,0 +1,12 @@
+import { Header } from '../Header';
+import Footer from './Footer';
+
+export default function Layout({ children }) {
+  return (
+    <div className="bg-white min-h-screen">
+      <Header />
+      {children}
+      <Footer />
+    </div>
+  );
+}

examples/openspending/content/about/contact.md

@@ -0,0 +1,23 @@
+---
+lead: true
+section: about
+title: Contact
+authors:
+- Anders Pedersen
+redirect_from:
+- /contact/
+---
+
+Connect with the OpenSpending community.
+
+The best place for OpenSpending-related discussions is our [forum](https://discuss.okfn.org/c/openspending/), but you can interact with OpenSpending on your favorite social network by following the links below:
+
+* Open Knowledge Discussion Forum
+  * [English](https://discuss.okfn.org/c/openspending/none)
+  * [Portuguese](https://discuss.okfn.org/c/openspending/gastos-abertos)
+* <abbr title="Gitter">Gitter</abbr> chat: [OpenSpending Gitter chatroom](https://gitter.im/openspending)
+* GitHub: 
+  * [OpenSpending](https://github.com/openspending) (OpenSpending-related code)
+
+* Twitter: [&#64;openspending](https://twitter.com/openspending)
+* Facebook: [OpenSpending](https://www.facebook.com/openspending?_rdr=p)

examples/openspending/content/about/contributors.md

@@ -0,0 +1,172 @@
+---
+section: about
+lead: true
+title: Contributors
+authors:
+- Anders Pedersen
+---
+
+There are many contributors to OpenSpending. There are thousands of registered OpenSpending users contributing data and analysis.
+
+It is impossible to adequately acknowledge the many individuals and organizations who have contributed.
+
+This page then is necessarily partial and is focused (though not limited to) those who have made special efforts to contribute through their participation in specific teams, in donating data or in other significant ways.
+
+If you'd like to be added as a contributor to this page please [get in touch]({{ site.baseurl }}/about/contact/).
+
+## Teams
+
+### News and Website Team
+
+#### News Editors - Descriptions
+
+These guys run the blog and manage our social media presence. We are
+still recruiting News Editor volunteers so if you'd like to join the
+team <a
+href="http://community.openspending.org/contribute/web/#Sign_up">apply
+here</a>.
+
+<strong>Burite Joseph</strong>, <a href="https://twitter.com/BuriteJoseph">@BuriteJoseph</a>
+
+Independent media practitioner and entrepreneur with over five years
+of journalism and research experience, Burite runs ZHENOBIA, a media
+integration and multimedia content aggregation company. She also
+consults for <a href="http://www.smsmedia.ug">SMS Media Uganda</a>,
+Ultimate Media Uganda, <a href="http://www.busiweek.com">East African
+Business Week</a> and <a href="http://www.dailymonitor.co.ug">Daily
+Monitor</a>.
+
+<blockquote>
+Working with data is my new passion. I am a quick learner and teamwork is my steroid.
+</blockquote>
+
+<strong>Anna Flagg</strong>, <a href="http://www.annaflagg.com">www.annaflagg.com</a>
+
+Data journalist at the Center for Responsive Politics, Anna has a background in computer science, data visualization, design and data-storytelling.
+
+<blockquote>
+I like working on projects that create awareness of issues important to the public. I'm excited to work with and learn from the Open Spending community.
+
+</blockquote>
+<strong>Laura S. García</strong>,<a href="https://twitter.com/Laura_S_Garcia">@laura_s_garcia</a>
+
+An experienced journalist, Laura has worked for more than ten years as a multi-media journalist in Spain. She has also taught Geography and History to high-school students. Laura speaks Spanish, Galician, English and a little Swedish.
+
+<blockquote>
+I’m looking to improve my knowledge of open data, as I’ve always thought this to be the best way to offer a good journalism and a good education as well.
+
+</blockquote>
+<strong>Karen Brzezinska</strong>, <a href="https://twitter.com/westofwarsaw">@westofwarsaw</a>
+
+Also a professional journalist, Karen (Kati) worked for international news services specialising in equity, commodity and currency markets. Her background is in PoliSci (East European studies), and, while originally from midwestern US, her life experience lists Italy, Hungary (1989-1992) and The Netherlands (since 1992) as home-countries. Kati is fluent in English (US) and Dutch.
+
+<blockquote>
+I'm interested in learning how open data can be used to enhance governance and education.
+
+</blockquote>
+<strong>Dominic Kornu</strong>, <a href="https://twitter.com/Qaphui">@qaphui</a>
+
+An IT and Maths tutor from Ghana, with an interest in web and social media technologies, Dominic blogs at <a href="http://dominicmary.blogspot.com">Qaphui’s Cafe</a> and volunteers in his free time.
+
+<blockquote>
+I am interested in learning how open data can be used to enhance governance and education.
+
+</blockquote>
+<strong>Mehmet Koksal</strong>, <a href="https://twitter.com/mehmetkoksal">@mehmetkoksal</a>
+
+Freelance journalist based in Brussels (Belgium) and conference interpreter, Mehmet also works as a fixer for the international press, including the French weekly <a href="http://www.courrierinternational.com/">Courrier int.</a>. In his free time he volunteers for <a href="http://www.ajp.be/">AJP</a> and acts as a campaign manager for the <a href="http://europe.ifj.org/en/pages/turkey-campaign-set-journalists-free">EFJ</a>.
+
+<strong>Teodora Beleaga</strong>, <a href="https://twitter.com/t30d0ra">@t30d0ra</a>
+
+A digital analyst and freelance data journalist based in London, <a href="http://teodorabeleaga.com">Teodora </a>is an alumna of City University’s Interactive Journalism MA and has completed work experience with <a href="http://www.theguardian.com/profile/teodora-beleaga">The Guardian</a>.
+
+<blockquote>
+I joined the Open Spending project to share my data analysis skills and expand my understanding of fiscal transparency and government spending.
+
+</blockquote>
+<strong>Miriam Ruhenstroth</strong>
+
+A Science and technology freelance journalist based in Berlin (Germany), Miriam has a background in biological sciences. In 2011 she attended a summer school for data journalism (organized by Initiative Wissenschaftsjournalismus).
+
+<blockquote>
+I found the field of data storytelling thrilling and joined OpenSpending, to learn more about it and participate for good.
+
+</blockquote>
+
+### Data Team
+
+
+#### Data Wranglers - Descriptions
+
+The Data Wranglers work to add, clean and visualise data in OpenSpending. They help community members who need assistance. Some data wranglers focus on cleaning and analysing data whereas others work to visualise data using the OpenSpending API. We are still recruiting Data Wrangler volunteers so if you'd like to join the team <a href="http://community.openspending.org/contribute/data/#Official_sign_up">apply here</a>.
+
+<strong>Concha Catalan</strong>, <a href="https://twitter.com/conchacatalan">@conchacatalan</a>
+
+An English teacher and freelance journalist based in Barcelona (Spain), Concha is currently working on a project to open the autonomous government of Catalonia (opengov.cat). She also blogs at <a href="http://barcelonalittleshell.blogspot.com.es">http://barcelonalittleshell.blogspot.com.es</a>.
+
+<blockquote>
+I would like to add the data set of the autonomous government of Catalonia budget to OpenSpending. I am coming to terms with lots of new concepts.
+
+</blockquote>
+<strong>Prakash Neupane</strong>, <a href="https://twitter.com/nprkshn">@nprkshn</a>
+
+OKFN Ambassador in Nepal and FOSS Enthusiastic, Prakash is working in social development empowering individuals and communities by using technology. He is an Open Data Researcher and Nepali Wikimedian, responsible for Wikimedia Education Program in Nepal. Find out more  about him <a href="http://www.prakashneupane.com.np/about-me">here</a>.
+
+<strong>Pierre Chrzanowski</strong>, @piezanowski
+
+A member of the French OKFN working in the field of Open Government Data, Pierre says he is really interested to work on Tax Heaven, Public Procurement and Aid Data.
+
+<blockquote>
+I want to learn more about tools to analyse the data sets and how best to do storytelling.
+
+</blockquote>
+<strong>Samuel S. Lee</strong>, <a href="https://twitter.com/OpenNotion">@OpenNotion</a>
+
+Currently based in Washington DC, Samuel is a member of the <a href="https://finances.worldbank.org">World Bank Group Open Finances</a> team. He loves data, innovation, transparency, photography and college football.
+
+<blockquote>
+I am passionate about “open” and its potential to transform civic engagement, international development, and the world. I am particularly interested in realizing the potential of open financial information.
+
+</blockquote>
+<strong>Adriana Homolova</strong>
+
+A data journalism student with a passion for open culture, Adriana is a member of the <a href="http://soit.sk">Society for Open Information Technologies</a>.
+
+<strong>Sipos Zoltán</strong>
+
+A Hungarian journalist working for an Internet news portal in Romania, Sipos specializes in investigative reporting.  His background includes philosophy, sociology and public policies. Sipos has experience working with data, filing FOI requests, and tackling spreadsheets.
+
+<blockquote>
+I am trying to learn as much as I can about data journalism through online groups, MOOCs and books purchased from Amazon. My ultimate goal is to set up a small investigative / data journalism start-up in Romania.
+</blockquote>
+
+<strong>Gabe Sawhney</strong>
+
+A member of <a href="http://betterbudget.ca">Better Budget
+Toronto</a> Gabe joined the Team to bring transparency to his
+city’s <a
+href="http://spacing.ca/toronto/2012/12/10/lorinc-building-a-better-budget-at-city-hall/">budget</a>.
+
+<blockquote>
+I want to mobilize action (citizens, elected officials and policymakers) for better process, better clarity, better formats, and more transparency around city budgets.
+</blockquote>
+
+<strong>Elaine Ayo</strong>, <a href="https://twitter.com/eieayo">@eieayo</a>
+
+Statistician student based in Washington, DC Elaine has spent the last
+three years in Seoul, South Korea as a copy editor for an English news
+wire. Prior to that Elaine reported for her hometown paper, the San
+Antonio Express-News, in Texas.
+
+<strong>Hans Loos</strong>
+
+An IT and telecom freelance journalist based in Belgium, Hans studied
+sociology and has a passion for statistics.
+
+<blockquote>
+I have started to learn to program and study R but without big results up till now.
+</blockquote>
+
+## Data Donors
+
+In progress ...

examples/openspending/content/about/exemplars.md

@@ -0,0 +1,329 @@
+---
+section: about
+lead: true
+title: Project portfolio
+authors:
+- Neil Ashton
+---
+Here are some of the projects, past and present, that we have been involved with.
+
+* <a href="#current-development">Current OpenSpending projects</a>
+* <a href="#past-development">Past projects</a>
+* <a href="#gov-tools">Tools for governments</a>
+* <a href="#gov-use-cases">Government use cases</a>
+* <a href="#publications">Publications and reports</a>
+* <a href="#standards">Technical standards</a>
+* <a href="#stories">Stories and data journalism</a>
+* <a href="#your-project">Collaborate! Your project here...</a>
+
+<a name="current-development"></a>
+
+## Current projects
+
+We're working on [OpenSpending Next](/next/), the next version of OpenSpending.  Learn how to [get involved](/get-involved/)!
+
+<a name="past-development"></a>
+
+## Already unleashed
+
+#### Spending Stories
+
+In 2011, the Open Knowledge Foundation was awarded a Knight News Challenge Grant to work on the <a href="http://blog.okfn.org/2011/01/18/spending-stories/">Spending Stories</a> Project. Large numbers are often meaningless to the general public, and despite a wealth of information around government spending, the topic of government finance is often overlooked by journalists. The Spending Stories project aims to facilitate reporting by speeding up fact-checking around spending data as well as connecting news stories about public spending to relevant datasets and visualisations to put these stories into context.
+
+We're into our second year of the project now. <a href="http://www.pbs.org/idealab/2012/12/follow-the-money-a-spending-stories-guide-for-journalists345.html">See what we are up to by following our updates on the PBS Idea Lab blog.</a>
+
+#### budzeti.ba: Bosnian Budget Bubbles
+
+<img class="pull-right" style="margin-left: 1em;" title="Balkan Budgets" src="http://farm9.staticflickr.com/8063/8219557569_cc12ebbdea_m.jpg" alt="" />
+
+In October 2013 OpenSpending launched the "Where Does My Money Go"-site <a href="http://budzeti.ba/">budzeti.ba</a> in Bosnia and Herzegovina. Thanks to a grant from the National Endowment for Democracy, the OpenSpending team collaborated with the Centre for Public Interest Advocacy in Bosnia to produce interactive graphics of national and subnational budgets in Bosnia.
+
+As part of the project OpenSpending:
+
+<ul>
+<li>Created visualisations of national, entity-, cantonal- and district-level budgets for Bosnia.</li>
+<li>Conducted onsite trainings and capacity building workshops with organisations from other Balkan countries, on getting, wrangling and presenting financial data (with OpenSpending and other tools).</li>
+<li>Developed a tax calculator for Bosnia (similar to the <a href="http://wheredoesmymoneygo.org/dailybread.html">Daily Bread</a>).</li>
+<li>Tested the <a href="http://openspending.org/resources/handbook/ch001_introduction.html">Spending Data Handbook</a> and added an <a href="http://community.openspending.org/help/guide/">OpenSpending guide</a> with a more technical guidance, for those organisations wishing to do more ambitious data work.</li>
+</ul>
+
+<a href="http://blog.openspending.org/2012/09/26/balkan-budget-bubbles/">Read more about the project.</a> Read the full announcement from <a href="http://community.openspending.org/2013/10/budzeti-ba-follow-the-money-bosnia-herzegovina/">the launch</a>.
+
+#### OpenSpending Slovakia
+
+In early 2013, we launched <a href="http://slovakia.openspending.org">Slovakia Openspending</a>, prepared in collaboration with transparency and anti-corruption watchdog <a href="http://www.transparency.sk/">Transparency International Slovakia</a>. The site contains budget and expenditure information gathered from some 20 cities across Slovakia allowing users to examine current municipality budgets and compare them to expenditures going back to 2009. Based on the openspending.org source code, we have added several improvements such as:
+
+<ul>
+<li>Year-on-year comparison for spending categories.</li>
+<li>Improved listing and more contextual presentation of the data.</li>
+<li>Embedable graphic drilldowns with accompanying tables.</li>
+</ul>
+Transparency Slovakia hopes to promote the site as an example of how budgets could be made more accessible and comprehensible to the public once the site is localized and tested further. While not in its current plans, additional spending data and a feature to allow users to compare cities would improve opportunities for doing crowdsourced benchmarking.
+
+#### Where Does My Money Go?
+
+<img class="pull-left" style="margin-right: 3em;" title="WDMMG original" src="http://farm8.staticflickr.com/7088/7315252358_7dae93b263.jpg" alt="" width="250" />
+
+<a href="http://wheredoesmymoneygo.org/">Where Does My Money Go?</a> (WDMMG) is the project from which OpenSpending was born. Funded originally by 4IP, it allows UK citizens to examine where their taxes are being spent through an interactive 'bubble tree' visualisation. They can even find out how much they contribute on a daily basis through their taxes to various sectors of society through the <a href="http://wheredoesmymoneygo.org/dailybread.html">Daily Bread</a> app.
+
+You can now build a site like WDMMG for your own country using the OpenSpending API and the WDMMG Toolkit:
+
+<ul>
+<li><a href="http://wheredoesmymoneygo.org/">Visit the Where Does My Money Go? project</a></li>
+<li><a href="https://twitter.com/#!/wdmmg">Follow Where Does My Money Go? on Twitter</a></li>
+<li><a href="http://blog.openspending.org/2012/02/16/thekit/">Read more about how to create your own Where Does My Money Go?site</a></li>
+</ul>
+
+#### Cameroon Budget Inquirer
+
+<img class="pull-left" style="margin-right: 2em;" title="Cameroon Budget Inquirer" src="http://farm9.staticflickr.com/8340/8287000485_4daf7d73da_n.jpg" alt="" />
+
+We collaborated with the World Bank to produce a citizen-friendly representation of <a href="http://cameroon.openspending.org/">Cameroon's budget</a>.
+
+The project incorporates the open-source 'bubble visualisations' first seen in Where Does My Money Go, as well as many new elements, such as a map based navigation and heatmaps. The project includes:
+
+<ul>
+<li>Visualisation of the National Investment Budget, the Northwest region's budget of Cameroon (including absolute and per-capita estimates, budgeted as well as actual figures) as well as a similar visualisation of the Tignere local council budget.</li>
+<li>A sub-national budget transparency index - similar to the work of the International Budget Partnership, but at sub-national levels. This highlights the availability of key budget documents and ranks regions on their availability.</li>
+<li>Explore the data: filter the data by categories such as amount, location, and activity.</li>
+</ul>
+
+#### OffenerHaushalt
+<img class="pull-right" style="margin-left: 1em;" title="Offener Haushalt" src="http://farm8.staticflickr.com/7077/7315281352_2c00d928d8.jpg" alt="" width="250" />
+
+<a href="http://bund.offenerhaushalt.de/">OffenerHaushalt</a> allows users to explore and drill down through the various layers of Germany's federal budget, comparing data from as far back as 2006. Through the new TreeMap visualisation, the user can easily see and explore the different departments and programmes and see how much is spent, proportions and statistics on changes between years.
+
+The success of OffenerHaushalt and the demand to roll it out on a local level was one of the prime motivations for the creation of OpenSpending. To date, the OffenerHaushalt team have received around 90 requests for a similar site in their area.
+
+<ul>
+<li><a href="http://bund.offenerhaushalt.de/">Visit the OffenerHaushalt Project</a></li>
+<li><a href="https://twitter.com/#!/offenerhaushalt">Follow OffenerHaushalt on Twitter</a></li>
+</ul>
+
+#### Uganda Aid Visualisation
+
+Aid flows often do not pass through a recipient government’s conventional budget mechanisms. When this happens, recipient governments themselves may not have the complete overview of where aid money goes and how donor priorities align with their own. This information is vital for governments and aid donors to be able to make the best use of scarce resources.
+
+<img class="pull-left" style="margin-right: 1em;" title="Uganda Aid" src="http://farm9.staticflickr.com/8159/7315314386_7940819de6.jpg" alt="" width="250" />
+
+Normally this overview is not available – leading to waste, overlap and inefficiency. The lack of comparable information means aid donors and recipient country governments can’t work together to coordinate their efforts or understand how donor priorities align with recipient priorities; it decreases developing country governments' ownership and undermines the potential for good governance and planning. Donors and governments need to know what others are doing - and crucially, what others are planning on doing - if they are to make sure that these resources are used most effectively. Otherwise, some sectors and areas will not receive enough funding, while others may have too many donors involved.
+
+The Uganda Aid visualisation project was a joint project between the OKFN and Publish What You Fund to combine two key types of fiscal data, revenues from aid together with spending information, and present them together in an informative way for the first time through an interactive visualisation.
+
+<ul>
+<li><a href="http://www.publishwhatyoufund.org/uganda/uganda-with-data.htm">Visit the Publish What You Fund Uganda Visualisation Project</a></li>
+<li><a href="http://www.publishwhatyoufund.org">Visit Publish What You Fund's website</a></li>
+<li><a href="http://openspending.org/ugandabudget">Uganda Budget and Aid to Uganda, 2003-2006 dataset in OpenSpending</a></li>
+<li>Coverage of the Uganda Visualisation: <a href="http://www.guardian.co.uk/global-development/poverty-matters/2011/nov/25/uganda-aid-confusion-analyse-spending?newsfeed=true">Guardian Poverty Matters Blog</a></li>
+</ul>
+
+#### IATI
+
+The International Aid Transparency Initiative is a common format for publishing aid information. 29 signatories representing 75% of global Official Development Finance have committed to reporting timely information about their aid activities in this standard format. Already, 13 signatories representing 45% of ODF have published.
+
+<img class="pull-right" style="margin-left: 1em;" title="IATI" src="http://farm9.staticflickr.com/8007/7315331330_4dafe5ea48.jpg" alt="" width="250" />
+
+IATI publishers release the data as open data feeds in a common XML format through their own websites. They then register their data with the IATI Registry - which runs on the Open Knowledge Foundation's CKAN software - making it easy for users to find this data.
+
+However, the nature of IATI as a distributed collection of raw data feeds also presents a challenge to non-technical users. The Open Knowledge Foundation worked with Publish What You Fund to transform the data into a format suitable for import into OpenSpending, where the data can be much more easily visualised and analysed.
+
+<a href="http://openspending.org/iati?_time=2011">View the IATI data on OpenSpending</a>
+
+<a name="gov-tools"></a>
+
+## Tools for governments
+
+#### Data.Gov.Uk Spend Browser
+
+<img class="pull-left" style="margin-right: 2em;" title="Reporting Tool" src="http://farm6.staticflickr.com/5523/9045231550_505140b175_n.jpg" alt="http://farm6.staticflickr.com/5523/9045231550_505140b175_n.jpg" />
+
+The OpenSpending team worked with Data.Gov.Uk to build the <a href="http://data.gov.uk/data/openspending-browse">transaction explorer</a> which forms part of the Data.Gov.Uk website.
+
+The transaction explorer allows any user to directly search the OpenSpending database for companies, departments or projects of interest and investigate how much money was spent on them.
+
+#### UK Government Spend Reporting Dashboard
+
+<img class="pull-left" style="margin-right: 2em;" title="Reporting Tool" src="http://farm9.staticflickr.com/8443/7980196066_d4aa29eb0d_z.jpg" alt="" />
+
+The OpenSpending team worked with Data.Gov.Uk to produce an automatic reporting tool to demonstrate which government departments were complying with their transparency obligations.
+
+The <a href="http://data.gov.uk/data/openspending-report/index">tool</a> lists departments registered as data publishers on data.gov.uk and details how precisely they have followed the <a href="https://www.gov.uk/government/publications/guidance-for-publishing-spend-over-25000">HM Treasury reporting guidelines</a>. It will also make the whole of the reported data available for search and analysis both on <a href="http://data.gov.uk/openspending">data.gov.uk</a> and on the OpenSpending site.
+
+<a name="gov-use-cases"></a>
+
+## Government use cases
+
+#### City of Bologna
+
+<img class="pull-left" style="margin-right: 2em;" title="Open Dati Bologna Tool" src="http://dati.comune.bologna.it/file/field/image/open_spending_pic_bologna.png" alt="http://dati.comune.bologna.it/file/field/image/open_spending_pic_bologna.png" />
+
+The open data team at the City of Bologna uses OpenSpending to visualise several years of <a href="http://openspending.org/bp_2012_entrate/views/table-of-aggregates-bilancio-di-previsione-2012-entrate">the city budget</a>.
+
+#### City of Berlin
+<img class="pull-left" style="margin-right: 2em;" title="The budget for Berlin city" src="http://stefanwehrmeyer.com/img/blog/2013/02/budget-berlin.png" alt="http://stefanwehrmeyer.com/img/blog/2013/02/budget-berlin.png" />
+
+The City of Berlin asked Open Knowledge Foundation Germany to visualise the budget for the city using OpenSpending. The result is featured on the <a href="https://imperia9.berlinonline.de/sen/finanzen/haushalt/haushaltsplan/artikel.5697.php">site for the city finances</a>.
+
+<a name="publications"></a>
+
+## Publications and Reports
+
+#### Technology for Transparent &amp; Accountable Public Finance
+<img class="pull-left" style="margin-right: 2em;" title="TTAPF" src="http://farm9.staticflickr.com/8031/8002530046_fe4354f76a_m.jpg" alt="" />
+
+The report, <a href="../../resources/gift/">“Technology for Transparent and Accountable Public Finance”</a>, was commissioned by the Global Initiative for Fiscal Transparency (GIFT) in February 2012 in order to assist GIFT in assessing the potential of technology to aid transparency and accountability in relation to governments’ fiscal activities.
+
+Read the <a href="../../resources/gift/chapter9-intro/#governments">recommendations</a>.
+
+<a href="http://content.openspending.org/resources/gift/pdf/ttapf_report_20120530.pdf">Download as PDF</a>
+
+#### Mapping the Open Spending Data Community
+<img class="pull-right" style="margin-left: 2em;" title="Spend Report" src="http://farm8.staticflickr.com/7414/8881834696_4176e6d2ea_m.jpg" alt="" />
+
+In early 2012, we set out on a mission. Our aim was to establish how CSO's currently work with spending data, how they would like to use it, and what they would like to achieve - including:
+
+<ul>
+<li>what existing tools are being used</li>
+<li>what current technical needs are unmet</li>
+<li>what would be required to meet these needs and how feasible is it to tackle them</li>
+</ul>
+This <a href="../../resources/mappingcommunity/">report</a> is the output of that research. Here, we bring together key case studies from organisations who have done pioneering work in using technology to put government data to best use.
+
+In this report we:
+
+<ul>
+<li>outline how the data could be improved in order to make it more usable</li>
+<li>examine some key case studies of how organisations are using technology to do groundbreaking research, citizen engagement, visualisation and tracking of accountability.</li>
+<li>talk about the training needs for CSO's to help them better use the data available, and to demand better data.</li>
+</ul>
+Read the <a href="../../resources/mappingcommunity/conclusions/">conclusions</a>.
+
+<a href="../../resources/mappingcommunity/introduction/videos/">Athens to Berlin research trip: Watch the video interviews</a>
+
+#### The Spending Data Handbook
+<img class="pull-left" style="margin-right: 2em;" title="Spending Handbook" src="http://farm8.staticflickr.com/7449/8754377372_77aed9107e_m.jpg" alt="" />
+
+The <a href="../../resources/handbook/">Spending Data Handbook</a> is addressed to people and organisations who want to use and understand government budgets and spending data in their work.
+
+The book covers:
+
+<ul>
+<li>Collaborating with other organizations to pool resources and strengthen your advocacy effort</li>
+<li>If you're just starting out, what data to look for and what to ask for (nay, demand!) from your government</li>
+<li>The 'Data Pipeline': Tricks and tips for finding, wrangling and systematically processing your data</li>
+<li>Getting ambitious, running a technology project</li>
+<li>Presenting your findings to engage the public, media and government</li>
+<li>Lists and appendices of technical and non-technical resources</li>
+</ul>
+
+<a name="standards"></a>
+
+## Standards
+
+#### [Draft] Standard for Transaction-level Spending Data
+The release of transaction-level data (i.e. information about individual disbursements or contract spending) is a relatively new idea, compared to the release of higher-level accounting information or budget overviews. The availability of such data allows for fine-grained analysis and oversight of activities and will, in the future, enable anyone inside or outside of government to reconstruct key reports from raw data. In order to perform these types of analysis, it is often necessary to combine spending information from several sources - either for completeness or comparison.
+
+Too often, standardization in this context appears to be supply-driven: every publisher wants to express the full range of data they hold and are willing to release. Necessarily, such an approach leads to a standard that is the superset of all the systems that feed into it.
+
+Such designs tend to be of little use to the intended consumers, as they raise the barriers to understanding the information considerably. Our approach therefore is to generate demand-side use cases first, ensuring that everything that is done will generate value for data users.
+
+Read our <a href="../../resources/standard/">proposed standard</a>.
+
+<a name="stories"></a>
+
+## Stories and Data Journalism
+
+#### The Guardian (UK)
+
+<h4>The Real Price of the London Olympics</h4>
+<a title="olympic-guardian by anderspedersenOKF, on Flickr" href="http://www.flickr.com/photos/94746900@N06/8915659698/"><img src="http://farm3.staticflickr.com/2806/8915659698_60f3b70eed_o.png" alt="olympic-guardian" /></a>
+
+The Guardian used OpenSpending's bubbletree visualisations to answer the questions: Where is the Olympics money coming from - and where's it being spent? How much is coming from sponsorship - and how much does the Olympic stadium cost?
+
+<a href="http://www.guardian.co.uk/sport/datablog/interactive/2012/jul/26/london-2012-price-olympic-games-visualised">See the interactive visualisation on the Guardian Datablog</a>.
+
+<h4>The Daily Bread</h4>
+<a href="http://www.guardian.co.uk/uk/datablog/interactive/2013/mar/20/budget-2013-how-taxes-spent-interactive">The Daily Bread</a> from Where Does My Money Go? Has made regular appearances on the Guardian Datablog around budget time in the UK.
+
+#### Le Monde (France)
+We experienced an extremely high traffic spike when a visualisation based on French data was featured in Le Monde in October 2012.
+The article <a href="http://www.lemonde.fr/politique/article/2012/10/16/plf-des-avions-au-bouclier-fiscal-la-java-des-amendements_1776093_823448.html?xtmc=depenses&amp;xtcr=52">PLF : des avions au bouclier fiscal, la java des amendements</a>, (PLF=Projet de loi de finances, the draft finance law) deals with suggested amendments to the draft finance law and which parties were demanding what amendments.
+
+The OpenSpending visualisation used in the article is intended to give a high-level representation of some of the main areas of government expenditure in France.
+
+#### Politiken (Denmark)
+<a title="politiken" href="http://farm6.staticflickr.com/5339/9044797934_366b1d8914_m.jpg"><img class="pull-right" style="margin-left: 2em;" src="http://farm6.staticflickr.com/5339/9044797934_366b1d8914_m.jpg" alt="politiken" /></a>
+
+Municipalities are central for the functioning of the welfare state Denmark. They take care of a range of important tasks like social- and health care, primary education, social benefits, traffic and much more. Even in a year with local elections, however, they do not attract much public attention. The open data web agency Buhl-Rasmussen developed a site of all 98 Danish municipalities for one of the biggest Danish news sites, Politiken.
+
+#### Sumy News, Ukraine
+<a title="politiken" href="http://sumynews.com/data-journalism2/budget-economy/item/4726-byudzhet-m-sumy-na-2013-rik-openspending.html&lt;br /&gt;
+"><img class="pull-left" style="margin-right: 1em;" src="http://farm6.staticflickr.com/5459/9045133060_9dbb04b97c_m.jpg" alt="" /></a>
+
+On December 26, 2012, Sumy City Council approved the city budget for 2013, but there were big concerns about how it would be executed.
+
+The expected revenues for the coming year did not appear to match the planned output, as Sumy would have to have generated 15.7% more than the expected revenues in 2012....
+
+<a href="http://sumynews.com/data-journalism2/budget-economy/item/4726-byudzhet-m-sumy-na-2013-rik-openspending.html">Read the article</a>.
+
+#### Privacy International
+
+In early 2012, Privacy International approached the Spending Stories team to ask for a search widget to be able to search across all of the government spending datasets held in OpenSpending. They had a list of companies which exhibited at the famous surveillance technology conferences in the US, the so-called Wiretappers' Ball, as well as a list of attendees of the conference, and they wanted to check which attendees also became customers of these companies.
+
+<img class="pull-right" style="margin-left: 1em;" title="Caelainn Barr" src="http://farm8.staticflickr.com/7213/7315271184_921d9ed606.jpg" alt="" width="250" />
+
+Some attendees posed no surprises – the FBI, the US Drug Enforcement Administration, the UK Serious Organized Crime Agency and Interpol to name a few – but there were a few that are downright baffling, like the US Department of Commerce, the US Fish &amp; Wildlife Service and the Clark County School District Police Department.
+
+As more datasets are loaded into OpenSpending, this universal search will get increasingly more powerful, and we look forward to hearing what other people use the search for.
+
+<ul>
+<li><a href="http://opendatalabs.org/spendbrowser">Try out the search for yourself in the spendbrowser</a></li>
+<li><a href="http://blog.openspending.org/2012/02/24/how-spending-stories-fact-checks-big-brother-the-wiretappers-ball/">Read more about the story</a></li>
+</ul>
+
+#### Italian Regional Accounts Data
+<img class="pull-left" style="margin-right: 1em;" title="Caelainn Barr" src="http://farm6.staticflickr.com/5224/5639223572_5451048271.jpg" alt="" width="250" />
+
+During the International Journalism Festival in Perugia in 2011, the OpenSpending team loaded and visualised the Italian regional accounts for 2008. The project received wide coverage in the Italian and International Press and was one of the earliest success stories for OpenSpending after Where Does My Money Go? went international.
+
+<a href="http://openspending.org/it-regional-accounts">Visit the Italian Regional Accounts</a> on OpenSpending.org.
+
+Coverage of the Italian Regional Accounts Data:
+
+<ul>
+<li><a href="http://www3.lastampa.it/economia/sezioni/articolo/lstp/398705/">La Stampa</a></li>
+<li><a href="http://www.guardian.co.uk/news/datablog/interactive/2011/apr/19/italy-public-spending-visualisation">The Guardian</a></li>
+<li><a href="http://daily.wired.it/news/economia/2011/04/19/open-spending.html">Daily Wired</a></li>
+</ul>
+
+## Training
+
+<img class="pull-right" style="margin-left: 3em;" title="Caelainn Barr" src="http://farm7.staticflickr.com/6166/6151919267_897ccfbd1c.jpg" alt="" width="250" />
+
+We regularly conduct training sessions for journalists and NGOs on how to locate, extract, work with and visualise spending and other types of data. If you are interested in exploring these possibilities, please [get in touch](/about/contact/).
+
+<ul>
+<li><a href="http://blog.okfn.org/2011/08/09/data-driven-journalism-workshop-on-eu-spending-tools-techniques-utrecht-8th-9th-september/">Workshop on EU Spending, Utrecht</a></li>
+<li><a href="http://blog.okfn.org/2011/09/27/eurohack-one-day-data-journalism-competition-and-workshop-on-eu-spending/">EuroHack: One-day data journalism competition and workshop on EU spending</a></li>
+<li><a href="http://blog.openspending.org/2012/11/26/day-1-openspending-cso-workshop-sarajevo/">Budget and Spending Visualisation Workshop, Sarajevo</a></li>
+</ul>
+You can find more of our training materials on the <a href="http://schoolofdata.org/handbook/courses/">School of Data website</a>.
+
+## Community
+
+<img class="pull-right" style="margin-left: 1em;" title="Kaitlin Lee presents Sunlight's Analysis OGDCamp" src="http://farm7.staticflickr.com/6166/6270108254_5875c8a7ed.jpg" alt="" width="250" />
+
+The OpenSpending community includes engaged citizens, dedicated journalists and members of civil society organisations working on developing best practices around opening up and using government financial data with experts from fields ranging from aid experts, participatory budgeting fields, governmental institutions and civic developer initiatives.
+
+We showcase and display some of the most interesting projects from all over the world on [the OpenSpending blog](/blog/).
+
+The working group is open to everyone with an interest in improving Government Financial Transparency around the world. If you're interested in joining, please join the <a href="https://discuss.okfn.org/c/openspending">forums</a>.
+
+<a name="your-project"></a>
+
+## Contribute
+
+#### Your project here
+Have some data? Have a Spending Story? Get in touch - we'd love to hear from you and are open to suggestions!
+
+

examples/openspending/content/about/faq.md

@@ -0,0 +1,43 @@
+---
+section: about
+lead: true
+title: Frequently Asked Questions
+authors:
+- Neil Ashton
+---
+#### What is OpenSpending?
+
+OpenSpending is about Mapping the Money.
+
+The aim is to help track every (public) government and corporate financial transaction across the world and present it in useful and engaging forms for everyone from a schoolchild to a data geek.
+
+What [OpenStreetMap](http://www.openstreetmap.org/) does for geography, OpenSpending does for money. OpenStreetMap has mapped the world in unprecedented levels of detail, harnessing the power of thousands of volunteers who each contribute data for their little corner of the world. However, as far as we know, there is no 'global atlas' of spending, no integrated, searchable database which would be a valuable resource for policy-makers and civil society alike. We want anyone to be able to go to their local council or national government, request the data, upload, understand and visualize it and contribute to this 'spending commons', which anyone can benefit from.
+
+#### What is the relationship to Where Does My Money Go?
+
+Simply put, OpenSpending is the international version of [Where Does My Money Go?](http://wheredoesmymoneygo.org/) (WDMMG). WDMMG used interactive 'bubble trees' to show the UK Taxpayer which spending sectors their tax money is put towards as well as having an interactive tax calculator to let users see for their level of income, how much they contribute to various spending areas through their taxes.
+
+##### Platform vs. project
+
+After the success of [the original project](http://jonathangray.org/2007/04/02/where-does-my-money-go-project-proposal/), it became clear that there was need for an international platform to allow people to build such sites at scale. Where Does My Money Go? was remodelled to become an instance of OpenSpending: the data is stored in the central OpenSpending database, and a custom frontend was developed using the OpenSpending API. Anyone can create one just like it, with their own branding at their own URL.
+
+#### Can anyone contribute?
+
+Yes!  Whether you are interested in international, national or local data, you can contribute, work with and visualize it. See [How can I get involved?](../../help/development/volunteer) for the various ways you can get involved.
+
+#### How can I get involved?
+
+The team has worked hard to make it possible to load and visualize your own data, translate the software and even build your own custom front-end sites like [Where Does My Money Go?](http://wheredoesmymoneygo.org/)
+
+Whether you are technical or non-technical, whether your aims are advocacy, journalism, better policy-making or just making pretty bubbles to help people understand your country's budget, there are plenty of ways to get involved. And don't be shy - if you spot a way to get involved that we haven't thought of, please don't hesitate to let us know.
+
+See the [get involved](../../help/development/volunteer) page for more information.
+
+#### What kind of language support do you offer?
+
+We currently provide language versions of the OpenSpending platform in Greek, Indonesian and Italian. We expect to enable French as well as other languages in the near future.
+You can help translate OpenSpending into another language at our [Transifex page](https://www.transifex.com/projects/p/openspending/).
+
+#### How are you funded?
+
+OpenSpending is a community-run project, but it would not be able to exist without the generous support of many organizations. See [this page](../funders/) for more information.

examples/openspending/content/about/fiscaldatapackage.md

@@ -0,0 +1,23 @@
+---
+title: The Fiscal Data Package
+redirect_from:
+  - "/about/governance/"
+---
+
+The Fiscal Data Package is a lightweight and user-oriented format for publishing and consuming fiscal data. Fiscal data packages are made of simple and universal components. They can be produced from ordinary spreadsheet software and used in any environment.
+
+The motivation behind the fiscal data package was to create a specification which is open by nature - based on other open standards, supported by open tools and software, modular, extensible and promoted transparently by a large community.
+
+It is designed to be simple to use - providing a small but flexible set of features, based on real-world requirements and not theoretical ones. All the while, the built-in extensibility allows this spec to adapt to many different use cases and domains. It is also possible to gradually use more and more part of this specification - thus making it easier to implement this spec with existing data while slowly improving the data quality.
+
+A main concern of this specification is the ability to work with data as it is currently exists, without forcing publishers to modify the contents or structure of their current data files in order to "adapt" them to the specification.
+
+#### The Open Fiscal Data Package serves two main purposes:
+Standardizing the structure and the content of fiscal data so that tools and services can be built over it for visualization, analysis or comparison;
+Driving data quality by providing a solid framework of publication.
+
+So the Open Fiscal Data Package specifies the form for fiscal data and offers a standardized framework for the content.
+
+- Read the [Fiscal Data Package specification](https://frictionlessdata.io/specs/fiscal-data-package/) on the Frictionless Data website
+- Meet the [OpenSpending tools](/about/tools)
+- Read about 

examples/openspending/content/about/funders.md

@@ -0,0 +1,36 @@
+---
+section: about
+lead: true
+title: Our funders
+authors:
+- Neil Ashton
+---
+OpenSpending is a community-driven project, but we would not be able to run it without the generous support of these organizations:
+
+#### 4IP
+
+<img src="http://blog.openspending.org/files/2013/09/4ip-300x79.jpg" alt="4ip" width="300" height="79" class="alignright size-medium wp-image-1056" />
+
+It was thanks to Channel 4's Information Program that [Where Does My Money Go?](http://wheredoesmymoneygo.org/) was born. 'Where Does My Money Go?' was the original version of OpenSpending for the United Kingdom and has inspired many international versions of the site.
+
+#### Open Society Foundations
+
+<img src="http://blog.openspending.org/files/2013/09/osf-300x70.jpg" alt="Basic CMYK" width="300" height="70" class="alignright size-medium wp-image-1057" />
+
+The Open Society Foundations support community building work around Open Spending Data. With this project we aim to help more groups and individuals around the world to use and work with spending data more effectively to do the things they care about – whether this is investigative journalism, evidence based policy-making, political campaigning, budgeting or creating new useful applications and services.
+
+In particular, we would like to document and spread best practices in the legal and technical aspects of reusing public information, and enabling re-use and better collaboration around this material.
+
+Read [the announce post]( http://blog.openspending.org/2012/01/12/civil-society-and-spending-data-who-is-mapping-the-money/) for more information.
+
+#### Knight Foundation
+
+<img src="http://blog.openspending.org/files/2013/09/kf-300x71.png" alt="kf" width="300" height="71" class="alignright size-medium wp-image-1058" />
+
+The Knight Foundation support the development of the [Spending Stories](http://blog.okfn.org/2011/06/22/spending-stories-is-a-winner-of-the-knight-news-challenge/) project. Spending Stories is a tool designed to speed up fact checking around spending data, helping journalists to connect raw data with their stories and build context around data with visualisations and auxilliary information.
+
+#### Omidyar Network
+
+<img src="http://blog.openspending.org/files/2013/09/on-300x100.jpg" alt="on" width="300" height="100" class="alignright size-medium wp-image-1059" />
+
+The Omidyar Network funded the research behind the report 'Technology for Transparent and Accountable Public Finance', which was presented at the Global Initiative for Fiscal Transparency meeting in Brasilia in April 2012.