[examples/openspending,home][m]: add min date and max date facets

- Also added an indexing example

examples/fivethirtyeight/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/fivethirtyeight/README.md

@@ -0,0 +1,44 @@
+This is a replica of the awesome data.fivethirtyeight.com using PortalJS.
+
+You might be asking why we did that, there are three main reasons:
+
+- The website has a great UI, with multiple datasets being displayed elegantly and with simplicity.
+- PortalJS allows us to add more functionality to it e.g dataset previews and search functionality.
+- The project follows our same principles of open sourcing and free data, with every dataset being publicly available on Github.
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file.
+
+[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
+
+The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

examples/fivethirtyeight/components/Breadcrumbs.tsx

@@ -0,0 +1,23 @@
+import Link from "next/link";
+
+function HomeIcon({ className = "" }) {
+  return <div className={`inline-block w-4 ${className}`}><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">    <path d="M 12 2 A 1 1 0 0 0 11.289062 2.296875 L 1.203125 11.097656 A 0.5 0.5 0 0 0 1 11.5 A 0.5 0.5 0 0 0 1.5 12 L 4 12 L 4 20 C 4 20.552 4.448 21 5 21 L 9 21 C 9.552 21 10 20.552 10 20 L 10 14 L 14 14 L 14 20 C 14 20.552 14.448 21 15 21 L 19 21 C 19.552 21 20 20.552 20 20 L 20 12 L 22.5 12 A 0.5 0.5 0 0 0 23 11.5 A 0.5 0.5 0 0 0 22.796875 11.097656 L 12.716797 2.3027344 A 1 1 0 0 0 12.710938 2.296875 A 1 1 0 0 0 12 2 z"/></svg></div>
+}
+
+export default function Breadcrumbs({ links }: { links: { title: string, href?: string, target?: string }[] }) {
+  const current = links.at(-1);
+
+  return <div className="flex items-center uppercase font-black text-xs">
+    <Link className="flex items-center" href='/'><HomeIcon /></Link>
+
+    {/* {links.length > 1 && links.slice(0, -1).map((link) => {
+      return <>
+        <span className="mx-4">/</span>
+        <Link href={link.href}>{link.title}</Link>
+      </>
+    })} */}
+
+    <span className="mx-4">/</span>
+    <span>{current?.title}</span>
+  </div >
+}

examples/fivethirtyeight/lib/octokit.ts

@@ -0,0 +1,38 @@
+import { Octokit } from 'octokit';
+
+export interface GithubProject {
+  owner: string;
+  repo: string;
+  branch: string;
+  files: string[];
+  readme: string;
+  description?: string;
+  name?: string;
+}
+
+export async function getProjectReadme(
+  owner: string,
+  repo: string,
+  branch: string,
+  readme: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.getContent({
+      owner,
+      repo,
+      path: readme,
+      ref: branch,
+    });
+    const data = response.data as { content?: string };
+    const fileContent = data.content ? data.content : '';
+    if (fileContent === '') {
+      return null;
+    }
+    const decodedContent = Buffer.from(fileContent, 'base64').toString();
+    return decodedContent;
+  } catch (error) {
+    return null;
+  }
+}

examples/fivethirtyeight/next.config.js

@@ -0,0 +1,9 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  reactStrictMode: true,
+  serverRuntimeConfig: {
+    github_pat: process.env.GITHUB_PAT ? process.env.GITHUB_PAT : null,
+  },
+}
+
+module.exports = nextConfig

examples/fivethirtyeight/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "fiverthirtyeight-example",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@portaljs/components": "^0.1.0",
+    "@tailwindcss/typography": "^0.5.9",
+    "@types/node": "20.1.1",
+    "@types/react": "18.2.6",
+    "@types/react-dom": "18.2.4",
+    "autoprefixer": "10.4.14",
+    "eslint": "8.40.0",
+    "eslint-config-next": "13.4.1",
+    "flexsearch": "^0.7.31",
+    "next": "13.4.1",
+    "next-mdx-remote": "^4.4.1",
+    "next-seo": "^6.0.0",
+    "octokit": "^2.0.14",
+    "postcss": "8.4.23",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-markdown": "^8.0.7",
+    "remark-code-frontmatter": "^1.0.0",
+    "remark-extract-frontmatter": "^3.2.0",
+    "remark-frontmatter": "^4.0.1",
+    "remark-gfm": "^3.0.1",
+    "tailwindcss": "3.3.2",
+    "timeago.js": "^4.0.2",
+    "typescript": "5.0.4"
+  }
+}

examples/fivethirtyeight/pages/_app.tsx

@@ -0,0 +1,8 @@
+import '@/styles/globals.css'
+import '@portaljs/components/styles.css'
+
+import type { AppProps } from 'next/app'
+
+export default function App({ Component, pageProps }: AppProps) {
+  return <Component {...pageProps} />
+}

examples/fivethirtyeight/pages/_document.tsx

@@ -0,0 +1,96 @@
+import { Html, Head, Main, NextScript } from 'next/document';
+
+export default function Document() {
+  return (
+    <Html lang="en">
+      <Head>
+        <link
+          rel="icon"
+          type="image/x-icon"
+          href="https://projects.fivethirtyeight.com/shared/favicon.ico"
+        />
+        <meta
+          property="og:image"
+          content="https://portaljs-fivethirtyeight.vercel.app/share_image.png"
+        />
+        <meta
+          property="twitter:image"
+          content="https://portaljs-fivethirtyeight.vercel.app/share_image.png"
+        />
+      </Head>
+      <body>
+        <div className="px-2 max-w-5xl mx-auto pb-2">
+          <div className="mt-2 px-2 bg-[#3c3c3c] text-white">
+            <div className="p-2 text-center">
+              This is a replica to the awesome{' '}
+              <a
+                className="hover:underline font-bold"
+                href="https://data.fivethirtyeight.com"
+              >
+                data.fivethirtyeight.com
+              </a>{' '}
+              website.{' '}
+              <a
+                className="hover:underline font-bold"
+                href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
+              >
+                Read more here
+              </a>{' '}
+            </div>
+          </div>
+        </div>
+        <header className="max-w-5xl mx-auto mt-8 w-full">
+          <div className="border-b-2 pb-2.5 mx-2 border-zinc-800 flex justify-between">
+            <h1 className="flex gap-x-1 items-end">
+              <span className="sr-only">FiveThirtyEight</span>
+              <img
+                width="197"
+                height="25"
+                alt="FiveThirtyEight"
+                src="data:image/svg+xml;base64,PHN2ZyBpZD0iTGF5ZXJfMSIgZGF0YS1uYW1lPSJMYXllciAxIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjEgNTMuNzYiPjxkZWZzPjxzdHlsZT4uY2xzLTF7ZmlsbDojMDEwMTAxO308L3N0eWxlPjwvZGVmcz48dGl0bGU+QXJ0Ym9hcmQgOTU8L3RpdGxlPjxwYXRoIGNsYXNzPSJjbHMtMSIgZD0iTTAgMGgyNXY4SDl2MTBoMTV2OEg5djE3SDBWMHpNMzEgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdIMzF6bTUtMzZoOHY4aC04ek0xNzkgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdoLTE3em01LTM2aDh2OGgtOHpNMzE2IDM2aDVWMThoLTV2LThoMTN2MjZoNHY3aC0xN3ptNS0zNmg4djhoLTh6TTU0IDI3VjEwaDh2MTVsNCA5Ljk4aDFMNzEgMjVWMTBoOHYxN2wtNyAxNkg2MWwtNy0xNnpNMTExIDQzSDk3LjQyQzg5LjIzIDQzIDg1IDM5LjE5IDg1IDMxLjE3VjIyYzAtNy41NyA0LjMtMTMgMTMtMTMgOS4zMyAwIDEzIDUuMDcgMTMgMTR2N0g5NHYxLjc0YzAgMi42MiAxIDQuMjYgMy40MiA0LjI2SDExMXpNOTQgMjNoOHYtMS41NWMwLTIuNjItMS4wNi01LjQ1LTQuMTMtNS40NS0yLjc5IDAtMy44NyAyLjItMy44NyA1LjQ1ek0xMjUgOGgtMTBWMGgyOXY4aC0xMHYzNWgtOVY4ek0yMDIgNDNWMTBoOHY0YzEuMTQtMi40NSAzLjc1LTQgNy4yMi00SDIyMHY4aC02Yy0yLjg0IDAtNCAuOTQtNCAzLjlWNDN6TTI0NSA0M2gtNC44NEMyMzMuMDUgNDMgMjMwIDM5LjMxIDIzMCAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0gyNDV6TTQyMSA0M2gtNC44NEM0MDkuMDUgNDMgNDA2IDM5LjMxIDQwNiAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0g0MjF6TTI1NC4yNiA1My43Nmw0LjYxLTkuNUwyNTEgMjdWMTBoOHYxNWw0IDEwaDFsNC0xMFYxMGg4djE3bC0xMi4zIDI2Ljc2aC05LjQ0ek0yODQgMGgyNXY4aC0xNnY5aDE1djhoLTE1djEwaDE2djhoLTI1VjB6TTMzNyA0OHYtMmgxNi4xYzIgMCAyLjktLjE4IDIuOS0xLjI3di0uMzRjMC0xLjA4LS45MS0xLjM5LTIuOS0xLjM5SDM0MHYtNWw1LTVjLTUuMjktMS40OC04LTUuNDMtOC0xMXYtMWMwLTcuNTYgNC40NC0xMiAxNC0xMmEyMS45MyAyMS45MyAwIDAgMSA1Ljk1IDFMMzYxIDRsNSAzLTQgNmMxLjM3IDEuOTMgMyA0LjkzIDMgOHYxYzAgNy0zLjMgMTAuNjYtMTIgMTFsLTMgNGg2YzUuOTIgMCA5IDIuNjIgOSA3LjY4di4xMWMwIDUuMDYtMi43MSA4LjIxLTguNjIgOC4yMWgtMTNjLTQuMjkgMC02LjM4LTEuODQtNi4zOC01em0xOS0yNXYtM2MwLTMuMy0xLjMzLTQtNS00cy01IC43LTUgNHYzYzAgMy4zIDEuMzkgNCA1IDRzNS0uNyA1LTR6TTM4MCA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuNC00IDctNCA2LjI2IDAgOSAzLjA4IDkgMTAuNzZWNDNoLThWMjJjMC0zLjEzLTEuMDctNS00LTVzLTQgMS44Ny00IDV6TTE1NyA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuOTEtNCA3LjQ5LTQgNi4yNiAwIDguNTEgMy4xMyA4LjUxIDEwLjgxVjQzaC04VjIxYzAtMy4xMy0xLjA3LTQuNDQtNC00LjQ0cy00IDIuMjYtNCA1LjM5eiIvPjwvc3ZnPg=="
+              />{' '}
+              <span className="-mb-0.5 text-[#3c3c3c]">replica</span>
+            </h1>
+            <div className="md:flex items-center gap-x-3 text-[#3c3c3c] -mb-1 hidden">
+              <a
+                className="hover:opacity-75 transition"
+                href="https://portaljs.org"
+              >
+                Built with 🌀PortalJS
+              </a>
+              <hr className="h-[80%] border border-[#3c3c3c] opacity-75 my-2"></hr>
+              <a
+                className="hover:opacity-75 transition"
+                href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
+              >
+                Github
+              </a>
+            </div>
+          </div>
+          <div className="mx-2 py-1.5 text-[14px] text-[#3c3c3c] md:hidden">
+            <ul className="flex gap-x-4">
+              <li>
+                <a
+                  className="hover:opacity-75 transition"
+                  href="https://portaljs.org"
+                >
+                  PortalJS
+                </a>
+              </li>
+              <li>
+                <a
+                  className="hover:opacity-75 transition"
+                  href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
+                >
+                  View on Github
+                </a>
+              </li>
+            </ul>
+          </div>
+        </header>
+        <Main />
+        <NextScript />
+      </body>
+    </Html>
+  );
+}

examples/fivethirtyeight/pages/datasets/[datasetName].tsx

@@ -0,0 +1,131 @@
+import { NextSeo } from 'next-seo';
+import { promises as fs } from 'fs';
+import path from 'path';
+import getConfig from 'next/config';
+import { getProjectReadme, GithubProject } from '@/lib/octokit';
+import remarkGfm from 'remark-gfm';
+import extract from 'remark-extract-frontmatter';
+import { Dataset } from '..';
+import { GetStaticProps } from 'next';
+import { Table } from '@portaljs/components';
+import Breadcrumbs from '@/components/Breadcrumbs';
+import { ReactMarkdown } from 'react-markdown/lib/react-markdown';
+import remarkFrontmatter from 'remark-frontmatter';
+
+export default function DatasetPage({
+  dataset,
+}: {
+  dataset: Dataset & {
+    readme: string | null;
+  };
+}) {
+  return (
+    <>
+      <NextSeo title={`${dataset.name} page`} />
+      <main className="max-w-5xl px-2 prose mx-auto my-8 prose-thead:border-b-4 prose-table:max-w-5xl prose-table:overflow-scroll prose-thead:overflow-scroll prose-tbody:overflow-scroll prose-thead:pb-2 prose-thead:border-zinc-900 prose-th:uppercase prose-th:text-left prose-th:font-light prose-th:text-xs">
+        <Breadcrumbs links={[{ title: dataset.name, href: '' }]} />
+        <h1 className="uppercase mb-0 mt-16">{dataset.name}</h1>
+        <p className="mb-8">
+          <span className="font-semibold">Repository:</span>{' '}
+          <a target="_blank" href={dataset.url}>
+            {dataset.url}
+          </a>
+        </p>
+
+        <h2 className="mb-0 mt-10">FILES</h2>
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="min-w-full divide-y divide-gray-300">
+            <thead className="border-b-4 pb-2 border-zinc-900">
+              <tr>
+                <th
+                  className="uppercase text-left font-light text-xs pb-3"
+                  scope="col"
+                >
+                  Name
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              {dataset.files?.map((file) => (
+                <tr key={file}>
+                  <td className="whitespace-nowrap text-left py-4 text-sm text-gray-500">
+                    <a href={file}>{file.split('/').slice(-1)}</a>
+                  </td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        </div>
+        {dataset.files && dataset.files.length > 0 && (
+          <>
+            <h2 className="mb-0 mt-10">DATA PREVIEWS</h2>
+            {dataset.files?.map((file) => (
+              <div key={file} className="preview-table my-8">
+                <h3>{file.split('/').slice(-1)}</h3>
+                <Table url={file} />
+              </div>
+            ))}
+          </>
+        )}
+        {dataset.readme && (
+          <>
+            <h2 className="uppercase font-black">Readme</h2>
+            {dataset.readme && (
+              <ReactMarkdown
+                remarkPlugins={[
+                  remarkFrontmatter,
+                  remarkGfm,
+                  [extract, { remove: true }],
+                ]}
+              >
+                {dataset.readme}
+              </ReactMarkdown>
+            )}
+          </>
+        )}
+      </main>
+    </>
+  );
+}
+
+export async function getStaticPaths() {
+  const datasetsFile = path.join(process.cwd(), 'datasets.json');
+  const datasets = await fs.readFile(datasetsFile, 'utf8');
+
+  return {
+    paths: JSON.parse(datasets).map((dataset: Dataset) => {
+      return {
+        params: { datasetName: dataset.name },
+      };
+    }),
+    fallback: false, // can also be true or 'blocking'
+  };
+}
+
+export const getStaticProps: GetStaticProps = async ({ params }) => {
+  const datasetsFile = path.join(process.cwd(), 'datasets.json');
+  const datasetsString = await fs.readFile(datasetsFile, 'utf8');
+  const datasets: Dataset[] = JSON.parse(datasetsString);
+  const dataset: Dataset | undefined = datasets.find(
+    (_dataset) => _dataset.name === params?.datasetName
+  );
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+  const readmes = await Promise.all(['/README.md', '/readme.md', '/Readme.md'].map(async (readme) => await getProjectReadme(
+    'fivethirtyeight',
+    'data',
+    'master',
+    dataset?.name + readme,
+    github_pat
+  )));
+  const readme = readmes.find(item => item !== null)
+  if (!readme) console.log('Readme not found for ' + dataset?.name)
+  return {
+    props: {
+      dataset: {
+        ...dataset,
+        readme,
+        files: dataset && dataset.files ? dataset.files : null,
+      },
+    },
+  };
+};

examples/fivethirtyeight/pages/index.tsx

@@ -0,0 +1,211 @@
+import Image from 'next/image';
+import { Inter } from 'next/font/google';
+import { format } from 'timeago.js';
+import { promises as fs } from 'fs';
+import path from 'path';
+import { NextSeo } from 'next-seo';
+
+const inter = Inter({ subsets: ['latin'] });
+
+export interface Article {
+  date: string;
+  title: string;
+  url: string;
+}
+
+export interface Dataset {
+  url: string;
+  name: string;
+  displayName: string;
+  articles: Article[];
+  files?: string[];
+}
+
+// Request a weekday along with a long date
+const options = {
+  year: 'numeric',
+  month: 'long',
+  day: 'numeric',
+} as const;
+
+export function MobileItem({ dataset }: { dataset: Dataset }) {
+  return (
+    <div className="flex gap-x-2 pb-2 py-4 items-center justify-between border-b border-zinc-600">
+      <div className="flex flex-col">
+        <span className="font-mono font-light">{dataset.name}</span>
+        {dataset.articles.map((article) => (
+          <div key={article.title} className="py-1 flex flex-col">
+            <span className="font-bold hover:underline">{article.title}</span>
+            <span className="font-light text-base">
+              {format(article.date).includes('years')
+                ? new Date(article.date).toLocaleString('en-US', options)
+                : format(article.date)}
+            </span>{' '}
+          </div>
+        ))}
+      </div>
+      <div className="flex flex-col justify-start">
+        <a
+          className="ml-2 border border-zinc-900 font-light px-4 py-1 text-sm transition hover:bg-zinc-900 hover:text-white"
+          href={dataset.url}
+        >
+          info
+        </a>
+        {/*
+                <button>
+                  <svg
+                    xmlns="http://www.w3.org/2000/svg"
+                    viewBox="0 0 24 24"
+                    fill="currentColor"
+                    className="w-12 h-12 text-blue-400 hover:text-blue-300 transition mt-1"
+                  >
+                    <path
+                      fillRule="evenodd"
+                      d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
+                      clipRule="evenodd"
+                    />
+                  </svg>
+                </button> */}
+      </div>
+    </div>
+  );
+}
+
+export function DesktopItem({ dataset }: { dataset: Dataset }) {
+  return (
+    <>
+      {dataset.articles.map((article, index) => (
+        <tr
+          key={article.url}
+          className={`${
+            index === dataset.articles.length - 1 ? 'border-b' : ''
+          } border-zinc-400`}
+        >
+          <td className="py-8 font-light font-mono text-[13px] text-zinc-700">
+            {index === 0 ? dataset.name : ''}
+          </td>
+          <td>
+            <a
+              className="py-8 font-bold hover:underline pr-2"
+              href={article.url}
+            >
+              {article.title}
+            </a>
+          </td>
+          <td className="py-8 font-light text-[14px] min-w-[138px] font-mono text-[#999]">
+            {format(article.date).includes('years')
+              ? new Date(article.date).toLocaleString('en-US', options)
+              : format(article.date)}
+          </td>
+          <td className="py-8">
+            {index === 0 && (
+              <a
+                className="ml-2 border border-zinc-900 font-light px-[25px] py-2.5 text-sm transition hover:bg-zinc-900 hover:text-white"
+                href={dataset.url}
+              >
+                info
+              </a>
+            )}
+          </td>
+          {/*
+              <td>
+                <button>
+                  <svg
+                    xmlns="http://www.w3.org/2000/svg"
+                    viewBox="0 0 24 24"
+                    fill="currentColor"
+                    className="w-12 h-12 text-blue-400 hover:text-blue-300 transition mt-1"
+                  >
+                    <path
+                      fillRule="evenodd"
+                      d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
+                      clipRule="evenodd"
+                    />
+                  </svg>
+                </button>
+              </td>*/}
+        </tr>
+      ))}
+    </>
+  );
+}
+
+export async function getStaticProps() {
+  const jsonDirectory = path.join(process.cwd(), '/datasets.json');
+  const datasetString = await fs.readFile(jsonDirectory, 'utf8');
+  const datasets = JSON.parse(datasetString);
+  return {
+    props: { datasets },
+  };
+}
+
+export default function Home({ datasets }: { datasets: Dataset[] }) {
+  return (
+    <>
+      <NextSeo title="FiveThirtyEight tribute by PortalJS" />
+      <main
+        className={`flex min-h-screen flex-col items-center max-w-5xl mx-auto pt-20 px-2.5 ${inter.className}`}
+      >
+        <div>
+          <h1 className="text-[40px] font-bold text-zinc-800 text-center">
+            Our Data
+          </h1>
+          <p className="max-w-[600px] text-[17px] text-center text-[#6d6f71]">
+            We’re sharing the data and code behind some of our articles and
+            graphics. We hope you’ll use it to check our work and to create
+            stories and visualizations of&nbsp;your&nbsp;own.
+          </p>
+        </div>
+        <article className="w-full px-2 md:hidden py-4">
+          {datasets.map((dataset) => (
+            <MobileItem key={dataset.name} dataset={dataset} />
+          ))}
+        </article>
+        <table className="w-full mt-10 mb-4 hidden md:table">
+          <thead className="border-b-4 pb-2 border-zinc-900">
+            <tr>
+              <th className="uppercase text-left font-normal text-xs pb-3">
+                data set
+              </th>
+              <th className="uppercase text-left font-normal text-xs pb-3">
+                related content
+              </th>
+              <th className="uppercase text-left font-normal text-xs pb-3">
+                last updated
+              </th>
+            </tr>
+          </thead>
+          <tbody>
+            {datasets.map((dataset) => (
+              <DesktopItem key={dataset.name} dataset={dataset} />
+            ))}
+          </tbody>
+        </table>
+        <p className="text-[13px] py-8">
+          Unless otherwise noted, our data sets are available under the{' '}
+          <a
+            className="text-blue-400 hover:underline"
+            href="http://creativecommons.org/licenses/by/4.0/"
+          >
+            Creative Commons Attribution 4.0 International license
+          </a>
+          , and the code is available under the{' '}
+          <a
+            className="text-blue-400 hover:underline"
+            href="http://opensource.org/licenses/MIT"
+          >
+            MIT license
+          </a>
+          . If you find this information useful, please{' '}
+          <a
+            className="text-blue-400 hover:underline"
+            href="mailto:data@fivethirtyeight.com"
+          >
+            let us know
+          </a>
+          .
+        </p>
+      </main>
+    </>
+  );
+}

examples/fivethirtyeight/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

examples/fivethirtyeight/public/vercel.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

examples/fivethirtyeight/styles/globals.css

@@ -0,0 +1,8 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+.preview-table > div {
+  overflow-x: scroll;
+  overflow-y: hidden;
+}

examples/fivethirtyeight/tailwind.config.js

@@ -0,0 +1,18 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    './pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './components/**/*.{js,ts,jsx,tsx,mdx}',
+    './app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      backgroundImage: {
+        'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
+        'gradient-conic':
+          'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
+      },
+    },
+  },
+  plugins: [require('@tailwindcss/typography')],
+};

examples/fivethirtyeight/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+  "exclude": ["node_modules"]
+}

examples/github-backed-catalog/README.md

@@ -0,0 +1,102 @@
+# A data catalog with data on GitHub
+
+This example showcases a simple data catalog that get its data from a list of GitHub repos that serve as datasets.
+
+A `datasets.json` file is used to specify which datasets are going to be part of the data catalog.
+
+The application contains an index page, which lists all the datasets specified in the `datasets.json` file, and users can see more information about each dataset, such as the list of data files in it and the README, by clicking the "info" button on the list.
+
+You can read more about it on the [Data catalog with data on GitHub](https://portaljs.org/docs/examples/github-backed-catalog) blog post.
+
+## Demo
+
+https://example.portaljs.org/
+
+## Deploy your own
+
+[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fdatopian%2Fportaljs%2Ftree%2Fmain%2Fexamples%2Fgithub-backed-catalog)
+
+By clicking on this button, you will be redirected to a page which will allow you to clone the content into your own GitHub/GitLab/Bitbucket account and automatically deploy everything.
+
+## How to use
+
+### Install
+
+Execute `create-next-app` to bootstrap the example:
+
+```
+npx create-next-app <app-name> --example https://github.com/datopian/portaljs/tree/main/examples/github-backed-catalog
+cd <app-name>
+```
+
+### Set environment variables
+
+This project uses the GitHub API, which for anonymous users will cap at 50 requests per hour, so you might want to get a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and add it to a `.env` file inside the folder like so
+
+```
+GITHUB_PAT=<github token>
+```
+
+### Change datasets
+
+You can change the datasets that will be displayed in the data catalog by editing the file `datasets.json`. Some examples can be found inside [this repo](https://github.com/datasets).
+
+### Run in development mode
+
+Run the app using:
+
+```
+npm run dev
+```
+
+Open http://localhost:3000 from your browser. You should see something similar to this:
+
+![](https://i.imgur.com/jAljJ9C.png)
+
+If click on the `info` button for a dataset you will see a page similar to this:
+
+![](https://i.imgur.com/AoJd4O0.png)
+
+## Notes
+
+### Structure of `datasets.json`
+
+The `datasets.json` file is simply a list of datasets, below you can see a minimal example of a dataset:
+
+```json
+{
+  "owner": "fivethirtyeight",
+  "repo": "data",
+  "branch": "master",
+  "files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"],
+  "readme": "nba-raptor/README.md"
+}
+```
+
+It has:
+
+- A `owner` which is going to be the github repo owner
+- A `repo` which is going to be the github repo name
+- A `branch` which is going to be the branch to which we need to get the files and the readme
+- A list of `files` which is going to be a list of paths with files that you want to show to the world
+- A `readme` which is going to be the path to your data description, it can also be a subpath eg: `example/README.md`
+
+You can also add:
+
+- A `description` which is useful if you have more than one dataset for each repo, if not provided we are just going to use the repo description
+- A `Name` which is useful if you want to give your dataset a nice name, if not provided we are going to use the junction of the `owner` the `repo` + the path of the README, in the exaple above it will be `fivethirtyeight/data/nba-raptor`
+
+### Extra commands
+
+You can also build the project for production with:
+
+```
+npm run build
+```
+
+And run the production build with:
+
+```
+npm run start
+```
+

examples/github-backed-catalog/components/_shared/Breadcrumbs.tsx

@@ -0,0 +1,20 @@
+import Link from "next/link";
+import HomeIcon from "../icons/HomeIcon";
+
+export default function Breadcrumbs({ links }: { links: { title: string, href?: string, target?: string }[] }) {
+  const current = links.at(-1);
+
+  return <div className="flex items-center uppercase font-black text-xs">
+    <Link className="flex items-center" href='/'><HomeIcon /></Link>
+
+    {/* {links.length > 1 && links.slice(0, -1).map((link) => {
+      return <>
+        <span className="mx-4">/</span>
+        <Link href={link.href}>{link.title}</Link>
+      </>
+    })} */}
+
+    <span className="mx-4">/</span>
+    <span>{current.title}</span>
+  </div >
+}

examples/github-backed-catalog/components/icons/ExternalLinkIcon.tsx

@@ -0,0 +1,3 @@
+export default function ExternalLinkIcon({ className = "" }) {
+  return <div className={`inline-block w-4 ${className}`}><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" fill="currentColor"><path d="M 40 10 C 38.896 10 38 10.896 38 12 C 38 13.104 38.896 14 40 14 L 47.171875 14 L 30.585938 30.585938 C 29.804938 31.366938 29.804938 32.633063 30.585938 33.414062 C 30.976938 33.805063 31.488 34 32 34 C 32.512 34 33.023063 33.805062 33.414062 33.414062 L 50 16.828125 L 50 24 C 50 25.104 50.896 26 52 26 C 53.104 26 54 25.104 54 24 L 54 12 C 54 10.896 53.104 10 52 10 L 40 10 z M 18 12 C 14.691 12 12 14.691 12 18 L 12 46 C 12 49.309 14.691 52 18 52 L 46 52 C 49.309 52 52 49.309 52 46 L 52 34 C 52 32.896 51.104 32 50 32 C 48.896 32 48 32.896 48 34 L 48 46 C 48 47.103 47.103 48 46 48 L 18 48 C 16.897 48 16 47.103 16 46 L 16 18 C 16 16.897 16.897 16 18 16 L 30 16 C 31.104 16 32 15.104 32 14 C 32 12.896 31.104 12 30 12 L 18 12 z"/></svg></div>
+}

examples/github-backed-catalog/components/icons/HomeIcon.tsx

@@ -0,0 +1,3 @@
+export default function HomeIcon({ className = "" }) {
+  return <div className={`inline-block w-4 ${className}`}><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">    <path d="M 12 2 A 1 1 0 0 0 11.289062 2.296875 L 1.203125 11.097656 A 0.5 0.5 0 0 0 1 11.5 A 0.5 0.5 0 0 0 1.5 12 L 4 12 L 4 20 C 4 20.552 4.448 21 5 21 L 9 21 C 9.552 21 10 20.552 10 20 L 10 14 L 14 14 L 14 20 C 14 20.552 14.448 21 15 21 L 19 21 C 19.552 21 20 20.552 20 20 L 20 12 L 22.5 12 A 0.5 0.5 0 0 0 23 11.5 A 0.5 0.5 0 0 0 22.796875 11.097656 L 12.716797 2.3027344 A 1 1 0 0 0 12.710938 2.296875 A 1 1 0 0 0 12 2 z"/></svg></div>
+}

examples/github-backed-catalog/package-lock.json

@@ -19,6 +19,7 @@
         "react": "18.2.0",
         "react-dom": "18.2.0",
         "react-markdown": "^8.0.7",
+        "react-timeago": "^7.1.0",
         "remark-gfm": "^3.0.1",
         "typescript": "5.0.4"
       },
@@ -4797,6 +4798,14 @@
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
       "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w=="
     },
+    "node_modules/react-timeago": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/react-timeago/-/react-timeago-7.1.0.tgz",
+      "integrity": "sha512-rouF7MiEm55fH791Y8cg+VobIJgx8gtNJ+gjr86R4ZqO1WKPkXiXjdT/lRzrvEkUzsxT1exHqV2V+Zdi114H3A==",
+      "peerDependencies": {
+        "react": "^16.0.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
     "node_modules/read-cache": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",

examples/github-backed-catalog/package.json

@@ -20,6 +20,7 @@
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-markdown": "^8.0.7",
+    "react-timeago": "^7.1.0",
     "remark-gfm": "^3.0.1",
     "typescript": "5.0.4"
   },

examples/github-backed-catalog/pages/@org/[org]/[...path].tsx

@@ -1,6 +1,3 @@
-import Head from 'next/head';
-import { useRouter } from 'next/router';
-
 import { NextSeo } from 'next-seo';
 import { promises as fs } from 'fs';
 import path from 'path';
@@ -8,15 +5,20 @@ import getConfig from 'next/config';
 import { getProject, GithubProject } from '../../../lib/octokit';
 import ReactMarkdown from 'react-markdown';
 import remarkGfm from 'remark-gfm';
-import Link from 'next/link';
+import Breadcrumbs from '../../../components/_shared/Breadcrumbs';
 
 export default function ProjectPage({ project }) {
+  const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`
+
   return (
     <>
-      <NextSeo title={`PortalJS - @${project.repo_config.owner}/${project.repo_config.repo}${project.base_path !== '/' ? '/' + project.base_path : ''}`} />
+      <NextSeo title={`${repoId}${project.base_path !== '/' ? '/' + project.base_path : ''} - GitHub Datasets`} />
       <main className="prose mx-auto my-8">
-        <Link href='/'>Back to homepage</Link>
-        <h1 className="mb-0">Data</h1>
+        <Breadcrumbs links={[{ title: repoId, href: "" }]} />
+        <h1 className="mb-0 mt-16">{project.repo_config.name || repoId}</h1>
+        <p className='mb-8'><span className='font-semibold'>Repository:</span> <a target="_blank" href={project.html_url}>{project.html_url}</a></p>
+
+        <h2 className="mb-0 mt-10">Files</h2>
         <div className="inline-block min-w-full py-2 align-middle">
           <table className="min-w-full divide-y divide-gray-300">
             <thead>
@@ -50,7 +52,9 @@ export default function ProjectPage({ project }) {
           </table>
         </div>
 
-        <h1>Readme</h1>
+        <hr />
+
+        <h2 className='uppercase font-black'>Readme</h2>
         <ReactMarkdown remarkPlugins={[remarkGfm]}>
           {project.readmeContent}
         </ReactMarkdown>

examples/github-backed-catalog/pages/_app.tsx

@@ -6,7 +6,7 @@ function CustomApp({ Component, pageProps }: AppProps) {
   return (
     <>
       <Head>
-        <title>Welcome to simple-example!</title>
+        <title>GitHub Datasets</title>
       </Head>
       <main className="app">
         <Component {...pageProps} />

examples/github-backed-catalog/pages/index.tsx

@@ -2,6 +2,9 @@ import { promises as fs } from 'fs';
 import path from 'path';
 import { getProject } from '../lib/octokit';
 import getConfig from 'next/config';
+import ExternalLinkIcon from '../components/icons/ExternalLinkIcon';
+import TimeAgo from 'react-timeago';
+import Link from 'next/link';
 
 export async function getStaticProps() {
   const jsonDirectory = path.join(
@@ -24,26 +27,18 @@ export async function getStaticProps() {
   };
 }
 
-const formatter = new Intl.DateTimeFormat('en-US', {
-  year: 'numeric',
-  month: 'long',
-  day: 'numeric',
-  hour: 'numeric',
-  minute: 'numeric',
-  second: 'numeric',
-  timeZone: 'UTC',
-});
-
 export function Datasets({ projects }) {
   return (
-    <div className="bg-white">
+    <div className="bg-white min-h-screen">
       <div className="mx-auto max-w-7xl px-6 py-16 sm:py-24 lg:px-8">
-        <h2 className="text-2xl font-bold leading-10 tracking-tight">
-          My Datasets
-        </h2>
-        <p className="mt-6 max-w-2xl text-base leading-7 text-gray-600">
-          Here is a list of all my datasets for easy access and sharing
-        </p>
+        <div className='text-center'>
+          <h2 className="text-3xl font-bold leading-10 tracking-tight">
+            GitHub Datasets
+          </h2>
+          <p className="mt-3 mx-auto max-w-2xl text-base leading-7 text-gray-500">
+            Data catalog with datasets hosted on GitHub by <Link target="_blank" className='underline' href="https://portaljs.org/">🌀 PortalJS</Link>
+          </p>
+        </div>
         <div className="mt-20">
           <div className="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
             <div className="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
@@ -60,7 +55,7 @@ export function Datasets({ projects }) {
                       scope="col"
                       className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
                     >
-                      Repo
+                      Repository
                     </th>
                     <th
                       scope="col"
@@ -83,27 +78,28 @@ export function Datasets({ projects }) {
                 <tbody className="divide-y divide-gray-200">
                   {projects.map((project) => (
                     <tr key={project.id}>
-                      <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      <td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
                         {project.repo_config.name
                           ? project.repo_config.name
                           : project.full_name + (project.base_path === '/' ? '' : '/' + project.base_path)}
                       </td>
-                      <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                        <a href={project.html_url}>{project.full_name}</a>
+                      <td className="whitespace-nowrap px-3 py-6 text-sm group text-gray-500 hover:text-gray-900 transition-all duration-250">
+                        <a href={project.html_url} target="_blank" className='flex items-center'>@{project.full_name} <ExternalLinkIcon className='ml-1' /></a>
                       </td>
                       <td className="px-3 py-4 text-sm text-gray-500">
                         {project.repo_config.description
                           ? project.repo_config.description
                           : project.description}
                       </td>
-                      <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                        {formatter.format(new Date(project.last_updated))}
+                      <td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
+                        <TimeAgo date={new Date(project.last_updated)} />
                       </td>
-                      <td className="relative whitespace-nowrap py-4 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
+                      <td className="relative whitespace-nowrap py-6 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
                         <a
                           href={`/@${project.repo_config.owner}/${project.repo_config.repo}/${project.base_path === '/' ? '' : project.base_path}`}
+                          className='border border-gray-900 text-gray-900 px-4 py-2 transition-all hover:bg-gray-900 hover:text-white'
                         >
-                          More info
+                          info
                         </a>
                       </td>
                     </tr>

examples/github-backed-catalog/pages/styles.css

@@ -0,0 +1,80 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+html {
+  -webkit-text-size-adjust: 100%;
+  font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont,
+    Segoe UI, Roboto, Helvetica Neue, Arial, Noto Sans, sans-serif,
+    Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol, Noto Color Emoji;
+  line-height: 1.5;
+  tab-size: 4;
+  scroll-behavior: smooth;
+}
+body {
+  font-family: inherit;
+  line-height: inherit;
+  margin: 0;
+}
+h1,
+h2,
+p,
+pre {
+  margin: 0;
+}
+*,
+::before,
+::after {
+  box-sizing: border-box;
+  border-width: 0;
+  border-style: solid;
+  border-color: currentColor;
+}
+h1,
+h2 {
+  font-size: inherit;
+  font-weight: inherit;
+}
+a {
+  color: inherit;
+  text-decoration: inherit;
+}
+pre {
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
+    Liberation Mono, Courier New, monospace;
+}
+svg {
+  display: block;
+  vertical-align: middle;
+  shape-rendering: auto;
+  text-rendering: optimizeLegibility;
+}
+pre {
+  background-color: rgba(55, 65, 81, 1);
+  border-radius: 0.25rem;
+  color: rgba(229, 231, 235, 1);
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
+    Liberation Mono, Courier New, monospace;
+  overflow: scroll;
+  padding: 0.5rem 0.75rem;
+}
+
+.shadow {
+  box-shadow: 0 0 #0000, 0 0 #0000, 0 10px 15px -3px rgba(0, 0, 0, 0.1),
+    0 4px 6px -2px rgba(0, 0, 0, 0.05);
+}
+.rounded {
+  border-radius: 1.5rem;
+}
+.wrapper {
+  width: 100%;
+}
+.container {
+  margin-left: auto;
+  margin-right: auto;
+  max-width: 768px;
+  padding-bottom: 3rem;
+  padding-left: 1rem;
+  padding-right: 1rem;
+  color: rgba(55, 65, 81, 1);
+  width: 100%;
+}

examples/github-backed-catalog/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

examples/openspending/.eslintrc.json

@@ -0,0 +1,32 @@
+{
+  "extends": [
+    "next",
+    "next/core-web-vitals"
+  ],
+  "ignorePatterns": ["!**/*", ".next/**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {
+        "@next/next/no-html-link-for-pages": [
+          "error",
+          "examples/simple-example/pages"
+        ]
+      }
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ],
+  "rules": {
+    "@next/next/no-html-link-for-pages": "off"
+  },
+  "env": {
+    "jest": true
+  }
+}

examples/openspending/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/openspending/README.md

@@ -0,0 +1,102 @@
+# A data catalog with data on GitHub
+
+This example showcases a simple data catalog that get its data from a list of GitHub repos that serve as datasets.
+
+A `datasets.json` file is used to specify which datasets are going to be part of the data catalog.
+
+The application contains an index page, which lists all the datasets specified in the `datasets.json` file, and users can see more information about each dataset, such as the list of data files in it and the README, by clicking the "info" button on the list.
+
+You can read more about it on the [Data catalog with data on GitHub](https://portaljs.org/docs/examples/github-backed-catalog) blog post.
+
+## Demo
+
+https://example.portaljs.org/
+
+## Deploy your own
+
+[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fdatopian%2Fportaljs%2Ftree%2Fmain%2Fexamples%2Fgithub-backed-catalog)
+
+By clicking on this button, you will be redirected to a page which will allow you to clone the content into your own GitHub/GitLab/Bitbucket account and automatically deploy everything.
+
+## How to use
+
+### Install
+
+Execute `create-next-app` to bootstrap the example:
+
+```
+npx create-next-app <app-name> --example https://github.com/datopian/portaljs/tree/main/examples/github-backed-catalog
+cd <app-name>
+```
+
+### Set environment variables
+
+This project uses the GitHub API, which for anonymous users will cap at 50 requests per hour, so you might want to get a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and add it to a `.env` file inside the folder like so
+
+```
+GITHUB_PAT=<github token>
+```
+
+### Change datasets
+
+You can change the datasets that will be displayed in the data catalog by editing the file `datasets.json`. Some examples can be found inside [this repo](https://github.com/datasets).
+
+### Run in development mode
+
+Run the app using:
+
+```
+npm run dev
+```
+
+Open http://localhost:3000 from your browser. You should see something similar to this:
+
+![](https://i.imgur.com/jAljJ9C.png)
+
+If click on the `info` button for a dataset you will see a page similar to this:
+
+![](https://i.imgur.com/AoJd4O0.png)
+
+## Notes
+
+### Structure of `datasets.json`
+
+The `datasets.json` file is simply a list of datasets, below you can see a minimal example of a dataset:
+
+```json
+{
+  "owner": "fivethirtyeight",
+  "repo": "data",
+  "branch": "master",
+  "files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"],
+  "readme": "nba-raptor/README.md"
+}
+```
+
+It has:
+
+- A `owner` which is going to be the github repo owner
+- A `repo` which is going to be the github repo name
+- A `branch` which is going to be the branch to which we need to get the files and the readme
+- A list of `files` which is going to be a list of paths with files that you want to show to the world
+- A `readme` which is going to be the path to your data description, it can also be a subpath eg: `example/README.md`
+
+You can also add:
+
+- A `description` which is useful if you have more than one dataset for each repo, if not provided we are just going to use the repo description
+- A `Name` which is useful if you want to give your dataset a nice name, if not provided we are going to use the junction of the `owner` the `repo` + the path of the README, in the exaple above it will be `fivethirtyeight/data/nba-raptor`
+
+### Extra commands
+
+You can also build the project for production with:
+
+```
+npm run build
+```
+
+And run the production build with:
+
+```
+npm run start
+```
+

examples/openspending/__tests__/os-data.test.ts

@@ -0,0 +1,45 @@
+import { Octokit } from 'octokit';
+import { assert, expect, test } from 'vitest'
+import { getProjectDataPackage } from '../lib/octokit';
+
+export async function getAllDataPackagesFromOrg(
+  org: string,
+  branch?: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  const repos = await octokit.rest.repos.listForOrg({ org, type: 'public', per_page: 100 });
+  let failedDataPackages = [];
+  const datapackages = await Promise.all(
+    repos.data.map(async (_repo) => {
+      const datapackage = await getProjectDataPackage(
+        org,
+        _repo.name,
+        branch ? branch : 'main',
+        github_pat
+      );
+      if (!datapackage) {
+        failedDataPackages.push(_repo.name)
+        return null
+      };
+      return {...datapackage, repo: _repo.name};
+    })
+  );
+  return {
+    datapackages: datapackages.filter((item) => item !== null),
+    failedDataPackages,
+  };
+}
+
+test('Test OS-Data', async () => {
+  const repos = await getAllDataPackagesFromOrg('os-data', 'main', process.env.VITE_GITHUB_PAT)
+  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
+  expect(repos.failedDataPackages.length).toBe(0)
+}, {timeout: 100000})
+
+test('Test Gift-Data', async () => {
+  const repos = await getAllDataPackagesFromOrg('gift-data', 'main', process.env.VITE_GITHUB_PAT)
+  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
+  expect(repos.failedDataPackages.length).toBe(0)
+}, {timeout: 100000})
+

examples/openspending/components/Button.tsx

@@ -0,0 +1,15 @@
+import Link from 'next/link';
+import clsx from 'clsx';
+
+export function Button({ href, className = '', ...props }) {
+  className = clsx(
+    'inline-flex justify-center rounded-2xl bg-emerald-600 p-4 text-base font-semibold text-white hover:bg-emerald-500 focus:outline-none focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-emerald-500 active:text-white/70',
+    className
+  );
+
+  return href ? (
+    <Link scroll={false} href={href} className={className} {...props} />
+  ) : (
+    <button className={className} {...props} />
+  );
+}

examples/openspending/components/Container.tsx

@@ -0,0 +1,10 @@
+import clsx from 'clsx'
+
+export function Container({ className = "", ...props }) {
+  return (
+    <div
+      className={clsx('mx-auto max-w-7xl px-4 sm:px-6 lg:px-8', className)}
+      {...props}
+    />
+  )
+}

examples/openspending/components/DatasetCard.tsx

@@ -0,0 +1,76 @@
+import Link from 'next/link';
+import { Project } from '../lib/project.interface';
+import ExternalLinkIcon from './icons/ExternalLinkIcon';
+
+export default function DatasetCard({ dataset }: { dataset: Project }) {
+  return (
+    <div
+      key={dataset.name}
+      className="overflow-hidden rounded-xl border border-gray-200"
+    >
+      <Link
+        href=""
+        className="flex items-center gap-x-4 border-b border-gray-900/5 bg-gray-50 p-6"
+      >
+        <img
+          src={dataset.owner.logo || '/assets/org-icon.svg'}
+          alt={dataset.owner.name}
+          className="h-12 w-12 flex-none rounded-lg bg-white object-cover ring-1 ring-gray-900/10 p-2"
+        />
+        <div className="text-sm font-medium leading-6">
+          <div className="text-gray-900 line-clamp-1">{dataset.title}</div>
+          <div className="text-gray-500 line-clamp-1">
+            {dataset.owner.title}
+          </div>
+        </div>
+      </Link>
+      <dl className="-my-3 divide-y divide-gray-100 px-6 py-4 text-sm leading-6">
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Name</dt>
+          <dd className="flex items-start gap-x-2">
+            <div className="font-medium text-gray-900 line-clamp-1">
+              {dataset.name}
+            </div>
+          </dd>
+        </div>
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Country</dt>
+          <dd className="flex items-start gap-x-2">
+            <div className="font-medium text-gray-900">
+              {dataset.countryCode}
+            </div>
+          </dd>
+        </div>
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Fiscal Period</dt>
+          <dd className="text-gray-700">
+            {dataset.fiscalPeriod?.start &&
+              new Date(dataset.fiscalPeriod.start).getFullYear()}
+            {dataset.fiscalPeriod?.end &&
+              dataset.fiscalPeriod?.start !== dataset.fiscalPeriod?.end && (
+                <>
+                  {' - '}
+                  {new Date(dataset.fiscalPeriod.end).getFullYear()}
+                </>
+              )}
+          </dd>
+        </div>
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Metadata</dt>
+          <dd className="flex items-start gap-x-2">
+            <div className="font-medium text-gray-900">
+              <Link
+                // TODO: where do we get the info needed for this link?
+                href=""
+                target="_blank"
+                className="flex items-center hover:text-gray-700"
+              >
+                datapackage.json <ExternalLinkIcon className="ml-1" />
+              </Link>
+            </div>
+          </dd>
+        </div>
+      </dl>
+    </div>
+  );
+}

examples/openspending/components/DatasetsGrid.tsx

@@ -0,0 +1,19 @@
+import { Project } from '../lib/project.interface';
+import DatasetCard from './DatasetCard';
+
+export default function DatasetsGrid({ datasets }: { datasets: Project[] }) {
+  return (
+    <ul
+      className="grid gap-x-6 gap-y-8 grid-cols-1 sm:grid-cols-2 md:grid-cols-3"
+      role="list"
+    >
+      {datasets.map((dataset, idx) => {
+        return (
+          <li key={`datasets-grid-item-${idx}`}>
+            <DatasetCard dataset={dataset} />
+          </li>
+        );
+      })}
+    </ul>
+  );
+}

examples/openspending/components/DatasetsSearch.tsx

@@ -0,0 +1,163 @@
+import { useForm } from 'react-hook-form';
+import DatasetsGrid from './DatasetsGrid';
+import { Project } from '../lib/project.interface';
+import { Index } from 'flexsearch';
+
+export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
+  const index = new Index({ tokenize: 'full' });
+  datasets.forEach((dataset: Project) =>
+    index.add(
+      dataset.name,
+      `${dataset.repo} ${dataset.name} ${dataset.title} ${dataset.author} ${dataset.title} ${dataset.cityCode} ${dataset.fiscalPeriod?.start} ${dataset.fiscalPeriod?.end}`
+    )
+  );
+
+  const { register, watch, handleSubmit, reset, resetField } = useForm({
+    defaultValues: {
+      searchTerm: '',
+      country: '',
+      minDate: '',
+      maxDate: '',
+    },
+  });
+
+  const allCountries = datasets
+    .map((item) => item.countryCode)
+    .filter((v) => v) //  Filters false values
+    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
+    //  TODO: title should be the full name
+    .map((code) => ({ code, title: code }));
+
+  return (
+    <>
+      <div className="flex flex-col gap-3 sm:flex-row">
+        <div className="min-w-0 flex-auto">
+          <br />
+          <div className="relative">
+            <input
+              placeholder="Search datasets"
+              aria-label="Search datasets"
+              {...register('searchTerm')}
+              className="h-[3em] relative w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+            {watch().searchTerm !== '' && (
+              <button
+                onClick={() => resetField('searchTerm')}
+                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
+              >
+                <CloseIcon />
+              </button>
+            )}
+          </div>
+        </div>
+        <div className="sm:basis-1/6">
+          {/* TODO: nicer select e.g. headlessui example */}
+          <label className="text-sm text-gray-600 font-medium">Country</label>
+          <select
+            className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            {...register('country')}
+          >
+            <option value="">All</option>
+            {allCountries.map((country) => {
+              return (
+                <option key={country.code} value={country.code}>
+                  {country.title}
+                </option>
+              );
+            })}
+          </select>
+        </div>
+        <div className="sm:basis-1/6">
+          <label className="text-sm text-gray-600 font-medium">Min. date</label>
+          <div className="relative">
+            <input
+              aria-label="Min. date"
+              type="date"
+              {...register('minDate')}
+              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+            {watch().minDate !== '' && (
+              <button
+                onClick={() => resetField('minDate')}
+                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
+              >
+                <CloseIcon />
+              </button>
+            )}
+          </div>
+        </div>
+        <div className="sm:basis-1/6">
+          <label className="text-sm text-gray-600 font-medium">Max. date</label>
+          <div className="relative">
+            <input
+              aria-label="Max. date"
+              type="date"
+              {...register('maxDate')}
+              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+            {watch().maxDate !== '' && (
+              <button
+                onClick={() => resetField('maxDate')}
+                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
+              >
+                <CloseIcon />
+              </button>
+            )}
+          </div>
+        </div>
+      </div>
+      <div className="min-w-full mt-10 align-middle">
+        <DatasetsGrid
+          datasets={datasets
+            .filter((dataset: Project) =>
+              watch().searchTerm && watch().searchTerm !== ''
+                ? index.search(watch().searchTerm).includes(dataset.name)
+                : true
+            )
+            .filter((dataset) =>
+              watch().country && watch().country !== ''
+                ? dataset.countryCode === watch().country
+                : true
+            )
+            // TODO: Does that really makes sense?
+            // What if the fiscalPeriod is 2015-2017 and inputs are
+            // set to 2015-2016. It's going to be filtered out but
+            // it shouldn't.
+            .filter((dataset) =>
+              watch().minDate && watch().minDate !== ''
+                ? dataset.fiscalPeriod?.start >= watch().minDate
+                : true
+            )
+            .filter((dataset) =>
+              watch().maxDate && watch().maxDate !== ''
+                ? dataset.fiscalPeriod?.end <= watch().maxDate
+                : true
+            )}
+        />
+      </div>
+    </>
+  );
+}
+
+const CloseIcon = () => {
+  return (
+    <svg
+      width={20}
+      height={20}
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g id="Menu / Close_MD">
+        <path
+          id="Vector"
+          d="M18 18L12 12M12 12L6 6M12 12L18 6M12 12L6 18"
+          stroke="currentColor"
+          stroke-width="2"
+          stroke-linecap="round"
+          stroke-linejoin="round"
+        />
+      </g>
+    </svg>
+  );
+};

examples/openspending/components/Header.tsx

@@ -0,0 +1,53 @@
+import Image from 'next/image'
+import { Button } from './Button'
+import { Container } from './Container'
+import logo from "../public/logo.svg"
+import Link from 'next/link'
+import { useRouter } from 'next/router'
+
+export function Header() {
+  const router = useRouter();
+
+  const isActive = (navLink) => {
+    return router.asPath.split("?")[0] == navLink.href;
+  }
+
+  const navLinks = [
+    {
+      title: "Home",
+      href: "/#header"
+    },
+    {
+      title: "Datasets",
+      href: "/#datasets"
+    },
+    {
+      title: "Community",
+      href: "https://community.openspending.org/"
+    }
+  ]
+
+  return (
+    <header className="z-50 pb-5 lg:pt-11 sticky top-0 backdrop-blur" id="header">
+      <Container className="flex flex-wrap items-center justify-center sm:justify-between lg:flex-nowrap">
+        <div className="mt-10 lg:mt-0 lg:grow lg:basis-0 flex items-center">
+          <Image src={logo} alt="OpenSpending" className="h-12 w-auto" />
+        </div>
+        <ul className='list-none flex gap-x-5 text-base font-medium'>
+          {navLinks.map((link, i) => (
+            <li key={`nav-link-${i}`}>
+              <Link
+                className={`text-emerald-900 hover:text-emerald-600 ${isActive(link) ? "text-emerald-600" : ""}`}
+                href={link.href}
+                scroll={false}
+              >
+                {link.title}
+              </Link>
+            </li>))}
+        </ul>
+        <div className="hidden sm:mt-10 sm:flex lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
+        </div>
+      </Container>
+    </header >
+  )
+}

examples/openspending/components/Hero.tsx

@@ -0,0 +1,47 @@
+import { Button } from './Button'
+import { Container } from './Container'
+
+export function Hero() {
+  return (
+    <div className="relative pb-20 pt-10 sm:py-40">
+      <div className="absolute inset-x-0 -bottom-14 -top-48 overflow-hidden bg-green-50 bg-opacity-50">
+        <div className="absolute inset-x-0 top-0 h-40 bg-gradient-to-b from-white" />
+        <div className="absolute inset-x-0 bottom-0 h-40 bg-gradient-to-t from-white" />
+      </div>
+      <Container className="relative">
+        <div className="mx-auto max-w-2xl lg:max-w-4xl lg:px-12">
+          <h1 className="font-display text-5xl font-bold tracking-tighter text-emerald-600 sm:text-7xl">
+            It's our money!
+          </h1>
+          <div className="mt-6 space-y-6 font-display text-2xl tracking-tight text-emerald-900">
+            <p>
+              By understanding how governments spend money in our name can we have a say
+              in how that money will affect our own lives. The journey starts here.
+            </p>
+            <p>
+              OpenSpending is a free, open and global platform to search, visualise and analyse
+              fiscal data in the public sphere.
+            </p>
+          </div>
+          <Button href="#datasets" className="mt-10">
+            Search datasets
+          </Button>
+          <dl className="mt-10 grid grid-cols-2 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
+            {[
+              ['Countries', '75'],
+              ['Datasets', '2091'],
+              ['Files', '9230'],
+            ].map(([name, value]) => (
+              <div key={name}>
+                <dt className="font-mono text-sm text-emerald-600">{name}</dt>
+                <dd className="mt-0.5 text-2xl font-semibold tracking-tight text-emerald-900">
+                  {value}
+                </dd>
+              </div>
+            ))}
+          </dl>
+        </div>
+      </Container>
+    </div>
+  )
+}

examples/openspending/components/_shared/Breadcrumbs.tsx

@@ -0,0 +1,20 @@
+import Link from "next/link";
+import HomeIcon from "../icons/HomeIcon";
+
+export default function Breadcrumbs({ links }: { links: { title: string, href?: string, target?: string }[] }) {
+  const current = links.at(-1);
+
+  return <div className="flex items-center uppercase font-black text-xs">
+    <Link className="flex items-center" href='/'><HomeIcon /></Link>
+
+    {/* {links.length > 1 && links.slice(0, -1).map((link) => {
+      return <>
+        <span className="mx-4">/</span>
+        <Link href={link.href}>{link.title}</Link>
+      </>
+    })} */}
+
+    <span className="mx-4">/</span>
+    <span>{current.title}</span>
+  </div >
+}

examples/openspending/components/icons/ExternalLinkIcon.tsx

@@ -0,0 +1,3 @@
+export default function ExternalLinkIcon({ className = "" }) {
+  return <div className={`inline-block w-4 ${className}`}><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" fill="currentColor"><path d="M 40 10 C 38.896 10 38 10.896 38 12 C 38 13.104 38.896 14 40 14 L 47.171875 14 L 30.585938 30.585938 C 29.804938 31.366938 29.804938 32.633063 30.585938 33.414062 C 30.976938 33.805063 31.488 34 32 34 C 32.512 34 33.023063 33.805062 33.414062 33.414062 L 50 16.828125 L 50 24 C 50 25.104 50.896 26 52 26 C 53.104 26 54 25.104 54 24 L 54 12 C 54 10.896 53.104 10 52 10 L 40 10 z M 18 12 C 14.691 12 12 14.691 12 18 L 12 46 C 12 49.309 14.691 52 18 52 L 46 52 C 49.309 52 52 49.309 52 46 L 52 34 C 52 32.896 51.104 32 50 32 C 48.896 32 48 32.896 48 34 L 48 46 C 48 47.103 47.103 48 46 48 L 18 48 C 16.897 48 16 47.103 16 46 L 16 18 C 16 16.897 16.897 16 18 16 L 30 16 C 31.104 16 32 15.104 32 14 C 32 12.896 31.104 12 30 12 L 18 12 z"/></svg></div>
+}

examples/openspending/components/icons/HomeIcon.tsx

@@ -0,0 +1,3 @@
+export default function HomeIcon({ className = "" }) {
+  return <div className={`inline-block w-4 ${className}`}><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">    <path d="M 12 2 A 1 1 0 0 0 11.289062 2.296875 L 1.203125 11.097656 A 0.5 0.5 0 0 0 1 11.5 A 0.5 0.5 0 0 0 1.5 12 L 4 12 L 4 20 C 4 20.552 4.448 21 5 21 L 9 21 C 9.552 21 10 20.552 10 20 L 10 14 L 14 14 L 14 20 C 14 20.552 14.448 21 15 21 L 19 21 C 19.552 21 20 20.552 20 20 L 20 12 L 22.5 12 A 0.5 0.5 0 0 0 23 11.5 A 0.5 0.5 0 0 0 22.796875 11.097656 L 12.716797 2.3027344 A 1 1 0 0 0 12.710938 2.296875 A 1 1 0 0 0 12 2 z"/></svg></div>
+}

examples/openspending/datasets.json

@@ -0,0 +1,27 @@
+[
+  {
+    "owner": "os-data",
+    "branch": "main",
+    "name": "mongolia-budget-2016-2017"
+  },
+  {
+    "owner": "os-data",
+    "branch": "main",
+    "name": "gb-country-regional-analysis"
+  },
+  {
+    "owner": "os-data",
+    "branch": "main",
+    "name": "berlin-berlin"
+  },
+  {
+    "owner": "os-data",
+    "branch": "main",
+    "name": "state-of-minas-gerais-brazil-planned-budget"
+  },
+  {
+    "owner": "os-data",
+    "branch": "main",
+    "name": "wesel"
+  }
+]

examples/openspending/index.d.ts

@@ -0,0 +1,6 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+declare module '*.svg' {
+  const content: any;
+  export const ReactComponent: any;
+  export default content;
+}

examples/openspending/lib/datapackage.interface.ts

@@ -0,0 +1,288 @@
+/**
+ * Fiscal Data Package is a simple specification for data access and delivery of fiscal data.
+ */
+export type FiscalDataPackage = TabularDataPackage & {
+  countryCode?: ISO31661Alpha2CountryCode
+  regionCode?: string
+  cityCode?: string
+  author?: string
+  readme?: string
+  granularity?: GranularityOfResources
+  fiscalPeriod?: FiscalPeriodForTheBudget
+  [k: string]: unknown
+}
+/**
+ * The profile of this descriptor.
+ */
+export type Profile = "tabular-data-package"
+/**
+ * An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.
+ */
+export type Name = string
+/**
+ * A property reserved for globally unique identifiers. Examples of identifiers that are unique include UUIDs and DOIs.
+ */
+export type ID = string
+/**
+ * A human-readable title.
+ */
+export type Title = string
+/**
+ * A text description. Markdown is encouraged.
+ */
+export type Description = string
+/**
+ * The home on the web that is related to this data package.
+ */
+export type HomePage = string
+/**
+ * The datetime on which this descriptor was created.
+ */
+export type Created = string
+/**
+ * The contributors to this descriptor.
+ */
+export type Contributors = [Contributor, ...Contributor[]]
+/**
+ * A human-readable title.
+ */
+export type Title1 = string
+/**
+ * A fully qualified URL, or a POSIX file path.
+ */
+export type Path = string
+/**
+ * An email address.
+ */
+export type Email = string
+/**
+ * An organizational affiliation for this contributor.
+ */
+export type Organization = string
+/**
+ * A list of keywords that describe this package.
+ */
+export type Keywords = [string, ...string[]]
+/**
+ * A image to represent this package.
+ */
+export type Image = string
+/**
+ * The license(s) under which this package is published.
+ */
+export type Licenses = [License, ...License[]]
+/**
+ * A license for this descriptor.
+ */
+export type License =
+  | {
+      [k: string]: unknown
+    }
+  | {
+      [k: string]: unknown
+    }
+/**
+ * An `array` of Tabular Data Resource objects, each compliant with the [Tabular Data Resource](/tabular-data-resource/) specification.
+ *
+/**
+ * A Tabular Data Resource.
+ */
+export interface TabularDataResource {
+  format?: string;
+  name: string;
+  description?: string;
+  title?: string;
+  schema?: Schema;
+  sample?: any[];
+  profile?: string;
+  key?: string;
+  path?: string;
+  size?: number;
+}
+
+export interface Field {
+  name: string;
+  type: FieldType;
+}
+
+export interface Schema {
+  fields: Field[];
+}
+
+export const OptionsFields = [
+  "any",
+  "array",
+  "boolean",
+  "date",
+  "datetime",
+  "duration",
+  "geojson",
+  "geopoint",
+  "integer",
+  "number",
+  "object",
+  "string",
+  "time",
+  "year",
+  "yearmonth",
+] as const;
+
+type FieldType = typeof OptionsFields[number];
+/**
+ * A human-readable title.
+ */
+export type Title2 = string
+/**
+ * A fully qualified URL, or a POSIX file path.
+ */
+export type Path1 = string
+/**
+ * An email address.
+ */
+export type Email1 = string
+/**
+ * The raw sources for this resource.
+ */
+export type Sources = Source[]
+
+/**
+ * A keyword that represents the direction of the spend, either expenditure or revenue.
+ */
+export type DirectionOfTheSpending = "expenditure" | "revenue"
+/**
+ * A keyword that represents the phase of the data, can be proposed for a budget proposal, approved for an approved budget, adjusted for modified budget or executed for the enacted budget
+ */
+export type BudgetPhase = "proposed" | "approved" | "adjusted" | "executed"
+/**
+ * Either an array of strings corresponding to the name attributes in a set of field objects in the fields array or a single string corresponding to one of these names. The value of primaryKey indicates the primary key or primary keys for the dimension.
+ */
+export type PrimaryKey = string | [string, ...string[]]
+/**
+ * Describes what kind of a dimension it is.
+ */
+export type DimensionType =
+  | "datetime"
+  | "entity"
+  | "classification"
+  | "activity"
+  | "fact"
+  | "location"
+  | "other"
+/**
+ * The type of the classification.
+ */
+export type ClassificationType = "functional" | "administrative" | "economic"
+/**
+ * A valid 2-digit ISO country code (ISO 3166-1 alpha-2), or, an array of valid ISO codes.
+ */
+export type ISO31661Alpha2CountryCode = string | [string, ...string[]]
+/**
+ * A keyword that represents the type of spend data, eiter aggregated or transactional
+ */
+export type GranularityOfResources = "aggregated" | "transactional"
+
+/**
+ * Tabular Data Package
+ */
+export interface TabularDataPackage {
+  profile: Profile
+  name?: Name
+  id?: ID
+  title?: Title
+  description?: Description
+  homepage?: HomePage
+  created?: Created
+  contributors?: Contributors
+  keywords?: Keywords
+  image?: Image
+  licenses?: Licenses
+  resources: TabularDataResource[]
+  sources?: Sources
+  [k: string]: unknown
+}
+/**
+ * A contributor to this descriptor.
+ */
+export interface Contributor {
+  title: Title1
+  path?: Path
+  email?: Email
+  organization?: Organization
+  role?: string
+  [k: string]: unknown
+}
+/**
+ * A source file.
+ */
+export interface Source {
+  title: Title2
+  path?: Path1
+  email?: Email1
+  [k: string]: unknown
+}
+/**
+ * Measures are numerical and correspond to financial amounts in the source data.
+ */
+export interface Measures {
+  [k: string]: Measure
+}
+/**
+ * Measure.
+ *
+ * This interface was referenced by `Measures`'s JSON-Schema definition
+ * via the `patternProperty` "^\w+".
+ */
+export interface Measure {
+  source: string
+  resource?: string
+  currency: string
+  factor?: number
+  direction?: DirectionOfTheSpending
+  phase?: BudgetPhase
+  [k: string]: unknown
+}
+/**
+ * Dimensions are groups of related fields. Dimensions cover all items other than the measure.
+ */
+export interface Dimensions {
+  [k: string]: Dimension
+}
+/**
+ * Dimension.
+ *
+ * This interface was referenced by `Dimensions`'s JSON-Schema definition
+ * via the `patternProperty` "^\w+".
+ */
+export interface Dimension {
+  attributes: Attributes
+  primaryKey: PrimaryKey
+  dimensionType?: DimensionType
+  classificationType?: ClassificationType
+  [k: string]: unknown
+}
+/**
+ * Attribute objects that make up the dimension
+ */
+export interface Attributes {
+  /**
+   * This interface was referenced by `Attributes`'s JSON-Schema definition
+   * via the `patternProperty` "^\w+".
+   */
+  [k: string]: {
+    source: string
+    resource?: string
+    constant?: string | number
+    parent?: string
+    labelfor?: string
+    [k: string]: unknown
+  }
+}
+/**
+ * The fiscal period of the dataset
+ */
+export interface FiscalPeriodForTheBudget {
+  start: string
+  end?: string
+  [k: string]: unknown
+}
+

examples/openspending/lib/loader.ts

@@ -0,0 +1,34 @@
+import { FiscalDataPackage } from './datapackage.interface';
+import { Project } from './project.interface';
+
+export function loadDataPackage(datapackage: FiscalDataPackage, repo): Project {
+  return {
+    name: datapackage.name,
+    title: datapackage.title,
+    owner: {
+      name: repo.owner.login,
+      logo: repo.owner.avatar_url,
+      // TODO: make this title work
+      title: repo.owner.login,
+    },
+    repo: { name: repo, full_name: repo.full_name },
+    files: datapackage.resources,
+    author: datapackage.author ? datapackage.author : null,
+    cityCode: datapackage.cityCode ? datapackage.cityCode : null,
+    countryCode: datapackage.countryCode
+      ? (datapackage.countryCode as string)
+      : null,
+    fiscalPeriod: datapackage.fiscalPeriod
+      ? {
+          start: datapackage.fiscalPeriod.start
+            ? datapackage.fiscalPeriod.start
+            : null,
+          end: datapackage.fiscalPeriod.end
+            ? datapackage.fiscalPeriod.end
+            : null,
+        }
+      : null,
+    readme: datapackage.readme ? datapackage.readme : '',
+    datapackage,
+  };
+}

examples/openspending/lib/octokit.ts

@@ -0,0 +1,192 @@
+import { Octokit } from 'octokit';
+
+export interface GithubProject {
+  owner: string;
+  repo: string;
+  branch: string;
+  files: string[];
+  readme: string;
+  description?: string;
+  name?: string;
+}
+
+export async function getProjectReadme(
+  owner: string,
+  repo: string,
+  branch: string,
+  readme: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.getContent({
+      owner,
+      repo,
+      path: readme,
+      ref: branch,
+    });
+    const data = response.data as { content?: string };
+    const fileContent = data.content ? data.content : '';
+    if (fileContent === '') {
+      return null;
+    }
+    const decodedContent = Buffer.from(fileContent, 'base64').toString();
+    return decodedContent;
+  } catch (error) {
+    console.log(error);
+    return null;
+  }
+}
+
+export async function getLastUpdated(
+  owner: string,
+  repo: string,
+  branch: string,
+  readme: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.listCommits({
+      owner,
+      repo,
+      path: readme,
+      ref: branch,
+    });
+    return response.data[0].commit.committer.date;
+  } catch (error) {
+    console.log(error);
+    return null;
+  }
+}
+export async function getProjectMetadata(
+  owner: string,
+  repo: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.get({
+      owner,
+      repo,
+    });
+    return response.data;
+  } catch (error) {
+    console.log(error);
+    return null;
+  }
+}
+
+export async function getRepoContents(
+  owner: string,
+  repo: string,
+  branch: string,
+  files: string[],
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const contents = [];
+    for (const path of files) {
+      const response = await octokit.rest.repos.getContent({
+        owner,
+        repo,
+        ref: branch,
+        path: path,
+      });
+      const data = response.data as {
+        download_url?: string;
+        name: string;
+        size: number;
+      };
+      contents.push({
+        download_url: data.download_url,
+        name: data.name,
+        size: data.size,
+      });
+    }
+    return contents;
+  } catch (error) {
+    console.log(error);
+    return null;
+  }
+}
+
+export async function getProject(project: GithubProject, github_pat?: string) {
+  const projectMetadata = await getProjectMetadata(
+    project.owner,
+    project.repo,
+    github_pat
+  );
+  if (!projectMetadata) {
+    return null;
+  }
+  const projectReadme = await getProjectReadme(
+    project.owner,
+    project.repo,
+    project.branch,
+    project.readme,
+    github_pat
+  );
+
+  const projectData = await getRepoContents(
+    project.owner,
+    project.repo,
+    project.branch,
+    project.files,
+    github_pat
+  );
+  if (!projectData) {
+    return null;
+  }
+
+  let projectBase = '',
+    last_updated = '';
+  if (projectReadme) {
+    projectBase =
+      project.readme.split('/').length > 1
+        ? project.readme.split('/').slice(0, -1).join('/')
+        : '/';
+    last_updated = await getLastUpdated(
+      project.owner,
+      project.repo,
+      project.branch,
+      projectBase,
+      github_pat
+    );
+  }
+  return {
+    ...projectMetadata,
+    files: projectData,
+    readmeContent: projectReadme,
+    last_updated,
+    base_path: projectBase,
+  };
+}
+
+export async function getProjectDataPackage(
+  owner: string,
+  repo: string,
+  branch: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.getContent({
+      owner,
+      repo,
+      path: 'datapackage.json',
+      ref: branch,
+    });
+    const data = response.data as { content?: string };
+    const fileContent = data.content ? data.content : '';
+    if (fileContent === '') {
+      return null;
+    }
+    const decodedContent = Buffer.from(fileContent, 'base64').toString();
+    const datapackage = JSON.parse(decodedContent);
+    return {...datapackage, repo };
+  } catch (error) {
+    return null;
+  }
+}

examples/openspending/lib/project.interface.ts

@@ -0,0 +1,21 @@
+import {
+  FiscalDataPackage,
+  TabularDataResource,
+} from './datapackage.interface';
+
+export interface Project {
+  owner: { name: string; logo?: string; title?: string }; // Info about the owner of the data repo
+  repo: { name: string; full_name: string }; // Info about the the data repo
+  files: TabularDataResource[];
+  name: string;
+  title?: string;
+  author?: string;
+  cityCode?: string;
+  countryCode?: string;
+  fiscalPeriod?: {
+    start: string;
+    end: string;
+  };
+  readme?: string;
+  datapackage: FiscalDataPackage;
+}

examples/openspending/next.config.js

@@ -0,0 +1,17 @@
+const nextConfig = {
+  async rewrites() {
+    return {
+      beforeFiles: [
+        {
+          source: '/@:org/:project*',
+          destination: '/@org/:org/:project*',
+        },
+      ],
+    };
+  },
+  serverRuntimeConfig: {
+    github_pat: process.env.GITHUB_PAT ? process.env.GITHUB_PAT : null,
+  },
+};
+
+module.exports = nextConfig;

examples/openspending/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "my-app",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint",
+    "test": "vitest"
+  },
+  "dependencies": {
+    "@octokit/plugin-throttling": "^5.2.2",
+    "@types/flexsearch": "^0.7.3",
+    "@types/node": "18.16.0",
+    "@types/react": "18.0.38",
+    "@types/react-dom": "18.0.11",
+    "@vitejs/plugin-react": "^4.0.0",
+    "clsx": "^1.2.1",
+    "eslint": "8.39.0",
+    "eslint-config-next": "13.3.1",
+    "flexsearch": "0.7.21",
+    "next": "13.3.1",
+    "next-seo": "^6.0.0",
+    "octokit": "^2.0.14",
+    "prettier": "^2.8.8",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-hook-form": "^7.43.9",
+    "react-markdown": "^8.0.7",
+    "react-timeago": "^7.1.0",
+    "remark-gfm": "^3.0.1",
+    "typescript": "5.0.4"
+  },
+  "devDependencies": {
+    "@tailwindcss/typography": "^0.5.9",
+    "autoprefixer": "^10.4.14",
+    "postcss": "^8.4.23",
+    "tailwindcss": "^3.3.1",
+    "vitest": "^0.31.0"
+  }
+}

examples/openspending/pages/@org/[org]/[...path].tsx

@@ -0,0 +1,126 @@
+import { NextSeo } from 'next-seo';
+import { promises as fs } from 'fs';
+import path from 'path';
+import getConfig from 'next/config';
+import { getProject, GithubProject } from '../../../lib/octokit';
+import ReactMarkdown from 'react-markdown';
+import remarkGfm from 'remark-gfm';
+import Breadcrumbs from '../../../components/_shared/Breadcrumbs';
+
+export default function ProjectPage({ project }) {
+  const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`
+
+  return (
+    <>
+      <NextSeo title={`${repoId}${project.base_path !== '/' ? '/' + project.base_path : ''} - GitHub Datasets`} />
+      <main className="prose mx-auto my-8">
+        <Breadcrumbs links={[{ title: repoId, href: "" }]} />
+        <h1 className="mb-0 mt-16">{project.repo_config.name || repoId}</h1>
+        <p className='mb-8'><span className='font-semibold'>Repository:</span> <a target="_blank" href={project.html_url}>{project.html_url}</a></p>
+
+        <h2 className="mb-0 mt-10">Files</h2>
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="min-w-full divide-y divide-gray-300">
+            <thead>
+              <tr>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Name
+                </th>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Size
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              {project.files?.map((file) => (
+                <tr key={file.download_url}>
+                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                    <a href={file.download_url}>{file.name}</a>
+                  </td>
+                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                    {file.size} Bytes
+                  </td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        </div>
+
+        {project.readmeContent && <>
+          <hr />
+
+          <h2 className='uppercase font-black'>Readme</h2>
+          <ReactMarkdown remarkPlugins={[remarkGfm]}>
+            {project.readmeContent}
+          </ReactMarkdown>
+        </>}
+      </main>
+    </>
+  );
+}
+
+// Generates `/posts/1` and `/posts/2`
+export async function getStaticPaths() {
+  const jsonDirectory = path.join(
+    process.cwd(),
+    'datasets.json'
+  );
+  const repos = await fs.readFile(jsonDirectory, 'utf8');
+
+  return {
+    paths: JSON.parse(repos).map((repo) => {
+      const projectPath =
+        repo.readme && repo.readme.split('/').length > 1
+          ? repo.readme.split('/').slice(0, -1)
+          : null;
+      let path = [repo.name];
+      if (projectPath) {
+        projectPath.forEach((element) => {
+          path.push(element);
+        });
+      }
+      return {
+        params: { org: repo.owner, path },
+      };
+    }),
+    fallback: false, // can also be true or 'blocking'
+  };
+}
+
+export async function getStaticProps({ params }) {
+  const jsonDirectory = path.join(
+    process.cwd(),
+    'datasets.json'
+  );
+  const reposFile = await fs.readFile(jsonDirectory, 'utf8');
+  const repos: GithubProject[] = JSON.parse(reposFile);
+  const repo = repos.find((_repo) => {
+    const projectPath =
+      _repo.readme && _repo.readme.split('/').length > 1
+        ? _repo.readme.split('/').slice(0, -1)
+        : null;
+    let path = [_repo.name];
+    if (projectPath) {
+      projectPath.forEach((element) => {
+        path.push(element);
+      });
+    }
+    return (
+      _repo.owner == params.org &&
+      JSON.stringify(path) === JSON.stringify(params.path)
+    );
+  });
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+  const project = await getProject(repo, github_pat);
+  return {
+    props: {
+      project: { ...project, repo_config: repo },
+    },
+  };
+}

examples/openspending/pages/_app.tsx

@@ -0,0 +1,18 @@
+import { AppProps } from 'next/app';
+import Head from 'next/head';
+import './styles.css';
+
+function CustomApp({ Component, pageProps }: AppProps) {
+  return (
+    <>
+      <Head>
+        <title>GitHub Datasets</title>
+      </Head>
+      <main className="app">
+        <Component {...pageProps} />
+      </main>
+    </>
+  );
+}
+
+export default CustomApp;

examples/openspending/pages/index.tsx

@@ -0,0 +1,85 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import {
+  GithubProject,
+  getProjectDataPackage,
+  getProjectMetadata,
+} from '../lib/octokit';
+import getConfig from 'next/config';
+import ExternalLinkIcon from '../components/icons/ExternalLinkIcon';
+import TimeAgo from 'react-timeago';
+import Link from 'next/link';
+import { Hero } from '../components/Hero';
+import { Header } from '../components/Header';
+import { Container } from '../components/Container';
+import { FiscalDataPackage } from '../lib/datapackage.interface';
+import { loadDataPackage } from '../lib/loader';
+import DatasetsSearch from '../components/DatasetsSearch';
+
+export async function getStaticProps() {
+  const jsonDirectory = path.join(process.cwd(), '/datasets.json');
+  const repos = await fs.readFile(jsonDirectory, 'utf8');
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+  const datapackages = await Promise.all(
+    JSON.parse(repos).map(async (_repo: GithubProject) => {
+      const datapackage = await getProjectDataPackage(
+        _repo.owner,
+        _repo.name,
+        'main',
+        github_pat
+      );
+      const repo = await getProjectMetadata(
+        _repo.owner,
+        _repo.name,
+        github_pat
+      );
+
+      return {
+        datapackage,
+        repo,
+      };
+    })
+  );
+
+  const projects = datapackages.map(
+    (item: { datapackage: FiscalDataPackage & { repo: string }; repo: any }) =>
+      loadDataPackage(item.datapackage, item.repo)
+  );
+
+  return {
+    props: {
+      projects: JSON.stringify(projects),
+    },
+  };
+}
+
+export function Datasets({ projects }) {
+  projects = JSON.parse(projects);
+
+  return (
+    <div className="bg-white min-h-screen">
+      <Header />
+      <Hero />
+      <section className="py-20 sm:py-32">
+        <Container>
+          <div className="mx-auto max-w-2xl lg:mx-0">
+            <h2
+              id="datasets"
+              className="font-display text-4xl font-medium tracking-tighter text-emerald-600 sm:text-5xl"
+            >
+              Datasets
+            </h2>
+            <p className="mt-4 font-display text-2xl tracking-tight text-emerald-900">
+              Find spending data about countries all around the world.
+            </p>
+          </div>
+          <div className="mt-10">
+            <DatasetsSearch datasets={projects} />
+          </div>
+        </Container>
+      </section>
+    </div>
+  );
+}
+
+export default Datasets;

examples/openspending/pages/styles.css

@@ -0,0 +1,80 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+html {
+  -webkit-text-size-adjust: 100%;
+  font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont,
+    Segoe UI, Roboto, Helvetica Neue, Arial, Noto Sans, sans-serif,
+    Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol, Noto Color Emoji;
+  line-height: 1.5;
+  tab-size: 4;
+  scroll-behavior: smooth;
+}
+body {
+  font-family: inherit;
+  line-height: inherit;
+  margin: 0;
+}
+h1,
+h2,
+p,
+pre {
+  margin: 0;
+}
+*,
+::before,
+::after {
+  box-sizing: border-box;
+  border-width: 0;
+  border-style: solid;
+  border-color: currentColor;
+}
+h1,
+h2 {
+  font-size: inherit;
+  font-weight: inherit;
+}
+a {
+  color: inherit;
+  text-decoration: inherit;
+}
+pre {
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
+    Liberation Mono, Courier New, monospace;
+}
+svg {
+  display: block;
+  vertical-align: middle;
+  shape-rendering: auto;
+  text-rendering: optimizeLegibility;
+}
+pre {
+  background-color: rgba(55, 65, 81, 1);
+  border-radius: 0.25rem;
+  color: rgba(229, 231, 235, 1);
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
+    Liberation Mono, Courier New, monospace;
+  overflow: scroll;
+  padding: 0.5rem 0.75rem;
+}
+
+.shadow {
+  box-shadow: 0 0 #0000, 0 0 #0000, 0 10px 15px -3px rgba(0, 0, 0, 0.1),
+    0 4px 6px -2px rgba(0, 0, 0, 0.05);
+}
+.rounded {
+  border-radius: 1.5rem;
+}
+.wrapper {
+  width: 100%;
+}
+.container {
+  margin-left: auto;
+  margin-right: auto;
+  max-width: 768px;
+  padding-bottom: 3rem;
+  padding-left: 1rem;
+  padding-right: 1rem;
+  color: rgba(55, 65, 81, 1);
+  width: 100%;
+}

examples/openspending/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

examples/openspending/public/assets/org-icon.svg

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg fill="#000000" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="800px" height="800px" viewBox="0 0 120 120" enable-background="new 0 0 120 120" xml:space="preserve">
+<rect x="2" y="108.1" width="116" height="11.9"/>
+<rect x="6.744" y="96.582" width="104.979" height="6.543"/>
+<rect x="15.288" y="38.532" width="17.639" height="52.925"/>
+<rect x="50.484" y="38.532" width="17.639" height="52.925"/>
+<rect x="84.33" y="38.532" width="17.639" height="52.925"/>
+<polygon points="0,26.96 60,0 120,26.96 119.946,33.912 0,34.01 "/>
+</svg>

examples/openspending/tailwind.config.js

@@ -0,0 +1,15 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    "./app/**/*.{js,ts,jsx,tsx,mdx}",
+    "./pages/**/*.{js,ts,jsx,tsx,mdx}",
+    "./components/**/*.{js,ts,jsx,tsx,mdx}",
+  ],
+  theme: {
+    extend: {},
+  },
+  plugins: [
+    require('@tailwindcss/typography')
+  ],
+}
+

examples/openspending/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": false,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+  "exclude": ["node_modules"]
+}

examples/openspending/vitest.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'vitest/config'
+import react from '@vitejs/plugin-react'
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  plugins: [react()],
+  test: {
+    environment: 'jsdom',
+  },
+})

examples/simple-example/README.md

@@ -1,75 +0,0 @@
-This is a repo intended to serve as a simple example of a data catalog that get its data from a series of github repos, you can init an example just like this one by.
-
-- Creating a new project with `create-next-app` like so:
-
-```
-npx create-next-app <app-name> --example https://github.com/datopian/portaljs/tree/main/examples/simple-example
-cd <app-name>
-```
-
-- This project uses the github api, which for anonymous users will cap at 50 requests per hour, so you might want to get a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and add it to a `.env` file inside the folder like so
-
-```
-GITHUB_PAT=<github token>
-```
-
-- Edit the file `datasets.json` to your liking, some examples can be found inside this [repo](https://github.com/datasets)
-- Run the app using:
-
-```
-npm run dev
-```
-
-Congratulations, you now have something similar to this running on `http://localhost:3000`
-![](https://i.imgur.com/jAljJ9C.png)
-If yo go to any one of those pages by clicking on `More info` you will see something similar to this
-![](https://i.imgur.com/AoJd4O0.png)
-
-## Deployment
-
-[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fdatopian%2Fportaljs%2Ftree%2Fmain%2Fexamples%2Fsimple-example)
-
-By clicking on this button, you will be redirected to a page which will allow you to clone the content into your own github/gitlab/bitbucket account and automatically deploy everything.
-
-
-## Structure of `datasets.json`
-
-The `datasets.json` file is simply a list of datasets, below you can see a minimal example of a dataset
-
-```json
-{
-  "owner": "fivethirtyeight",
-  "repo": "data",
-  "branch": "master",
-  "files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"],
-  "readme": "nba-raptor/README.md"
-}
-```
-
-It has
-
-- A `owner` which is going to be the github repo owner
-- A `repo` which is going to be the github repo name
-- A `branch` which is going to be the branch to which we need to get the files and the readme
-- A list of `files` which is going to be a list of paths with files that you want to show to the world
-- A `readme` which is going to be the path to your data description, it can also be a subpath eg: `example/README.md`
-
-You can also add
-
-- A `description` which is useful if you have more than one dataset for each repo, if not provided we are just going to use the repo description
-- A `Name` which is useful if you want to give your dataset a nice name, if not provided we are going to use the junction of the `owner` the `repo` + the path of the README, in the exaple above it will be `fivethirtyeight/data/nba-raptor`
-
-## Extra commands
-
-You can also build the project for production with
-
-```
-npm run build
-```
-
-And run using the production build like so:
-
-```
-npm run start
-```
-

examples/simple-example/pages/styles.css

@@ -1,403 +0,0 @@
-@tailwind base;
-@tailwind components;
-@tailwind utilities;
-html {
-  -webkit-text-size-adjust: 100%;
-  font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont,
-    Segoe UI, Roboto, Helvetica Neue, Arial, Noto Sans, sans-serif,
-    Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol, Noto Color Emoji;
-  line-height: 1.5;
-  tab-size: 4;
-  scroll-behavior: smooth;
-}
-body {
-  font-family: inherit;
-  line-height: inherit;
-  margin: 0;
-}
-h1,
-h2,
-p,
-pre {
-  margin: 0;
-}
-*,
-::before,
-::after {
-  box-sizing: border-box;
-  border-width: 0;
-  border-style: solid;
-  border-color: currentColor;
-}
-h1,
-h2 {
-  font-size: inherit;
-  font-weight: inherit;
-}
-a {
-  color: inherit;
-  text-decoration: inherit;
-}
-pre {
-  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
-    Liberation Mono, Courier New, monospace;
-}
-svg {
-  display: block;
-  vertical-align: middle;
-  shape-rendering: auto;
-  text-rendering: optimizeLegibility;
-}
-pre {
-  background-color: rgba(55, 65, 81, 1);
-  border-radius: 0.25rem;
-  color: rgba(229, 231, 235, 1);
-  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
-    Liberation Mono, Courier New, monospace;
-  overflow: scroll;
-  padding: 0.5rem 0.75rem;
-}
-
-.shadow {
-  box-shadow: 0 0 #0000, 0 0 #0000, 0 10px 15px -3px rgba(0, 0, 0, 0.1),
-    0 4px 6px -2px rgba(0, 0, 0, 0.05);
-}
-.rounded {
-  border-radius: 1.5rem;
-}
-.wrapper {
-  width: 100%;
-}
-.container {
-  margin-left: auto;
-  margin-right: auto;
-  max-width: 768px;
-  padding-bottom: 3rem;
-  padding-left: 1rem;
-  padding-right: 1rem;
-  color: rgba(55, 65, 81, 1);
-  width: 100%;
-}
-#welcome {
-  margin-top: 2.5rem;
-}
-#welcome h1 {
-  font-size: 3rem;
-  font-weight: 500;
-  letter-spacing: -0.025em;
-  line-height: 1;
-}
-#welcome span {
-  display: block;
-  font-size: 1.875rem;
-  font-weight: 300;
-  line-height: 2.25rem;
-  margin-bottom: 0.5rem;
-}
-#hero {
-  align-items: center;
-  background-color: hsla(214, 62%, 21%, 1);
-  border: none;
-  box-sizing: border-box;
-  color: rgba(55, 65, 81, 1);
-  display: grid;
-  grid-template-columns: 1fr;
-  margin-top: 3.5rem;
-}
-#hero .text-container {
-  color: rgba(255, 255, 255, 1);
-  padding: 3rem 2rem;
-}
-#hero .text-container h2 {
-  font-size: 1.5rem;
-  line-height: 2rem;
-  position: relative;
-}
-#hero .text-container h2 svg {
-  color: hsla(162, 47%, 50%, 1);
-  height: 2rem;
-  left: -0.25rem;
-  position: absolute;
-  top: 0;
-  width: 2rem;
-}
-#hero .text-container h2 span {
-  margin-left: 2.5rem;
-}
-#hero .text-container a {
-  background-color: rgba(255, 255, 255, 1);
-  border-radius: 0.75rem;
-  color: rgba(55, 65, 81, 1);
-  display: inline-block;
-  margin-top: 1.5rem;
-  padding: 1rem 2rem;
-  text-decoration: inherit;
-}
-#hero .logo-container {
-  display: none;
-  justify-content: center;
-  padding-left: 2rem;
-  padding-right: 2rem;
-}
-#hero .logo-container svg {
-  color: rgba(255, 255, 255, 1);
-  width: 66.666667%;
-}
-#middle-content {
-  align-items: flex-start;
-  display: grid;
-  gap: 4rem;
-  grid-template-columns: 1fr;
-  margin-top: 3.5rem;
-}
-#learning-materials {
-  padding: 2.5rem 2rem;
-}
-#learning-materials h2 {
-  font-weight: 500;
-  font-size: 1.25rem;
-  letter-spacing: -0.025em;
-  line-height: 1.75rem;
-  padding-left: 1rem;
-  padding-right: 1rem;
-}
-.list-item-link {
-  align-items: center;
-  border-radius: 0.75rem;
-  display: flex;
-  margin-top: 1rem;
-  padding: 1rem;
-  transition-property: background-color, border-color, color, fill, stroke,
-    opacity, box-shadow, transform, filter, backdrop-filter,
-    -webkit-backdrop-filter;
-  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
-  transition-duration: 150ms;
-  width: 100%;
-}
-.list-item-link svg:first-child {
-  margin-right: 1rem;
-  height: 1.5rem;
-  transition-property: background-color, border-color, color, fill, stroke,
-    opacity, box-shadow, transform, filter, backdrop-filter,
-    -webkit-backdrop-filter;
-  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
-  transition-duration: 150ms;
-  width: 1.5rem;
-}
-.list-item-link > span {
-  flex-grow: 1;
-  font-weight: 400;
-  transition-property: background-color, border-color, color, fill, stroke,
-    opacity, box-shadow, transform, filter, backdrop-filter,
-    -webkit-backdrop-filter;
-  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
-  transition-duration: 150ms;
-}
-.list-item-link > span > span {
-  color: rgba(107, 114, 128, 1);
-  display: block;
-  flex-grow: 1;
-  font-size: 0.75rem;
-  font-weight: 300;
-  line-height: 1rem;
-  transition-property: background-color, border-color, color, fill, stroke,
-    opacity, box-shadow, transform, filter, backdrop-filter,
-    -webkit-backdrop-filter;
-  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
-  transition-duration: 150ms;
-}
-.list-item-link svg:last-child {
-  height: 1rem;
-  transition-property: all;
-  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
-  transition-duration: 150ms;
-  width: 1rem;
-}
-.list-item-link:hover {
-  color: rgba(255, 255, 255, 1);
-  background-color: hsla(162, 47%, 50%, 1);
-}
-.list-item-link:hover > span {
-}
-.list-item-link:hover > span > span {
-  color: rgba(243, 244, 246, 1);
-}
-.list-item-link:hover svg:last-child {
-  transform: translateX(0.25rem);
-}
-#other-links {
-}
-.button-pill {
-  padding: 1.5rem 2rem;
-  transition-duration: 300ms;
-  transition-property: background-color, border-color, color, fill, stroke,
-    opacity, box-shadow, transform, filter, backdrop-filter,
-    -webkit-backdrop-filter;
-  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
-  align-items: center;
-  display: flex;
-}
-.button-pill svg {
-  transition-property: background-color, border-color, color, fill, stroke,
-    opacity, box-shadow, transform, filter, backdrop-filter,
-    -webkit-backdrop-filter;
-  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
-  transition-duration: 150ms;
-  flex-shrink: 0;
-  width: 3rem;
-}
-.button-pill > span {
-  letter-spacing: -0.025em;
-  font-weight: 400;
-  font-size: 1.125rem;
-  line-height: 1.75rem;
-  padding-left: 1rem;
-  padding-right: 1rem;
-}
-.button-pill span span {
-  display: block;
-  font-size: 0.875rem;
-  font-weight: 300;
-  line-height: 1.25rem;
-}
-.button-pill:hover svg,
-.button-pill:hover {
-  color: rgba(255, 255, 255, 1) !important;
-}
-#nx-console:hover {
-  background-color: rgba(0, 122, 204, 1);
-}
-#nx-console svg {
-  color: rgba(0, 122, 204, 1);
-}
-#nx-repo:hover {
-  background-color: rgba(24, 23, 23, 1);
-}
-#nx-repo svg {
-  color: rgba(24, 23, 23, 1);
-}
-#nx-cloud {
-  margin-bottom: 2rem;
-  margin-top: 2rem;
-  padding: 2.5rem 2rem;
-}
-#nx-cloud > div {
-  align-items: center;
-  display: flex;
-}
-#nx-cloud > div svg {
-  border-radius: 0.375rem;
-  flex-shrink: 0;
-  width: 3rem;
-}
-#nx-cloud > div h2 {
-  font-size: 1.125rem;
-  font-weight: 400;
-  letter-spacing: -0.025em;
-  line-height: 1.75rem;
-  padding-left: 1rem;
-  padding-right: 1rem;
-}
-#nx-cloud > div h2 span {
-  display: block;
-  font-size: 0.875rem;
-  font-weight: 300;
-  line-height: 1.25rem;
-}
-#nx-cloud p {
-  font-size: 1rem;
-  line-height: 1.5rem;
-  margin-top: 1rem;
-}
-#nx-cloud pre {
-  margin-top: 1rem;
-}
-#nx-cloud a {
-  color: rgba(107, 114, 128, 1);
-  display: block;
-  font-size: 0.875rem;
-  line-height: 1.25rem;
-  margin-top: 1.5rem;
-  text-align: right;
-}
-#nx-cloud a:hover {
-  text-decoration: underline;
-}
-#commands {
-  padding: 2.5rem 2rem;
-  margin-top: 3.5rem;
-}
-#commands h2 {
-  font-size: 1.25rem;
-  font-weight: 400;
-  letter-spacing: -0.025em;
-  line-height: 1.75rem;
-  padding-left: 1rem;
-  padding-right: 1rem;
-}
-#commands p {
-  font-size: 1rem;
-  font-weight: 300;
-  line-height: 1.5rem;
-  margin-top: 1rem;
-  padding-left: 1rem;
-  padding-right: 1rem;
-}
-details {
-  align-items: center;
-  display: flex;
-  margin-top: 1rem;
-  padding-left: 1rem;
-  padding-right: 1rem;
-  width: 100%;
-}
-details pre > span {
-  color: rgba(181, 181, 181, 1);
-  display: block;
-}
-summary {
-  border-radius: 0.5rem;
-  display: flex;
-  font-weight: 400;
-  padding: 0.5rem;
-  cursor: pointer;
-  transition-property: background-color, border-color, color, fill, stroke,
-    opacity, box-shadow, transform, filter, backdrop-filter,
-    -webkit-backdrop-filter;
-  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
-  transition-duration: 150ms;
-}
-summary:hover {
-  background-color: rgba(243, 244, 246, 1);
-}
-summary svg {
-  height: 1.5rem;
-  margin-right: 1rem;
-  width: 1.5rem;
-}
-#love {
-  color: rgba(107, 114, 128, 1);
-  font-size: 0.875rem;
-  line-height: 1.25rem;
-  margin-top: 3.5rem;
-  opacity: 0.6;
-  text-align: center;
-}
-#love svg {
-  color: rgba(252, 165, 165, 1);
-  width: 1.25rem;
-  height: 1.25rem;
-  display: inline;
-  margin-top: -0.25rem;
-}
-@media screen and (min-width: 768px) {
-  #hero {
-    grid-template-columns: repeat(2, minmax(0, 1fr));
-  }
-  #hero .logo-container {
-    display: flex;
-  }
-  #middle-content {
-    grid-template-columns: repeat(2, minmax(0, 1fr));
-  }
-}

examples/turing/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

examples/turing/components/Header.jsx

@@ -140,7 +140,7 @@ function MobileNavigation(props) {
             </div>
             <nav className="mt-6">
               <ul className="-my-2 divide-y divide-zinc-100 text-base text-zinc-800 dark:divide-zinc-100/5 dark:text-zinc-300">
-                <MobileNavItem href="https://github.com/leondz/hatespeechdata">
+                <MobileNavItem href="https://github.com/datopian/portaljs/tree/main/examples/turing">
                   View on Github <GithubIcon />
                 </MobileNavItem>
               </ul>
@@ -179,7 +179,7 @@ function DesktopNavigation(props) {
   return (
     <nav {...props}>
       <ul className="flex rounded-full bg-white/90 px-3 text-sm font-medium text-zinc-800 shadow-lg shadow-zinc-800/5 ring-1 ring-zinc-900/5 backdrop-blur dark:bg-zinc-800/90 dark:text-zinc-200 dark:ring-white/10">
-        <NavItem href="https://github.com/leondz/hatespeechdata">
+        <NavItem href="https://github.com/datopian/portaljs/tree/main/examples/turing">
           View on Github <GithubIcon />
         </NavItem>
       </ul>

examples/turing/content/index.mdx

@@ -36,7 +36,7 @@ In the following page type `content/datasets/<name-of-the-file>.md`. if you want
 
 ### Fill in content
 
-Copy the contents of `templates/dataset.md` or `templates/keywords.md` respectively to the camp below, filling out the fields with the correct data format
+Copy the contents of `templates/dataset.md` or `templates/keywords.md` respectively to the camp below, filling out the fields with the correct data format. Everything below the second `---` will automatically get rendered into the page, so you may add any standard markdown fields e.g tables, headings, lists...
 
 ![](https://i.imgur.com/x6JIjhz.png)