add whole tutorial content

dsf
[site/content/blog][s]: blob post about basic tutorial on markdowndb
2023-05-26 15:26:07 +02:00 · 2023-05-26 12:29:41 +02:00 · 2023-05-26 12:24:56 +02:00 · 2023-05-19 19:03:35 +02:00 · 2023-05-19 07:04:16 -03:00 · 2023-05-19 11:15:52 +02:00
28 changed files with 5539 additions and 375 deletions
--- a/examples/openspending/tests/os-data.test.ts
+++ b/examples/openspending/tests/os-data.test.ts
@@ -0,0 +1,138 @@
+import { expect, test } from 'vitest';
+import { getAllProjectsFromOrg, getProjectDataPackage } from '../lib/project';
+import { loadDataPackage } from '../lib/loader';
+import { getProjectMetadata } from '../lib/project';
+import { getCsv, parseCsv } from '../components/Table';
+
+test(
+  'Test OS-Data',
+  async () => {
+    const repos = await getAllProjectsFromOrg(
+      'os-data',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    if (repos.failed.length > 0) console.log(repos.failed);
+    expect(repos.failed.length).toBe(0);
+  },
+  { timeout: 100000 }
+);
+
+test(
+  'Test Gift-Data',
+  async () => {
+    const repos = await getAllProjectsFromOrg(
+      'gift-data',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    if (repos.failed.length > 0) console.log(repos.failed);
+    expect(repos.failed.length).toBe(0);
+  },
+  { timeout: 100000 }
+);
+
+test(
+  'Test getting one dataset from github',
+  async () => {
+    const datapackage = await getProjectDataPackage(
+      'os-data',
+      'berlin-berlin',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    const repo = await getProjectMetadata(
+      'os-data',
+      'berlin-berlin',
+      process.env.VITE_GITHUB_PAT
+    );
+    const project = loadDataPackage(datapackage, repo);
+    delete project['datapackage'];
+    delete project.files[0]['dialect'];
+    delete project.files[0]['schema'];
+    expect(project).toStrictEqual({
+      name: 'berlin-berlin',
+      title: 'Berlin-Berlin',
+      description: null,
+      owner: {
+        name: 'os-data',
+        logo: 'https://avatars.githubusercontent.com/u/13695166?v=4',
+        title: 'os-data',
+      },
+      repo: {
+        name: 'berlin-berlin',
+        full_name: 'os-data/berlin-berlin',
+        url: 'https://github.com/os-data/berlin-berlin',
+      },
+      files: [
+        {
+          name: 'berlin-gesamt',
+          format: 'csv',
+          path: 'https://storage.openspending.org/berlin-berlin/berlin-gesamt.csv',
+          mediatype: 'text/csv',
+          bytes: 81128743,
+          encoding: 'utf-8',
+        },
+      ],
+      author: 'Michael Peters <michael.peters@okfn.de>',
+      cityCode: 'Berlin',
+      countryCode: 'DE',
+      fiscalPeriod: { start: '2014-01-01', end: '2019-12-31' },
+      readme: '',
+    });
+  },
+  { timeout: 100000 }
+);
+
+test(
+  'Test getting one section of csv from R2',
+  async () => {
+    const rawCsv = await getCsv(
+      'https://storage.openspending.org/state-of-minas-gerais-brazil-planned-budget/__os_imported__br-mg-ppagloc.csv'
+    );
+    const parsedCsv = await parseCsv(rawCsv);
+    expect(parsedCsv.errors.length).toBe(1);
+    expect(parsedCsv.data.length).toBe(10165);
+    expect(parsedCsv.meta.fields).toStrictEqual([
+      'function_name',
+      'function_label',
+      'product_name',
+      'product_label',
+      'area_name',
+      'area_label',
+      'subaction_name',
+      'subaction_label',
+      'region_label_map',
+      'region_reg_map',
+      'region_name',
+      'region_label',
+      'municipality_map_id',
+      'municipality_name',
+      'municipality_map_code',
+      'municipality_label',
+      'municipality_map_name_simple',
+      'municipality_map_name',
+      'cofog1_label_en',
+      'cofog1_name',
+      'cofog1_label',
+      'amount',
+      'subprogramme_name',
+      'subprogramme_label',
+      'time_name',
+      'time_year',
+      'time_month',
+      'time_day',
+      'time_week',
+      'time_yearmonth',
+      'time_quarter',
+      'time',
+      'action_name',
+      'action_label',
+      'subfunction_name',
+      'subfunction_label',
+      'programme_name',
+      'programme_label',
+    ]);
+  },
+  { timeout: 100000 }
+);
--- a/examples/openspending/components/Button.tsx
+++ b/examples/openspending/components/Button.tsx
@@ -1,15 +1,15 @@
-import Link from 'next/link'
-import clsx from 'clsx'
+import Link from 'next/link';
+import clsx from 'clsx';

-export function Button({ href, className = "", ...props }) {
+export function Button({ href, className = '', ...props }) {
  className = clsx(
    'inline-flex justify-center rounded-2xl bg-emerald-600 p-4 text-base font-semibold text-white hover:bg-emerald-500 focus:outline-none focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-emerald-500 active:text-white/70',
    className
-  )
+  );

  return href ? (
-    <Link href={href} className={className} {...props} />
+    <Link scroll={false} href={href} className={className} {...props} />
  ) : (
    <button className={className} {...props} />
-  )
+  );
 }
--- a/examples/openspending/components/DatasetCard.tsx
+++ b/examples/openspending/components/DatasetCard.tsx
@@ -0,0 +1,76 @@
+import Link from 'next/link';
+import { Project } from '../lib/project.interface';
+import ExternalLinkIcon from './icons/ExternalLinkIcon';
+
+export default function DatasetCard({ dataset }: { dataset: Project }) {
+  return (
+    <div
+      key={dataset.name}
+      className="overflow-hidden rounded-xl border border-gray-200"
+    >
+      <Link
+        href={`/@${dataset.owner.name}/${dataset.repo.name}`}
+        className="flex items-center gap-x-4 border-b border-gray-900/5 bg-gray-50 p-6"
+      >
+        <img
+          src={dataset.owner.logo || '/assets/org-icon.svg'}
+          alt={dataset.owner.name}
+          className="h-12 w-12 flex-none rounded-lg bg-white object-cover ring-1 ring-gray-900/10 p-2"
+        />
+        <div className="text-sm font-medium leading-6">
+          <div className="text-gray-900 line-clamp-1">{dataset.title}</div>
+          <div className="text-gray-500 line-clamp-1">
+            {dataset.owner.title}
+          </div>
+        </div>
+      </Link>
+      <dl className="-my-3 divide-y divide-gray-100 px-6 py-4 text-sm leading-6">
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Name</dt>
+          <dd className="flex items-start gap-x-2">
+            <div className="font-medium text-gray-900 line-clamp-1">
+              {dataset.name}
+            </div>
+          </dd>
+        </div>
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Country</dt>
+          <dd className="flex items-start gap-x-2">
+            <div className="font-medium text-gray-900">
+              {dataset.countryCode}
+            </div>
+          </dd>
+        </div>
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Fiscal Period</dt>
+          <dd className="text-gray-700">
+            {dataset.fiscalPeriod?.start &&
+              new Date(dataset.fiscalPeriod.start).getFullYear()}
+            {dataset.fiscalPeriod?.end &&
+              dataset.fiscalPeriod?.start !== dataset.fiscalPeriod?.end && (
+                <>
+                  {' - '}
+                  {new Date(dataset.fiscalPeriod.end).getFullYear()}
+                </>
+              )}
+          </dd>
+        </div>
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Metadata</dt>
+          <dd className="flex items-start gap-x-2">
+            <div className="font-medium text-gray-900">
+              <Link
+                // TODO: this link may be incorrect for some datasets
+                href={`https://github.com/${dataset.owner.name}/${dataset.repo.name}/blob/main/datapackage.json`}
+                target="_blank"
+                className="flex items-center hover:text-gray-700"
+              >
+                datapackage.json <ExternalLinkIcon className="ml-1" />
+              </Link>
+            </div>
+          </dd>
+        </div>
+      </dl>
+    </div>
+  );
+}
--- a/examples/openspending/components/DatasetsGrid.tsx
+++ b/examples/openspending/components/DatasetsGrid.tsx
@@ -0,0 +1,19 @@
+import { Project } from '../lib/project.interface';
+import DatasetCard from './DatasetCard';
+
+export default function DatasetsGrid({ datasets }: { datasets: Project[] }) {
+  return (
+    <ul
+      className="grid gap-x-6 gap-y-8 grid-cols-1 sm:grid-cols-2 md:grid-cols-3"
+      role="list"
+    >
+      {datasets.map((dataset, idx) => {
+        return (
+          <li key={`datasets-grid-item-${idx}`}>
+            <DatasetCard dataset={dataset} />
+          </li>
+        );
+      })}
+    </ul>
+  );
+}
--- a/examples/openspending/components/DatasetsSearch.tsx
+++ b/examples/openspending/components/DatasetsSearch.tsx
@@ -0,0 +1,206 @@
+import { useForm } from 'react-hook-form';
+import DatasetsGrid from './DatasetsGrid';
+import { Project } from '../lib/project.interface';
+import { Index } from 'flexsearch';
+import {
+  ChevronDoubleLeftIcon,
+  ChevronDoubleRightIcon,
+  ChevronLeftIcon,
+  ChevronRightIcon,
+} from '@heroicons/react/24/solid';
+import { useState } from 'react';
+
+export default function DatasetsSearch({
+  datasets,
+  availableCountries,
+}: {
+  datasets: Project[];
+  availableCountries;
+}) {
+  const itemsPerPage = 6;
+  const [page, setPage] = useState(1);
+
+  const index = new Index({ tokenize: 'full' });
+
+  datasets.forEach((dataset: Project) =>
+    index.add(
+      dataset.name,
+      `${dataset.repo} ${dataset.name} ${dataset.title} ${dataset.author} ${dataset.title} ${dataset.cityCode} ${dataset.fiscalPeriod?.start} ${dataset.fiscalPeriod?.end}`
+    )
+  );
+
+  const { register, watch, handleSubmit, reset, resetField } = useForm({
+    defaultValues: {
+      searchTerm: '',
+      country: '',
+      minDate: '',
+      maxDate: '',
+    },
+  });
+
+  const filteredDatasets = datasets
+    .filter((dataset: Project) =>
+      watch().searchTerm && watch().searchTerm !== ''
+        ? index.search(watch().searchTerm).includes(dataset.name)
+        : true
+    )
+    .filter((dataset) =>
+      watch().country && watch().country !== ''
+        ? dataset.countryCode === watch().country
+        : true
+    )
+    // TODO: Does that really makes sense?
+    // What if the fiscalPeriod is 2015-2017 and inputs are
+    // set to 2015-2016. It's going to be filtered out but
+    // it shouldn't.
+    .filter((dataset) =>
+      watch().minDate && watch().minDate !== ''
+        ? dataset.fiscalPeriod?.start >= watch().minDate
+        : true
+    )
+    .filter((dataset) =>
+      watch().maxDate && watch().maxDate !== ''
+        ? dataset.fiscalPeriod?.end <= watch().maxDate
+        : true
+    );
+
+  const paginatedDatasets = filteredDatasets.slice(
+    (page - 1) * itemsPerPage,
+    (page - 1) * itemsPerPage + itemsPerPage
+  );
+
+  const pageCount = Math.ceil(filteredDatasets.length / itemsPerPage) || 1;
+
+  return (
+    <>
+      <div className="flex flex-col gap-3 sm:flex-row">
+        <div className="min-w-0 flex-auto">
+          <br />
+          <div className="relative">
+            <input
+              placeholder="Search datasets"
+              aria-label="Search datasets"
+              {...register('searchTerm', { onChange: () => setPage(1) })}
+              className="h-[3em] relative w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+            {watch().searchTerm !== '' && (
+              <button
+                onClick={() => resetField('searchTerm')}
+                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
+              >
+                <CloseIcon />
+              </button>
+            )}
+          </div>
+        </div>
+        <div className="sm:basis-1/6">
+          {/* TODO: nicer select e.g. headlessui example */}
+          <label className="text-sm text-gray-600 font-medium">Country</label>
+          <select
+            className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            {...register('country', { onChange: () => setPage(1) })}
+          >
+            <option value="">All</option>
+            {availableCountries.map((country) => {
+              return (
+                <option key={country.code} value={country.code}>
+                  {country.title}
+                </option>
+              );
+            })}
+          </select>
+        </div>
+        <div className="sm:basis-1/6">
+          <label className="text-sm text-gray-600 font-medium">Min. date</label>
+          <div className="relative">
+            <input
+              aria-label="Min. date"
+              type="date"
+              {...register('minDate', { onChange: () => setPage(1) })}
+              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+          </div>
+        </div>
+        <div className="sm:basis-1/6">
+          <label className="text-sm text-gray-600 font-medium">Max. date</label>
+          <div className="relative">
+            <input
+              aria-label="Max. date"
+              type="date"
+              {...register('maxDate', { onChange: () => setPage(1) })}
+              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+          </div>
+        </div>
+      </div>
+      <div className="mt-5 mb-5">
+        <span className="text-lg font-medium">
+          {filteredDatasets.length} datasets found
+        </span>
+      </div>
+      <div className="min-w-full align-middle">
+        <DatasetsGrid datasets={paginatedDatasets} />
+        <div className="w-full flex justify-center mt-10">
+          <button
+            onClick={() => setPage(1)}
+            disabled={page <= 1}
+            className="disabled:text-gray-400"
+          >
+            <ChevronDoubleLeftIcon className="w-6 h-6" />
+          </button>
+          <button
+            onClick={() => {
+              if (page > 1) setPage((prev) => --prev);
+            }}
+            disabled={page <= 1}
+            className="disabled:text-gray-400"
+          >
+            <ChevronLeftIcon className="w-6 h-6" />
+          </button>
+          <span className="mx-5">
+            Page {page} of {pageCount}
+          </span>
+          <button
+            onClick={() => {
+              if (page < pageCount) setPage((prev) => ++prev);
+            }}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronRightIcon className="w-6 h-6" />
+          </button>
+          <button
+            onClick={() => setPage(pageCount)}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronDoubleRightIcon className="w-6 h-6" />
+          </button>
+        </div>
+      </div>
+    </>
+  );
+}
+
+const CloseIcon = () => {
+  return (
+    <svg
+      width={20}
+      height={20}
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g id="Menu / Close_MD">
+        <path
+          id="Vector"
+          d="M18 18L12 12M12 12L6 6M12 12L18 6M12 12L6 18"
+          stroke="currentColor"
+          stroke-width="2"
+          stroke-linecap="round"
+          stroke-linejoin="round"
+        />
+      </g>
+    </svg>
+  );
+};
--- a/examples/openspending/components/Header.tsx
+++ b/examples/openspending/components/Header.tsx
@@ -1,54 +1,82 @@
-import Image from 'next/image'
-import { Button } from './Button'
-import { Container } from './Container'
-import logo from "../public/logo.svg"
-import Link from 'next/link'
-import { useRouter } from 'next/router'
+import Image from 'next/image';
+import { Container } from './Container';
+import logo from '../public/logo.svg';
+import Link from 'next/link';
+import { useRouter } from 'next/router';
+import { Bars3Icon } from '@heroicons/react/24/outline';
+import { useState } from 'react';

 export function Header() {
+  const [menuOpen, setMenuOpen] = useState<boolean>(false);
+
  const router = useRouter();

  const isActive = (navLink) => {
-    return router.asPath.split("?")[0] == navLink.href;
-  }
+    return router.asPath.split('?')[0] == navLink.href;
+  };

  const navLinks = [
    {
-      title: "Home",
-      href: "/#header"
+      title: 'Home',
+      href: '/',
    },
    {
-      title: "Datasets",
-      href: "/#datasets"
+      title: 'Datasets',
+      href: '/#datasets',
    },
-    {
-      title: "Community",
-      href: "https://community.openspending.org/"
-    }
-  ]
+    // {
+    //   title: "Community",
+    //   href: "https://community.openspending.org/"
+    // }
+  ];

  return (
-    <header className="z-50 pb-5 lg:pt-11 sticky top-0 backdrop-blur" id="header">
-      <Container className="flex flex-wrap items-center justify-center sm:justify-between lg:flex-nowrap">
-        <div className="mt-10 lg:mt-0 lg:grow lg:basis-0 flex items-center">
+    <header className="relative z-50 pb-11 lg:pt-11">
+      <Container className="flex flex-wrap items-center justify-between lg:flex-nowrap mt-10 lg:mt-0">
+        <Link href="/" className="lg:mt-0 lg:grow lg:basis-0 flex items-center">
          <Image src={logo} alt="OpenSpending" className="h-12 w-auto" />
-        </div>
-        <ul className='list-none flex gap-x-5 text-base font-medium'>
+        </Link>
+        <ul className="hidden list-none sm:flex gap-x-5 text-base font-medium">
          {navLinks.map((link, i) => (
            <li key={`nav-link-${i}`}>
              <Link
-                className={`text-emerald-900 hover:text-emerald-600 ${isActive(link) ? "text-emerald-600" : ""}`}
+                className={`text-emerald-900 hover:text-emerald-600 ${
+                  isActive(link) ? 'text-emerald-600' : ''
+                }`}
                href={link.href}
                scroll={false}
              >
                {link.title}
              </Link>
-            </li>))}
+            </li>
+          ))}
        </ul>
-        <div className="hidden sm:mt-10 sm:flex lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
-          <Button href="#">View on GitHub</Button>
+        <div className="hidden xl:block xl:grow"></div>
+        <div className="sm:hidden sm:mt-10 lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
+          <button onClick={() => setMenuOpen(!menuOpen)}>
+            <Bars3Icon className="w-8 h-8" />
+          </button>
        </div>
+        {menuOpen && (
+          <div className={`sm:hidden basis-full mt-5 text-center`}>
+            <ul className="gap-x-5 text-base font-medium">
+              {navLinks.map((link, i) => (
+                <li key={`nav-link-${i}`}>
+                  <Link
+                    className={`text-emerald-900 hover:text-emerald-600 ${
+                      isActive(link) ? 'text-emerald-600' : ''
+                    }`}
+                    href={link.href}
+                    scroll={false}
+                  >
+                    {link.title}
+                  </Link>
+                </li>
+              ))}
+            </ul>
+          </div>
+        )}
      </Container>
-    </header >
-  )
+    </header>
+  );
 }
--- a/examples/openspending/components/Hero.tsx
+++ b/examples/openspending/components/Hero.tsx
@@ -1,9 +1,9 @@
-import { Button } from './Button'
-import { Container } from './Container'
+import { Button } from './Button';
+import { Container } from './Container';

-export function Hero() {
+export function Hero({ countriesCount, datasetsCount, filesCount }) {
  return (
-    <div className="relative pb-20 pt-10 sm:py-40">
+    <div className="relative pb-20 pt-10 sm:py-40" id="hero">
      <div className="absolute inset-x-0 -bottom-14 -top-48 overflow-hidden bg-green-50 bg-opacity-50">
        <div className="absolute inset-x-0 top-0 h-40 bg-gradient-to-b from-white" />
        <div className="absolute inset-x-0 bottom-0 h-40 bg-gradient-to-t from-white" />
@@ -15,22 +15,25 @@ export function Hero() {
          </h1>
          <div className="mt-6 space-y-6 font-display text-2xl tracking-tight text-emerald-900">
            <p>
-              By understanding how governments spend money in our name can we have a say
-              in how that money will affect our own lives. The journey starts here.
+              By understanding how governments spend money in our name can we
+              have a say in how that money will affect our own lives. The
+              journey starts here.
            </p>
            <p>
-              OpenSpending is a free, open and global platform to search, visualise and analyse
-              fiscal data in the public sphere.
+              OpenSpending is a free, open and global platform to search,
+              visualise and analyse fiscal data in the public sphere.
            </p>
          </div>
-          <Button href="#" className="mt-10 w-full sm:hidden">
-            View on GitHub
+          <Button href="#datasets" className="mt-10">
+            Search datasets
          </Button>
          <dl className="mt-10 grid grid-cols-2 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
            {[
-              ['Countries', '75'],
-              ['Datasets', '2091'],
-              ['Files', '9230'],
+              //  Added the plus sign because some datasets do not
+              //  contain defined countries
+              ['Countries', '+' + countriesCount],
+              ['Datasets', datasetsCount],
+              ['Files', filesCount],
            ].map(([name, value]) => (
              <div key={name}>
                <dt className="font-mono text-sm text-emerald-600">{name}</dt>
@@ -43,5 +46,5 @@ export function Hero() {
        </div>
      </Container>
    </div>
-  )
+  );
 }
--- a/examples/openspending/components/Table.tsx
+++ b/examples/openspending/components/Table.tsx
@@ -0,0 +1,94 @@
+import {
+  QueryClient,
+  QueryClientProvider,
+  useQuery,
+  useQueryClient,
+} from 'react-query';
+import Papa from 'papaparse';
+import { Grid } from '@githubocto/flat-ui';
+
+const queryClient = new QueryClient();
+
+export async function getCsv(url: string) {
+  const response = await fetch(url, {
+    headers: {
+      Range: 'bytes=0-5132288',
+    },
+  });
+  const data = await response.text();
+  return data;
+}
+
+export async function parseCsv(file: string): Promise<any> {
+  return new Promise((resolve, reject) => {
+    Papa.parse(file, {
+      header: true,
+      dynamicTyping: true,
+      skipEmptyLines: true,
+      transform: (value: string): string => {
+        return value.trim();
+      },
+      complete: (results: any) => {
+        return resolve(results);
+      },
+      error: (error: any) => {
+        return reject(error);
+      },
+    });
+  });
+}
+
+const Spinning = () => {
+  return (
+    <div role="status w-fit mx-auto">
+      <svg
+        aria-hidden="true"
+        className="w-8 h-8 mr-2 text-gray-200 animate-spin dark:text-gray-600 fill-emerald-600"
+        viewBox="0 0 100 101"
+        fill="none"
+        xmlns="http://www.w3.org/2000/svg"
+      >
+        <path
+          d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
+          fill="currentColor"
+        />
+        <path
+          d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
+          fill="currentFill"
+        />
+      </svg>
+      <span className="sr-only">Loading...</span>
+    </div>
+  );
+};
+
+export const Table: React.FC<{ url: string }> = ({ url }) => {
+  return (
+    // Provide the client to your App
+    <QueryClientProvider client={queryClient}>
+      <TableInner url={url} />
+    </QueryClientProvider>
+  );
+};
+
+const TableInner: React.FC<{ url: string }> = ({ url }) => {
+  const { data: csvString, isLoading: isDownloadingCSV } = useQuery(
+    ['dataCsv', url],
+    () => getCsv(url)
+  );
+  const { data: parsedData, isLoading: isParsing } = useQuery(
+    ['dataPreview', csvString],
+    () => parseCsv(csvString),
+    { enabled: !!csvString }
+  );
+  if (isParsing || isDownloadingCSV)
+    <div className="w-full">
+      <Spinning />
+    </div>;
+  if (parsedData)
+    return (
+      <div className="h-[500px] overflow-scroll">
+        <Grid data={parsedData.data} />
+      </div>
+    );
+};
--- a/examples/openspending/components/_shared/Layout.tsx
+++ b/examples/openspending/components/_shared/Layout.tsx
@@ -0,0 +1,10 @@
+import { Header } from '../Header';
+
+export default function Layout({ children }) {
+  return (
+    <div className="bg-white min-h-screen">
+      <Header />
+      {children}
+    </div>
+  );
+}
--- a/examples/openspending/datasets.json
+++ b/examples/openspending/datasets.json
@@ -1,25 +0,0 @@
-[
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "repo": "mongolia-budget-2016-2017",
-    "files": [
-      "data/mongolia-2017.csv",
-      "data/mongolia-2017__2017.csv"
-    ]
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "repo": "gb-country-regional-analysis",
-    "files": [
-      "data/cofog.csv",
-      "data/cofog_dejargonise.csv",
-      "data/cra.csv",
-      "data/departments.csv",
-      "data/nuts_pop.csv",
-      "data/pogs.csv"
-    ],
-    "readme": "README.md"
-  }
-]
--- a/examples/openspending/lib/datapackage.interface.ts
+++ b/examples/openspending/lib/datapackage.interface.ts
@@ -0,0 +1,289 @@
+/**
+ * Fiscal Data Package is a simple specification for data access and delivery of fiscal data.
+ */
+export type FiscalDataPackage = TabularDataPackage & {
+  countryCode?: ISO31661Alpha2CountryCode
+  regionCode?: string
+  cityCode?: string
+  author?: string
+  readme?: string
+  granularity?: GranularityOfResources
+  fiscalPeriod?: FiscalPeriodForTheBudget
+  [k: string]: unknown
+}
+/**
+ * The profile of this descriptor.
+ */
+export type Profile = "tabular-data-package"
+/**
+ * An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.
+ */
+export type Name = string
+/**
+ * A property reserved for globally unique identifiers. Examples of identifiers that are unique include UUIDs and DOIs.
+ */
+export type ID = string
+/**
+ * A human-readable title.
+ */
+export type Title = string
+/**
+ * A text description. Markdown is encouraged.
+ */
+export type Description = string
+/**
+ * The home on the web that is related to this data package.
+ */
+export type HomePage = string
+/**
+ * The datetime on which this descriptor was created.
+ */
+export type Created = string
+/**
+ * The contributors to this descriptor.
+ */
+export type Contributors = [Contributor, ...Contributor[]]
+/**
+ * A human-readable title.
+ */
+export type Title1 = string
+/**
+ * A fully qualified URL, or a POSIX file path.
+ */
+export type Path = string
+/**
+ * An email address.
+ */
+export type Email = string
+/**
+ * An organizational affiliation for this contributor.
+ */
+export type Organization = string
+/**
+ * A list of keywords that describe this package.
+ */
+export type Keywords = [string, ...string[]]
+/**
+ * A image to represent this package.
+ */
+export type Image = string
+/**
+ * The license(s) under which this package is published.
+ */
+export type Licenses = [License, ...License[]]
+/**
+ * A license for this descriptor.
+ */
+export type License =
+  | {
+      [k: string]: unknown
+    }
+  | {
+      [k: string]: unknown
+    }
+/**
+ * An `array` of Tabular Data Resource objects, each compliant with the [Tabular Data Resource](/tabular-data-resource/) specification.
+ *
+/**
+ * A Tabular Data Resource.
+ */
+export interface TabularDataResource {
+  format?: string;
+  name: string;
+  description?: string;
+  title?: string;
+  schema?: Schema;
+  sample?: any[];
+  profile?: string;
+  key?: string;
+  path?: string;
+  size?: number;
+  bytes?: number;
+}
+
+export interface Field {
+  name: string;
+  type: FieldType;
+}
+
+export interface Schema {
+  fields: Field[];
+}
+
+export const OptionsFields = [
+  "any",
+  "array",
+  "boolean",
+  "date",
+  "datetime",
+  "duration",
+  "geojson",
+  "geopoint",
+  "integer",
+  "number",
+  "object",
+  "string",
+  "time",
+  "year",
+  "yearmonth",
+] as const;
+
+type FieldType = typeof OptionsFields[number];
+/**
+ * A human-readable title.
+ */
+export type Title2 = string
+/**
+ * A fully qualified URL, or a POSIX file path.
+ */
+export type Path1 = string
+/**
+ * An email address.
+ */
+export type Email1 = string
+/**
+ * The raw sources for this resource.
+ */
+export type Sources = Source[]
+
+/**
+ * A keyword that represents the direction of the spend, either expenditure or revenue.
+ */
+export type DirectionOfTheSpending = "expenditure" | "revenue"
+/**
+ * A keyword that represents the phase of the data, can be proposed for a budget proposal, approved for an approved budget, adjusted for modified budget or executed for the enacted budget
+ */
+export type BudgetPhase = "proposed" | "approved" | "adjusted" | "executed"
+/**
+ * Either an array of strings corresponding to the name attributes in a set of field objects in the fields array or a single string corresponding to one of these names. The value of primaryKey indicates the primary key or primary keys for the dimension.
+ */
+export type PrimaryKey = string | [string, ...string[]]
+/**
+ * Describes what kind of a dimension it is.
+ */
+export type DimensionType =
+  | "datetime"
+  | "entity"
+  | "classification"
+  | "activity"
+  | "fact"
+  | "location"
+  | "other"
+/**
+ * The type of the classification.
+ */
+export type ClassificationType = "functional" | "administrative" | "economic"
+/**
+ * A valid 2-digit ISO country code (ISO 3166-1 alpha-2), or, an array of valid ISO codes.
+ */
+export type ISO31661Alpha2CountryCode = string | [string, ...string[]]
+/**
+ * A keyword that represents the type of spend data, eiter aggregated or transactional
+ */
+export type GranularityOfResources = "aggregated" | "transactional"
+
+/**
+ * Tabular Data Package
+ */
+export interface TabularDataPackage {
+  profile: Profile
+  name?: Name
+  id?: ID
+  title?: Title
+  description?: Description
+  homepage?: HomePage
+  created?: Created
+  contributors?: Contributors
+  keywords?: Keywords
+  image?: Image
+  licenses?: Licenses
+  resources: TabularDataResource[]
+  sources?: Sources
+  [k: string]: unknown
+}
+/**
+ * A contributor to this descriptor.
+ */
+export interface Contributor {
+  title: Title1
+  path?: Path
+  email?: Email
+  organization?: Organization
+  role?: string
+  [k: string]: unknown
+}
+/**
+ * A source file.
+ */
+export interface Source {
+  title: Title2
+  path?: Path1
+  email?: Email1
+  [k: string]: unknown
+}
+/**
+ * Measures are numerical and correspond to financial amounts in the source data.
+ */
+export interface Measures {
+  [k: string]: Measure
+}
+/**
+ * Measure.
+ *
+ * This interface was referenced by `Measures`'s JSON-Schema definition
+ * via the `patternProperty` "^\w+".
+ */
+export interface Measure {
+  source: string
+  resource?: string
+  currency: string
+  factor?: number
+  direction?: DirectionOfTheSpending
+  phase?: BudgetPhase
+  [k: string]: unknown
+}
+/**
+ * Dimensions are groups of related fields. Dimensions cover all items other than the measure.
+ */
+export interface Dimensions {
+  [k: string]: Dimension
+}
+/**
+ * Dimension.
+ *
+ * This interface was referenced by `Dimensions`'s JSON-Schema definition
+ * via the `patternProperty` "^\w+".
+ */
+export interface Dimension {
+  attributes: Attributes
+  primaryKey: PrimaryKey
+  dimensionType?: DimensionType
+  classificationType?: ClassificationType
+  [k: string]: unknown
+}
+/**
+ * Attribute objects that make up the dimension
+ */
+export interface Attributes {
+  /**
+   * This interface was referenced by `Attributes`'s JSON-Schema definition
+   * via the `patternProperty` "^\w+".
+   */
+  [k: string]: {
+    source: string
+    resource?: string
+    constant?: string | number
+    parent?: string
+    labelfor?: string
+    [k: string]: unknown
+  }
+}
+/**
+ * The fiscal period of the dataset
+ */
+export interface FiscalPeriodForTheBudget {
+  start: string
+  end?: string
+  [k: string]: unknown
+}
+
--- a/examples/openspending/lib/loader.ts
+++ b/examples/openspending/lib/loader.ts
@@ -0,0 +1,35 @@
+import { FiscalDataPackage } from './datapackage.interface';
+import { Project } from './project.interface';
+
+export function loadDataPackage(datapackage: FiscalDataPackage, repo): Project {
+  return {
+    name: datapackage.name,
+    title: datapackage.title,
+    description: datapackage.description || null,
+    owner: {
+      name: repo.owner.login,
+      logo: repo.owner.avatar_url,
+      // TODO: make this title work
+      title: repo.owner.login,
+    },
+    repo: { name: repo.name, full_name: repo.full_name, url: repo.html_url },
+    files: datapackage.resources,
+    author: datapackage.author ? datapackage.author : null,
+    cityCode: datapackage.cityCode ? datapackage.cityCode : null,
+    countryCode: datapackage.countryCode
+      ? (datapackage.countryCode as string)
+      : null,
+    fiscalPeriod: datapackage.fiscalPeriod
+      ? {
+          start: datapackage.fiscalPeriod.start
+            ? datapackage.fiscalPeriod.start
+            : null,
+          end: datapackage.fiscalPeriod.end
+            ? datapackage.fiscalPeriod.end
+            : null,
+        }
+      : null,
+    readme: datapackage.readme ? datapackage.readme : '',
+    datapackage,
+  };
+}
--- a/examples/openspending/lib/project.interface.ts
+++ b/examples/openspending/lib/project.interface.ts
@@ -0,0 +1,22 @@
+import {
+  FiscalDataPackage,
+  TabularDataResource,
+} from './datapackage.interface';
+
+export interface Project {
+  owner: { name: string; logo?: string; title?: string }; // Info about the owner of the data repo
+  repo: { name: string; full_name: string; url: string }; // Info about the the data repo
+  files: TabularDataResource[];
+  name: string;
+  title?: string;
+  description?: string;
+  author?: string;
+  cityCode?: string;
+  countryCode?: string;
+  fiscalPeriod?: {
+    start: string;
+    end: string;
+  };
+  readme?: string;
+  datapackage: FiscalDataPackage;
+}
--- a/examples/openspending/lib/project.ts
+++ b/examples/openspending/lib/project.ts
@@ -13,8 +13,7 @@ export interface GithubProject {
 export async function getProjectReadme(
  owner: string,
  repo: string,
-  branch: string,
-  readme: string,
+  branch: string = 'main',
  github_pat?: string
 ) {
  const octokit = new Octokit({ auth: github_pat });
@@ -22,7 +21,7 @@ export async function getProjectReadme(
    const response = await octokit.rest.repos.getContent({
      owner,
      repo,
-      path: readme,
+      path: 'README.md',
      ref: branch,
    });
    const data = response.data as { content?: string };
@@ -125,7 +124,6 @@ export async function getProject(project: GithubProject, github_pat?: string) {
    project.owner,
    project.repo,
    project.branch,
-    project.readme,
    github_pat
  );

@@ -140,7 +138,8 @@ export async function getProject(project: GithubProject, github_pat?: string) {
    return null;
  }

-  let projectBase = "", last_updated = "";
+  let projectBase = '',
+    last_updated = '';
  if (projectReadme) {
    projectBase =
      project.readme.split('/').length > 1
@@ -162,3 +161,65 @@ export async function getProject(project: GithubProject, github_pat?: string) {
    base_path: projectBase,
  };
 }
+
+export async function getProjectDataPackage(
+  owner: string,
+  repo: string,
+  branch: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.getContent({
+      owner,
+      repo,
+      path: 'datapackage.json',
+      ref: branch,
+    });
+    const data = response.data as { content?: string };
+    const fileContent = data.content ? data.content : '';
+    if (fileContent === '') {
+      return null;
+    }
+    const decodedContent = Buffer.from(fileContent, 'base64').toString();
+    const datapackage = JSON.parse(decodedContent);
+
+    return { ...datapackage, repo };
+  } catch (error) {
+    return null;
+  }
+}
+
+export async function getAllProjectsFromOrg(
+  org: string,
+  branch?: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  const repos = await octokit.rest.repos.listForOrg({
+    org,
+    type: 'public',
+    per_page: 100,
+  });
+  let failedProjects = [];
+  const projects = await Promise.all(
+    repos.data.map(async (_repo) => {
+      const project = await getProjectDataPackage(
+        org,
+        _repo.name,
+        branch ? branch : 'main',
+        github_pat
+      );
+      if (!project) {
+        failedProjects.push(_repo.name);
+        return null;
+      }
+
+      return { datapackage: project, repo: _repo };
+    })
+  );
+  return {
+    results: projects.filter((item) => item !== null),
+    failed: failedProjects,
+  };
+}
--- a/examples/openspending/orgs.json
+++ b/examples/openspending/orgs.json
@@ -0,0 +1,3 @@
+[
+ "os-data"
+]
--- a/examples/openspending/package-lock.json
+++ b/examples/openspending/package-lock.json
--- a/examples/openspending/package.json
+++ b/examples/openspending/package.json
@@ -6,30 +6,42 @@
    "dev": "next dev",
    "build": "next build",
    "start": "next start",
-    "lint": "next lint"
+    "lint": "next lint",
+    "test": "vitest"
  },
  "dependencies": {
+    "@githubocto/flat-ui": "^0.14.1",
+    "@heroicons/react": "^2.0.18",
+    "@octokit/plugin-throttling": "^5.2.2",
+    "@types/flexsearch": "^0.7.3",
    "@types/node": "18.16.0",
    "@types/react": "18.0.38",
    "@types/react-dom": "18.0.11",
+    "@vitejs/plugin-react": "^4.0.0",
    "clsx": "^1.2.1",
    "eslint": "8.39.0",
    "eslint-config-next": "13.3.1",
+    "flexsearch": "0.7.21",
    "next": "13.3.1",
    "next-seo": "^6.0.0",
    "octokit": "^2.0.14",
+    "papaparse": "^5.4.1",
    "prettier": "^2.8.8",
    "react": "18.2.0",
    "react-dom": "18.2.0",
+    "react-hook-form": "^7.43.9",
    "react-markdown": "^8.0.7",
+    "react-query": "^3.39.3",
    "react-timeago": "^7.1.0",
    "remark-gfm": "^3.0.1",
    "typescript": "5.0.4"
  },
  "devDependencies": {
    "@tailwindcss/typography": "^0.5.9",
+    "@types/papaparse": "^5.3.7",
    "autoprefixer": "^10.4.14",
    "postcss": "^8.4.23",
-    "tailwindcss": "^3.3.1"
+    "tailwindcss": "^3.3.1",
+    "vitest": "^0.31.0"
  }
 }
--- a/examples/openspending/pages/@org/[org]/[...path].tsx
+++ b/examples/openspending/pages/@org/[org]/[...path].tsx
@@ -1,126 +0,0 @@
-import { NextSeo } from 'next-seo';
-import { promises as fs } from 'fs';
-import path from 'path';
-import getConfig from 'next/config';
-import { getProject, GithubProject } from '../../../lib/octokit';
-import ReactMarkdown from 'react-markdown';
-import remarkGfm from 'remark-gfm';
-import Breadcrumbs from '../../../components/_shared/Breadcrumbs';
-
-export default function ProjectPage({ project }) {
-  const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`
-
-  return (
-    <>
-      <NextSeo title={`${repoId}${project.base_path !== '/' ? '/' + project.base_path : ''} - GitHub Datasets`} />
-      <main className="prose mx-auto my-8">
-        <Breadcrumbs links={[{ title: repoId, href: "" }]} />
-        <h1 className="mb-0 mt-16">{project.repo_config.name || repoId}</h1>
-        <p className='mb-8'><span className='font-semibold'>Repository:</span> <a target="_blank" href={project.html_url}>{project.html_url}</a></p>
-
-        <h2 className="mb-0 mt-10">Files</h2>
-        <div className="inline-block min-w-full py-2 align-middle">
-          <table className="min-w-full divide-y divide-gray-300">
-            <thead>
-              <tr>
-                <th
-                  scope="col"
-                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                >
-                  Name
-                </th>
-                <th
-                  scope="col"
-                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                >
-                  Size
-                </th>
-              </tr>
-            </thead>
-            <tbody className="divide-y divide-gray-200">
-              {project.files.map((file) => (
-                <tr key={file.download_url}>
-                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                    <a href={file.download_url}>{file.name}</a>
-                  </td>
-                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                    {file.size} Bytes
-                  </td>
-                </tr>
-              ))}
-            </tbody>
-          </table>
-        </div>
-
-        {project.readmeContent && <>
-          <hr />
-
-          <h2 className='uppercase font-black'>Readme</h2>
-          <ReactMarkdown remarkPlugins={[remarkGfm]}>
-            {project.readmeContent}
-          </ReactMarkdown>
-        </>}
-      </main>
-    </>
-  );
-}
-
-// Generates `/posts/1` and `/posts/2`
-export async function getStaticPaths() {
-  const jsonDirectory = path.join(
-    process.cwd(),
-    'datasets.json'
-  );
-  const repos = await fs.readFile(jsonDirectory, 'utf8');
-
-  return {
-    paths: JSON.parse(repos).map((repo) => {
-      const projectPath =
-        repo.readme && repo.readme.split('/').length > 1
-          ? repo.readme.split('/').slice(0, -1)
-          : null;
-      let path = [repo.repo];
-      if (projectPath) {
-        projectPath.forEach((element) => {
-          path.push(element);
-        });
-      }
-      return {
-        params: { org: repo.owner, path },
-      };
-    }),
-    fallback: false, // can also be true or 'blocking'
-  };
-}
-
-export async function getStaticProps({ params }) {
-  const jsonDirectory = path.join(
-    process.cwd(),
-    'datasets.json'
-  );
-  const reposFile = await fs.readFile(jsonDirectory, 'utf8');
-  const repos: GithubProject[] = JSON.parse(reposFile);
-  const repo = repos.find((_repo) => {
-    const projectPath =
-      _repo.readme && _repo.readme.split('/').length > 1
-        ? _repo.readme.split('/').slice(0, -1)
-        : null;
-    let path = [_repo.repo];
-    if (projectPath) {
-      projectPath.forEach((element) => {
-        path.push(element);
-      });
-    }
-    return (
-      _repo.owner == params.org &&
-      JSON.stringify(path) === JSON.stringify(params.path)
-    );
-  });
-  const github_pat = getConfig().serverRuntimeConfig.github_pat;
-  const project = await getProject(repo, github_pat);
-  return {
-    props: {
-      project: { ...project, repo_config: repo },
-    },
-  };
-}
--- a/examples/openspending/pages/@org/[org]/[project].tsx
+++ b/examples/openspending/pages/@org/[org]/[project].tsx
@@ -0,0 +1,234 @@
+import { NextSeo } from 'next-seo';
+import getConfig from 'next/config';
+import {
+  getAllProjectsFromOrg,
+  getProjectDataPackage,
+  getProjectMetadata,
+  getProjectReadme,
+} from '../../../lib/project';
+import ReactMarkdown from 'react-markdown';
+import remarkGfm from 'remark-gfm';
+import { loadDataPackage } from '../../../lib/loader';
+import Layout from '../../../components/_shared/Layout';
+import Link from 'next/link';
+import { Project } from '../../../lib/project.interface';
+import ExternalLinkIcon from '../../../components/icons/ExternalLinkIcon';
+
+export default function ProjectPage({
+  project,
+  readme,
+}: {
+  project: Project;
+  readme: string;
+}) {
+
+  //  Get description from datapackage or  calculate
+  //  excerpt from README by getting all the content
+  //  up to the first dot.
+  const description =
+    project.description || (readme && readme.slice(0, readme.indexOf('.') + 1));
+
+  return (
+    <Layout>
+      <NextSeo title={`${project.title} - OpenSpending`} />
+      <main className="prose mx-auto my-8">
+        <h1 className="mb-1 mt-16">{project.title || project.name}</h1>
+        <Link target="_blank" href={project.repo.url}>
+          @{project.repo.full_name}
+        </Link>
+
+        {description && (
+          <div className="inline-block min-w-full py-2 align-middle mt-5">
+            {description}
+          </div>
+        )}
+
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="min-w-full divide-y divide-gray-300">
+            <thead>
+              <tr>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Name
+                </th>
+                {project.datapackage.countryCode && (
+                  <th
+                    scope="col"
+                    className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                  >
+                    Country
+                  </th>
+                )}
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Metadata
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              <tr>
+                <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                  {project.name}
+                </td>
+                {project.datapackage.countryCode && (
+                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                    {project.datapackage.countryCode}
+                  </td>
+                )}
+
+                <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                  <Link
+                    // TODO: this link may be incorrect for some datasets
+                    href={`https://github.com/${project.owner.name}/${project.repo.name}/blob/main/datapackage.json`}
+                    target="_blank"
+                    className="flex items-center hover:text-gray-700"
+                  >
+                    datapackage.json <ExternalLinkIcon className="ml-1" />
+                  </Link>
+                </td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+
+        <h3 className="mb-1 mt-10">Data files</h3>
+        <p>
+          This dataset contains {project.files.length} file
+          {project.files.length != 1 ? '' : 's'}
+        </p>
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="mt-0 min-w-full divide-y divide-gray-300">
+            <thead>
+              <tr>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Name
+                </th>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Format
+                </th>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Size
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              {project.files?.map((file) => {
+                let size: number | string = file.size;
+
+                if (!size) {
+                  if (file.bytes) {
+                    if (file.bytes > 1000000) {
+                      size = (file.bytes / 1000000).toFixed(2) + ' MB';
+                    } else {
+                      size = (file.bytes / 1000).toFixed(2) + ' kB';
+                    }
+                  }
+                }
+
+                return (
+                  <tr key={file.name}>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {file.name}
+                    </td>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {file.format}
+                    </td>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {size}
+                    </td>
+
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      <Link
+                        target="_blank"
+                        href={
+                          file.path.startsWith('http')
+                            ? file.path
+                            : `https://raw.githubusercontent.com/${project.owner.name}/${project.repo.name}/main/${file.path}`
+                        }
+                      >
+                        Download
+                      </Link>
+                    </td>
+                  </tr>
+                );
+              })}
+            </tbody>
+          </table>
+        </div>
+
+        {readme && (
+          <>
+            <hr />
+
+            <h2 className="uppercase font-black">Readme</h2>
+            <ReactMarkdown remarkPlugins={[remarkGfm]}>{readme}</ReactMarkdown>
+          </>
+        )}
+      </main>
+    </Layout>
+  );
+}
+
+// Generates `/posts/1` and `/posts/2`
+export async function getStaticPaths() {
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+
+  const allProjects = await getAllProjectsFromOrg(
+    'os-data',
+    'main',
+    github_pat
+  );
+
+  console.log(allProjects)
+  const paths = allProjects.results.map((project) => ({
+    params: {
+      // TODO: dynamize the org
+      org: 'os-data',
+      project: project.repo.name,
+    },
+  }));
+
+  return {
+    paths,
+    fallback: false, // can also be true or 'blocking'
+  };
+}
+
+export async function getStaticProps({ params }) {
+  const { org: orgName, project: projectName } = params;
+
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+  const datapackage = await getProjectDataPackage(
+    orgName,
+    projectName,
+    'main',
+    github_pat
+  );
+
+  const repo = await getProjectMetadata(orgName, projectName, github_pat);
+
+  const project = loadDataPackage(datapackage, repo);
+
+  // TODO: should this be moved to the loader?
+  const readme = await getProjectReadme(orgName, projectName, 'main', github_pat);
+
+  return {
+    props: {
+      project,
+      readme,
+    },
+  };
+}
--- a/examples/openspending/pages/_app.tsx
+++ b/examples/openspending/pages/_app.tsx
@@ -1,13 +1,11 @@
 import { AppProps } from 'next/app';
-import Head from 'next/head';
 import './styles.css';
+import { NextSeo } from 'next-seo';

 function CustomApp({ Component, pageProps }: AppProps) {
  return (
    <>
-      <Head>
-        <title>GitHub Datasets</title>
-      </Head>
+      <NextSeo title="OpenSpending" />
      <main className="app">
        <Component {...pageProps} />
      </main>
--- a/examples/openspending/pages/index.tsx
+++ b/examples/openspending/pages/index.tsx
@@ -1,44 +1,59 @@
-import { promises as fs } from 'fs';
-import path from 'path';
-import { getProject } from '../lib/octokit';
+import { getAllProjectsFromOrg } from '../lib/project';
 import getConfig from 'next/config';
-import ExternalLinkIcon from '../components/icons/ExternalLinkIcon';
-import TimeAgo from 'react-timeago';
-import Link from 'next/link';
 import { Hero } from '../components/Hero';
-import { Header } from '../components/Header';
 import { Container } from '../components/Container';
+import { FiscalDataPackage } from '../lib/datapackage.interface';
+import { loadDataPackage } from '../lib/loader';
+import DatasetsSearch from '../components/DatasetsSearch';
+import Layout from '../components/_shared/Layout';

 export async function getStaticProps() {
-  const jsonDirectory = path.join(
-    process.cwd(),
-    '/datasets.json'
-  );
-  const repos = await fs.readFile(jsonDirectory, 'utf8');
+  //  TODO: support other orgs
+  // const orgsListPath = path.join(process.cwd(), '/orgs.json');
+  // const orgs = await fs.readFile(orgsListPath, 'utf8');
+
  const github_pat = getConfig().serverRuntimeConfig.github_pat;

-  const projects = await Promise.all(
-    (JSON.parse(repos)).map(async (repo) => {
-      const project = await getProject(repo, github_pat);
-      return { ...project, repo_config: repo };
-    })
+  const allProjects = await getAllProjectsFromOrg(
+    'os-data',
+    'main',
+    github_pat
  );

+  const projects = allProjects.results.map(
+    (item: { datapackage: FiscalDataPackage & { repo: string }; repo: any }) =>
+      loadDataPackage(item.datapackage, item.repo)
+  );
+
+  const availableCountries = projects
+    .map((item) => item.countryCode)
+    .filter((v) => v) //  Filters false values
+    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
+    //  TODO: title should be the full name
+    .map((code) => ({ code, title: code }));
+
  return {
    props: {
-      projects,
+      projects: JSON.stringify(projects),
+      availableCountries,
    },
  };
 }

-export function Datasets({ projects }) {
+export function Home({ projects, availableCountries }) {
+  projects = JSON.parse(projects);
+
  return (
-    <div className="bg-white min-h-screen">
-      <Header />
-      <Hero />
-      <section
-        className="py-20 sm:py-32"
-      >
+    <Layout>
+      <Hero
+        countriesCount={availableCountries.length}
+        datasetsCount={projects.length}
+        filesCount={projects.reduce(
+          (partialSum, a) => partialSum + a.files.length,
+          0
+        )}
+      />
+      <section className="py-20 sm:py-32">
        <Container>
          <div className="mx-auto max-w-2xl lg:mx-0">
            <h2
@@ -51,80 +66,16 @@ export function Datasets({ projects }) {
              Find spending data about countries all around the world.
            </p>
          </div>
-          <div className="mt-5">
-            <div className="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
-              <div className="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
-                <table className="min-w-full divide-y divide-gray-300">
-                  <thead>
-                    <tr>
-                      <th
-                        scope="col"
-                        className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                      >
-                        Name
-                      </th>
-                      <th
-                        scope="col"
-                        className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                      >
-                        Repository
-                      </th>
-                      <th
-                        scope="col"
-                        className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                      >
-                        Description
-                      </th>
-                      <th
-                        scope="col"
-                        className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                      >
-                        Last updated
-                      </th>
-                      <th
-                        scope="col"
-                        className="relative py-3.5 pl-3 pr-4 sm:pr-0"
-                      ></th>
-                    </tr>
-                  </thead>
-                  <tbody className="divide-y divide-gray-200">
-                    {projects.map((project) => (
-                      <tr key={project.id}>
-                        <td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
-                          {project.repo_config.name
-                            ? project.repo_config.name
-                            : project.full_name + (project.base_path === '/' ? '' : '/' + project.base_path)}
-                        </td>
-                        <td className="whitespace-nowrap px-3 py-6 text-sm group text-gray-500 hover:text-gray-900 transition-all duration-250">
-                          <a href={project.html_url} target="_blank" className='flex items-center'>@{project.full_name} <ExternalLinkIcon className='ml-1' /></a>
-                        </td>
-                        <td className="px-3 py-4 text-sm text-gray-500">
-                          {project.repo_config.description
-                            ? project.repo_config.description
-                            : project.description}
-                        </td>
-                        <td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
-                          <TimeAgo date={new Date(project.last_updated)} />
-                        </td>
-                        <td className="relative whitespace-nowrap py-6 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
-                          <a
-                            href={`/@${project.repo_config.owner}/${project.repo_config.repo}/${project.base_path === '/' ? '' : project.base_path}`}
-                            className='border border-gray-900 text-gray-900 px-4 py-2 transition-all hover:bg-gray-900 hover:text-white'
-                          >
-                            info
-                          </a>
-                        </td>
-                      </tr>
-                    ))}
-                  </tbody>
-                </table>
-              </div>
-            </div>
+          <div className="mt-10">
+            <DatasetsSearch
+              datasets={projects}
+              availableCountries={availableCountries}
+            />
          </div>
        </Container>
      </section>
-    </div>
+    </Layout>
  );
 }

-export default Datasets;
+export default Home;
--- a/examples/openspending/public/assets/org-icon.svg
+++ b/examples/openspending/public/assets/org-icon.svg
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg fill="#000000" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="800px" height="800px" viewBox="0 0 120 120" enable-background="new 0 0 120 120" xml:space="preserve">
+<rect x="2" y="108.1" width="116" height="11.9"/>
+<rect x="6.744" y="96.582" width="104.979" height="6.543"/>
+<rect x="15.288" y="38.532" width="17.639" height="52.925"/>
+<rect x="50.484" y="38.532" width="17.639" height="52.925"/>
+<rect x="84.33" y="38.532" width="17.639" height="52.925"/>
+<polygon points="0,26.96 60,0 120,26.96 119.946,33.912 0,34.01 "/>
+</svg>
--- a/examples/openspending/vitest.config.ts
+++ b/examples/openspending/vitest.config.ts
@@ -0,0 +1,10 @@
+import { defineConfig } from 'vitest/config'
+import react from '@vitejs/plugin-react'
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  plugins: [react()],
+  test: {
+    environment: 'jsdom',
+  },
+})
--- a/site/content/assets/blog/markdowndb.gif
+++ b/site/content/assets/blog/markdowndb.gif
--- a/site/content/assets/blog/output.png
+++ b/site/content/assets/blog/output.png
--- a/site/content/assets/blog/sqlite-viewer.png
+++ b/site/content/assets/blog/sqlite-viewer.png
--- a/site/content/blog/markdowndb-basics-tutorial-2023.md
+++ b/site/content/blog/markdowndb-basics-tutorial-2023.md
@@ -0,0 +1,270 @@
+---
+title: Learn how to use MarkdownDB with our First Tutorial!
+description: We've just released our first tutorial that covers the fundamentals of MarkdownDB - our new package for treating markdown files as a database. If you've been curious about how to manage your markdown files more effectively, check it out!
+date: 2023-05-26
+author: Ola Rubaj
+---
+
+We've just released our first tutorial that covers the fundamentals of [MarkdownDB](https://github.com/datopian/markdowndb) - our new package for treating markdown files as a database. If you've been curious about how to manage your markdown files more effectively, this tutorial is an excellent starting point!
+
+![Gif](/assets/blog/markdowndb.gif)
+
+## What is MarkdownDB?
+
+MarkdownDB can parse your markdown files, extract structured data (such as frontmatter, tags, back- and forward links and more), and create an index in a local SQLite database. It also provides a lightweight JavaScript API for querying the database and importing files into your application. With MarkdownDB, you have a powerful tool that allows you to efficiently query your markdown data.
+
+[🔍 Click here to learn more!](https://github.com/datopian/markdowndb)
+
+## What you're going to learn
+
+This tutorial guides you through the steps of creating a small project catalog. In our case, we used our open-source projects on GitHub - but you could use anything! We use simple markdown files to describe each project and then display them with the MarkdownDB package.
+
+## Step 1: Create a markdown file for each project
+
+First, let's create a markdown file for each project. You can use real project details or make up some examples. For the sake of this tutorial, we'll create files for some of the projects we've built at Datopian.
+
+```bash
+mkdir projects
+cd projects
+touch markdowndb.md portaljs.md flowershow.md datapipes.md giftless.md data-cli.md
+```
+
+In each file we'll write some short info about a given project, like so:
+
+```md
+# What is MarkdownDB
+
+MarkdownDB is a javascript library for treating markdown files as a database -- as a "MarkdownDB". Specifically, it:
+
+- Parses your markdown files to extract structured data (frontmatter, tags, etc) and creates an index in a local SQLite database
+- Provides a lightweight javascript API for querying the database and importing files into your application
+```
+
+## Step 2: Index markdown files into SQLite database
+
+Once we have prepared our markdown files, we can store them (or more precisely - their metadata) in a database, so that we can then query it later for specific project files.
+
+```bash
+# npx @flowershow/markdowndb <path-to-folder-with-md-files>
+npx @flowershow/markdowndb ./projects
+```
+
+The above command will output a `markdown.db` file in the directory where it was executed. So, in our case, the folder structure will look like this:
+
+```
+.
+├── markdown.db
+└── projects
+    ├── data-cli.md
+    ├── ...
+    └── portaljs.md
+```
+
+## Step 3: Explore the SQLite database
+
+Now, let's explore the database. We can do it with any SQLite viewer, e.g. https://sqlitebrowser.org/. When we open the `markdown.db` file in the viewer, we'll see a list of tables:
+
+- `files`: containing metadata of our markdown,
+- `file_tags`: containing tags set in the frontmatter of our markdown files,
+- `links`: containing wiki links, i.e. links between our markdown files,
+- `tags`: containing all tags used in our markdown files.
+
+In our case, the `files` table will look like this:
+
+![[/assets/blog/sqlite-viewer.png]]
+
+You can also explore the database from the command line, e.g.:
+
+```bash
+sqlite3 markdown.db
+```
+
+And then run SQL queries, e.g.:
+
+```sql
+SELECT * FROM files;
+```
+
+Which will output:
+
+```bash
+27ce406aac24e59af7a9f3c0c0a437c1d024152b|projects/data-cli.md|md|data-cli||{}
+26b1b0b06a4f450646f9e22fc18ec069bf577d8c|projects/datapipes.md|md|datapipes||{}
+dafdd0daf71a1b06db1988c57848dc36947375f4|projects/flowershow.md|md|flowershow||{}
+32c8db33fb8758516bfefb6ab1f22d03b1e53a08|projects/giftless.md|md|giftless||{}
+7e01cae193f12f5a4a9be2b89f22b429761bd313|projects/markdowndb.md|md|markdowndb||{}
+5445349c6822704d6f531a83c2aca2e4f90de864|projects/portaljs.md|md|portaljs||{}
+```
+
+## Step 4: Query the database in the Node.js app
+
+Now, let's write a simple script that will query the database for our projects and display them on the terminal.
+
+First, let's create a new Node.js project:
+
+```bash
+mkdir projects-list
+cd projects-list
+npm init -y
+```
+
+Then, let's install the `@flowershow/markdowndb` package:
+
+```bash
+npm install @flowershow/markdowndb
+```
+
+Now, let's create a new file `index.js` and add the following code:
+
+```js
+import { MarkdownDB } from '@flowershow/markdowndb';
+
+// change this to the path to your markdown.db file
+const dbPath = 'markdown.db';
+
+const client = new MarkdownDB({
+  client: 'sqlite3',
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const mddb = await client.init();
+
+// get all projects
+const projects = await mddb.getFiles();
+
+console.log(JSON.stringify(projects, null, 2));
+
+process.exit(0);
+```
+
+Since we're using ES6 modules, we also need to add `"type": "module"` to our `package.json` file.
+
+Before we run the above script, we need to make sure that the `dbPath` variable is pointing to our `markdown.db` file. If you want to store the database outside of your project folder, you can update the `dbPath` variable to point to the correct location. If you want to have it inside your project folder, you can copy it there, or simply re-run the `npx @flowershow/markdowndb <path-to-markdown-folder>` command from within your project folder.
+
+Now, let's run the script:
+
+```bash
+node index.js
+```
+
+It should output the JSON with all our projects.
+
+```json
+[
+  {
+    "_id": "7e01cae193f12f5a4a9be2b89f22b429761bd313",
+    "file_path": "projects/markdowndb.md",
+    "extension": "md",
+    "url_path": "markdowndb",
+    "filetype": null,
+    "metadata": {}
+  },
+  ...
+]
+```
+
+## Step 5: Add metadata to project files
+
+Now, let's add some metadata to our project files. We'll use frontmatter for that. Since we're creating a catalog of our GitHub projects, we'll add the following frontmatter fields to each file:
+
+```md
+---
+name: markdowndb
+description: Javascript library for treating markdown files as a database.
+stars: 6
+forks: 0
+---
+```
+
+After adding the metadata, we need to re-index our markdown files into the database:
+
+```bash
+npx @flowershow/markdowndb ../projects
+```
+
+Now, if we run our script again, we'll see that the `metadata` field in the output contains the metadata we've added to our project files:
+
+```json
+[
+  {
+    "_id": "7e01cae193f12f5a4a9be2b89f22b429761bd313",
+    "file_path": "projects/markdowndb.md",
+    "extension": "md",
+    "url_path": "markdowndb",
+    "metadata": {
+      "name": "markdowndb",
+      "description": "Javascript library for treating markdown files as a database",
+      "stars": 6,
+      "forks": 0
+    }
+  },
+  ...
+]
+```
+
+## Step 6: Pretty print the output
+
+Now, let's make the output a bit more readable. We'll use the `columnify` package for that:
+
+```bash
+npm install columnify
+```
+
+And then we'll update our `index.js` file:
+
+```js {2,16-38}
+import { MarkdownDB } from '@flowershow/markdowndb';
+import columnify from 'columnify';
+
+const dbPath = 'markdown.db';
+
+const client = new MarkdownDB({
+  client: 'sqlite3',
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const mddb = await client.init();
+const projects = await mddb.getFiles();
+
+// console.log(JSON.stringify(projects, null, 2));
+
+const projects2 = projects.map((project) => {
+  const { file_path, metadata } = project;
+  return {
+    file_path,
+    ...metadata,
+  };
+});
+
+const columns = columnify(projects2, {
+  truncate: true,
+  columnSplitter: ' | ',
+  config: {
+    description: {
+      maxWidth: 80,
+    },
+  },
+});
+
+console.log('\n');
+console.log(columns);
+console.log('\n');
+
+process.exit(0);
+```
+
+The above script will output the following to the terminal:
+
+![[/assets/blog/output.png]]
+
+## Done!
+
+That's it! We've just created a simple catalog of our GitHub projects using markdown files and the MarkdownDB package. You can find the full code for this tutorial [here](https://github.com/datopian/markdowndb/tree/main/examples/basic-example).
+
+We look forward to seeing the amazing applications you'll build with this tool!
+
+Happy coding!
--- a/site/content/config.js
+++ b/site/content/config.js
@@ -20,7 +20,9 @@ const config = {
    // { name: "Components", href: "/docs/components" },
    { name: "Blog", href: "/blog" },
    { name: "Showcases", href: "/#showcases" },
+    { name: "Howtos", href: "/howto" },
    { name: "Examples", href: "https://github.com/datopian/portaljs/tree/main/examples", target: "_blank" },
+    { name: "Components", href: "https://storybook.portaljs.org", target: "_blank" },
    // { name: "DL Demo", href: "/data-literate/demo" },
    // { name: "Excel Viewer", href: "/excel-viewer" },
  ],
Author	SHA1	Message	Date
olayway	adcafc7d6a	add whole tutorial content	2023-05-26 15:26:07 +02:00
olayway	e563782dc4	dsf	2023-05-26 12:29:41 +02:00
olayway	730e160951	[site/content/blog][s]: blob post about basic tutorial on markdowndb	2023-05-26 12:24:56 +02:00
olayway	418b4bfe52	[content/config.js][xs]: add howtos to navlinks	2023-05-19 19:03:35 +02:00
Luccas Mateus de Medeiros Gomes	ee6efc7431	[site][xs] - add components tab in navbar	2023-05-19 07:04:16 -03:00
Ola Rubaj	a62addbfbb	Dev how-to pages (#890 ) Related to #878 ## Changes How-to docs pages: - /howto/index: home page for guides - /howto/analytics: "How to add Google Analytics?" - /howto/seo: "How to customize page metadata for SEO?" - /howto/sitemap: "How to build a sitemap?" - /howto/blog: "How to add a simple blog?" - blog index page - blog layout - comments -> link to the page below - /howto/comments: "How to add user comments?" - /howto/markdown: "How to add markdown-based content pages?" - /howto/drd: "How to create data-rich documents with charts and tables?"	2023-05-19 11:15:52 +02:00
João Demenech	adb6d1bb0e	Implement dataset page v0.1 + Home improvements (#892 ) * [examples/openspending][xl]: implement dataset page v0.1, add pagination to the datasets grid * [examples/openspending][m] - fix build + add tests --------- Co-authored-by: Luccas Mateus de Medeiros Gomes <luccasmmg@gmail.com>	2023-05-18 21:19:01 -03:00
Luccas Mateus	2115a3fdb3	[examples/openspending][m] - table preview (#891 ) * [examples/openspending][m] - table preview * [examples/openspending][xs] - fix typo	2023-05-18 15:09:57 -03:00
João Demenech	efd8c85926	Load datasets + Datasets grid + Datasets search (#889 ) * [examples/openspending][m] - added loader + fetching from datapackage - Also added an indexing example * [examples/openspending,home][xl]: removes datasets table, implement dataset cards grid, implement country facet * [examples/openspending,home][m]: add min date and max date facets --------- Co-authored-by: Luccas Mateus de Medeiros Gomes <luccasmmg@gmail.com>	2023-05-18 07:21:30 -03:00