Implement dataset page v0.1 + Home improvements (#892)

* [examples/openspending][xl]: implement dataset page v0.1, add pagination to the datasets grid * [examples/openspending][m] - fix build + add tests --------- Co-authored-by: Luccas Mateus de Medeiros Gomes <luccasmmg@gmail.com>
2023-05-18 21:19:01 -03:00 · 2023-05-18 21:19:01 -03:00 · adb6d1bb0e
commit adb6d1bb0e
parent 2115a3fdb3
19 changed files with 646 additions and 344 deletions
--- a/examples/openspending/tests/os-data.test.ts
+++ b/examples/openspending/tests/os-data.test.ts
@ -1,45 +1,138 @@
-import { Octokit } from 'octokit';
-import { assert, expect, test } from 'vitest'
-import { getProjectDataPackage } from '../lib/octokit';
+import { expect, test } from 'vitest';
+import { getAllProjectsFromOrg, getProjectDataPackage } from '../lib/project';
+import { loadDataPackage } from '../lib/loader';
+import { getProjectMetadata } from '../lib/project';
+import { getCsv, parseCsv } from '../components/Table';

-export async function getAllDataPackagesFromOrg(
-  org: string,
-  branch?: string,
-  github_pat?: string
-) {
-  const octokit = new Octokit({ auth: github_pat });
-  const repos = await octokit.rest.repos.listForOrg({ org, type: 'public', per_page: 100 });
-  let failedDataPackages = [];
-  const datapackages = await Promise.all(
-    repos.data.map(async (_repo) => {
-      const datapackage = await getProjectDataPackage(
-        org,
-        _repo.name,
-        branch ? branch : 'main',
-        github_pat
-      );
-      if (!datapackage) {
-        failedDataPackages.push(_repo.name)
-        return null
-      };
-      return {...datapackage, repo: _repo.name};
-    })
-  );
-  return {
-    datapackages: datapackages.filter((item) => item !== null),
-    failedDataPackages,
-  };
-}
+test(
+  'Test OS-Data',
+  async () => {
+    const repos = await getAllProjectsFromOrg(
+      'os-data',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    if (repos.failed.length > 0) console.log(repos.failed);
+    expect(repos.failed.length).toBe(0);
+  },
+  { timeout: 100000 }
+);

-test('Test OS-Data', async () => {
-  const repos = await getAllDataPackagesFromOrg('os-data', 'main', process.env.VITE_GITHUB_PAT)
-  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
-  expect(repos.failedDataPackages.length).toBe(0)
-}, {timeout: 100000})
+test(
+  'Test Gift-Data',
+  async () => {
+    const repos = await getAllProjectsFromOrg(
+      'gift-data',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    if (repos.failed.length > 0) console.log(repos.failed);
+    expect(repos.failed.length).toBe(0);
+  },
+  { timeout: 100000 }
+);

-test('Test Gift-Data', async () => {
-  const repos = await getAllDataPackagesFromOrg('gift-data', 'main', process.env.VITE_GITHUB_PAT)
-  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
-  expect(repos.failedDataPackages.length).toBe(0)
-}, {timeout: 100000})
+test(
+  'Test getting one dataset from github',
+  async () => {
+    const datapackage = await getProjectDataPackage(
+      'os-data',
+      'berlin-berlin',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    const repo = await getProjectMetadata(
+      'os-data',
+      'berlin-berlin',
+      process.env.VITE_GITHUB_PAT
+    );
+    const project = loadDataPackage(datapackage, repo);
+    delete project['datapackage'];
+    delete project.files[0]['dialect'];
+    delete project.files[0]['schema'];
+    expect(project).toStrictEqual({
+      name: 'berlin-berlin',
+      title: 'Berlin-Berlin',
+      description: null,
+      owner: {
+        name: 'os-data',
+        logo: 'https://avatars.githubusercontent.com/u/13695166?v=4',
+        title: 'os-data',
+      },
+      repo: {
+        name: 'berlin-berlin',
+        full_name: 'os-data/berlin-berlin',
+        url: 'https://github.com/os-data/berlin-berlin',
+      },
+      files: [
+        {
+          name: 'berlin-gesamt',
+          format: 'csv',
+          path: 'https://storage.openspending.org/berlin-berlin/berlin-gesamt.csv',
+          mediatype: 'text/csv',
+          bytes: 81128743,
+          encoding: 'utf-8',
+        },
+      ],
+      author: 'Michael Peters <michael.peters@okfn.de>',
+      cityCode: 'Berlin',
+      countryCode: 'DE',
+      fiscalPeriod: { start: '2014-01-01', end: '2019-12-31' },
+      readme: '',
+    });
+  },
+  { timeout: 100000 }
+);

+test(
+  'Test getting one section of csv from R2',
+  async () => {
+    const rawCsv = await getCsv(
+      'https://storage.openspending.org/state-of-minas-gerais-brazil-planned-budget/__os_imported__br-mg-ppagloc.csv'
+    );
+    const parsedCsv = await parseCsv(rawCsv);
+    expect(parsedCsv.errors.length).toBe(1);
+    expect(parsedCsv.data.length).toBe(10165);
+    expect(parsedCsv.meta.fields).toStrictEqual([
+      'function_name',
+      'function_label',
+      'product_name',
+      'product_label',
+      'area_name',
+      'area_label',
+      'subaction_name',
+      'subaction_label',
+      'region_label_map',
+      'region_reg_map',
+      'region_name',
+      'region_label',
+      'municipality_map_id',
+      'municipality_name',
+      'municipality_map_code',
+      'municipality_label',
+      'municipality_map_name_simple',
+      'municipality_map_name',
+      'cofog1_label_en',
+      'cofog1_name',
+      'cofog1_label',
+      'amount',
+      'subprogramme_name',
+      'subprogramme_label',
+      'time_name',
+      'time_year',
+      'time_month',
+      'time_day',
+      'time_week',
+      'time_yearmonth',
+      'time_quarter',
+      'time',
+      'action_name',
+      'action_label',
+      'subfunction_name',
+      'subfunction_label',
+      'programme_name',
+      'programme_label',
+    ]);
+  },
+  { timeout: 100000 }
+);
--- a/examples/openspending/components/DatasetCard.tsx
+++ b/examples/openspending/components/DatasetCard.tsx
@ -9,7 +9,7 @@ export default function DatasetCard({ dataset }: { dataset: Project }) {
      className="overflow-hidden rounded-xl border border-gray-200"
    >
      <Link
-        href=""
+        href={`/@${dataset.owner.name}/${dataset.repo.name}`}
        className="flex items-center gap-x-4 border-b border-gray-900/5 bg-gray-50 p-6"
      >
        <img
@ -60,8 +60,8 @@ export default function DatasetCard({ dataset }: { dataset: Project }) {
          <dd className="flex items-start gap-x-2">
            <div className="font-medium text-gray-900">
              <Link
-                // TODO: where do we get the info needed for this link?
-                href=""
+                // TODO: this link may be incorrect for some datasets
+                href={`https://github.com/${dataset.owner.name}/${dataset.repo.name}/blob/main/datapackage.json`}
                target="_blank"
                className="flex items-center hover:text-gray-700"
              >
--- a/examples/openspending/components/DatasetsSearch.tsx
+++ b/examples/openspending/components/DatasetsSearch.tsx
@ -2,9 +2,26 @@ import { useForm } from 'react-hook-form';
 import DatasetsGrid from './DatasetsGrid';
 import { Project } from '../lib/project.interface';
 import { Index } from 'flexsearch';
+import {
+  ChevronDoubleLeftIcon,
+  ChevronDoubleRightIcon,
+  ChevronLeftIcon,
+  ChevronRightIcon,
+} from '@heroicons/react/24/solid';
+import { useState } from 'react';
+
+export default function DatasetsSearch({
+  datasets,
+  availableCountries,
+}: {
+  datasets: Project[];
+  availableCountries;
+}) {
+  const itemsPerPage = 6;
+  const [page, setPage] = useState(1);

-export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
  const index = new Index({ tokenize: 'full' });
+
  datasets.forEach((dataset: Project) =>
    index.add(
      dataset.name,
@ -21,12 +38,38 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
    },
  });

-  const allCountries = datasets
-    .map((item) => item.countryCode)
-    .filter((v) => v) //  Filters false values
-    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
-    //  TODO: title should be the full name
-    .map((code) => ({ code, title: code }));
+  const filteredDatasets = datasets
+    .filter((dataset: Project) =>
+      watch().searchTerm && watch().searchTerm !== ''
+        ? index.search(watch().searchTerm).includes(dataset.name)
+        : true
+    )
+    .filter((dataset) =>
+      watch().country && watch().country !== ''
+        ? dataset.countryCode === watch().country
+        : true
+    )
+    // TODO: Does that really makes sense?
+    // What if the fiscalPeriod is 2015-2017 and inputs are
+    // set to 2015-2016. It's going to be filtered out but
+    // it shouldn't.
+    .filter((dataset) =>
+      watch().minDate && watch().minDate !== ''
+        ? dataset.fiscalPeriod?.start >= watch().minDate
+        : true
+    )
+    .filter((dataset) =>
+      watch().maxDate && watch().maxDate !== ''
+        ? dataset.fiscalPeriod?.end <= watch().maxDate
+        : true
+    );
+
+  const paginatedDatasets = filteredDatasets.slice(
+    (page - 1) * itemsPerPage,
+    (page - 1) * itemsPerPage + itemsPerPage
+  );
+
+  const pageCount = Math.ceil(filteredDatasets.length / itemsPerPage) || 1;

  return (
    <>
@ -37,7 +80,7 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
            <input
              placeholder="Search datasets"
              aria-label="Search datasets"
-              {...register('searchTerm')}
+              {...register('searchTerm', { onChange: () => setPage(1) })}
              className="h-[3em] relative w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
            />
            {watch().searchTerm !== '' && (
@ -55,10 +98,10 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
          <label className="text-sm text-gray-600 font-medium">Country</label>
          <select
            className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
-            {...register('country')}
+            {...register('country', { onChange: () => setPage(1) })}
          >
            <option value="">All</option>
-            {allCountries.map((country) => {
+            {availableCountries.map((country) => {
              return (
                <option key={country.code} value={country.code}>
                  {country.title}
@ -73,17 +116,9 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
            <input
              aria-label="Min. date"
              type="date"
-              {...register('minDate')}
+              {...register('minDate', { onChange: () => setPage(1) })}
              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
            />
-            {watch().minDate !== '' && (
-              <button
-                onClick={() => resetField('minDate')}
-                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
-              >
-                <CloseIcon />
-              </button>
-            )}
          </div>
        </div>
        <div className="sm:basis-1/6">
@ -92,48 +127,56 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
            <input
              aria-label="Max. date"
              type="date"
-              {...register('maxDate')}
+              {...register('maxDate', { onChange: () => setPage(1) })}
              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
            />
-            {watch().maxDate !== '' && (
-              <button
-                onClick={() => resetField('maxDate')}
-                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
-              >
-                <CloseIcon />
-              </button>
-            )}
          </div>
        </div>
      </div>
-      <div className="min-w-full mt-10 align-middle">
-        <DatasetsGrid
-          datasets={datasets
-            .filter((dataset: Project) =>
-              watch().searchTerm && watch().searchTerm !== ''
-                ? index.search(watch().searchTerm).includes(dataset.name)
-                : true
-            )
-            .filter((dataset) =>
-              watch().country && watch().country !== ''
-                ? dataset.countryCode === watch().country
-                : true
-            )
-            // TODO: Does that really makes sense?
-            // What if the fiscalPeriod is 2015-2017 and inputs are
-            // set to 2015-2016. It's going to be filtered out but
-            // it shouldn't.
-            .filter((dataset) =>
-              watch().minDate && watch().minDate !== ''
-                ? dataset.fiscalPeriod?.start >= watch().minDate
-                : true
-            )
-            .filter((dataset) =>
-              watch().maxDate && watch().maxDate !== ''
-                ? dataset.fiscalPeriod?.end <= watch().maxDate
-                : true
-            )}
-        />
+      <div className="mt-5 mb-5">
+        <span className="text-lg font-medium">
+          {filteredDatasets.length} datasets found
+        </span>
+      </div>
+      <div className="min-w-full align-middle">
+        <DatasetsGrid datasets={paginatedDatasets} />
+        <div className="w-full flex justify-center mt-10">
+          <button
+            onClick={() => setPage(1)}
+            disabled={page <= 1}
+            className="disabled:text-gray-400"
+          >
+            <ChevronDoubleLeftIcon className="w-6 h-6" />
+          </button>
+          <button
+            onClick={() => {
+              if (page > 1) setPage((prev) => --prev);
+            }}
+            disabled={page <= 1}
+            className="disabled:text-gray-400"
+          >
+            <ChevronLeftIcon className="w-6 h-6" />
+          </button>
+          <span className="mx-5">
+            Page {page} of {pageCount}
+          </span>
+          <button
+            onClick={() => {
+              if (page < pageCount) setPage((prev) => ++prev);
+            }}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronRightIcon className="w-6 h-6" />
+          </button>
+          <button
+            onClick={() => setPage(pageCount)}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronDoubleRightIcon className="w-6 h-6" />
+          </button>
+        </div>
      </div>
    </>
  );
--- a/examples/openspending/components/Header.tsx
+++ b/examples/openspending/components/Header.tsx
@ -1,53 +1,82 @@
-import Image from 'next/image'
-import { Button } from './Button'
-import { Container } from './Container'
-import logo from "../public/logo.svg"
-import Link from 'next/link'
-import { useRouter } from 'next/router'
+import Image from 'next/image';
+import { Container } from './Container';
+import logo from '../public/logo.svg';
+import Link from 'next/link';
+import { useRouter } from 'next/router';
+import { Bars3Icon } from '@heroicons/react/24/outline';
+import { useState } from 'react';

 export function Header() {
+  const [menuOpen, setMenuOpen] = useState<boolean>(false);
+
  const router = useRouter();

  const isActive = (navLink) => {
-    return router.asPath.split("?")[0] == navLink.href;
-  }
+    return router.asPath.split('?')[0] == navLink.href;
+  };

  const navLinks = [
    {
-      title: "Home",
-      href: "/#header"
+      title: 'Home',
+      href: '/',
    },
    {
-      title: "Datasets",
-      href: "/#datasets"
+      title: 'Datasets',
+      href: '/#datasets',
    },
-    {
-      title: "Community",
-      href: "https://community.openspending.org/"
-    }
-  ]
+    // {
+    //   title: "Community",
+    //   href: "https://community.openspending.org/"
+    // }
+  ];

  return (
-    <header className="z-50 pb-5 lg:pt-11 sticky top-0 backdrop-blur" id="header">
-      <Container className="flex flex-wrap items-center justify-center sm:justify-between lg:flex-nowrap">
-        <div className="mt-10 lg:mt-0 lg:grow lg:basis-0 flex items-center">
+    <header className="relative z-50 pb-11 lg:pt-11">
+      <Container className="flex flex-wrap items-center justify-between lg:flex-nowrap mt-10 lg:mt-0">
+        <Link href="/" className="lg:mt-0 lg:grow lg:basis-0 flex items-center">
          <Image src={logo} alt="OpenSpending" className="h-12 w-auto" />
-        </div>
-        <ul className='list-none flex gap-x-5 text-base font-medium'>
+        </Link>
+        <ul className="hidden list-none sm:flex gap-x-5 text-base font-medium">
          {navLinks.map((link, i) => (
            <li key={`nav-link-${i}`}>
              <Link
-                className={`text-emerald-900 hover:text-emerald-600 ${isActive(link) ? "text-emerald-600" : ""}`}
+                className={`text-emerald-900 hover:text-emerald-600 ${
+                  isActive(link) ? 'text-emerald-600' : ''
+                }`}
                href={link.href}
                scroll={false}
              >
                {link.title}
              </Link>
-            </li>))}
+            </li>
+          ))}
        </ul>
-        <div className="hidden sm:mt-10 sm:flex lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
+        <div className="hidden xl:block xl:grow"></div>
+        <div className="sm:hidden sm:mt-10 lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
+          <button onClick={() => setMenuOpen(!menuOpen)}>
+            <Bars3Icon className="w-8 h-8" />
+          </button>
        </div>
+        {menuOpen && (
+          <div className={`sm:hidden basis-full mt-5 text-center`}>
+            <ul className="gap-x-5 text-base font-medium">
+              {navLinks.map((link, i) => (
+                <li key={`nav-link-${i}`}>
+                  <Link
+                    className={`text-emerald-900 hover:text-emerald-600 ${
+                      isActive(link) ? 'text-emerald-600' : ''
+                    }`}
+                    href={link.href}
+                    scroll={false}
+                  >
+                    {link.title}
+                  </Link>
+                </li>
+              ))}
+            </ul>
+          </div>
+        )}
      </Container>
-    </header >
-  )
+    </header>
+  );
 }
--- a/examples/openspending/components/Hero.tsx
+++ b/examples/openspending/components/Hero.tsx
@ -1,9 +1,9 @@
-import { Button } from './Button'
-import { Container } from './Container'
+import { Button } from './Button';
+import { Container } from './Container';

-export function Hero() {
+export function Hero({ countriesCount, datasetsCount, filesCount }) {
  return (
-    <div className="relative pb-20 pt-10 sm:py-40">
+    <div className="relative pb-20 pt-10 sm:py-40" id="hero">
      <div className="absolute inset-x-0 -bottom-14 -top-48 overflow-hidden bg-green-50 bg-opacity-50">
        <div className="absolute inset-x-0 top-0 h-40 bg-gradient-to-b from-white" />
        <div className="absolute inset-x-0 bottom-0 h-40 bg-gradient-to-t from-white" />
@ -15,12 +15,13 @@ export function Hero() {
          </h1>
          <div className="mt-6 space-y-6 font-display text-2xl tracking-tight text-emerald-900">
            <p>
-              By understanding how governments spend money in our name can we have a say
-              in how that money will affect our own lives. The journey starts here.
+              By understanding how governments spend money in our name can we
+              have a say in how that money will affect our own lives. The
+              journey starts here.
            </p>
            <p>
-              OpenSpending is a free, open and global platform to search, visualise and analyse
-              fiscal data in the public sphere.
+              OpenSpending is a free, open and global platform to search,
+              visualise and analyse fiscal data in the public sphere.
            </p>
          </div>
          <Button href="#datasets" className="mt-10">
@ -28,9 +29,11 @@ export function Hero() {
          </Button>
          <dl className="mt-10 grid grid-cols-2 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
            {[
-              ['Countries', '75'],
-              ['Datasets', '2091'],
-              ['Files', '9230'],
+              //  Added the plus sign because some datasets do not
+              //  contain defined countries
+              ['Countries', '+' + countriesCount],
+              ['Datasets', datasetsCount],
+              ['Files', filesCount],
            ].map(([name, value]) => (
              <div key={name}>
                <dt className="font-mono text-sm text-emerald-600">{name}</dt>
@ -43,5 +46,5 @@ export function Hero() {
        </div>
      </Container>
    </div>
-  )
+  );
 }
--- a/examples/openspending/components/Table.tsx
+++ b/examples/openspending/components/Table.tsx
@ -9,7 +9,7 @@ import { Grid } from '@githubocto/flat-ui';

 const queryClient = new QueryClient();

-async function getCsv(url: string) {
+export async function getCsv(url: string) {
  const response = await fetch(url, {
    headers: {
      Range: 'bytes=0-5132288',
--- a/examples/openspending/components/_shared/Layout.tsx
+++ b/examples/openspending/components/_shared/Layout.tsx
@ -0,0 +1,10 @@
+import { Header } from '../Header';
+
+export default function Layout({ children }) {
+  return (
+    <div className="bg-white min-h-screen">
+      <Header />
+      {children}
+    </div>
+  );
+}
--- a/examples/openspending/datasets.json
+++ b/examples/openspending/datasets.json
@ -1,27 +0,0 @@
-[
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "mongolia-budget-2016-2017"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "gb-country-regional-analysis"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "berlin-berlin"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "state-of-minas-gerais-brazil-planned-budget"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "wesel"
-  }
-]
--- a/examples/openspending/lib/datapackage.interface.ts
+++ b/examples/openspending/lib/datapackage.interface.ts
@ -98,6 +98,7 @@ export interface TabularDataResource {
  key?: string;
  path?: string;
  size?: number;
+  bytes?: number;
 }

 export interface Field {
--- a/examples/openspending/lib/loader.ts
+++ b/examples/openspending/lib/loader.ts
@ -5,13 +5,14 @@ export function loadDataPackage(datapackage: FiscalDataPackage, repo): Project {
  return {
    name: datapackage.name,
    title: datapackage.title,
+    description: datapackage.description || null,
    owner: {
      name: repo.owner.login,
      logo: repo.owner.avatar_url,
      // TODO: make this title work
      title: repo.owner.login,
    },
-    repo: { name: repo, full_name: repo.full_name },
+    repo: { name: repo.name, full_name: repo.full_name, url: repo.html_url },
    files: datapackage.resources,
    author: datapackage.author ? datapackage.author : null,
    cityCode: datapackage.cityCode ? datapackage.cityCode : null,
--- a/examples/openspending/lib/project.interface.ts
+++ b/examples/openspending/lib/project.interface.ts
@ -5,10 +5,11 @@ import {

 export interface Project {
  owner: { name: string; logo?: string; title?: string }; // Info about the owner of the data repo
-  repo: { name: string; full_name: string }; // Info about the the data repo
+  repo: { name: string; full_name: string; url: string }; // Info about the the data repo
  files: TabularDataResource[];
  name: string;
  title?: string;
+  description?: string;
  author?: string;
  cityCode?: string;
  countryCode?: string;
--- a/examples/openspending/lib/project.ts
+++ b/examples/openspending/lib/project.ts
@ -13,8 +13,7 @@ export interface GithubProject {
 export async function getProjectReadme(
  owner: string,
  repo: string,
-  branch: string,
-  readme: string,
+  branch: string = 'main',
  github_pat?: string
 ) {
  const octokit = new Octokit({ auth: github_pat });
@ -22,7 +21,7 @@ export async function getProjectReadme(
    const response = await octokit.rest.repos.getContent({
      owner,
      repo,
-      path: readme,
+      path: 'README.md',
      ref: branch,
    });
    const data = response.data as { content?: string };
@ -125,7 +124,6 @@ export async function getProject(project: GithubProject, github_pat?: string) {
    project.owner,
    project.repo,
    project.branch,
-    project.readme,
    github_pat
  );

@ -185,8 +183,43 @@ export async function getProjectDataPackage(
    }
    const decodedContent = Buffer.from(fileContent, 'base64').toString();
    const datapackage = JSON.parse(decodedContent);
-    return {...datapackage, repo };
+
+    return { ...datapackage, repo };
  } catch (error) {
    return null;
  }
 }
+
+export async function getAllProjectsFromOrg(
+  org: string,
+  branch?: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  const repos = await octokit.rest.repos.listForOrg({
+    org,
+    type: 'public',
+    per_page: 100,
+  });
+  let failedProjects = [];
+  const projects = await Promise.all(
+    repos.data.map(async (_repo) => {
+      const project = await getProjectDataPackage(
+        org,
+        _repo.name,
+        branch ? branch : 'main',
+        github_pat
+      );
+      if (!project) {
+        failedProjects.push(_repo.name);
+        return null;
+      }
+
+      return { datapackage: project, repo: _repo };
+    })
+  );
+  return {
+    results: projects.filter((item) => item !== null),
+    failed: failedProjects,
+  };
+}
--- a/examples/openspending/orgs.json
+++ b/examples/openspending/orgs.json
@ -0,0 +1,3 @@
+[
+ "os-data"
+]
--- a/examples/openspending/package-lock.json
+++ b/examples/openspending/package-lock.json
@ -9,6 +9,7 @@
      "version": "0.1.0",
      "dependencies": {
        "@githubocto/flat-ui": "^0.14.1",
+        "@heroicons/react": "^2.0.18",
        "@octokit/plugin-throttling": "^5.2.2",
        "@types/flexsearch": "^0.7.3",
        "@types/node": "18.16.0",
@ -1622,6 +1623,14 @@
        "object-assign": "^4.1.1"
      }
    },
+    "node_modules/@heroicons/react": {
+      "version": "2.0.18",
+      "resolved": "https://registry.npmjs.org/@heroicons/react/-/react-2.0.18.tgz",
+      "integrity": "sha512-7TyMjRrZZMBPa+/5Y8lN0iyvUU/01PeMGX2+RE7cQWpEUIcb4QotzUObFkJDejj/HUH4qjP/eQ0gzzKs2f+6Yw==",
+      "peerDependencies": {
+        "react": ">= 16"
+      }
+    },
    "node_modules/@humanwhocodes/config-array": {
      "version": "0.11.8",
      "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz",
--- a/examples/openspending/package.json
+++ b/examples/openspending/package.json
@ -11,6 +11,7 @@
  },
  "dependencies": {
    "@githubocto/flat-ui": "^0.14.1",
+    "@heroicons/react": "^2.0.18",
    "@octokit/plugin-throttling": "^5.2.2",
    "@types/flexsearch": "^0.7.3",
    "@types/node": "18.16.0",
--- a/examples/openspending/pages/@org/[org]/[...path].tsx
+++ b/examples/openspending/pages/@org/[org]/[...path].tsx
@ -1,126 +0,0 @@
-import { NextSeo } from 'next-seo';
-import { promises as fs } from 'fs';
-import path from 'path';
-import getConfig from 'next/config';
-import { getProject, GithubProject } from '../../../lib/octokit';
-import ReactMarkdown from 'react-markdown';
-import remarkGfm from 'remark-gfm';
-import Breadcrumbs from '../../../components/_shared/Breadcrumbs';
-
-export default function ProjectPage({ project }) {
-  const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`
-
-  return (
-    <>
-      <NextSeo title={`${repoId}${project.base_path !== '/' ? '/' + project.base_path : ''} - GitHub Datasets`} />
-      <main className="prose mx-auto my-8">
-        <Breadcrumbs links={[{ title: repoId, href: "" }]} />
-        <h1 className="mb-0 mt-16">{project.repo_config.name || repoId}</h1>
-        <p className='mb-8'><span className='font-semibold'>Repository:</span> <a target="_blank" href={project.html_url}>{project.html_url}</a></p>
-
-        <h2 className="mb-0 mt-10">Files</h2>
-        <div className="inline-block min-w-full py-2 align-middle">
-          <table className="min-w-full divide-y divide-gray-300">
-            <thead>
-              <tr>
-                <th
-                  scope="col"
-                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                >
-                  Name
-                </th>
-                <th
-                  scope="col"
-                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                >
-                  Size
-                </th>
-              </tr>
-            </thead>
-            <tbody className="divide-y divide-gray-200">
-              {project.files?.map((file) => (
-                <tr key={file.download_url}>
-                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                    <a href={file.download_url}>{file.name}</a>
-                  </td>
-                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                    {file.size} Bytes
-                  </td>
-                </tr>
-              ))}
-            </tbody>
-          </table>
-        </div>
-
-        {project.readmeContent && <>
-          <hr />
-
-          <h2 className='uppercase font-black'>Readme</h2>
-          <ReactMarkdown remarkPlugins={[remarkGfm]}>
-            {project.readmeContent}
-          </ReactMarkdown>
-        </>}
-      </main>
-    </>
-  );
-}
-
-// Generates `/posts/1` and `/posts/2`
-export async function getStaticPaths() {
-  const jsonDirectory = path.join(
-    process.cwd(),
-    'datasets.json'
-  );
-  const repos = await fs.readFile(jsonDirectory, 'utf8');
-
-  return {
-    paths: JSON.parse(repos).map((repo) => {
-      const projectPath =
-        repo.readme && repo.readme.split('/').length > 1
-          ? repo.readme.split('/').slice(0, -1)
-          : null;
-      let path = [repo.name];
-      if (projectPath) {
-        projectPath.forEach((element) => {
-          path.push(element);
-        });
-      }
-      return {
-        params: { org: repo.owner, path },
-      };
-    }),
-    fallback: false, // can also be true or 'blocking'
-  };
-}
-
-export async function getStaticProps({ params }) {
-  const jsonDirectory = path.join(
-    process.cwd(),
-    'datasets.json'
-  );
-  const reposFile = await fs.readFile(jsonDirectory, 'utf8');
-  const repos: GithubProject[] = JSON.parse(reposFile);
-  const repo = repos.find((_repo) => {
-    const projectPath =
-      _repo.readme && _repo.readme.split('/').length > 1
-        ? _repo.readme.split('/').slice(0, -1)
-        : null;
-    let path = [_repo.name];
-    if (projectPath) {
-      projectPath.forEach((element) => {
-        path.push(element);
-      });
-    }
-    return (
-      _repo.owner == params.org &&
-      JSON.stringify(path) === JSON.stringify(params.path)
-    );
-  });
-  const github_pat = getConfig().serverRuntimeConfig.github_pat;
-  const project = await getProject(repo, github_pat);
-  return {
-    props: {
-      project: { ...project, repo_config: repo },
-    },
-  };
-}
--- a/examples/openspending/pages/@org/[org]/[project].tsx
+++ b/examples/openspending/pages/@org/[org]/[project].tsx
@ -0,0 +1,234 @@
+import { NextSeo } from 'next-seo';
+import getConfig from 'next/config';
+import {
+  getAllProjectsFromOrg,
+  getProjectDataPackage,
+  getProjectMetadata,
+  getProjectReadme,
+} from '../../../lib/project';
+import ReactMarkdown from 'react-markdown';
+import remarkGfm from 'remark-gfm';
+import { loadDataPackage } from '../../../lib/loader';
+import Layout from '../../../components/_shared/Layout';
+import Link from 'next/link';
+import { Project } from '../../../lib/project.interface';
+import ExternalLinkIcon from '../../../components/icons/ExternalLinkIcon';
+
+export default function ProjectPage({
+  project,
+  readme,
+}: {
+  project: Project;
+  readme: string;
+}) {
+
+  //  Get description from datapackage or  calculate
+  //  excerpt from README by getting all the content
+  //  up to the first dot.
+  const description =
+    project.description || (readme && readme.slice(0, readme.indexOf('.') + 1));
+
+  return (
+    <Layout>
+      <NextSeo title={`${project.title} - OpenSpending`} />
+      <main className="prose mx-auto my-8">
+        <h1 className="mb-1 mt-16">{project.title || project.name}</h1>
+        <Link target="_blank" href={project.repo.url}>
+          @{project.repo.full_name}
+        </Link>
+
+        {description && (
+          <div className="inline-block min-w-full py-2 align-middle mt-5">
+            {description}
+          </div>
+        )}
+
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="min-w-full divide-y divide-gray-300">
+            <thead>
+              <tr>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Name
+                </th>
+                {project.datapackage.countryCode && (
+                  <th
+                    scope="col"
+                    className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                  >
+                    Country
+                  </th>
+                )}
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Metadata
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              <tr>
+                <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                  {project.name}
+                </td>
+                {project.datapackage.countryCode && (
+                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                    {project.datapackage.countryCode}
+                  </td>
+                )}
+
+                <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                  <Link
+                    // TODO: this link may be incorrect for some datasets
+                    href={`https://github.com/${project.owner.name}/${project.repo.name}/blob/main/datapackage.json`}
+                    target="_blank"
+                    className="flex items-center hover:text-gray-700"
+                  >
+                    datapackage.json <ExternalLinkIcon className="ml-1" />
+                  </Link>
+                </td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+
+        <h3 className="mb-1 mt-10">Data files</h3>
+        <p>
+          This dataset contains {project.files.length} file
+          {project.files.length != 1 ? '' : 's'}
+        </p>
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="mt-0 min-w-full divide-y divide-gray-300">
+            <thead>
+              <tr>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Name
+                </th>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Format
+                </th>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Size
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              {project.files?.map((file) => {
+                let size: number | string = file.size;
+
+                if (!size) {
+                  if (file.bytes) {
+                    if (file.bytes > 1000000) {
+                      size = (file.bytes / 1000000).toFixed(2) + ' MB';
+                    } else {
+                      size = (file.bytes / 1000).toFixed(2) + ' kB';
+                    }
+                  }
+                }
+
+                return (
+                  <tr key={file.name}>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {file.name}
+                    </td>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {file.format}
+                    </td>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {size}
+                    </td>
+
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      <Link
+                        target="_blank"
+                        href={
+                          file.path.startsWith('http')
+                            ? file.path
+                            : `https://raw.githubusercontent.com/${project.owner.name}/${project.repo.name}/main/${file.path}`
+                        }
+                      >
+                        Download
+                      </Link>
+                    </td>
+                  </tr>
+                );
+              })}
+            </tbody>
+          </table>
+        </div>
+
+        {readme && (
+          <>
+            <hr />
+
+            <h2 className="uppercase font-black">Readme</h2>
+            <ReactMarkdown remarkPlugins={[remarkGfm]}>{readme}</ReactMarkdown>
+          </>
+        )}
+      </main>
+    </Layout>
+  );
+}
+
+// Generates `/posts/1` and `/posts/2`
+export async function getStaticPaths() {
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+
+  const allProjects = await getAllProjectsFromOrg(
+    'os-data',
+    'main',
+    github_pat
+  );
+
+  console.log(allProjects)
+  const paths = allProjects.results.map((project) => ({
+    params: {
+      // TODO: dynamize the org
+      org: 'os-data',
+      project: project.repo.name,
+    },
+  }));
+
+  return {
+    paths,
+    fallback: false, // can also be true or 'blocking'
+  };
+}
+
+export async function getStaticProps({ params }) {
+  const { org: orgName, project: projectName } = params;
+
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+  const datapackage = await getProjectDataPackage(
+    orgName,
+    projectName,
+    'main',
+    github_pat
+  );
+
+  const repo = await getProjectMetadata(orgName, projectName, github_pat);
+
+  const project = loadDataPackage(datapackage, repo);
+
+  // TODO: should this be moved to the loader?
+  const readme = await getProjectReadme(orgName, projectName, 'main', github_pat);
+
+  return {
+    props: {
+      project,
+      readme,
+    },
+  };
+}
--- a/examples/openspending/pages/_app.tsx
+++ b/examples/openspending/pages/_app.tsx
@ -1,13 +1,11 @@
 import { AppProps } from 'next/app';
-import Head from 'next/head';
 import './styles.css';
+import { NextSeo } from 'next-seo';

 function CustomApp({ Component, pageProps }: AppProps) {
  return (
    <>
-      <Head>
-        <title>GitHub Datasets</title>
-      </Head>
+      <NextSeo title="OpenSpending" />
      <main className="app">
        <Component {...pageProps} />
      </main>
--- a/examples/openspending/pages/index.tsx
+++ b/examples/openspending/pages/index.tsx
@ -1,65 +1,58 @@
-import { promises as fs } from 'fs';
-import path from 'path';
-import {
-  GithubProject,
-  getProjectDataPackage,
-  getProjectMetadata,
-} from '../lib/octokit';
+import { getAllProjectsFromOrg } from '../lib/project';
 import getConfig from 'next/config';
-import ExternalLinkIcon from '../components/icons/ExternalLinkIcon';
-import TimeAgo from 'react-timeago';
-import Link from 'next/link';
 import { Hero } from '../components/Hero';
-import { Header } from '../components/Header';
 import { Container } from '../components/Container';
 import { FiscalDataPackage } from '../lib/datapackage.interface';
 import { loadDataPackage } from '../lib/loader';
 import DatasetsSearch from '../components/DatasetsSearch';
+import Layout from '../components/_shared/Layout';

 export async function getStaticProps() {
-  const jsonDirectory = path.join(process.cwd(), '/datasets.json');
-  const repos = await fs.readFile(jsonDirectory, 'utf8');
-  const github_pat = getConfig().serverRuntimeConfig.github_pat;
-  const datapackages = await Promise.all(
-    JSON.parse(repos).map(async (_repo: GithubProject) => {
-      const datapackage = await getProjectDataPackage(
-        _repo.owner,
-        _repo.name,
-        'main',
-        github_pat
-      );
-      const repo = await getProjectMetadata(
-        _repo.owner,
-        _repo.name,
-        github_pat
-      );
+  //  TODO: support other orgs
+  // const orgsListPath = path.join(process.cwd(), '/orgs.json');
+  // const orgs = await fs.readFile(orgsListPath, 'utf8');

-      return {
-        datapackage,
-        repo,
-      };
-    })
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+
+  const allProjects = await getAllProjectsFromOrg(
+    'os-data',
+    'main',
+    github_pat
  );

-  const projects = datapackages.map(
+  const projects = allProjects.results.map(
    (item: { datapackage: FiscalDataPackage & { repo: string }; repo: any }) =>
      loadDataPackage(item.datapackage, item.repo)
  );

+  const availableCountries = projects
+    .map((item) => item.countryCode)
+    .filter((v) => v) //  Filters false values
+    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
+    //  TODO: title should be the full name
+    .map((code) => ({ code, title: code }));
+
  return {
    props: {
      projects: JSON.stringify(projects),
+      availableCountries,
    },
  };
 }

-export function Datasets({ projects }) {
+export function Home({ projects, availableCountries }) {
  projects = JSON.parse(projects);

  return (
-    <div className="bg-white min-h-screen">
-      <Header />
-      <Hero />
+    <Layout>
+      <Hero
+        countriesCount={availableCountries.length}
+        datasetsCount={projects.length}
+        filesCount={projects.reduce(
+          (partialSum, a) => partialSum + a.files.length,
+          0
+        )}
+      />
      <section className="py-20 sm:py-32">
        <Container>
          <div className="mx-auto max-w-2xl lg:mx-0">
@ -74,12 +67,15 @@ export function Datasets({ projects }) {
            </p>
          </div>
          <div className="mt-10">
-            <DatasetsSearch datasets={projects} />
+            <DatasetsSearch
+              datasets={projects}
+              availableCountries={availableCountries}
+            />
          </div>
        </Container>
      </section>
-    </div>
+    </Layout>
  );
 }

-export default Datasets;
+export default Home;