Implement dataset page v0.1 + Home improvements (#892)

* [examples/openspending][xl]: implement dataset page v0.1, add pagination to the datasets grid

* [examples/openspending][m] - fix build + add tests

---------

Co-authored-by: Luccas Mateus de Medeiros Gomes <luccasmmg@gmail.com>

examples/openspending/__tests__/os-data.test.ts

@@ -1,45 +1,138 @@
-import { Octokit } from 'octokit';
+import { expect, test } from 'vitest';
-import { assert, expect, test } from 'vitest'
+import { getAllProjectsFromOrg, getProjectDataPackage } from '../lib/project';
-import { getProjectDataPackage } from '../lib/octokit';
+import { loadDataPackage } from '../lib/loader';
+import { getProjectMetadata } from '../lib/project';
+import { getCsv, parseCsv } from '../components/Table';
 
-export async function getAllDataPackagesFromOrg(
+test(
-  org: string,
+  'Test OS-Data',
-  branch?: string,
+  async () => {
-  github_pat?: string
+    const repos = await getAllProjectsFromOrg(
-) {
+      'os-data',
-  const octokit = new Octokit({ auth: github_pat });
+      'main',
-  const repos = await octokit.rest.repos.listForOrg({ org, type: 'public', per_page: 100 });
+      process.env.VITE_GITHUB_PAT
-  let failedDataPackages = [];
+    );
-  const datapackages = await Promise.all(
+    if (repos.failed.length > 0) console.log(repos.failed);
-    repos.data.map(async (_repo) => {
+    expect(repos.failed.length).toBe(0);
+  },
+  { timeout: 100000 }
+);
+
+test(
+  'Test Gift-Data',
+  async () => {
+    const repos = await getAllProjectsFromOrg(
+      'gift-data',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    if (repos.failed.length > 0) console.log(repos.failed);
+    expect(repos.failed.length).toBe(0);
+  },
+  { timeout: 100000 }
+);
+
+test(
+  'Test getting one dataset from github',
+  async () => {
     const datapackage = await getProjectDataPackage(
-        org,
+      'os-data',
-        _repo.name,
+      'berlin-berlin',
-        branch ? branch : 'main',
+      'main',
-        github_pat
+      process.env.VITE_GITHUB_PAT
     );
-      if (!datapackage) {
+    const repo = await getProjectMetadata(
-        failedDataPackages.push(_repo.name)
+      'os-data',
-        return null
+      'berlin-berlin',
-      };
+      process.env.VITE_GITHUB_PAT
-      return {...datapackage, repo: _repo.name};
-    })
     );
-  return {
+    const project = loadDataPackage(datapackage, repo);
-    datapackages: datapackages.filter((item) => item !== null),
+    delete project['datapackage'];
-    failedDataPackages,
+    delete project.files[0]['dialect'];
-  };
+    delete project.files[0]['schema'];
-}
+    expect(project).toStrictEqual({
-
+      name: 'berlin-berlin',
-test('Test OS-Data', async () => {
+      title: 'Berlin-Berlin',
-  const repos = await getAllDataPackagesFromOrg('os-data', 'main', process.env.VITE_GITHUB_PAT)
+      description: null,
-  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
+      owner: {
-  expect(repos.failedDataPackages.length).toBe(0)
+        name: 'os-data',
-}, {timeout: 100000})
+        logo: 'https://avatars.githubusercontent.com/u/13695166?v=4',
-
+        title: 'os-data',
-test('Test Gift-Data', async () => {
+      },
-  const repos = await getAllDataPackagesFromOrg('gift-data', 'main', process.env.VITE_GITHUB_PAT)
+      repo: {
-  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
+        name: 'berlin-berlin',
-  expect(repos.failedDataPackages.length).toBe(0)
+        full_name: 'os-data/berlin-berlin',
-}, {timeout: 100000})
+        url: 'https://github.com/os-data/berlin-berlin',
+      },
+      files: [
+        {
+          name: 'berlin-gesamt',
+          format: 'csv',
+          path: 'https://storage.openspending.org/berlin-berlin/berlin-gesamt.csv',
+          mediatype: 'text/csv',
+          bytes: 81128743,
+          encoding: 'utf-8',
+        },
+      ],
+      author: 'Michael Peters <michael.peters@okfn.de>',
+      cityCode: 'Berlin',
+      countryCode: 'DE',
+      fiscalPeriod: { start: '2014-01-01', end: '2019-12-31' },
+      readme: '',
+    });
+  },
+  { timeout: 100000 }
+);
 
+test(
+  'Test getting one section of csv from R2',
+  async () => {
+    const rawCsv = await getCsv(
+      'https://storage.openspending.org/state-of-minas-gerais-brazil-planned-budget/__os_imported__br-mg-ppagloc.csv'
+    );
+    const parsedCsv = await parseCsv(rawCsv);
+    expect(parsedCsv.errors.length).toBe(1);
+    expect(parsedCsv.data.length).toBe(10165);
+    expect(parsedCsv.meta.fields).toStrictEqual([
+      'function_name',
+      'function_label',
+      'product_name',
+      'product_label',
+      'area_name',
+      'area_label',
+      'subaction_name',
+      'subaction_label',
+      'region_label_map',
+      'region_reg_map',
+      'region_name',
+      'region_label',
+      'municipality_map_id',
+      'municipality_name',
+      'municipality_map_code',
+      'municipality_label',
+      'municipality_map_name_simple',
+      'municipality_map_name',
+      'cofog1_label_en',
+      'cofog1_name',
+      'cofog1_label',
+      'amount',
+      'subprogramme_name',
+      'subprogramme_label',
+      'time_name',
+      'time_year',
+      'time_month',
+      'time_day',
+      'time_week',
+      'time_yearmonth',
+      'time_quarter',
+      'time',
+      'action_name',
+      'action_label',
+      'subfunction_name',
+      'subfunction_label',
+      'programme_name',
+      'programme_label',
+    ]);
+  },
+  { timeout: 100000 }
+);

examples/openspending/components/DatasetCard.tsx

@@ -9,7 +9,7 @@ export default function DatasetCard({ dataset }: { dataset: Project }) {
       className="overflow-hidden rounded-xl border border-gray-200"
     >
       <Link
-        href=""
+        href={`/@${dataset.owner.name}/${dataset.repo.name}`}
         className="flex items-center gap-x-4 border-b border-gray-900/5 bg-gray-50 p-6"
       >
         <img
@@ -60,8 +60,8 @@ export default function DatasetCard({ dataset }: { dataset: Project }) {
           <dd className="flex items-start gap-x-2">
             <div className="font-medium text-gray-900">
               <Link
-                // TODO: where do we get the info needed for this link?
+                // TODO: this link may be incorrect for some datasets
-                href=""
+                href={`https://github.com/${dataset.owner.name}/${dataset.repo.name}/blob/main/datapackage.json`}
                 target="_blank"
                 className="flex items-center hover:text-gray-700"
               >

examples/openspending/components/DatasetsSearch.tsx

@@ -2,9 +2,26 @@ import { useForm } from 'react-hook-form';
 import DatasetsGrid from './DatasetsGrid';
 import { Project } from '../lib/project.interface';
 import { Index } from 'flexsearch';
+import {
+  ChevronDoubleLeftIcon,
+  ChevronDoubleRightIcon,
+  ChevronLeftIcon,
+  ChevronRightIcon,
+} from '@heroicons/react/24/solid';
+import { useState } from 'react';
+
+export default function DatasetsSearch({
+  datasets,
+  availableCountries,
+}: {
+  datasets: Project[];
+  availableCountries;
+}) {
+  const itemsPerPage = 6;
+  const [page, setPage] = useState(1);
 
-export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
   const index = new Index({ tokenize: 'full' });
+
   datasets.forEach((dataset: Project) =>
     index.add(
       dataset.name,
@@ -21,94 +38,7 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
     },
   });
 
-  const allCountries = datasets
+  const filteredDatasets = datasets
-    .map((item) => item.countryCode)
-    .filter((v) => v) //  Filters false values
-    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
-    //  TODO: title should be the full name
-    .map((code) => ({ code, title: code }));
-
-  return (
-    <>
-      <div className="flex flex-col gap-3 sm:flex-row">
-        <div className="min-w-0 flex-auto">
-          <br />
-          <div className="relative">
-            <input
-              placeholder="Search datasets"
-              aria-label="Search datasets"
-              {...register('searchTerm')}
-              className="h-[3em] relative w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
-            />
-            {watch().searchTerm !== '' && (
-              <button
-                onClick={() => resetField('searchTerm')}
-                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
-              >
-                <CloseIcon />
-              </button>
-            )}
-          </div>
-        </div>
-        <div className="sm:basis-1/6">
-          {/* TODO: nicer select e.g. headlessui example */}
-          <label className="text-sm text-gray-600 font-medium">Country</label>
-          <select
-            className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
-            {...register('country')}
-          >
-            <option value="">All</option>
-            {allCountries.map((country) => {
-              return (
-                <option key={country.code} value={country.code}>
-                  {country.title}
-                </option>
-              );
-            })}
-          </select>
-        </div>
-        <div className="sm:basis-1/6">
-          <label className="text-sm text-gray-600 font-medium">Min. date</label>
-          <div className="relative">
-            <input
-              aria-label="Min. date"
-              type="date"
-              {...register('minDate')}
-              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
-            />
-            {watch().minDate !== '' && (
-              <button
-                onClick={() => resetField('minDate')}
-                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
-              >
-                <CloseIcon />
-              </button>
-            )}
-          </div>
-        </div>
-        <div className="sm:basis-1/6">
-          <label className="text-sm text-gray-600 font-medium">Max. date</label>
-          <div className="relative">
-            <input
-              aria-label="Max. date"
-              type="date"
-              {...register('maxDate')}
-              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
-            />
-            {watch().maxDate !== '' && (
-              <button
-                onClick={() => resetField('maxDate')}
-                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
-              >
-                <CloseIcon />
-              </button>
-            )}
-          </div>
-        </div>
-      </div>
-      <div className="min-w-full mt-10 align-middle">
-        <DatasetsGrid
-          datasets={datasets
     .filter((dataset: Project) =>
       watch().searchTerm && watch().searchTerm !== ''
         ? index.search(watch().searchTerm).includes(dataset.name)
@@ -132,8 +62,121 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
       watch().maxDate && watch().maxDate !== ''
         ? dataset.fiscalPeriod?.end <= watch().maxDate
         : true
-            )}
+    );
+
+  const paginatedDatasets = filteredDatasets.slice(
+    (page - 1) * itemsPerPage,
+    (page - 1) * itemsPerPage + itemsPerPage
+  );
+
+  const pageCount = Math.ceil(filteredDatasets.length / itemsPerPage) || 1;
+
+  return (
+    <>
+      <div className="flex flex-col gap-3 sm:flex-row">
+        <div className="min-w-0 flex-auto">
+          <br />
+          <div className="relative">
+            <input
+              placeholder="Search datasets"
+              aria-label="Search datasets"
+              {...register('searchTerm', { onChange: () => setPage(1) })}
+              className="h-[3em] relative w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
             />
+            {watch().searchTerm !== '' && (
+              <button
+                onClick={() => resetField('searchTerm')}
+                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
+              >
+                <CloseIcon />
+              </button>
+            )}
+          </div>
+        </div>
+        <div className="sm:basis-1/6">
+          {/* TODO: nicer select e.g. headlessui example */}
+          <label className="text-sm text-gray-600 font-medium">Country</label>
+          <select
+            className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            {...register('country', { onChange: () => setPage(1) })}
+          >
+            <option value="">All</option>
+            {availableCountries.map((country) => {
+              return (
+                <option key={country.code} value={country.code}>
+                  {country.title}
+                </option>
+              );
+            })}
+          </select>
+        </div>
+        <div className="sm:basis-1/6">
+          <label className="text-sm text-gray-600 font-medium">Min. date</label>
+          <div className="relative">
+            <input
+              aria-label="Min. date"
+              type="date"
+              {...register('minDate', { onChange: () => setPage(1) })}
+              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+          </div>
+        </div>
+        <div className="sm:basis-1/6">
+          <label className="text-sm text-gray-600 font-medium">Max. date</label>
+          <div className="relative">
+            <input
+              aria-label="Max. date"
+              type="date"
+              {...register('maxDate', { onChange: () => setPage(1) })}
+              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+          </div>
+        </div>
+      </div>
+      <div className="mt-5 mb-5">
+        <span className="text-lg font-medium">
+          {filteredDatasets.length} datasets found
+        </span>
+      </div>
+      <div className="min-w-full align-middle">
+        <DatasetsGrid datasets={paginatedDatasets} />
+        <div className="w-full flex justify-center mt-10">
+          <button
+            onClick={() => setPage(1)}
+            disabled={page <= 1}
+            className="disabled:text-gray-400"
+          >
+            <ChevronDoubleLeftIcon className="w-6 h-6" />
+          </button>
+          <button
+            onClick={() => {
+              if (page > 1) setPage((prev) => --prev);
+            }}
+            disabled={page <= 1}
+            className="disabled:text-gray-400"
+          >
+            <ChevronLeftIcon className="w-6 h-6" />
+          </button>
+          <span className="mx-5">
+            Page {page} of {pageCount}
+          </span>
+          <button
+            onClick={() => {
+              if (page < pageCount) setPage((prev) => ++prev);
+            }}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronRightIcon className="w-6 h-6" />
+          </button>
+          <button
+            onClick={() => setPage(pageCount)}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronDoubleRightIcon className="w-6 h-6" />
+          </button>
+        </div>
       </div>
     </>
   );

examples/openspending/components/Header.tsx

@@ -1,53 +1,82 @@
-import Image from 'next/image'
+import Image from 'next/image';
-import { Button } from './Button'
+import { Container } from './Container';
-import { Container } from './Container'
+import logo from '../public/logo.svg';
-import logo from "../public/logo.svg"
+import Link from 'next/link';
-import Link from 'next/link'
+import { useRouter } from 'next/router';
-import { useRouter } from 'next/router'
+import { Bars3Icon } from '@heroicons/react/24/outline';
+import { useState } from 'react';
 
 export function Header() {
+  const [menuOpen, setMenuOpen] = useState<boolean>(false);
+
   const router = useRouter();
 
   const isActive = (navLink) => {
-    return router.asPath.split("?")[0] == navLink.href;
+    return router.asPath.split('?')[0] == navLink.href;
-  }
+  };
 
   const navLinks = [
     {
-      title: "Home",
+      title: 'Home',
-      href: "/#header"
+      href: '/',
     },
     {
-      title: "Datasets",
+      title: 'Datasets',
-      href: "/#datasets"
+      href: '/#datasets',
     },
-    {
+    // {
-      title: "Community",
+    //   title: "Community",
-      href: "https://community.openspending.org/"
+    //   href: "https://community.openspending.org/"
-    }
+    // }
-  ]
+  ];
 
   return (
-    <header className="z-50 pb-5 lg:pt-11 sticky top-0 backdrop-blur" id="header">
+    <header className="relative z-50 pb-11 lg:pt-11">
-      <Container className="flex flex-wrap items-center justify-center sm:justify-between lg:flex-nowrap">
+      <Container className="flex flex-wrap items-center justify-between lg:flex-nowrap mt-10 lg:mt-0">
-        <div className="mt-10 lg:mt-0 lg:grow lg:basis-0 flex items-center">
+        <Link href="/" className="lg:mt-0 lg:grow lg:basis-0 flex items-center">
           <Image src={logo} alt="OpenSpending" className="h-12 w-auto" />
-        </div>
+        </Link>
-        <ul className='list-none flex gap-x-5 text-base font-medium'>
+        <ul className="hidden list-none sm:flex gap-x-5 text-base font-medium">
           {navLinks.map((link, i) => (
             <li key={`nav-link-${i}`}>
               <Link
-                className={`text-emerald-900 hover:text-emerald-600 ${isActive(link) ? "text-emerald-600" : ""}`}
+                className={`text-emerald-900 hover:text-emerald-600 ${
+                  isActive(link) ? 'text-emerald-600' : ''
+                }`}
                 href={link.href}
                 scroll={false}
               >
                 {link.title}
               </Link>
-            </li>))}
+            </li>
+          ))}
         </ul>
-        <div className="hidden sm:mt-10 sm:flex lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
+        <div className="hidden xl:block xl:grow"></div>
+        <div className="sm:hidden sm:mt-10 lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
+          <button onClick={() => setMenuOpen(!menuOpen)}>
+            <Bars3Icon className="w-8 h-8" />
+          </button>
         </div>
+        {menuOpen && (
+          <div className={`sm:hidden basis-full mt-5 text-center`}>
+            <ul className="gap-x-5 text-base font-medium">
+              {navLinks.map((link, i) => (
+                <li key={`nav-link-${i}`}>
+                  <Link
+                    className={`text-emerald-900 hover:text-emerald-600 ${
+                      isActive(link) ? 'text-emerald-600' : ''
+                    }`}
+                    href={link.href}
+                    scroll={false}
+                  >
+                    {link.title}
+                  </Link>
+                </li>
+              ))}
+            </ul>
+          </div>
+        )}
       </Container>
-    </header >
+    </header>
-  )
+  );
 }

examples/openspending/components/Hero.tsx

@@ -1,9 +1,9 @@
-import { Button } from './Button'
+import { Button } from './Button';
-import { Container } from './Container'
+import { Container } from './Container';
 
-export function Hero() {
+export function Hero({ countriesCount, datasetsCount, filesCount }) {
   return (
-    <div className="relative pb-20 pt-10 sm:py-40">
+    <div className="relative pb-20 pt-10 sm:py-40" id="hero">
       <div className="absolute inset-x-0 -bottom-14 -top-48 overflow-hidden bg-green-50 bg-opacity-50">
         <div className="absolute inset-x-0 top-0 h-40 bg-gradient-to-b from-white" />
         <div className="absolute inset-x-0 bottom-0 h-40 bg-gradient-to-t from-white" />
@@ -15,12 +15,13 @@ export function Hero() {
           </h1>
           <div className="mt-6 space-y-6 font-display text-2xl tracking-tight text-emerald-900">
             <p>
-              By understanding how governments spend money in our name can we have a say
+              By understanding how governments spend money in our name can we
-              in how that money will affect our own lives. The journey starts here.
+              have a say in how that money will affect our own lives. The
+              journey starts here.
             </p>
             <p>
-              OpenSpending is a free, open and global platform to search, visualise and analyse
+              OpenSpending is a free, open and global platform to search,
-              fiscal data in the public sphere.
+              visualise and analyse fiscal data in the public sphere.
             </p>
           </div>
           <Button href="#datasets" className="mt-10">
@@ -28,9 +29,11 @@ export function Hero() {
           </Button>
           <dl className="mt-10 grid grid-cols-2 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
             {[
-              ['Countries', '75'],
+              //  Added the plus sign because some datasets do not
-              ['Datasets', '2091'],
+              //  contain defined countries
-              ['Files', '9230'],
+              ['Countries', '+' + countriesCount],
+              ['Datasets', datasetsCount],
+              ['Files', filesCount],
             ].map(([name, value]) => (
               <div key={name}>
                 <dt className="font-mono text-sm text-emerald-600">{name}</dt>
@@ -43,5 +46,5 @@ export function Hero() {
         </div>
       </Container>
     </div>
-  )
+  );
 }

examples/openspending/components/Table.tsx

@@ -9,7 +9,7 @@ import { Grid } from '@githubocto/flat-ui';
 
 const queryClient = new QueryClient();
 
-async function getCsv(url: string) {
+export async function getCsv(url: string) {
   const response = await fetch(url, {
     headers: {
       Range: 'bytes=0-5132288',

examples/openspending/components/_shared/Layout.tsx

@@ -0,0 +1,10 @@
+import { Header } from '../Header';
+
+export default function Layout({ children }) {
+  return (
+    <div className="bg-white min-h-screen">
+      <Header />
+      {children}
+    </div>
+  );
+}

examples/openspending/datasets.json

@@ -1,27 +0,0 @@
-[
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "mongolia-budget-2016-2017"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "gb-country-regional-analysis"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "berlin-berlin"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "state-of-minas-gerais-brazil-planned-budget"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "wesel"
-  }
-]

examples/openspending/lib/datapackage.interface.ts

@@ -98,6 +98,7 @@ export interface TabularDataResource {
   key?: string;
   path?: string;
   size?: number;
+  bytes?: number;
 }
 
 export interface Field {

examples/openspending/lib/loader.ts

@@ -5,13 +5,14 @@ export function loadDataPackage(datapackage: FiscalDataPackage, repo): Project {
   return {
     name: datapackage.name,
     title: datapackage.title,
+    description: datapackage.description || null,
     owner: {
       name: repo.owner.login,
       logo: repo.owner.avatar_url,
       // TODO: make this title work
       title: repo.owner.login,
     },
-    repo: { name: repo, full_name: repo.full_name },
+    repo: { name: repo.name, full_name: repo.full_name, url: repo.html_url },
     files: datapackage.resources,
     author: datapackage.author ? datapackage.author : null,
     cityCode: datapackage.cityCode ? datapackage.cityCode : null,

examples/openspending/lib/project.interface.ts

@@ -5,10 +5,11 @@ import {
 
 export interface Project {
   owner: { name: string; logo?: string; title?: string }; // Info about the owner of the data repo
-  repo: { name: string; full_name: string }; // Info about the the data repo
+  repo: { name: string; full_name: string; url: string }; // Info about the the data repo
   files: TabularDataResource[];
   name: string;
   title?: string;
+  description?: string;
   author?: string;
   cityCode?: string;
   countryCode?: string;

examples/openspending/lib/project.ts

@@ -13,8 +13,7 @@ export interface GithubProject {
 export async function getProjectReadme(
   owner: string,
   repo: string,
-  branch: string,
+  branch: string = 'main',
-  readme: string,
   github_pat?: string
 ) {
   const octokit = new Octokit({ auth: github_pat });
@@ -22,7 +21,7 @@ export async function getProjectReadme(
     const response = await octokit.rest.repos.getContent({
       owner,
       repo,
-      path: readme,
+      path: 'README.md',
       ref: branch,
     });
     const data = response.data as { content?: string };
@@ -125,7 +124,6 @@ export async function getProject(project: GithubProject, github_pat?: string) {
     project.owner,
     project.repo,
     project.branch,
-    project.readme,
     github_pat
   );
 
@@ -185,8 +183,43 @@ export async function getProjectDataPackage(
     }
     const decodedContent = Buffer.from(fileContent, 'base64').toString();
     const datapackage = JSON.parse(decodedContent);
-    return {...datapackage, repo };
+
+    return { ...datapackage, repo };
   } catch (error) {
     return null;
   }
 }
+
+export async function getAllProjectsFromOrg(
+  org: string,
+  branch?: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  const repos = await octokit.rest.repos.listForOrg({
+    org,
+    type: 'public',
+    per_page: 100,
+  });
+  let failedProjects = [];
+  const projects = await Promise.all(
+    repos.data.map(async (_repo) => {
+      const project = await getProjectDataPackage(
+        org,
+        _repo.name,
+        branch ? branch : 'main',
+        github_pat
+      );
+      if (!project) {
+        failedProjects.push(_repo.name);
+        return null;
+      }
+
+      return { datapackage: project, repo: _repo };
+    })
+  );
+  return {
+    results: projects.filter((item) => item !== null),
+    failed: failedProjects,
+  };
+}

examples/openspending/orgs.json

@@ -0,0 +1,3 @@
+[
+ "os-data"
+]

examples/openspending/package-lock.json

@@ -9,6 +9,7 @@
       "version": "0.1.0",
       "dependencies": {
         "@githubocto/flat-ui": "^0.14.1",
+        "@heroicons/react": "^2.0.18",
         "@octokit/plugin-throttling": "^5.2.2",
         "@types/flexsearch": "^0.7.3",
         "@types/node": "18.16.0",
@@ -1622,6 +1623,14 @@
         "object-assign": "^4.1.1"
       }
     },
+    "node_modules/@heroicons/react": {
+      "version": "2.0.18",
+      "resolved": "https://registry.npmjs.org/@heroicons/react/-/react-2.0.18.tgz",
+      "integrity": "sha512-7TyMjRrZZMBPa+/5Y8lN0iyvUU/01PeMGX2+RE7cQWpEUIcb4QotzUObFkJDejj/HUH4qjP/eQ0gzzKs2f+6Yw==",
+      "peerDependencies": {
+        "react": ">= 16"
+      }
+    },
     "node_modules/@humanwhocodes/config-array": {
       "version": "0.11.8",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz",

examples/openspending/package.json

@@ -11,6 +11,7 @@
   },
   "dependencies": {
     "@githubocto/flat-ui": "^0.14.1",
+    "@heroicons/react": "^2.0.18",
     "@octokit/plugin-throttling": "^5.2.2",
     "@types/flexsearch": "^0.7.3",
     "@types/node": "18.16.0",

examples/openspending/pages/@org/[org]/[...path].tsx

@@ -1,126 +0,0 @@
-import { NextSeo } from 'next-seo';
-import { promises as fs } from 'fs';
-import path from 'path';
-import getConfig from 'next/config';
-import { getProject, GithubProject } from '../../../lib/octokit';
-import ReactMarkdown from 'react-markdown';
-import remarkGfm from 'remark-gfm';
-import Breadcrumbs from '../../../components/_shared/Breadcrumbs';
-
-export default function ProjectPage({ project }) {
-  const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`
-
-  return (
-    <>
-      <NextSeo title={`${repoId}${project.base_path !== '/' ? '/' + project.base_path : ''} - GitHub Datasets`} />
-      <main className="prose mx-auto my-8">
-        <Breadcrumbs links={[{ title: repoId, href: "" }]} />
-        <h1 className="mb-0 mt-16">{project.repo_config.name || repoId}</h1>
-        <p className='mb-8'><span className='font-semibold'>Repository:</span> <a target="_blank" href={project.html_url}>{project.html_url}</a></p>
-
-        <h2 className="mb-0 mt-10">Files</h2>
-        <div className="inline-block min-w-full py-2 align-middle">
-          <table className="min-w-full divide-y divide-gray-300">
-            <thead>
-              <tr>
-                <th
-                  scope="col"
-                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                >
-                  Name
-                </th>
-                <th
-                  scope="col"
-                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                >
-                  Size
-                </th>
-              </tr>
-            </thead>
-            <tbody className="divide-y divide-gray-200">
-              {project.files?.map((file) => (
-                <tr key={file.download_url}>
-                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                    <a href={file.download_url}>{file.name}</a>
-                  </td>
-                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                    {file.size} Bytes
-                  </td>
-                </tr>
-              ))}
-            </tbody>
-          </table>
-        </div>
-
-        {project.readmeContent && <>
-          <hr />
-
-          <h2 className='uppercase font-black'>Readme</h2>
-          <ReactMarkdown remarkPlugins={[remarkGfm]}>
-            {project.readmeContent}
-          </ReactMarkdown>
-        </>}
-      </main>
-    </>
-  );
-}
-
-// Generates `/posts/1` and `/posts/2`
-export async function getStaticPaths() {
-  const jsonDirectory = path.join(
-    process.cwd(),
-    'datasets.json'
-  );
-  const repos = await fs.readFile(jsonDirectory, 'utf8');
-
-  return {
-    paths: JSON.parse(repos).map((repo) => {
-      const projectPath =
-        repo.readme && repo.readme.split('/').length > 1
-          ? repo.readme.split('/').slice(0, -1)
-          : null;
-      let path = [repo.name];
-      if (projectPath) {
-        projectPath.forEach((element) => {
-          path.push(element);
-        });
-      }
-      return {
-        params: { org: repo.owner, path },
-      };
-    }),
-    fallback: false, // can also be true or 'blocking'
-  };
-}
-
-export async function getStaticProps({ params }) {
-  const jsonDirectory = path.join(
-    process.cwd(),
-    'datasets.json'
-  );
-  const reposFile = await fs.readFile(jsonDirectory, 'utf8');
-  const repos: GithubProject[] = JSON.parse(reposFile);
-  const repo = repos.find((_repo) => {
-    const projectPath =
-      _repo.readme && _repo.readme.split('/').length > 1
-        ? _repo.readme.split('/').slice(0, -1)
-        : null;
-    let path = [_repo.name];
-    if (projectPath) {
-      projectPath.forEach((element) => {
-        path.push(element);
-      });
-    }
-    return (
-      _repo.owner == params.org &&
-      JSON.stringify(path) === JSON.stringify(params.path)
-    );
-  });
-  const github_pat = getConfig().serverRuntimeConfig.github_pat;
-  const project = await getProject(repo, github_pat);
-  return {
-    props: {
-      project: { ...project, repo_config: repo },
-    },
-  };
-}

examples/openspending/pages/@org/[org]/[project].tsx

@@ -0,0 +1,234 @@
+import { NextSeo } from 'next-seo';
+import getConfig from 'next/config';
+import {
+  getAllProjectsFromOrg,
+  getProjectDataPackage,
+  getProjectMetadata,
+  getProjectReadme,
+} from '../../../lib/project';
+import ReactMarkdown from 'react-markdown';
+import remarkGfm from 'remark-gfm';
+import { loadDataPackage } from '../../../lib/loader';
+import Layout from '../../../components/_shared/Layout';
+import Link from 'next/link';
+import { Project } from '../../../lib/project.interface';
+import ExternalLinkIcon from '../../../components/icons/ExternalLinkIcon';
+
+export default function ProjectPage({
+  project,
+  readme,
+}: {
+  project: Project;
+  readme: string;
+}) {
+
+  //  Get description from datapackage or  calculate
+  //  excerpt from README by getting all the content
+  //  up to the first dot.
+  const description =
+    project.description || (readme && readme.slice(0, readme.indexOf('.') + 1));
+
+  return (
+    <Layout>
+      <NextSeo title={`${project.title} - OpenSpending`} />
+      <main className="prose mx-auto my-8">
+        <h1 className="mb-1 mt-16">{project.title || project.name}</h1>
+        <Link target="_blank" href={project.repo.url}>
+          @{project.repo.full_name}
+        </Link>
+
+        {description && (
+          <div className="inline-block min-w-full py-2 align-middle mt-5">
+            {description}
+          </div>
+        )}
+
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="min-w-full divide-y divide-gray-300">
+            <thead>
+              <tr>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Name
+                </th>
+                {project.datapackage.countryCode && (
+                  <th
+                    scope="col"
+                    className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                  >
+                    Country
+                  </th>
+                )}
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Metadata
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              <tr>
+                <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                  {project.name}
+                </td>
+                {project.datapackage.countryCode && (
+                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                    {project.datapackage.countryCode}
+                  </td>
+                )}
+
+                <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                  <Link
+                    // TODO: this link may be incorrect for some datasets
+                    href={`https://github.com/${project.owner.name}/${project.repo.name}/blob/main/datapackage.json`}
+                    target="_blank"
+                    className="flex items-center hover:text-gray-700"
+                  >
+                    datapackage.json <ExternalLinkIcon className="ml-1" />
+                  </Link>
+                </td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+
+        <h3 className="mb-1 mt-10">Data files</h3>
+        <p>
+          This dataset contains {project.files.length} file
+          {project.files.length != 1 ? '' : 's'}
+        </p>
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="mt-0 min-w-full divide-y divide-gray-300">
+            <thead>
+              <tr>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Name
+                </th>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Format
+                </th>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Size
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              {project.files?.map((file) => {
+                let size: number | string = file.size;
+
+                if (!size) {
+                  if (file.bytes) {
+                    if (file.bytes > 1000000) {
+                      size = (file.bytes / 1000000).toFixed(2) + ' MB';
+                    } else {
+                      size = (file.bytes / 1000).toFixed(2) + ' kB';
+                    }
+                  }
+                }
+
+                return (
+                  <tr key={file.name}>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {file.name}
+                    </td>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {file.format}
+                    </td>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {size}
+                    </td>
+
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      <Link
+                        target="_blank"
+                        href={
+                          file.path.startsWith('http')
+                            ? file.path
+                            : `https://raw.githubusercontent.com/${project.owner.name}/${project.repo.name}/main/${file.path}`
+                        }
+                      >
+                        Download
+                      </Link>
+                    </td>
+                  </tr>
+                );
+              })}
+            </tbody>
+          </table>
+        </div>
+
+        {readme && (
+          <>
+            <hr />
+
+            <h2 className="uppercase font-black">Readme</h2>
+            <ReactMarkdown remarkPlugins={[remarkGfm]}>{readme}</ReactMarkdown>
+          </>
+        )}
+      </main>
+    </Layout>
+  );
+}
+
+// Generates `/posts/1` and `/posts/2`
+export async function getStaticPaths() {
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+
+  const allProjects = await getAllProjectsFromOrg(
+    'os-data',
+    'main',
+    github_pat
+  );
+
+  console.log(allProjects)
+  const paths = allProjects.results.map((project) => ({
+    params: {
+      // TODO: dynamize the org
+      org: 'os-data',
+      project: project.repo.name,
+    },
+  }));
+
+  return {
+    paths,
+    fallback: false, // can also be true or 'blocking'
+  };
+}
+
+export async function getStaticProps({ params }) {
+  const { org: orgName, project: projectName } = params;
+
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+  const datapackage = await getProjectDataPackage(
+    orgName,
+    projectName,
+    'main',
+    github_pat
+  );
+
+  const repo = await getProjectMetadata(orgName, projectName, github_pat);
+
+  const project = loadDataPackage(datapackage, repo);
+
+  // TODO: should this be moved to the loader?
+  const readme = await getProjectReadme(orgName, projectName, 'main', github_pat);
+
+  return {
+    props: {
+      project,
+      readme,
+    },
+  };
+}

examples/openspending/pages/_app.tsx

@@ -1,13 +1,11 @@
 import { AppProps } from 'next/app';
-import Head from 'next/head';
 import './styles.css';
+import { NextSeo } from 'next-seo';
 
 function CustomApp({ Component, pageProps }: AppProps) {
   return (
     <>
-      <Head>
+      <NextSeo title="OpenSpending" />
-        <title>GitHub Datasets</title>
-      </Head>
       <main className="app">
         <Component {...pageProps} />
       </main>

examples/openspending/pages/index.tsx

@@ -1,65 +1,58 @@
-import { promises as fs } from 'fs';
+import { getAllProjectsFromOrg } from '../lib/project';
-import path from 'path';
-import {
-  GithubProject,
-  getProjectDataPackage,
-  getProjectMetadata,
-} from '../lib/octokit';
 import getConfig from 'next/config';
-import ExternalLinkIcon from '../components/icons/ExternalLinkIcon';
-import TimeAgo from 'react-timeago';
-import Link from 'next/link';
 import { Hero } from '../components/Hero';
-import { Header } from '../components/Header';
 import { Container } from '../components/Container';
 import { FiscalDataPackage } from '../lib/datapackage.interface';
 import { loadDataPackage } from '../lib/loader';
 import DatasetsSearch from '../components/DatasetsSearch';
+import Layout from '../components/_shared/Layout';
 
 export async function getStaticProps() {
-  const jsonDirectory = path.join(process.cwd(), '/datasets.json');
+  //  TODO: support other orgs
-  const repos = await fs.readFile(jsonDirectory, 'utf8');
+  // const orgsListPath = path.join(process.cwd(), '/orgs.json');
+  // const orgs = await fs.readFile(orgsListPath, 'utf8');
+
   const github_pat = getConfig().serverRuntimeConfig.github_pat;
-  const datapackages = await Promise.all(
+
-    JSON.parse(repos).map(async (_repo: GithubProject) => {
+  const allProjects = await getAllProjectsFromOrg(
-      const datapackage = await getProjectDataPackage(
+    'os-data',
-        _repo.owner,
-        _repo.name,
     'main',
     github_pat
   );
-      const repo = await getProjectMetadata(
-        _repo.owner,
-        _repo.name,
-        github_pat
-      );
 
-      return {
+  const projects = allProjects.results.map(
-        datapackage,
-        repo,
-      };
-    })
-  );
-
-  const projects = datapackages.map(
     (item: { datapackage: FiscalDataPackage & { repo: string }; repo: any }) =>
       loadDataPackage(item.datapackage, item.repo)
   );
 
+  const availableCountries = projects
+    .map((item) => item.countryCode)
+    .filter((v) => v) //  Filters false values
+    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
+    //  TODO: title should be the full name
+    .map((code) => ({ code, title: code }));
+
   return {
     props: {
       projects: JSON.stringify(projects),
+      availableCountries,
     },
   };
 }
 
-export function Datasets({ projects }) {
+export function Home({ projects, availableCountries }) {
   projects = JSON.parse(projects);
 
   return (
-    <div className="bg-white min-h-screen">
+    <Layout>
-      <Header />
+      <Hero
-      <Hero />
+        countriesCount={availableCountries.length}
+        datasetsCount={projects.length}
+        filesCount={projects.reduce(
+          (partialSum, a) => partialSum + a.files.length,
+          0
+        )}
+      />
       <section className="py-20 sm:py-32">
         <Container>
           <div className="mx-auto max-w-2xl lg:mx-0">
@@ -74,12 +67,15 @@ export function Datasets({ projects }) {
             </p>
           </div>
           <div className="mt-10">
-            <DatasetsSearch datasets={projects} />
+            <DatasetsSearch
+              datasets={projects}
+              availableCountries={availableCountries}
+            />
           </div>
         </Container>
       </section>
-    </div>
+    </Layout>
   );
 }
 
-export default Datasets;
+export default Home;