[examples/openspending][m] - fix build + add tests

* [examples/openspending][m] - table preview

* [examples/openspending][xs] - fix typo

examples/openspending/__tests__/os-data.test.ts

@@ -1,45 +1,138 @@
-import { Octokit } from 'octokit';
+import { expect, test } from 'vitest';
-import { assert, expect, test } from 'vitest'
+import { getAllProjectsFromOrg, getProjectDataPackage } from '../lib/project';
-import { getProjectDataPackage } from '../lib/octokit';
+import { loadDataPackage } from '../lib/loader';
+import { getProjectMetadata } from '../lib/project';
+import { getCsv, parseCsv } from '../components/Table';
 
-export async function getAllDataPackagesFromOrg(
+test(
-  org: string,
+  'Test OS-Data',
-  branch?: string,
+  async () => {
-  github_pat?: string
+    const repos = await getAllProjectsFromOrg(
-) {
+      'os-data',
-  const octokit = new Octokit({ auth: github_pat });
+      'main',
-  const repos = await octokit.rest.repos.listForOrg({ org, type: 'public', per_page: 100 });
+      process.env.VITE_GITHUB_PAT
-  let failedDataPackages = [];
+    );
-  const datapackages = await Promise.all(
+    if (repos.failed.length > 0) console.log(repos.failed);
-    repos.data.map(async (_repo) => {
+    expect(repos.failed.length).toBe(0);
-      const datapackage = await getProjectDataPackage(
+  },
-        org,
+  { timeout: 100000 }
-        _repo.name,
+);
-        branch ? branch : 'main',
-        github_pat
-      );
-      if (!datapackage) {
-        failedDataPackages.push(_repo.name)
-        return null
-      };
-      return {...datapackage, repo: _repo.name};
-    })
-  );
-  return {
-    datapackages: datapackages.filter((item) => item !== null),
-    failedDataPackages,
-  };
-}
 
-test('Test OS-Data', async () => {
+test(
-  const repos = await getAllDataPackagesFromOrg('os-data', 'main', process.env.VITE_GITHUB_PAT)
+  'Test Gift-Data',
-  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
+  async () => {
-  expect(repos.failedDataPackages.length).toBe(0)
+    const repos = await getAllProjectsFromOrg(
-}, {timeout: 100000})
+      'gift-data',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    if (repos.failed.length > 0) console.log(repos.failed);
+    expect(repos.failed.length).toBe(0);
+  },
+  { timeout: 100000 }
+);
 
-test('Test Gift-Data', async () => {
+test(
-  const repos = await getAllDataPackagesFromOrg('gift-data', 'main', process.env.VITE_GITHUB_PAT)
+  'Test getting one dataset from github',
-  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
+  async () => {
-  expect(repos.failedDataPackages.length).toBe(0)
+    const datapackage = await getProjectDataPackage(
-}, {timeout: 100000})
+      'os-data',
+      'berlin-berlin',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    const repo = await getProjectMetadata(
+      'os-data',
+      'berlin-berlin',
+      process.env.VITE_GITHUB_PAT
+    );
+    const project = loadDataPackage(datapackage, repo);
+    delete project['datapackage'];
+    delete project.files[0]['dialect'];
+    delete project.files[0]['schema'];
+    expect(project).toStrictEqual({
+      name: 'berlin-berlin',
+      title: 'Berlin-Berlin',
+      description: null,
+      owner: {
+        name: 'os-data',
+        logo: 'https://avatars.githubusercontent.com/u/13695166?v=4',
+        title: 'os-data',
+      },
+      repo: {
+        name: 'berlin-berlin',
+        full_name: 'os-data/berlin-berlin',
+        url: 'https://github.com/os-data/berlin-berlin',
+      },
+      files: [
+        {
+          name: 'berlin-gesamt',
+          format: 'csv',
+          path: 'https://storage.openspending.org/berlin-berlin/berlin-gesamt.csv',
+          mediatype: 'text/csv',
+          bytes: 81128743,
+          encoding: 'utf-8',
+        },
+      ],
+      author: 'Michael Peters <michael.peters@okfn.de>',
+      cityCode: 'Berlin',
+      countryCode: 'DE',
+      fiscalPeriod: { start: '2014-01-01', end: '2019-12-31' },
+      readme: '',
+    });
+  },
+  { timeout: 100000 }
+);
 
+test(
+  'Test getting one section of csv from R2',
+  async () => {
+    const rawCsv = await getCsv(
+      'https://storage.openspending.org/state-of-minas-gerais-brazil-planned-budget/__os_imported__br-mg-ppagloc.csv'
+    );
+    const parsedCsv = await parseCsv(rawCsv);
+    expect(parsedCsv.errors.length).toBe(1);
+    expect(parsedCsv.data.length).toBe(10165);
+    expect(parsedCsv.meta.fields).toStrictEqual([
+      'function_name',
+      'function_label',
+      'product_name',
+      'product_label',
+      'area_name',
+      'area_label',
+      'subaction_name',
+      'subaction_label',
+      'region_label_map',
+      'region_reg_map',
+      'region_name',
+      'region_label',
+      'municipality_map_id',
+      'municipality_name',
+      'municipality_map_code',
+      'municipality_label',
+      'municipality_map_name_simple',
+      'municipality_map_name',
+      'cofog1_label_en',
+      'cofog1_name',
+      'cofog1_label',
+      'amount',
+      'subprogramme_name',
+      'subprogramme_label',
+      'time_name',
+      'time_year',
+      'time_month',
+      'time_day',
+      'time_week',
+      'time_yearmonth',
+      'time_quarter',
+      'time',
+      'action_name',
+      'action_label',
+      'subfunction_name',
+      'subfunction_label',
+      'programme_name',
+      'programme_label',
+    ]);
+  },
+  { timeout: 100000 }
+);

examples/openspending/components/DatasetCard.tsx

@@ -9,7 +9,7 @@ export default function DatasetCard({ dataset }: { dataset: Project }) {
       className="overflow-hidden rounded-xl border border-gray-200"
     >
       <Link
-        href=""
+        href={`/@${dataset.owner.name}/${dataset.repo.name}`}
         className="flex items-center gap-x-4 border-b border-gray-900/5 bg-gray-50 p-6"
       >
         <img
@@ -60,8 +60,8 @@ export default function DatasetCard({ dataset }: { dataset: Project }) {
           <dd className="flex items-start gap-x-2">
             <div className="font-medium text-gray-900">
               <Link
-                // TODO: where do we get the info needed for this link?
+                // TODO: this link may be incorrect for some datasets
-                href=""
+                href={`https://github.com/${dataset.owner.name}/${dataset.repo.name}/blob/main/datapackage.json`}
                 target="_blank"
                 className="flex items-center hover:text-gray-700"
               >

examples/openspending/components/DatasetsSearch.tsx

@@ -2,9 +2,26 @@ import { useForm } from 'react-hook-form';
 import DatasetsGrid from './DatasetsGrid';
 import { Project } from '../lib/project.interface';
 import { Index } from 'flexsearch';
+import {
+  ChevronDoubleLeftIcon,
+  ChevronDoubleRightIcon,
+  ChevronLeftIcon,
+  ChevronRightIcon,
+} from '@heroicons/react/24/solid';
+import { useState } from 'react';
+
+export default function DatasetsSearch({
+  datasets,
+  availableCountries,
+}: {
+  datasets: Project[];
+  availableCountries;
+}) {
+  const itemsPerPage = 6;
+  const [page, setPage] = useState(1);
 
-export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
   const index = new Index({ tokenize: 'full' });
+
   datasets.forEach((dataset: Project) =>
     index.add(
       dataset.name,
@@ -21,12 +38,38 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
     },
   });
 
-  const allCountries = datasets
+  const filteredDatasets = datasets
-    .map((item) => item.countryCode)
+    .filter((dataset: Project) =>
-    .filter((v) => v) //  Filters false values
+      watch().searchTerm && watch().searchTerm !== ''
-    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
+        ? index.search(watch().searchTerm).includes(dataset.name)
-    //  TODO: title should be the full name
+        : true
-    .map((code) => ({ code, title: code }));
+    )
+    .filter((dataset) =>
+      watch().country && watch().country !== ''
+        ? dataset.countryCode === watch().country
+        : true
+    )
+    // TODO: Does that really makes sense?
+    // What if the fiscalPeriod is 2015-2017 and inputs are
+    // set to 2015-2016. It's going to be filtered out but
+    // it shouldn't.
+    .filter((dataset) =>
+      watch().minDate && watch().minDate !== ''
+        ? dataset.fiscalPeriod?.start >= watch().minDate
+        : true
+    )
+    .filter((dataset) =>
+      watch().maxDate && watch().maxDate !== ''
+        ? dataset.fiscalPeriod?.end <= watch().maxDate
+        : true
+    );
+
+  const paginatedDatasets = filteredDatasets.slice(
+    (page - 1) * itemsPerPage,
+    (page - 1) * itemsPerPage + itemsPerPage
+  );
+
+  const pageCount = Math.ceil(filteredDatasets.length / itemsPerPage) || 1;
 
   return (
     <>
@@ -37,7 +80,7 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
             <input
               placeholder="Search datasets"
               aria-label="Search datasets"
-              {...register('searchTerm')}
+              {...register('searchTerm', { onChange: () => setPage(1) })}
               className="h-[3em] relative w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
             />
             {watch().searchTerm !== '' && (
@@ -55,10 +98,10 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
           <label className="text-sm text-gray-600 font-medium">Country</label>
           <select
             className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
-            {...register('country')}
+            {...register('country', { onChange: () => setPage(1) })}
           >
             <option value="">All</option>
-            {allCountries.map((country) => {
+            {availableCountries.map((country) => {
               return (
                 <option key={country.code} value={country.code}>
                   {country.title}
@@ -73,17 +116,9 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
             <input
               aria-label="Min. date"
               type="date"
-              {...register('minDate')}
+              {...register('minDate', { onChange: () => setPage(1) })}
               className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
             />
-            {watch().minDate !== '' && (
-              <button
-                onClick={() => resetField('minDate')}
-                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
-              >
-                <CloseIcon />
-              </button>
-            )}
           </div>
         </div>
         <div className="sm:basis-1/6">
@@ -92,48 +127,56 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
             <input
               aria-label="Max. date"
               type="date"
-              {...register('maxDate')}
+              {...register('maxDate', { onChange: () => setPage(1) })}
               className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
             />
-            {watch().maxDate !== '' && (
-              <button
-                onClick={() => resetField('maxDate')}
-                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
-              >
-                <CloseIcon />
-              </button>
-            )}
           </div>
         </div>
       </div>
-      <div className="min-w-full mt-10 align-middle">
+      <div className="mt-5 mb-5">
-        <DatasetsGrid
+        <span className="text-lg font-medium">
-          datasets={datasets
+          {filteredDatasets.length} datasets found
-            .filter((dataset: Project) =>
+        </span>
-              watch().searchTerm && watch().searchTerm !== ''
+      </div>
-                ? index.search(watch().searchTerm).includes(dataset.name)
+      <div className="min-w-full align-middle">
-                : true
+        <DatasetsGrid datasets={paginatedDatasets} />
-            )
+        <div className="w-full flex justify-center mt-10">
-            .filter((dataset) =>
+          <button
-              watch().country && watch().country !== ''
+            onClick={() => setPage(1)}
-                ? dataset.countryCode === watch().country
+            disabled={page <= 1}
-                : true
+            className="disabled:text-gray-400"
-            )
+          >
-            // TODO: Does that really makes sense?
+            <ChevronDoubleLeftIcon className="w-6 h-6" />
-            // What if the fiscalPeriod is 2015-2017 and inputs are
+          </button>
-            // set to 2015-2016. It's going to be filtered out but
+          <button
-            // it shouldn't.
+            onClick={() => {
-            .filter((dataset) =>
+              if (page > 1) setPage((prev) => --prev);
-              watch().minDate && watch().minDate !== ''
+            }}
-                ? dataset.fiscalPeriod?.start >= watch().minDate
+            disabled={page <= 1}
-                : true
+            className="disabled:text-gray-400"
-            )
+          >
-            .filter((dataset) =>
+            <ChevronLeftIcon className="w-6 h-6" />
-              watch().maxDate && watch().maxDate !== ''
+          </button>
-                ? dataset.fiscalPeriod?.end <= watch().maxDate
+          <span className="mx-5">
-                : true
+            Page {page} of {pageCount}
-            )}
+          </span>
-        />
+          <button
+            onClick={() => {
+              if (page < pageCount) setPage((prev) => ++prev);
+            }}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronRightIcon className="w-6 h-6" />
+          </button>
+          <button
+            onClick={() => setPage(pageCount)}
+            disabled={page >= pageCount}
+            className="disabled:text-gray-400"
+          >
+            <ChevronDoubleRightIcon className="w-6 h-6" />
+          </button>
+        </div>
       </div>
     </>
   );

examples/openspending/components/Header.tsx

@@ -1,53 +1,82 @@
-import Image from 'next/image'
+import Image from 'next/image';
-import { Button } from './Button'
+import { Container } from './Container';
-import { Container } from './Container'
+import logo from '../public/logo.svg';
-import logo from "../public/logo.svg"
+import Link from 'next/link';
-import Link from 'next/link'
+import { useRouter } from 'next/router';
-import { useRouter } from 'next/router'
+import { Bars3Icon } from '@heroicons/react/24/outline';
+import { useState } from 'react';
 
 export function Header() {
+  const [menuOpen, setMenuOpen] = useState<boolean>(false);
+
   const router = useRouter();
 
   const isActive = (navLink) => {
-    return router.asPath.split("?")[0] == navLink.href;
+    return router.asPath.split('?')[0] == navLink.href;
-  }
+  };
 
   const navLinks = [
     {
-      title: "Home",
+      title: 'Home',
-      href: "/#header"
+      href: '/',
     },
     {
-      title: "Datasets",
+      title: 'Datasets',
-      href: "/#datasets"
+      href: '/#datasets',
     },
-    {
+    // {
-      title: "Community",
+    //   title: "Community",
-      href: "https://community.openspending.org/"
+    //   href: "https://community.openspending.org/"
-    }
+    // }
-  ]
+  ];
 
   return (
-    <header className="z-50 pb-5 lg:pt-11 sticky top-0 backdrop-blur" id="header">
+    <header className="relative z-50 pb-11 lg:pt-11">
-      <Container className="flex flex-wrap items-center justify-center sm:justify-between lg:flex-nowrap">
+      <Container className="flex flex-wrap items-center justify-between lg:flex-nowrap mt-10 lg:mt-0">
-        <div className="mt-10 lg:mt-0 lg:grow lg:basis-0 flex items-center">
+        <Link href="/" className="lg:mt-0 lg:grow lg:basis-0 flex items-center">
           <Image src={logo} alt="OpenSpending" className="h-12 w-auto" />
-        </div>
+        </Link>
-        <ul className='list-none flex gap-x-5 text-base font-medium'>
+        <ul className="hidden list-none sm:flex gap-x-5 text-base font-medium">
           {navLinks.map((link, i) => (
             <li key={`nav-link-${i}`}>
               <Link
-                className={`text-emerald-900 hover:text-emerald-600 ${isActive(link) ? "text-emerald-600" : ""}`}
+                className={`text-emerald-900 hover:text-emerald-600 ${
+                  isActive(link) ? 'text-emerald-600' : ''
+                }`}
                 href={link.href}
                 scroll={false}
               >
                 {link.title}
               </Link>
-            </li>))}
+            </li>
+          ))}
         </ul>
-        <div className="hidden sm:mt-10 sm:flex lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
+        <div className="hidden xl:block xl:grow"></div>
+        <div className="sm:hidden sm:mt-10 lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
+          <button onClick={() => setMenuOpen(!menuOpen)}>
+            <Bars3Icon className="w-8 h-8" />
+          </button>
         </div>
+        {menuOpen && (
+          <div className={`sm:hidden basis-full mt-5 text-center`}>
+            <ul className="gap-x-5 text-base font-medium">
+              {navLinks.map((link, i) => (
+                <li key={`nav-link-${i}`}>
+                  <Link
+                    className={`text-emerald-900 hover:text-emerald-600 ${
+                      isActive(link) ? 'text-emerald-600' : ''
+                    }`}
+                    href={link.href}
+                    scroll={false}
+                  >
+                    {link.title}
+                  </Link>
+                </li>
+              ))}
+            </ul>
+          </div>
+        )}
       </Container>
-    </header >
+    </header>
-  )
+  );
 }

examples/openspending/components/Hero.tsx

@@ -1,9 +1,9 @@
-import { Button } from './Button'
+import { Button } from './Button';
-import { Container } from './Container'
+import { Container } from './Container';
 
-export function Hero() {
+export function Hero({ countriesCount, datasetsCount, filesCount }) {
   return (
-    <div className="relative pb-20 pt-10 sm:py-40">
+    <div className="relative pb-20 pt-10 sm:py-40" id="hero">
       <div className="absolute inset-x-0 -bottom-14 -top-48 overflow-hidden bg-green-50 bg-opacity-50">
         <div className="absolute inset-x-0 top-0 h-40 bg-gradient-to-b from-white" />
         <div className="absolute inset-x-0 bottom-0 h-40 bg-gradient-to-t from-white" />
@@ -15,12 +15,13 @@ export function Hero() {
           </h1>
           <div className="mt-6 space-y-6 font-display text-2xl tracking-tight text-emerald-900">
             <p>
-              By understanding how governments spend money in our name can we have a say
+              By understanding how governments spend money in our name can we
-              in how that money will affect our own lives. The journey starts here.
+              have a say in how that money will affect our own lives. The
+              journey starts here.
             </p>
             <p>
-              OpenSpending is a free, open and global platform to search, visualise and analyse
+              OpenSpending is a free, open and global platform to search,
-              fiscal data in the public sphere.
+              visualise and analyse fiscal data in the public sphere.
             </p>
           </div>
           <Button href="#datasets" className="mt-10">
@@ -28,9 +29,11 @@ export function Hero() {
           </Button>
           <dl className="mt-10 grid grid-cols-2 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
             {[
-              ['Countries', '75'],
+              //  Added the plus sign because some datasets do not
-              ['Datasets', '2091'],
+              //  contain defined countries
-              ['Files', '9230'],
+              ['Countries', '+' + countriesCount],
+              ['Datasets', datasetsCount],
+              ['Files', filesCount],
             ].map(([name, value]) => (
               <div key={name}>
                 <dt className="font-mono text-sm text-emerald-600">{name}</dt>
@@ -43,5 +46,5 @@ export function Hero() {
         </div>
       </Container>
     </div>
-  )
+  );
 }

examples/openspending/components/Table.tsx

@@ -0,0 +1,94 @@
+import {
+  QueryClient,
+  QueryClientProvider,
+  useQuery,
+  useQueryClient,
+} from 'react-query';
+import Papa from 'papaparse';
+import { Grid } from '@githubocto/flat-ui';
+
+const queryClient = new QueryClient();
+
+export async function getCsv(url: string) {
+  const response = await fetch(url, {
+    headers: {
+      Range: 'bytes=0-5132288',
+    },
+  });
+  const data = await response.text();
+  return data;
+}
+
+export async function parseCsv(file: string): Promise<any> {
+  return new Promise((resolve, reject) => {
+    Papa.parse(file, {
+      header: true,
+      dynamicTyping: true,
+      skipEmptyLines: true,
+      transform: (value: string): string => {
+        return value.trim();
+      },
+      complete: (results: any) => {
+        return resolve(results);
+      },
+      error: (error: any) => {
+        return reject(error);
+      },
+    });
+  });
+}
+
+const Spinning = () => {
+  return (
+    <div role="status w-fit mx-auto">
+      <svg
+        aria-hidden="true"
+        className="w-8 h-8 mr-2 text-gray-200 animate-spin dark:text-gray-600 fill-emerald-600"
+        viewBox="0 0 100 101"
+        fill="none"
+        xmlns="http://www.w3.org/2000/svg"
+      >
+        <path
+          d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
+          fill="currentColor"
+        />
+        <path
+          d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
+          fill="currentFill"
+        />
+      </svg>
+      <span className="sr-only">Loading...</span>
+    </div>
+  );
+};
+
+export const Table: React.FC<{ url: string }> = ({ url }) => {
+  return (
+    // Provide the client to your App
+    <QueryClientProvider client={queryClient}>
+      <TableInner url={url} />
+    </QueryClientProvider>
+  );
+};
+
+const TableInner: React.FC<{ url: string }> = ({ url }) => {
+  const { data: csvString, isLoading: isDownloadingCSV } = useQuery(
+    ['dataCsv', url],
+    () => getCsv(url)
+  );
+  const { data: parsedData, isLoading: isParsing } = useQuery(
+    ['dataPreview', csvString],
+    () => parseCsv(csvString),
+    { enabled: !!csvString }
+  );
+  if (isParsing || isDownloadingCSV)
+    <div className="w-full">
+      <Spinning />
+    </div>;
+  if (parsedData)
+    return (
+      <div className="h-[500px] overflow-scroll">
+        <Grid data={parsedData.data} />
+      </div>
+    );
+};

examples/openspending/components/_shared/Layout.tsx

@@ -0,0 +1,10 @@
+import { Header } from '../Header';
+
+export default function Layout({ children }) {
+  return (
+    <div className="bg-white min-h-screen">
+      <Header />
+      {children}
+    </div>
+  );
+}

examples/openspending/datasets.json

@@ -1,27 +0,0 @@
-[
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "mongolia-budget-2016-2017"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "gb-country-regional-analysis"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "berlin-berlin"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "state-of-minas-gerais-brazil-planned-budget"
-  },
-  {
-    "owner": "os-data",
-    "branch": "main",
-    "name": "wesel"
-  }
-]

examples/openspending/lib/datapackage.interface.ts

@@ -98,6 +98,7 @@ export interface TabularDataResource {
   key?: string;
   path?: string;
   size?: number;
+  bytes?: number;
 }
 
 export interface Field {

examples/openspending/lib/loader.ts

@@ -5,13 +5,14 @@ export function loadDataPackage(datapackage: FiscalDataPackage, repo): Project {
   return {
     name: datapackage.name,
     title: datapackage.title,
+    description: datapackage.description || null,
     owner: {
       name: repo.owner.login,
       logo: repo.owner.avatar_url,
       // TODO: make this title work
       title: repo.owner.login,
     },
-    repo: { name: repo, full_name: repo.full_name },
+    repo: { name: repo.name, full_name: repo.full_name, url: repo.html_url },
     files: datapackage.resources,
     author: datapackage.author ? datapackage.author : null,
     cityCode: datapackage.cityCode ? datapackage.cityCode : null,

examples/openspending/lib/project.interface.ts

@@ -5,10 +5,11 @@ import {
 
 export interface Project {
   owner: { name: string; logo?: string; title?: string }; // Info about the owner of the data repo
-  repo: { name: string; full_name: string }; // Info about the the data repo
+  repo: { name: string; full_name: string; url: string }; // Info about the the data repo
   files: TabularDataResource[];
   name: string;
   title?: string;
+  description?: string;
   author?: string;
   cityCode?: string;
   countryCode?: string;

examples/openspending/lib/project.ts

@@ -13,8 +13,7 @@ export interface GithubProject {
 export async function getProjectReadme(
   owner: string,
   repo: string,
-  branch: string,
+  branch: string = 'main',
-  readme: string,
   github_pat?: string
 ) {
   const octokit = new Octokit({ auth: github_pat });
@@ -22,7 +21,7 @@ export async function getProjectReadme(
     const response = await octokit.rest.repos.getContent({
       owner,
       repo,
-      path: readme,
+      path: 'README.md',
       ref: branch,
     });
     const data = response.data as { content?: string };
@@ -125,7 +124,6 @@ export async function getProject(project: GithubProject, github_pat?: string) {
     project.owner,
     project.repo,
     project.branch,
-    project.readme,
     github_pat
   );
 
@@ -185,8 +183,43 @@ export async function getProjectDataPackage(
     }
     const decodedContent = Buffer.from(fileContent, 'base64').toString();
     const datapackage = JSON.parse(decodedContent);
-    return {...datapackage, repo };
+
+    return { ...datapackage, repo };
   } catch (error) {
     return null;
   }
 }
+
+export async function getAllProjectsFromOrg(
+  org: string,
+  branch?: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  const repos = await octokit.rest.repos.listForOrg({
+    org,
+    type: 'public',
+    per_page: 100,
+  });
+  let failedProjects = [];
+  const projects = await Promise.all(
+    repos.data.map(async (_repo) => {
+      const project = await getProjectDataPackage(
+        org,
+        _repo.name,
+        branch ? branch : 'main',
+        github_pat
+      );
+      if (!project) {
+        failedProjects.push(_repo.name);
+        return null;
+      }
+
+      return { datapackage: project, repo: _repo };
+    })
+  );
+  return {
+    results: projects.filter((item) => item !== null),
+    failed: failedProjects,
+  };
+}

examples/openspending/orgs.json

@@ -0,0 +1,3 @@
+[
+ "os-data"
+]

examples/openspending/package.json

@@ -10,6 +10,8 @@
     "test": "vitest"
   },
   "dependencies": {
+    "@githubocto/flat-ui": "^0.14.1",
+    "@heroicons/react": "^2.0.18",
     "@octokit/plugin-throttling": "^5.2.2",
     "@types/flexsearch": "^0.7.3",
     "@types/node": "18.16.0",
@@ -23,17 +25,20 @@
     "next": "13.3.1",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
+    "papaparse": "^5.4.1",
     "prettier": "^2.8.8",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-hook-form": "^7.43.9",
     "react-markdown": "^8.0.7",
+    "react-query": "^3.39.3",
     "react-timeago": "^7.1.0",
     "remark-gfm": "^3.0.1",
     "typescript": "5.0.4"
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",
+    "@types/papaparse": "^5.3.7",
     "autoprefixer": "^10.4.14",
     "postcss": "^8.4.23",
     "tailwindcss": "^3.3.1",

examples/openspending/pages/@org/[org]/[...path].tsx

@@ -1,126 +0,0 @@
-import { NextSeo } from 'next-seo';
-import { promises as fs } from 'fs';
-import path from 'path';
-import getConfig from 'next/config';
-import { getProject, GithubProject } from '../../../lib/octokit';
-import ReactMarkdown from 'react-markdown';
-import remarkGfm from 'remark-gfm';
-import Breadcrumbs from '../../../components/_shared/Breadcrumbs';
-
-export default function ProjectPage({ project }) {
-  const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`
-
-  return (
-    <>
-      <NextSeo title={`${repoId}${project.base_path !== '/' ? '/' + project.base_path : ''} - GitHub Datasets`} />
-      <main className="prose mx-auto my-8">
-        <Breadcrumbs links={[{ title: repoId, href: "" }]} />
-        <h1 className="mb-0 mt-16">{project.repo_config.name || repoId}</h1>
-        <p className='mb-8'><span className='font-semibold'>Repository:</span> <a target="_blank" href={project.html_url}>{project.html_url}</a></p>
-
-        <h2 className="mb-0 mt-10">Files</h2>
-        <div className="inline-block min-w-full py-2 align-middle">
-          <table className="min-w-full divide-y divide-gray-300">
-            <thead>
-              <tr>
-                <th
-                  scope="col"
-                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                >
-                  Name
-                </th>
-                <th
-                  scope="col"
-                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                >
-                  Size
-                </th>
-              </tr>
-            </thead>
-            <tbody className="divide-y divide-gray-200">
-              {project.files?.map((file) => (
-                <tr key={file.download_url}>
-                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                    <a href={file.download_url}>{file.name}</a>
-                  </td>
-                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
-                    {file.size} Bytes
-                  </td>
-                </tr>
-              ))}
-            </tbody>
-          </table>
-        </div>
-
-        {project.readmeContent && <>
-          <hr />
-
-          <h2 className='uppercase font-black'>Readme</h2>
-          <ReactMarkdown remarkPlugins={[remarkGfm]}>
-            {project.readmeContent}
-          </ReactMarkdown>
-        </>}
-      </main>
-    </>
-  );
-}
-
-// Generates `/posts/1` and `/posts/2`
-export async function getStaticPaths() {
-  const jsonDirectory = path.join(
-    process.cwd(),
-    'datasets.json'
-  );
-  const repos = await fs.readFile(jsonDirectory, 'utf8');
-
-  return {
-    paths: JSON.parse(repos).map((repo) => {
-      const projectPath =
-        repo.readme && repo.readme.split('/').length > 1
-          ? repo.readme.split('/').slice(0, -1)
-          : null;
-      let path = [repo.name];
-      if (projectPath) {
-        projectPath.forEach((element) => {
-          path.push(element);
-        });
-      }
-      return {
-        params: { org: repo.owner, path },
-      };
-    }),
-    fallback: false, // can also be true or 'blocking'
-  };
-}
-
-export async function getStaticProps({ params }) {
-  const jsonDirectory = path.join(
-    process.cwd(),
-    'datasets.json'
-  );
-  const reposFile = await fs.readFile(jsonDirectory, 'utf8');
-  const repos: GithubProject[] = JSON.parse(reposFile);
-  const repo = repos.find((_repo) => {
-    const projectPath =
-      _repo.readme && _repo.readme.split('/').length > 1
-        ? _repo.readme.split('/').slice(0, -1)
-        : null;
-    let path = [_repo.name];
-    if (projectPath) {
-      projectPath.forEach((element) => {
-        path.push(element);
-      });
-    }
-    return (
-      _repo.owner == params.org &&
-      JSON.stringify(path) === JSON.stringify(params.path)
-    );
-  });
-  const github_pat = getConfig().serverRuntimeConfig.github_pat;
-  const project = await getProject(repo, github_pat);
-  return {
-    props: {
-      project: { ...project, repo_config: repo },
-    },
-  };
-}

examples/openspending/pages/@org/[org]/[project].tsx

@@ -0,0 +1,234 @@
+import { NextSeo } from 'next-seo';
+import getConfig from 'next/config';
+import {
+  getAllProjectsFromOrg,
+  getProjectDataPackage,
+  getProjectMetadata,
+  getProjectReadme,
+} from '../../../lib/project';
+import ReactMarkdown from 'react-markdown';
+import remarkGfm from 'remark-gfm';
+import { loadDataPackage } from '../../../lib/loader';
+import Layout from '../../../components/_shared/Layout';
+import Link from 'next/link';
+import { Project } from '../../../lib/project.interface';
+import ExternalLinkIcon from '../../../components/icons/ExternalLinkIcon';
+
+export default function ProjectPage({
+  project,
+  readme,
+}: {
+  project: Project;
+  readme: string;
+}) {
+
+  //  Get description from datapackage or  calculate
+  //  excerpt from README by getting all the content
+  //  up to the first dot.
+  const description =
+    project.description || (readme && readme.slice(0, readme.indexOf('.') + 1));
+
+  return (
+    <Layout>
+      <NextSeo title={`${project.title} - OpenSpending`} />
+      <main className="prose mx-auto my-8">
+        <h1 className="mb-1 mt-16">{project.title || project.name}</h1>
+        <Link target="_blank" href={project.repo.url}>
+          @{project.repo.full_name}
+        </Link>
+
+        {description && (
+          <div className="inline-block min-w-full py-2 align-middle mt-5">
+            {description}
+          </div>
+        )}
+
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="min-w-full divide-y divide-gray-300">
+            <thead>
+              <tr>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Name
+                </th>
+                {project.datapackage.countryCode && (
+                  <th
+                    scope="col"
+                    className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                  >
+                    Country
+                  </th>
+                )}
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Metadata
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              <tr>
+                <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                  {project.name}
+                </td>
+                {project.datapackage.countryCode && (
+                  <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                    {project.datapackage.countryCode}
+                  </td>
+                )}
+
+                <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                  <Link
+                    // TODO: this link may be incorrect for some datasets
+                    href={`https://github.com/${project.owner.name}/${project.repo.name}/blob/main/datapackage.json`}
+                    target="_blank"
+                    className="flex items-center hover:text-gray-700"
+                  >
+                    datapackage.json <ExternalLinkIcon className="ml-1" />
+                  </Link>
+                </td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+
+        <h3 className="mb-1 mt-10">Data files</h3>
+        <p>
+          This dataset contains {project.files.length} file
+          {project.files.length != 1 ? '' : 's'}
+        </p>
+        <div className="inline-block min-w-full py-2 align-middle">
+          <table className="mt-0 min-w-full divide-y divide-gray-300">
+            <thead>
+              <tr>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Name
+                </th>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Format
+                </th>
+                <th
+                  scope="col"
+                  className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
+                >
+                  Size
+                </th>
+              </tr>
+            </thead>
+            <tbody className="divide-y divide-gray-200">
+              {project.files?.map((file) => {
+                let size: number | string = file.size;
+
+                if (!size) {
+                  if (file.bytes) {
+                    if (file.bytes > 1000000) {
+                      size = (file.bytes / 1000000).toFixed(2) + ' MB';
+                    } else {
+                      size = (file.bytes / 1000).toFixed(2) + ' kB';
+                    }
+                  }
+                }
+
+                return (
+                  <tr key={file.name}>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {file.name}
+                    </td>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {file.format}
+                    </td>
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      {size}
+                    </td>
+
+                    <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
+                      <Link
+                        target="_blank"
+                        href={
+                          file.path.startsWith('http')
+                            ? file.path
+                            : `https://raw.githubusercontent.com/${project.owner.name}/${project.repo.name}/main/${file.path}`
+                        }
+                      >
+                        Download
+                      </Link>
+                    </td>
+                  </tr>
+                );
+              })}
+            </tbody>
+          </table>
+        </div>
+
+        {readme && (
+          <>
+            <hr />
+
+            <h2 className="uppercase font-black">Readme</h2>
+            <ReactMarkdown remarkPlugins={[remarkGfm]}>{readme}</ReactMarkdown>
+          </>
+        )}
+      </main>
+    </Layout>
+  );
+}
+
+// Generates `/posts/1` and `/posts/2`
+export async function getStaticPaths() {
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+
+  const allProjects = await getAllProjectsFromOrg(
+    'os-data',
+    'main',
+    github_pat
+  );
+
+  console.log(allProjects)
+  const paths = allProjects.results.map((project) => ({
+    params: {
+      // TODO: dynamize the org
+      org: 'os-data',
+      project: project.repo.name,
+    },
+  }));
+
+  return {
+    paths,
+    fallback: false, // can also be true or 'blocking'
+  };
+}
+
+export async function getStaticProps({ params }) {
+  const { org: orgName, project: projectName } = params;
+
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+  const datapackage = await getProjectDataPackage(
+    orgName,
+    projectName,
+    'main',
+    github_pat
+  );
+
+  const repo = await getProjectMetadata(orgName, projectName, github_pat);
+
+  const project = loadDataPackage(datapackage, repo);
+
+  // TODO: should this be moved to the loader?
+  const readme = await getProjectReadme(orgName, projectName, 'main', github_pat);
+
+  return {
+    props: {
+      project,
+      readme,
+    },
+  };
+}

examples/openspending/pages/_app.tsx

@@ -1,13 +1,11 @@
 import { AppProps } from 'next/app';
-import Head from 'next/head';
 import './styles.css';
+import { NextSeo } from 'next-seo';
 
 function CustomApp({ Component, pageProps }: AppProps) {
   return (
     <>
-      <Head>
+      <NextSeo title="OpenSpending" />
-        <title>GitHub Datasets</title>
-      </Head>
       <main className="app">
         <Component {...pageProps} />
       </main>

examples/openspending/pages/index.tsx

@@ -1,65 +1,58 @@
-import { promises as fs } from 'fs';
+import { getAllProjectsFromOrg } from '../lib/project';
-import path from 'path';
-import {
-  GithubProject,
-  getProjectDataPackage,
-  getProjectMetadata,
-} from '../lib/octokit';
 import getConfig from 'next/config';
-import ExternalLinkIcon from '../components/icons/ExternalLinkIcon';
-import TimeAgo from 'react-timeago';
-import Link from 'next/link';
 import { Hero } from '../components/Hero';
-import { Header } from '../components/Header';
 import { Container } from '../components/Container';
 import { FiscalDataPackage } from '../lib/datapackage.interface';
 import { loadDataPackage } from '../lib/loader';
 import DatasetsSearch from '../components/DatasetsSearch';
+import Layout from '../components/_shared/Layout';
 
 export async function getStaticProps() {
-  const jsonDirectory = path.join(process.cwd(), '/datasets.json');
+  //  TODO: support other orgs
-  const repos = await fs.readFile(jsonDirectory, 'utf8');
+  // const orgsListPath = path.join(process.cwd(), '/orgs.json');
-  const github_pat = getConfig().serverRuntimeConfig.github_pat;
+  // const orgs = await fs.readFile(orgsListPath, 'utf8');
-  const datapackages = await Promise.all(
-    JSON.parse(repos).map(async (_repo: GithubProject) => {
-      const datapackage = await getProjectDataPackage(
-        _repo.owner,
-        _repo.name,
-        'main',
-        github_pat
-      );
-      const repo = await getProjectMetadata(
-        _repo.owner,
-        _repo.name,
-        github_pat
-      );
 
-      return {
+  const github_pat = getConfig().serverRuntimeConfig.github_pat;
-        datapackage,
+
-        repo,
+  const allProjects = await getAllProjectsFromOrg(
-      };
+    'os-data',
-    })
+    'main',
+    github_pat
   );
 
-  const projects = datapackages.map(
+  const projects = allProjects.results.map(
     (item: { datapackage: FiscalDataPackage & { repo: string }; repo: any }) =>
       loadDataPackage(item.datapackage, item.repo)
   );
 
+  const availableCountries = projects
+    .map((item) => item.countryCode)
+    .filter((v) => v) //  Filters false values
+    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
+    //  TODO: title should be the full name
+    .map((code) => ({ code, title: code }));
+
   return {
     props: {
       projects: JSON.stringify(projects),
+      availableCountries,
     },
   };
 }
 
-export function Datasets({ projects }) {
+export function Home({ projects, availableCountries }) {
   projects = JSON.parse(projects);
 
   return (
-    <div className="bg-white min-h-screen">
+    <Layout>
-      <Header />
+      <Hero
-      <Hero />
+        countriesCount={availableCountries.length}
+        datasetsCount={projects.length}
+        filesCount={projects.reduce(
+          (partialSum, a) => partialSum + a.files.length,
+          0
+        )}
+      />
       <section className="py-20 sm:py-32">
         <Container>
           <div className="mx-auto max-w-2xl lg:mx-0">
@@ -74,12 +67,15 @@ export function Datasets({ projects }) {
             </p>
           </div>
           <div className="mt-10">
-            <DatasetsSearch datasets={projects} />
+            <DatasetsSearch
+              datasets={projects}
+              availableCountries={availableCountries}
+            />
           </div>
         </Container>
       </section>
-    </div>
+    </Layout>
   );
 }
 
-export default Datasets;
+export default Home;