[examples/openspending,home][m]: add min date and max date facets

- Also added an indexing example

examples/openspending/__tests__/os-data.test.ts

@@ -0,0 +1,45 @@
+import { Octokit } from 'octokit';
+import { assert, expect, test } from 'vitest'
+import { getProjectDataPackage } from '../lib/octokit';
+
+export async function getAllDataPackagesFromOrg(
+  org: string,
+  branch?: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  const repos = await octokit.rest.repos.listForOrg({ org, type: 'public', per_page: 100 });
+  let failedDataPackages = [];
+  const datapackages = await Promise.all(
+    repos.data.map(async (_repo) => {
+      const datapackage = await getProjectDataPackage(
+        org,
+        _repo.name,
+        branch ? branch : 'main',
+        github_pat
+      );
+      if (!datapackage) {
+        failedDataPackages.push(_repo.name)
+        return null
+      };
+      return {...datapackage, repo: _repo.name};
+    })
+  );
+  return {
+    datapackages: datapackages.filter((item) => item !== null),
+    failedDataPackages,
+  };
+}
+
+test('Test OS-Data', async () => {
+  const repos = await getAllDataPackagesFromOrg('os-data', 'main', process.env.VITE_GITHUB_PAT)
+  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
+  expect(repos.failedDataPackages.length).toBe(0)
+}, {timeout: 100000})
+
+test('Test Gift-Data', async () => {
+  const repos = await getAllDataPackagesFromOrg('gift-data', 'main', process.env.VITE_GITHUB_PAT)
+  if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages)
+  expect(repos.failedDataPackages.length).toBe(0)
+}, {timeout: 100000})
+

examples/openspending/components/Button.tsx

@@ -1,15 +1,15 @@
-import Link from 'next/link'
+import Link from 'next/link';
-import clsx from 'clsx'
+import clsx from 'clsx';
 
-export function Button({ href, className = "", ...props }) {
+export function Button({ href, className = '', ...props }) {
   className = clsx(
     'inline-flex justify-center rounded-2xl bg-emerald-600 p-4 text-base font-semibold text-white hover:bg-emerald-500 focus:outline-none focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-emerald-500 active:text-white/70',
     className
-  )
+  );
 
   return href ? (
-    <Link href={href} className={className} {...props} />
+    <Link scroll={false} href={href} className={className} {...props} />
   ) : (
     <button className={className} {...props} />
-  )
+  );
 }

examples/openspending/components/DatasetCard.tsx

@@ -0,0 +1,76 @@
+import Link from 'next/link';
+import { Project } from '../lib/project.interface';
+import ExternalLinkIcon from './icons/ExternalLinkIcon';
+
+export default function DatasetCard({ dataset }: { dataset: Project }) {
+  return (
+    <div
+      key={dataset.name}
+      className="overflow-hidden rounded-xl border border-gray-200"
+    >
+      <Link
+        href=""
+        className="flex items-center gap-x-4 border-b border-gray-900/5 bg-gray-50 p-6"
+      >
+        <img
+          src={dataset.owner.logo || '/assets/org-icon.svg'}
+          alt={dataset.owner.name}
+          className="h-12 w-12 flex-none rounded-lg bg-white object-cover ring-1 ring-gray-900/10 p-2"
+        />
+        <div className="text-sm font-medium leading-6">
+          <div className="text-gray-900 line-clamp-1">{dataset.title}</div>
+          <div className="text-gray-500 line-clamp-1">
+            {dataset.owner.title}
+          </div>
+        </div>
+      </Link>
+      <dl className="-my-3 divide-y divide-gray-100 px-6 py-4 text-sm leading-6">
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Name</dt>
+          <dd className="flex items-start gap-x-2">
+            <div className="font-medium text-gray-900 line-clamp-1">
+              {dataset.name}
+            </div>
+          </dd>
+        </div>
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Country</dt>
+          <dd className="flex items-start gap-x-2">
+            <div className="font-medium text-gray-900">
+              {dataset.countryCode}
+            </div>
+          </dd>
+        </div>
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Fiscal Period</dt>
+          <dd className="text-gray-700">
+            {dataset.fiscalPeriod?.start &&
+              new Date(dataset.fiscalPeriod.start).getFullYear()}
+            {dataset.fiscalPeriod?.end &&
+              dataset.fiscalPeriod?.start !== dataset.fiscalPeriod?.end && (
+                <>
+                  {' - '}
+                  {new Date(dataset.fiscalPeriod.end).getFullYear()}
+                </>
+              )}
+          </dd>
+        </div>
+        <div className="flex justify-between gap-x-4 py-3">
+          <dt className="text-gray-500">Metadata</dt>
+          <dd className="flex items-start gap-x-2">
+            <div className="font-medium text-gray-900">
+              <Link
+                // TODO: where do we get the info needed for this link?
+                href=""
+                target="_blank"
+                className="flex items-center hover:text-gray-700"
+              >
+                datapackage.json <ExternalLinkIcon className="ml-1" />
+              </Link>
+            </div>
+          </dd>
+        </div>
+      </dl>
+    </div>
+  );
+}

examples/openspending/components/DatasetsGrid.tsx

@@ -0,0 +1,19 @@
+import { Project } from '../lib/project.interface';
+import DatasetCard from './DatasetCard';
+
+export default function DatasetsGrid({ datasets }: { datasets: Project[] }) {
+  return (
+    <ul
+      className="grid gap-x-6 gap-y-8 grid-cols-1 sm:grid-cols-2 md:grid-cols-3"
+      role="list"
+    >
+      {datasets.map((dataset, idx) => {
+        return (
+          <li key={`datasets-grid-item-${idx}`}>
+            <DatasetCard dataset={dataset} />
+          </li>
+        );
+      })}
+    </ul>
+  );
+}

examples/openspending/components/DatasetsSearch.tsx

@@ -0,0 +1,163 @@
+import { useForm } from 'react-hook-form';
+import DatasetsGrid from './DatasetsGrid';
+import { Project } from '../lib/project.interface';
+import { Index } from 'flexsearch';
+
+export default function DatasetsSearch({ datasets }: { datasets: Project[] }) {
+  const index = new Index({ tokenize: 'full' });
+  datasets.forEach((dataset: Project) =>
+    index.add(
+      dataset.name,
+      `${dataset.repo} ${dataset.name} ${dataset.title} ${dataset.author} ${dataset.title} ${dataset.cityCode} ${dataset.fiscalPeriod?.start} ${dataset.fiscalPeriod?.end}`
+    )
+  );
+
+  const { register, watch, handleSubmit, reset, resetField } = useForm({
+    defaultValues: {
+      searchTerm: '',
+      country: '',
+      minDate: '',
+      maxDate: '',
+    },
+  });
+
+  const allCountries = datasets
+    .map((item) => item.countryCode)
+    .filter((v) => v) //  Filters false values
+    .filter((v, i, a) => a.indexOf(v) === i) //  Remove duplicates
+    //  TODO: title should be the full name
+    .map((code) => ({ code, title: code }));
+
+  return (
+    <>
+      <div className="flex flex-col gap-3 sm:flex-row">
+        <div className="min-w-0 flex-auto">
+          <br />
+          <div className="relative">
+            <input
+              placeholder="Search datasets"
+              aria-label="Search datasets"
+              {...register('searchTerm')}
+              className="h-[3em] relative w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+            {watch().searchTerm !== '' && (
+              <button
+                onClick={() => resetField('searchTerm')}
+                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
+              >
+                <CloseIcon />
+              </button>
+            )}
+          </div>
+        </div>
+        <div className="sm:basis-1/6">
+          {/* TODO: nicer select e.g. headlessui example */}
+          <label className="text-sm text-gray-600 font-medium">Country</label>
+          <select
+            className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            {...register('country')}
+          >
+            <option value="">All</option>
+            {allCountries.map((country) => {
+              return (
+                <option key={country.code} value={country.code}>
+                  {country.title}
+                </option>
+              );
+            })}
+          </select>
+        </div>
+        <div className="sm:basis-1/6">
+          <label className="text-sm text-gray-600 font-medium">Min. date</label>
+          <div className="relative">
+            <input
+              aria-label="Min. date"
+              type="date"
+              {...register('minDate')}
+              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+            {watch().minDate !== '' && (
+              <button
+                onClick={() => resetField('minDate')}
+                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
+              >
+                <CloseIcon />
+              </button>
+            )}
+          </div>
+        </div>
+        <div className="sm:basis-1/6">
+          <label className="text-sm text-gray-600 font-medium">Max. date</label>
+          <div className="relative">
+            <input
+              aria-label="Max. date"
+              type="date"
+              {...register('maxDate')}
+              className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
+            />
+            {watch().maxDate !== '' && (
+              <button
+                onClick={() => resetField('maxDate')}
+                className="absolute right-3 top-1/2 -translate-y-1/2 text-gray-500"
+              >
+                <CloseIcon />
+              </button>
+            )}
+          </div>
+        </div>
+      </div>
+      <div className="min-w-full mt-10 align-middle">
+        <DatasetsGrid
+          datasets={datasets
+            .filter((dataset: Project) =>
+              watch().searchTerm && watch().searchTerm !== ''
+                ? index.search(watch().searchTerm).includes(dataset.name)
+                : true
+            )
+            .filter((dataset) =>
+              watch().country && watch().country !== ''
+                ? dataset.countryCode === watch().country
+                : true
+            )
+            // TODO: Does that really makes sense?
+            // What if the fiscalPeriod is 2015-2017 and inputs are
+            // set to 2015-2016. It's going to be filtered out but
+            // it shouldn't.
+            .filter((dataset) =>
+              watch().minDate && watch().minDate !== ''
+                ? dataset.fiscalPeriod?.start >= watch().minDate
+                : true
+            )
+            .filter((dataset) =>
+              watch().maxDate && watch().maxDate !== ''
+                ? dataset.fiscalPeriod?.end <= watch().maxDate
+                : true
+            )}
+        />
+      </div>
+    </>
+  );
+}
+
+const CloseIcon = () => {
+  return (
+    <svg
+      width={20}
+      height={20}
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g id="Menu / Close_MD">
+        <path
+          id="Vector"
+          d="M18 18L12 12M12 12L6 6M12 12L18 6M12 12L6 18"
+          stroke="currentColor"
+          stroke-width="2"
+          stroke-linecap="round"
+          stroke-linejoin="round"
+        />
+      </g>
+    </svg>
+  );
+};

examples/openspending/components/Header.tsx

@@ -46,7 +46,6 @@ export function Header() {
             </li>))}
         </ul>
         <div className="hidden sm:mt-10 sm:flex lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
-          <Button href="#">View on GitHub</Button>
         </div>
       </Container>
     </header >

examples/openspending/components/Hero.tsx

@@ -23,8 +23,8 @@ export function Hero() {
               fiscal data in the public sphere.
             </p>
           </div>
-          <Button href="#" className="mt-10 w-full sm:hidden">
+          <Button href="#datasets" className="mt-10">
-            View on GitHub
+            Search datasets
           </Button>
           <dl className="mt-10 grid grid-cols-2 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
             {[

examples/openspending/datasets.json

@@ -2,24 +2,26 @@
   {
     "owner": "os-data",
     "branch": "main",
-    "repo": "mongolia-budget-2016-2017",
+    "name": "mongolia-budget-2016-2017"
-    "files": [
-      "data/mongolia-2017.csv",
-      "data/mongolia-2017__2017.csv"
-    ]
   },
   {
     "owner": "os-data",
     "branch": "main",
-    "repo": "gb-country-regional-analysis",
+    "name": "gb-country-regional-analysis"
-    "files": [
+  },
-      "data/cofog.csv",
+  {
-      "data/cofog_dejargonise.csv",
+    "owner": "os-data",
-      "data/cra.csv",
+    "branch": "main",
-      "data/departments.csv",
+    "name": "berlin-berlin"
-      "data/nuts_pop.csv",
+  },
-      "data/pogs.csv"
+  {
-    ],
+    "owner": "os-data",
-    "readme": "README.md"
+    "branch": "main",
+    "name": "state-of-minas-gerais-brazil-planned-budget"
+  },
+  {
+    "owner": "os-data",
+    "branch": "main",
+    "name": "wesel"
   }
 ]

examples/openspending/lib/datapackage.interface.ts

@@ -0,0 +1,288 @@
+/**
+ * Fiscal Data Package is a simple specification for data access and delivery of fiscal data.
+ */
+export type FiscalDataPackage = TabularDataPackage & {
+  countryCode?: ISO31661Alpha2CountryCode
+  regionCode?: string
+  cityCode?: string
+  author?: string
+  readme?: string
+  granularity?: GranularityOfResources
+  fiscalPeriod?: FiscalPeriodForTheBudget
+  [k: string]: unknown
+}
+/**
+ * The profile of this descriptor.
+ */
+export type Profile = "tabular-data-package"
+/**
+ * An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.
+ */
+export type Name = string
+/**
+ * A property reserved for globally unique identifiers. Examples of identifiers that are unique include UUIDs and DOIs.
+ */
+export type ID = string
+/**
+ * A human-readable title.
+ */
+export type Title = string
+/**
+ * A text description. Markdown is encouraged.
+ */
+export type Description = string
+/**
+ * The home on the web that is related to this data package.
+ */
+export type HomePage = string
+/**
+ * The datetime on which this descriptor was created.
+ */
+export type Created = string
+/**
+ * The contributors to this descriptor.
+ */
+export type Contributors = [Contributor, ...Contributor[]]
+/**
+ * A human-readable title.
+ */
+export type Title1 = string
+/**
+ * A fully qualified URL, or a POSIX file path.
+ */
+export type Path = string
+/**
+ * An email address.
+ */
+export type Email = string
+/**
+ * An organizational affiliation for this contributor.
+ */
+export type Organization = string
+/**
+ * A list of keywords that describe this package.
+ */
+export type Keywords = [string, ...string[]]
+/**
+ * A image to represent this package.
+ */
+export type Image = string
+/**
+ * The license(s) under which this package is published.
+ */
+export type Licenses = [License, ...License[]]
+/**
+ * A license for this descriptor.
+ */
+export type License =
+  | {
+      [k: string]: unknown
+    }
+  | {
+      [k: string]: unknown
+    }
+/**
+ * An `array` of Tabular Data Resource objects, each compliant with the [Tabular Data Resource](/tabular-data-resource/) specification.
+ *
+/**
+ * A Tabular Data Resource.
+ */
+export interface TabularDataResource {
+  format?: string;
+  name: string;
+  description?: string;
+  title?: string;
+  schema?: Schema;
+  sample?: any[];
+  profile?: string;
+  key?: string;
+  path?: string;
+  size?: number;
+}
+
+export interface Field {
+  name: string;
+  type: FieldType;
+}
+
+export interface Schema {
+  fields: Field[];
+}
+
+export const OptionsFields = [
+  "any",
+  "array",
+  "boolean",
+  "date",
+  "datetime",
+  "duration",
+  "geojson",
+  "geopoint",
+  "integer",
+  "number",
+  "object",
+  "string",
+  "time",
+  "year",
+  "yearmonth",
+] as const;
+
+type FieldType = typeof OptionsFields[number];
+/**
+ * A human-readable title.
+ */
+export type Title2 = string
+/**
+ * A fully qualified URL, or a POSIX file path.
+ */
+export type Path1 = string
+/**
+ * An email address.
+ */
+export type Email1 = string
+/**
+ * The raw sources for this resource.
+ */
+export type Sources = Source[]
+
+/**
+ * A keyword that represents the direction of the spend, either expenditure or revenue.
+ */
+export type DirectionOfTheSpending = "expenditure" | "revenue"
+/**
+ * A keyword that represents the phase of the data, can be proposed for a budget proposal, approved for an approved budget, adjusted for modified budget or executed for the enacted budget
+ */
+export type BudgetPhase = "proposed" | "approved" | "adjusted" | "executed"
+/**
+ * Either an array of strings corresponding to the name attributes in a set of field objects in the fields array or a single string corresponding to one of these names. The value of primaryKey indicates the primary key or primary keys for the dimension.
+ */
+export type PrimaryKey = string | [string, ...string[]]
+/**
+ * Describes what kind of a dimension it is.
+ */
+export type DimensionType =
+  | "datetime"
+  | "entity"
+  | "classification"
+  | "activity"
+  | "fact"
+  | "location"
+  | "other"
+/**
+ * The type of the classification.
+ */
+export type ClassificationType = "functional" | "administrative" | "economic"
+/**
+ * A valid 2-digit ISO country code (ISO 3166-1 alpha-2), or, an array of valid ISO codes.
+ */
+export type ISO31661Alpha2CountryCode = string | [string, ...string[]]
+/**
+ * A keyword that represents the type of spend data, eiter aggregated or transactional
+ */
+export type GranularityOfResources = "aggregated" | "transactional"
+
+/**
+ * Tabular Data Package
+ */
+export interface TabularDataPackage {
+  profile: Profile
+  name?: Name
+  id?: ID
+  title?: Title
+  description?: Description
+  homepage?: HomePage
+  created?: Created
+  contributors?: Contributors
+  keywords?: Keywords
+  image?: Image
+  licenses?: Licenses
+  resources: TabularDataResource[]
+  sources?: Sources
+  [k: string]: unknown
+}
+/**
+ * A contributor to this descriptor.
+ */
+export interface Contributor {
+  title: Title1
+  path?: Path
+  email?: Email
+  organization?: Organization
+  role?: string
+  [k: string]: unknown
+}
+/**
+ * A source file.
+ */
+export interface Source {
+  title: Title2
+  path?: Path1
+  email?: Email1
+  [k: string]: unknown
+}
+/**
+ * Measures are numerical and correspond to financial amounts in the source data.
+ */
+export interface Measures {
+  [k: string]: Measure
+}
+/**
+ * Measure.
+ *
+ * This interface was referenced by `Measures`'s JSON-Schema definition
+ * via the `patternProperty` "^\w+".
+ */
+export interface Measure {
+  source: string
+  resource?: string
+  currency: string
+  factor?: number
+  direction?: DirectionOfTheSpending
+  phase?: BudgetPhase
+  [k: string]: unknown
+}
+/**
+ * Dimensions are groups of related fields. Dimensions cover all items other than the measure.
+ */
+export interface Dimensions {
+  [k: string]: Dimension
+}
+/**
+ * Dimension.
+ *
+ * This interface was referenced by `Dimensions`'s JSON-Schema definition
+ * via the `patternProperty` "^\w+".
+ */
+export interface Dimension {
+  attributes: Attributes
+  primaryKey: PrimaryKey
+  dimensionType?: DimensionType
+  classificationType?: ClassificationType
+  [k: string]: unknown
+}
+/**
+ * Attribute objects that make up the dimension
+ */
+export interface Attributes {
+  /**
+   * This interface was referenced by `Attributes`'s JSON-Schema definition
+   * via the `patternProperty` "^\w+".
+   */
+  [k: string]: {
+    source: string
+    resource?: string
+    constant?: string | number
+    parent?: string
+    labelfor?: string
+    [k: string]: unknown
+  }
+}
+/**
+ * The fiscal period of the dataset
+ */
+export interface FiscalPeriodForTheBudget {
+  start: string
+  end?: string
+  [k: string]: unknown
+}
+

examples/openspending/lib/loader.ts

@@ -0,0 +1,34 @@
+import { FiscalDataPackage } from './datapackage.interface';
+import { Project } from './project.interface';
+
+export function loadDataPackage(datapackage: FiscalDataPackage, repo): Project {
+  return {
+    name: datapackage.name,
+    title: datapackage.title,
+    owner: {
+      name: repo.owner.login,
+      logo: repo.owner.avatar_url,
+      // TODO: make this title work
+      title: repo.owner.login,
+    },
+    repo: { name: repo, full_name: repo.full_name },
+    files: datapackage.resources,
+    author: datapackage.author ? datapackage.author : null,
+    cityCode: datapackage.cityCode ? datapackage.cityCode : null,
+    countryCode: datapackage.countryCode
+      ? (datapackage.countryCode as string)
+      : null,
+    fiscalPeriod: datapackage.fiscalPeriod
+      ? {
+          start: datapackage.fiscalPeriod.start
+            ? datapackage.fiscalPeriod.start
+            : null,
+          end: datapackage.fiscalPeriod.end
+            ? datapackage.fiscalPeriod.end
+            : null,
+        }
+      : null,
+    readme: datapackage.readme ? datapackage.readme : '',
+    datapackage,
+  };
+}

examples/openspending/lib/octokit.ts

@@ -140,7 +140,8 @@ export async function getProject(project: GithubProject, github_pat?: string) {
     return null;
   }
 
-  let projectBase = "", last_updated = "";
+  let projectBase = '',
+    last_updated = '';
   if (projectReadme) {
     projectBase =
       project.readme.split('/').length > 1
@@ -162,3 +163,30 @@ export async function getProject(project: GithubProject, github_pat?: string) {
     base_path: projectBase,
   };
 }
+
+export async function getProjectDataPackage(
+  owner: string,
+  repo: string,
+  branch: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.getContent({
+      owner,
+      repo,
+      path: 'datapackage.json',
+      ref: branch,
+    });
+    const data = response.data as { content?: string };
+    const fileContent = data.content ? data.content : '';
+    if (fileContent === '') {
+      return null;
+    }
+    const decodedContent = Buffer.from(fileContent, 'base64').toString();
+    const datapackage = JSON.parse(decodedContent);
+    return {...datapackage, repo };
+  } catch (error) {
+    return null;
+  }
+}

examples/openspending/lib/project.interface.ts

@@ -0,0 +1,21 @@
+import {
+  FiscalDataPackage,
+  TabularDataResource,
+} from './datapackage.interface';
+
+export interface Project {
+  owner: { name: string; logo?: string; title?: string }; // Info about the owner of the data repo
+  repo: { name: string; full_name: string }; // Info about the the data repo
+  files: TabularDataResource[];
+  name: string;
+  title?: string;
+  author?: string;
+  cityCode?: string;
+  countryCode?: string;
+  fiscalPeriod?: {
+    start: string;
+    end: string;
+  };
+  readme?: string;
+  datapackage: FiscalDataPackage;
+}

examples/openspending/package.json

@@ -6,21 +6,27 @@
     "dev": "next dev",
     "build": "next build",
     "start": "next start",
-    "lint": "next lint"
+    "lint": "next lint",
+    "test": "vitest"
   },
   "dependencies": {
+    "@octokit/plugin-throttling": "^5.2.2",
+    "@types/flexsearch": "^0.7.3",
     "@types/node": "18.16.0",
     "@types/react": "18.0.38",
     "@types/react-dom": "18.0.11",
+    "@vitejs/plugin-react": "^4.0.0",
     "clsx": "^1.2.1",
     "eslint": "8.39.0",
     "eslint-config-next": "13.3.1",
+    "flexsearch": "0.7.21",
     "next": "13.3.1",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
     "prettier": "^2.8.8",
     "react": "18.2.0",
     "react-dom": "18.2.0",
+    "react-hook-form": "^7.43.9",
     "react-markdown": "^8.0.7",
     "react-timeago": "^7.1.0",
     "remark-gfm": "^3.0.1",
@@ -30,6 +36,7 @@
     "@tailwindcss/typography": "^0.5.9",
     "autoprefixer": "^10.4.14",
     "postcss": "^8.4.23",
-    "tailwindcss": "^3.3.1"
+    "tailwindcss": "^3.3.1",
+    "vitest": "^0.31.0"
   }
 }

examples/openspending/pages/@org/[org]/[...path].tsx

@@ -38,7 +38,7 @@ export default function ProjectPage({ project }) {
               </tr>
             </thead>
             <tbody className="divide-y divide-gray-200">
-              {project.files.map((file) => (
+              {project.files?.map((file) => (
                 <tr key={file.download_url}>
                   <td className="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
                     <a href={file.download_url}>{file.name}</a>
@@ -79,7 +79,7 @@ export async function getStaticPaths() {
         repo.readme && repo.readme.split('/').length > 1
           ? repo.readme.split('/').slice(0, -1)
           : null;
-      let path = [repo.repo];
+      let path = [repo.name];
       if (projectPath) {
         projectPath.forEach((element) => {
           path.push(element);
@@ -105,7 +105,7 @@ export async function getStaticProps({ params }) {
       _repo.readme && _repo.readme.split('/').length > 1
         ? _repo.readme.split('/').slice(0, -1)
         : null;
-    let path = [_repo.repo];
+    let path = [_repo.name];
     if (projectPath) {
       projectPath.forEach((element) => {
         path.push(element);

examples/openspending/pages/index.tsx

@@ -1,6 +1,10 @@
 import { promises as fs } from 'fs';
 import path from 'path';
-import { getProject } from '../lib/octokit';
+import {
+  GithubProject,
+  getProjectDataPackage,
+  getProjectMetadata,
+} from '../lib/octokit';
 import getConfig from 'next/config';
 import ExternalLinkIcon from '../components/icons/ExternalLinkIcon';
 import TimeAgo from 'react-timeago';
@@ -8,37 +12,55 @@ import Link from 'next/link';
 import { Hero } from '../components/Hero';
 import { Header } from '../components/Header';
 import { Container } from '../components/Container';
+import { FiscalDataPackage } from '../lib/datapackage.interface';
+import { loadDataPackage } from '../lib/loader';
+import DatasetsSearch from '../components/DatasetsSearch';
 
 export async function getStaticProps() {
-  const jsonDirectory = path.join(
+  const jsonDirectory = path.join(process.cwd(), '/datasets.json');
-    process.cwd(),
-    '/datasets.json'
-  );
   const repos = await fs.readFile(jsonDirectory, 'utf8');
   const github_pat = getConfig().serverRuntimeConfig.github_pat;
+  const datapackages = await Promise.all(
+    JSON.parse(repos).map(async (_repo: GithubProject) => {
+      const datapackage = await getProjectDataPackage(
+        _repo.owner,
+        _repo.name,
+        'main',
+        github_pat
+      );
+      const repo = await getProjectMetadata(
+        _repo.owner,
+        _repo.name,
+        github_pat
+      );
 
-  const projects = await Promise.all(
+      return {
-    (JSON.parse(repos)).map(async (repo) => {
+        datapackage,
-      const project = await getProject(repo, github_pat);
+        repo,
-      return { ...project, repo_config: repo };
+      };
     })
   );
 
+  const projects = datapackages.map(
+    (item: { datapackage: FiscalDataPackage & { repo: string }; repo: any }) =>
+      loadDataPackage(item.datapackage, item.repo)
+  );
+
   return {
     props: {
-      projects,
+      projects: JSON.stringify(projects),
     },
   };
 }
 
 export function Datasets({ projects }) {
+  projects = JSON.parse(projects);
+
   return (
     <div className="bg-white min-h-screen">
       <Header />
       <Hero />
-      <section
+      <section className="py-20 sm:py-32">
-        className="py-20 sm:py-32"
-      >
         <Container>
           <div className="mx-auto max-w-2xl lg:mx-0">
             <h2
@@ -51,75 +73,8 @@ export function Datasets({ projects }) {
               Find spending data about countries all around the world.
             </p>
           </div>
-          <div className="mt-5">
+          <div className="mt-10">
-            <div className="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
+            <DatasetsSearch datasets={projects} />
-              <div className="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
-                <table className="min-w-full divide-y divide-gray-300">
-                  <thead>
-                    <tr>
-                      <th
-                        scope="col"
-                        className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                      >
-                        Name
-                      </th>
-                      <th
-                        scope="col"
-                        className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                      >
-                        Repository
-                      </th>
-                      <th
-                        scope="col"
-                        className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                      >
-                        Description
-                      </th>
-                      <th
-                        scope="col"
-                        className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
-                      >
-                        Last updated
-                      </th>
-                      <th
-                        scope="col"
-                        className="relative py-3.5 pl-3 pr-4 sm:pr-0"
-                      ></th>
-                    </tr>
-                  </thead>
-                  <tbody className="divide-y divide-gray-200">
-                    {projects.map((project) => (
-                      <tr key={project.id}>
-                        <td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
-                          {project.repo_config.name
-                            ? project.repo_config.name
-                            : project.full_name + (project.base_path === '/' ? '' : '/' + project.base_path)}
-                        </td>
-                        <td className="whitespace-nowrap px-3 py-6 text-sm group text-gray-500 hover:text-gray-900 transition-all duration-250">
-                          <a href={project.html_url} target="_blank" className='flex items-center'>@{project.full_name} <ExternalLinkIcon className='ml-1' /></a>
-                        </td>
-                        <td className="px-3 py-4 text-sm text-gray-500">
-                          {project.repo_config.description
-                            ? project.repo_config.description
-                            : project.description}
-                        </td>
-                        <td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
-                          <TimeAgo date={new Date(project.last_updated)} />
-                        </td>
-                        <td className="relative whitespace-nowrap py-6 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
-                          <a
-                            href={`/@${project.repo_config.owner}/${project.repo_config.repo}/${project.base_path === '/' ? '' : project.base_path}`}
-                            className='border border-gray-900 text-gray-900 px-4 py-2 transition-all hover:bg-gray-900 hover:text-white'
-                          >
-                            info
-                          </a>
-                        </td>
-                      </tr>
-                    ))}
-                  </tbody>
-                </table>
-              </div>
-            </div>
           </div>
         </Container>
       </section>

examples/openspending/public/assets/org-icon.svg

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg fill="#000000" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="800px" height="800px" viewBox="0 0 120 120" enable-background="new 0 0 120 120" xml:space="preserve">
+<rect x="2" y="108.1" width="116" height="11.9"/>
+<rect x="6.744" y="96.582" width="104.979" height="6.543"/>
+<rect x="15.288" y="38.532" width="17.639" height="52.925"/>
+<rect x="50.484" y="38.532" width="17.639" height="52.925"/>
+<rect x="84.33" y="38.532" width="17.639" height="52.925"/>
+<polygon points="0,26.96 60,0 120,26.96 119.946,33.912 0,34.01 "/>
+</svg>

examples/openspending/vitest.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'vitest/config'
+import react from '@vitejs/plugin-react'
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  plugins: [react()],
+  test: {
+    environment: 'jsdom',
+  },
+})