[examples/openspending][m] - fix build + add tests

examples/openspending/__tests__/os-data.test.ts

@@ -1,5 +1,8 @@
 import { expect, test } from 'vitest';
-import { getAllProjectsFromOrg } from '../lib/project';
+import { getAllProjectsFromOrg, getProjectDataPackage } from '../lib/project';
+import { loadDataPackage } from '../lib/loader';
+import { getProjectMetadata } from '../lib/project';
+import { getCsv, parseCsv } from '../components/Table';
 
 test(
   'Test OS-Data',
@@ -9,8 +12,7 @@ test(
       'main',
       process.env.VITE_GITHUB_PAT
     );
-    if (repos.failed.length > 0)
+    if (repos.failed.length > 0) console.log(repos.failed);
-      console.log(repos.failed);
     expect(repos.failed.length).toBe(0);
   },
   { timeout: 100000 }
@@ -24,9 +26,113 @@ test(
       'main',
       process.env.VITE_GITHUB_PAT
     );
-    if (repos.failed.length > 0)
+    if (repos.failed.length > 0) console.log(repos.failed);
-      console.log(repos.failed);
     expect(repos.failed.length).toBe(0);
   },
   { timeout: 100000 }
 );
+
+test(
+  'Test getting one dataset from github',
+  async () => {
+    const datapackage = await getProjectDataPackage(
+      'os-data',
+      'berlin-berlin',
+      'main',
+      process.env.VITE_GITHUB_PAT
+    );
+    const repo = await getProjectMetadata(
+      'os-data',
+      'berlin-berlin',
+      process.env.VITE_GITHUB_PAT
+    );
+    const project = loadDataPackage(datapackage, repo);
+    delete project['datapackage'];
+    delete project.files[0]['dialect'];
+    delete project.files[0]['schema'];
+    expect(project).toStrictEqual({
+      name: 'berlin-berlin',
+      title: 'Berlin-Berlin',
+      description: null,
+      owner: {
+        name: 'os-data',
+        logo: 'https://avatars.githubusercontent.com/u/13695166?v=4',
+        title: 'os-data',
+      },
+      repo: {
+        name: 'berlin-berlin',
+        full_name: 'os-data/berlin-berlin',
+        url: 'https://github.com/os-data/berlin-berlin',
+      },
+      files: [
+        {
+          name: 'berlin-gesamt',
+          format: 'csv',
+          path: 'https://storage.openspending.org/berlin-berlin/berlin-gesamt.csv',
+          mediatype: 'text/csv',
+          bytes: 81128743,
+          encoding: 'utf-8',
+        },
+      ],
+      author: 'Michael Peters <michael.peters@okfn.de>',
+      cityCode: 'Berlin',
+      countryCode: 'DE',
+      fiscalPeriod: { start: '2014-01-01', end: '2019-12-31' },
+      readme: '',
+    });
+  },
+  { timeout: 100000 }
+);
+
+test(
+  'Test getting one section of csv from R2',
+  async () => {
+    const rawCsv = await getCsv(
+      'https://storage.openspending.org/state-of-minas-gerais-brazil-planned-budget/__os_imported__br-mg-ppagloc.csv'
+    );
+    const parsedCsv = await parseCsv(rawCsv);
+    expect(parsedCsv.errors.length).toBe(1);
+    expect(parsedCsv.data.length).toBe(10165);
+    expect(parsedCsv.meta.fields).toStrictEqual([
+      'function_name',
+      'function_label',
+      'product_name',
+      'product_label',
+      'area_name',
+      'area_label',
+      'subaction_name',
+      'subaction_label',
+      'region_label_map',
+      'region_reg_map',
+      'region_name',
+      'region_label',
+      'municipality_map_id',
+      'municipality_name',
+      'municipality_map_code',
+      'municipality_label',
+      'municipality_map_name_simple',
+      'municipality_map_name',
+      'cofog1_label_en',
+      'cofog1_name',
+      'cofog1_label',
+      'amount',
+      'subprogramme_name',
+      'subprogramme_label',
+      'time_name',
+      'time_year',
+      'time_month',
+      'time_day',
+      'time_week',
+      'time_yearmonth',
+      'time_quarter',
+      'time',
+      'action_name',
+      'action_label',
+      'subfunction_name',
+      'subfunction_label',
+      'programme_name',
+      'programme_label',
+    ]);
+  },
+  { timeout: 100000 }
+);

examples/openspending/components/Table.tsx

@@ -9,7 +9,7 @@ import { Grid } from '@githubocto/flat-ui';
 
 const queryClient = new QueryClient();
 
-async function getCsv(url: string) {
+export async function getCsv(url: string) {
   const response = await fetch(url, {
     headers: {
       Range: 'bytes=0-5132288',

examples/openspending/pages/@org/[org]/[project].tsx

@@ -192,9 +192,10 @@ export async function getStaticPaths() {
     github_pat
   );
 
+  console.log(allProjects)
   const paths = allProjects.results.map((project) => ({
     params: {
-      // TODO: dybamize the org
+      // TODO: dynamize the org
       org: 'os-data',
       project: project.repo.name,
     },
@@ -219,10 +220,10 @@ export async function getStaticProps({ params }) {
 
   const repo = await getProjectMetadata(orgName, projectName, github_pat);
 
-  const project = await loadDataPackage(datapackage, repo);
+  const project = loadDataPackage(datapackage, repo);
 
   // TODO: should this be moved to the loader?
-  const readme = await getProjectReadme(orgName, projectName);
+  const readme = await getProjectReadme(orgName, projectName, 'main', github_pat);
 
   return {
     props: {