diff --git a/examples/openspending/__tests__/os-data.test.ts b/examples/openspending/__tests__/os-data.test.ts index b36549ac..de652c6a 100644 --- a/examples/openspending/__tests__/os-data.test.ts +++ b/examples/openspending/__tests__/os-data.test.ts @@ -1,5 +1,8 @@ import { expect, test } from 'vitest'; -import { getAllProjectsFromOrg } from '../lib/project'; +import { getAllProjectsFromOrg, getProjectDataPackage } from '../lib/project'; +import { loadDataPackage } from '../lib/loader'; +import { getProjectMetadata } from '../lib/project'; +import { getCsv, parseCsv } from '../components/Table'; test( 'Test OS-Data', @@ -9,8 +12,7 @@ test( 'main', process.env.VITE_GITHUB_PAT ); - if (repos.failed.length > 0) - console.log(repos.failed); + if (repos.failed.length > 0) console.log(repos.failed); expect(repos.failed.length).toBe(0); }, { timeout: 100000 } @@ -24,9 +26,113 @@ test( 'main', process.env.VITE_GITHUB_PAT ); - if (repos.failed.length > 0) - console.log(repos.failed); + if (repos.failed.length > 0) console.log(repos.failed); expect(repos.failed.length).toBe(0); }, { timeout: 100000 } ); + +test( + 'Test getting one dataset from github', + async () => { + const datapackage = await getProjectDataPackage( + 'os-data', + 'berlin-berlin', + 'main', + process.env.VITE_GITHUB_PAT + ); + const repo = await getProjectMetadata( + 'os-data', + 'berlin-berlin', + process.env.VITE_GITHUB_PAT + ); + const project = loadDataPackage(datapackage, repo); + delete project['datapackage']; + delete project.files[0]['dialect']; + delete project.files[0]['schema']; + expect(project).toStrictEqual({ + name: 'berlin-berlin', + title: 'Berlin-Berlin', + description: null, + owner: { + name: 'os-data', + logo: 'https://avatars.githubusercontent.com/u/13695166?v=4', + title: 'os-data', + }, + repo: { + name: 'berlin-berlin', + full_name: 'os-data/berlin-berlin', + url: 'https://github.com/os-data/berlin-berlin', + }, + files: [ + { + name: 'berlin-gesamt', + format: 'csv', + path: 'https://storage.openspending.org/berlin-berlin/berlin-gesamt.csv', + mediatype: 'text/csv', + bytes: 81128743, + encoding: 'utf-8', + }, + ], + author: 'Michael Peters ', + cityCode: 'Berlin', + countryCode: 'DE', + fiscalPeriod: { start: '2014-01-01', end: '2019-12-31' }, + readme: '', + }); + }, + { timeout: 100000 } +); + +test( + 'Test getting one section of csv from R2', + async () => { + const rawCsv = await getCsv( + 'https://storage.openspending.org/state-of-minas-gerais-brazil-planned-budget/__os_imported__br-mg-ppagloc.csv' + ); + const parsedCsv = await parseCsv(rawCsv); + expect(parsedCsv.errors.length).toBe(1); + expect(parsedCsv.data.length).toBe(10165); + expect(parsedCsv.meta.fields).toStrictEqual([ + 'function_name', + 'function_label', + 'product_name', + 'product_label', + 'area_name', + 'area_label', + 'subaction_name', + 'subaction_label', + 'region_label_map', + 'region_reg_map', + 'region_name', + 'region_label', + 'municipality_map_id', + 'municipality_name', + 'municipality_map_code', + 'municipality_label', + 'municipality_map_name_simple', + 'municipality_map_name', + 'cofog1_label_en', + 'cofog1_name', + 'cofog1_label', + 'amount', + 'subprogramme_name', + 'subprogramme_label', + 'time_name', + 'time_year', + 'time_month', + 'time_day', + 'time_week', + 'time_yearmonth', + 'time_quarter', + 'time', + 'action_name', + 'action_label', + 'subfunction_name', + 'subfunction_label', + 'programme_name', + 'programme_label', + ]); + }, + { timeout: 100000 } +); diff --git a/examples/openspending/components/Table.tsx b/examples/openspending/components/Table.tsx index 77612e03..9b646206 100644 --- a/examples/openspending/components/Table.tsx +++ b/examples/openspending/components/Table.tsx @@ -9,7 +9,7 @@ import { Grid } from '@githubocto/flat-ui'; const queryClient = new QueryClient(); -async function getCsv(url: string) { +export async function getCsv(url: string) { const response = await fetch(url, { headers: { Range: 'bytes=0-5132288', diff --git a/examples/openspending/pages/@org/[org]/[project].tsx b/examples/openspending/pages/@org/[org]/[project].tsx index ca21981a..bedc6a89 100644 --- a/examples/openspending/pages/@org/[org]/[project].tsx +++ b/examples/openspending/pages/@org/[org]/[project].tsx @@ -192,9 +192,10 @@ export async function getStaticPaths() { github_pat ); + console.log(allProjects) const paths = allProjects.results.map((project) => ({ params: { - // TODO: dybamize the org + // TODO: dynamize the org org: 'os-data', project: project.repo.name, }, @@ -219,10 +220,10 @@ export async function getStaticProps({ params }) { const repo = await getProjectMetadata(orgName, projectName, github_pat); - const project = await loadDataPackage(datapackage, repo); + const project = loadDataPackage(datapackage, repo); // TODO: should this be moved to the loader? - const readme = await getProjectReadme(orgName, projectName); + const readme = await getProjectReadme(orgName, projectName, 'main', github_pat); return { props: {