diff --git a/examples/openspending/__tests__/os-data.test.ts b/examples/openspending/__tests__/os-data.test.ts index 73da58c8..de652c6a 100644 --- a/examples/openspending/__tests__/os-data.test.ts +++ b/examples/openspending/__tests__/os-data.test.ts @@ -1,45 +1,138 @@ -import { Octokit } from 'octokit'; -import { assert, expect, test } from 'vitest' -import { getProjectDataPackage } from '../lib/octokit'; +import { expect, test } from 'vitest'; +import { getAllProjectsFromOrg, getProjectDataPackage } from '../lib/project'; +import { loadDataPackage } from '../lib/loader'; +import { getProjectMetadata } from '../lib/project'; +import { getCsv, parseCsv } from '../components/Table'; -export async function getAllDataPackagesFromOrg( - org: string, - branch?: string, - github_pat?: string -) { - const octokit = new Octokit({ auth: github_pat }); - const repos = await octokit.rest.repos.listForOrg({ org, type: 'public', per_page: 100 }); - let failedDataPackages = []; - const datapackages = await Promise.all( - repos.data.map(async (_repo) => { - const datapackage = await getProjectDataPackage( - org, - _repo.name, - branch ? branch : 'main', - github_pat - ); - if (!datapackage) { - failedDataPackages.push(_repo.name) - return null - }; - return {...datapackage, repo: _repo.name}; - }) - ); - return { - datapackages: datapackages.filter((item) => item !== null), - failedDataPackages, - }; -} +test( + 'Test OS-Data', + async () => { + const repos = await getAllProjectsFromOrg( + 'os-data', + 'main', + process.env.VITE_GITHUB_PAT + ); + if (repos.failed.length > 0) console.log(repos.failed); + expect(repos.failed.length).toBe(0); + }, + { timeout: 100000 } +); -test('Test OS-Data', async () => { - const repos = await getAllDataPackagesFromOrg('os-data', 'main', process.env.VITE_GITHUB_PAT) - if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages) - expect(repos.failedDataPackages.length).toBe(0) -}, {timeout: 100000}) +test( + 'Test Gift-Data', + async () => { + const repos = await getAllProjectsFromOrg( + 'gift-data', + 'main', + process.env.VITE_GITHUB_PAT + ); + if (repos.failed.length > 0) console.log(repos.failed); + expect(repos.failed.length).toBe(0); + }, + { timeout: 100000 } +); -test('Test Gift-Data', async () => { - const repos = await getAllDataPackagesFromOrg('gift-data', 'main', process.env.VITE_GITHUB_PAT) - if (repos.failedDataPackages.length > 0) console.log(repos.failedDataPackages) - expect(repos.failedDataPackages.length).toBe(0) -}, {timeout: 100000}) +test( + 'Test getting one dataset from github', + async () => { + const datapackage = await getProjectDataPackage( + 'os-data', + 'berlin-berlin', + 'main', + process.env.VITE_GITHUB_PAT + ); + const repo = await getProjectMetadata( + 'os-data', + 'berlin-berlin', + process.env.VITE_GITHUB_PAT + ); + const project = loadDataPackage(datapackage, repo); + delete project['datapackage']; + delete project.files[0]['dialect']; + delete project.files[0]['schema']; + expect(project).toStrictEqual({ + name: 'berlin-berlin', + title: 'Berlin-Berlin', + description: null, + owner: { + name: 'os-data', + logo: 'https://avatars.githubusercontent.com/u/13695166?v=4', + title: 'os-data', + }, + repo: { + name: 'berlin-berlin', + full_name: 'os-data/berlin-berlin', + url: 'https://github.com/os-data/berlin-berlin', + }, + files: [ + { + name: 'berlin-gesamt', + format: 'csv', + path: 'https://storage.openspending.org/berlin-berlin/berlin-gesamt.csv', + mediatype: 'text/csv', + bytes: 81128743, + encoding: 'utf-8', + }, + ], + author: 'Michael Peters ', + cityCode: 'Berlin', + countryCode: 'DE', + fiscalPeriod: { start: '2014-01-01', end: '2019-12-31' }, + readme: '', + }); + }, + { timeout: 100000 } +); +test( + 'Test getting one section of csv from R2', + async () => { + const rawCsv = await getCsv( + 'https://storage.openspending.org/state-of-minas-gerais-brazil-planned-budget/__os_imported__br-mg-ppagloc.csv' + ); + const parsedCsv = await parseCsv(rawCsv); + expect(parsedCsv.errors.length).toBe(1); + expect(parsedCsv.data.length).toBe(10165); + expect(parsedCsv.meta.fields).toStrictEqual([ + 'function_name', + 'function_label', + 'product_name', + 'product_label', + 'area_name', + 'area_label', + 'subaction_name', + 'subaction_label', + 'region_label_map', + 'region_reg_map', + 'region_name', + 'region_label', + 'municipality_map_id', + 'municipality_name', + 'municipality_map_code', + 'municipality_label', + 'municipality_map_name_simple', + 'municipality_map_name', + 'cofog1_label_en', + 'cofog1_name', + 'cofog1_label', + 'amount', + 'subprogramme_name', + 'subprogramme_label', + 'time_name', + 'time_year', + 'time_month', + 'time_day', + 'time_week', + 'time_yearmonth', + 'time_quarter', + 'time', + 'action_name', + 'action_label', + 'subfunction_name', + 'subfunction_label', + 'programme_name', + 'programme_label', + ]); + }, + { timeout: 100000 } +); diff --git a/examples/openspending/components/DatasetCard.tsx b/examples/openspending/components/DatasetCard.tsx index e0ca77f8..5af7c0c3 100644 --- a/examples/openspending/components/DatasetCard.tsx +++ b/examples/openspending/components/DatasetCard.tsx @@ -9,7 +9,7 @@ export default function DatasetCard({ dataset }: { dataset: Project }) { className="overflow-hidden rounded-xl border border-gray-200" >
diff --git a/examples/openspending/components/DatasetsSearch.tsx b/examples/openspending/components/DatasetsSearch.tsx index c04e3dee..7e7d364c 100644 --- a/examples/openspending/components/DatasetsSearch.tsx +++ b/examples/openspending/components/DatasetsSearch.tsx @@ -2,9 +2,26 @@ import { useForm } from 'react-hook-form'; import DatasetsGrid from './DatasetsGrid'; import { Project } from '../lib/project.interface'; import { Index } from 'flexsearch'; +import { + ChevronDoubleLeftIcon, + ChevronDoubleRightIcon, + ChevronLeftIcon, + ChevronRightIcon, +} from '@heroicons/react/24/solid'; +import { useState } from 'react'; + +export default function DatasetsSearch({ + datasets, + availableCountries, +}: { + datasets: Project[]; + availableCountries; +}) { + const itemsPerPage = 6; + const [page, setPage] = useState(1); -export default function DatasetsSearch({ datasets }: { datasets: Project[] }) { const index = new Index({ tokenize: 'full' }); + datasets.forEach((dataset: Project) => index.add( dataset.name, @@ -21,12 +38,38 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) { }, }); - const allCountries = datasets - .map((item) => item.countryCode) - .filter((v) => v) // Filters false values - .filter((v, i, a) => a.indexOf(v) === i) // Remove duplicates - // TODO: title should be the full name - .map((code) => ({ code, title: code })); + const filteredDatasets = datasets + .filter((dataset: Project) => + watch().searchTerm && watch().searchTerm !== '' + ? index.search(watch().searchTerm).includes(dataset.name) + : true + ) + .filter((dataset) => + watch().country && watch().country !== '' + ? dataset.countryCode === watch().country + : true + ) + // TODO: Does that really makes sense? + // What if the fiscalPeriod is 2015-2017 and inputs are + // set to 2015-2016. It's going to be filtered out but + // it shouldn't. + .filter((dataset) => + watch().minDate && watch().minDate !== '' + ? dataset.fiscalPeriod?.start >= watch().minDate + : true + ) + .filter((dataset) => + watch().maxDate && watch().maxDate !== '' + ? dataset.fiscalPeriod?.end <= watch().maxDate + : true + ); + + const paginatedDatasets = filteredDatasets.slice( + (page - 1) * itemsPerPage, + (page - 1) * itemsPerPage + itemsPerPage + ); + + const pageCount = Math.ceil(filteredDatasets.length / itemsPerPage) || 1; return ( <> @@ -37,7 +80,7 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) { setPage(1) })} className="h-[3em] relative w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm" /> {watch().searchTerm !== '' && ( @@ -55,10 +98,10 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) { setPage(1) })} className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm" /> - {watch().minDate !== '' && ( - - )}
@@ -92,48 +127,56 @@ export default function DatasetsSearch({ datasets }: { datasets: Project[] }) { setPage(1) })} className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm" /> - {watch().maxDate !== '' && ( - - )}
-
- - watch().searchTerm && watch().searchTerm !== '' - ? index.search(watch().searchTerm).includes(dataset.name) - : true - ) - .filter((dataset) => - watch().country && watch().country !== '' - ? dataset.countryCode === watch().country - : true - ) - // TODO: Does that really makes sense? - // What if the fiscalPeriod is 2015-2017 and inputs are - // set to 2015-2016. It's going to be filtered out but - // it shouldn't. - .filter((dataset) => - watch().minDate && watch().minDate !== '' - ? dataset.fiscalPeriod?.start >= watch().minDate - : true - ) - .filter((dataset) => - watch().maxDate && watch().maxDate !== '' - ? dataset.fiscalPeriod?.end <= watch().maxDate - : true - )} - /> +
+ + {filteredDatasets.length} datasets found + +
+
+ +
+ + + + Page {page} of {pageCount} + + + +
); diff --git a/examples/openspending/components/Header.tsx b/examples/openspending/components/Header.tsx index c9fb47a0..50009713 100644 --- a/examples/openspending/components/Header.tsx +++ b/examples/openspending/components/Header.tsx @@ -1,53 +1,82 @@ -import Image from 'next/image' -import { Button } from './Button' -import { Container } from './Container' -import logo from "../public/logo.svg" -import Link from 'next/link' -import { useRouter } from 'next/router' +import Image from 'next/image'; +import { Container } from './Container'; +import logo from '../public/logo.svg'; +import Link from 'next/link'; +import { useRouter } from 'next/router'; +import { Bars3Icon } from '@heroicons/react/24/outline'; +import { useState } from 'react'; export function Header() { + const [menuOpen, setMenuOpen] = useState(false); + const router = useRouter(); const isActive = (navLink) => { - return router.asPath.split("?")[0] == navLink.href; - } + return router.asPath.split('?')[0] == navLink.href; + }; const navLinks = [ { - title: "Home", - href: "/#header" + title: 'Home', + href: '/', }, { - title: "Datasets", - href: "/#datasets" + title: 'Datasets', + href: '/#datasets', }, - { - title: "Community", - href: "https://community.openspending.org/" - } - ] + // { + // title: "Community", + // href: "https://community.openspending.org/" + // } + ]; return ( - - ) + + ); } diff --git a/examples/openspending/components/Hero.tsx b/examples/openspending/components/Hero.tsx index b09b321c..6c28ee1f 100644 --- a/examples/openspending/components/Hero.tsx +++ b/examples/openspending/components/Hero.tsx @@ -1,9 +1,9 @@ -import { Button } from './Button' -import { Container } from './Container' +import { Button } from './Button'; +import { Container } from './Container'; -export function Hero() { +export function Hero({ countriesCount, datasetsCount, filesCount }) { return ( -
+
@@ -15,12 +15,13 @@ export function Hero() {

- By understanding how governments spend money in our name can we have a say - in how that money will affect our own lives. The journey starts here. + By understanding how governments spend money in our name can we + have a say in how that money will affect our own lives. The + journey starts here.

- OpenSpending is a free, open and global platform to search, visualise and analyse - fiscal data in the public sphere. + OpenSpending is a free, open and global platform to search, + visualise and analyse fiscal data in the public sphere.

{[ - ['Countries', '75'], - ['Datasets', '2091'], - ['Files', '9230'], + // Added the plus sign because some datasets do not + // contain defined countries + ['Countries', '+' + countriesCount], + ['Datasets', datasetsCount], + ['Files', filesCount], ].map(([name, value]) => (
{name}
@@ -43,5 +46,5 @@ export function Hero() {
- ) + ); } diff --git a/examples/openspending/components/Table.tsx b/examples/openspending/components/Table.tsx index 77612e03..9b646206 100644 --- a/examples/openspending/components/Table.tsx +++ b/examples/openspending/components/Table.tsx @@ -9,7 +9,7 @@ import { Grid } from '@githubocto/flat-ui'; const queryClient = new QueryClient(); -async function getCsv(url: string) { +export async function getCsv(url: string) { const response = await fetch(url, { headers: { Range: 'bytes=0-5132288', diff --git a/examples/openspending/components/_shared/Layout.tsx b/examples/openspending/components/_shared/Layout.tsx new file mode 100644 index 00000000..7c10c66f --- /dev/null +++ b/examples/openspending/components/_shared/Layout.tsx @@ -0,0 +1,10 @@ +import { Header } from '../Header'; + +export default function Layout({ children }) { + return ( +
+
+ {children} +
+ ); +} diff --git a/examples/openspending/datasets.json b/examples/openspending/datasets.json deleted file mode 100644 index cb2786f6..00000000 --- a/examples/openspending/datasets.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - { - "owner": "os-data", - "branch": "main", - "name": "mongolia-budget-2016-2017" - }, - { - "owner": "os-data", - "branch": "main", - "name": "gb-country-regional-analysis" - }, - { - "owner": "os-data", - "branch": "main", - "name": "berlin-berlin" - }, - { - "owner": "os-data", - "branch": "main", - "name": "state-of-minas-gerais-brazil-planned-budget" - }, - { - "owner": "os-data", - "branch": "main", - "name": "wesel" - } -] diff --git a/examples/openspending/lib/datapackage.interface.ts b/examples/openspending/lib/datapackage.interface.ts index 432c9cf1..ecbd004e 100644 --- a/examples/openspending/lib/datapackage.interface.ts +++ b/examples/openspending/lib/datapackage.interface.ts @@ -98,6 +98,7 @@ export interface TabularDataResource { key?: string; path?: string; size?: number; + bytes?: number; } export interface Field { diff --git a/examples/openspending/lib/loader.ts b/examples/openspending/lib/loader.ts index 2d5243b2..0dff4da9 100644 --- a/examples/openspending/lib/loader.ts +++ b/examples/openspending/lib/loader.ts @@ -5,13 +5,14 @@ export function loadDataPackage(datapackage: FiscalDataPackage, repo): Project { return { name: datapackage.name, title: datapackage.title, + description: datapackage.description || null, owner: { name: repo.owner.login, logo: repo.owner.avatar_url, // TODO: make this title work title: repo.owner.login, }, - repo: { name: repo, full_name: repo.full_name }, + repo: { name: repo.name, full_name: repo.full_name, url: repo.html_url }, files: datapackage.resources, author: datapackage.author ? datapackage.author : null, cityCode: datapackage.cityCode ? datapackage.cityCode : null, diff --git a/examples/openspending/lib/project.interface.ts b/examples/openspending/lib/project.interface.ts index 05c894cf..fadd9ea2 100644 --- a/examples/openspending/lib/project.interface.ts +++ b/examples/openspending/lib/project.interface.ts @@ -5,10 +5,11 @@ import { export interface Project { owner: { name: string; logo?: string; title?: string }; // Info about the owner of the data repo - repo: { name: string; full_name: string }; // Info about the the data repo + repo: { name: string; full_name: string; url: string }; // Info about the the data repo files: TabularDataResource[]; name: string; title?: string; + description?: string; author?: string; cityCode?: string; countryCode?: string; diff --git a/examples/openspending/lib/octokit.ts b/examples/openspending/lib/project.ts similarity index 82% rename from examples/openspending/lib/octokit.ts rename to examples/openspending/lib/project.ts index 1c3787b3..4c1913c5 100644 --- a/examples/openspending/lib/octokit.ts +++ b/examples/openspending/lib/project.ts @@ -13,8 +13,7 @@ export interface GithubProject { export async function getProjectReadme( owner: string, repo: string, - branch: string, - readme: string, + branch: string = 'main', github_pat?: string ) { const octokit = new Octokit({ auth: github_pat }); @@ -22,7 +21,7 @@ export async function getProjectReadme( const response = await octokit.rest.repos.getContent({ owner, repo, - path: readme, + path: 'README.md', ref: branch, }); const data = response.data as { content?: string }; @@ -125,7 +124,6 @@ export async function getProject(project: GithubProject, github_pat?: string) { project.owner, project.repo, project.branch, - project.readme, github_pat ); @@ -185,8 +183,43 @@ export async function getProjectDataPackage( } const decodedContent = Buffer.from(fileContent, 'base64').toString(); const datapackage = JSON.parse(decodedContent); - return {...datapackage, repo }; + + return { ...datapackage, repo }; } catch (error) { return null; } } + +export async function getAllProjectsFromOrg( + org: string, + branch?: string, + github_pat?: string +) { + const octokit = new Octokit({ auth: github_pat }); + const repos = await octokit.rest.repos.listForOrg({ + org, + type: 'public', + per_page: 100, + }); + let failedProjects = []; + const projects = await Promise.all( + repos.data.map(async (_repo) => { + const project = await getProjectDataPackage( + org, + _repo.name, + branch ? branch : 'main', + github_pat + ); + if (!project) { + failedProjects.push(_repo.name); + return null; + } + + return { datapackage: project, repo: _repo }; + }) + ); + return { + results: projects.filter((item) => item !== null), + failed: failedProjects, + }; +} diff --git a/examples/openspending/orgs.json b/examples/openspending/orgs.json new file mode 100644 index 00000000..ff52b40c --- /dev/null +++ b/examples/openspending/orgs.json @@ -0,0 +1,3 @@ +[ + "os-data" +] diff --git a/examples/openspending/package-lock.json b/examples/openspending/package-lock.json index d30c362e..d8530d23 100644 --- a/examples/openspending/package-lock.json +++ b/examples/openspending/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "dependencies": { "@githubocto/flat-ui": "^0.14.1", + "@heroicons/react": "^2.0.18", "@octokit/plugin-throttling": "^5.2.2", "@types/flexsearch": "^0.7.3", "@types/node": "18.16.0", @@ -1622,6 +1623,14 @@ "object-assign": "^4.1.1" } }, + "node_modules/@heroicons/react": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/@heroicons/react/-/react-2.0.18.tgz", + "integrity": "sha512-7TyMjRrZZMBPa+/5Y8lN0iyvUU/01PeMGX2+RE7cQWpEUIcb4QotzUObFkJDejj/HUH4qjP/eQ0gzzKs2f+6Yw==", + "peerDependencies": { + "react": ">= 16" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", diff --git a/examples/openspending/package.json b/examples/openspending/package.json index ae260582..afea1a9c 100644 --- a/examples/openspending/package.json +++ b/examples/openspending/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@githubocto/flat-ui": "^0.14.1", + "@heroicons/react": "^2.0.18", "@octokit/plugin-throttling": "^5.2.2", "@types/flexsearch": "^0.7.3", "@types/node": "18.16.0", diff --git a/examples/openspending/pages/@org/[org]/[...path].tsx b/examples/openspending/pages/@org/[org]/[...path].tsx deleted file mode 100644 index 8227290c..00000000 --- a/examples/openspending/pages/@org/[org]/[...path].tsx +++ /dev/null @@ -1,126 +0,0 @@ -import { NextSeo } from 'next-seo'; -import { promises as fs } from 'fs'; -import path from 'path'; -import getConfig from 'next/config'; -import { getProject, GithubProject } from '../../../lib/octokit'; -import ReactMarkdown from 'react-markdown'; -import remarkGfm from 'remark-gfm'; -import Breadcrumbs from '../../../components/_shared/Breadcrumbs'; - -export default function ProjectPage({ project }) { - const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}` - - return ( - <> - -
- -

{project.repo_config.name || repoId}

-

Repository: {project.html_url}

- -

Files

-
- - - - - - - - - {project.files?.map((file) => ( - - - - - ))} - -
- Name - - Size -
- {file.name} - - {file.size} Bytes -
-
- - {project.readmeContent && <> -
- -

Readme

- - {project.readmeContent} - - } -
- - ); -} - -// Generates `/posts/1` and `/posts/2` -export async function getStaticPaths() { - const jsonDirectory = path.join( - process.cwd(), - 'datasets.json' - ); - const repos = await fs.readFile(jsonDirectory, 'utf8'); - - return { - paths: JSON.parse(repos).map((repo) => { - const projectPath = - repo.readme && repo.readme.split('/').length > 1 - ? repo.readme.split('/').slice(0, -1) - : null; - let path = [repo.name]; - if (projectPath) { - projectPath.forEach((element) => { - path.push(element); - }); - } - return { - params: { org: repo.owner, path }, - }; - }), - fallback: false, // can also be true or 'blocking' - }; -} - -export async function getStaticProps({ params }) { - const jsonDirectory = path.join( - process.cwd(), - 'datasets.json' - ); - const reposFile = await fs.readFile(jsonDirectory, 'utf8'); - const repos: GithubProject[] = JSON.parse(reposFile); - const repo = repos.find((_repo) => { - const projectPath = - _repo.readme && _repo.readme.split('/').length > 1 - ? _repo.readme.split('/').slice(0, -1) - : null; - let path = [_repo.name]; - if (projectPath) { - projectPath.forEach((element) => { - path.push(element); - }); - } - return ( - _repo.owner == params.org && - JSON.stringify(path) === JSON.stringify(params.path) - ); - }); - const github_pat = getConfig().serverRuntimeConfig.github_pat; - const project = await getProject(repo, github_pat); - return { - props: { - project: { ...project, repo_config: repo }, - }, - }; -} diff --git a/examples/openspending/pages/@org/[org]/[project].tsx b/examples/openspending/pages/@org/[org]/[project].tsx new file mode 100644 index 00000000..bedc6a89 --- /dev/null +++ b/examples/openspending/pages/@org/[org]/[project].tsx @@ -0,0 +1,234 @@ +import { NextSeo } from 'next-seo'; +import getConfig from 'next/config'; +import { + getAllProjectsFromOrg, + getProjectDataPackage, + getProjectMetadata, + getProjectReadme, +} from '../../../lib/project'; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; +import { loadDataPackage } from '../../../lib/loader'; +import Layout from '../../../components/_shared/Layout'; +import Link from 'next/link'; +import { Project } from '../../../lib/project.interface'; +import ExternalLinkIcon from '../../../components/icons/ExternalLinkIcon'; + +export default function ProjectPage({ + project, + readme, +}: { + project: Project; + readme: string; +}) { + + // Get description from datapackage or calculate + // excerpt from README by getting all the content + // up to the first dot. + const description = + project.description || (readme && readme.slice(0, readme.indexOf('.') + 1)); + + return ( + + +
+

{project.title || project.name}

+ + @{project.repo.full_name} + + + {description && ( +
+ {description} +
+ )} + +
+ + + + + {project.datapackage.countryCode && ( + + )} + + + + + + + {project.datapackage.countryCode && ( + + )} + + + + +
+ Name + + Country + + Metadata +
+ {project.name} + + {project.datapackage.countryCode} + + + datapackage.json + +
+
+ +

Data files

+

+ This dataset contains {project.files.length} file + {project.files.length != 1 ? '' : 's'} +

+
+ + + + + + + + + + {project.files?.map((file) => { + let size: number | string = file.size; + + if (!size) { + if (file.bytes) { + if (file.bytes > 1000000) { + size = (file.bytes / 1000000).toFixed(2) + ' MB'; + } else { + size = (file.bytes / 1000).toFixed(2) + ' kB'; + } + } + } + + return ( + + + + + + + + ); + })} + +
+ Name + + Format + + Size +
+ {file.name} + + {file.format} + + {size} + + + Download + +
+
+ + {readme && ( + <> +
+ +

Readme

+ {readme} + + )} +
+ + ); +} + +// Generates `/posts/1` and `/posts/2` +export async function getStaticPaths() { + const github_pat = getConfig().serverRuntimeConfig.github_pat; + + const allProjects = await getAllProjectsFromOrg( + 'os-data', + 'main', + github_pat + ); + + console.log(allProjects) + const paths = allProjects.results.map((project) => ({ + params: { + // TODO: dynamize the org + org: 'os-data', + project: project.repo.name, + }, + })); + + return { + paths, + fallback: false, // can also be true or 'blocking' + }; +} + +export async function getStaticProps({ params }) { + const { org: orgName, project: projectName } = params; + + const github_pat = getConfig().serverRuntimeConfig.github_pat; + const datapackage = await getProjectDataPackage( + orgName, + projectName, + 'main', + github_pat + ); + + const repo = await getProjectMetadata(orgName, projectName, github_pat); + + const project = loadDataPackage(datapackage, repo); + + // TODO: should this be moved to the loader? + const readme = await getProjectReadme(orgName, projectName, 'main', github_pat); + + return { + props: { + project, + readme, + }, + }; +} diff --git a/examples/openspending/pages/_app.tsx b/examples/openspending/pages/_app.tsx index b14578b1..23b6c668 100644 --- a/examples/openspending/pages/_app.tsx +++ b/examples/openspending/pages/_app.tsx @@ -1,13 +1,11 @@ import { AppProps } from 'next/app'; -import Head from 'next/head'; import './styles.css'; +import { NextSeo } from 'next-seo'; function CustomApp({ Component, pageProps }: AppProps) { return ( <> - - GitHub Datasets - +
diff --git a/examples/openspending/pages/index.tsx b/examples/openspending/pages/index.tsx index bb163944..b3b30902 100644 --- a/examples/openspending/pages/index.tsx +++ b/examples/openspending/pages/index.tsx @@ -1,65 +1,58 @@ -import { promises as fs } from 'fs'; -import path from 'path'; -import { - GithubProject, - getProjectDataPackage, - getProjectMetadata, -} from '../lib/octokit'; +import { getAllProjectsFromOrg } from '../lib/project'; import getConfig from 'next/config'; -import ExternalLinkIcon from '../components/icons/ExternalLinkIcon'; -import TimeAgo from 'react-timeago'; -import Link from 'next/link'; import { Hero } from '../components/Hero'; -import { Header } from '../components/Header'; import { Container } from '../components/Container'; import { FiscalDataPackage } from '../lib/datapackage.interface'; import { loadDataPackage } from '../lib/loader'; import DatasetsSearch from '../components/DatasetsSearch'; +import Layout from '../components/_shared/Layout'; export async function getStaticProps() { - const jsonDirectory = path.join(process.cwd(), '/datasets.json'); - const repos = await fs.readFile(jsonDirectory, 'utf8'); - const github_pat = getConfig().serverRuntimeConfig.github_pat; - const datapackages = await Promise.all( - JSON.parse(repos).map(async (_repo: GithubProject) => { - const datapackage = await getProjectDataPackage( - _repo.owner, - _repo.name, - 'main', - github_pat - ); - const repo = await getProjectMetadata( - _repo.owner, - _repo.name, - github_pat - ); + // TODO: support other orgs + // const orgsListPath = path.join(process.cwd(), '/orgs.json'); + // const orgs = await fs.readFile(orgsListPath, 'utf8'); - return { - datapackage, - repo, - }; - }) + const github_pat = getConfig().serverRuntimeConfig.github_pat; + + const allProjects = await getAllProjectsFromOrg( + 'os-data', + 'main', + github_pat ); - const projects = datapackages.map( + const projects = allProjects.results.map( (item: { datapackage: FiscalDataPackage & { repo: string }; repo: any }) => loadDataPackage(item.datapackage, item.repo) ); + const availableCountries = projects + .map((item) => item.countryCode) + .filter((v) => v) // Filters false values + .filter((v, i, a) => a.indexOf(v) === i) // Remove duplicates + // TODO: title should be the full name + .map((code) => ({ code, title: code })); + return { props: { projects: JSON.stringify(projects), + availableCountries, }, }; } -export function Datasets({ projects }) { +export function Home({ projects, availableCountries }) { projects = JSON.parse(projects); return ( -
-
- + + partialSum + a.files.length, + 0 + )} + />
@@ -74,12 +67,15 @@ export function Datasets({ projects }) {

- +
-
+ ); } -export default Datasets; +export default Home;