[examples/github-backed-catalog] - improvements after issue #871 (#887)

This commit is contained in:
Luccas Mateus 2023-05-17 11:10:18 -03:00 committed by GitHub
parent ebcb93c996
commit 4e91e88f2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 276 additions and 182 deletions

View File

@ -1,8 +1,5 @@
{
"extends": [
"next",
"next/core-web-vitals"
],
"extends": ["next", "next/core-web-vitals"],
"ignorePatterns": ["!**/*", ".next/**/*"],
"overrides": [
{

View File

@ -0,0 +1,7 @@
node_modules
**/.next/**
**/_next/**
**/dist/**
**/__tmp__/**
lerna.json
.github

View File

@ -0,0 +1 @@
{}

View File

@ -99,4 +99,3 @@ And run the production build with:
```
npm run start
```

View File

@ -1,20 +1,28 @@
import Link from "next/link";
import HomeIcon from "../icons/HomeIcon";
export default function Breadcrumbs({ links }: { links: { title: string, href?: string, target?: string }[] }) {
export default function Breadcrumbs({
links,
}: {
links: { title: string; href?: string; target?: string }[];
}) {
const current = links.at(-1);
return <div className="flex items-center uppercase font-black text-xs">
<Link className="flex items-center" href='/'><HomeIcon /></Link>
return (
<div className="flex items-center uppercase font-black text-xs">
<Link className="flex items-center" href="/">
<HomeIcon />
</Link>
{/* {links.length > 1 && links.slice(0, -1).map((link) => {
{/* {links.length > 1 && links.slice(0, -1).map((link) => {
return <>
<span className="mx-4">/</span>
<Link href={link.href}>{link.title}</Link>
</>
})} */}
<span className="mx-4">/</span>
<span>{current.title}</span>
</div >
}
<span className="mx-4">/</span>
<span>{current.title}</span>
</div>
);
}

View File

@ -1,3 +1,13 @@
export default function ExternalLinkIcon({ className = "" }) {
return <div className={`inline-block w-4 ${className}`}><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" fill="currentColor"><path d="M 40 10 C 38.896 10 38 10.896 38 12 C 38 13.104 38.896 14 40 14 L 47.171875 14 L 30.585938 30.585938 C 29.804938 31.366938 29.804938 32.633063 30.585938 33.414062 C 30.976938 33.805063 31.488 34 32 34 C 32.512 34 33.023063 33.805062 33.414062 33.414062 L 50 16.828125 L 50 24 C 50 25.104 50.896 26 52 26 C 53.104 26 54 25.104 54 24 L 54 12 C 54 10.896 53.104 10 52 10 L 40 10 z M 18 12 C 14.691 12 12 14.691 12 18 L 12 46 C 12 49.309 14.691 52 18 52 L 46 52 C 49.309 52 52 49.309 52 46 L 52 34 C 52 32.896 51.104 32 50 32 C 48.896 32 48 32.896 48 34 L 48 46 C 48 47.103 47.103 48 46 48 L 18 48 C 16.897 48 16 47.103 16 46 L 16 18 C 16 16.897 16.897 16 18 16 L 30 16 C 31.104 16 32 15.104 32 14 C 32 12.896 31.104 12 30 12 L 18 12 z"/></svg></div>
}
return (
<div className={`inline-block w-4 ${className}`}>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 64 64"
fill="currentColor"
>
<path d="M 40 10 C 38.896 10 38 10.896 38 12 C 38 13.104 38.896 14 40 14 L 47.171875 14 L 30.585938 30.585938 C 29.804938 31.366938 29.804938 32.633063 30.585938 33.414062 C 30.976938 33.805063 31.488 34 32 34 C 32.512 34 33.023063 33.805062 33.414062 33.414062 L 50 16.828125 L 50 24 C 50 25.104 50.896 26 52 26 C 53.104 26 54 25.104 54 24 L 54 12 C 54 10.896 53.104 10 52 10 L 40 10 z M 18 12 C 14.691 12 12 14.691 12 18 L 12 46 C 12 49.309 14.691 52 18 52 L 46 52 C 49.309 52 52 49.309 52 46 L 52 34 C 52 32.896 51.104 32 50 32 C 48.896 32 48 32.896 48 34 L 48 46 C 48 47.103 47.103 48 46 48 L 18 48 C 16.897 48 16 47.103 16 46 L 16 18 C 16 16.897 16.897 16 18 16 L 30 16 C 31.104 16 32 15.104 32 14 C 32 12.896 31.104 12 30 12 L 18 12 z" />
</svg>
</div>
);
}

View File

@ -1,3 +1,10 @@
export default function HomeIcon({ className = "" }) {
return <div className={`inline-block w-4 ${className}`}><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"> <path d="M 12 2 A 1 1 0 0 0 11.289062 2.296875 L 1.203125 11.097656 A 0.5 0.5 0 0 0 1 11.5 A 0.5 0.5 0 0 0 1.5 12 L 4 12 L 4 20 C 4 20.552 4.448 21 5 21 L 9 21 C 9.552 21 10 20.552 10 20 L 10 14 L 14 14 L 14 20 C 14 20.552 14.448 21 15 21 L 19 21 C 19.552 21 20 20.552 20 20 L 20 12 L 22.5 12 A 0.5 0.5 0 0 0 23 11.5 A 0.5 0.5 0 0 0 22.796875 11.097656 L 12.716797 2.3027344 A 1 1 0 0 0 12.710938 2.296875 A 1 1 0 0 0 12 2 z"/></svg></div>
}
return (
<div className={`inline-block w-4 ${className}`}>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
{" "}
<path d="M 12 2 A 1 1 0 0 0 11.289062 2.296875 L 1.203125 11.097656 A 0.5 0.5 0 0 0 1 11.5 A 0.5 0.5 0 0 0 1.5 12 L 4 12 L 4 20 C 4 20.552 4.448 21 5 21 L 9 21 C 9.552 21 10 20.552 10 20 L 10 14 L 14 14 L 14 20 C 14 20.552 14.448 21 15 21 L 19 21 C 19.552 21 20 20.552 20 20 L 20 12 L 22.5 12 A 0.5 0.5 0 0 0 23 11.5 A 0.5 0.5 0 0 0 22.796875 11.097656 L 12.716797 2.3027344 A 1 1 0 0 0 12.710938 2.296875 A 1 1 0 0 0 12 2 z" />
</svg>
</div>
);
}

View File

@ -19,10 +19,7 @@
"owner": "datasets",
"branch": "main",
"repo": "investor-flow-of-funds-us",
"files": [
"data/monthly.csv",
"data/weekly.csv"
],
"files": ["data/monthly.csv", "data/weekly.csv"],
"readme": "README.md"
},
{
@ -38,7 +35,10 @@
"owner": "fivethirtyeight",
"repo": "data",
"branch": "master",
"files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"],
"files": [
"nba-raptor/historical_RAPTOR_by_player.csv",
"nba-raptor/historical_RAPTOR_by_team.csv"
],
"readme": "nba-raptor/README.md"
}
]

View File

@ -1,5 +1,5 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
declare module '*.svg' {
declare module "*.svg" {
const content: any;
export const ReactComponent: any;
export default content;

View File

@ -1,4 +1,4 @@
import { Octokit } from 'octokit';
import { Octokit } from "octokit";
export interface GithubProject {
owner: string;
@ -26,15 +26,16 @@ export async function getProjectReadme(
ref: branch,
});
const data = response.data as { content?: string };
const fileContent = data.content ? data.content : '';
if (fileContent === '') {
const fileContent = data.content ? data.content : "";
if (fileContent === "") {
return null;
}
const decodedContent = Buffer.from(fileContent, 'base64').toString();
const decodedContent = Buffer.from(fileContent, "base64").toString();
return decodedContent;
} catch (error) {
console.log(error);
return null;
throw new Error(
"Couldn't get project readme please make sure that you are pointing to a valid repo and that the repo in question contains a README.md"
);
}
}
@ -50,13 +51,13 @@ export async function getLastUpdated(
const response = await octokit.rest.repos.listCommits({
owner,
repo,
path: readme,
ref: branch,
});
return response.data[0].commit.committer.date;
} catch (error) {
console.log(error);
return null;
throw new Error(
"Couldn't get project list of commits please make sure that you are pointing to a valid repo"
);
}
}
export async function getProjectMetadata(
@ -72,8 +73,9 @@ export async function getProjectMetadata(
});
return response.data;
} catch (error) {
console.log(error);
return null;
throw new Error(
"Couldn't get project metadata please make sure that you are pointing to a valid repo"
);
}
}
@ -94,13 +96,32 @@ export async function getRepoContents(
ref: branch,
path: path,
});
const data = response.data as { download_url?: string, name: string, size: number };
contents.push({ download_url: data.download_url, name: data.name, size: data.size});
const data = response.data as {
download_url?: string;
name: string;
size: number;
};
contents.push({
download_url: data.download_url,
name: data.name,
size: data.size,
});
}
return contents;
} catch (error) {
console.log(error);
return null;
if (
error.message ===
'This endpoint can only return blobs smaller than 100 MB in size. The requested blob is too large to fetch via the API, but you can always clone the repository via Git to obtain it.: {"resource":"Blob","field":"data","code":"too_large"}'
) {
throw new Error(
`The requested files ${files.join(
", "
)} are too big making it impossible to fetch via Github API`
);
}
throw new Error(
"Couldn't get project contents please make sure that you are pointing to a valid repo"
);
}
}
@ -120,22 +141,20 @@ export async function getProject(project: GithubProject, github_pat?: string) {
project.readme,
github_pat
);
if (!projectReadme) {
return null;
let projectData = [];
if (project.files) {
projectData = await getRepoContents(
project.owner,
project.repo,
project.branch,
project.files,
github_pat
);
}
const projectData = await getRepoContents(
project.owner,
project.repo,
project.branch,
project.files,
github_pat
);
if (!projectData) {
return null;
}
const projectBase = project.readme.split('/').length > 1
? project.readme.split('/').slice(0, -1).join('/')
: '/'
const projectBase =
project.readme && project.readme.split("/").length > 1
? project.readme.split("/").slice(0, -1).join("/")
: "/";
const last_updated = await getLastUpdated(
project.owner,
project.repo,
@ -143,5 +162,11 @@ export async function getProject(project: GithubProject, github_pat?: string) {
projectBase,
github_pat
);
return { ...projectMetadata, files: projectData, readmeContent: projectReadme, last_updated, base_path: projectBase };
return {
...projectMetadata,
files: projectData,
readmeContent: projectReadme,
last_updated,
base_path: projectBase,
};
}

View File

@ -3,8 +3,8 @@ const nextConfig = {
return {
beforeFiles: [
{
source: '/@:org/:project*',
destination: '/@org/:org/:project*',
source: "/@:org/:project*",
destination: "/@org/:org/:project*",
},
],
};

View File

@ -27,6 +27,7 @@
"@tailwindcss/typography": "^0.5.9",
"autoprefixer": "^10.4.14",
"postcss": "^8.4.23",
"prettier": "2.8.8",
"tailwindcss": "^3.3.1"
}
},
@ -4677,6 +4678,21 @@
"node": ">= 0.8.0"
}
},
"node_modules/prettier": {
"version": "2.8.8",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.8.tgz",
"integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==",
"dev": true,
"bin": {
"prettier": "bin-prettier.js"
},
"engines": {
"node": ">=10.13.0"
},
"funding": {
"url": "https://github.com/prettier/prettier?sponsor=1"
}
},
"node_modules/prop-types": {
"version": "15.8.1",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",

View File

@ -6,7 +6,8 @@
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint"
"lint": "next lint",
"prettier": "prettier --write ."
},
"dependencies": {
"@types/node": "18.16.0",
@ -28,6 +29,7 @@
"@tailwindcss/typography": "^0.5.9",
"autoprefixer": "^10.4.14",
"postcss": "^8.4.23",
"prettier": "2.8.8",
"tailwindcss": "^3.3.1"
}
}

View File

@ -1,22 +1,31 @@
import { NextSeo } from 'next-seo';
import { promises as fs } from 'fs';
import path from 'path';
import getConfig from 'next/config';
import { getProject, GithubProject } from '../../../lib/octokit';
import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
import Breadcrumbs from '../../../components/_shared/Breadcrumbs';
import { NextSeo } from "next-seo";
import { promises as fs } from "fs";
import path from "path";
import getConfig from "next/config";
import { getProject, GithubProject } from "../../../lib/octokit";
import ReactMarkdown from "react-markdown";
import remarkGfm from "remark-gfm";
import Breadcrumbs from "../../../components/_shared/Breadcrumbs";
export default function ProjectPage({ project }) {
const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`
const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`;
return (
<>
<NextSeo title={`${repoId}${project.base_path !== '/' ? '/' + project.base_path : ''} - GitHub Datasets`} />
<NextSeo
title={`${repoId}${
project.base_path !== "/" ? "/" + project.base_path : ""
} - GitHub Datasets`}
/>
<main className="prose mx-auto my-8">
<Breadcrumbs links={[{ title: repoId, href: "" }]} />
<h1 className="mb-0 mt-16">{project.repo_config.name || repoId}</h1>
<p className='mb-8'><span className='font-semibold'>Repository:</span> <a target="_blank" href={project.html_url}>{project.html_url}</a></p>
<p className="mb-8">
<span className="font-semibold">Repository:</span>{" "}
<a target="_blank" href={project.html_url}>
{project.html_url}
</a>
</p>
<h2 className="mb-0 mt-10">Files</h2>
<div className="inline-block min-w-full py-2 align-middle">
@ -54,7 +63,7 @@ export default function ProjectPage({ project }) {
<hr />
<h2 className='uppercase font-black'>Readme</h2>
<h2 className="uppercase font-black">Readme</h2>
<ReactMarkdown remarkPlugins={[remarkGfm]}>
{project.readmeContent}
</ReactMarkdown>
@ -65,17 +74,14 @@ export default function ProjectPage({ project }) {
// Generates `/posts/1` and `/posts/2`
export async function getStaticPaths() {
const jsonDirectory = path.join(
process.cwd(),
'datasets.json'
);
const repos = await fs.readFile(jsonDirectory, 'utf8');
const jsonDirectory = path.join(process.cwd(), "datasets.json");
const repos = await fs.readFile(jsonDirectory, "utf8");
return {
paths: JSON.parse(repos).map((repo) => {
const projectPath =
repo.readme.split('/').length > 1
? repo.readme.split('/').slice(0, -1)
repo.readme && repo.readme.split("/").length > 1
? repo.readme.split("/").slice(0, -1)
: null;
let path = [repo.repo];
if (projectPath) {
@ -92,16 +98,13 @@ export async function getStaticPaths() {
}
export async function getStaticProps({ params }) {
const jsonDirectory = path.join(
process.cwd(),
'datasets.json'
);
const reposFile = await fs.readFile(jsonDirectory, 'utf8');
const jsonDirectory = path.join(process.cwd(), "datasets.json");
const reposFile = await fs.readFile(jsonDirectory, "utf8");
const repos: GithubProject[] = JSON.parse(reposFile);
const repo = repos.find((_repo) => {
const projectPath =
_repo.readme.split('/').length > 1
? _repo.readme.split('/').slice(0, -1)
_repo.readme && _repo.readme.split("/").length > 1
? _repo.readme.split("/").slice(0, -1)
: null;
let path = [_repo.repo];
if (projectPath) {

View File

@ -1,6 +1,6 @@
import { AppProps } from 'next/app';
import Head from 'next/head';
import './styles.css';
import { AppProps } from "next/app";
import Head from "next/head";
import "./styles.css";
function CustomApp({ Component, pageProps }: AppProps) {
return (

View File

@ -1,21 +1,19 @@
import { promises as fs } from 'fs';
import path from 'path';
import { getProject } from '../lib/octokit';
import getConfig from 'next/config';
import ExternalLinkIcon from '../components/icons/ExternalLinkIcon';
import TimeAgo from 'react-timeago';
import Link from 'next/link';
import { promises as fs } from "fs";
import path from "path";
import { getProject } from "../lib/octokit";
import getConfig from "next/config";
import ExternalLinkIcon from "../components/icons/ExternalLinkIcon";
import TimeAgo from "react-timeago";
import Link from "next/link";
import { NextSeo } from "next-seo";
export async function getStaticProps() {
const jsonDirectory = path.join(
process.cwd(),
'/datasets.json'
);
const repos = await fs.readFile(jsonDirectory, 'utf8');
const jsonDirectory = path.join(process.cwd(), "/datasets.json");
const repos = await fs.readFile(jsonDirectory, "utf8");
const github_pat = getConfig().serverRuntimeConfig.github_pat;
const projects = await Promise.all(
(JSON.parse(repos)).map(async (repo) => {
JSON.parse(repos).map(async (repo) => {
const project = await getProject(repo, github_pat);
return { ...project, repo_config: repo };
})
@ -29,88 +27,112 @@ export async function getStaticProps() {
export function Datasets({ projects }) {
return (
<div className="bg-white min-h-screen">
<div className="mx-auto max-w-7xl px-6 py-16 sm:py-24 lg:px-8">
<div className='text-center'>
<h2 className="text-3xl font-bold leading-10 tracking-tight">
GitHub Datasets
</h2>
<p className="mt-3 mx-auto max-w-2xl text-base leading-7 text-gray-500">
Data catalog with datasets hosted on GitHub by <Link target="_blank" className='underline' href="https://portaljs.org/">🌀 PortalJS</Link>
</p>
</div>
<div className="mt-20">
<div className="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
<div className="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
<table className="min-w-full divide-y divide-gray-300">
<thead>
<tr>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Name
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Repository
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Description
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Last updated
</th>
<th
scope="col"
className="relative py-3.5 pl-3 pr-4 sm:pr-0"
></th>
</tr>
</thead>
<tbody className="divide-y divide-gray-200">
{projects.map((project) => (
<tr key={project.id}>
<td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
{project.repo_config.name
? project.repo_config.name
: project.full_name + (project.base_path === '/' ? '' : '/' + project.base_path)}
</td>
<td className="whitespace-nowrap px-3 py-6 text-sm group text-gray-500 hover:text-gray-900 transition-all duration-250">
<a href={project.html_url} target="_blank" className='flex items-center'>@{project.full_name} <ExternalLinkIcon className='ml-1' /></a>
</td>
<td className="px-3 py-4 text-sm text-gray-500">
{project.repo_config.description
? project.repo_config.description
: project.description}
</td>
<td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
<TimeAgo date={new Date(project.last_updated)} />
</td>
<td className="relative whitespace-nowrap py-6 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
<a
href={`/@${project.repo_config.owner}/${project.repo_config.repo}/${project.base_path === '/' ? '' : project.base_path}`}
className='border border-gray-900 text-gray-900 px-4 py-2 transition-all hover:bg-gray-900 hover:text-white'
>
info
</a>
</td>
<>
<NextSeo title="GitHub Datasets" />
<div className="bg-white min-h-screen">
<div className="mx-auto max-w-7xl px-6 py-16 sm:py-24 lg:px-8">
<div className="text-center">
<h2 className="text-3xl font-bold leading-10 tracking-tight">
GitHub Datasets
</h2>
<p className="mt-3 mx-auto max-w-2xl text-base leading-7 text-gray-500">
Data catalog with datasets hosted on GitHub by{" "}
<Link
target="_blank"
className="underline"
href="https://portaljs.org/"
>
🌀 PortalJS
</Link>
</p>
</div>
<div className="mt-20">
<div className="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
<div className="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
<table className="min-w-full divide-y divide-gray-300">
<thead>
<tr>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Name
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Repository
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Description
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Last updated
</th>
<th
scope="col"
className="relative py-3.5 pl-3 pr-4 sm:pr-0"
></th>
</tr>
))}
</tbody>
</table>
</thead>
<tbody className="divide-y divide-gray-200">
{projects.map((project) => (
<tr key={project.id}>
<td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
{project.repo_config.name
? project.repo_config.name
: project.full_name +
(project.base_path === "/"
? ""
: "/" + project.base_path)}
</td>
<td className="whitespace-nowrap px-3 py-6 text-sm group text-gray-500 hover:text-gray-900 transition-all duration-250">
<a
href={project.html_url}
target="_blank"
className="flex items-center"
>
@{project.full_name}{" "}
<ExternalLinkIcon className="ml-1" />
</a>
</td>
<td className="px-3 py-4 text-sm text-gray-500">
{project.repo_config.description
? project.repo_config.description
: project.description}
</td>
<td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
<TimeAgo date={new Date(project.last_updated)} />
</td>
<td className="relative whitespace-nowrap py-6 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
<a
href={`/@${project.repo_config.owner}/${
project.repo_config.repo
}/${
project.base_path === "/" ? "" : project.base_path
}`}
className="border border-gray-900 text-gray-900 px-4 py-2 transition-all hover:bg-gray-900 hover:text-white"
>
info
</a>
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</div>
</>
);
}

View File

@ -3,4 +3,4 @@ module.exports = {
tailwindcss: {},
autoprefixer: {},
},
}
};

View File

@ -8,8 +8,5 @@ module.exports = {
theme: {
extend: {},
},
plugins: [
require('@tailwindcss/typography')
],
}
plugins: [require("@tailwindcss/typography")],
};