Compare commits

..

15 Commits

Author SHA1 Message Date
Luccas Mateus de Medeiros Gomes
8a7be6e402 Remove things from openspending 2023-05-18 13:48:01 -03:00
Luccas Mateus de Medeiros Gomes
fa05e374c9 [docs][howto] - [!info] on vegalite 2023-05-18 13:45:07 -03:00
Luccas Mateus de Medeiros Gomes
f52832624f [docs][howto] - added catalog code 2023-05-18 13:43:29 -03:00
Luccas Mateus
f2e7f157b9
Update site/content/howto/drd.md
Co-authored-by: Ola Rubaj <52197250+olayway@users.noreply.github.com>
2023-05-18 13:26:43 -03:00
Luccas Mateus
ddc954d6bb
Update site/content/howto/drd.md
Co-authored-by: Ola Rubaj <52197250+olayway@users.noreply.github.com>
2023-05-18 13:26:33 -03:00
Luccas Mateus
7c62a8c93f
Update site/content/howto/drd.md
Co-authored-by: Ola Rubaj <52197250+olayway@users.noreply.github.com>
2023-05-18 13:26:08 -03:00
Luccas Mateus
64cc1355bb
Update site/content/howto/drd.md
Co-authored-by: Ola Rubaj <52197250+olayway@users.noreply.github.com>
2023-05-18 13:25:56 -03:00
Luccas Mateus
8945e7dd85
Update site/content/howto/drd.md
Co-authored-by: Ola Rubaj <52197250+olayway@users.noreply.github.com>
2023-05-18 13:25:37 -03:00
Luccas Mateus
92f6c5eb47
Update site/content/howto/drd.md
Co-authored-by: Ola Rubaj <52197250+olayway@users.noreply.github.com>
2023-05-18 13:25:29 -03:00
Luccas Mateus
ff8157bf57
Update site/content/howto/drd.md
Co-authored-by: Ola Rubaj <52197250+olayway@users.noreply.github.com>
2023-05-18 13:25:16 -03:00
Luccas Mateus de Medeiros Gomes
ba3efc9ec7 [docs][xs] - remove my version of files 2023-05-18 11:51:01 -03:00
Luccas Mateus de Medeiros Gomes
87c46aba04 [docs][xs] - added catalog example 2023-05-18 11:48:59 -03:00
Luccas Mateus de Medeiros Gomes
964eb5b3ee [docs][m] - start of drd page 2023-05-18 07:57:20 -03:00
Luccas Mateus de Medeiros Gomes
f93d4aa6bd [site][m] - start of developer faq 2023-05-17 19:41:27 -03:00
Luccas Mateus de Medeiros Gomes
902e5e07a0 [examples/openspending][m] - added loader + fetching from datapackage
- Also added an indexing example
2023-05-17 14:57:50 -03:00
1475 changed files with 34556 additions and 620432 deletions

View File

@ -1,8 +0,0 @@
# Changesets
Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
with multi-package repos, or single-package repos to help you version and publish your code. You can
find the full documentation for it [in our repository](https://github.com/changesets/changesets)
We have a quick list of common questions to get you started engaging with this project in
[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)

View File

@ -1,14 +0,0 @@
{
"$schema": "https://unpkg.com/@changesets/config@2.3.0/schema.json",
"changelog": [
"@changesets/changelog-github",
{ "repo": "datopian/portaljs" }
],
"commit": false,
"fixed": [],
"linked": [],
"access": "restricted",
"baseBranch": "main",
"updateInternalDependencies": "patch",
"ignore": []
}

View File

@ -1,39 +0,0 @@
name: Release
on:
push:
branches:
- main
concurrency: release-${{ github.ref }}
jobs:
release:
name: Release
runs-on: ubuntu-latest
steps:
- name: Checkout Repo
uses: actions/checkout@v3
- name: Setup Node.js 16.x
uses: actions/setup-node@v3
with:
node-version: 16.x
- name: Install Dependencies
run: npm ci
- name: Create Release Pull Request or Publish to npm
id: changesets
uses: changesets/action@v1
with:
publish: npm run release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
# - name: Send a Discord notification if a publish happens
# if: steps.changesets.outputs.published == 'true'
# uses: Ilshidur/action-discord@0.3.2
# with:
# args: 'The project {{ EVENT_PAYLOAD.repository.full_name }} has been deployed.'

2
.gitignore vendored
View File

@ -4,7 +4,6 @@
dist
tmp
/out-tsc
**/*.tgz
# dependencies
node_modules
@ -17,7 +16,6 @@ node_modules
*.launch
.settings/
*.sublime-workspace
.obsidian
# IDE - VSCode
.vscode/*

8
.vscode/extensions.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"recommendations": [
"nrwl.angular-console",
"esbenp.prettier-vscode",
"firsttris.vscode-jest-runner",
"dbaeumer.vscode-eslint"
]
}

View File

@ -4,7 +4,7 @@ title: Developer docs for contributors
## Our repository
https://github.com/datopian/datahub
https://github.com/datopian/portaljs
Structure:
@ -13,11 +13,11 @@ Structure:
- **dataset-frictionless**: Example utilizing a frictionless dataset as an example
- **site**: the website for the project, with a landing page and the docs
- **packages**:
- **portaljs-components**: the library of components for creating a data portal
- **portaljs-components**: the library of components for creating a data portal
## How to contribute
You can start by checking our [issues board](https://github.com/datopian/datahub/issues).
You can start by checking our [issues board](https://github.com/datopian/portaljs/issues).
If you'd like to work on one of the issues you can:
@ -26,16 +26,15 @@ If you'd like to work on one of the issues you can:
3. Clone the forked repository to your machine.
4. Create a feature branch (e.g. `50-update-readme`, where `50` is the number of the related issue).
5. Commit your changes to the feature branch.
6. Add changeset file describing the changes. (See section below)
7. Push the feature branch to your forked repository.
8. Create a Pull Request against the original repository.
6. Push the feature branch to your forked repository.
7. Create a Pull Request against the original repository.
- add a short description of the changes included in the PR
9. Address review comments if requested by our demanding reviewers 😜.
8. Address review comments if requested by our demanding reviewers 😜.
If you have an idea for improvement, and it doesn't have a corresponding issue yet, simply submit a new one.
> [!note]
> Join our [Discord channel](https://discord.gg/KZSf3FG4EZ) do discuss existing issues and to ask for help.
> Join our [Discord channel](https://discord.gg/rTxfCutu) do discuss existing issues and to ask for help.
## Nx
@ -63,7 +62,6 @@ or you can use just:
nx <target> <project>
# e.g. npx nx serve ckan
```
if you have the `nx` binary installed globally in your machine
#### Running multiple tasks
@ -111,7 +109,7 @@ To check code formatting in selected projects:
```sh
npx nx format:check --projects=<array of projects>
# npx nx format:check --projects=ckan,dataset-frictionless
# npx nx format:check --projects=ckan,dataset-frictionless
```
To check code formatting in all projects:
@ -126,7 +124,7 @@ To fix code formatting in selected projects:
```sh
npx nx format:write --projects=<array projects>
# npx nx format:write --projects=ckan,dataset-frictionless
# npx nx format:write --projects=ckan,dataset-frictionless
```
To fix formatting in all projects:
@ -176,23 +174,3 @@ To learn more see this [offical docs page](https://nx.dev/reference/nx-json).
Each project also has it's own configuration file - `project.json`, where you can define and configure it's targets (and more).
To learn more see this [offical docs page](https://nx.dev/reference/project-configuration).
## Changesets and publishing packages
> This monorepo is set up with changesets versioning tool. See their [github repository](https://github.com/changesets/changesets) to learn more.
### What are Changesets?
Changesets are files that describe the intention of a contributor to bump a version of the package according to their changes. Changeset file holds two key bits of information: a version type (following semver), and change information to be added to a changelog.
### Adding changesets
In the root directory of the repo, run:
```
npx changeset
```
Select the package that has been changed, the semver version that should be bumped with it and a description of your changes. Please make sure to add the most accurate but also concise information.
To learn about semantic versioning standards see [this semver doc page](https://semver.org/).

View File

@ -1,51 +1,31 @@
<p align="center">
Bugs, issues and suggestions re PortalJS framework
<br />
<br /><a href="https://discord.gg/xfFDMPU9dC"><img src="https://dcbadge.vercel.app/api/server/xfFDMPU9dC" /></a>
</p>
<h1 align="center">
🌀 Portal.JS
<br />
Rapidly build rich data portals using a modern frontend framework
</h1>
## PortalJS framework
* [What is Portal.JS ?](#What-is-Portal.JS)
* [Features](#Features)
* [For developers](#For-developers)
* [Docs](#Docs)
* [Community](#Community)
* [Appendix](#Appendix)
* [What happened to Recline?](#What-happened-to-Recline?)
This repo and issue tracker are for
# What is Portal.JS
- PortalJS 🌀 - https://www.portaljs.com/
- DataHub Cloud ☁️ - https://datahub.io/
🌀 Portal.JS is a framework for rapidly building rich data portal frontends using a modern frontend approach. Portal.JS can be used to present a single dataset or build a full-scale data catalog/portal.
### Issues
Built in JavaScript and React on top of the popular [Next.js](https://nextjs.com/) framework. Portal.JS assumes a "decoupled" approach where the frontend is a separate service from the backend and interacts with backend(s) via an API. It can be used with any backend and has out of the box support for [CKAN](https://ckan.org/).
Found a bug: 👉 https://github.com/datopian/portaljs/issues/new
### Discussions
Got a suggestion, a question, want some support or just want to shoot the breeze 🙂
Head to the discussion forum: 👉 https://github.com/datopian/portaljs/discussions
### Chat on Discord
If you would prefer to get help via live chat check out our discord 👉
[Discord](https://discord.gg/xfFDMPU9dC)
### Docs
- For PortalJS go to https://www.portaljs.com/opensource
- For DataHub Cloud https://datahub.io/docs
## PortalJS Cloud 🌀
PortalJS Cloud 🌀 is a platform for rapidly creating rich data portal and publishing systems using a modern frontend approach. PortalJS Cloud can be used to publish a single dataset or build a full-scale data catalog/portal.
PortalJS Cloud is built in JavaScript and React on top of the popular [Next.js](https://nextjs.org) framework. PortalJS Cloud assumes a "decoupled" approach where the frontend is a separate service from the backend and interacts with backend(s) via an API. It can be used with any backend and has out of the box support for [CKAN](https://ckan.org/), GitHub, Frictionless Data Packages and more.
### Features
## Features
- 🗺️ Unified sites: present data and content in one seamless site, pulling datasets from a DMS (e.g. CKAN) and content from a CMS (e.g. Wordpress) with a common internal API.
- 👩‍💻 Developer friendly: built with familiar frontend tech (JavaScript, React, Next.js).
- 🔋 Batteries included: full set of portal components out of the box e.g. catalog search, dataset showcase, blog, etc.
- 🎨 Easy to theme and customize: installable themes, use standard CSS and React+CSS tooling. Add new routes quickly.
- 🧱 Extensible: quickly extend and develop/import your own React components
- 📝 Well documented: full set of documentation plus the documentation of Next.js.
- 📝 Well documented: full set of documentation plus the documentation of Next.js and Apollo.
### For developers
@ -53,3 +33,25 @@ PortalJS Cloud is built in JavaScript and React on top of the popular [Next.js](
- 🚀 Next.js framework: so everything in Next.js for free: Server Side Rendering, Static Site Generation, huge number of examples and integrations, etc.
- Server Side Rendering (SSR) => Unlimited number of pages, SEO and more whilst still using React.
- Static Site Generation (SSG) => Ultra-simple deployment, great performance, great lighthouse scores and more (good for small sites)
#### **Check out the [Portal.JS website](https://portaljs.org/) for a gallery of live portals**
___
# Docs
Access the Portal.JS documentation at:
https://portaljs.org/docs
- [Examples](https://portaljs.org/docs#examples)
# Community
If you have questions about anything related to Portal.JS, you're always welcome to ask our community on [GitHub Discussions](https://github.com/datopian/portal.js/discussions) or on our [Discord server](https://discord.gg/EeyfGrGu4U).
# Appendix
## What happened to Recline?
Portal.JS used to be Recline(JS). If you are looking for the old Recline codebase it still exists: see the [`recline` branch](https://github.com/datopian/portal.js/tree/recline). If you want context for the rename see [this issue](https://github.com/datopian/portal.js/issues/520).

View File

@ -2,7 +2,7 @@
**🚩 UPDATE April 2023: This example is now deprecated - though still works!. Please use the [new CKAN examples](https://github.com/datopian/portaljs/tree/main/examples)**
This example shows how you can build a full data portal using a CKAN Backend with a Next.JS Frontend powered by Apollo, a full fledged guide is available as a [blog post](https://portaljs.com/blog/example-ckan-2021)
This example shows how you can build a full data portal using a CKAN Backend with a Next.JS Frontend powered by Apollo, a full fledged guide is available as a [blog post](https://portaljs.org/blog/example-ckan-2021)
## Developers

View File

@ -0,0 +1 @@
DMS=https://demo.dev.datopian.com

View File

@ -1,7 +1,7 @@
This is a repo intended to serve as an example of a data catalog that get its data from a CKAN Instance.
```
npx create-next-app <app-name> --example https://github.com/datopian/datahub/tree/main/examples/ckan-ssg
npx create-next-app <app-name> --example https://github.com/datopian/portaljs/tree/main/examples/ckan-example
cd <app-name>
```
@ -19,7 +19,7 @@ npm run dev
Congratulations, you now have something similar to this running on `http://localhost:4200`
![](https://media.discordapp.net/attachments/1069718983604977754/1098252297726865408/image.png?width=853&height=461)
If you go to any one of those pages by clicking on `More info` you will see something similar to this
If yo go to any one of those pages by clicking on `More info` you will see something similar to this
![](https://media.discordapp.net/attachments/1069718983604977754/1098252298074988595/image.png?width=853&height=461)
## Deployment

View File

@ -10,8 +10,6 @@
},
"dependencies": {
"@heroicons/react": "^2.0.17",
"@portaljs/ckan": "^0.0.2",
"@portaljs/remark-wiki-link": "^1.0.4",
"next": "13.3.1",
"next-seo": "^6.0.0",
"octokit": "^2.0.14",
@ -22,14 +20,14 @@
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.9",
"@types/node": "18.16.0",
"@types/react": "18.0.38",
"@types/react-dom": "18.0.11",
"autoprefixer": "^10.4.14",
"eslint": "8.39.0",
"eslint-config-next": "13.3.1",
"postcss": "^8.4.23",
"tailwindcss": "^3.3.1",
"typescript": "5.0.4"
"eslint": "8.39.0",
"eslint-config-next": "13.3.1",
"typescript": "5.0.4",
"@types/node": "18.16.0",
"@types/react": "18.0.38",
"@types/react-dom": "18.0.11"
}
}

View File

@ -11,9 +11,8 @@ import {
ServerIcon,
UserIcon,
} from '@heroicons/react/20/solid';
import { CKAN } from '@portaljs/ckan';
const backend_url = getConfig().publicRuntimeConfig.DMS;
const dms = getConfig().publicRuntimeConfig.DMS;
const formatter = new Intl.DateTimeFormat('en-US', {
year: 'numeric',
@ -26,12 +25,14 @@ const formatter = new Intl.DateTimeFormat('en-US', {
});
export const getServerSideProps: GetServerSideProps = async (context) => {
const ckan = new CKAN(backend_url)
const { dataset } = context.query;
const _dataset = await ckan.getDatasetDetails(dataset as string)
const response = await fetch(
`${dms}/api/3/action/package_show?id=${dataset}`
);
const _dataset = await response.json();
return {
props: {
dataset: _dataset,
dataset: _dataset.result,
},
};
};

View File

@ -1,8 +1,7 @@
import getConfig from 'next/config';
import styles from './index.module.css';
import { CKAN } from '@portaljs/ckan';
const backend_url = getConfig().publicRuntimeConfig.DMS
const dms = getConfig().publicRuntimeConfig.DMS
const formatter = new Intl.DateTimeFormat('en-US', {
year: 'numeric',
@ -16,11 +15,12 @@ const formatter = new Intl.DateTimeFormat('en-US', {
export async function getServerSideProps() {
const ckan = new CKAN(backend_url)
const { datasets } = await ckan.packageSearch({ limit: 1000, offset: 0, groups:[], orgs: [], tags: []})
const datasetsWithDetails = await Promise.all(datasets.map(async (dataset) => {
const _dataset = await ckan.getDatasetDetails(dataset.name)
return _dataset
const response = await fetch(`${dms}/api/3/action/package_search`)
const datasets = await response.json()
const datasetsWithDetails = await Promise.all(datasets.result.results.map(async (dataset) => {
const response = await fetch(`${dms}/api/3/action/package_show?id=` + dataset.name)
const json = await response.json()
return json.result
}))
return {
@ -79,7 +79,7 @@ export function Index({ datasets }) {
</thead>
<tbody className="divide-y divide-gray-200">
{datasets.map((dataset) => (
<tr key={dataset.name}>
<tr>
<td className="px-3 py-4 text-sm text-gray-500">
{dataset.title}
</td>

View File

@ -1,3 +0,0 @@
{
"extends": "next/core-web-vitals"
}

View File

@ -1,35 +0,0 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.js
# testing
/coverage
# next.js
/.next/
/out/
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# local env files
.env*.local
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts

View File

@ -1,38 +0,0 @@
This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
## Getting Started
First, run the development server:
```bash
npm run dev
# or
yarn dev
# or
pnpm dev
```
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file.
[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
## Learn More
To learn more about Next.js, take a look at the following resources:
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
## Deploy on Vercel
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

View File

@ -1,21 +0,0 @@
import { MDXRemote } from 'next-mdx-remote';
import dynamic from 'next/dynamic';
import { Mermaid } from '@portaljs/core';
// Custom components/renderers to pass to MDX.
// Since the MDX files aren't loaded by webpack, they have no knowledge of how
// to handle import statements. Instead, you must include components in scope
// here.
const components = {
Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
mermaid: Mermaid,
Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),
LineChart: dynamic(() => import('@portaljs/components').then(mod => mod.LineChart)),
} as any;
export default function DRD({ source }: { source: any }) {
return <MDXRemote {...source} components={components} />;
}

View File

@ -1,3 +0,0 @@
# Test
Test Data Rich Stories

View File

@ -1,105 +0,0 @@
import matter from "gray-matter";
import mdxmermaid from "mdx-mermaid";
import { h } from "hastscript";
import remarkCallouts from "@portaljs/remark-callouts";
import remarkEmbed from "@portaljs/remark-embed";
import remarkGfm from "remark-gfm";
import remarkMath from "remark-math";
import remarkSmartypants from "remark-smartypants";
import remarkToc from "remark-toc";
import remarkWikiLink from "@portaljs/remark-wiki-link";
import rehypeAutolinkHeadings from "rehype-autolink-headings";
import rehypeKatex from "rehype-katex";
import rehypeSlug from "rehype-slug";
import rehypePrismPlus from "rehype-prism-plus";
import { serialize } from "next-mdx-remote/serialize";
/**
* Parse a markdown or MDX file to an MDX source form + front matter data
*
* @source: the contents of a markdown or mdx file
* @format: used to indicate to next-mdx-remote which format to use (md or mdx)
* @returns: { mdxSource: mdxSource, frontMatter: ...}
*/
const parse = async function (source, format, scope) {
const { content, data, excerpt } = matter(source, {
excerpt: (file, options) => {
// Generate an excerpt for the file
file.excerpt = file.content.split("\n\n")[0];
},
});
const mdxSource = await serialize(
{ value: content, path: format },
{
// Optionally pass remark/rehype plugins
mdxOptions: {
remarkPlugins: [
remarkEmbed,
remarkGfm,
[remarkSmartypants, { quotes: false, dashes: "oldschool" }],
remarkMath,
remarkCallouts,
remarkWikiLink,
[
remarkToc,
{
heading: "Table of contents",
tight: true,
},
],
[mdxmermaid, {}],
],
rehypePlugins: [
rehypeSlug,
[
rehypeAutolinkHeadings,
{
properties: { className: 'heading-link' },
test(element) {
return (
["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
element.properties?.id !== "table-of-contents" &&
element.properties?.className !== "blockquote-heading"
);
},
content() {
return [
h(
"svg",
{
xmlns: "http:www.w3.org/2000/svg",
fill: "#ab2b65",
viewBox: "0 0 20 20",
className: "w-5 h-5",
},
[
h("path", {
fillRule: "evenodd",
clipRule: "evenodd",
d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
}),
]
),
];
},
},
],
[rehypeKatex, { output: "mathml" }],
[rehypePrismPlus, { ignoreMissing: true }],
],
format,
},
scope,
}
);
return {
mdxSource: mdxSource,
frontMatter: data,
excerpt,
};
};
export default parse;

View File

@ -1,14 +0,0 @@
import { MarkdownDB } from "mddb";
const dbPath = "markdown.db";
const client = new MarkdownDB({
client: "sqlite3",
connection: {
filename: dbPath,
},
});
const clientPromise = client.init();
export default clientPromise;

View File

@ -1,11 +0,0 @@
/** @type {import('next').NextConfig} */
const nextConfig = {
reactStrictMode: true,
publicRuntimeConfig: {
DMS: process.env.DMS
? process.env.DMS.replace(/\/?$/, '')
: 'https://demo.dev.datopian.com/',
},
};
module.exports = nextConfig;

File diff suppressed because it is too large Load Diff

View File

@ -1,48 +0,0 @@
{
"name": "ckan",
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "next dev",
"prebuild": "npm run mddb",
"build": "next build",
"start": "next start",
"lint": "next lint",
"mddb": "mddb ./content"
},
"dependencies": {
"@githubocto/flat-ui": "^0.14.1",
"@heroicons/react": "^2.0.18",
"@portaljs/ckan": "^0.0.2",
"@portaljs/components": "0.1.6",
"@portaljs/core": "^1.0.5",
"@portaljs/remark-callouts": "^1.0.5",
"@portaljs/remark-embed": "^1.0.4",
"@portaljs/remark-wiki-link": "^1.0.4",
"@tailwindcss/typography": "^0.5.9",
"@types/node": "20.2.3",
"@types/react": "18.2.6",
"@types/react-dom": "18.2.4",
"autoprefixer": "10.4.14",
"eslint": "8.41.0",
"eslint-config-next": "13.4.3",
"isomorphic-unfetch": "^4.0.2",
"mddb": "^0.1.9",
"next": "13.4.3",
"next-mdx-remote": "^4.4.1",
"papaparse": "^5.4.1",
"postcss": "8.4.23",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-query": "^3.39.3",
"rehype-autolink-headings": "^6.1.1",
"rehype-katex": "^6.0.3",
"rehype-prism-plus": "^1.5.1",
"rehype-slug": "^5.1.0",
"remark-math": "^5.1.1",
"remark-smartypants": "^2.0.0",
"remark-toc": "^8.0.1",
"tailwindcss": "3.3.2",
"typescript": "5.0.4"
}
}

View File

@ -1,179 +0,0 @@
import Head from "next/head";
import { CKAN, Dataset } from "@portaljs/ckan";
import {
ChevronRightIcon,
HomeIcon,
PaperClipIcon,
} from "@heroicons/react/20/solid";
import Link from "next/link";
import getConfig from "next/config";
const backend_url = getConfig().publicRuntimeConfig.DMS
export const getServerSideProps = async (context: any) => {
try {
const datasetName = context.params?.dataset;
if (!datasetName) {
return {
notFound: true,
};
}
const ckan = new CKAN(backend_url);
const dataset = await ckan.getDatasetDetails(datasetName as string);
if (!dataset) {
return {
notFound: true,
};
}
return {
props: { dataset },
};
} catch {
return {
notFound: true,
};
}
};
export default function DatasetPage({
dataset,
}: {
dataset: Dataset;
}): JSX.Element {
return (
<>
<Head>
<title>{`${dataset.title || dataset.name} - Dataset`}</title>
<meta name="description" content="Generated by create next app" />
<link rel="icon" href="/favicon.ico" />
</Head>
<main className="flex min-h-screen flex-col items-center justify-between p-24 bg-zinc-900">
<div className="bg-white p-8 my-4 rounded-lg">
<nav className="flex px-4 py-8" aria-label="Breadcrumb">
<ol role="list" className="flex items-center space-x-4">
<li>
<div>
<Link href="/" className="text-gray-400 hover:text-gray-500">
<HomeIcon
className="h-5 w-5 flex-shrink-0"
aria-hidden="true"
/>
<span className="sr-only">Home</span>
</Link>
</div>
</li>
<li>
<div className="flex items-center">
<ChevronRightIcon
className="h-5 w-5 flex-shrink-0 text-gray-400"
aria-hidden="true"
/>
<span
className="ml-4 text-sm font-medium text-gray-500 hover:text-gray-700"
aria-current={"page"}
>
{dataset.name}
</span>
</div>
</li>
</ol>
</nav>
{dataset && (
<div>
<div className="px-4 sm:px-0">
<h3 className="text-base font-semibold leading-7 text-gray-900">
{dataset.title || dataset.name}
</h3>
<p className="mt-1 max-w-2xl text-sm leading-6 text-gray-500">
Dataset details
</p>
</div>
<div className="mt-6 border-t border-gray-100">
<dl className="divide-y divide-gray-100">
<div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
<dt className="text-sm font-medium leading-6 text-gray-900">
Title
</dt>
<dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
{dataset.title}
</dd>
</div>
{dataset.tags && dataset.tags.length > 0 && (
<div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
<dt className="text-sm font-medium leading-6 text-gray-900">
Tags
</dt>
<dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
{dataset.tags.map((tag) => tag.display_name).join(", ")}
</dd>
</div>
)}
{dataset.tags && dataset.tags.length > 0 && (
<div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
<dt className="text-sm font-medium leading-6 text-gray-900">
URL
</dt>
<dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
{dataset.url}
</dd>
</div>
)}
<div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
{dataset.notes && (
<>
<dt className="text-sm font-medium leading-6 text-gray-900">
Description
</dt>
<dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
{dataset.notes}
</dd>
</>
)}
</div>
<div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
<dt className="text-sm font-medium leading-6 text-gray-900">
Files
</dt>
<dd className="mt-2 text-sm text-gray-900 sm:col-span-2 sm:mt-0">
<ul
role="list"
className="divide-y divide-gray-100 rounded-md border border-gray-200"
>
{dataset.resources.map((resource) => (
<li key={resource.id} className="flex items-center justify-between py-4 pl-4 pr-5 text-sm leading-6">
<div className="flex w-0 flex-1 items-center">
<PaperClipIcon
className="h-5 w-5 flex-shrink-0 text-gray-400"
aria-hidden="true"
/>
<div className="ml-4 flex min-w-0 flex-1 gap-2">
<span className="truncate font-medium">
{resource.name || resource.id}
</span>
<span className="flex-shrink-0 text-gray-400">
{resource.size}
</span>
</div>
</div>
<div className="ml-4 flex-shrink-0">
<a
href={resource.url}
className="font-medium hover:text-indigo-500 mr-4"
>
Download
</a>
</div>
</li>
))}
</ul>
</dd>
</div>
</dl>
</div>
</div>
)}
</div>
</main>
</>
);
}

View File

@ -1,7 +0,0 @@
import '@/styles/globals.css'
import '@portaljs/ckan/styles.css'
import type { AppProps } from 'next/app'
export default function App({ Component, pageProps }: AppProps) {
return <Component {...pageProps} />
}

View File

@ -1,13 +0,0 @@
import { Html, Head, Main, NextScript } from 'next/document'
export default function Document() {
return (
<Html lang="en">
<Head />
<body>
<Main />
<NextScript />
</body>
</Html>
)
}

View File

@ -1,18 +0,0 @@
import fetch from 'isomorphic-unfetch';
const Cors = async (req: any, res: any) => {
const { url } = req.query;
try {
const resProxy = await fetch(url, {
headers: {
Range: 'bytes=0-5132288',
},
});
const data = await resProxy.text();
return res.status(200).send(data);
} catch (error: any) {
res.status(400).send(error.toString());
}
};
export default Cors;

View File

@ -1,64 +0,0 @@
import {
CKAN,
DatasetSearchForm,
ListOfDatasets,
PackageSearchOptions,
Organization,
Group,
} from '@portaljs/ckan';
import getConfig from 'next/config';
import { useState } from 'react';
const backend_url = getConfig().publicRuntimeConfig.DMS;
export async function getServerSideProps() {
const ckan = new CKAN(backend_url);
const groups = await ckan.getGroupsWithDetails();
const orgs = await ckan.getOrgsWithDetails();
return {
props: {
groups,
orgs,
},
};
}
export default function Home({
orgs,
groups,
}: {
orgs: Organization[];
groups: Group[];
}) {
const ckan = new CKAN(backend_url);
const [options, setOptions] = useState<PackageSearchOptions>({
offset: 0,
limit: 5,
tags: [],
groups: [],
orgs: [],
});
return (
<div>
<main className="py-12 bg-zinc-900">
<DatasetSearchForm
options={options}
setOptions={setOptions}
groups={groups}
orgs={orgs}
/>
<div
className="bg-white p-8 mx-auto my-4 rounded-lg"
style={{ width: 'min(1100px, 95vw)' }}
>
<ListOfDatasets
options={options}
setOptions={setOptions}
ckan={ckan}
/>{' '}
</div>
</main>
</div>
);
}

View File

@ -1,129 +0,0 @@
import { existsSync, promises as fs } from 'fs';
import path from 'path';
import parse from '../../lib/markdown';
import DataRichDocument from '../../components/DataRichDocument';
import clientPromise from '../../lib/mddb';
import getConfig from 'next/config';
import { CKAN } from '@portaljs/ckan';
export const getStaticPaths = async () => {
const contentDir = path.join(process.cwd(), '/content/');
const contentFolders = await fs.readdir(contentDir, 'utf8');
const paths = contentFolders.map((folder: string) => ({
params: { path: [folder.split('.')[0]] },
}));
return {
paths,
fallback: false,
};
};
const backend_url = getConfig().publicRuntimeConfig.DMS;
export const getStaticProps = async (context) => {
const mddb = await clientPromise;
const storyFile = await mddb.getFileByUrl(context.params.path);
const md = await fs.readFile(
`${process.cwd()}/${storyFile.file_path}`,
'utf8'
);
const ckan = new CKAN(backend_url);
const datasets = storyFile.metadata.datasets ? await Promise.all(
storyFile.metadata.datasets.map(
async (datasetName: string) => await ckan.getDatasetDetails(datasetName)
)
) : [];
const orgs = storyFile.metadata.orgs ? await Promise.all(
storyFile.metadata.orgs.map(
async (orgName: string) => await ckan.getOrgDetails(orgName)
)
) : [];
let { mdxSource, frontMatter } = await parse(md, '.mdx', { datasets, orgs });
return {
props: {
mdxSource,
frontMatter: JSON.stringify(frontMatter),
},
};
};
export default function DatasetPage({ mdxSource, frontMatter }) {
frontMatter = JSON.parse(frontMatter);
return (
<main className="flex min-h-screen flex-col justify-between p-16 bg-zinc-900">
<div className="bg-white p-8 my-4 rounded-lg">
<div className="prose mx-auto py-8">
<header>
<div className="mb-6">
<>
<h1 className="mb-2">{frontMatter.title}</h1>
{frontMatter.author && (
<p className="my-0">
<span className="font-semibold">Author: </span>
<span className="my-0">{frontMatter.author}</span>
</p>
)}
{frontMatter.description && (
<p className="my-0">
<span className="font-semibold">Description: </span>
<span className="description my-0">
{frontMatter.description}
</span>
</p>
)}
{frontMatter.modified && (
<p className="my-0">
<span className="font-semibold">Modified: </span>
<span className="description my-0">
{new Date(frontMatter.modified).toLocaleDateString()}
</span>
</p>
)}
{frontMatter.files && (
<section className="py-6">
<h2 className="mt-0">Data files</h2>
<table className="table-auto">
<thead>
<tr>
<th>File</th>
<th>Format</th>
</tr>
</thead>
<tbody>
{frontMatter.files.map((f) => {
const fileName = f.split('/').slice(-1);
return (
<tr key={`resources-list-${f}`}>
<td>
<a target="_blank" href={f}>
{fileName}
</a>
</td>
<td>
{fileName[0]
.split('.')
.slice(-1)[0]
.toUpperCase()}
</td>
</tr>
);
})}
</tbody>
</table>
</section>
)}
</>
</div>
</header>
<main>
<DataRichDocument source={mdxSource} />
</main>
</div>
</div>
</main>
);
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 25 KiB

View File

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

Before

Width:  |  Height:  |  Size: 1.3 KiB

View File

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

Before

Width:  |  Height:  |  Size: 629 B

View File

@ -1,71 +0,0 @@
@tailwind base;
@tailwind components;
@tailwind utilities;
@import "@portaljs/remark-callouts/styles.css";
/* mathjax */
.math-inline > mjx-container > svg {
display: inline;
align-items: center;
}
/* smooth scrolling in modern browsers */
html {
scroll-behavior: smooth !important;
}
/* tooltip fade-out clip */
.tooltip-body::after {
content: "";
position: absolute;
right: 0;
top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
width: 10rem;
background: linear-gradient(
to right,
rgba(255, 255, 255, 0),
rgba(255, 255, 255, 1) 100%
);
}
:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
margin-left: -2rem !important;
padding-left: 2rem !important;
scroll-margin-top: 4.5rem;
position: relative;
}
.heading-link {
padding: 1px;
position: absolute;
left: 0;
top: 50%;
transform: translateY(-50%);
margin: auto 0;
border-radius: 5px;
background: #1e293b;
opacity: 0;
transition: opacity 0.2s;
}
.light .heading-link {
/* border: 1px solid #ab2b65; */
/* background: none; */
background: #e2e8f0;
}
:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
opacity: 100;
}
.heading-link svg {
transform: scale(0.75);
}
@media screen and (max-width: 640px) {
.heading-link {
visibility: hidden;
}
}

View File

@ -1,18 +0,0 @@
/** @type {import('tailwindcss').Config} */
module.exports = {
content: [
'./pages/**/*.{js,ts,jsx,tsx,mdx}',
'./components/**/*.{js,ts,jsx,tsx,mdx}',
'./app/**/*.{js,ts,jsx,tsx,mdx}',
],
theme: {
extend: {
backgroundImage: {
'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
'gradient-conic':
'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
},
},
},
plugins: [require('@tailwindcss/typography')],
}

View File

@ -1,23 +0,0 @@
{
"compilerOptions": {
"target": "es5",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": false,
"forceConsistentCasingInFileNames": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "node",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"paths": {
"@/*": ["./*"]
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
"exclude": ["node_modules"]
}

View File

@ -1,6 +1,6 @@
This example creates a portal/showcase for a single dataset. The dataset should be a [Frictionless dataset (data package)][fd] i.e. there should be a `datapackage.json`.
[fd]: https://specs.frictionlessdata.io/data-package/
[fd]: https://frictionlessdata.io/data-packages/
## How to use

View File

@ -1,9 +1,3 @@
# PortalJS Demo replicating the FiveThirtyEight data portal
## 👉 https://fivethirtyeight.portaljs.org 👈
Here's a blog post we wrote about it: https://www.datopian.com/blog/fivethirtyeight-replica
This is a replica of the awesome data.fivethirtyeight.com using PortalJS.
You might be asking why we did that, there are three main reasons:

View File

@ -1,99 +0,0 @@
import { useState } from 'react';
import { XMarkIcon } from '@heroicons/react/20/solid';
import { Transition } from '@headlessui/react';
export default function Layout({ children }: { children: React.ReactNode }) {
const [isShowing, setShow] = useState(true);
return (
<>
<Transition
show={isShowing}
enter="transition-opacity duration-75"
enterFrom="opacity-0"
enterTo="opacity-100"
leave="transition-opacity duration-150"
leaveFrom="opacity-100"
leaveTo="opacity-0"
>
<div className="flex items-center gap-x-6 bg-[#3c3c3c] px-6 py-2.5 sm:px-3.5 sm:before:flex-1">
<p className="text-sm leading-6 text-white">
This is a replica to the awesome{' '}
<a
className="hover:underline font-bold"
href="https://data.fivethirtyeight.com"
>
data.fivethirtyeight.com
</a>{' '}
website.{' '}
<a
className="hover:underline font-bold"
href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight#readme"
>
Read more here
</a>{' '}
</p>
<div className="flex flex-1 justify-end">
<button
type="button"
onClick={() => setShow(false)}
className="-m-3 p-3 focus-visible:outline-offset-[-4px]"
>
<span className="sr-only">Dismiss</span>
<XMarkIcon className="h-5 w-5 text-white" aria-hidden="true" />
</button>
</div>
</div>
</Transition>
<header className="max-w-5xl mx-auto mt-8 w-full">
<div className="border-b-2 pb-2.5 mx-2 border-zinc-800 flex justify-between">
<h1 className="flex gap-x-1 items-end">
<span className="sr-only">FiveThirtyEight</span>
<img
width="197"
height="25"
alt="FiveThirtyEight"
src="data:image/svg+xml;base64,PHN2ZyBpZD0iTGF5ZXJfMSIgZGF0YS1uYW1lPSJMYXllciAxIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjEgNTMuNzYiPjxkZWZzPjxzdHlsZT4uY2xzLTF7ZmlsbDojMDEwMTAxO308L3N0eWxlPjwvZGVmcz48dGl0bGU+QXJ0Ym9hcmQgOTU8L3RpdGxlPjxwYXRoIGNsYXNzPSJjbHMtMSIgZD0iTTAgMGgyNXY4SDl2MTBoMTV2OEg5djE3SDBWMHpNMzEgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdIMzF6bTUtMzZoOHY4aC04ek0xNzkgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdoLTE3em01LTM2aDh2OGgtOHpNMzE2IDM2aDVWMThoLTV2LThoMTN2MjZoNHY3aC0xN3ptNS0zNmg4djhoLTh6TTU0IDI3VjEwaDh2MTVsNCA5Ljk4aDFMNzEgMjVWMTBoOHYxN2wtNyAxNkg2MWwtNy0xNnpNMTExIDQzSDk3LjQyQzg5LjIzIDQzIDg1IDM5LjE5IDg1IDMxLjE3VjIyYzAtNy41NyA0LjMtMTMgMTMtMTMgOS4zMyAwIDEzIDUuMDcgMTMgMTR2N0g5NHYxLjc0YzAgMi42MiAxIDQuMjYgMy40MiA0LjI2SDExMXpNOTQgMjNoOHYtMS41NWMwLTIuNjItMS4wNi01LjQ1LTQuMTMtNS40NS0yLjc5IDAtMy44NyAyLjItMy44NyA1LjQ1ek0xMjUgOGgtMTBWMGgyOXY4aC0xMHYzNWgtOVY4ek0yMDIgNDNWMTBoOHY0YzEuMTQtMi40NSAzLjc1LTQgNy4yMi00SDIyMHY4aC02Yy0yLjg0IDAtNCAuOTQtNCAzLjlWNDN6TTI0NSA0M2gtNC44NEMyMzMuMDUgNDMgMjMwIDM5LjMxIDIzMCAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0gyNDV6TTQyMSA0M2gtNC44NEM0MDkuMDUgNDMgNDA2IDM5LjMxIDQwNiAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0g0MjF6TTI1NC4yNiA1My43Nmw0LjYxLTkuNUwyNTEgMjdWMTBoOHYxNWw0IDEwaDFsNC0xMFYxMGg4djE3bC0xMi4zIDI2Ljc2aC05LjQ0ek0yODQgMGgyNXY4aC0xNnY5aDE1djhoLTE1djEwaDE2djhoLTI1VjB6TTMzNyA0OHYtMmgxNi4xYzIgMCAyLjktLjE4IDIuOS0xLjI3di0uMzRjMC0xLjA4LS45MS0xLjM5LTIuOS0xLjM5SDM0MHYtNWw1LTVjLTUuMjktMS40OC04LTUuNDMtOC0xMXYtMWMwLTcuNTYgNC40NC0xMiAxNC0xMmEyMS45MyAyMS45MyAwIDAgMSA1Ljk1IDFMMzYxIDRsNSAzLTQgNmMxLjM3IDEuOTMgMyA0LjkzIDMgOHYxYzAgNy0zLjMgMTAuNjYtMTIgMTFsLTMgNGg2YzUuOTIgMCA5IDIuNjIgOSA3LjY4di4xMWMwIDUuMDYtMi43MSA4LjIxLTguNjIgOC4yMWgtMTNjLTQuMjkgMC02LjM4LTEuODQtNi4zOC01em0xOS0yNXYtM2MwLTMuMy0xLjMzLTQtNS00cy01IC43LTUgNHYzYzAgMy4zIDEuMzkgNCA1IDRzNS0uNyA1LTR6TTM4MCA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuNC00IDctNCA2LjI2IDAgOSAzLjA4IDkgMTAuNzZWNDNoLThWMjJjMC0zLjEzLTEuMDctNS00LTVzLTQgMS44Ny00IDV6TTE1NyA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuOTEtNCA3LjQ5LTQgNi4yNiAwIDguNTEgMy4xMyA4LjUxIDEwLjgxVjQzaC04VjIxYzAtMy4xMy0xLjA3LTQuNDQtNC00LjQ0cy00IDIuMjYtNCA1LjM5eiIvPjwvc3ZnPg=="
/>{' '}
<span className="-mb-0.5 text-[#3c3c3c]">replica</span>
</h1>
<div className="md:flex items-center gap-x-3 text-[#3c3c3c] -mb-1 hidden">
<a
className="hover:opacity-75 transition"
href="https://portaljs.com"
>
Built with 🌀PortalJS
</a>
<hr className="h-[80%] border border-[#3c3c3c] opacity-75 my-2"></hr>
<a
className="hover:opacity-75 transition"
href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
>
Github
</a>
</div>
</div>
<div className="mx-2 py-1.5 text-[14px] text-[#3c3c3c] md:hidden">
<ul className="flex gap-x-4">
<li>
<a
className="hover:opacity-75 transition"
href="https://portaljs.com"
>
PortalJS
</a>
</li>
<li>
<a
className="hover:opacity-75 transition"
href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
>
View on Github
</a>
</li>
</ul>
</div>
</header>
{children}
</>
);
}

View File

@ -1,4 +1,33 @@
[
{
"url": "https://github.com/fivethirtyeight/data/tree/master/polls",
"name": "polls",
"displayName": "<span class=\"lastword\">polls</span>",
"articles": [
{
"date": "2023-05-11T14:35:40.000Z",
"title": "Latest Polls",
"url": "https://projects.fivethirtyeight.com/polls/"
}
],
"files": [
"https://projects.fivethirtyeight.com/polls-page/data/president_primary_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/president_primary_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/president_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/president_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/senate_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/senate_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/house_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/house_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/governor_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/governor_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/president_approval_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/generic_ballot_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/generic_ballot_polls_historical.csv",
"https://projects.fivethirtyeight.com/2020-primary-data/pres_primary_avgs_2020.csv",
"https://projects.fivethirtyeight.com/2020-general-data/presidential_poll_averages_2020.csv"
]
},
{
"url": "https://github.com/fivethirtyeight/data/tree/master/congress-generic-ballot",
"name": "congress-generic-ballot",
@ -166,35 +195,6 @@
}
]
},
{
"url": "https://github.com/fivethirtyeight/data/tree/master/polls",
"name": "polls",
"displayName": "<span class=\"lastword\">polls</span>",
"articles": [
{
"date": "2023-05-11T14:35:40.000Z",
"title": "Latest Polls",
"url": "https://projects.fivethirtyeight.com/polls/"
}
],
"files": [
"https://projects.fivethirtyeight.com/polls-page/data/president_primary_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/president_primary_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/president_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/president_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/senate_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/senate_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/house_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/house_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/governor_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/governor_polls_historical.csv",
"https://projects.fivethirtyeight.com/polls-page/data/president_approval_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/generic_ballot_polls.csv",
"https://projects.fivethirtyeight.com/polls-page/data/generic_ballot_polls_historical.csv",
"https://projects.fivethirtyeight.com/2020-primary-data/pres_primary_avgs_2020.csv",
"https://projects.fivethirtyeight.com/2020-general-data/presidential_poll_averages_2020.csv"
]
},
{
"url": "https://github.com/fivethirtyeight/data/tree/master/nfl-elo",
"name": "nfl-elo",
@ -1169,6 +1169,18 @@
}
]
},
{
"url": "https://github.com/fivethirtyeight/data/tree/master/undefeated-boxers",
"name": "undefeated-boxers",
"displayName": "undefeated-<span class=\"lastword\">boxers</span>",
"articles": [
{
"date": "2017-08-18T18:47:32.000Z",
"title": "Mayweather Is Defined By The Zero Next To His Name",
"url": "https://fivethirtyeight.com/features/mayweather-is-defined-by-the-zero-next-to-his-name/"
}
]
},
{
"url": "https://github.com/fivethirtyeight/data/tree/master/chess-transfers",
"name": "chess-transfers",
@ -2127,18 +2139,6 @@
}
]
},
{
"url": "https://github.com/fivethirtyeight/data/tree/master/undefeated-boxers",
"name": "undefeated-boxers",
"displayName": "undefeated-<span class=\"lastword\">boxers</span>",
"articles": [
{
"date": "2017-08-18T18:47:32.000Z",
"title": "Mayweather Is Defined By The Zero Next To His Name",
"url": "https://fivethirtyeight.com/features/mayweather-is-defined-by-the-zero-next-to-his-name/"
}
]
},
{
"url": "https://github.com/fivethirtyeight/data/tree/master/march-madness-predictions",
"name": "march-madness-predictions",

File diff suppressed because it is too large Load Diff

View File

@ -9,11 +9,7 @@
"lint": "next lint"
},
"dependencies": {
"@headlessui/react": "^1.7.14",
"@heroicons/react": "^2.0.18",
"@portaljs/components": "^0.1.8",
"@portaljs/core": "^1.0.5",
"@portaljs/remark-wiki-link": "^1.0.4",
"@portaljs/components": "^0.1.0",
"@tailwindcss/typography": "^0.5.9",
"@types/node": "20.1.1",
"@types/react": "18.2.6",
@ -30,15 +26,12 @@
"react": "18.2.0",
"react-dom": "18.2.0",
"react-markdown": "^8.0.7",
"remark": "^14.0.3",
"remark-code-frontmatter": "^1.0.0",
"remark-excerpt": "^1.0.0-beta.1",
"remark-extract-frontmatter": "^3.2.0",
"remark-frontmatter": "^4.0.1",
"remark-gfm": "^3.0.1",
"tailwindcss": "3.3.2",
"timeago.js": "^4.0.2",
"to-vfile": "^7.2.4",
"typescript": "5.0.4"
}
}

View File

@ -1,49 +1,8 @@
import '@/styles/globals.css';
import '@portaljs/components/styles.css';
import { useEffect } from 'react';
import { pageview } from '@portaljs/core';
import Script from 'next/script';
import Head from 'next/head';
import { useRouter } from 'next/router';
import '@/styles/globals.css'
import '@portaljs/components/styles.css'
import type { AppProps } from 'next/app';
import type { AppProps } from 'next/app'
export default function App({ Component, pageProps }: AppProps) {
const router = useRouter();
useEffect(() => {
const handleRouteChange = (url: any) => {
pageview(url);
};
router.events.on('routeChangeComplete', handleRouteChange);
return () => {
router.events.off('routeChangeComplete', handleRouteChange);
};
}, [router.events]);
return (
<>
<Head>
<link rel="shortcut icon" href="/squared_logo.png" />
</Head>
<Script
strategy="afterInteractive"
src="https://www.googletagmanager.com/gtag/js?id=G-3N9SXTC7GS"
/>
<Script
id="gtag-init"
strategy="afterInteractive"
dangerouslySetInnerHTML={{
__html: `
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-3N9SXTC7GS', {
page_path: window.location.pathname,
});
`,
}}
/>
<Component {...pageProps} />
</>
);
return <Component {...pageProps} />
}

View File

@ -19,9 +19,78 @@ export default function Document() {
/>
</Head>
<body>
<div className="px-2 max-w-5xl mx-auto pb-2">
<div className="mt-2 px-2 bg-[#3c3c3c] text-white">
<div className="p-2 text-center">
This is a replica to the awesome{' '}
<a
className="hover:underline font-bold"
href="https://data.fivethirtyeight.com"
>
data.fivethirtyeight.com
</a>{' '}
website.{' '}
<a
className="hover:underline font-bold"
href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
>
Read more here
</a>{' '}
</div>
</div>
</div>
<header className="max-w-5xl mx-auto mt-8 w-full">
<div className="border-b-2 pb-2.5 mx-2 border-zinc-800 flex justify-between">
<h1 className="flex gap-x-1 items-end">
<span className="sr-only">FiveThirtyEight</span>
<img
width="197"
height="25"
alt="FiveThirtyEight"
src="data:image/svg+xml;base64,PHN2ZyBpZD0iTGF5ZXJfMSIgZGF0YS1uYW1lPSJMYXllciAxIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjEgNTMuNzYiPjxkZWZzPjxzdHlsZT4uY2xzLTF7ZmlsbDojMDEwMTAxO308L3N0eWxlPjwvZGVmcz48dGl0bGU+QXJ0Ym9hcmQgOTU8L3RpdGxlPjxwYXRoIGNsYXNzPSJjbHMtMSIgZD0iTTAgMGgyNXY4SDl2MTBoMTV2OEg5djE3SDBWMHpNMzEgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdIMzF6bTUtMzZoOHY4aC04ek0xNzkgMzZoNVYxOGgtNXYtOGgxM3YyNmg0djdoLTE3em01LTM2aDh2OGgtOHpNMzE2IDM2aDVWMThoLTV2LThoMTN2MjZoNHY3aC0xN3ptNS0zNmg4djhoLTh6TTU0IDI3VjEwaDh2MTVsNCA5Ljk4aDFMNzEgMjVWMTBoOHYxN2wtNyAxNkg2MWwtNy0xNnpNMTExIDQzSDk3LjQyQzg5LjIzIDQzIDg1IDM5LjE5IDg1IDMxLjE3VjIyYzAtNy41NyA0LjMtMTMgMTMtMTMgOS4zMyAwIDEzIDUuMDcgMTMgMTR2N0g5NHYxLjc0YzAgMi42MiAxIDQuMjYgMy40MiA0LjI2SDExMXpNOTQgMjNoOHYtMS41NWMwLTIuNjItMS4wNi01LjQ1LTQuMTMtNS40NS0yLjc5IDAtMy44NyAyLjItMy44NyA1LjQ1ek0xMjUgOGgtMTBWMGgyOXY4aC0xMHYzNWgtOVY4ek0yMDIgNDNWMTBoOHY0YzEuMTQtMi40NSAzLjc1LTQgNy4yMi00SDIyMHY4aC02Yy0yLjg0IDAtNCAuOTQtNCAzLjlWNDN6TTI0NSA0M2gtNC44NEMyMzMuMDUgNDMgMjMwIDM5LjMxIDIzMCAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0gyNDV6TTQyMSA0M2gtNC44NEM0MDkuMDUgNDMgNDA2IDM5LjMxIDQwNiAzMS44NVYxOGgtNnYtOGg2VjNoOHY3aDd2OGgtN2wtLjA3IDEzLjkzYzAgMi4yMi45MyA0LjA3IDMuNjYgNC4wN0g0MjF6TTI1NC4yNiA1My43Nmw0LjYxLTkuNUwyNTEgMjdWMTBoOHYxNWw0IDEwaDFsNC0xMFYxMGg4djE3bC0xMi4zIDI2Ljc2aC05LjQ0ek0yODQgMGgyNXY4aC0xNnY5aDE1djhoLTE1djEwaDE2djhoLTI1VjB6TTMzNyA0OHYtMmgxNi4xYzIgMCAyLjktLjE4IDIuOS0xLjI3di0uMzRjMC0xLjA4LS45MS0xLjM5LTIuOS0xLjM5SDM0MHYtNWw1LTVjLTUuMjktMS40OC04LTUuNDMtOC0xMXYtMWMwLTcuNTYgNC40NC0xMiAxNC0xMmEyMS45MyAyMS45MyAwIDAgMSA1Ljk1IDFMMzYxIDRsNSAzLTQgNmMxLjM3IDEuOTMgMyA0LjkzIDMgOHYxYzAgNy0zLjMgMTAuNjYtMTIgMTFsLTMgNGg2YzUuOTIgMCA5IDIuNjIgOSA3LjY4di4xMWMwIDUuMDYtMi43MSA4LjIxLTguNjIgOC4yMWgtMTNjLTQuMjkgMC02LjM4LTEuODQtNi4zOC01em0xOS0yNXYtM2MwLTMuMy0xLjMzLTQtNS00cy01IC43LTUgNHYzYzAgMy4zIDEuMzkgNCA1IDRzNS0uNyA1LTR6TTM4MCA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuNC00IDctNCA2LjI2IDAgOSAzLjA4IDkgMTAuNzZWNDNoLThWMjJjMC0zLjEzLTEuMDctNS00LTVzLTQgMS44Ny00IDV6TTE1NyA0M2gtOFYwaDh2MTRjMS4xNC0yLjY3IDMuOTEtNCA3LjQ5LTQgNi4yNiAwIDguNTEgMy4xMyA4LjUxIDEwLjgxVjQzaC04VjIxYzAtMy4xMy0xLjA3LTQuNDQtNC00LjQ0cy00IDIuMjYtNCA1LjM5eiIvPjwvc3ZnPg=="
/>{' '}
<span className="-mb-0.5 text-[#3c3c3c]">replica</span>
</h1>
<div className="md:flex items-center gap-x-3 text-[#3c3c3c] -mb-1 hidden">
<a
className="hover:opacity-75 transition"
href="https://portaljs.org"
>
Built with 🌀PortalJS
</a>
<hr className="h-[80%] border border-[#3c3c3c] opacity-75 my-2"></hr>
<a
className="hover:opacity-75 transition"
href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
>
Github
</a>
</div>
</div>
<div className="mx-2 py-1.5 text-[14px] text-[#3c3c3c] md:hidden">
<ul className="flex gap-x-4">
<li>
<a
className="hover:opacity-75 transition"
href="https://portaljs.org"
>
PortalJS
</a>
</li>
<li>
<a
className="hover:opacity-75 transition"
href="https://github.com/datopian/portaljs/tree/main/examples/fivethirtyeight"
>
View on Github
</a>
</li>
</ul>
</div>
</header>
<Main />
<NextScript />
</body>
<NextScript />
</Html>
);
}

View File

@ -7,19 +7,10 @@ import remarkGfm from 'remark-gfm';
import extract from 'remark-extract-frontmatter';
import { Dataset } from '..';
import { GetStaticProps } from 'next';
import { FlatUiTable } from '@portaljs/components';
import { Table } from '@portaljs/components';
import Breadcrumbs from '@/components/Breadcrumbs';
import { ReactMarkdown } from 'react-markdown/lib/react-markdown';
import remarkFrontmatter from 'remark-frontmatter';
import Layout from '@/components/Layout';
import { format } from 'timeago.js';
// Request a weekday along with a long date
const options = {
year: 'numeric',
month: 'long',
day: 'numeric',
} as const;
export default function DatasetPage({
dataset,
@ -31,146 +22,68 @@ export default function DatasetPage({
return (
<>
<NextSeo title={`${dataset.name} page`} />
<Layout>
<main className="max-w-5xl px-2 prose mx-auto my-8 pb-8 prose-thead:border-b-4 prose-table:max-w-5xl prose-table:overflow-scroll prose-thead:overflow-scroll prose-tbody:overflow-scroll prose-thead:pb-2 prose-thead:border-zinc-900 prose-th:uppercase prose-th:text-left prose-th:font-light prose-th:text-xs prose-a:no-underline">
<Breadcrumbs links={[{ title: dataset.name, href: '' }]} />
<h1 className="uppercase mb-0 mt-16">{dataset.name}</h1>
<table className="w-full my-10 mb-8 hidden md:table">
<main className="max-w-5xl px-2 prose mx-auto my-8 prose-thead:border-b-4 prose-table:max-w-5xl prose-table:overflow-scroll prose-thead:overflow-scroll prose-tbody:overflow-scroll prose-thead:pb-2 prose-thead:border-zinc-900 prose-th:uppercase prose-th:text-left prose-th:font-light prose-th:text-xs">
<Breadcrumbs links={[{ title: dataset.name, href: '' }]} />
<h1 className="uppercase mb-0 mt-16">{dataset.name}</h1>
<p className="mb-8">
<span className="font-semibold">Repository:</span>{' '}
<a target="_blank" href={dataset.url}>
{dataset.url}
</a>
</p>
<h2 className="mb-0 mt-10">FILES</h2>
<div className="inline-block min-w-full py-2 align-middle">
<table className="min-w-full divide-y divide-gray-300">
<thead className="border-b-4 pb-2 border-zinc-900">
<tr>
<th className="uppercase text-left font-normal text-xs pb-3">
related content
</th>
<th className="uppercase text-left font-normal text-xs pb-3">
last updated
<th
className="uppercase text-left font-light text-xs pb-3"
scope="col"
>
Name
</th>
</tr>
</thead>
<tbody>
<DesktopItem key={dataset.name} dataset={dataset} />
<tbody className="divide-y divide-gray-200">
{dataset.files?.map((file) => (
<tr key={file}>
<td className="whitespace-nowrap text-left py-4 text-sm text-gray-500">
<a href={file}>{file.split('/').slice(-1)}</a>
</td>
</tr>
))}
</tbody>
</table>
{dataset.readme && (
<>
{dataset.readme && (
<ReactMarkdown
remarkPlugins={[
remarkFrontmatter,
remarkGfm,
[extract, { remove: true }],
]}
>
{dataset.readme}
</ReactMarkdown>
)}
</>
)}
<h2 className="mb-0 mt-10">Files</h2>
<div className="inline-block min-w-full py-2 align-middle">
<table className="min-w-full divide-y divide-gray-300">
<thead className="border-b-4 pb-2 border-zinc-900">
<tr>
<th
className="uppercase text-left font-light text-xs pb-3"
scope="col"
>
Name
</th>
<th
className="uppercase text-left font-light text-xs pb-3"
scope="col"
>
Download
</th>
</tr>
</thead>
<tbody className="divide-y divide-gray-200">
{dataset.files?.map((file) => (
<tr key={file}>
<td className="whitespace-nowrap text-left py-4 text-sm text-gray-500">
<a href={`#${file.split('/').slice(-1)}`}>
{file.split('/').slice(-1)}
</a>
</td>
<td className="whitespace-nowrap py-4 text-sm text-gray-500">
<a href={file}>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
className="w-8 h-8 text-blue-400 hover:text-blue-300 transition mt-1 ml-3"
>
<path
fillRule="evenodd"
d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
clipRule="evenodd"
/>
</svg>
</a>
</td>
</tr>
))}
</tbody>
</table>
</div>
{dataset.files && dataset.files.length > 0 && (
<>
<h2 className="mb-0 mt-8">Data Previews</h2>
{dataset.files?.map((file) => (
<div
key={file}
id={file.split('/').slice(-1).join('')}
className="preview-table my-8"
>
<h3>{file.split('/').slice(-1)}</h3>
<FlatUiTable url={file} />
</div>
))}
</>
)}
</main>
</Layout>
</>
);
}
export function DesktopItem({ dataset }: { dataset: Dataset }) {
return (
<>
{dataset.articles.map((article, index) => (
<tr
key={article.url}
className={`${
index === dataset.articles.length - 1 ? 'border-b' : ''
} border-zinc-400`}
>
<td>
<a
className="py-8 font-bold hover:underline pr-2"
href={article.url}
>
{article.title}
</a>
</td>
<td className="py-8 font-light text-[14px] min-w-[138px] font-mono text-[#999]">
{format(article.date).includes('years')
? new Date(article.date).toLocaleString('en-US', options)
: format(article.date)}
</td>
<td className="py-8 text-end">
{index === 0 && (
<a
className="ml-auto border border-zinc-900 font-light px-[25px] py-2.5 text-sm transition hover:bg-zinc-900 hover:text-white"
href={dataset.url}
</div>
{dataset.files && dataset.files.length > 0 && (
<>
<h2 className="mb-0 mt-10">DATA PREVIEWS</h2>
{dataset.files?.map((file) => (
<div key={file} className="preview-table my-8">
<h3>{file.split('/').slice(-1)}</h3>
<Table url={file} />
</div>
))}
</>
)}
{dataset.readme && (
<>
<h2 className="uppercase font-black">Readme</h2>
{dataset.readme && (
<ReactMarkdown
remarkPlugins={[
remarkFrontmatter,
remarkGfm,
[extract, { remove: true }],
]}
>
info
</a>
{dataset.readme}
</ReactMarkdown>
)}
</td>
</tr>
))}
</>
)}
</main>
</>
);
}
@ -188,7 +101,6 @@ export async function getStaticPaths() {
fallback: false, // can also be true or 'blocking'
};
}
// change href base check datahub-next
export const getStaticProps: GetStaticProps = async ({ params }) => {
const datasetsFile = path.join(process.cwd(), 'datasets.json');
@ -198,20 +110,15 @@ export const getStaticProps: GetStaticProps = async ({ params }) => {
(_dataset) => _dataset.name === params?.datasetName
);
const github_pat = getConfig().serverRuntimeConfig.github_pat;
const readmes = await Promise.all(
['/README.md', '/readme.md', '/Readme.md'].map(
async (readme) =>
await getProjectReadme(
'fivethirtyeight',
'data',
'master',
dataset?.name + readme,
github_pat
)
)
);
const readme = readmes.find((item) => item !== null);
if (!readme) console.log('Readme not found for ' + dataset?.name);
const readmes = await Promise.all(['/README.md', '/readme.md', '/Readme.md'].map(async (readme) => await getProjectReadme(
'fivethirtyeight',
'data',
'master',
dataset?.name + readme,
github_pat
)));
const readme = readmes.find(item => item !== null)
if (!readme) console.log('Readme not found for ' + dataset?.name)
return {
props: {
dataset: {

View File

@ -4,7 +4,6 @@ import { format } from 'timeago.js';
import { promises as fs } from 'fs';
import path from 'path';
import { NextSeo } from 'next-seo';
import Layout from '@/components/Layout';
const inter = Inter({ subsets: ['latin'] });
@ -52,12 +51,21 @@ export function MobileItem({ dataset }: { dataset: Dataset }) {
>
info
</a>
<a
className="ml-2 border border-[#3c3c3c] px-[25px] py-2.5 text-sm transition bg-[#3c3c3c] text-white hover:bg-zinc-900"
href={`/datasets/${dataset.name}`}
>
explore
</a>
{/*
<button>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
className="w-12 h-12 text-blue-400 hover:text-blue-300 transition mt-1"
>
<path
fillRule="evenodd"
d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
clipRule="evenodd"
/>
</svg>
</button> */}
</div>
</div>
);
@ -89,16 +97,6 @@ export function DesktopItem({ dataset }: { dataset: Dataset }) {
? new Date(article.date).toLocaleString('en-US', options)
: format(article.date)}
</td>
<td>
{index === 0 && (
<a
className="ml-2 border border-[#3c3c3c] px-[25px] py-2.5 text-sm transition bg-[#3c3c3c] text-white hover:bg-zinc-900"
href={`/datasets/${dataset.name}`}
>
explore
</a>
)}
</td>
<td className="py-8">
{index === 0 && (
<a
@ -109,6 +107,23 @@ export function DesktopItem({ dataset }: { dataset: Dataset }) {
</a>
)}
</td>
{/*
<td>
<button>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
className="w-12 h-12 text-blue-400 hover:text-blue-300 transition mt-1"
>
<path
fillRule="evenodd"
d="M12 2.25c-5.385 0-9.75 4.365-9.75 9.75s4.365 9.75 9.75 9.75 9.75-4.365 9.75-9.75S17.385 2.25 12 2.25zm-.53 14.03a.75.75 0 001.06 0l3-3a.75.75 0 10-1.06-1.06l-1.72 1.72V8.25a.75.75 0 00-1.5 0v5.69l-1.72-1.72a.75.75 0 00-1.06 1.06l3 3z"
clipRule="evenodd"
/>
</svg>
</button>
</td>*/}
</tr>
))}
</>
@ -128,7 +143,6 @@ export default function Home({ datasets }: { datasets: Dataset[] }) {
return (
<>
<NextSeo title="FiveThirtyEight tribute by PortalJS" />
<Layout>
<main
className={`flex min-h-screen flex-col items-center max-w-5xl mx-auto pt-20 px-2.5 ${inter.className}`}
>
@ -192,7 +206,6 @@ export default function Home({ datasets }: { datasets: Dataset[] }) {
.
</p>
</main>
</Layout>
</>
);
}

View File

@ -3,9 +3,6 @@
@tailwind utilities;
.preview-table > div {
overflow-x: scroll;
overflow-y: hidden;
}
.prose h1 {
font-size: 1.5em !important;
}

View File

@ -1,5 +1,8 @@
{
"extends": ["next", "next/core-web-vitals"],
"extends": [
"next",
"next/core-web-vitals"
],
"ignorePatterns": ["!**/*", ".next/**/*"],
"overrides": [
{

View File

@ -1,7 +0,0 @@
node_modules
**/.next/**
**/_next/**
**/dist/**
**/__tmp__/**
lerna.json
.github

View File

@ -6,7 +6,7 @@ A `datasets.json` file is used to specify which datasets are going to be part of
The application contains an index page, which lists all the datasets specified in the `datasets.json` file, and users can see more information about each dataset, such as the list of data files in it and the README, by clicking the "info" button on the list.
You can read more about it on the [Data catalog with data on GitHub](https://portaljs.com/docs/examples/github-backed-catalog) blog post.
You can read more about it on the [Data catalog with data on GitHub](https://portaljs.org/docs/examples/github-backed-catalog) blog post.
## Demo
@ -99,3 +99,4 @@ And run the production build with:
```
npm run start
```

View File

@ -1,21 +0,0 @@
import { MDXRemote } from 'next-mdx-remote';
import dynamic from 'next/dynamic';
import { Mermaid } from '@portaljs/core';
// Custom components/renderers to pass to MDX.
// Since the MDX files aren't loaded by webpack, they have no knowledge of how
// to handle import statements. Instead, you must include components in scope
// here.
const components = {
Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
mermaid: Mermaid,
Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),
LineChart: dynamic(() => import('@portaljs/components').then(mod => mod.LineChart)),
} as any;
export default function DRD({ source }: { source: any }) {
return <MDXRemote {...source} components={components} />;
}

View File

@ -1,28 +1,20 @@
import Link from "next/link";
import HomeIcon from "../icons/HomeIcon";
export default function Breadcrumbs({
links,
}: {
links: { title: string; href?: string; target?: string }[];
}) {
export default function Breadcrumbs({ links }: { links: { title: string, href?: string, target?: string }[] }) {
const current = links.at(-1);
return (
<div className="flex items-center uppercase font-black text-xs">
<Link className="flex items-center" href="/">
<HomeIcon />
</Link>
return <div className="flex items-center uppercase font-black text-xs">
<Link className="flex items-center" href='/'><HomeIcon /></Link>
{/* {links.length > 1 && links.slice(0, -1).map((link) => {
{/* {links.length > 1 && links.slice(0, -1).map((link) => {
return <>
<span className="mx-4">/</span>
<Link href={link.href}>{link.title}</Link>
</>
})} */}
<span className="mx-4">/</span>
<span>{current.title}</span>
</div>
);
}
<span className="mx-4">/</span>
<span>{current.title}</span>
</div >
}

View File

@ -1,13 +1,3 @@
export default function ExternalLinkIcon({ className = "" }) {
return (
<div className={`inline-block w-4 ${className}`}>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 64 64"
fill="currentColor"
>
<path d="M 40 10 C 38.896 10 38 10.896 38 12 C 38 13.104 38.896 14 40 14 L 47.171875 14 L 30.585938 30.585938 C 29.804938 31.366938 29.804938 32.633063 30.585938 33.414062 C 30.976938 33.805063 31.488 34 32 34 C 32.512 34 33.023063 33.805062 33.414062 33.414062 L 50 16.828125 L 50 24 C 50 25.104 50.896 26 52 26 C 53.104 26 54 25.104 54 24 L 54 12 C 54 10.896 53.104 10 52 10 L 40 10 z M 18 12 C 14.691 12 12 14.691 12 18 L 12 46 C 12 49.309 14.691 52 18 52 L 46 52 C 49.309 52 52 49.309 52 46 L 52 34 C 52 32.896 51.104 32 50 32 C 48.896 32 48 32.896 48 34 L 48 46 C 48 47.103 47.103 48 46 48 L 18 48 C 16.897 48 16 47.103 16 46 L 16 18 C 16 16.897 16.897 16 18 16 L 30 16 C 31.104 16 32 15.104 32 14 C 32 12.896 31.104 12 30 12 L 18 12 z" />
</svg>
</div>
);
}
return <div className={`inline-block w-4 ${className}`}><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" fill="currentColor"><path d="M 40 10 C 38.896 10 38 10.896 38 12 C 38 13.104 38.896 14 40 14 L 47.171875 14 L 30.585938 30.585938 C 29.804938 31.366938 29.804938 32.633063 30.585938 33.414062 C 30.976938 33.805063 31.488 34 32 34 C 32.512 34 33.023063 33.805062 33.414062 33.414062 L 50 16.828125 L 50 24 C 50 25.104 50.896 26 52 26 C 53.104 26 54 25.104 54 24 L 54 12 C 54 10.896 53.104 10 52 10 L 40 10 z M 18 12 C 14.691 12 12 14.691 12 18 L 12 46 C 12 49.309 14.691 52 18 52 L 46 52 C 49.309 52 52 49.309 52 46 L 52 34 C 52 32.896 51.104 32 50 32 C 48.896 32 48 32.896 48 34 L 48 46 C 48 47.103 47.103 48 46 48 L 18 48 C 16.897 48 16 47.103 16 46 L 16 18 C 16 16.897 16.897 16 18 16 L 30 16 C 31.104 16 32 15.104 32 14 C 32 12.896 31.104 12 30 12 L 18 12 z"/></svg></div>
}

View File

@ -1,10 +1,3 @@
export default function HomeIcon({ className = "" }) {
return (
<div className={`inline-block w-4 ${className}`}>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
{" "}
<path d="M 12 2 A 1 1 0 0 0 11.289062 2.296875 L 1.203125 11.097656 A 0.5 0.5 0 0 0 1 11.5 A 0.5 0.5 0 0 0 1.5 12 L 4 12 L 4 20 C 4 20.552 4.448 21 5 21 L 9 21 C 9.552 21 10 20.552 10 20 L 10 14 L 14 14 L 14 20 C 14 20.552 14.448 21 15 21 L 19 21 C 19.552 21 20 20.552 20 20 L 20 12 L 22.5 12 A 0.5 0.5 0 0 0 23 11.5 A 0.5 0.5 0 0 0 22.796875 11.097656 L 12.716797 2.3027344 A 1 1 0 0 0 12.710938 2.296875 A 1 1 0 0 0 12 2 z" />
</svg>
</div>
);
}
return <div className={`inline-block w-4 ${className}`}><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"> <path d="M 12 2 A 1 1 0 0 0 11.289062 2.296875 L 1.203125 11.097656 A 0.5 0.5 0 0 0 1 11.5 A 0.5 0.5 0 0 0 1.5 12 L 4 12 L 4 20 C 4 20.552 4.448 21 5 21 L 9 21 C 9.552 21 10 20.552 10 20 L 10 14 L 14 14 L 14 20 C 14 20.552 14.448 21 15 21 L 19 21 C 19.552 21 20 20.552 20 20 L 20 12 L 22.5 12 A 0.5 0.5 0 0 0 23 11.5 A 0.5 0.5 0 0 0 22.796875 11.097656 L 12.716797 2.3027344 A 1 1 0 0 0 12.710938 2.296875 A 1 1 0 0 0 12 2 z"/></svg></div>
}

View File

@ -15,18 +15,14 @@
],
"readme": "README.md"
},
{
"owner": "luccasmmg",
"branch": "main",
"repo": "test-data-repo-1",
"files": ["data_1.csv", "data_2.csv"],
"readme": "README.md"
},
{
"owner": "datasets",
"branch": "main",
"repo": "investor-flow-of-funds-us",
"files": ["data/monthly.csv", "data/weekly.csv"],
"files": [
"data/monthly.csv",
"data/weekly.csv"
],
"readme": "README.md"
},
{
@ -42,10 +38,7 @@
"owner": "fivethirtyeight",
"repo": "data",
"branch": "master",
"files": [
"nba-raptor/historical_RAPTOR_by_player.csv",
"nba-raptor/historical_RAPTOR_by_team.csv"
],
"files": ["nba-raptor/historical_RAPTOR_by_player.csv", "nba-raptor/historical_RAPTOR_by_team.csv"],
"readme": "nba-raptor/README.md"
}
]

View File

@ -1,5 +1,5 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
declare module "*.svg" {
declare module '*.svg' {
const content: any;
export const ReactComponent: any;
export default content;

View File

@ -1,105 +0,0 @@
import matter from "gray-matter";
import mdxmermaid from "mdx-mermaid";
import { h } from "hastscript";
import remarkCallouts from "@portaljs/remark-callouts";
import remarkEmbed from "@portaljs/remark-embed";
import remarkGfm from "remark-gfm";
import remarkMath from "remark-math";
import remarkSmartypants from "remark-smartypants";
import remarkToc from "remark-toc";
import remarkWikiLink from "@portaljs/remark-wiki-link";
import rehypeAutolinkHeadings from "rehype-autolink-headings";
import rehypeKatex from "rehype-katex";
import rehypeSlug from "rehype-slug";
import rehypePrismPlus from "rehype-prism-plus";
import { serialize } from "next-mdx-remote/serialize";
/**
* Parse a markdown or MDX file to an MDX source form + front matter data
*
* @source: the contents of a markdown or mdx file
* @format: used to indicate to next-mdx-remote which format to use (md or mdx)
* @returns: { mdxSource: mdxSource, frontMatter: ...}
*/
const parse = async function (source, format, scope) {
const { content, data, excerpt } = matter(source, {
excerpt: (file, options) => {
// Generate an excerpt for the file
file.excerpt = file.content.split("\n\n")[0];
},
});
const mdxSource = await serialize(
{ value: content, path: format },
{
// Optionally pass remark/rehype plugins
mdxOptions: {
remarkPlugins: [
remarkEmbed,
remarkGfm,
[remarkSmartypants, { quotes: false, dashes: "oldschool" }],
remarkMath,
remarkCallouts,
remarkWikiLink,
[
remarkToc,
{
heading: "Table of contents",
tight: true,
},
],
[mdxmermaid, {}],
],
rehypePlugins: [
rehypeSlug,
[
rehypeAutolinkHeadings,
{
properties: { className: 'heading-link' },
test(element) {
return (
["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
element.properties?.id !== "table-of-contents" &&
element.properties?.className !== "blockquote-heading"
);
},
content() {
return [
h(
"svg",
{
xmlns: "http:www.w3.org/2000/svg",
fill: "#ab2b65",
viewBox: "0 0 20 20",
className: "w-5 h-5",
},
[
h("path", {
fillRule: "evenodd",
clipRule: "evenodd",
d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
}),
]
),
];
},
},
],
[rehypeKatex, { output: "mathml" }],
[rehypePrismPlus, { ignoreMissing: true }],
],
format,
},
scope,
}
);
return {
mdxSource: mdxSource,
frontMatter: data,
excerpt,
};
};
export default parse;

View File

@ -1,14 +0,0 @@
import { MarkdownDB } from "mddb";
const dbPath = "markdown.db";
const client = new MarkdownDB({
client: "sqlite3",
connection: {
filename: dbPath,
},
});
const clientPromise = client.init();
export default clientPromise;

View File

@ -1,4 +1,4 @@
import { Octokit } from "octokit";
import { Octokit } from 'octokit';
export interface GithubProject {
owner: string;
@ -26,42 +26,15 @@ export async function getProjectReadme(
ref: branch,
});
const data = response.data as { content?: string };
const fileContent = data.content ? data.content : "";
if (fileContent === "") {
const fileContent = data.content ? data.content : '';
if (fileContent === '') {
return null;
}
const decodedContent = Buffer.from(fileContent, "base64").toString();
const decodedContent = Buffer.from(fileContent, 'base64').toString();
return decodedContent;
} catch (error) {
throw new Error(
"Couldn't get project readme please make sure that you are pointing to a valid repo and that the repo in question contains a README.md"
);
}
}
export async function getProjectDatapackage(
owner: string,
repo: string,
branch: string,
github_pat?: string
) {
const octokit = new Octokit({ auth: github_pat });
try {
const response = await octokit.rest.repos.getContent({
owner,
repo,
path: "datapackage.json",
ref: branch,
});
const data = response.data as { content?: string };
const fileContent = data.content ? data.content : "";
if (fileContent === "") {
return null;
}
const decodedContent = Buffer.from(fileContent, "base64").toString();
return JSON.parse(decodedContent);
} catch (error) {
return null
console.log(error);
return null;
}
}
@ -77,13 +50,13 @@ export async function getLastUpdated(
const response = await octokit.rest.repos.listCommits({
owner,
repo,
path: readme,
ref: branch,
});
return response.data[0].commit.committer.date;
} catch (error) {
throw new Error(
"Couldn't get project list of commits please make sure that you are pointing to a valid repo"
);
console.log(error);
return null;
}
}
export async function getProjectMetadata(
@ -99,9 +72,8 @@ export async function getProjectMetadata(
});
return response.data;
} catch (error) {
throw new Error(
"Couldn't get project metadata please make sure that you are pointing to a valid repo"
);
console.log(error);
return null;
}
}
@ -122,32 +94,13 @@ export async function getRepoContents(
ref: branch,
path: path,
});
const data = response.data as {
download_url?: string;
name: string;
size: number;
};
contents.push({
download_url: data.download_url,
name: data.name,
size: data.size,
});
const data = response.data as { download_url?: string, name: string, size: number };
contents.push({ download_url: data.download_url, name: data.name, size: data.size});
}
return contents;
} catch (error) {
if (
error.message ===
'This endpoint can only return blobs smaller than 100 MB in size. The requested blob is too large to fetch via the API, but you can always clone the repository via Git to obtain it.: {"resource":"Blob","field":"data","code":"too_large"}'
) {
throw new Error(
`The requested files ${files.join(
", "
)} are too big making it impossible to fetch via Github API`
);
}
throw new Error(
"Couldn't get project contents please make sure that you are pointing to a valid repo"
);
console.log(error);
return null;
}
}
@ -167,20 +120,22 @@ export async function getProject(project: GithubProject, github_pat?: string) {
project.readme,
github_pat
);
let projectData = [];
if (project.files) {
projectData = await getRepoContents(
project.owner,
project.repo,
project.branch,
project.files,
github_pat
);
if (!projectReadme) {
return null;
}
const projectBase =
project.readme && project.readme.split("/").length > 1
? project.readme.split("/").slice(0, -1).join("/")
: "/";
const projectData = await getRepoContents(
project.owner,
project.repo,
project.branch,
project.files,
github_pat
);
if (!projectData) {
return null;
}
const projectBase = project.readme.split('/').length > 1
? project.readme.split('/').slice(0, -1).join('/')
: '/'
const last_updated = await getLastUpdated(
project.owner,
project.repo,
@ -188,20 +143,5 @@ export async function getProject(project: GithubProject, github_pat?: string) {
projectBase,
github_pat
);
const projectDatapackage = await getProjectDatapackage(
project.owner,
project.repo,
project.branch,
github_pat
);
return {
...projectMetadata,
files: projectData,
readmeContent: projectReadme,
last_updated,
base_path: projectBase,
datapackage: projectDatapackage
};
return { ...projectMetadata, files: projectData, readmeContent: projectReadme, last_updated, base_path: projectBase };
}

View File

@ -3,8 +3,8 @@ const nextConfig = {
return {
beforeFiles: [
{
source: "/@:org/:project*",
destination: "/@org/:org/:project*",
source: '/@:org/:project*',
destination: '/@org/:org/:project*',
},
],
};

File diff suppressed because it is too large Load Diff

View File

@ -6,44 +6,28 @@
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint",
"prettier": "prettier --write ."
"lint": "next lint"
},
"dependencies": {
"@portaljs/components": "^0.1.6",
"@portaljs/core": "^1.0.5",
"@portaljs/remark-callouts": "^1.0.5",
"@portaljs/remark-embed": "^1.0.4",
"@portaljs/remark-wiki-link": "^1.0.4",
"@types/node": "18.16.0",
"@types/react": "18.0.38",
"@types/react-dom": "18.0.11",
"eslint": "8.39.0",
"eslint-config-next": "13.3.1",
"mddb": "^0.1.9",
"next": "13.4.3",
"next-mdx-remote": "^4.4.1",
"next": "13.3.1",
"next-seo": "^6.0.0",
"octokit": "^2.0.14",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-markdown": "^8.0.7",
"react-timeago": "^7.1.0",
"rehype-autolink-headings": "^6.1.1",
"rehype-katex": "^6.0.3",
"rehype-prism-plus": "^1.5.1",
"rehype-slug": "^5.1.0",
"remark-gfm": "^3.0.1",
"remark-math": "^5.1.1",
"remark-smartypants": "^2.0.0",
"remark-toc": "^8.0.1",
"typescript": "5.0.4"
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.9",
"autoprefixer": "^10.4.14",
"postcss": "^8.4.23",
"prettier": "2.8.8",
"tailwindcss": "^3.3.1"
}
}

View File

@ -1,33 +1,22 @@
import { NextSeo } from "next-seo";
import { promises as fs } from "fs";
import path from "path";
import getConfig from "next/config";
import { getProject, GithubProject } from "../../../lib/octokit";
import ReactMarkdown from "react-markdown";
import remarkGfm from "remark-gfm";
import Breadcrumbs from "../../../components/_shared/Breadcrumbs";
import parse from '../../../lib/markdown';
import DataRichDocument from '../../../components/DataRichDocument'
import { NextSeo } from 'next-seo';
import { promises as fs } from 'fs';
import path from 'path';
import getConfig from 'next/config';
import { getProject, GithubProject } from '../../../lib/octokit';
import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
import Breadcrumbs from '../../../components/_shared/Breadcrumbs';
export default function ProjectPage({ project }) {
const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`;
const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`
return (
<>
<NextSeo
title={`${repoId}${
project.base_path !== "/" ? "/" + project.base_path : ""
} - GitHub Datasets`}
/>
<NextSeo title={`${repoId}${project.base_path !== '/' ? '/' + project.base_path : ''} - GitHub Datasets`} />
<main className="prose mx-auto my-8">
<Breadcrumbs links={[{ title: repoId, href: "" }]} />
<h1 className="mb-0 mt-16">{project.repo_config.name || repoId}</h1>
<p className="mb-8">
<span className="font-semibold">Repository:</span>{" "}
<a target="_blank" href={project.html_url}>
{project.html_url}
</a>
</p>
<p className='mb-8'><span className='font-semibold'>Repository:</span> <a target="_blank" href={project.html_url}>{project.html_url}</a></p>
<h2 className="mb-0 mt-10">Files</h2>
<div className="inline-block min-w-full py-2 align-middle">
@ -65,8 +54,10 @@ export default function ProjectPage({ project }) {
<hr />
<h2 className="uppercase font-black">Readme</h2>
<DataRichDocument source={project.mdxSource} />
<h2 className='uppercase font-black'>Readme</h2>
<ReactMarkdown remarkPlugins={[remarkGfm]}>
{project.readmeContent}
</ReactMarkdown>
</main>
</>
);
@ -74,14 +65,17 @@ export default function ProjectPage({ project }) {
// Generates `/posts/1` and `/posts/2`
export async function getStaticPaths() {
const jsonDirectory = path.join(process.cwd(), "datasets.json");
const repos = await fs.readFile(jsonDirectory, "utf8");
const jsonDirectory = path.join(
process.cwd(),
'datasets.json'
);
const repos = await fs.readFile(jsonDirectory, 'utf8');
return {
paths: JSON.parse(repos).map((repo) => {
const projectPath =
repo.readme && repo.readme.split("/").length > 1
? repo.readme.split("/").slice(0, -1)
repo.readme.split('/').length > 1
? repo.readme.split('/').slice(0, -1)
: null;
let path = [repo.repo];
if (projectPath) {
@ -98,13 +92,16 @@ export async function getStaticPaths() {
}
export async function getStaticProps({ params }) {
const jsonDirectory = path.join(process.cwd(), "datasets.json");
const reposFile = await fs.readFile(jsonDirectory, "utf8");
const jsonDirectory = path.join(
process.cwd(),
'datasets.json'
);
const reposFile = await fs.readFile(jsonDirectory, 'utf8');
const repos: GithubProject[] = JSON.parse(reposFile);
const repo = repos.find((_repo) => {
const projectPath =
_repo.readme && _repo.readme.split("/").length > 1
? _repo.readme.split("/").slice(0, -1)
_repo.readme.split('/').length > 1
? _repo.readme.split('/').slice(0, -1)
: null;
let path = [_repo.repo];
if (projectPath) {
@ -119,10 +116,9 @@ export async function getStaticProps({ params }) {
});
const github_pat = getConfig().serverRuntimeConfig.github_pat;
const project = await getProject(repo, github_pat);
let { mdxSource, frontMatter } = await parse(project.readmeContent, '.mdx', { project });
return {
props: {
project: { ...project, repo_config: repo, mdxSource },
project: { ...project, repo_config: repo },
},
};
}

View File

@ -1,6 +1,6 @@
import { AppProps } from "next/app";
import Head from "next/head";
import "./styles.css";
import { AppProps } from 'next/app';
import Head from 'next/head';
import './styles.css';
function CustomApp({ Component, pageProps }: AppProps) {
return (

View File

@ -1,19 +1,21 @@
import { promises as fs } from "fs";
import path from "path";
import { getProject } from "../lib/octokit";
import getConfig from "next/config";
import ExternalLinkIcon from "../components/icons/ExternalLinkIcon";
import TimeAgo from "react-timeago";
import Link from "next/link";
import { NextSeo } from "next-seo";
import { promises as fs } from 'fs';
import path from 'path';
import { getProject } from '../lib/octokit';
import getConfig from 'next/config';
import ExternalLinkIcon from '../components/icons/ExternalLinkIcon';
import TimeAgo from 'react-timeago';
import Link from 'next/link';
export async function getStaticProps() {
const jsonDirectory = path.join(process.cwd(), "/datasets.json");
const repos = await fs.readFile(jsonDirectory, "utf8");
const jsonDirectory = path.join(
process.cwd(),
'/datasets.json'
);
const repos = await fs.readFile(jsonDirectory, 'utf8');
const github_pat = getConfig().serverRuntimeConfig.github_pat;
const projects = await Promise.all(
JSON.parse(repos).map(async (repo) => {
(JSON.parse(repos)).map(async (repo) => {
const project = await getProject(repo, github_pat);
return { ...project, repo_config: repo };
})
@ -27,112 +29,88 @@ export async function getStaticProps() {
export function Datasets({ projects }) {
return (
<>
<NextSeo title="GitHub Datasets" />
<div className="bg-white min-h-screen">
<div className="mx-auto max-w-7xl px-6 py-16 sm:py-24 lg:px-8">
<div className="text-center">
<h2 className="text-3xl font-bold leading-10 tracking-tight">
GitHub Datasets
</h2>
<p className="mt-3 mx-auto max-w-2xl text-base leading-7 text-gray-500">
Data catalog with datasets hosted on GitHub by{" "}
<Link
target="_blank"
className="underline"
href="https://portaljs.com/"
>
🌀 PortalJS
</Link>
</p>
</div>
<div className="mt-20">
<div className="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
<div className="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
<table className="min-w-full divide-y divide-gray-300">
<thead>
<tr>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Name
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Repository
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Description
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Last updated
</th>
<th
scope="col"
className="relative py-3.5 pl-3 pr-4 sm:pr-0"
></th>
<div className="bg-white min-h-screen">
<div className="mx-auto max-w-7xl px-6 py-16 sm:py-24 lg:px-8">
<div className='text-center'>
<h2 className="text-3xl font-bold leading-10 tracking-tight">
GitHub Datasets
</h2>
<p className="mt-3 mx-auto max-w-2xl text-base leading-7 text-gray-500">
Data catalog with datasets hosted on GitHub by <Link target="_blank" className='underline' href="https://portaljs.org/">🌀 PortalJS</Link>
</p>
</div>
<div className="mt-20">
<div className="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
<div className="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
<table className="min-w-full divide-y divide-gray-300">
<thead>
<tr>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Name
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Repository
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Description
</th>
<th
scope="col"
className="px-3 py-3.5 text-left text-sm font-semibold text-gray-900"
>
Last updated
</th>
<th
scope="col"
className="relative py-3.5 pl-3 pr-4 sm:pr-0"
></th>
</tr>
</thead>
<tbody className="divide-y divide-gray-200">
{projects.map((project) => (
<tr key={project.id}>
<td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
{project.repo_config.name
? project.repo_config.name
: project.full_name + (project.base_path === '/' ? '' : '/' + project.base_path)}
</td>
<td className="whitespace-nowrap px-3 py-6 text-sm group text-gray-500 hover:text-gray-900 transition-all duration-250">
<a href={project.html_url} target="_blank" className='flex items-center'>@{project.full_name} <ExternalLinkIcon className='ml-1' /></a>
</td>
<td className="px-3 py-4 text-sm text-gray-500">
{project.repo_config.description
? project.repo_config.description
: project.description}
</td>
<td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
<TimeAgo date={new Date(project.last_updated)} />
</td>
<td className="relative whitespace-nowrap py-6 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
<a
href={`/@${project.repo_config.owner}/${project.repo_config.repo}/${project.base_path === '/' ? '' : project.base_path}`}
className='border border-gray-900 text-gray-900 px-4 py-2 transition-all hover:bg-gray-900 hover:text-white'
>
info
</a>
</td>
</tr>
</thead>
<tbody className="divide-y divide-gray-200">
{projects.map((project) => (
<tr key={project.id}>
<td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
{project.repo_config.name
? project.repo_config.name
: project.full_name +
(project.base_path === "/"
? ""
: "/" + project.base_path)}
</td>
<td className="whitespace-nowrap px-3 py-6 text-sm group text-gray-500 hover:text-gray-900 transition-all duration-250">
<a
href={project.html_url}
target="_blank"
className="flex items-center"
>
@{project.full_name}{" "}
<ExternalLinkIcon className="ml-1" />
</a>
</td>
<td className="px-3 py-4 text-sm text-gray-500">
{project.repo_config.description
? project.repo_config.description
: project.description}
</td>
<td className="whitespace-nowrap px-3 py-6 text-sm text-gray-500">
<TimeAgo date={new Date(project.last_updated)} />
</td>
<td className="relative whitespace-nowrap py-6 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
<a
href={`/@${project.repo_config.owner}/${
project.repo_config.repo
}/${
project.base_path === "/" ? "" : project.base_path
}`}
className="border border-gray-900 text-gray-900 px-4 py-2 transition-all hover:bg-gray-900 hover:text-white"
>
info
</a>
</td>
</tr>
))}
</tbody>
</table>
</div>
))}
</tbody>
</table>
</div>
</div>
</div>
</div>
</>
</div>
);
}

View File

@ -78,72 +78,3 @@ pre {
color: rgba(55, 65, 81, 1);
width: 100%;
}
@import "@portaljs/remark-callouts/styles.css";
/* mathjax */
.math-inline > mjx-container > svg {
display: inline;
align-items: center;
}
/* smooth scrolling in modern browsers */
html {
scroll-behavior: smooth !important;
}
/* tooltip fade-out clip */
.tooltip-body::after {
content: "";
position: absolute;
right: 0;
top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
width: 10rem;
background: linear-gradient(
to right,
rgba(255, 255, 255, 0),
rgba(255, 255, 255, 1) 100%
);
}
:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
margin-left: -2rem !important;
padding-left: 2rem !important;
scroll-margin-top: 4.5rem;
position: relative;
}
.heading-link {
padding: 1px;
position: absolute;
left: 0;
top: 50%;
transform: translateY(-50%);
margin: auto 0;
border-radius: 5px;
background: #1e293b;
opacity: 0;
transition: opacity 0.2s;
}
.light .heading-link {
/* border: 1px solid #ab2b65; */
/* background: none; */
background: #e2e8f0;
}
:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
opacity: 100;
}
.heading-link svg {
transform: scale(0.75);
}
@media screen and (max-width: 640px) {
.heading-link {
visibility: hidden;
}
}

View File

@ -3,4 +3,4 @@ module.exports = {
tailwindcss: {},
autoprefixer: {},
},
};
}

View File

@ -8,5 +8,8 @@ module.exports = {
theme: {
extend: {},
},
plugins: [require("@tailwindcss/typography")],
};
plugins: [
require('@tailwindcss/typography')
],
}

View File

@ -0,0 +1 @@
PortalJS Learn Example - https://portaljs.org/docs

View File

@ -1,6 +1,6 @@
import { MDXRemote } from 'next-mdx-remote';
import dynamic from 'next/dynamic';
import { Mermaid } from '@portaljs/core';
import { Mermaid } from '@flowershow/core';
// Custom components/renderers to pass to MDX.
// Since the MDX files aren't loaded by webpack, they have no knowledge of how
@ -13,7 +13,6 @@ const components = {
Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),
LineChart: dynamic(() => import('@portaljs/components').then(mod => mod.LineChart)),
FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
} as any;
export default function DRD({ source }: { source: any }) {

View File

@ -1,13 +1,13 @@
import matter from "gray-matter";
import mdxmermaid from "mdx-mermaid";
import { h } from "hastscript";
import remarkCallouts from "@portaljs/remark-callouts";
import remarkEmbed from "@portaljs/remark-embed";
import remarkCallouts from "@flowershow/remark-callouts";
import remarkEmbed from "@flowershow/remark-embed";
import remarkGfm from "remark-gfm";
import remarkMath from "remark-math";
import remarkSmartypants from "remark-smartypants";
import remarkToc from "remark-toc";
import remarkWikiLink from "@portaljs/remark-wiki-link";
import remarkWikiLink from "@flowershow/remark-wiki-link";
import rehypeAutolinkHeadings from "rehype-autolink-headings";
import rehypeKatex from "rehype-katex";
import rehypeSlug from "rehype-slug";

View File

@ -1,4 +1,4 @@
import { MarkdownDB } from "mddb";
import { MarkdownDB } from "@flowershow/markdowndb";
const dbPath = "markdown.db";

File diff suppressed because it is too large Load Diff

View File

@ -12,27 +12,25 @@
"mddb": "mddb ./content"
},
"dependencies": {
"@githubocto/flat-ui": "^0.14.1",
"@flowershow/core": "^0.4.10",
"@flowershow/markdowndb": "^0.1.1",
"@flowershow/remark-callouts": "^1.0.0",
"@flowershow/remark-embed": "^1.0.0",
"@flowershow/remark-wiki-link": "^1.1.2",
"@heroicons/react": "^2.0.17",
"@opentelemetry/api": "^1.4.0",
"@portaljs/components": "^0.1.8",
"@portaljs/core": "^1.0.5",
"@portaljs/remark-callouts": "^1.0.5",
"@portaljs/remark-embed": "^1.0.4",
"@portaljs/remark-wiki-link": "^1.0.4",
"@portaljs/components": "^0.1.0",
"@tanstack/react-table": "^8.8.5",
"flexsearch": "0.7.21",
"gray-matter": "^4.0.3",
"hastscript": "^7.2.0",
"mddb": "^0.1.9",
"mdx-mermaid": "2.0.0-rc7",
"next": "13.2.1",
"next-mdx-remote": "^4.4.1",
"papaparse": "^5.4.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-hook-form": "^7.43.9",
"react-query": "^3.39.3",
"react-vega": "^7.6.0",
"rehype-autolink-headings": "^6.1.1",
"rehype-katex": "^6.0.3",
@ -42,9 +40,7 @@
"remark-math": "^5.1.1",
"remark-smartypants": "^2.0.0",
"remark-toc": "^8.0.1",
"typescript": "5.0.4",
"vega": "5.25.0",
"vega-lite": "5.1.0"
"typescript": "5.0.4"
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.9",

View File

@ -81,7 +81,7 @@ export default function DatasetPage({ mdxSource, frontMatter }) {
<p className="my-0">
<span className="font-semibold">Modified: </span>
<span className="description my-0">
{new Date(frontMatter.modified).toLocaleDateString("en-US")}
{new Date(frontMatter.modified).toLocaleDateString()}
</span>
</p>
)}

View File

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 37 KiB

View File

@ -1,7 +1,7 @@
@tailwind base;
@tailwind components;
@tailwind utilities;
@import "@portaljs/remark-callouts/styles.css";
@import "@flowershow/remark-callouts/styles.css";
.w-5 {
width: 1.25rem

Some files were not shown because too many files have changed in this diff Show More