knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #850 from datopian/feature/lhs-navigation

Browse Source 
LHS Navigation
This commit is contained in:
Anuar Ustayev (aka Anu) 2023-05-06 10:35:00 +06:00 committed by GitHub
commit 45c07f829a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 847 additions and 3680 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
site/components/DocsPagination.tsx Normal file
View File

@ -0,0 +1,44 @@
export default function DocsPagination({ prev = '', next = '' }) {
return (
<div className="w-full flex my-20">
{prev && (
<a href={prev} className="mr-10 no-underline">
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
className="w-6 h-6 inline mr-2"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M15.75 19.5L8.25 12l7.5-7.5"
/>
</svg>
Prev
</a>
)}
{next && (
<a href={next} className="no-underline ml-auto">
Next Lesson
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
strokeWidth={1.5}
stroke="currentColor"
className="w-6 h-6 inline ml-2"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M8.25 4.5l7.5 7.5-7.5 7.5"
/>
</svg>
</a>
)}
</div>
);
}

13
site/components/Layout.tsx
View File

@ -5,6 +5,7 @@ import Link from 'next/link';
import { useCallback, useEffect, useState } from 'react';
import Nav from './Nav';
import { SiteToc } from '@/components/SiteToc';
function useTableOfContents(tableOfContents) {
const [currentSection, setCurrentSection] = useState(tableOfContents[0]?.id);
@ -53,10 +54,14 @@ export default function Layout({
children,
title,
tableOfContents = [],
urlPath,
sidebarTree = []
}: {
children;
title?: string;
tableOfContents?;
urlPath?: string;
sidebarTree?: [];
}) {
// const { toc } = children.props;
const { theme, setTheme } = useTheme();
@ -129,7 +134,7 @@ export default function Layout({
href={`#${subSection.id}`}
className={
isActive(subSection)
? 'text-sky-500'
? 'text-secondary'
: 'hover:text-slate-600 dark:hover:text-slate-300'
}
>
@ -145,6 +150,12 @@ export default function Layout({
</nav>
</div>
)}
{/* LHS NAVIGATION */}
{/* {showSidebar && ( */}
<div className="hidden lg:block fixed z-20 w-[18rem] top-[4.6rem] right-auto bottom-0 left-[max(0px,calc(50%-44rem))] pt-8 pl-8 overflow-y-auto">
<SiteToc currentPath={urlPath} nav={sidebarTree} />
</div>
{/* )} */}
</>
);
}

7
site/components/MDXPage.tsx
View File

@ -1,5 +1,6 @@
import { MDXRemote } from "next-mdx-remote";
import layouts from "layouts";
import { MDXRemote } from 'next-mdx-remote';
import layouts from 'layouts';
import DocsPagination from './DocsPagination';
export default function MDXPage({ source, frontMatter }) {
const Layout = ({ children }) => {
@ -32,7 +33,7 @@ export default function MDXPage({ source, frontMatter }) {
</header>
<main>
<Layout>
<MDXRemote {...source} />
<MDXRemote {...source} components={{ DocsPagination }} />
</Layout>
</main>
</div>

110
site/components/SiteToc.tsx Normal file
View File

@ -0,0 +1,110 @@
import Link from 'next/link.js';
import clsx from 'clsx';
import { Disclosure, Transition } from '@headlessui/react';
export interface NavItem {
name: string;
href: string;
}
export interface NavGroup {
name: string;
path: string;
level: number;
children: Array<NavItem | NavGroup>;
}
interface Props {
currentPath: string;
nav: Array<NavItem | NavGroup>;
}
function isNavGroup(item: NavItem | NavGroup): item is NavGroup {
return (item as NavGroup).children !== undefined;
}
function navItemBeforeNavGroup(a, b) {
if (isNavGroup(a) === isNavGroup(b)) {
return 0;
}
if (isNavGroup(a) && !isNavGroup(b)) {
return 1;
}
return -1;
}
function sortNavGroupChildren(items: Array<NavItem | NavGroup>) {
return items.sort(
(a, b) => navItemBeforeNavGroup(a, b) || a.name.localeCompare(b.name)
);
}
export const SiteToc: React.FC<Props> = ({ currentPath, nav }) => {
return (
<nav data-testid="lhs-sidebar" className="flex flex-col space-y-3 text-sm">
{/* {sortNavGroupChildren(nav).map((n) => ( */}
{nav.map((n) => (
<NavComponent item={n} currentPath={currentPath} />
))}
</nav>
);
};
const NavComponent: React.FC<{
item: NavItem | NavGroup;
currentPath: string;
}> = ({ item, currentPath }) => {
function isActiveItem(item: NavItem) {
return item.href === "/" + currentPath;
}
return !isNavGroup(item) ? (
<Link
key={item.name}
href={item.href}
className={clsx(
isActiveItem(item)
? 'text-secondary'
: 'font-normal text-slate-500 hover:text-slate-700 dark:text-slate-400 dark:hover:text-slate-300',
'block'
)}
>
{item.name}
</Link>
) : (
<Disclosure as="div" key={item.name} className="flex flex-col space-y-3" defaultOpen={true}>
{({ open }) => (
<div>
<Disclosure.Button className="group w-full flex items-center text-left text-md font-medium text-slate-900 dark:text-white">
<svg
className={clsx(
open ? 'text-slate-400 rotate-90' : 'text-slate-300',
'h-3 w-3 mr-2 flex-shrink-0 transform transition-colors duration-150 ease-in-out group-hover:text-slate-400'
)}
viewBox="0 0 20 20"
aria-hidden="true"
>
<path d="M6 6L14 10L6 14V6Z" fill="currentColor" />
</svg>
{item.name}
</Disclosure.Button>
<Transition
enter="transition duration-100 ease-out"
enterFrom="transform scale-95 opacity-0"
enterTo="transform scale-100 opacity-100"
leave="transition duration-75 ease-out"
leaveFrom="transform scale-100 opacity-100"
leaveTo="transform scale-95 opacity-0"
>
<Disclosure.Panel className="flex flex-col space-y-3 pl-5 mt-3">
{/* {sortNavGroupChildren(item.children).map((subItem) => ( */}
{item.children.map((subItem) => (
<NavComponent item={subItem} currentPath={currentPath} />
))}
</Disclosure.Panel>
</Transition>
</div>
)}
</Disclosure>
);
};

21
site/content/data-literate.md
View File

@ -1,21 +0,0 @@
---
title: Data Literate Documents
author: Rufus Pollock and Friends
---
**What?** An experiment in simple, lightweight approach to creating, displaying and sharing datasets and data-driven stories.
**Why?** a simple, fast, extensible way to present data(sets) and author data-driven content. I want to work with markdown for content and quickly add data in the simplest way possible e.g. dropping in links, pasting tables or adding links to the metadata.
**How?** Technically the essence is Markdown+React (MDX) + a curated toolkit of components for data-presentation + NextJS for framework and deployment.
Check out the [demo](/data-literate/demo).
## Background
I have observed two converging data-rich use cases:
* **Data Publishing**: quickly presenting data whether a single file or a full dataset.
* **Data Stories**: creating data-driven content from the simplest of a blog post with a graph to high end there is sophisticated data journalism and visualization.
Both of these can now be well served by a simple markdown-plus approach. Taking data publishing first. I've long been a fan of ultra-simple `README + metadata + csv` datasets. With the evolution of frontmatter we can merge the metadata into the README. However, we still need to "present" the dataset and the key thing for a dataset is the data and this is not something markdown ever supported well ... But now with MDX and the richness of the javascript ecosystem it's quite easy to enhance our markdown and build a rendering pipeleine.

268
site/content/data-literate/demo.mdx
View File

@ -1,268 +0,0 @@
---
title: Demo
---
This demos and documents Data Literate features live.
You can see the raw source of this page here: https://raw.githubusercontent.com/datopian/data-literate/main/content/demo.mdx
## Table of Contents
## GFM
We can have github-flavored markdown including markdown tables, auto-linked links and checklists:
```
https://github.com/datopian/portaljs
| a | b |
|---|---|
| 1 | 2 |
* [x] one thing to do
* [ ] a second thing to do
```
https://github.com/datopian/portaljs
| a | b |
|---|---|
| 1 | 2 |
* [x] one thing to do
* [ ] a second thing to do
## Footnotes
```
here is a footnote reference[^1]
[^1]: a very interesting footnote.
```
here is a footnote reference[^1]
[^1]: a very interesting footnote.
## Frontmatter
Posts can have frontmatter like:
```
---
title: Hello World
author: Rufus Pollock
---
```
The title and description are pulled from the MDX file and processed using `gray-matter`. Additionally, links are rendered using a custom component passed to `next-mdx-remote`.
## A Table of Contents
You can create a table of contents by having a markdown heading named `Table of Contents`. You can see an example at the start of this post.
## A Table
You can create tables ...
```
<Table cols={[
{ key: 'id', name: 'ID' },
{ key: 'firstName', name: 'First name' },
{ key: 'lastName', name: 'Last name' },
{ key: 'age', name: 'Age' }
]} data={[
{ id: 1, lastName: 'Snow', firstName: 'Jon', age: 35 },
{ id: 2, lastName: 'Lannister', firstName: 'Cersei', age: 42 },
{ id: 3, lastName: 'Lannister', firstName: 'Jaime', age: 45 },
{ id: 4, lastName: 'Stark', firstName: 'Arya', age: 16 },
{ id: 7, lastName: 'Clifford', firstName: 'Ferrara', age: 44 },
{ id: 8, lastName: 'Frances', firstName: 'Rossini', age: 36 },
{ id: 9, lastName: 'Roxie', firstName: 'Harvey', age: 65 },
]}
/>
```
<Table cols={[
{ key: 'id', name: 'ID' },
{ key: 'firstName', name: 'First name' },
{ key: 'lastName', name: 'Last name' },
{ key: 'age', name: 'Age' }
]} data={[
{ id: 1, lastName: 'Snow', firstName: 'Jon', age: 35 },
{ id: 2, lastName: 'Lannister', firstName: 'Cersei', age: 42 },
{ id: 3, lastName: 'Lannister', firstName: 'Jaime', age: 45 },
{ id: 4, lastName: 'Stark', firstName: 'Arya', age: 16 },
{ id: 7, lastName: 'Clifford', firstName: 'Ferrara', age: 44 },
{ id: 8, lastName: 'Frances', firstName: 'Rossini', age: 36 },
{ id: 9, lastName: 'Roxie', firstName: 'Harvey', age: 65 },
]}
/>
### Table from Raw CSV
You can also pass raw CSV as the content ...
```
<Table csv={`
Year,Temp Anomaly
1850,-0.418
2020,0.923
`} />
```
<Table csv={`
Year,Temp Anomaly,
1850,-0.418
2020,0.923
`} />
### Table from a URL
<Table url='https://raw.githubusercontent.com/datopian/data-literate/main/public/_files/HadCRUT.5.0.1.0.analysis.summary_series.global.annual.csv' />
```
<Table url='https://raw.githubusercontent.com/datopian/data-literate/main/public/_files/HadCRUT.5.0.1.0.analysis.summary_series.global.annual.csv' />
```
## Charts
You can create charts using a simple syntax.
### Line Chart
<LineChart data={
[
["1850",-0.41765878],
["1851",-0.2333498],
["1852",-0.22939907],
["1853",-0.27035445],
["1854",-0.29163003]
]
}
/>
```
<LineChart data={
[
["1850",-0.41765878],
["1851",-0.2333498],
["1852",-0.22939907],
["1853",-0.27035445],
["1854",-0.29163003]
]
}
/>
```
NB: we have quoted years as otherwise not interpreted as dates but as integers ...
### Vega and Vega Lite
You can using vega or vega-lite. Here's an example using vega-lite:
<VegaLite data={ { "table": [
{
"y": -0.418,
"x": 1850
},
{
"y": 0.923,
"x": 2020
}
]
}
} spec={
{
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
"mark": "bar",
"data": {
"name": "table"
},
"encoding": {
"x": {
"field": "x",
"type": "ordinal"
},
"y": {
"field": "y",
"type": "quantitative"
}
}
}
} />
```jsx
<VegaLite data={ { "table": [
{
"y": -0.418,
"x": 1850
},
{
"y": 0.923,
"x": 2020
}
]
}
} spec={
{
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
"mark": "bar",
"data": {
"name": "table"
},
"encoding": {
"x": {
"field": "x",
"type": "ordinal"
},
"y": {
"field": "y",
"type": "quantitative"
}
}
}
} />
```
#### Line Chart from URL with Tooltip
https://vega.github.io/vega-lite/examples/interactive_multi_line_pivot_tooltip.html
<VegaLite spec={
{
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
"data": {"url": "/_files/HadCRUT.5.0.1.0.analysis.summary_series.global.annual.csv"},
"width": 600,
"height": 250,
"mark": "line",
"encoding": {
"x": {"field": "Time", "type": "temporal"},
"y": {"field": "Anomaly (deg C)", "type": "quantitative"},
"tooltip": {"field": "Anomaly (deg C)", "type": "quantitative"}
}
}
} />
## Display Excel Files
Local file ...
```
<Excel src='/_files/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx' />
```
<Excel src='/_files/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx' />
Remote files work too (even without CORS) thanks to proxying:
```
<Excel src='https://github.com/datasets/awesome-data/files/6604635/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx' />
```
<Excel src='https://github.com/datasets/awesome-data/files/6604635/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx' />

66
site/content/docs/creating-new-datasets.md Normal file
View File

@ -0,0 +1,66 @@
# Creating new datasets
So far, the PortalJS app we created only has a single page displaying a dataset. Data catalogs and data portals generally showcase many different datasets.
Let's explore how to add and display more datasets to our portal.
## Pages in PortalJS
As you have seen, in this example a dataset page is just a markdown file on disk plus a data file.
To create a new data showcase page we just create a new markdown file in the `content/` folder and a new data file in the `public/` folder.
Let's do that now. Create a `content/my-incredible-dataset` folder, and inside this new folder create a `index.md` file with the following content:
```markdown
# My Incredible Dataset
This is my incredible dataset.
## Chart
<LineChart
title="US Population By Decade"
xAxis="Year"
yAxis="Population (mi)"
data="my-incredible-data.csv"
/>
```
Now, create a file in `public/` named `my-incredible-data.csv` and put the following content inside it:
```bash
Year,Population (mi)
1980,227
1990,249
2000,281
2010,309
2020,331
```
Note that pages are associated with a route based on their pathname, so, to see the new data page, access http://localhost:3000/my-incredible-dataset from the browser. You should see the following:
<img src="/assets/docs/my-incredible-dataset.png" />
> [!tip]
> In this tutorial we opted for storing content as markdown files and data as CSV files in the app, but PortalJS can have metadata, data and content stored anywhere.
## Create an index page
Now, let's create an index page. First, create a new folder `content/my-awesome-dataset/` and move `content/index.md` to it. Then, create a new file `content/index.md` and put the following content inside it:
```markdown
# Welcome to my data portal!
List of available datasets:
- [My Awesome Dataset](/my-awesome-dataset)
- [My Incredible Dataset](/my-incredible-dataset)
```
From the browser, access http://localhost:3000. You should see the following:
<img src="/assets/docs/datasets-index-page.png" />
<DocsPagination prev="/docs" next="/docs/searching-datasets" />

56
site/content/docs/deploying-your-portaljs-app.md Normal file
View File

@ -0,0 +1,56 @@
# Deploying your PortalJS app
Finally, let's learn how to deploy PortalJS apps to Vercel or Cloudflare Pages.
> [!tip]
> Although we are using Vercel and Cloudflare Pages in this tutorial, you can deploy apps in any hosting solution you want as a static website by running `npm run export` and distributing the contents of the `out/` folder.
## Push to a GitHub repo
The PortalJS app we built up to this point is stored locally. To allow Vercel or Cloudflare Pages to deploy it, we have to push it to GitHub (or another SCM supported by these hosting solutions).
- Create a new repository under your GitHub account
- Add the new remote origin to your PortalJS app
- Push the app to the repository
If you are not sure about how to do it, follow this guide: https://nextjs.org/learn/basics/deploying-nextjs-app/github
> [!tip]
> You can also deploy using our Vercel deploy button. In this case, a new repository will be created under your GitHub account automatically.
> [Click here](#one-click-deploy) to scroll to the deploy button.
## Deploy to Vercel
The easiest way to deploy a PortalJS app is to use Vercel, a serverless platform for static and hybrid applications developed by the creators of Next.js.
To deploy your PortalJS app:
- Create a Vercel account by going to https://vercel.com/signup and choosing "Continue with GitHub"
- Import the repository you created for the PortalJS app at https://vercel.com/new
- During the setup process you can use the default settings - no need to change anything.
When you deploy, your PortalJS app will start building. It should finish in under a minute.
When it’s done, you’ll get deployment URLs. Click on one of the URLs and you should see your PortaJS app live.
>[!tip]
> You can find a more in-depth explanation about this process at https://nextjs.org/learn/basics/deploying-nextjs-app/deploy
### One-Click Deploy
You can instantly deploy our example app to your Vercel account by clicking the button below:
[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fdatopian%2Fportaljs%2Ftree%2Fmain%2Fexamples%2Flearn-example&project-name=my-data-portal&repository-name=my-data-portal&demo-title=PortalJS%20Learn%20Example&demo-description=PortalJS%20Learn%20Example%20-%20https%3A%2F%2Fportaljs.org%2Fdocs&demo-url=learn-example.portaljs.org&demo-image=https%3A%2F%2Fportaljs.org%2Fassets%2Fexamples%2Fbasic-example.png)
This will create a new repository on your GitHub account and deploy it to Vercel. If you are following the tutorial, you can replicate the changes done on your local app to this new repository.
## Deploy to Cloudflare Pages
To deploy your PortalJS app to Cloudflare Pages, follow this guide:
https://developers.cloudflare.com/pages/framework-guides/deploy-a-nextjs-site/#deploy-with-cloudflare-pages-1
Note that you don't have to change anything - just follow the steps, choosing the repository you created.
<DocsPagination prev="/docs/showing-metadata" />

0
site/content/docs/example-ckan.md → site/content/docs/examples/example-ckan.md
View File

0
site/content/docs/example-data-catalog.md → site/content/docs/examples/example-data-catalog.md
View File

0
site/content/docs/example-frictionless-dataset.md.bkp → site/content/docs/examples/example-frictionless-dataset.md.bkp
View File

260
site/content/docs/index.md
View File

@ -51,262 +51,4 @@ As soon as you save the file, the browser automatically updates the page with th
<img src="/assets/docs/editing-the-page-1.png" />
## Creating new datasets
So far, the PortalJS app we created only has a single page displaying a dataset. Data catalogs and data portals generally showcase many different datasets.
Let's explore how to add and display more datasets to our portal.
### Pages in PortalJS
As you have seen, in this example a dataset page is just a markdown file on disk plus a data file.
To create a new data showcase page we just create a new markdown file in the `content/` folder and a new data file in the `public/` folder.
Let's do that now. Create a `content/my-incredible-dataset` folder, and inside this new folder create a `index.md` file with the following content:
```markdown
# My Incredible Dataset
This is my incredible dataset.
## Chart
<LineChart
title="US Population By Decade"
xAxis="Year"
yAxis="Population (mi)"
data="my-incredible-data.csv"
/>
```
Now, create a file in `public/` named `my-incredible-data.csv` and put the following content inside it:
```bash
Year,Population (mi)
1980,227
1990,249
2000,281
2010,309
2020,331
```
Note that pages are associated with a route based on their pathname, so, to see the new data page, access http://localhost:3000/my-incredible-dataset from the browser. You should see the following:
<img src="/assets/docs/my-incredible-dataset.png" />
> [!tip]
> In this tutorial we opted for storing content as markdown files and data as CSV files in the app, but PortalJS can have metadata, data and content stored anywhere.
### Create an index page
Now, let's create an index page. First, create a new folder `content/my-awesome-dataset/` and move `content/index.md` to it. Then, create a new file `content/index.md` and put the following content inside it:
```markdown
# Welcome to my data portal!
List of available datasets:
- [My Awesome Dataset](/my-awesome-dataset)
- [My Incredible Dataset](/my-incredible-dataset)
```
From the browser, access http://localhost:3000. You should see the following:
<img src="/assets/docs/datasets-index-page.png" />
## Search page
Typing out every link in the index page will get cumbersome eventually, and as the portal grows, finding the datasets you are looking for on the index page will become harder and harder, for that we will need search functionality.
### Creating a search page
Luckily we have a component for that. Change your `content/index.md` file to this:
```
# Welcome to my data portal!
List of available datasets:
<Catalog datasets={datasets} />
```
Before you refresh the page, however, you will need to run the following command:
```
npm run mddb
```
This example makes use of the [markdowndb](https://github.com/datopian/markdowndb) library. For now the only thing you need to know is that you should run the command above everytime you make some change to `/content`.
From the browser, access http://localhost:3000. You should see the following, you now have a searchable automatic list of your datasets:
![](https://i.imgur.com/9HfSPIx.png)
To make this catalog look even better, we can change the text that is being displayed for each dataset to a title. Let's do that by adding the "title" [frontmatter field](https://daily-dev-tips.com/posts/what-exactly-is-frontmatter/) to the first dataset in the list. Change `content/my-awesome-dataset/index.md` to the following:
```
---
title: 'My awesome dataset'
---
# My Awesome Dataset
Built with PortalJS
## Table
<Table url="data.csv" />
```
Rerun `npm run mddb` and, from the browser, access http://localhost:3000. You should see the title appearing instead of the folder name:
![](https://i.imgur.com/nvmSnJ5.png)
Any frontmatter attribute that you add will automatically get indexed and be usable in the search box.
### Adding filters
Sometimes contextual search is not enough. Let's add a filter. To do so, lets add a new metadata field called "group", add it to your `content/my-incredible-dataset/index.md` like so:
```
---
group: 'Incredible'
---
# My Incredible Dataset
This is my incredible dataset.
## Chart
<LineChart
title="US Population By Decade"
xAxis="Year"
yAxis="Population (mi)"
data="my-incredible-data.csv"
/>
```
Also add it to your `content/my-awesome-dataset/index.md` like so:
```
---
title: 'My awesome dataset'
group: 'Awesome'
---
# My Awesome Dataset
Built with PortalJS
## Table
<Table url="data.csv" />
```
Now on your `content/index.md` you can add a "facet" to the `Catalog` component, like so:
```
# Welcome to my data portal!
List of available datasets:
<Catalog datasets={datasets} facets={['group']}/>
```
You now have a filter in your page with all possible values automatically added to it.
![](https://i.imgur.com/p2miSdg.png)
## Showing metadata
If you go now to `http://localhost:3000/my-awesome-dataset`, you will see that we now have two titles on the page. That's because `title` is one of the default metadata fields supported by PortalJS.
![](https://i.imgur.com/O145uuc.png)
Change the content inside `/content/my-awesome-dataset/index.md` to this.
```
---
title: 'My awesome dataset'
author: 'Rufus Pollock'
description: 'An awesome dataset displaying some awesome data'
modified: '2023-05-04'
files: ['data.csv']
groups: ['Awesome']
---
Built with PortalJS
## Table
<Table url="data.csv" />
```
Once you refresh the page at `http://localhost:3000/my-awesome-dataset` you should see something like this at the top:
![](https://i.imgur.com/nvDYJQT.png)
These are the standard metadata fields that will be shown at the top of the page if you add them.
- `title` that gets displayed as a big header at the top of the page
- `author`, `description`, and `modified` which gets displayed below the title
- `files` that get displayed as a table with two columns: `File` which is linked directly to the file, and `Format` which show the file format.
## Deploying your PortalJS app
Finally, let's learn how to deploy PortalJS apps to Vercel or Cloudflare Pages.
> [!tip]
> Although we are using Vercel and Cloudflare Pages in this tutorial, you can deploy apps in any hosting solution you want as a static website by running `npm run export` and distributing the contents of the `out/` folder.
### Push to a GitHub repo
The PortalJS app we built up to this point is stored locally. To allow Vercel or Cloudflare Pages to deploy it, we have to push it to GitHub (or another SCM supported by these hosting solutions).
- Create a new repository under your GitHub account
- Add the new remote origin to your PortalJS app
- Push the app to the repository
If you are not sure about how to do it, follow this guide: https://nextjs.org/learn/basics/deploying-nextjs-app/github
> [!tip]
> You can also deploy using our Vercel deploy button. In this case, a new repository will be created under your GitHub account automatically.
> [Click here](#one-click-deploy) to scroll to the deploy button.
### Deploy to Vercel
The easiest way to deploy a PortalJS app is to use Vercel, a serverless platform for static and hybrid applications developed by the creators of Next.js.
To deploy your PortalJS app:
- Create a Vercel account by going to https://vercel.com/signup and choosing "Continue with GitHub"
- Import the repository you created for the PortalJS app at https://vercel.com/new
- During the setup process you can use the default settings - no need to change anything.
When you deploy, your PortalJS app will start building. It should finish in under a minute.
When it’s done, you’ll get deployment URLs. Click on one of the URLs and you should see your PortaJS app live.
>[!tip]
> You can find a more in-depth explanation about this process at https://nextjs.org/learn/basics/deploying-nextjs-app/deploy
#### One-Click Deploy
You can instantly deploy our example app to your Vercel account by clicking the button below:
[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fdatopian%2Fportaljs%2Ftree%2Fmain%2Fexamples%2Flearn-example&project-name=my-data-portal&repository-name=my-data-portal&demo-title=PortalJS%20Learn%20Example&demo-description=PortalJS%20Learn%20Example%20-%20https%3A%2F%2Fportaljs.org%2Fdocs&demo-url=learn-example.portaljs.org&demo-image=https%3A%2F%2Fportaljs.org%2Fassets%2Fexamples%2Fbasic-example.png)
This will create a new repository on your GitHub account and deploy it to Vercel. If you are following the tutorial, you can replicate the changes done on your local app to this new repository.
### Deploy to Cloudflare Pages
To deploy your PortalJS app to Cloudflare Pages, follow this guide:
https://developers.cloudflare.com/pages/framework-guides/deploy-a-nextjs-site/#deploy-with-cloudflare-pages-1
Note that you don't have to change anything - just follow the steps, choosing the repository you created.
<DocsPagination next="/docs/creating-new-datasets" />

105
site/content/docs/searching-datasets.md Normal file
View File

@ -0,0 +1,105 @@
# Searching datasets
Typing out every link in the index page will get cumbersome eventually, and as the portal grows, finding the datasets you are looking for on the index page will become harder and harder, for that we will need search functionality.
## Creating a search page
Luckily we have a component for that. Change your `content/index.md` file to this:
```
# Welcome to my data portal!
List of available datasets:
<Catalog datasets={datasets} />
```
Before you refresh the page, however, you will need to run the following command:
```
npm run mddb
```
This example makes use of the [markdowndb](https://github.com/datopian/markdowndb) library. For now the only thing you need to know is that you should run the command above everytime you make some change to `/content`.
From the browser, access http://localhost:3000. You should see the following, you now have a searchable automatic list of your datasets:
![](https://i.imgur.com/9HfSPIx.png)
To make this catalog look even better, we can change the text that is being displayed for each dataset to a title. Let's do that by adding the "title" [frontmatter field](https://daily-dev-tips.com/posts/what-exactly-is-frontmatter/) to the first dataset in the list. Change `content/my-awesome-dataset/index.md` to the following:
```
---
title: 'My awesome dataset'
---
# My Awesome Dataset
Built with PortalJS
## Table
<Table url="data.csv" />
```
Rerun `npm run mddb` and, from the browser, access http://localhost:3000. You should see the title appearing instead of the folder name:
![](https://i.imgur.com/nvmSnJ5.png)
Any frontmatter attribute that you add will automatically get indexed and be usable in the search box.
## Adding filters
Sometimes contextual search is not enough. Let's add a filter. To do so, lets add a new metadata field called "group", add it to your `content/my-incredible-dataset/index.md` like so:
```
---
group: 'Incredible'
---
# My Incredible Dataset
This is my incredible dataset.
## Chart
<LineChart
title="US Population By Decade"
xAxis="Year"
yAxis="Population (mi)"
data="my-incredible-data.csv"
/>
```
Also add it to your `content/my-awesome-dataset/index.md` like so:
```
---
title: 'My awesome dataset'
group: 'Awesome'
---
# My Awesome Dataset
Built with PortalJS
## Table
<Table url="data.csv" />
```
Now on your `content/index.md` you can add a "facet" to the `Catalog` component, like so:
```
# Welcome to my data portal!
List of available datasets:
<Catalog datasets={datasets} facets={['group']}/>
```
You now have a filter in your page with all possible values automatically added to it.
![](https://i.imgur.com/p2miSdg.png)
<DocsPagination prev="/docs/creating-new-datasets" next="/docs/showing-metadata" />

36
site/content/docs/showing-metadata.md Normal file
View File

@ -0,0 +1,36 @@
# Showing metadata
If you go now to `http://localhost:3000/my-awesome-dataset`, you will see that we now have two titles on the page. That's because `title` is one of the default metadata fields supported by PortalJS.
![](https://i.imgur.com/O145uuc.png)
Change the content inside `/content/my-awesome-dataset/index.md` to this.
```
---
title: 'My awesome dataset'
author: 'Rufus Pollock'
description: 'An awesome dataset displaying some awesome data'
modified: '2023-05-04'
files: ['data.csv']
groups: ['Awesome']
---
Built with PortalJS
## Table
<Table url="data.csv" />
```
Once you refresh the page at `http://localhost:3000/my-awesome-dataset` you should see something like this at the top:
![](https://i.imgur.com/nvDYJQT.png)
These are the standard metadata fields that will be shown at the top of the page if you add them.
- `title` that gets displayed as a big header at the top of the page
- `author`, `description`, and `modified` which gets displayed below the title
- `files` that get displayed as a table with two columns: `File` which is linked directly to the file, and `Format` which show the file format.
<DocsPagination prev="/docs/searching-datasets" next="/docs/deploying-your-portaljs-app" />

40
site/content/docs/sidebar.json Normal file
View File

@ -0,0 +1,40 @@
[
{
"name": "Getting started",
"children": [
{
"name": "Setup",
"href": "/docs"
},
{
"name": "Creating new datasets",
"href": "/docs/creating-new-datasets"
},
{
"name": "Searching datasets",
"href": "/docs/searching-datasets"
},
{
"name": "Showing metadata",
"href": "/docs/showing-metadata"
},
{
"name": "Deploying your PortalJS app",
"href": "/docs/deploying-your-portaljs-app"
}
]
},
{
"name": "Examples",
"children": [
{
"name": "Data Catalog w/ CKAN datasets",
"href": "/docs/examples/example-ckan"
},
{
"name": "Data Catalog w/ GitHub datasets",
"href": "/docs/examples/example-data-catalog"
}
]
}
]

5
site/content/gallery.md
View File

@ -1,5 +0,0 @@
---
title: Gallery
---
Come back soon!

121
site/lib/computeFields.ts Normal file
View File

@ -0,0 +1,121 @@
// This file is a temporary replacement for legacy contentlayer's computeFields + default fields values
import { remark } from "remark";
import stripMarkdown, { Options } from "strip-markdown";
import { siteConfig } from "../config/siteConfig";
import { getAuthorsDetails } from "./getAuthorsDetails";
import sluggify from "./sluggify";
// TODO return type
const computeFields = async ({
frontMatter,
urlPath,
filePath,
source,
}: {
frontMatter: Record<string, any>;
urlPath: string;
filePath: string;
source: string;
}) => {
// Fields with corresponding config options
// TODO see _app.tsx
const showComments =
frontMatter.showComments ?? siteConfig.showComments ?? false;
const showEditLink =
frontMatter.showEditLink ?? siteConfig.showEditLink ?? false;
// TODO take config into accout
const showLinkPreviews =
frontMatter.showLinkPreviews ?? siteConfig.showLinkPreviews ?? false;
const showToc = frontMatter.showToc ?? siteConfig.showToc ?? false;
const showSidebar =
frontMatter.showSidebar ?? siteConfig.showSidebar ?? false;
const sidebarTreeFile = frontMatter.sidebarTreeFile ?? null;
// Computed fields
// const title = frontMatter.title ?? (await extractTitle(source));
const title = frontMatter.title ?? null;
const description =
frontMatter.description ?? (await extractDescription(source));
const date = frontMatter.date ?? frontMatter.created ?? null;
const layout = (() => {
if (frontMatter.layout) return frontMatter.layout;
if (urlPath.startsWith("blog/")) return "blog";
// if (urlPath.startsWith("docs/")) return "docs";
return "docs"; // TODO default layout from config?
})();
// TODO Temporary, should probably be a column in the database
const slug = sluggify(urlPath);
// TODO take into accout include/exclude fields in config
const isDraft = frontMatter.isDraft ?? false;
const editUrl =
(siteConfig.editLinkRoot && `${siteConfig.editLinkRoot}/${filePath}`) ||
null;
const authors = await getAuthorsDetails(frontMatter.authors);
return {
...frontMatter,
authors,
title,
description,
date,
layout,
slug,
urlPath, // extra for blogs index page; temporary here
isDraft,
editUrl,
showComments,
showEditLink,
showLinkPreviews,
showToc,
showSidebar,
sidebarTreeFile
};
};
const extractTitle = async (source: string) => {
const heading = source.trim().match(/^#\s+(.*)/);
if (heading) {
const title = heading[1]
// replace wikilink with only text value
.replace(/\[\[([\S]*?)]]/, "$1");
const stripTitle = await remark().use(stripMarkdown).process(title);
return stripTitle.toString().trim();
}
return null;
};
const extractDescription = async (source: string) => {
const content = source
// remove commented lines
.replace(/{\/\*.*\*\/}/g, "")
// remove import statements
.replace(
/^import\s*(?:\{\s*[\w\s,\n]+\s*\})?(\s*(\w+))?\s*from\s*("|')[^"]+("|');?$/gm,
""
)
// remove youtube links
.replace(/^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.?be)\/.+$/gm, "")
// replace wikilinks with only text
.replace(/([^!])\[\[(\S*?)\]]/g, "$1$2")
// remove wikilink images
.replace(/!\[[\S]*?]]/g, "");
// remove markdown formatting
const stripped = await remark()
.use(stripMarkdown, {
remove: ["heading", "blockquote", "list", "image", "html", "code"],
} as Options)
.process(content);
if (stripped.value) {
const description: string = stripped.value.toString().slice(0, 200);
return description + "...";
}
return null;
};
export default computeFields;

5
site/lib/sluggify.ts Normal file
View File

@ -0,0 +1,5 @@
const sluggify = (urlPath: string) => {
return urlPath.replace(/^(.+?\/)*/, "");
};
export default sluggify;

3152
site/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

10
site/package.json
View File

@ -10,8 +10,8 @@
"mddb": "mddb content"
},
"dependencies": {
"@flowershow/core": "^0.4.9",
"@flowershow/markdowndb": "^0.1.0",
"@flowershow/core": "^0.4.11",
"@flowershow/markdowndb": "^0.1.1",
"@flowershow/remark-callouts": "^1.0.0",
"@flowershow/remark-embed": "^1.0.0",
"@flowershow/remark-wiki-link": "^1.0.1",
@ -46,7 +46,8 @@
"remark-smartypants": "^2.0.0",
"remark-toc": "^7.2.0",
"vega": "^5.20.2",
"vega-lite": "^5.1.0"
"vega-lite": "^5.1.0",
"strip-markdown": "^5.0.0"
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.9",
@ -54,6 +55,7 @@
"postcss": "^8.4.22",
"prettier": "^2.8.7",
"tailwindcss": "^3.3.1",
"typescript": "^5.0.4"
"typescript": "^5.0.4",
"remark": "^14.0.2"
}
}

166
site/pages/[...slug].tsx
View File

@ -4,15 +4,17 @@ import parse from '../lib/markdown.mjs';
import MDXPage from '../components/MDXPage';
import clientPromise from '@/lib/mddb';
import { getAuthorsDetails } from 'lib/getAuthorsDetails';
import Layout from 'components/Layout';
import { useEffect, useState } from 'react';
import { useRouter } from 'next/router.js';
import { collectHeadings } from '@flowershow/core';
import { NavGroup, NavItem, collectHeadings } from '@flowershow/core';
import { GetStaticProps, GetStaticPropsResult } from 'next';
import { CustomAppProps } from './_app.jsx';
import computeFields from '@/lib/computeFields';
import { getAuthorsDetails } from '@/lib/getAuthorsDetails';
export default function DRDPage({ source, frontMatter }) {
export default function Page({ source, meta, sidebarTree }) {
source = JSON.parse(source);
frontMatter = JSON.parse(frontMatter);
const router = useRouter();
@ -27,53 +29,81 @@ export default function DRDPage({ source, frontMatter }) {
}, [router.asPath]); // update table of contents on route change with next/link
return (
<Layout tableOfContents={tableOfContents} title={frontMatter.title}>
<MDXPage source={source} frontMatter={frontMatter} />
<Layout
tableOfContents={tableOfContents}
title={meta.title}
sidebarTree={sidebarTree}
urlPath={meta.urlPath}
>
<MDXPage source={source} frontMatter={meta} />
</Layout>
);
}
export const getStaticProps = async ({ params }) => {
const urlPath = params.slug ? params.slug.join('/') : '';
interface SlugPageProps extends CustomAppProps {
source: any;
}
export const getStaticProps: GetStaticProps = async ({
params,
}): Promise<GetStaticPropsResult<SlugPageProps>> => {
const urlPath = params?.slug ? (params.slug as string[]).join('/') : '/';
const mddb = await clientPromise;
const dbFile = await mddb.getFileByUrl(urlPath);
const dbBacklinks = await mddb.getLinks({
fileId: dbFile._id,
direction: 'backward',
});
// TODO temporary solution, we will have a method on MddbFile to get these links
const dbBacklinkFilesPromises = dbBacklinks.map((link) =>
mddb.getFileById(link.from)
);
const dbBacklinkFiles = await Promise.all(dbBacklinkFilesPromises);
const dbBacklinkUrls = dbBacklinkFiles.map(
(file) => file.toObject().url_path
);
// TODO we can already get frontmatter from dbFile.metadata
// so parse could only return mdxSource
const source = fs.readFileSync(dbFile.file_path, { encoding: 'utf-8' });
const { mdxSource, frontMatter } = await parse(source, 'mdx', {
backlinks: dbBacklinkUrls,
});
const filePath = dbFile!.file_path;
const frontMatter = dbFile!.metadata ?? {};
// Temporary, so that blogs work properly
if (
dbFile.url_path.startsWith('blog/') ||
(dbFile.url_path.startsWith('docs/') && dbFile.metadata.filetype === 'blog')
) {
if (dbFile.metadata.filetype === 'blog') {
frontMatter.layout = 'blog';
frontMatter.authorsDetails = await getAuthorsDetails(
dbFile.metadata.authors
);
}
// Temporary, docs pages should present the LHS sidebar
if (dbFile.url_path.startsWith('docs')) {
frontMatter.showSidebar = true;
frontMatter.sidebarTreeFile = 'content/docs/sidebar.json';
}
const source = fs.readFileSync(filePath, { encoding: 'utf-8' });
const { mdxSource } = await parse(source, 'mdx', {});
// TODO temporary replacement for contentlayer's computedFields
const frontMatterWithComputedFields = await computeFields({
frontMatter,
urlPath,
filePath,
source,
});
let sidebarTree: Array<NavGroup | NavItem> = [];
if (frontMatterWithComputedFields?.showSidebar) {
let sidebarTreeFile = frontMatterWithComputedFields?.sidebarTreeFile;
// Added this file funcionality so that we can control
// which items appear in the sidebar and the order via
// a json file
if (sidebarTreeFile) {
const tree = fs.readFileSync(sidebarTreeFile, { encoding: 'utf-8' });
sidebarTree = JSON.parse(tree);
} else {
const allPages = await mddb.getFiles({ extensions: ['md', 'mdx'] });
const pages = allPages.filter((p) => !p.metadata?.isDraft);
pages.forEach((page) => {
addPageToSitemap(page, sidebarTree);
});
}
}
return {
props: {
source: JSON.stringify(mdxSource),
frontMatter: JSON.stringify(frontMatter),
meta: frontMatterWithComputedFields,
sidebarTree,
},
};
};
@ -82,18 +112,70 @@ export async function getStaticPaths() {
const mddb = await clientPromise;
let allDocuments = await mddb.getFiles({ extensions: ['md', 'mdx'] });
// Avoid duplicate path
allDocuments = allDocuments.filter(
(doc) => !doc.url_path.startsWith('data-literate/')
);
const paths = allDocuments.map((page) => {
const parts = page.url_path.split('/');
return { params: { slug: parts } };
});
const paths = allDocuments
.filter((page) => page.metadata?.isDraft !== true)
.map((page) => {
const parts = page.url_path!.split('/');
return { params: { slug: parts } };
});
return {
paths,
fallback: false,
};
}
function capitalize(str: string) {
return str.charAt(0).toUpperCase() + str.slice(1);
}
/* function addPageToGroup(page: MddbFile, sitemap: Array<NavGroup>) { */
function addPageToSitemap(page: any, sitemap: Array<NavGroup | NavItem>) {
const urlParts = page.url_path!.split('/').filter((part) => part);
// don't add home page to the sitemap
if (urlParts.length === 0) return;
// top level, root pages
if (urlParts.length === 1) {
sitemap.push({
name: page.metadata?.title || urlParts[0],
href: page.url_path,
});
} else {
// /blog/blogtest
const nestingLevel = urlParts.length - 1; // 1
let currArray: Array<NavItem | NavGroup> = sitemap;
for (let level = 0; level <= nestingLevel; level++) {
if (level === nestingLevel) {
currArray.push({
name: urlParts[level],
href: page.url_path,
});
continue;
}
const matchingGroup = currArray
.filter(isNavGroup)
.find(
(group) =>
group.path !== undefined && page.url_path.startsWith(group.path)
);
if (!matchingGroup) {
const newGroup: NavGroup = {
name: capitalize(urlParts[level]),
path: urlParts.slice(0, level + 1).join('/'),
level,
children: [],
};
currArray.push(newGroup);
currArray = newGroup.children;
} else {
currArray = matchingGroup.children;
}
}
}
}
function isNavGroup(item: NavItem | NavGroup): item is NavGroup {
return (item as NavGroup).children !== undefined;
}

16
site/pages/_app.tsx
View File

@ -5,11 +5,25 @@ import Script from "next/script";
import { DefaultSeo } from "next-seo";
import { pageview, ThemeProvider } from "@flowershow/core";
import { NavGroup, NavItem, pageview, ThemeProvider } from "@flowershow/core";
import { siteConfig } from "../config/siteConfig";
import { useEffect } from "react";
import { useRouter } from "next/dist/client/router";
export interface CustomAppProps {
meta: {
showToc: boolean;
showEditLink: boolean;
showSidebar: boolean;
showComments: boolean;
urlPath: string; // not sure what's this for
editUrl?: string;
[key: string]: any;
};
siteMap?: Array<NavItem | NavGroup>;
[key: string]: any;
}
function MyApp({ Component, pageProps }) {
const router = useRouter();

26
site/pages/blog.tsx
View File

@ -1,6 +1,8 @@
import Layout from '@/components/Layout';
import computeFields from '@/lib/computeFields';
import clientPromise from '@/lib/mddb';
import { BlogsList, SimpleLayout } from '@flowershow/core';
import * as fs from 'fs';
export default function Blog({ blogs }) {
return (
@ -32,19 +34,29 @@ export async function getStaticProps() {
blogs = [...blogs, ...docs];
const blogsSorted = blogs.sort(
const blogsWithComputedFields = blogs.map(async (blog) => {
const source = fs.readFileSync(blog.file_path, { encoding: 'utf-8' });
return await computeFields({
frontMatter: blog.metadata,
urlPath: blog.url_path,
filePath: blog.file_path,
source,
});
});
const blogList = await Promise.all(blogsWithComputedFields);
const blogsSorted = blogList.sort(
(a, b) =>
new Date(b.metadata.date).getTime() - new Date(a.metadata.date).getTime()
new Date(b?.date).getTime() -
new Date(a?.date).getTime()
);
// Temporary, flowershow/BlogsList expects the contentlayer fields
const blogsObjects = blogsSorted.map((b) => {
return { ...b, ...b.metadata };
});
return {
props: {
blogs: blogsObjects,
blogs: blogsSorted,
},
};
}