Part 3 tutorial - Creating an index page (#834)

* [learn-example][m] - code section for the tutorial part 3

* [learn-example][sm] - dont panic when no markdown.db file found

* [docs][m] - creating an inedx page

examples/learn-example/components/Catalog.tsx

@@ -0,0 +1,40 @@
+import { Index } from 'flexsearch';
+import { useState } from 'react';
+import DebouncedInput from './DebouncedInput';
+
+export default function Catalog({ datasets }: { datasets: any[] }) {
+  const [indexFilter, setIndexFilter] = useState('');
+  const index = new Index({ tokenize: "full"});
+  datasets.forEach((dataset) =>
+    index.add(
+      dataset._id,
+      Object.entries(dataset.metadata).reduce(
+            (acc, curr) => acc + ' ' + curr.toString(),
+            ''
+          ) + ' ' + dataset.url_path
+    )
+  );
+  return (
+    <>
+      <DebouncedInput
+        value={indexFilter ?? ''}
+        onChange={(value) => setIndexFilter(String(value))}
+        className="p-2 text-sm shadow border border-block"
+        placeholder="Search all datasets..."
+      />
+      <ul>
+        {datasets
+          .filter((dataset) =>
+            indexFilter !== ''
+              ? index.search(indexFilter).includes(dataset._id)
+              : true
+          )
+          .map((dataset) => (
+            <li key={dataset.id}>
+              <a href={dataset.url_path}>{dataset.metadata.title ? dataset.metadata.title : dataset.url_path}</a>
+            </li>
+          ))}
+      </ul>
+    </>
+  );
+}

examples/learn-example/components/DRD.tsx

@@ -8,6 +8,7 @@ import { Mermaid } from '@flowershow/core';
 // here.
 const components = {
   Table: dynamic(() => import('./Table')),
+  Catalog: dynamic(() => import('./Catalog')),
   mermaid: Mermaid,
   // Excel: dynamic(() => import('../components/Excel')),
   // TODO: try and make these dynamic ...

examples/learn-example/lib/markdown.js

@@ -22,7 +22,7 @@ import { serialize } from "next-mdx-remote/serialize";
  * @format: used to indicate to next-mdx-remote which format to use (md or mdx)
  * @returns: { mdxSource: mdxSource, frontMatter: ...}
  */
-const parse = async function (source, format) {
+const parse = async function (source, format, scope) {
   const { content, data, excerpt } = matter(source, {
     excerpt: (file, options) => {
       // Generate an excerpt for the file
@@ -91,7 +91,7 @@ const parse = async function (source, format) {
         ],
         format,
       },
-      scope: data,
+      scope: { ...scope, ...data},
     }
   );
 

examples/learn-example/lib/mddb.ts

@@ -0,0 +1,14 @@
+import { MarkdownDB } from "@flowershow/markdowndb";
+
+const dbPath = "markdown.db";
+
+const client = new MarkdownDB({
+  client: "sqlite3",
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const clientPromise = client.init();
+
+export default clientPromise;

examples/learn-example/package.json

@@ -7,21 +7,26 @@
     "build": "next build",
     "start": "next start",
     "lint": "next lint",
-    "export": "npm run build && next export -o out"
+    "export": "npm run build && next export -o out",
+    "prebuild": "npm run mddb",
+    "mddb": "mddb ./content"
   },
   "dependencies": {
     "@flowershow/core": "^0.4.10",
+    "@flowershow/markdowndb": "^0.1.1",
     "@flowershow/remark-callouts": "^1.0.0",
     "@flowershow/remark-embed": "^1.0.0",
     "@flowershow/remark-wiki-link": "^1.1.2",
     "@heroicons/react": "^2.0.17",
     "@opentelemetry/api": "^1.4.0",
     "@tanstack/react-table": "^8.8.5",
+    "@types/flexsearch": "^0.7.3",
     "@types/node": "18.16.0",
     "@types/react": "18.2.0",
     "@types/react-dom": "18.2.0",
     "eslint": "8.39.0",
     "eslint-config-next": "13.3.1",
+    "flexsearch": "^0.7.31",
     "gray-matter": "^4.0.3",
     "hastscript": "^7.2.0",
     "mdx-mermaid": "2.0.0-rc7",

examples/learn-example/pages/[[...path]].tsx

@@ -1,7 +1,8 @@
-import { promises as fs } from 'fs';
+import { existsSync, promises as fs } from 'fs';
 import path from 'path';
 import parse from '../lib/markdown';
 import DRD from '../components/DRD';
+import clientPromise from '../lib/mddb';
 
 export const getStaticPaths = async () => {
   const contentDir = path.join(process.cwd(), '/content/');
@@ -23,9 +24,28 @@ export const getStaticProps = async (context) => {
     pathToFile = context.params.path.join('/') + '/index.md';
   }
 
+  let datasets = [];
+  const mddbFileExists = existsSync('markdown.db');
+  if (mddbFileExists) {
+    const mddb = await clientPromise;
+    const datasetsFiles = await mddb.getFiles({
+      extensions: ['md', 'mdx'],
+    });
+    datasets = datasetsFiles
+      .filter((dataset) => dataset.url_path !== '/')
+      .map((dataset) => ({
+        _id: dataset._id,
+        url_path: dataset.url_path,
+        file_path: dataset.file_path,
+        metadata: dataset.metadata,
+      }));
+  }
+
   const indexFile = path.join(process.cwd(), '/content/' + pathToFile);
   const readme = await fs.readFile(indexFile, 'utf8');
-  let { mdxSource, frontMatter } = await parse(readme, '.mdx');
+
+  let { mdxSource, frontMatter } = await parse(readme, '.mdx', { datasets });
+
   return {
     props: {
       mdxSource,

site/content/docs/index.md

@@ -116,6 +116,52 @@ From the browser, access http://localhost:3000. You should see the following:
 
 <img src="/assets/docs/datasets-index-page.png" />
 
+### Creating a search page
+
+Typing out every link in the index page will get cumbersome eventually, and as the portal grows, finding the datasets you are looking for on the index page will become harder and harder. Luckily we have a component for that. Change your `content/index.md` file to this:
+
+```
+# Welcome to my data portal!
+
+List of available datasets:
+
+<Catalog datasets={datasets} />
+```
+
+Before you refresh the page, however, you will need to run the following command:
+
+```
+npm run mddb
+```
+
+This example makes use of the [markdowndb](https://github.com/datopian/markdowndb) library. For now the only thing you need to know is that you should run the command above everytime you make some change to `/content`.
+
+From the browser, access http://localhost:3000. You should see the following, you now have a searchable automatic list of your datasets:
+
+![](https://i.imgur.com/9HfSPIx.png)
+
+To make this catalog look even better, we can change the text that is being displayed for each dataset to a title. Let's do that by adding the "title" [frontmatter field](https://daily-dev-tips.com/posts/what-exactly-is-frontmatter/) to the first dataset in the list. Change `content/my-awesome-dataset/index.md` to the following:
+
+```
+---
+title: 'My awesome dataset'
+---
+
+# My Awesome Dataset
+
+Built with PortalJS 
+
+## Table 
+
+<Table url="data.csv" />
+```
+
+Rerun `npm run mddb` and, from the browser, access http://localhost:3000. You should see the title appearing instead of the folder name:
+
+![](https://i.imgur.com/nvmSnJ5.png)
+
+Any frontmatter attribute that you add will automatically get indexed and be usable in the search box.
+
 ## Deploying your PortalJS app
 
 Finally, let's learn how to deploy PortalJS apps to Vercel or Cloudflare Pages.