Example of how to create a data portal in 5 minutes (#769)

* [example][m] - start of a simple-example

* Empty-Commit

* [simple-example][sm] - change from repos.json to datasets.json

* [example][m] - changed styling and added octokit

* [build][sm] - fix build

examples/simple-example/lib/loadUrlProxied.tsx

@@ -0,0 +1,11 @@
+import axios from "axios";
+
+export default function loadUrlProxied(url: string) {
+  // HACK: duplicate of Excel code - maybe refactor
+  // if url is external may have CORS issue so we proxy it ...
+  if (url.startsWith("http")) {
+    const PROXY_URL = "/api/proxy";
+    url = PROXY_URL + "?url=" + encodeURIComponent(url);
+  }
+  return axios.get(url).then((res) => res.data);
+}

examples/simple-example/lib/markdown.js

@@ -0,0 +1,105 @@
+import matter from "gray-matter";
+import mdxmermaid from "mdx-mermaid";
+import { h } from "hastscript";
+import remarkCallouts from "@flowershow/remark-callouts";
+import remarkEmbed from "@flowershow/remark-embed";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import remarkSmartypants from "remark-smartypants";
+import remarkToc from "remark-toc";
+import remarkWikiLink from "@flowershow/remark-wiki-link";
+import rehypeAutolinkHeadings from "rehype-autolink-headings";
+import rehypeKatex from "rehype-katex";
+import rehypeSlug from "rehype-slug";
+import rehypePrismPlus from "rehype-prism-plus";
+
+import { serialize } from "next-mdx-remote/serialize";
+
+/**
+ * Parse a markdown or MDX file to an MDX source form + front matter data
+ *
+ * @source: the contents of a markdown or mdx file
+ * @format: used to indicate to next-mdx-remote which format to use (md or mdx)
+ * @returns: { mdxSource: mdxSource, frontMatter: ...}
+ */
+const parse = async function (source, format) {
+  const { content, data, excerpt } = matter(source, {
+    excerpt: (file, options) => {
+      // Generate an excerpt for the file
+      file.excerpt = file.content.split("\n\n")[0];
+    },
+  });
+
+  const mdxSource = await serialize(
+    { value: content, path: format },
+    {
+      // Optionally pass remark/rehype plugins
+      mdxOptions: {
+        remarkPlugins: [
+          remarkEmbed,
+          remarkGfm,
+          [remarkSmartypants, { quotes: false, dashes: "oldschool" }],
+          remarkMath,
+          remarkCallouts,
+          remarkWikiLink,
+          [
+            remarkToc,
+            {
+              heading: "Table of contents",
+              tight: true,
+            },
+          ],
+          [mdxmermaid, {}],
+        ],
+        rehypePlugins: [
+          rehypeSlug,
+          [
+            rehypeAutolinkHeadings,
+            {
+              properties: { className: 'heading-link' },
+              test(element) {
+                return (
+                  ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
+                  element.properties?.id !== "table-of-contents" &&
+                  element.properties?.className !== "blockquote-heading"
+                );
+              },
+              content() {
+                return [
+                  h(
+                    "svg",
+                    {
+                      xmlns: "http:www.w3.org/2000/svg",
+                      fill: "#ab2b65",
+                      viewBox: "0 0 20 20",
+                      className: "w-5 h-5",
+                    },
+                    [
+                      h("path", {
+                        fillRule: "evenodd",
+                        clipRule: "evenodd",
+                        d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
+                      }),
+                    ]
+                  ),
+                ];
+              },
+            },
+          ],
+          [rehypeKatex, { output: "mathml" }],
+          [rehypePrismPlus, { ignoreMissing: true }],
+        ],
+        format,
+      },
+      scope: data,
+    }
+  );
+
+  return {
+    mdxSource: mdxSource,
+    frontMatter: data,
+    excerpt,
+  };
+};
+
+export default parse;

examples/simple-example/lib/parseCsv.ts

@@ -0,0 +1,16 @@
+import papa from "papaparse";
+
+const parseCsv = (csv) => {
+  csv = csv.trim();
+  const rawdata = papa.parse(csv, { header: true });
+  const cols = rawdata.meta.fields.map((r, i) => {
+    return { key: r, name: r };
+  });
+
+  return {
+    rows: rawdata.data,
+    fields: cols,
+  };
+};
+
+export default parseCsv;

examples/simple-example/lib/project.ts

@@ -0,0 +1,60 @@
+import * as crypto from "crypto";
+import axios from "axios";
+import { Octokit } from "octokit"
+
+export default class Project {
+  id: string;
+  name: string;
+  owner: string;
+  github_repo: string;
+  readme: string;
+  metadata: any;
+  repo_metadata: any;
+
+  constructor(owner: string, name: string) {
+    this.name = name;
+    this.owner = owner;
+    this.github_repo = `https://github.com/${owner}/${name}`;
+
+    //  TODO: using the GitHub repo to set the id is not a good idea
+    //  since repos can be renamed and then we are going to end up with
+    //  a duplicate
+    const encodedGHRepo = Buffer.from(this.github_repo, "utf-8").toString();
+    this.id = crypto.createHash("sha1").update(encodedGHRepo).digest("hex");
+  }
+
+  initFromGitHub = async () => {
+    const octokit = new Octokit()
+    //  TODO: what if the repo doesn't exist?
+    await this.getFileContent("README.md")
+      .then((content) => (this.readme = content))
+      .catch((e) => (this.readme = null));
+
+    await this.getFileContent("datapackage.json")
+      .then((content) => (this.metadata = content))
+      .catch((e) => (this.metadata = {}));
+
+    const github_metadata = await octokit.rest.repos.get({ owner: this.owner, repo: this.name })
+    this.repo_metadata = github_metadata.data ? github_metadata.data : null
+  };
+
+  getFileContent = (path, branch = "main") => {
+    return axios
+      .get(
+        `https://raw.githubusercontent.com/${this.owner}/${this.name}/${branch}/${path}`
+      )
+      .then((res) => res.data);
+  };
+
+  serialize() {
+    return JSON.parse(JSON.stringify(this));
+  }
+
+  static async getFromGitHub(owner: string, name: string) {
+    const project = new Project(owner, name);
+    await project.initFromGitHub();
+
+    return project;
+  }
+}
+

examples/simple-example/lib/viewSpecConversion.ts

@@ -0,0 +1,47 @@
+export function convertSimpleToVegaLite(view, resource) {
+  const x = resource.schema.fields.find((f) => f.name === view.spec.group);
+  const y = resource.schema.fields.find((f) => f.name === view.spec.series[0]);
+
+  const xType = inferVegaType(x.type);
+  const yType = inferVegaType(y.type);
+
+  let vegaLiteSpec = {
+    $schema: "https://vega.github.io/schema/vega-lite/v5.json",
+    mark: {
+      type: view.spec.type,
+      color: "black",
+      strokeWidth: 1,
+      tooltip: true,
+    },
+    title: view.title,
+    width: "container",
+    height: 300,
+    selection: {
+      grid: {
+        type: "interval",
+        bind: "scales",
+      },
+    },
+    encoding: {
+      x: {
+        field: x.name,
+        type: xType,
+      },
+      y: {
+        field: y.name,
+        type: yType,
+      },
+    },
+  };
+
+  return vegaLiteSpec;
+}
+
+const inferVegaType = (fieldType) => {
+  switch (fieldType) {
+    case "date":
+      return "Temporal";
+    case "number":
+      return "Quantitative";
+  }
+};