Run prettier

* [examples/openspending] - openspending v0.2

* [examples/openspending][m] - fix build

* [examples/openspending][xs] - fix build

* [examples/openspending][xs] - add prebuild step

* [examples/openspending][m] - fix requested by demenech

* [examples/openspending][sm] - remove links + fix bug

.gitignore

@@ -16,6 +16,7 @@ node_modules
 *.launch
 .settings/
 *.sublime-workspace
+.obsidian
 
 # IDE - VSCode
 .vscode/*

examples/ckan-example/.env

@@ -1 +0,0 @@
-DMS=https://demo.dev.datopian.com

examples/ckan-ssg/package.json

@@ -10,6 +10,7 @@
   },
   "dependencies": {
     "@heroicons/react": "^2.0.17",
+    "@portaljs/ckan": "^0.0.2",
     "next": "13.3.1",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
@@ -20,14 +21,14 @@
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",
-    "autoprefixer": "^10.4.14",
-    "postcss": "^8.4.23",
-    "tailwindcss": "^3.3.1",
-    "eslint": "8.39.0",
-    "eslint-config-next": "13.3.1",
-    "typescript": "5.0.4",
     "@types/node": "18.16.0",
     "@types/react": "18.0.38",
-    "@types/react-dom": "18.0.11"
+    "@types/react-dom": "18.0.11",
+    "autoprefixer": "^10.4.14",
+    "eslint": "8.39.0",
+    "eslint-config-next": "13.3.1",
+    "postcss": "^8.4.23",
+    "tailwindcss": "^3.3.1",
+    "typescript": "5.0.4"
   }
 }

examples/ckan-ssg/pages/@org/[org]/[dataset]/index.tsx

@@ -11,8 +11,9 @@ import {
   ServerIcon,
   UserIcon,
 } from '@heroicons/react/20/solid';
+import { CKAN } from '@portaljs/ckan';
 
-const dms = getConfig().publicRuntimeConfig.DMS;
+const backend_url = getConfig().publicRuntimeConfig.DMS;
 
 const formatter = new Intl.DateTimeFormat('en-US', {
   year: 'numeric',
@@ -25,14 +26,12 @@ const formatter = new Intl.DateTimeFormat('en-US', {
 });
 
 export const getServerSideProps: GetServerSideProps = async (context) => {
+  const ckan = new CKAN(backend_url)
   const { dataset } = context.query;
-  const response = await fetch(
-    `${dms}/api/3/action/package_show?id=${dataset}`
-  );
-  const _dataset = await response.json();
+  const _dataset = await ckan.getDatasetDetails(dataset as string)
   return {
     props: {
-      dataset: _dataset.result,
+      dataset: _dataset,
     },
   };
 };

examples/ckan-ssg/pages/index.tsx

@@ -1,7 +1,8 @@
 import getConfig from 'next/config';
 import styles from './index.module.css';
+import { CKAN } from '@portaljs/ckan';
 
-const dms = getConfig().publicRuntimeConfig.DMS
+const backend_url = getConfig().publicRuntimeConfig.DMS
 
 const formatter = new Intl.DateTimeFormat('en-US', {
   year: 'numeric',
@@ -15,12 +16,11 @@ const formatter = new Intl.DateTimeFormat('en-US', {
 
 
 export async function getServerSideProps() {
-  const response = await fetch(`${dms}/api/3/action/package_search`)
-  const datasets = await response.json()
-  const datasetsWithDetails = await Promise.all(datasets.result.results.map(async (dataset) => {
-    const response = await fetch(`${dms}/api/3/action/package_show?id=` + dataset.name)
-    const json = await response.json()
-    return json.result
+  const ckan = new CKAN(backend_url)
+  const { datasets } = await ckan.packageSearch({ limit: 1000, offset: 0, groups:[], orgs: [], tags: []})
+  const datasetsWithDetails = await Promise.all(datasets.map(async (dataset) => {
+    const _dataset = await ckan.getDatasetDetails(dataset.name)
+    return _dataset
   }))
 
   return {
@@ -79,7 +79,7 @@ export function Index({ datasets }) {
                 </thead>
                 <tbody className="divide-y divide-gray-200">
                   {datasets.map((dataset) => (
-                    <tr>
+                    <tr key={dataset.name}>
                       <td className="px-3 py-4 text-sm text-gray-500">
                           {dataset.title}
                       </td>

examples/ckan/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

examples/ckan/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/ckan/README.md

@@ -0,0 +1,38 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file.
+
+[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
+
+The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

examples/ckan/components/DataRichDocument.tsx

@@ -9,6 +9,7 @@ import { Mermaid } from '@flowershow/core';
 const components = {
   Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
   Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
+  FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
   mermaid: Mermaid,
   Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
   VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),

examples/ckan/content/test.md

@@ -0,0 +1,3 @@
+# Test
+
+Test Data Rich Stories

examples/ckan/next.config.js

@@ -0,0 +1,11 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  reactStrictMode: true,
+  publicRuntimeConfig: {
+    DMS: process.env.DMS
+      ? process.env.DMS.replace(/\/?$/, '')
+      : 'https://demo.dev.datopian.com/',
+  },
+};
+
+module.exports = nextConfig;

examples/ckan/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "ckan",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "prebuild": "npm run mddb",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint",
+    "mddb": "mddb ./content"
+  },
+  "dependencies": {
+    "@flowershow/core": "^0.4.13",
+    "@flowershow/markdowndb": "^0.1.5",
+    "@flowershow/remark-callouts": "^1.0.0",
+    "@flowershow/remark-embed": "^1.0.0",
+    "@githubocto/flat-ui": "^0.14.1",
+    "@heroicons/react": "^2.0.18",
+    "@portaljs/ckan": "^0.0.2",
+    "@portaljs/components": "0.1.6",
+    "@tailwindcss/typography": "^0.5.9",
+    "@types/node": "20.2.3",
+    "@types/react": "18.2.6",
+    "@types/react-dom": "18.2.4",
+    "autoprefixer": "10.4.14",
+    "eslint": "8.41.0",
+    "eslint-config-next": "13.4.3",
+    "isomorphic-unfetch": "^4.0.2",
+    "next": "13.4.3",
+    "next-mdx-remote": "^4.4.1",
+    "papaparse": "^5.4.1",
+    "postcss": "8.4.23",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-query": "^3.39.3",
+    "rehype-autolink-headings": "^6.1.1",
+    "rehype-katex": "^6.0.3",
+    "rehype-prism-plus": "^1.5.1",
+    "rehype-slug": "^5.1.0",
+    "remark-math": "^5.1.1",
+    "remark-smartypants": "^2.0.0",
+    "remark-toc": "^8.0.1",
+    "tailwindcss": "3.3.2",
+    "typescript": "5.0.4"
+  }
+}

examples/ckan/pages/[org]/[dataset]/index.tsx

@@ -0,0 +1,179 @@
+import Head from "next/head";
+import { CKAN, Dataset } from "@portaljs/ckan";
+import {
+  ChevronRightIcon,
+  HomeIcon,
+  PaperClipIcon,
+} from "@heroicons/react/20/solid";
+import Link from "next/link";
+import getConfig from "next/config";
+
+const backend_url = getConfig().publicRuntimeConfig.DMS
+
+export const getServerSideProps = async (context: any) => {
+  try {
+    const datasetName = context.params?.dataset;
+    if (!datasetName) {
+      return {
+        notFound: true,
+      };
+    }
+    const ckan = new CKAN(backend_url);
+    const dataset = await ckan.getDatasetDetails(datasetName as string);
+    if (!dataset) {
+      return {
+        notFound: true,
+      };
+    }
+    return {
+      props: { dataset },
+    };
+  } catch {
+    return {
+      notFound: true,
+    };
+  }
+};
+
+export default function DatasetPage({
+  dataset,
+}: {
+  dataset: Dataset;
+}): JSX.Element {
+  return (
+    <>
+      <Head>
+        <title>{`${dataset.title || dataset.name} - Dataset`}</title>
+        <meta name="description" content="Generated by create next app" />
+        <link rel="icon" href="/favicon.ico" />
+      </Head>
+      <main className="flex min-h-screen flex-col items-center justify-between p-24 bg-zinc-900">
+        <div className="bg-white p-8 my-4 rounded-lg">
+          <nav className="flex px-4 py-8" aria-label="Breadcrumb">
+            <ol role="list" className="flex items-center space-x-4">
+              <li>
+                <div>
+                  <Link href="/" className="text-gray-400 hover:text-gray-500">
+                    <HomeIcon
+                      className="h-5 w-5 flex-shrink-0"
+                      aria-hidden="true"
+                    />
+                    <span className="sr-only">Home</span>
+                  </Link>
+                </div>
+              </li>
+              <li>
+                <div className="flex items-center">
+                  <ChevronRightIcon
+                    className="h-5 w-5 flex-shrink-0 text-gray-400"
+                    aria-hidden="true"
+                  />
+                  <span
+                    className="ml-4 text-sm font-medium text-gray-500 hover:text-gray-700"
+                    aria-current={"page"}
+                  >
+                    {dataset.name}
+                  </span>
+                </div>
+              </li>
+            </ol>
+          </nav>
+          {dataset && (
+            <div>
+              <div className="px-4 sm:px-0">
+                <h3 className="text-base font-semibold leading-7 text-gray-900">
+                  {dataset.title || dataset.name}
+                </h3>
+                <p className="mt-1 max-w-2xl text-sm leading-6 text-gray-500">
+                  Dataset details
+                </p>
+              </div>
+              <div className="mt-6 border-t border-gray-100">
+                <dl className="divide-y divide-gray-100">
+                  <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                    <dt className="text-sm font-medium leading-6 text-gray-900">
+                      Title
+                    </dt>
+                    <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                      {dataset.title}
+                    </dd>
+                  </div>
+                  {dataset.tags && dataset.tags.length > 0 && (
+                    <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                      <dt className="text-sm font-medium leading-6 text-gray-900">
+                        Tags
+                      </dt>
+                      <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                        {dataset.tags.map((tag) => tag.display_name).join(", ")}
+                      </dd>
+                    </div>
+                  )}
+                  {dataset.tags && dataset.tags.length > 0 && (
+                    <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                      <dt className="text-sm font-medium leading-6 text-gray-900">
+                        URL
+                      </dt>
+                      <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                        {dataset.url}
+                      </dd>
+                    </div>
+                  )}
+                  <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                    {dataset.notes && (
+                      <>
+                        <dt className="text-sm font-medium leading-6 text-gray-900">
+                          Description
+                        </dt>
+                        <dd className="mt-1 text-sm leading-6 text-gray-700 sm:col-span-2 sm:mt-0">
+                          {dataset.notes}
+                        </dd>
+                      </>
+                    )}
+                  </div>
+                  <div className="px-4 py-6 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0">
+                    <dt className="text-sm font-medium leading-6 text-gray-900">
+                      Files
+                    </dt>
+                    <dd className="mt-2 text-sm text-gray-900 sm:col-span-2 sm:mt-0">
+                      <ul
+                        role="list"
+                        className="divide-y divide-gray-100 rounded-md border border-gray-200"
+                      >
+                        {dataset.resources.map((resource) => (
+                          <li key={resource.id} className="flex items-center justify-between py-4 pl-4 pr-5 text-sm leading-6">
+                            <div className="flex w-0 flex-1 items-center">
+                              <PaperClipIcon
+                                className="h-5 w-5 flex-shrink-0 text-gray-400"
+                                aria-hidden="true"
+                              />
+                              <div className="ml-4 flex min-w-0 flex-1 gap-2">
+                                <span className="truncate font-medium">
+                                  {resource.name || resource.id}
+                                </span>
+                                <span className="flex-shrink-0 text-gray-400">
+                                  {resource.size}
+                                </span>
+                              </div>
+                            </div>
+                            <div className="ml-4 flex-shrink-0">
+                              <a
+                                href={resource.url}
+                                className="font-medium hover:text-indigo-500 mr-4"
+                              >
+                                Download
+                              </a>
+                            </div>
+                          </li>
+                        ))}
+                      </ul>
+                    </dd>
+                  </div>
+                </dl>
+              </div>
+            </div>
+          )}
+        </div>
+      </main>
+    </>
+  );
+}

examples/ckan/pages/_app.tsx

@@ -0,0 +1,7 @@
+import '@/styles/globals.css'
+import '@portaljs/ckan/styles.css'
+import type { AppProps } from 'next/app'
+
+export default function App({ Component, pageProps }: AppProps) {
+  return <Component {...pageProps} />
+}

examples/ckan/pages/_document.tsx

@@ -0,0 +1,13 @@
+import { Html, Head, Main, NextScript } from 'next/document'
+
+export default function Document() {
+  return (
+    <Html lang="en">
+      <Head />
+      <body>
+        <Main />
+        <NextScript />
+      </body>
+    </Html>
+  )
+}

examples/ckan/pages/api/cors.ts

@@ -0,0 +1,18 @@
+import fetch from 'isomorphic-unfetch';
+
+const Cors = async (req: any, res: any) => {
+  const { url } = req.query;
+  try {
+    const resProxy = await fetch(url, {
+      headers: {
+        Range: 'bytes=0-5132288',
+      },
+    });
+    const data = await resProxy.text();
+    return res.status(200).send(data);
+  } catch (error: any) {
+    res.status(400).send(error.toString());
+  }
+};
+
+export default Cors;

examples/ckan/pages/index.tsx

@@ -0,0 +1,55 @@
+import {
+  CKAN,
+  DatasetSearchForm,
+  ListOfDatasets,
+  PackageSearchOptions,
+  Organization,
+  Group,
+} from '@portaljs/ckan';
+import getConfig from 'next/config';
+import { useState } from 'react';
+
+const backend_url = getConfig().publicRuntimeConfig.DMS;
+
+export async function getServerSideProps() {
+  const ckan = new CKAN(backend_url);
+  const groups = await ckan.getGroupsWithDetails();
+  const orgs = await ckan.getOrgsWithDetails();
+  return {
+    props: {
+      groups,
+      orgs,
+    },
+  };
+}
+
+export default function Home({
+  orgs,
+  groups,
+}: {
+  orgs: Organization[];
+  groups: Group[];
+}) {
+  const ckan = new CKAN(backend_url);
+  const [options, setOptions] = useState<PackageSearchOptions>({
+    offset: 0,
+    limit: 5,
+    tags: [],
+    groups: [],
+    orgs: [],
+  });
+
+  return (
+    <main className="flex min-h-screen flex-col items-center p-24 bg-zinc-900">
+      <DatasetSearchForm
+        options={options}
+        setOptions={setOptions}
+        groups={groups}
+        orgs={orgs}
+      />
+      <div className="bg-white p-8 my-4 rounded-lg">
+        <ListOfDatasets options={options} setOptions={setOptions} ckan={ckan} />{' '}
+      </div>
+    </main>
+  );
+}

examples/ckan/pages/stories/[[...path]].tsx

@@ -0,0 +1,129 @@
+import { existsSync, promises as fs } from 'fs';
+import path from 'path';
+import parse from '../../lib/markdown';
+
+import DataRichDocument from '../../components/DataRichDocument';
+import clientPromise from '../../lib/mddb';
+import getConfig from 'next/config';
+import { CKAN } from '@portaljs/ckan';
+
+export const getStaticPaths = async () => {
+  const contentDir = path.join(process.cwd(), '/content/');
+  const contentFolders = await fs.readdir(contentDir, 'utf8');
+  const paths = contentFolders.map((folder: string) => ({
+    params: { path: [folder.split('.')[0]] },
+  }));
+  return {
+    paths,
+    fallback: false,
+  };
+};
+
+const backend_url = getConfig().publicRuntimeConfig.DMS;
+
+export const getStaticProps = async (context) => {
+  const mddb = await clientPromise;
+  const storyFile = await mddb.getFileByUrl(context.params.path);
+  const md = await fs.readFile(
+    `${process.cwd()}/${storyFile.file_path}`,
+    'utf8'
+  );
+
+  const ckan = new CKAN(backend_url);
+  const datasets = storyFile.metadata.datasets ? await Promise.all(
+    storyFile.metadata.datasets.map(
+      async (datasetName: string) => await ckan.getDatasetDetails(datasetName)
+    )
+  ) : [];
+  const orgs = storyFile.metadata.orgs ? await Promise.all(
+    storyFile.metadata.orgs.map(
+      async (orgName: string) => await ckan.getOrgDetails(orgName)
+    )
+  ) : [];
+
+  let { mdxSource, frontMatter } = await parse(md, '.mdx', { datasets, orgs });
+
+  return {
+    props: {
+      mdxSource,
+      frontMatter: JSON.stringify(frontMatter),
+    },
+  };
+};
+
+export default function DatasetPage({ mdxSource, frontMatter }) {
+  frontMatter = JSON.parse(frontMatter);
+  return (
+    <main className="flex min-h-screen flex-col justify-between p-16 bg-zinc-900">
+      <div className="bg-white p-8 my-4 rounded-lg">
+        <div className="prose mx-auto py-8">
+          <header>
+            <div className="mb-6">
+              <>
+                <h1 className="mb-2">{frontMatter.title}</h1>
+                {frontMatter.author && (
+                  <p className="my-0">
+                    <span className="font-semibold">Author: </span>
+                    <span className="my-0">{frontMatter.author}</span>
+                  </p>
+                )}
+                {frontMatter.description && (
+                  <p className="my-0">
+                    <span className="font-semibold">Description: </span>
+                    <span className="description my-0">
+                      {frontMatter.description}
+                    </span>
+                  </p>
+                )}
+                {frontMatter.modified && (
+                  <p className="my-0">
+                    <span className="font-semibold">Modified: </span>
+                    <span className="description my-0">
+                      {new Date(frontMatter.modified).toLocaleDateString()}
+                    </span>
+                  </p>
+                )}
+                {frontMatter.files && (
+                  <section className="py-6">
+                    <h2 className="mt-0">Data files</h2>
+                    <table className="table-auto">
+                      <thead>
+                        <tr>
+                          <th>File</th>
+                          <th>Format</th>
+                        </tr>
+                      </thead>
+                      <tbody>
+                        {frontMatter.files.map((f) => {
+                          const fileName = f.split('/').slice(-1);
+                          return (
+                            <tr key={`resources-list-${f}`}>
+                              <td>
+                                <a target="_blank" href={f}>
+                                  {fileName}
+                                </a>
+                              </td>
+                              <td>
+                                {fileName[0]
+                                  .split('.')
+                                  .slice(-1)[0]
+                                  .toUpperCase()}
+                              </td>
+                            </tr>
+                          );
+                        })}
+                      </tbody>
+                    </table>
+                  </section>
+                )}
+              </>
+            </div>
+          </header>
+          <main>
+            <DataRichDocument source={mdxSource} />
+          </main>
+        </div>
+      </div>
+    </main>
+  );
+}

examples/ckan/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

examples/ckan/public/vercel.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

examples/ckan/styles/globals.css

@@ -0,0 +1,71 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+@import "@flowershow/remark-callouts/styles.css";
+
+/* mathjax */
+.math-inline > mjx-container > svg {
+  display: inline;
+  align-items: center;
+}
+
+/* smooth scrolling in modern browsers */
+html {
+  scroll-behavior: smooth !important;
+}
+
+/* tooltip fade-out clip */
+.tooltip-body::after {
+  content: "";
+  position: absolute;
+  right: 0;
+  top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
+  height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
+  width: 10rem;
+  background: linear-gradient(
+    to right,
+    rgba(255, 255, 255, 0),
+    rgba(255, 255, 255, 1) 100%
+  );
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
+  margin-left: -2rem !important;
+  padding-left: 2rem !important;
+  scroll-margin-top: 4.5rem;
+  position: relative;
+}
+
+.heading-link {
+  padding: 1px;
+  position: absolute;
+  left: 0;
+  top: 50%;
+  transform: translateY(-50%);
+  margin: auto 0;
+  border-radius: 5px;
+  background: #1e293b;
+  opacity: 0;
+  transition: opacity 0.2s;
+}
+
+.light .heading-link {
+  /* border: 1px solid #ab2b65; */
+  /* background: none; */
+  background: #e2e8f0;
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
+  opacity: 100;
+}
+
+.heading-link svg {
+  transform: scale(0.75);
+}
+
+@media screen and (max-width: 640px) {
+  .heading-link {
+    visibility: hidden;
+  }
+}

examples/ckan/tailwind.config.js

@@ -0,0 +1,18 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    './pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './components/**/*.{js,ts,jsx,tsx,mdx}',
+    './app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      backgroundImage: {
+        'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
+        'gradient-conic':
+          'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
+      },
+    },
+  },
+  plugins: [require('@tailwindcss/typography')],
+}

examples/ckan/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": false,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+  "exclude": ["node_modules"]
+}

examples/github-backed-catalog/components/DataRichDocument.tsx

@@ -0,0 +1,21 @@
+import { MDXRemote } from 'next-mdx-remote';
+import dynamic from 'next/dynamic';
+import { Mermaid } from '@flowershow/core';
+
+// Custom components/renderers to pass to MDX.
+// Since the MDX files aren't loaded by webpack, they have no knowledge of how
+// to handle import statements. Instead, you must include components in scope
+// here.
+const components = {
+  Table: dynamic(() => import('@portaljs/components').then(mod => mod.Table)),
+  Catalog: dynamic(() => import('@portaljs/components').then(mod => mod.Catalog)),
+  FlatUiTable: dynamic(() => import('@portaljs/components').then(mod => mod.FlatUiTable)),
+  mermaid: Mermaid,
+  Vega: dynamic(() => import('@portaljs/components').then(mod => mod.Vega)),
+  VegaLite: dynamic(() => import('@portaljs/components').then(mod => mod.VegaLite)),
+  LineChart: dynamic(() => import('@portaljs/components').then(mod => mod.LineChart)),
+} as any;
+
+export default function DRD({ source }: { source: any }) {
+  return <MDXRemote {...source} components={components} />;
+}

examples/github-backed-catalog/datasets.json

@@ -15,6 +15,13 @@
     ],
     "readme": "README.md"
   },
+  {
+    "owner": "luccasmmg",
+    "branch": "main",
+    "repo": "test-data-repo-1",
+    "files": ["data_1.csv", "data_2.csv"],
+    "readme": "README.md"
+  },
   {
     "owner": "datasets",
     "branch": "main",

examples/github-backed-catalog/lib/markdown.js

@@ -0,0 +1,105 @@
+import matter from "gray-matter";
+import mdxmermaid from "mdx-mermaid";
+import { h } from "hastscript";
+import remarkCallouts from "@flowershow/remark-callouts";
+import remarkEmbed from "@flowershow/remark-embed";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import remarkSmartypants from "remark-smartypants";
+import remarkToc from "remark-toc";
+import remarkWikiLink from "@flowershow/remark-wiki-link";
+import rehypeAutolinkHeadings from "rehype-autolink-headings";
+import rehypeKatex from "rehype-katex";
+import rehypeSlug from "rehype-slug";
+import rehypePrismPlus from "rehype-prism-plus";
+
+import { serialize } from "next-mdx-remote/serialize";
+
+/**
+ * Parse a markdown or MDX file to an MDX source form + front matter data
+ *
+ * @source: the contents of a markdown or mdx file
+ * @format: used to indicate to next-mdx-remote which format to use (md or mdx)
+ * @returns: { mdxSource: mdxSource, frontMatter: ...}
+ */
+const parse = async function (source, format, scope) {
+  const { content, data, excerpt } = matter(source, {
+    excerpt: (file, options) => {
+      // Generate an excerpt for the file
+      file.excerpt = file.content.split("\n\n")[0];
+    },
+  });
+
+  const mdxSource = await serialize(
+    { value: content, path: format },
+    {
+      // Optionally pass remark/rehype plugins
+      mdxOptions: {
+        remarkPlugins: [
+          remarkEmbed,
+          remarkGfm,
+          [remarkSmartypants, { quotes: false, dashes: "oldschool" }],
+          remarkMath,
+          remarkCallouts,
+          remarkWikiLink,
+          [
+            remarkToc,
+            {
+              heading: "Table of contents",
+              tight: true,
+            },
+          ],
+          [mdxmermaid, {}],
+        ],
+        rehypePlugins: [
+          rehypeSlug,
+          [
+            rehypeAutolinkHeadings,
+            {
+              properties: { className: 'heading-link' },
+              test(element) {
+                return (
+                  ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
+                  element.properties?.id !== "table-of-contents" &&
+                  element.properties?.className !== "blockquote-heading"
+                );
+              },
+              content() {
+                return [
+                  h(
+                    "svg",
+                    {
+                      xmlns: "http:www.w3.org/2000/svg",
+                      fill: "#ab2b65",
+                      viewBox: "0 0 20 20",
+                      className: "w-5 h-5",
+                    },
+                    [
+                      h("path", {
+                        fillRule: "evenodd",
+                        clipRule: "evenodd",
+                        d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
+                      }),
+                    ]
+                  ),
+                ];
+              },
+            },
+          ],
+          [rehypeKatex, { output: "mathml" }],
+          [rehypePrismPlus, { ignoreMissing: true }],
+        ],
+        format,
+      },
+      scope,
+    }
+  );
+
+  return {
+    mdxSource: mdxSource,
+    frontMatter: data,
+    excerpt,
+  };
+};
+
+export default parse;

examples/github-backed-catalog/lib/mddb.ts

@@ -0,0 +1,14 @@
+import { MarkdownDB } from "@flowershow/markdowndb";
+
+const dbPath = "markdown.db";
+
+const client = new MarkdownDB({
+  client: "sqlite3",
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const clientPromise = client.init();
+
+export default clientPromise;

examples/github-backed-catalog/lib/octokit.ts

@@ -39,6 +39,32 @@ export async function getProjectReadme(
   }
 }
 
+export async function getProjectDatapackage(
+  owner: string,
+  repo: string,
+  branch: string,
+  github_pat?: string
+) {
+  const octokit = new Octokit({ auth: github_pat });
+  try {
+    const response = await octokit.rest.repos.getContent({
+      owner,
+      repo,
+      path: "datapackage.json",
+      ref: branch,
+    });
+    const data = response.data as { content?: string };
+    const fileContent = data.content ? data.content : "";
+    if (fileContent === "") {
+      return null;
+    }
+    const decodedContent = Buffer.from(fileContent, "base64").toString();
+    return JSON.parse(decodedContent);
+  } catch (error) {
+    return null
+  }
+}
+
 export async function getLastUpdated(
   owner: string,
   repo: string,
@@ -162,11 +188,20 @@ export async function getProject(project: GithubProject, github_pat?: string) {
     projectBase,
     github_pat
   );
+
+  const projectDatapackage = await getProjectDatapackage(
+    project.owner,
+    project.repo,
+    project.branch,
+    github_pat
+  );
+
   return {
     ...projectMetadata,
     files: projectData,
     readmeContent: projectReadme,
     last_updated,
     base_path: projectBase,
+    datapackage: projectDatapackage
   };
 }

examples/github-backed-catalog/package.json

@@ -10,19 +10,32 @@
     "prettier": "prettier --write ."
   },
   "dependencies": {
+    "@flowershow/core": "^0.4.13",
+    "@flowershow/markdowndb": "^0.1.5",
+    "@flowershow/remark-callouts": "^1.0.0",
+    "@flowershow/remark-embed": "^1.0.0",
+    "@portaljs/components": "^0.1.6",
     "@types/node": "18.16.0",
     "@types/react": "18.0.38",
     "@types/react-dom": "18.0.11",
     "eslint": "8.39.0",
     "eslint-config-next": "13.3.1",
-    "next": "13.3.1",
+    "next": "13.4.3",
+    "next-mdx-remote": "^4.4.1",
     "next-seo": "^6.0.0",
     "octokit": "^2.0.14",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-markdown": "^8.0.7",
     "react-timeago": "^7.1.0",
+    "rehype-autolink-headings": "^6.1.1",
+    "rehype-katex": "^6.0.3",
+    "rehype-prism-plus": "^1.5.1",
+    "rehype-slug": "^5.1.0",
     "remark-gfm": "^3.0.1",
+    "remark-math": "^5.1.1",
+    "remark-smartypants": "^2.0.0",
+    "remark-toc": "^8.0.1",
     "typescript": "5.0.4"
   },
   "devDependencies": {

examples/github-backed-catalog/pages/@org/[org]/[...path].tsx

@@ -6,6 +6,8 @@ import { getProject, GithubProject } from "../../../lib/octokit";
 import ReactMarkdown from "react-markdown";
 import remarkGfm from "remark-gfm";
 import Breadcrumbs from "../../../components/_shared/Breadcrumbs";
+import parse from '../../../lib/markdown';
+import DataRichDocument from '../../../components/DataRichDocument'
 
 export default function ProjectPage({ project }) {
   const repoId = `@${project.repo_config.owner}/${project.repo_config.repo}`;
@@ -64,9 +66,7 @@ export default function ProjectPage({ project }) {
         <hr />
 
         <h2 className="uppercase font-black">Readme</h2>
-        <ReactMarkdown remarkPlugins={[remarkGfm]}>
-          {project.readmeContent}
-        </ReactMarkdown>
+        <DataRichDocument source={project.mdxSource} />
       </main>
     </>
   );
@@ -119,9 +119,10 @@ export async function getStaticProps({ params }) {
   });
   const github_pat = getConfig().serverRuntimeConfig.github_pat;
   const project = await getProject(repo, github_pat);
+  let { mdxSource, frontMatter } = await parse(project.readmeContent, '.mdx', { project });
   return {
     props: {
-      project: { ...project, repo_config: repo },
+      project: { ...project, repo_config: repo, mdxSource },
     },
   };
 }

examples/github-backed-catalog/pages/styles.css

@@ -78,3 +78,72 @@ pre {
   color: rgba(55, 65, 81, 1);
   width: 100%;
 }
+
+@import "@flowershow/remark-callouts/styles.css";
+
+/* mathjax */
+.math-inline > mjx-container > svg {
+  display: inline;
+  align-items: center;
+}
+
+/* smooth scrolling in modern browsers */
+html {
+  scroll-behavior: smooth !important;
+}
+
+/* tooltip fade-out clip */
+.tooltip-body::after {
+  content: "";
+  position: absolute;
+  right: 0;
+  top: 3.6rem; /* multiple of $line-height used on the tooltip body (defined in tooltipBodyStyle) */
+  height: 1.2rem; /* ($top + $height)/$line-height is the number of lines we want to clip tooltip text at*/
+  width: 10rem;
+  background: linear-gradient(
+    to right,
+    rgba(255, 255, 255, 0),
+    rgba(255, 255, 255, 1) 100%
+  );
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title) {
+  margin-left: -2rem !important;
+  padding-left: 2rem !important;
+  scroll-margin-top: 4.5rem;
+  position: relative;
+}
+
+.heading-link {
+  padding: 1px;
+  position: absolute;
+  left: 0;
+  top: 50%;
+  transform: translateY(-50%);
+  margin: auto 0;
+  border-radius: 5px;
+  background: #1e293b;
+  opacity: 0;
+  transition: opacity 0.2s;
+}
+
+.light .heading-link {
+  /* border: 1px solid #ab2b65; */
+  /* background: none; */
+  background: #e2e8f0;
+}
+
+:is(h2, h3, h4, h5, h6):not(.blogitem-title):hover .heading-link {
+  opacity: 100;
+}
+
+.heading-link svg {
+  transform: scale(0.75);
+}
+
+@media screen and (max-width: 640px) {
+  .heading-link {
+    visibility: hidden;
+  }
+}
+

examples/learn/components/DataRichDocument.tsx

@@ -0,0 +1,21 @@
+import { MDXRemote } from 'next-mdx-remote';
+import dynamic from 'next/dynamic';
+import { Mermaid } from '@flowershow/core';
+
+// Custom components/renderers to pass to MDX.
+// Since the MDX files aren't loaded by webpack, they have no knowledge of how
+// to handle import statements. Instead, you must include components in scope
+// here.
+const components = {
+  Table: dynamic(() => import('../portaljs/components/Table').then(mod => mod.Table)),
+  Catalog: dynamic(() => import('../portaljs/components/Catalog').then(mod => mod.Catalog)),
+  mermaid: Mermaid,
+  Vega: dynamic(() => import('../portaljs/components/Vega').then(mod => mod.Vega)),
+  VegaLite: dynamic(() => import('../portaljs/components/VegaLite').then(mod => mod.VegaLite)),
+  LineChart: dynamic(() => import('../portaljs/components/LineChart').then(mod => mod.LineChart)),
+  FlatUiTable: dynamic(() => import('../portaljs/components/FlatUiTable').then(mod => mod.FlatUiTable)),
+} as any;
+
+export default function DRD({ source }: { source: any }) {
+  return <MDXRemote {...source} components={components} />;
+}

examples/learn/lib/markdown.js

@@ -0,0 +1,105 @@
+import matter from "gray-matter";
+import mdxmermaid from "mdx-mermaid";
+import { h } from "hastscript";
+import remarkCallouts from "@flowershow/remark-callouts";
+import remarkEmbed from "@flowershow/remark-embed";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import remarkSmartypants from "remark-smartypants";
+import remarkToc from "remark-toc";
+import remarkWikiLink from "@flowershow/remark-wiki-link";
+import rehypeAutolinkHeadings from "rehype-autolink-headings";
+import rehypeKatex from "rehype-katex";
+import rehypeSlug from "rehype-slug";
+import rehypePrismPlus from "rehype-prism-plus";
+
+import { serialize } from "next-mdx-remote/serialize";
+
+/**
+ * Parse a markdown or MDX file to an MDX source form + front matter data
+ *
+ * @source: the contents of a markdown or mdx file
+ * @format: used to indicate to next-mdx-remote which format to use (md or mdx)
+ * @returns: { mdxSource: mdxSource, frontMatter: ...}
+ */
+const parse = async function (source, format, scope) {
+  const { content, data, excerpt } = matter(source, {
+    excerpt: (file, options) => {
+      // Generate an excerpt for the file
+      file.excerpt = file.content.split("\n\n")[0];
+    },
+  });
+
+  const mdxSource = await serialize(
+    { value: content, path: format },
+    {
+      // Optionally pass remark/rehype plugins
+      mdxOptions: {
+        remarkPlugins: [
+          remarkEmbed,
+          remarkGfm,
+          [remarkSmartypants, { quotes: false, dashes: "oldschool" }],
+          remarkMath,
+          remarkCallouts,
+          remarkWikiLink,
+          [
+            remarkToc,
+            {
+              heading: "Table of contents",
+              tight: true,
+            },
+          ],
+          [mdxmermaid, {}],
+        ],
+        rehypePlugins: [
+          rehypeSlug,
+          [
+            rehypeAutolinkHeadings,
+            {
+              properties: { className: 'heading-link' },
+              test(element) {
+                return (
+                  ["h2", "h3", "h4", "h5", "h6"].includes(element.tagName) &&
+                  element.properties?.id !== "table-of-contents" &&
+                  element.properties?.className !== "blockquote-heading"
+                );
+              },
+              content() {
+                return [
+                  h(
+                    "svg",
+                    {
+                      xmlns: "http:www.w3.org/2000/svg",
+                      fill: "#ab2b65",
+                      viewBox: "0 0 20 20",
+                      className: "w-5 h-5",
+                    },
+                    [
+                      h("path", {
+                        fillRule: "evenodd",
+                        clipRule: "evenodd",
+                        d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z",
+                      }),
+                    ]
+                  ),
+                ];
+              },
+            },
+          ],
+          [rehypeKatex, { output: "mathml" }],
+          [rehypePrismPlus, { ignoreMissing: true }],
+        ],
+        format,
+      },
+      scope,
+    }
+  );
+
+  return {
+    mdxSource: mdxSource,
+    frontMatter: data,
+    excerpt,
+  };
+};
+
+export default parse;

examples/learn/lib/mddb.ts

@@ -0,0 +1,14 @@
+import { MarkdownDB } from "@flowershow/markdowndb";
+
+const dbPath = "markdown.db";
+
+const client = new MarkdownDB({
+  client: "sqlite3",
+  connection: {
+    filename: dbPath,
+  },
+});
+
+const clientPromise = client.init();
+
+export default clientPromise;

examples/learn/package.json

@@ -17,21 +17,12 @@
     "@flowershow/remark-callouts": "^1.0.0",
     "@flowershow/remark-embed": "^1.0.0",
     "@flowershow/remark-wiki-link": "^1.1.2",
-    "@heroicons/react": "^2.0.17",
     "@opentelemetry/api": "^1.4.0",
     "@portaljs/components": "^0.1.0",
-    "@tanstack/react-table": "^8.8.5",
-    "flexsearch": "0.7.21",
     "gray-matter": "^4.0.3",
     "hastscript": "^7.2.0",
     "mdx-mermaid": "2.0.0-rc7",
     "next": "13.2.1",
-    "next-mdx-remote": "^4.4.1",
-    "papaparse": "^5.4.1",
-    "react": "18.2.0",
-    "react-dom": "18.2.0",
-    "react-hook-form": "^7.43.9",
-    "react-vega": "^7.6.0",
     "rehype-autolink-headings": "^6.1.1",
     "rehype-katex": "^6.0.3",
     "rehype-prism-plus": "^1.5.1",
@@ -40,7 +31,20 @@
     "remark-math": "^5.1.1",
     "remark-smartypants": "^2.0.0",
     "remark-toc": "^8.0.1",
-    "typescript": "5.0.4"
+    "typescript": "5.0.4",
+    "@githubocto/flat-ui": "^0.14.1",
+    "@heroicons/react": "^2.0.17",
+    "@tanstack/react-table": "^8.8.5",
+    "flexsearch": "0.7.21",
+    "next-mdx-remote": "^4.4.1",
+    "papaparse": "^5.4.1",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-hook-form": "^7.43.9",
+    "react-query": "^3.39.3",
+    "react-vega": "^7.6.0",
+    "vega": "5.25.0",
+    "vega-lite": "5.1.0"
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",

examples/learn/portaljs/components/Catalog.tsx

@@ -0,0 +1,119 @@
+import { Index } from 'flexsearch';
+import { useState } from 'react';
+import DebouncedInput from './DebouncedInput';
+import { useForm } from 'react-hook-form';
+
+export function Catalog({
+  datasets,
+  facets,
+}: {
+  datasets: any[];
+  facets: string[];
+}) {
+  const [indexFilter, setIndexFilter] = useState('');
+  const index = new Index({ tokenize: 'full' });
+  datasets.forEach((dataset) =>
+    index.add(
+      dataset._id,
+      //This will join every metadata value + the url_path into one big string and index that
+      Object.entries(dataset.metadata).reduce(
+        (acc, curr) => acc + ' ' + curr[1].toString(),
+        ''
+      ) +
+        ' ' +
+        dataset.url_path
+    )
+  );
+
+  const facetValues = facets
+    ? facets.reduce((acc, facet) => {
+        const possibleValues = datasets.reduce((acc, curr) => {
+          const facetValue = curr.metadata[facet];
+          if (facetValue) {
+            return Array.isArray(facetValue)
+              ? acc.concat(facetValue)
+              : acc.concat([facetValue]);
+          }
+          return acc;
+        }, []);
+        acc[facet] = {
+          possibleValues: [...new Set(possibleValues)],
+          selectedValue: null,
+        };
+        return acc;
+      }, {})
+    : [];
+
+  const { register, watch } = useForm(facetValues);
+
+  const filteredDatasets = datasets
+    // First filter by flex search
+    .filter((dataset) =>
+      indexFilter !== ''
+        ? index.search(indexFilter).includes(dataset._id)
+        : true
+    )
+    //Then check if the selectedValue for the given facet is included in the dataset metadata
+    .filter((dataset) => {
+      //Avoids a server rendering breakage
+      if (!watch() || Object.keys(watch()).length === 0) return true
+      //This will filter only the key pairs of the metadata values that were selected as facets
+      const datasetFacets = Object.entries(dataset.metadata).filter((entry) =>
+        facets.includes(entry[0])
+      );
+      //Check if the value present is included in the selected value in the form
+      return datasetFacets.every((elem) =>
+        watch()[elem[0]].selectedValue
+          ? (elem[1] as string | string[]).includes(
+              watch()[elem[0]].selectedValue
+            )
+          : true
+      );
+    });
+
+  return (
+    <>
+      <DebouncedInput
+        value={indexFilter ?? ''}
+        onChange={(value) => setIndexFilter(String(value))}
+        className="p-2 text-sm shadow border border-block mr-1"
+        placeholder="Search all datasets..."
+      />
+      {Object.entries(facetValues).map((elem) => (
+        <select
+          key={elem[0]}
+          defaultValue=""
+          className="p-2 ml-1 text-sm shadow border border-block"
+          {...register(elem[0] + '.selectedValue')}
+        >
+          <option value="">
+            Filter by {elem[0]}
+          </option>
+          {(elem[1] as { possibleValues: string[] }).possibleValues.map(
+            (val) => (
+              <option
+                key={val}
+                className="dark:bg-white dark:text-black"
+                value={val}
+              >
+                {val}
+              </option>
+            )
+          )}
+        </select>
+      ))}
+      <ul className='mb-5 pl-6 mt-5 list-disc'>
+        {filteredDatasets.map((dataset) => (
+          <li className='py-2' key={dataset._id}>
+            <a className='font-medium underline' href={dataset.url_path}>
+              {dataset.metadata.title
+                ? dataset.metadata.title
+                : dataset.url_path}
+            </a>
+          </li>
+        ))}
+      </ul>
+    </>
+  );
+}
+

examples/learn/portaljs/components/DebouncedInput.tsx

@@ -0,0 +1,32 @@
+import { useEffect, useState } from "react";
+
+const DebouncedInput = ({
+  value: initialValue,
+  onChange,
+  debounce = 500,
+  ...props
+}) => {
+  const [value, setValue] = useState(initialValue);
+
+  useEffect(() => {
+    setValue(initialValue);
+  }, [initialValue]);
+
+  useEffect(() => {
+    const timeout = setTimeout(() => {
+      onChange(value);
+    }, debounce);
+
+    return () => clearTimeout(timeout);
+  }, [value]);
+
+  return (
+    <input
+      {...props}
+      value={value}
+      onChange={(e) => setValue(e.target.value)}
+    />
+  );
+};
+
+export default DebouncedInput;

examples/learn/portaljs/components/FlatUiTable.tsx

@@ -0,0 +1,113 @@
+import { QueryClient, QueryClientProvider, useQuery } from 'react-query';
+import Papa from 'papaparse';
+import { Grid } from '@githubocto/flat-ui';
+
+const queryClient = new QueryClient();
+
+export async function getCsv(url: string, corsProxy?: string) {
+  if (corsProxy) {
+    url = corsProxy + url
+  }
+  const response = await fetch(url, {
+    headers: {
+      Range: 'bytes=0-5132288',
+    },
+  });
+  const data = await response.text();
+  return data;
+}
+
+export async function parseCsv(file: string): Promise<any> {
+  return new Promise((resolve, reject) => {
+    Papa.parse(file, {
+      header: true,
+      dynamicTyping: true,
+      skipEmptyLines: true,
+      transform: (value: string): string => {
+        return value.trim();
+      },
+      complete: (results: any) => {
+        return resolve(results);
+      },
+      error: (error: any) => {
+        return reject(error);
+      },
+    });
+  });
+}
+
+const Spinning = () => {
+  return (
+    <div role="status w-fit mx-auto">
+      <svg
+        aria-hidden="true"
+        className="w-8 h-8 mr-2 text-gray-200 animate-spin dark:text-gray-600 fill-emerald-600"
+        viewBox="0 0 100 101"
+        fill="none"
+        xmlns="http://www.w3.org/2000/svg"
+      >
+        <path
+          d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
+          fill="currentColor"
+        />
+        <path
+          d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
+          fill="currentFill"
+        />
+      </svg>
+      <span className="sr-only">Loading...</span>
+    </div>
+  );
+};
+
+export interface FlatUiTableProps {
+  url?: string;
+  data?: { [key: string]: number | string }[];
+  rawCsv?: string;
+  corsProxy?: string;
+}
+export const FlatUiTable: React.FC<FlatUiTableProps> = ({
+  url,
+  data,
+  rawCsv,
+  corsProxy,
+}) => {
+  return (
+    // Provide the client to your App
+    <QueryClientProvider client={queryClient}>
+      <TableInner corsProxy={corsProxy} url={url} data={data} rawCsv={rawCsv} />
+    </QueryClientProvider>
+  );
+};
+
+const TableInner: React.FC<FlatUiTableProps> = ({ url, data, rawCsv, corsProxy }) => {
+  if (data) {
+    return (
+      <div className="w-full" style={{height: '500px'}}>
+        <Grid data={data} />
+      </div>
+    );
+  }
+  const { data: csvString, isLoading: isDownloadingCSV } = useQuery(
+    ['dataCsv', url],
+    () => getCsv(url as string, corsProxy),
+    { enabled: !!url }
+  );
+  const { data: parsedData, isLoading: isParsing } = useQuery(
+    ['dataPreview', csvString],
+    () => parseCsv(rawCsv ? rawCsv as string : csvString as string),
+    { enabled: rawCsv ? true : !!csvString }
+  );
+  if (isParsing || isDownloadingCSV)
+    <div className="w-full">
+      <Spinning />
+    </div>;
+  if (parsedData)
+    return (
+      <div className="w-full" style={{height: '500px'}}>
+        <Grid data={parsedData.data} />
+      </div>
+    );
+  return <Spinning />
+};
+

examples/learn/portaljs/components/LineChart.tsx

@@ -0,0 +1,63 @@
+import { VegaLite } from './VegaLite';
+
+export type LineChartProps = {
+  data: Array<Array<string | number>> | string | { x: string; y: number }[];
+  title?: string;
+  xAxis?: string;
+  yAxis?: string;
+  fullWidth?: boolean;
+};
+
+export function LineChart({
+  data = [],
+  fullWidth = false,
+  title = '',
+  xAxis = 'x',
+  yAxis = 'y',
+}: LineChartProps) {
+  var tmp = data;
+  if (Array.isArray(data)) {
+    tmp = data.map((r) => {
+      return { x: r[0], y: r[1] };
+    });
+  }
+  const vegaData = { table: tmp };
+  const spec = {
+    $schema: 'https://vega.github.io/schema/vega-lite/v5.json',
+    title,
+    width: 600,
+    height: 300,
+    mark: {
+      type: 'line',
+      color: 'black',
+      strokeWidth: 1,
+      tooltip: true,
+    },
+    data: {
+      name: 'table',
+    },
+    selection: {
+      grid: {
+        type: 'interval',
+        bind: 'scales',
+      },
+    },
+    encoding: {
+      x: {
+        field: xAxis,
+        timeUnit: 'year',
+        type: 'temporal',
+      },
+      y: {
+        field: yAxis,
+        type: 'quantitative',
+      },
+    },
+  };
+  if (typeof data === 'string') {
+    spec.data = { url: data } as any;
+    return <VegaLite fullWidth={fullWidth} spec={spec} />;
+  }
+
+  return <VegaLite fullWidth={fullWidth} data={vegaData} spec={spec} />;
+}

examples/learn/portaljs/components/Table.tsx

@@ -0,0 +1,195 @@
+import {
+  createColumnHelper,
+  FilterFn,
+  flexRender,
+  getCoreRowModel,
+  getFilteredRowModel,
+  getPaginationRowModel,
+  getSortedRowModel,
+  useReactTable,
+} from '@tanstack/react-table';
+
+import {
+  ArrowDownIcon,
+  ArrowUpIcon,
+  ChevronDoubleLeftIcon,
+  ChevronDoubleRightIcon,
+  ChevronLeftIcon,
+  ChevronRightIcon,
+} from '@heroicons/react/24/solid';
+
+import React, { useEffect, useMemo, useState } from 'react';
+
+import parseCsv from '../lib/parseCsv';
+import DebouncedInput from './DebouncedInput';
+import loadData from '../lib/loadData';
+
+export type TableProps = {
+  data?: Array<{ [key: string]: number | string }>;
+  cols?: Array<{ [key: string]: string }>;
+  csv?: string;
+  url?: string;
+  fullWidth?: boolean;
+};
+
+export const Table = ({
+  data: ogData = [],
+  cols: ogCols = [],
+  csv = '',
+  url = '',
+  fullWidth = false,
+}: TableProps) => {
+  if (csv) {
+    const out = parseCsv(csv);
+    ogData = out.rows;
+    ogCols = out.fields;
+  }
+
+  const [data, setData] = React.useState(ogData);
+  const [cols, setCols] = React.useState(ogCols);
+  // const [error, setError] = React.useState(""); //  TODO: add error handling
+
+  const tableCols = useMemo(() => {
+    const columnHelper = createColumnHelper();
+    return cols.map((c) =>
+      columnHelper.accessor<any, string>(c.key, {
+        header: () => c.name,
+        cell: (info) => info.getValue(),
+      })
+    );
+  }, [data, cols]);
+
+  const [globalFilter, setGlobalFilter] = useState('');
+
+  const table = useReactTable({
+    data,
+    columns: tableCols,
+    getCoreRowModel: getCoreRowModel(),
+    state: {
+      globalFilter,
+    },
+    globalFilterFn: globalFilterFn,
+    onGlobalFilterChange: setGlobalFilter,
+    getFilteredRowModel: getFilteredRowModel(),
+    getPaginationRowModel: getPaginationRowModel(),
+    getSortedRowModel: getSortedRowModel(),
+  });
+
+  useEffect(() => {
+    if (url) {
+      loadData(url).then((data) => {
+        const { rows, fields } = parseCsv(data);
+        setData(rows);
+        setCols(fields);
+      });
+    }
+  }, [url]);
+
+  return (
+    <div className={`${fullWidth ? 'w-[90vw] ml-[calc(50%-45vw)]' : 'w-full'}`}>
+      <DebouncedInput
+        value={globalFilter ?? ''}
+        onChange={(value: any) => setGlobalFilter(String(value))}
+        className="p-2 text-sm shadow border border-block"
+        placeholder="Search all columns..."
+      />
+      <table className="w-full mt-10">
+        <thead className="text-left border-b border-b-slate-300">
+          {table.getHeaderGroups().map((hg) => (
+            <tr key={hg.id}>
+              {hg.headers.map((h) => (
+                <th key={h.id} className="pr-2 pb-2">
+                  <div
+                    {...{
+                      className: h.column.getCanSort()
+                        ? 'cursor-pointer select-none'
+                        : '',
+                      onClick: h.column.getToggleSortingHandler(),
+                    }}
+                  >
+                    {flexRender(h.column.columnDef.header, h.getContext())}
+                    {{
+                      asc: (
+                        <ArrowUpIcon className="inline-block ml-2 h-4 w-4" />
+                      ),
+                      desc: (
+                        <ArrowDownIcon className="inline-block ml-2 h-4 w-4" />
+                      ),
+                    }[h.column.getIsSorted() as string] ?? (
+                      <div className="inline-block ml-2 h-4 w-4" />
+                    )}
+                  </div>
+                </th>
+              ))}
+            </tr>
+          ))}
+        </thead>
+        <tbody>
+          {table.getRowModel().rows.map((r) => (
+            <tr key={r.id} className="border-b border-b-slate-200">
+              {r.getVisibleCells().map((c) => (
+                <td key={c.id} className="py-2">
+                  {flexRender(c.column.columnDef.cell, c.getContext())}
+                </td>
+              ))}
+            </tr>
+          ))}
+        </tbody>
+      </table>
+      <div className="flex gap-2 items-center justify-center mt-10">
+        <button
+          className={`w-6 h-6 ${
+            !table.getCanPreviousPage() ? 'opacity-25' : 'opacity-100'
+          }`}
+          onClick={() => table.setPageIndex(0)}
+          disabled={!table.getCanPreviousPage()}
+        >
+          <ChevronDoubleLeftIcon />
+        </button>
+        <button
+          className={`w-6 h-6 ${
+            !table.getCanPreviousPage() ? 'opacity-25' : 'opacity-100'
+          }`}
+          onClick={() => table.previousPage()}
+          disabled={!table.getCanPreviousPage()}
+        >
+          <ChevronLeftIcon />
+        </button>
+        <span className="flex items-center gap-1">
+          <div>Page</div>
+          <strong>
+            {table.getState().pagination.pageIndex + 1} of{' '}
+            {table.getPageCount()}
+          </strong>
+        </span>
+        <button
+          className={`w-6 h-6 ${
+            !table.getCanNextPage() ? 'opacity-25' : 'opacity-100'
+          }`}
+          onClick={() => table.nextPage()}
+          disabled={!table.getCanNextPage()}
+        >
+          <ChevronRightIcon />
+        </button>
+        <button
+          className={`w-6 h-6 ${
+            !table.getCanNextPage() ? 'opacity-25' : 'opacity-100'
+          }`}
+          onClick={() => table.setPageIndex(table.getPageCount() - 1)}
+          disabled={!table.getCanNextPage()}
+        >
+          <ChevronDoubleRightIcon />
+        </button>
+      </div>
+    </div>
+  );
+};
+
+const globalFilterFn: FilterFn<any> = (row, columnId, filterValue: string) => {
+  const search = filterValue.toLowerCase();
+
+  let value = row.getValue(columnId) as string;
+  if (typeof value === 'number') value = String(value);
+
+  return value?.toLowerCase().includes(search);
+};

examples/learn/portaljs/components/Vega.tsx

@@ -0,0 +1,6 @@
+//  Wrapper for the Vega component
+import { Vega as VegaOg } from "react-vega";
+
+export function Vega(props) {
+  return <VegaOg {...props} />;
+}

examples/learn/portaljs/components/VegaLite.tsx

@@ -0,0 +1,9 @@
+//  Wrapper for the Vega Lite component
+import { VegaLite as VegaLiteOg } from "react-vega";
+import applyFullWidthDirective from "../lib/applyFullWidthDirective";
+
+export function VegaLite(props) {
+  const Component = applyFullWidthDirective({ Component: VegaLiteOg });
+
+  return <Component {...props} />;
+}

examples/learn/portaljs/lib/applyFullWidthDirective.tsx

@@ -0,0 +1,21 @@
+export default function applyFullWidthDirective({
+  Component,
+  defaultWFull = true,
+}) {
+  return (props) => {
+    const newProps = { ...props };
+
+    let newClassName = newProps.className || "";
+    if (newProps.fullWidth === true) {
+      newClassName += " w-[90vw] ml-[calc(50%-45vw)] max-w-none";
+    } else if (defaultWFull) {
+      //  So that charts and tables will have the
+      //  same width as the text content, but images
+      //  can have its width set using the width prop
+      newClassName += " w-full";
+    }
+    newProps.className = newClassName;
+
+    return <Component {...newProps} />;
+  };
+}

examples/learn/portaljs/lib/loadData.tsx

@@ -0,0 +1,5 @@
+export default async function loadData(url: string) {
+  const response = await fetch(url)
+  const data = await response.text()
+  return data
+}

examples/learn/portaljs/lib/parseCsv.ts

@@ -0,0 +1,20 @@
+import papa from "papaparse";
+
+const parseCsv = (csv: string) => {
+  csv = csv.trim();
+  const rawdata = papa.parse(csv, { header: true });
+
+  let cols: any[] = [];
+  if(rawdata.meta.fields) {
+    cols = rawdata.meta.fields.map((r: string) => {
+      return { key: r, name: r };
+    });
+  }
+
+  return {
+    rows: rawdata.data as any,
+    fields: cols,
+  };
+};
+
+export default parseCsv;

examples/learn/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

examples/openspending/__tests__/os-data.test.ts

@@ -2,7 +2,7 @@ import { expect, test } from 'vitest';
 import { getAllProjectsFromOrg, getProjectDataPackage } from '../lib/project';
 import { loadDataPackage } from '../lib/loader';
 import { getProjectMetadata } from '../lib/project';
-import { getCsv, parseCsv } from '../components/Table';
+import { validate } from 'datapackage';
 
 test(
   'Test OS-Data',
@@ -12,8 +12,24 @@ test(
       'main',
       process.env.VITE_GITHUB_PAT
     );
-    if (repos.failed.length > 0) console.log(repos.failed);
-    expect(repos.failed.length).toBe(0);
+    if (repos.failed.length > 0)
+      console.log('Failed to get datapackage on', repos.failed);
+    let failedDatapackages = await Promise.all(
+      repos.results.map(async (item) => {
+        try {
+          const { valid, errors } = await validate(item.datapackage);
+          return errors.length > 0 ? item.repo.name : null;
+        } catch {
+          return item.repo.name;
+        }
+      })
+    );
+    failedDatapackages = failedDatapackages.filter((item) => item !== null);
+    if (failedDatapackages.length > 0) {
+      console.log('Failed to validate datapackage on ', failedDatapackages);
+    } else {
+      console.log('No invalid packages');
+    }
   },
   { timeout: 100000 }
 );
@@ -27,7 +43,22 @@ test(
       process.env.VITE_GITHUB_PAT
     );
     if (repos.failed.length > 0) console.log(repos.failed);
-    expect(repos.failed.length).toBe(0);
+    let failedDatapackages = await Promise.all(
+      repos.results.map(async (item) => {
+        try {
+          const { valid, errors } = await validate(item.datapackage);
+          return errors.length > 0 ? item.repo.name : null;
+        } catch {
+          return item.repo.name;
+        }
+      })
+    );
+    failedDatapackages = failedDatapackages.filter((item) => item !== null);
+    if (failedDatapackages.length > 0) {
+      console.log('Failed to validate datapackage on ', failedDatapackages);
+    } else {
+      console.log('No invalid packages');
+    }
   },
   { timeout: 100000 }
 );
@@ -83,56 +114,3 @@ test(
   },
   { timeout: 100000 }
 );
-
-test(
-  'Test getting one section of csv from R2',
-  async () => {
-    const rawCsv = await getCsv(
-      'https://storage.openspending.org/state-of-minas-gerais-brazil-planned-budget/__os_imported__br-mg-ppagloc.csv'
-    );
-    const parsedCsv = await parseCsv(rawCsv);
-    expect(parsedCsv.errors.length).toBe(1);
-    expect(parsedCsv.data.length).toBe(10165);
-    expect(parsedCsv.meta.fields).toStrictEqual([
-      'function_name',
-      'function_label',
-      'product_name',
-      'product_label',
-      'area_name',
-      'area_label',
-      'subaction_name',
-      'subaction_label',
-      'region_label_map',
-      'region_reg_map',
-      'region_name',
-      'region_label',
-      'municipality_map_id',
-      'municipality_name',
-      'municipality_map_code',
-      'municipality_label',
-      'municipality_map_name_simple',
-      'municipality_map_name',
-      'cofog1_label_en',
-      'cofog1_name',
-      'cofog1_label',
-      'amount',
-      'subprogramme_name',
-      'subprogramme_label',
-      'time_name',
-      'time_year',
-      'time_month',
-      'time_day',
-      'time_week',
-      'time_yearmonth',
-      'time_quarter',
-      'time',
-      'action_name',
-      'action_label',
-      'subfunction_name',
-      'subfunction_label',
-      'programme_name',
-      'programme_label',
-    ]);
-  },
-  { timeout: 100000 }
-);

examples/openspending/components/DatasetsSearch.tsx

@@ -13,9 +13,13 @@ import { useState } from 'react';
 export default function DatasetsSearch({
   datasets,
   availableCountries,
+  minPeriod,
+  maxPeriod,
 }: {
   datasets: Project[];
   availableCountries;
+  minPeriod: string;
+  maxPeriod: string;
 }) {
   const itemsPerPage = 6;
   const [page, setPage] = useState(1);
@@ -49,20 +53,35 @@ export default function DatasetsSearch({
         ? dataset.countryCode === watch().country
         : true
     )
-    // TODO: Does that really makes sense?
-    // What if the fiscalPeriod is 2015-2017 and inputs are
-    // set to 2015-2016. It's going to be filtered out but
-    // it shouldn't.
-    .filter((dataset) =>
-      watch().minDate && watch().minDate !== ''
-        ? dataset.fiscalPeriod?.start >= watch().minDate
-        : true
-    )
-    .filter((dataset) =>
-      watch().maxDate && watch().maxDate !== ''
-        ? dataset.fiscalPeriod?.end <= watch().maxDate
-        : true
-    );
+    .filter((dataset) => {
+      const filterMinDate = watch().minDate;
+      const filterMaxDate = watch().maxDate;
+
+      const datasetMinDate = dataset.fiscalPeriod?.start;
+      const datasetMaxDate = dataset.fiscalPeriod?.end;
+
+      let datasetStartOverlaps = false;
+      if (datasetMinDate) {
+        datasetStartOverlaps =
+          datasetMinDate >= filterMinDate && datasetMinDate <= filterMaxDate;
+      }
+
+      let datasetEndOverlaps = false;
+      if (datasetMaxDate) {
+        datasetEndOverlaps =
+          datasetMaxDate >= filterMinDate && datasetMaxDate <= filterMaxDate;
+      }
+
+      if (filterMinDate && filterMaxDate) {
+        return datasetStartOverlaps || datasetEndOverlaps;
+      } else if (filterMinDate) {
+        return datasetMinDate >= filterMinDate;
+      } else if (filterMaxDate) {
+        return datasetMinDate <= filterMaxDate;
+      }
+
+      return true;
+    });
 
   const paginatedDatasets = filteredDatasets.slice(
     (page - 1) * itemsPerPage,
@@ -111,22 +130,32 @@ export default function DatasetsSearch({
           </select>
         </div>
         <div className="sm:basis-1/6">
-          <label className="text-sm text-gray-600 font-medium">Min. date</label>
+          <label className="text-sm text-gray-600 font-medium">
+            Fiscal Period Start
+          </label>
           <div className="relative">
             <input
               aria-label="Min. date"
-              type="date"
+              type="text"
+              placeholder={minPeriod}
+              onFocus={(e) => (e.target.type = 'date')}
+              onBlur={(e) => (e.target.type = 'text')}
               {...register('minDate', { onChange: () => setPage(1) })}
               className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
             />
           </div>
         </div>
         <div className="sm:basis-1/6">
-          <label className="text-sm text-gray-600 font-medium">Max. date</label>
+          <label className="text-sm text-gray-600 font-medium">
+            Fiscal Period End
+          </label>
           <div className="relative">
             <input
               aria-label="Max. date"
-              type="date"
+              type="text"
+              placeholder={maxPeriod}
+              onFocus={(e) => (e.target.type = 'date')}
+              onBlur={(e) => (e.target.type = 'text')}
               {...register('maxDate', { onChange: () => setPage(1) })}
               className="h-[3em] w-full rounded-lg bg-white py-2 pl-3 pr-10 text-left shadow-md focus:outline-none focus-visible:border-emerald-500 focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75 focus-visible:ring-offset-2 focus-visible:ring-offset-emerald-400 sm:text-sm"
             />
@@ -196,9 +225,9 @@ const CloseIcon = () => {
           id="Vector"
           d="M18 18L12 12M12 12L6 6M12 12L18 6M12 12L6 18"
           stroke="currentColor"
-          stroke-width="2"
-          stroke-linecap="round"
-          stroke-linejoin="round"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
         />
       </g>
     </svg>

examples/openspending/components/FlatUiTable.tsx

@@ -1,9 +1,4 @@
-import {
-  QueryClient,
-  QueryClientProvider,
-  useQuery,
-  useQueryClient,
-} from 'react-query';
+import { QueryClient, QueryClientProvider, useQuery } from 'react-query';
 import Papa from 'papaparse';
 import { Grid } from '@githubocto/flat-ui';
 
@@ -62,24 +57,41 @@ const Spinning = () => {
   );
 };
 
-export const Table: React.FC<{ url: string }> = ({ url }) => {
+export interface FlatUiTableProps {
+  url?: string;
+  data?: { [key: string]: number | string }[];
+  rawCsv?: string;
+}
+export const FlatUiTable: React.FC<FlatUiTableProps> = ({
+  url,
+  data,
+  rawCsv,
+}) => {
   return (
     // Provide the client to your App
     <QueryClientProvider client={queryClient}>
-      <TableInner url={url} />
+      <TableInner url={url} data={data} rawCsv={rawCsv} />
     </QueryClientProvider>
   );
 };
 
-const TableInner: React.FC<{ url: string }> = ({ url }) => {
+const TableInner: React.FC<FlatUiTableProps> = ({ url, data, rawCsv }) => {
+  if (data) {
+    return (
+      <div className="w-full" style={{height: '500px'}}>
+        <Grid data={data} />
+      </div>
+    );
+  }
   const { data: csvString, isLoading: isDownloadingCSV } = useQuery(
     ['dataCsv', url],
-    () => getCsv(url)
+    () => getCsv(url),
+    { enabled: !!url }
   );
   const { data: parsedData, isLoading: isParsing } = useQuery(
     ['dataPreview', csvString],
-    () => parseCsv(csvString),
-    { enabled: !!csvString }
+    () => parseCsv(rawCsv ? rawCsv : csvString),
+    { enabled: rawCsv ? true : !!csvString }
   );
   if (isParsing || isDownloadingCSV)
     <div className="w-full">
@@ -87,7 +99,7 @@ const TableInner: React.FC<{ url: string }> = ({ url }) => {
     </div>;
   if (parsedData)
     return (
-      <div className="h-[500px] overflow-scroll">
+      <div className="w-full" style={{height: '500px'}}>
         <Grid data={parsedData.data} />
       </div>
     );

examples/openspending/components/Header.tsx

@@ -5,6 +5,9 @@ import Link from 'next/link';
 import { useRouter } from 'next/router';
 import { Bars3Icon } from '@heroicons/react/24/outline';
 import { useState } from 'react';
+import { Fragment } from 'react';
+import { Menu, Transition } from '@headlessui/react';
+import { ChevronDownIcon } from '@heroicons/react/20/solid';
 
 export function Header() {
   const [menuOpen, setMenuOpen] = useState<boolean>(false);
@@ -16,42 +19,69 @@ export function Header() {
   };
 
   const navLinks = [
-    {
-      title: 'Home',
-      href: '/',
-    },
     {
       title: 'Datasets',
       href: '/#datasets',
     },
-    // {
-    //   title: "Community",
-    //   href: "https://community.openspending.org/"
-    // }
+    {
+      title: 'Blog',
+      href: '/blog',
+    },
+    {
+      title: 'About',
+      href: '/about',
+      children: [
+        {
+          title: 'Fiscal Data Package',
+          href: '/about/fiscaldatapackage/',
+        },
+        {
+          title: 'Tools',
+          href: '/about/tools/',
+        },
+        {
+          title: 'Funders',
+          href: '/about/funders/',
+        },
+        {
+          title: 'Presentations',
+          href: '/about/presentations/',
+        },
+      ],
+    },
+    {
+      title: 'Resources',
+      href: '/resources',
+      children: [
+        {
+          title: 'Follow the money',
+          href: '/resources/journo',
+        },
+        {
+          title: 'Map of Spending Projects',
+          href: '/resources/map-of-spending-projects/',
+        },
+        {
+          title: 'Working Group On Open Spending Data',
+          href: '/resources/wg/',
+        },
+      ],
+    },
   ];
 
   return (
     <header className="relative z-50 pb-11 lg:pt-11">
-      <Container className="flex flex-wrap items-center justify-between lg:flex-nowrap mt-10 lg:mt-0">
+      <Container className="flex flex-wrap justify-between lg:flex-nowrap mt-10 lg:mt-0">
         <Link href="/" className="lg:mt-0 lg:grow lg:basis-0 flex items-center">
           <Image src={logo} alt="OpenSpending" className="h-12 w-auto" />
         </Link>
         <ul className="hidden list-none sm:flex gap-x-5 text-base font-medium">
           {navLinks.map((link, i) => (
             <li key={`nav-link-${i}`}>
-              <Link
-                className={`text-emerald-900 hover:text-emerald-600 ${
-                  isActive(link) ? 'text-emerald-600' : ''
-                }`}
-                href={link.href}
-                scroll={false}
-              >
-                {link.title}
-              </Link>
+              <Dropdown navItem={link} />
             </li>
           ))}
         </ul>
-        <div className="hidden xl:block xl:grow"></div>
         <div className="sm:hidden sm:mt-10 lg:mt-0 lg:grow lg:basis-0 lg:justify-end">
           <button onClick={() => setMenuOpen(!menuOpen)}>
             <Bars3Icon className="w-8 h-8" />
@@ -80,3 +110,77 @@ export function Header() {
     </header>
   );
 }
+
+function classNames(...classes) {
+  return classes.filter(Boolean).join(' ');
+}
+
+function Dropdown({ navItem }: { navItem: any }) {
+  const [showDropDown, setShowDropDown] = useState(false);
+  return (
+    <Menu as="div" className="relative inline-block text-left">
+      {({ open }) => (
+        <>
+          <div>
+            <Menu.Button
+              onMouseEnter={() => setShowDropDown(true)}
+              onMouseLeave={() => setShowDropDown(false)}
+              className="text-emerald-900 hover:text-emerald-600 inline-flex w-full justify-center gap-x-1.5 px-3 py-2 text-sm font-semibold"
+            >
+              <Link href={navItem.href}>{navItem.title}</Link>
+              {navItem.children && (
+                <ChevronDownIcon
+                  className="-mr-1 h-5 w-5 text-gray-400"
+                  aria-hidden="true"
+                />
+              )}
+            </Menu.Button>
+          </div>
+
+          {navItem.children && (
+            <Transition
+              as={Fragment}
+              show={showDropDown}
+              enter="transition ease-out duration-100"
+              enterFrom="transform opacity-0 scale-95"
+              enterTo="transform opacity-100 scale-100"
+              leave="transition ease-in duration-75"
+              leaveFrom="transform opacity-100 scale-100"
+              leaveTo="transform opacity-0 scale-95"
+            >
+              <div>
+                <Menu.Items
+                  static
+                  onMouseEnter={() => setShowDropDown(true)}
+                  onMouseLeave={() => setShowDropDown(false)}
+                  className="absolute right-0 z-10 w-56 origin-top-right rounded-md bg-white shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none"
+                >
+                  <div className="py-1">
+                    {navItem.children.map((item) => (
+                      <Menu.Item>
+                        {({ active }) => (
+                          <a
+                            key={item.href}
+                            href={item.href}
+                            className={classNames(
+                              active
+                                ? 'bg-gray-100 text-emerald-900 hover:text-emerald-600'
+                                : 'text-gray-700',
+                              'block px-4 py-2 text-sm'
+                            )}
+                          >
+                            {item.title}
+                          </a>
+                        )}
+                      </Menu.Item>
+                    ))}
+                  </div>
+                </Menu.Items>
+              </div>
+            </Transition>
+          )}
+        </>
+      )}
+    </Menu>
+  );
+}

examples/openspending/components/Hero.tsx

@@ -27,7 +27,7 @@ export function Hero({ countriesCount, datasetsCount, filesCount }) {
           <Button href="#datasets" className="mt-10">
             Search datasets
           </Button>
-          <dl className="mt-10 grid grid-cols-2 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
+          <dl className="mt-10 grid grid-cols-1 sm:grid-cols-3 gap-x-10 gap-y-6 sm:mt-16 sm:gap-x-16 sm:gap-y-10 sm:text-center lg:auto-cols-auto lg:grid-flow-col lg:grid-cols-none lg:justify-start lg:text-left">
             {[
               //  Added the plus sign because some datasets do not
               //  contain defined countries
@@ -36,10 +36,18 @@ export function Hero({ countriesCount, datasetsCount, filesCount }) {
               ['Files', filesCount],
             ].map(([name, value]) => (
               <div key={name}>
-                <dt className="font-mono text-sm text-emerald-600">{name}</dt>
-                <dd className="mt-0.5 text-2xl font-semibold tracking-tight text-emerald-900">
-                  {value}
-                </dd>
+                <div className='flex gap-x-2 items-center sm:hidden' key={name}>
+                  <dd className="mt-0.5 text-2xl font-semibold tracking-tight text-emerald-900">
+                    {value}
+                  </dd>
+                  <dt className="font-mono text-sm text-emerald-600">{name}</dt>
+                </div>
+                <div className='hidden sm:block' key={name}>
+                  <dt className="font-mono text-sm text-emerald-600">{name}</dt>
+                  <dd className="mt-0.5 text-2xl font-semibold tracking-tight text-emerald-900">
+                    {value}
+                  </dd>
+                </div>
               </div>
             ))}
           </dl>

examples/openspending/components/_shared/Footer.tsx

@@ -0,0 +1,22 @@
+import Image from 'next/image';
+import Link from 'next/link';
+export default function Footer() {
+  return (
+    <footer>
+      <div className="mx-auto max-w-7xl px-4 sm:px-6 lg:px-8 flex flex-col items-center justify-between md:flex-row">
+        <div className="flex gap-x-2 items-center mx-auto">
+          <p className="mt-8 text-base text-slate-500 md:mt-0">Maintained by</p>
+          <a href="https://www.datopian.com/" target="_blank">
+            <Image
+              alt="Datopian logo"
+              className="mb-2"
+              src="/datopian-logotype.png"
+              width={120}
+              height={30}
+            />
+          </a>
+        </div>
+      </div>
+    </footer>
+  );
+}

examples/openspending/components/_shared/Layout.tsx

@@ -1,10 +1,12 @@
 import { Header } from '../Header';
+import Footer from './Footer';
 
 export default function Layout({ children }) {
   return (
-    <div className="bg-white min-h-screen">
+    <div className="bg-white min-h-screen pb-12">
       <Header />
       {children}
+      <Footer />
     </div>
   );
 }

examples/openspending/content/about/contact.md

@@ -0,0 +1,23 @@
+---
+lead: true
+section: about
+title: Contact
+authors:
+- Anders Pedersen
+redirect_from:
+- /contact/
+---
+
+Connect with the OpenSpending community.
+
+The best place for OpenSpending-related discussions is our [forum](https://discuss.okfn.org/c/openspending/), but you can interact with OpenSpending on your favorite social network by following the links below:
+
+* Open Knowledge Discussion Forum
+  * [English](https://discuss.okfn.org/c/openspending/none)
+  * [Portuguese](https://discuss.okfn.org/c/openspending/gastos-abertos)
+* <abbr title="Gitter">Gitter</abbr> chat: [OpenSpending Gitter chatroom](https://gitter.im/openspending)
+* GitHub: 
+  * [OpenSpending](https://github.com/openspending) (OpenSpending-related code)
+
+* Twitter: [&#64;openspending](https://twitter.com/openspending)
+* Facebook: [OpenSpending](https://www.facebook.com/openspending?_rdr=p)

examples/openspending/content/about/contributors.md

@@ -0,0 +1,172 @@
+---
+section: about
+lead: true
+title: Contributors
+authors:
+- Anders Pedersen
+---
+
+There are many contributors to OpenSpending. There are thousands of registered OpenSpending users contributing data and analysis.
+
+It is impossible to adequately acknowledge the many individuals and organizations who have contributed.
+
+This page then is necessarily partial and is focused (though not limited to) those who have made special efforts to contribute through their participation in specific teams, in donating data or in other significant ways.
+
+If you'd like to be added as a contributor to this page please [get in touch]({{ site.baseurl }}/about/contact/).
+
+## Teams
+
+### News and Website Team
+
+#### News Editors - Descriptions
+
+These guys run the blog and manage our social media presence. We are
+still recruiting News Editor volunteers so if you'd like to join the
+team <a
+href="http://community.openspending.org/contribute/web/#Sign_up">apply
+here</a>.
+
+<strong>Burite Joseph</strong>, <a href="https://twitter.com/BuriteJoseph">@BuriteJoseph</a>
+
+Independent media practitioner and entrepreneur with over five years
+of journalism and research experience, Burite runs ZHENOBIA, a media
+integration and multimedia content aggregation company. She also
+consults for <a href="http://www.smsmedia.ug">SMS Media Uganda</a>,
+Ultimate Media Uganda, <a href="http://www.busiweek.com">East African
+Business Week</a> and <a href="http://www.dailymonitor.co.ug">Daily
+Monitor</a>.
+
+<blockquote>
+Working with data is my new passion. I am a quick learner and teamwork is my steroid.
+</blockquote>
+
+<strong>Anna Flagg</strong>, <a href="http://www.annaflagg.com">www.annaflagg.com</a>
+
+Data journalist at the Center for Responsive Politics, Anna has a background in computer science, data visualization, design and data-storytelling.
+
+<blockquote>
+I like working on projects that create awareness of issues important to the public. I'm excited to work with and learn from the Open Spending community.
+
+</blockquote>
+<strong>Laura S. García</strong>,<a href="https://twitter.com/Laura_S_Garcia">@laura_s_garcia</a>
+
+An experienced journalist, Laura has worked for more than ten years as a multi-media journalist in Spain. She has also taught Geography and History to high-school students. Laura speaks Spanish, Galician, English and a little Swedish.
+
+<blockquote>
+I’m looking to improve my knowledge of open data, as I’ve always thought this to be the best way to offer a good journalism and a good education as well.
+
+</blockquote>
+<strong>Karen Brzezinska</strong>, <a href="https://twitter.com/westofwarsaw">@westofwarsaw</a>
+
+Also a professional journalist, Karen (Kati) worked for international news services specialising in equity, commodity and currency markets. Her background is in PoliSci (East European studies), and, while originally from midwestern US, her life experience lists Italy, Hungary (1989-1992) and The Netherlands (since 1992) as home-countries. Kati is fluent in English (US) and Dutch.
+
+<blockquote>
+I'm interested in learning how open data can be used to enhance governance and education.
+
+</blockquote>
+<strong>Dominic Kornu</strong>, <a href="https://twitter.com/Qaphui">@qaphui</a>
+
+An IT and Maths tutor from Ghana, with an interest in web and social media technologies, Dominic blogs at <a href="http://dominicmary.blogspot.com">Qaphui’s Cafe</a> and volunteers in his free time.
+
+<blockquote>
+I am interested in learning how open data can be used to enhance governance and education.
+
+</blockquote>
+<strong>Mehmet Koksal</strong>, <a href="https://twitter.com/mehmetkoksal">@mehmetkoksal</a>
+
+Freelance journalist based in Brussels (Belgium) and conference interpreter, Mehmet also works as a fixer for the international press, including the French weekly <a href="http://www.courrierinternational.com/">Courrier int.</a>. In his free time he volunteers for <a href="http://www.ajp.be/">AJP</a> and acts as a campaign manager for the <a href="http://europe.ifj.org/en/pages/turkey-campaign-set-journalists-free">EFJ</a>.
+
+<strong>Teodora Beleaga</strong>, <a href="https://twitter.com/t30d0ra">@t30d0ra</a>
+
+A digital analyst and freelance data journalist based in London, <a href="http://teodorabeleaga.com">Teodora </a>is an alumna of City University’s Interactive Journalism MA and has completed work experience with <a href="http://www.theguardian.com/profile/teodora-beleaga">The Guardian</a>.
+
+<blockquote>
+I joined the Open Spending project to share my data analysis skills and expand my understanding of fiscal transparency and government spending.
+
+</blockquote>
+<strong>Miriam Ruhenstroth</strong>
+
+A Science and technology freelance journalist based in Berlin (Germany), Miriam has a background in biological sciences. In 2011 she attended a summer school for data journalism (organized by Initiative Wissenschaftsjournalismus).
+
+<blockquote>
+I found the field of data storytelling thrilling and joined OpenSpending, to learn more about it and participate for good.
+
+</blockquote>
+
+### Data Team
+
+
+#### Data Wranglers - Descriptions
+
+The Data Wranglers work to add, clean and visualise data in OpenSpending. They help community members who need assistance. Some data wranglers focus on cleaning and analysing data whereas others work to visualise data using the OpenSpending API. We are still recruiting Data Wrangler volunteers so if you'd like to join the team <a href="http://community.openspending.org/contribute/data/#Official_sign_up">apply here</a>.
+
+<strong>Concha Catalan</strong>, <a href="https://twitter.com/conchacatalan">@conchacatalan</a>
+
+An English teacher and freelance journalist based in Barcelona (Spain), Concha is currently working on a project to open the autonomous government of Catalonia (opengov.cat). She also blogs at <a href="http://barcelonalittleshell.blogspot.com.es">http://barcelonalittleshell.blogspot.com.es</a>.
+
+<blockquote>
+I would like to add the data set of the autonomous government of Catalonia budget to OpenSpending. I am coming to terms with lots of new concepts.
+
+</blockquote>
+<strong>Prakash Neupane</strong>, <a href="https://twitter.com/nprkshn">@nprkshn</a>
+
+OKFN Ambassador in Nepal and FOSS Enthusiastic, Prakash is working in social development empowering individuals and communities by using technology. He is an Open Data Researcher and Nepali Wikimedian, responsible for Wikimedia Education Program in Nepal. Find out more  about him <a href="http://www.prakashneupane.com.np/about-me">here</a>.
+
+<strong>Pierre Chrzanowski</strong>, @piezanowski
+
+A member of the French OKFN working in the field of Open Government Data, Pierre says he is really interested to work on Tax Heaven, Public Procurement and Aid Data.
+
+<blockquote>
+I want to learn more about tools to analyse the data sets and how best to do storytelling.
+
+</blockquote>
+<strong>Samuel S. Lee</strong>, <a href="https://twitter.com/OpenNotion">@OpenNotion</a>
+
+Currently based in Washington DC, Samuel is a member of the <a href="https://finances.worldbank.org">World Bank Group Open Finances</a> team. He loves data, innovation, transparency, photography and college football.
+
+<blockquote>
+I am passionate about “open” and its potential to transform civic engagement, international development, and the world. I am particularly interested in realizing the potential of open financial information.
+
+</blockquote>
+<strong>Adriana Homolova</strong>
+
+A data journalism student with a passion for open culture, Adriana is a member of the <a href="http://soit.sk">Society for Open Information Technologies</a>.
+
+<strong>Sipos Zoltán</strong>
+
+A Hungarian journalist working for an Internet news portal in Romania, Sipos specializes in investigative reporting.  His background includes philosophy, sociology and public policies. Sipos has experience working with data, filing FOI requests, and tackling spreadsheets.
+
+<blockquote>
+I am trying to learn as much as I can about data journalism through online groups, MOOCs and books purchased from Amazon. My ultimate goal is to set up a small investigative / data journalism start-up in Romania.
+</blockquote>
+
+<strong>Gabe Sawhney</strong>
+
+A member of <a href="http://betterbudget.ca">Better Budget
+Toronto</a> Gabe joined the Team to bring transparency to his
+city’s <a
+href="http://spacing.ca/toronto/2012/12/10/lorinc-building-a-better-budget-at-city-hall/">budget</a>.
+
+<blockquote>
+I want to mobilize action (citizens, elected officials and policymakers) for better process, better clarity, better formats, and more transparency around city budgets.
+</blockquote>
+
+<strong>Elaine Ayo</strong>, <a href="https://twitter.com/eieayo">@eieayo</a>
+
+Statistician student based in Washington, DC Elaine has spent the last
+three years in Seoul, South Korea as a copy editor for an English news
+wire. Prior to that Elaine reported for her hometown paper, the San
+Antonio Express-News, in Texas.
+
+<strong>Hans Loos</strong>
+
+An IT and telecom freelance journalist based in Belgium, Hans studied
+sociology and has a passion for statistics.
+
+<blockquote>
+I have started to learn to program and study R but without big results up till now.
+</blockquote>
+
+## Data Donors
+
+In progress ...

examples/openspending/content/about/exemplars.md

@@ -0,0 +1,329 @@
+---
+section: about
+lead: true
+title: Project portfolio
+authors:
+- Neil Ashton
+---
+Here are some of the projects, past and present, that we have been involved with.
+
+* <a href="#current-development">Current OpenSpending projects</a>
+* <a href="#past-development">Past projects</a>
+* <a href="#gov-tools">Tools for governments</a>
+* <a href="#gov-use-cases">Government use cases</a>
+* <a href="#publications">Publications and reports</a>
+* <a href="#standards">Technical standards</a>
+* <a href="#stories">Stories and data journalism</a>
+* <a href="#your-project">Collaborate! Your project here...</a>
+
+<a name="current-development"></a>
+
+## Current projects
+
+We're working on [OpenSpending Next](/next/), the next version of OpenSpending.  Learn how to [get involved](/get-involved/)!
+
+<a name="past-development"></a>
+
+## Already unleashed
+
+#### Spending Stories
+
+In 2011, the Open Knowledge Foundation was awarded a Knight News Challenge Grant to work on the <a href="http://blog.okfn.org/2011/01/18/spending-stories/">Spending Stories</a> Project. Large numbers are often meaningless to the general public, and despite a wealth of information around government spending, the topic of government finance is often overlooked by journalists. The Spending Stories project aims to facilitate reporting by speeding up fact-checking around spending data as well as connecting news stories about public spending to relevant datasets and visualisations to put these stories into context.
+
+We're into our second year of the project now. <a href="http://www.pbs.org/idealab/2012/12/follow-the-money-a-spending-stories-guide-for-journalists345.html">See what we are up to by following our updates on the PBS Idea Lab blog.</a>
+
+#### budzeti.ba: Bosnian Budget Bubbles
+
+<img class="pull-right" style="margin-left: 1em;" title="Balkan Budgets" src="http://farm9.staticflickr.com/8063/8219557569_cc12ebbdea_m.jpg" alt="" />
+
+In October 2013 OpenSpending launched the "Where Does My Money Go"-site <a href="http://budzeti.ba/">budzeti.ba</a> in Bosnia and Herzegovina. Thanks to a grant from the National Endowment for Democracy, the OpenSpending team collaborated with the Centre for Public Interest Advocacy in Bosnia to produce interactive graphics of national and subnational budgets in Bosnia.
+
+As part of the project OpenSpending:
+
+<ul>
+<li>Created visualisations of national, entity-, cantonal- and district-level budgets for Bosnia.</li>
+<li>Conducted onsite trainings and capacity building workshops with organisations from other Balkan countries, on getting, wrangling and presenting financial data (with OpenSpending and other tools).</li>
+<li>Developed a tax calculator for Bosnia (similar to the <a href="http://wheredoesmymoneygo.org/dailybread.html">Daily Bread</a>).</li>
+<li>Tested the <a href="http://openspending.org/resources/handbook/ch001_introduction.html">Spending Data Handbook</a> and added an <a href="http://community.openspending.org/help/guide/">OpenSpending guide</a> with a more technical guidance, for those organisations wishing to do more ambitious data work.</li>
+</ul>
+
+<a href="http://blog.openspending.org/2012/09/26/balkan-budget-bubbles/">Read more about the project.</a> Read the full announcement from <a href="http://community.openspending.org/2013/10/budzeti-ba-follow-the-money-bosnia-herzegovina/">the launch</a>.
+
+#### OpenSpending Slovakia
+
+In early 2013, we launched <a href="http://slovakia.openspending.org">Slovakia Openspending</a>, prepared in collaboration with transparency and anti-corruption watchdog <a href="http://www.transparency.sk/">Transparency International Slovakia</a>. The site contains budget and expenditure information gathered from some 20 cities across Slovakia allowing users to examine current municipality budgets and compare them to expenditures going back to 2009. Based on the openspending.org source code, we have added several improvements such as:
+
+<ul>
+<li>Year-on-year comparison for spending categories.</li>
+<li>Improved listing and more contextual presentation of the data.</li>
+<li>Embedable graphic drilldowns with accompanying tables.</li>
+</ul>
+Transparency Slovakia hopes to promote the site as an example of how budgets could be made more accessible and comprehensible to the public once the site is localized and tested further. While not in its current plans, additional spending data and a feature to allow users to compare cities would improve opportunities for doing crowdsourced benchmarking.
+
+#### Where Does My Money Go?
+
+<img class="pull-left" style="margin-right: 3em;" title="WDMMG original" src="http://farm8.staticflickr.com/7088/7315252358_7dae93b263.jpg" alt="" width="250" />
+
+<a href="http://wheredoesmymoneygo.org/">Where Does My Money Go?</a> (WDMMG) is the project from which OpenSpending was born. Funded originally by 4IP, it allows UK citizens to examine where their taxes are being spent through an interactive 'bubble tree' visualisation. They can even find out how much they contribute on a daily basis through their taxes to various sectors of society through the <a href="http://wheredoesmymoneygo.org/dailybread.html">Daily Bread</a> app.
+
+You can now build a site like WDMMG for your own country using the OpenSpending API and the WDMMG Toolkit:
+
+<ul>
+<li><a href="http://wheredoesmymoneygo.org/">Visit the Where Does My Money Go? project</a></li>
+<li><a href="https://twitter.com/#!/wdmmg">Follow Where Does My Money Go? on Twitter</a></li>
+<li><a href="http://blog.openspending.org/2012/02/16/thekit/">Read more about how to create your own Where Does My Money Go?site</a></li>
+</ul>
+
+#### Cameroon Budget Inquirer
+
+<img class="pull-left" style="margin-right: 2em;" title="Cameroon Budget Inquirer" src="http://farm9.staticflickr.com/8340/8287000485_4daf7d73da_n.jpg" alt="" />
+
+We collaborated with the World Bank to produce a citizen-friendly representation of <a href="http://cameroon.openspending.org/">Cameroon's budget</a>.
+
+The project incorporates the open-source 'bubble visualisations' first seen in Where Does My Money Go, as well as many new elements, such as a map based navigation and heatmaps. The project includes:
+
+<ul>
+<li>Visualisation of the National Investment Budget, the Northwest region's budget of Cameroon (including absolute and per-capita estimates, budgeted as well as actual figures) as well as a similar visualisation of the Tignere local council budget.</li>
+<li>A sub-national budget transparency index - similar to the work of the International Budget Partnership, but at sub-national levels. This highlights the availability of key budget documents and ranks regions on their availability.</li>
+<li>Explore the data: filter the data by categories such as amount, location, and activity.</li>
+</ul>
+
+#### OffenerHaushalt
+<img class="pull-right" style="margin-left: 1em;" title="Offener Haushalt" src="http://farm8.staticflickr.com/7077/7315281352_2c00d928d8.jpg" alt="" width="250" />
+
+<a href="http://bund.offenerhaushalt.de/">OffenerHaushalt</a> allows users to explore and drill down through the various layers of Germany's federal budget, comparing data from as far back as 2006. Through the new TreeMap visualisation, the user can easily see and explore the different departments and programmes and see how much is spent, proportions and statistics on changes between years.
+
+The success of OffenerHaushalt and the demand to roll it out on a local level was one of the prime motivations for the creation of OpenSpending. To date, the OffenerHaushalt team have received around 90 requests for a similar site in their area.
+
+<ul>
+<li><a href="http://bund.offenerhaushalt.de/">Visit the OffenerHaushalt Project</a></li>
+<li><a href="https://twitter.com/#!/offenerhaushalt">Follow OffenerHaushalt on Twitter</a></li>
+</ul>
+
+#### Uganda Aid Visualisation
+
+Aid flows often do not pass through a recipient government’s conventional budget mechanisms. When this happens, recipient governments themselves may not have the complete overview of where aid money goes and how donor priorities align with their own. This information is vital for governments and aid donors to be able to make the best use of scarce resources.
+
+<img class="pull-left" style="margin-right: 1em;" title="Uganda Aid" src="http://farm9.staticflickr.com/8159/7315314386_7940819de6.jpg" alt="" width="250" />
+
+Normally this overview is not available – leading to waste, overlap and inefficiency. The lack of comparable information means aid donors and recipient country governments can’t work together to coordinate their efforts or understand how donor priorities align with recipient priorities; it decreases developing country governments' ownership and undermines the potential for good governance and planning. Donors and governments need to know what others are doing - and crucially, what others are planning on doing - if they are to make sure that these resources are used most effectively. Otherwise, some sectors and areas will not receive enough funding, while others may have too many donors involved.
+
+The Uganda Aid visualisation project was a joint project between the OKFN and Publish What You Fund to combine two key types of fiscal data, revenues from aid together with spending information, and present them together in an informative way for the first time through an interactive visualisation.
+
+<ul>
+<li><a href="http://www.publishwhatyoufund.org/uganda/uganda-with-data.htm">Visit the Publish What You Fund Uganda Visualisation Project</a></li>
+<li><a href="http://www.publishwhatyoufund.org">Visit Publish What You Fund's website</a></li>
+<li><a href="http://openspending.org/ugandabudget">Uganda Budget and Aid to Uganda, 2003-2006 dataset in OpenSpending</a></li>
+<li>Coverage of the Uganda Visualisation: <a href="http://www.guardian.co.uk/global-development/poverty-matters/2011/nov/25/uganda-aid-confusion-analyse-spending?newsfeed=true">Guardian Poverty Matters Blog</a></li>
+</ul>
+
+#### IATI
+
+The International Aid Transparency Initiative is a common format for publishing aid information. 29 signatories representing 75% of global Official Development Finance have committed to reporting timely information about their aid activities in this standard format. Already, 13 signatories representing 45% of ODF have published.
+
+<img class="pull-right" style="margin-left: 1em;" title="IATI" src="http://farm9.staticflickr.com/8007/7315331330_4dafe5ea48.jpg" alt="" width="250" />
+
+IATI publishers release the data as open data feeds in a common XML format through their own websites. They then register their data with the IATI Registry - which runs on the Open Knowledge Foundation's CKAN software - making it easy for users to find this data.
+
+However, the nature of IATI as a distributed collection of raw data feeds also presents a challenge to non-technical users. The Open Knowledge Foundation worked with Publish What You Fund to transform the data into a format suitable for import into OpenSpending, where the data can be much more easily visualised and analysed.
+
+<a href="http://openspending.org/iati?_time=2011">View the IATI data on OpenSpending</a>
+
+<a name="gov-tools"></a>
+
+## Tools for governments
+
+#### Data.Gov.Uk Spend Browser
+
+<img class="pull-left" style="margin-right: 2em;" title="Reporting Tool" src="http://farm6.staticflickr.com/5523/9045231550_505140b175_n.jpg" alt="http://farm6.staticflickr.com/5523/9045231550_505140b175_n.jpg" />
+
+The OpenSpending team worked with Data.Gov.Uk to build the <a href="http://data.gov.uk/data/openspending-browse">transaction explorer</a> which forms part of the Data.Gov.Uk website.
+
+The transaction explorer allows any user to directly search the OpenSpending database for companies, departments or projects of interest and investigate how much money was spent on them.
+
+#### UK Government Spend Reporting Dashboard
+
+<img class="pull-left" style="margin-right: 2em;" title="Reporting Tool" src="http://farm9.staticflickr.com/8443/7980196066_d4aa29eb0d_z.jpg" alt="" />
+
+The OpenSpending team worked with Data.Gov.Uk to produce an automatic reporting tool to demonstrate which government departments were complying with their transparency obligations.
+
+The <a href="http://data.gov.uk/data/openspending-report/index">tool</a> lists departments registered as data publishers on data.gov.uk and details how precisely they have followed the <a href="https://www.gov.uk/government/publications/guidance-for-publishing-spend-over-25000">HM Treasury reporting guidelines</a>. It will also make the whole of the reported data available for search and analysis both on <a href="http://data.gov.uk/openspending">data.gov.uk</a> and on the OpenSpending site.
+
+<a name="gov-use-cases"></a>
+
+## Government use cases
+
+#### City of Bologna
+
+<img class="pull-left" style="margin-right: 2em;" title="Open Dati Bologna Tool" src="http://dati.comune.bologna.it/file/field/image/open_spending_pic_bologna.png" alt="http://dati.comune.bologna.it/file/field/image/open_spending_pic_bologna.png" />
+
+The open data team at the City of Bologna uses OpenSpending to visualise several years of <a href="http://openspending.org/bp_2012_entrate/views/table-of-aggregates-bilancio-di-previsione-2012-entrate">the city budget</a>.
+
+#### City of Berlin
+<img class="pull-left" style="margin-right: 2em;" title="The budget for Berlin city" src="http://stefanwehrmeyer.com/img/blog/2013/02/budget-berlin.png" alt="http://stefanwehrmeyer.com/img/blog/2013/02/budget-berlin.png" />
+
+The City of Berlin asked Open Knowledge Foundation Germany to visualise the budget for the city using OpenSpending. The result is featured on the <a href="https://imperia9.berlinonline.de/sen/finanzen/haushalt/haushaltsplan/artikel.5697.php">site for the city finances</a>.
+
+<a name="publications"></a>
+
+## Publications and Reports
+
+#### Technology for Transparent &amp; Accountable Public Finance
+<img class="pull-left" style="margin-right: 2em;" title="TTAPF" src="http://farm9.staticflickr.com/8031/8002530046_fe4354f76a_m.jpg" alt="" />
+
+The report, <a href="../../resources/gift/">“Technology for Transparent and Accountable Public Finance”</a>, was commissioned by the Global Initiative for Fiscal Transparency (GIFT) in February 2012 in order to assist GIFT in assessing the potential of technology to aid transparency and accountability in relation to governments’ fiscal activities.
+
+Read the <a href="../../resources/gift/chapter9-intro/#governments">recommendations</a>.
+
+<a href="http://content.openspending.org/resources/gift/pdf/ttapf_report_20120530.pdf">Download as PDF</a>
+
+#### Mapping the Open Spending Data Community
+<img class="pull-right" style="margin-left: 2em;" title="Spend Report" src="http://farm8.staticflickr.com/7414/8881834696_4176e6d2ea_m.jpg" alt="" />
+
+In early 2012, we set out on a mission. Our aim was to establish how CSO's currently work with spending data, how they would like to use it, and what they would like to achieve - including:
+
+<ul>
+<li>what existing tools are being used</li>
+<li>what current technical needs are unmet</li>
+<li>what would be required to meet these needs and how feasible is it to tackle them</li>
+</ul>
+This <a href="../../resources/mappingcommunity/">report</a> is the output of that research. Here, we bring together key case studies from organisations who have done pioneering work in using technology to put government data to best use.
+
+In this report we:
+
+<ul>
+<li>outline how the data could be improved in order to make it more usable</li>
+<li>examine some key case studies of how organisations are using technology to do groundbreaking research, citizen engagement, visualisation and tracking of accountability.</li>
+<li>talk about the training needs for CSO's to help them better use the data available, and to demand better data.</li>
+</ul>
+Read the <a href="../../resources/mappingcommunity/conclusions/">conclusions</a>.
+
+<a href="../../resources/mappingcommunity/introduction/videos/">Athens to Berlin research trip: Watch the video interviews</a>
+
+#### The Spending Data Handbook
+<img class="pull-left" style="margin-right: 2em;" title="Spending Handbook" src="http://farm8.staticflickr.com/7449/8754377372_77aed9107e_m.jpg" alt="" />
+
+The <a href="../../resources/handbook/">Spending Data Handbook</a> is addressed to people and organisations who want to use and understand government budgets and spending data in their work.
+
+The book covers:
+
+<ul>
+<li>Collaborating with other organizations to pool resources and strengthen your advocacy effort</li>
+<li>If you're just starting out, what data to look for and what to ask for (nay, demand!) from your government</li>
+<li>The 'Data Pipeline': Tricks and tips for finding, wrangling and systematically processing your data</li>
+<li>Getting ambitious, running a technology project</li>
+<li>Presenting your findings to engage the public, media and government</li>
+<li>Lists and appendices of technical and non-technical resources</li>
+</ul>
+
+<a name="standards"></a>
+
+## Standards
+
+#### [Draft] Standard for Transaction-level Spending Data
+The release of transaction-level data (i.e. information about individual disbursements or contract spending) is a relatively new idea, compared to the release of higher-level accounting information or budget overviews. The availability of such data allows for fine-grained analysis and oversight of activities and will, in the future, enable anyone inside or outside of government to reconstruct key reports from raw data. In order to perform these types of analysis, it is often necessary to combine spending information from several sources - either for completeness or comparison.
+
+Too often, standardization in this context appears to be supply-driven: every publisher wants to express the full range of data they hold and are willing to release. Necessarily, such an approach leads to a standard that is the superset of all the systems that feed into it.
+
+Such designs tend to be of little use to the intended consumers, as they raise the barriers to understanding the information considerably. Our approach therefore is to generate demand-side use cases first, ensuring that everything that is done will generate value for data users.
+
+Read our <a href="../../resources/standard/">proposed standard</a>.
+
+<a name="stories"></a>
+
+## Stories and Data Journalism
+
+#### The Guardian (UK)
+
+<h4>The Real Price of the London Olympics</h4>
+<a title="olympic-guardian by anderspedersenOKF, on Flickr" href="http://www.flickr.com/photos/94746900@N06/8915659698/"><img src="http://farm3.staticflickr.com/2806/8915659698_60f3b70eed_o.png" alt="olympic-guardian" /></a>
+
+The Guardian used OpenSpending's bubbletree visualisations to answer the questions: Where is the Olympics money coming from - and where's it being spent? How much is coming from sponsorship - and how much does the Olympic stadium cost?
+
+<a href="http://www.guardian.co.uk/sport/datablog/interactive/2012/jul/26/london-2012-price-olympic-games-visualised">See the interactive visualisation on the Guardian Datablog</a>.
+
+<h4>The Daily Bread</h4>
+<a href="http://www.guardian.co.uk/uk/datablog/interactive/2013/mar/20/budget-2013-how-taxes-spent-interactive">The Daily Bread</a> from Where Does My Money Go? Has made regular appearances on the Guardian Datablog around budget time in the UK.
+
+#### Le Monde (France)
+We experienced an extremely high traffic spike when a visualisation based on French data was featured in Le Monde in October 2012.
+The article <a href="http://www.lemonde.fr/politique/article/2012/10/16/plf-des-avions-au-bouclier-fiscal-la-java-des-amendements_1776093_823448.html?xtmc=depenses&amp;xtcr=52">PLF : des avions au bouclier fiscal, la java des amendements</a>, (PLF=Projet de loi de finances, the draft finance law) deals with suggested amendments to the draft finance law and which parties were demanding what amendments.
+
+The OpenSpending visualisation used in the article is intended to give a high-level representation of some of the main areas of government expenditure in France.
+
+#### Politiken (Denmark)
+<a title="politiken" href="http://farm6.staticflickr.com/5339/9044797934_366b1d8914_m.jpg"><img class="pull-right" style="margin-left: 2em;" src="http://farm6.staticflickr.com/5339/9044797934_366b1d8914_m.jpg" alt="politiken" /></a>
+
+Municipalities are central for the functioning of the welfare state Denmark. They take care of a range of important tasks like social- and health care, primary education, social benefits, traffic and much more. Even in a year with local elections, however, they do not attract much public attention. The open data web agency Buhl-Rasmussen developed a site of all 98 Danish municipalities for one of the biggest Danish news sites, Politiken.
+
+#### Sumy News, Ukraine
+<a title="politiken" href="http://sumynews.com/data-journalism2/budget-economy/item/4726-byudzhet-m-sumy-na-2013-rik-openspending.html&lt;br /&gt;
+"><img class="pull-left" style="margin-right: 1em;" src="http://farm6.staticflickr.com/5459/9045133060_9dbb04b97c_m.jpg" alt="" /></a>
+
+On December 26, 2012, Sumy City Council approved the city budget for 2013, but there were big concerns about how it would be executed.
+
+The expected revenues for the coming year did not appear to match the planned output, as Sumy would have to have generated 15.7% more than the expected revenues in 2012....
+
+<a href="http://sumynews.com/data-journalism2/budget-economy/item/4726-byudzhet-m-sumy-na-2013-rik-openspending.html">Read the article</a>.
+
+#### Privacy International
+
+In early 2012, Privacy International approached the Spending Stories team to ask for a search widget to be able to search across all of the government spending datasets held in OpenSpending. They had a list of companies which exhibited at the famous surveillance technology conferences in the US, the so-called Wiretappers' Ball, as well as a list of attendees of the conference, and they wanted to check which attendees also became customers of these companies.
+
+<img class="pull-right" style="margin-left: 1em;" title="Caelainn Barr" src="http://farm8.staticflickr.com/7213/7315271184_921d9ed606.jpg" alt="" width="250" />
+
+Some attendees posed no surprises – the FBI, the US Drug Enforcement Administration, the UK Serious Organized Crime Agency and Interpol to name a few – but there were a few that are downright baffling, like the US Department of Commerce, the US Fish &amp; Wildlife Service and the Clark County School District Police Department.
+
+As more datasets are loaded into OpenSpending, this universal search will get increasingly more powerful, and we look forward to hearing what other people use the search for.
+
+<ul>
+<li><a href="http://opendatalabs.org/spendbrowser">Try out the search for yourself in the spendbrowser</a></li>
+<li><a href="http://blog.openspending.org/2012/02/24/how-spending-stories-fact-checks-big-brother-the-wiretappers-ball/">Read more about the story</a></li>
+</ul>
+
+#### Italian Regional Accounts Data
+<img class="pull-left" style="margin-right: 1em;" title="Caelainn Barr" src="http://farm6.staticflickr.com/5224/5639223572_5451048271.jpg" alt="" width="250" />
+
+During the International Journalism Festival in Perugia in 2011, the OpenSpending team loaded and visualised the Italian regional accounts for 2008. The project received wide coverage in the Italian and International Press and was one of the earliest success stories for OpenSpending after Where Does My Money Go? went international.
+
+<a href="http://openspending.org/it-regional-accounts">Visit the Italian Regional Accounts</a> on OpenSpending.org.
+
+Coverage of the Italian Regional Accounts Data:
+
+<ul>
+<li><a href="http://www3.lastampa.it/economia/sezioni/articolo/lstp/398705/">La Stampa</a></li>
+<li><a href="http://www.guardian.co.uk/news/datablog/interactive/2011/apr/19/italy-public-spending-visualisation">The Guardian</a></li>
+<li><a href="http://daily.wired.it/news/economia/2011/04/19/open-spending.html">Daily Wired</a></li>
+</ul>
+
+## Training
+
+<img class="pull-right" style="margin-left: 3em;" title="Caelainn Barr" src="http://farm7.staticflickr.com/6166/6151919267_897ccfbd1c.jpg" alt="" width="250" />
+
+We regularly conduct training sessions for journalists and NGOs on how to locate, extract, work with and visualise spending and other types of data. If you are interested in exploring these possibilities, please [get in touch](/about/contact/).
+
+<ul>
+<li><a href="http://blog.okfn.org/2011/08/09/data-driven-journalism-workshop-on-eu-spending-tools-techniques-utrecht-8th-9th-september/">Workshop on EU Spending, Utrecht</a></li>
+<li><a href="http://blog.okfn.org/2011/09/27/eurohack-one-day-data-journalism-competition-and-workshop-on-eu-spending/">EuroHack: One-day data journalism competition and workshop on EU spending</a></li>
+<li><a href="http://blog.openspending.org/2012/11/26/day-1-openspending-cso-workshop-sarajevo/">Budget and Spending Visualisation Workshop, Sarajevo</a></li>
+</ul>
+You can find more of our training materials on the <a href="http://schoolofdata.org/handbook/courses/">School of Data website</a>.
+
+## Community
+
+<img class="pull-right" style="margin-left: 1em;" title="Kaitlin Lee presents Sunlight's Analysis OGDCamp" src="http://farm7.staticflickr.com/6166/6270108254_5875c8a7ed.jpg" alt="" width="250" />
+
+The OpenSpending community includes engaged citizens, dedicated journalists and members of civil society organisations working on developing best practices around opening up and using government financial data with experts from fields ranging from aid experts, participatory budgeting fields, governmental institutions and civic developer initiatives.
+
+We showcase and display some of the most interesting projects from all over the world on [the OpenSpending blog](/blog/).
+
+The working group is open to everyone with an interest in improving Government Financial Transparency around the world. If you're interested in joining, please join the <a href="https://discuss.okfn.org/c/openspending">forums</a>.
+
+<a name="your-project"></a>
+
+## Contribute
+
+#### Your project here
+Have some data? Have a Spending Story? Get in touch - we'd love to hear from you and are open to suggestions!
+
+

examples/openspending/content/about/faq.md

@@ -0,0 +1,43 @@
+---
+section: about
+lead: true
+title: Frequently Asked Questions
+authors:
+- Neil Ashton
+---
+#### What is OpenSpending?
+
+OpenSpending is about Mapping the Money.
+
+The aim is to help track every (public) government and corporate financial transaction across the world and present it in useful and engaging forms for everyone from a schoolchild to a data geek.
+
+What [OpenStreetMap](http://www.openstreetmap.org/) does for geography, OpenSpending does for money. OpenStreetMap has mapped the world in unprecedented levels of detail, harnessing the power of thousands of volunteers who each contribute data for their little corner of the world. However, as far as we know, there is no 'global atlas' of spending, no integrated, searchable database which would be a valuable resource for policy-makers and civil society alike. We want anyone to be able to go to their local council or national government, request the data, upload, understand and visualize it and contribute to this 'spending commons', which anyone can benefit from.
+
+#### What is the relationship to Where Does My Money Go?
+
+Simply put, OpenSpending is the international version of [Where Does My Money Go?](http://wheredoesmymoneygo.org/) (WDMMG). WDMMG used interactive 'bubble trees' to show the UK Taxpayer which spending sectors their tax money is put towards as well as having an interactive tax calculator to let users see for their level of income, how much they contribute to various spending areas through their taxes.
+
+##### Platform vs. project
+
+After the success of [the original project](http://jonathangray.org/2007/04/02/where-does-my-money-go-project-proposal/), it became clear that there was need for an international platform to allow people to build such sites at scale. Where Does My Money Go? was remodelled to become an instance of OpenSpending: the data is stored in the central OpenSpending database, and a custom frontend was developed using the OpenSpending API. Anyone can create one just like it, with their own branding at their own URL.
+
+#### Can anyone contribute?
+
+Yes!  Whether you are interested in international, national or local data, you can contribute, work with and visualize it. See [How can I get involved?](../../help/development/volunteer) for the various ways you can get involved.
+
+#### How can I get involved?
+
+The team has worked hard to make it possible to load and visualize your own data, translate the software and even build your own custom front-end sites like [Where Does My Money Go?](http://wheredoesmymoneygo.org/)
+
+Whether you are technical or non-technical, whether your aims are advocacy, journalism, better policy-making or just making pretty bubbles to help people understand your country's budget, there are plenty of ways to get involved. And don't be shy - if you spot a way to get involved that we haven't thought of, please don't hesitate to let us know.
+
+See the [get involved](../../help/development/volunteer) page for more information.
+
+#### What kind of language support do you offer?
+
+We currently provide language versions of the OpenSpending platform in Greek, Indonesian and Italian. We expect to enable French as well as other languages in the near future.
+You can help translate OpenSpending into another language at our [Transifex page](https://www.transifex.com/projects/p/openspending/).
+
+#### How are you funded?
+
+OpenSpending is a community-run project, but it would not be able to exist without the generous support of many organizations. See [this page](../funders/) for more information.

examples/openspending/content/about/fiscaldatapackage.md

@@ -0,0 +1,23 @@
+---
+title: The Fiscal Data Package
+redirect_from:
+  - "/about/governance/"
+---
+
+The Fiscal Data Package is a lightweight and user-oriented format for publishing and consuming fiscal data. Fiscal data packages are made of simple and universal components. They can be produced from ordinary spreadsheet software and used in any environment.
+
+The motivation behind the fiscal data package was to create a specification which is open by nature - based on other open standards, supported by open tools and software, modular, extensible and promoted transparently by a large community.
+
+It is designed to be simple to use - providing a small but flexible set of features, based on real-world requirements and not theoretical ones. All the while, the built-in extensibility allows this spec to adapt to many different use cases and domains. It is also possible to gradually use more and more part of this specification - thus making it easier to implement this spec with existing data while slowly improving the data quality.
+
+A main concern of this specification is the ability to work with data as it is currently exists, without forcing publishers to modify the contents or structure of their current data files in order to "adapt" them to the specification.
+
+#### The Open Fiscal Data Package serves two main purposes:
+Standardizing the structure and the content of fiscal data so that tools and services can be built over it for visualization, analysis or comparison;
+Driving data quality by providing a solid framework of publication.
+
+So the Open Fiscal Data Package specifies the form for fiscal data and offers a standardized framework for the content.
+
+- Read the [Fiscal Data Package specification](https://frictionlessdata.io/specs/fiscal-data-package/) on the Frictionless Data website
+- Meet the [OpenSpending tools](/about/tools)
+- Read about 

examples/openspending/content/about/funders.md

@@ -0,0 +1,36 @@
+---
+section: about
+lead: true
+title: Our funders
+authors:
+- Neil Ashton
+---
+OpenSpending is a community-driven project, but we would not be able to run it without the generous support of these organizations:
+
+#### 4IP
+
+<img src="http://blog.openspending.org/files/2013/09/4ip-300x79.jpg" alt="4ip" width="300" height="79" class="alignright size-medium wp-image-1056" />
+
+It was thanks to Channel 4's Information Program that [Where Does My Money Go?](http://wheredoesmymoneygo.org/) was born. 'Where Does My Money Go?' was the original version of OpenSpending for the United Kingdom and has inspired many international versions of the site.
+
+#### Open Society Foundations
+
+<img src="http://blog.openspending.org/files/2013/09/osf-300x70.jpg" alt="Basic CMYK" width="300" height="70" class="alignright size-medium wp-image-1057" />
+
+The Open Society Foundations support community building work around Open Spending Data. With this project we aim to help more groups and individuals around the world to use and work with spending data more effectively to do the things they care about – whether this is investigative journalism, evidence based policy-making, political campaigning, budgeting or creating new useful applications and services.
+
+In particular, we would like to document and spread best practices in the legal and technical aspects of reusing public information, and enabling re-use and better collaboration around this material.
+
+Read [the announce post]( http://blog.openspending.org/2012/01/12/civil-society-and-spending-data-who-is-mapping-the-money/) for more information.
+
+#### Knight Foundation
+
+<img src="http://blog.openspending.org/files/2013/09/kf-300x71.png" alt="kf" width="300" height="71" class="alignright size-medium wp-image-1058" />
+
+The Knight Foundation support the development of the [Spending Stories](http://blog.okfn.org/2011/06/22/spending-stories-is-a-winner-of-the-knight-news-challenge/) project. Spending Stories is a tool designed to speed up fact checking around spending data, helping journalists to connect raw data with their stories and build context around data with visualisations and auxilliary information.
+
+#### Omidyar Network
+
+<img src="http://blog.openspending.org/files/2013/09/on-300x100.jpg" alt="on" width="300" height="100" class="alignright size-medium wp-image-1059" />
+
+The Omidyar Network funded the research behind the report 'Technology for Transparent and Accountable Public Finance', which was presented at the Global Initiative for Fiscal Transparency meeting in Brasilia in April 2012.

examples/openspending/content/about/index.md

@@ -0,0 +1,19 @@
+---
+title: About OpenSpending
+redirect_from:
+  - "/about/governance/"
+---
+### What is OpenSpending?
+OpenSpending is a free, open and global platform to search, visualise and analyse fiscal data in the public sphere. With OpenSpending, Open Knowledge Foundation created the opportunity for governments, civil society organizations and communities to publish and visualize their revenue, budgets, spending and procurements data in an open source platform.
+
+The OpenSpending platform consists of a core platform with a large, centralised database that allows for deep analysis across a range of datasets. It is designed, developed and maintained by Open Knowledge Foundation. At the same time, OpenSpending offers tools that enable to establish an ecosystem around fiscal data, which is tailored to the specific aspects and local contexts the data is embedded in.
+
+As an open source and a community-driven project, also reflects the valuable contributions of an active, passionate and committed community.
+
+### Got it, what now?
+
+- Meet the [OpenSpending tools](/about/tools)
+- Learn about the [Fiscal Data Package](/about/fiscaldatapackage)
+- Read other resources about Fiscal Transparency on the [GIFT website](http://www.fiscaltransparency.net/resources/)
+- Read the [OpenSpending documentation](https://docs.openspending.org/en/latest/)
+- Learn more about [this website](/about/meta)

examples/openspending/content/about/meta/changes/index.md

@@ -0,0 +1,13 @@
+---
+title: Recent Changes
+githubactivity: true
+section: meta
+---
+
+A list of the most recent changes or issues raised on the site as
+documented on the site's
+[GitHub repository](https://github.com/openspending/community.openspending.org/).
+
+
+
+<div id="github-activity" style="width: 100%"></div>

examples/openspending/content/about/meta/contribute/add-resource.html

@@ -0,0 +1,48 @@
+---
+title: Adding a resource to the library
+authors:
+ - Mor Rubinstein
+lang: en
+section: meta
+---
+
+<p>Our resource library is a curated collection open data reosurces from across the community. Everyone can add a resource to the library. This is how to do so.</p>
+
+<h2>1. Add a folder</h2>
+
+<p>Log in to github and head to the following <a href=https://github.com https://github.com/{{ site.github_username }}/tree/gh-pages/resources> link: </a></p>
+
+Every resource has a slug number. To add your resource, you need to give it a number. Look at the list and give you resource the number that follows  the current last number in the list (e.g - if the number is 056, your resource should be named 057).
+Click on the + sign in the directory line and add the number and a “/”. For example: 060/.
+
+<h2>2. Add an index file</h2>
+
+<p>Click on the + sign again. Add a file named index.md to your new folder.</p>
+
+<h2>3. Add the resource</h2>
+
+<p>In the text editor, add the front matter fields in this pattern:</p>
+<pre>
+---
+section: resources
+lang: //Two first letters of the language, according to language code in this table.//
+Author: //The name(s) of the person(s) who wrote the text//
+Country: //One or more country by full name  separated with a comma:  “,”. If there is no specific country, write ‘global’//
+Description: //1-5 lines that summarizes the text. //
+Keywords: //Important descriptors of the text, separated with a comma, “,”.//
+Link: //The link to the resource online//
+MediaType: // List one out of these four types: Presentation, Article,  / Publication, Video//
+Notes: //Any notes or comments.//'
+Publishing_date: //The year the resource was published, e.g. 2015.//
+Publishing_entity: //The organisation(s) which publish the resource//
+Region: North America,Latin America,Asia,Europe,Africa,Mena,Global
+Title: //The name of the resource//
+Topic: //Choose one out of these nine : The Basics,Advocacy,Privacy,civic engagement,Right for information,Data training,PolicyStandards.
+---
+</pre>
+
+<h2>4. Make a pull request</h2>
+
+<p>Click on “Create a new branch for this commit and start a pull request.” This will allow us to review your changes.</p>
+
+Thank you! All done!

examples/openspending/content/about/meta/contribute/adding-terms.html

@@ -0,0 +1,47 @@
+---
+lang: en
+title: Adding a term to the glossary
+authors:
+ - Mor Rubinstein
+section: meta
+---
+<p>Each glossary (meaning, each translated instance of the glossary), has three components:
+<ul>
+<li> A layout template for the glossary homepage: 'glossary.html'</li>
+<li> A layout template for the glossary in each language: 'glossary/{lang}/index.md'</li>
+<li> A directory of the glossary terms. Each term in the directory is listed as the url slug (In English, all lower case letters, and hyphens instead of white spaces).</li>
+</ul>
+
+<p></p>Currently, the English glossary has been updated and organized. Other languages please follow these <a href=http://new.opendatahandbook.org/contribute/translate-glossary/>instructions</a>
+To add a new term, all you need is to have a Github account.</p>
+
+<h3>1: Create a folder for the term</h3>
+
+<p>Log - in to Github and go to this <a href="https://github.com/{{ site.github_repo }}tree/gh-pages/glossary/en/terms">link</a></p>
+
+<p>You will see a the branch name (“gh-pages”) and a directory. You will also see the breadcrumb <code>{{ site.github_repo }} / glossary / en / terms / + </code>. Click on the "+" to create a new folder.<p/>
+
+<p>  Write the name of the term that you want to add in a new slug (In English, all lower case letters, and hyphens instead of white spaces) and add a “/” at the end of the terms name. This will create a new folder name. </p>
+
+<h3>2: Create a file for your term</h3>
+
+<p> Now you will see the breadcrumb - <code>{{ site.github_repo }} / glossary / en / terms / your-new-term / +  </code> Click on the “+” sign again. Now write in the path “index.md”. This will save the whole file as a markdown file.  </p>
+
+<h3>3: Write the term definition </h3>
+<p> In  the text editor below, add the front matter (Jekyll way to mark the page) - </p>
+<pre>
+---
+section: terms
+lang: en
+title: // the term name //
+---
+</pre>
+
+Write the term definition after the front matter as usual.
+
+
+<h3>4: Make a pull request</h3>
+
+<p> Click on “Create a new branch for this commit and start a pull request.” This will allow us to review your changes. <p/>
+
+<p>Thank you! All done! If the handbook editors are happy with your term, it will be added to the glossary. </p>

examples/openspending/content/about/meta/contribute/adding.html

@@ -0,0 +1,126 @@
+---
+lang: en
+title: Adding a page
+authors:
+ - Sam Smith
+section: meta
+---
+
+<p class="lead">Most of what you need to know to add a new page is covered under ‘<a href="{{ "/meta/contribute/editing/" | prepend: site.baseurl }}">Editing a page</a>’, so make sure you have first read through that section.</p>
+
+<p>Adding a page follows a similar process to editing. Instead of locating your page, your first step is to locate your new page’s parent directory.</p>
+
+<h2>1: Create the page</h2>
+
+<p>First locate the parent directory, or folder, in which your file will live. If you are adding a page to the "Get Involved" section, for example, you should be in the '<strong>get-involved</strong>' directory, where you’ll see a list of all the pages related to contributions.</p>
+
+<p>Above the list of pages is a breadcrumb trail <code>{{ site.github_repo }} / get-involved / </code>, to the right of which is a plus symbol <code><strong>+</strong></code>. Click the plus symbol to create your new page.</p>
+
+<h2>2: Name your file</h2>
+
+<p>You should now have a new file editor page open. The first editable section you are presented with is the ‘<em>Name your file</em>’ field. As we touched upon when looking at <a href="{{ "/meta/contribute/editing/" | prepend: site.baseurl }}">editing files</a>, the file name corresponds to the URL of the page on the site. There are a few of rules to follow here:</p>
+
+<ul>
+  <li>The file name should reflect the title of the new page</li>
+  <li>Must be unique</li>
+  <li>Should be all lowercase</li>
+  <li>Words should be separated by hyphens (-)</li>
+  <li>File name should end with the extension '<em>.md</em>' (the .md extension indicates a markdown file)</li>
+</ul>
+
+<p>As an example, if you were creating a page titled '<em>My Cool Page</em>', you would use a file name of:</p>
+
+<pre>
+<code>my-cool-page.md</code>
+</pre>
+
+<p>Assuming you are creating this in the '<em>get-involved</em>' directory, this would result in a URL of <em>{{ site.url }}/get-involved/my-cool-page</em></p>
+
+<div class="note">
+  <h6>Note</h6>
+  <p>The actual words used in your file name are not crucial. It’s fine to use a more succinct version of your page title, for example.</p>
+</div>
+
+<h2>3: Formatting your content</h2>
+
+<p>This step is the same as when editing a page. You need to start your file with the Front Matter, then add your content, formatting it using Markdown. Here is a template to get you started:</p>
+
+<pre>
+<code>---
+title: My Cool Page
+authors:
+ - Fred Bloggs
+---
+
+##A large introductory paragraph.
+
+Regular paragraphs, separated by empty lines.
+
+###A heading
+
+Another paragraph.
+
+ * Maybe
+ * a
+ * list</code>
+</pre>
+
+<p>When you’re done, click ‘<em>Propose new file</em>’.</p>
+
+<h2>4: Make a pull request</h2>
+
+<p>Once you have created your page(s) and updated the contents document, you're ready to make your pull request. Click the pull request icon to the right of the screen <code class="icon-git-pull-request"><span>[git pull-request icon]</span></code>, then click ‘<em>New pull request</em>’.</p>
+
+<p>At the top of the resulting comparison screen, you’ll see a row of select boxes. You want to make sure these are configured like so:</p>
+
+</article>
+
+<div class="github panel">
+  <div class="range-editor">
+    <span class="icon-git-compare range-editor-icon"></span>
+    <div class="range">
+
+      <div class="range-cross-repo-pair">
+        <div class="select-menu js-menu-container js-select-menu fork-suggester">
+          <span class="minibutton select-menu-button js-menu-target" role="button" aria-label="Choose a Base Repository" aria-haspopup="true">
+            <i>base fork:</i>
+            <span class="js-select-button css-truncate css-truncate-target" title="base: ckan/ckan">{{ site.github_username }}/{{ site.github_repo }}</span>
+          </span>
+        </div>
+
+        <div class="select-menu js-menu-container js-select-menu commitish-suggester">
+          <span class="minibutton select-menu-button js-menu-target branch" role="button" aria-label="Choose a base branch" aria-haspopup="true">
+            <i>base:</i>
+            <span class="js-select-button css-truncate css-truncate-target" title="base: master">gh-pages</span>
+          </span>
+        </div>
+      </div>
+
+      <span class="dots">...</span>
+
+      <div class="range-cross-repo-pair">
+        <div class="select-menu js-menu-container js-select-menu fork-suggester">
+          <span class="minibutton select-menu-button js-menu-target" role="button" aria-label="Choose a Head Repository" aria-haspopup="true">
+            <i>head fork:</i>
+            <span class="js-select-button css-truncate css-truncate-target" title="head: mintcanary/ckan"><em>username</em>/{{ site.github_repo }}</span>
+          </span>
+        </div>
+
+        <div class="select-menu js-menu-container js-select-menu commitish-suggester">
+          <span class="minibutton select-menu-button js-menu-target branch" role="button" aria-label="Choose a head branch" aria-haspopup="true">
+            <i>compare:</i>
+            <span class="js-select-button css-truncate css-truncate-target" title="compare: master"><em>branch</em></span>
+          </span>
+        </div>
+
+      </div>
+    </div>
+  </div>
+</div>
+
+<article class="post-content">
+<p><em><strong>username</strong> being your github username, <strong>branch</strong> being the branch you have been working on.</em></p>
+
+<p>You should now be able to see listed below, all the changes that you wish to contribute. If everything looks as it should, click ‘<em>Create pull request</em>’.</p>
+
+<p>Give your pull request a title and description, then click ‘<em>Create pul request</em>’. You have now contributed your pages to the OpenSpending Community site :)</p>

examples/openspending/content/about/meta/contribute/editing.html

@@ -0,0 +1,103 @@
+---
+lang: en
+title: Editing a page
+authors:
+ - Sam Smith
+section: meta
+---
+
+<p class="lead">If you haven’t done so already, the first thing you need to do is head over to <a href="https://github.com/" rel="external">Github</a> and create your free account.</p>
+
+<p>There are three steps to editing a page. First you need to locate the page you wish to edit. There are a couple of ways to do this. <strong>Method A</strong> is probably the simplest, and most likely way you’ll do it. <strong>Method B</strong> will serve as a primer for the next section: <em>Adding a page</em>.</p>
+
+<h2>1: Locate the page</h2>
+
+<h4>Method A: Browse the website</h4>
+
+<p>While reading any section of the Handbook you’ll see an '<em>Edit this page</em>' link in the bottom left of the page. Following this link will take you directly to an editable version of that page. Easy huh?</p>
+
+<div class="note">
+  <h6>Note</h6>
+  <p>When the editable page opens it will (most likely) contain a message saying <em>“You're editing a file in a project you don't have write access to”</em>. If this is your first edit to The Open Data Handbook it will say <em>“We've created a fork of this project for you to commit your proposed changes to”</em>. This is normal and part of the workflow.</p>
+</div>
+
+<h4>Method B: Browse the Github repository</h4>
+
+<p>The entire file structure of this site can be browsed on Github. For example, the root of the site is <a href="https://github.com/{{ site.github_username }}/{{ site.github_repo }}" rel="external">here</a>, and the English language handbook section is <a href="https://github.com/{{ site.github_username }}/{{ site.github_repo }}/tree/gh-pages/en" rel="external">here</a>. It’s helpful to understand that the page URLs correspond to the file structure you see here. So, if you wanted to edit the Handbook introduction page, given that it’s URL is <code>{{ site.url }}/<strong>en</strong>/<strong>introduction</strong>/</code> we know this file can be found in the <code><strong>en</strong></code> directory with the filename <code><strong>introduction</strong>.md</code> <em>Note: the extension (.md) is stripped from the URL.</em> Following these links you should see a preview of the page you wish to edit. From here click the edit icon <code class="icon-pencil"><span>[pencil icon]</span></code> to start editing.</p>
+
+<div class="note">
+  <h6>Pro Tip!</h6>
+  <p>Press <code>t</code> on any tree or blob page to launch the file finder.</p>
+</div>
+
+<h2>2: Make your changes</h2>
+
+With the editable content in front of you, you’re probably either thinking “great, let’s get editing”, or “hang on, this looks a bit weird”. In case it’s the latter, let’s have a closer look.
+
+The first thing to recognise is the ‘Front Matter’, which will look like this:
+
+<pre>
+<code>---
+title: Introduction
+---</code>
+</pre>
+
+<p>The front matter must be the first thing in the file, must adhere to the above syntax, and must be set between triple-dashed lines. Numerous variables can be set here, but you’ll usually just need <code>title</code>. The title set here will be used as the main heading for the page, as well as in the browser tab.</p>
+
+<p>The other important thing to recognise is the Markdown syntax. For example, where you see a line commencing with two hash marks:</p>
+
+<pre>
+<code>##Do you know exactly how much of your tax money is spent on street lights?</code>
+</pre>
+
+<p>This is the Markdown way of creating a level two heading. On the site it will be outputted like so:</p>
+
+<h2>Do you know exactly how much of your tax money is spent on street lights?</h2>
+
+<p>Another common formatting requirement is bullet points, or lists. These are achieved in Markdown by using asterisks, like so:</p>
+
+<pre>
+<code>* civil servants
+* journalists
+* politicians</code>
+</pre>
+
+<p>giving you:</p>
+
+<ul>
+  <li>civil servants</li>
+  <li>journalists</li>
+  <li>politicians</li>
+</ul>
+<br>
+<p>Links are created like so:</p>
+
+<pre>
+<code>Give your data a home at the [Datahub](http://datahub.io/).</code>
+</pre>
+
+<p>result:</p>
+
+<p>Give your data a home at the <a href="http://datahub.io/">Datahub</a>.</p>
+
+<div class="note">
+  <h6>Pro Tip!</h6>
+  <p>To get a link to a specific heading on this site, hover over it then click the section icon <code class="icon-section"><span>[section icon]</span></code>. This will put the URL into your address bar.</p>
+</div>
+
+<p>More Markdown examples can be found <a href="{{ "/meta/contribute/markdown-examples/" | prepend: site.baseurl }}">here</a>, and a more detailed overview <a href="http://daringfireball.net/projects/markdown/syntax" rel="external">here</a>.</p>
+
+<p>If you are unsure of your markup while editing, you can switch to the preview tab <code class="icon-eye"><span>[eye icon]</span> Preview changes</code> to see how it will be rendered.</p>
+
+<div class="note">
+  <h6>Note</h6>
+  <p>The Github previews will look stylistically different from the live site. A different font will be used for example.</p>
+</div>
+
+<p>Once you are happy with your changes, add a summary of what you've changed in the field below the editable text. Then click ‘<em>Propose file change</em>’.</p>
+
+<h2>3: Make a pull request</h2>
+
+<p>You will now be presented with a ‘pull request’ form. So far, the changes you have made are to your own copy, or fork of the handbook. A pull request simply sends a request to the authors/maintainers of the live handbook, asking them to include your changes - and put them live! Add any comments you have for the handbook team, then press ‘<em>Create pull request</em>’.</p>
+
+<p>Your work here is done :) If you need to make related changes though, any new commits pushed to your branch will automatically be added to the pull request.</p>