knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #642 from datopian/fix/dataset-loading

Browse Source 
Fix/dataset loading
This commit is contained in:
Anuar Ustayev
2022-02-01 10:26:19 +06:00
committed by GitHub
parent 3cd6e2aad7 3ac0ee4653
commit 02dd3dd2da
2 changed files with 89 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
examples/dataset-frictionless/lib/dataset.js
View File

@@ -5,28 +5,66 @@ import toArray from 'stream-to-array'
export async function getDataset(directory) { export async function getDataset(directory) {
// get dataset descriptor and resources try {
const f11sDataset = await Dataset.load(directory)
const descriptor = f11sDataset.descriptor
const resources = await Promise.all(f11sDataset.resources.map(async (resource) => { if (!directory) {
let _tmp = resource.descriptor throw new Error('No directory provided.')
let rowStream = await resource.rows({ keyed: true }) }
_tmp.sample = await toArray(rowStream)
_tmp.size = resource.size
return _tmp
}))
const readme = descriptor.readme
const processed = await remark()
.use(html)
.process(readme)
const readmeHtml = processed.toString() const f11sDataset = await Dataset.load(directory)
const dataset = { const descriptor = f11sDataset.descriptor
readme: readme,
readmeHtml: readmeHtml, const resources = await Promise.all(f11sDataset.resources.map(async (resource) => {
descriptor: descriptor, let _tmp = resource.descriptor
resources: resources let rowStream = await resource.rows({ keyed: true })
_tmp.sample = await toArray(rowStream)
_tmp.size = resource.size
return _tmp
}))
const readme = descriptor.readme || ""
const processed = await remark()
.use(html)
.process(readme)
const readmeHtml = processed.toString()
const dataset = {
readme: readme,
readmeHtml: readmeHtml,
descriptor: descriptor,
resources: resources,
hasError: false,
errorMsg: ""
}
return dataset
} catch (err) {
console.log(err)
return {
hasError: true,
errorMsg: errorMessageMappings[err.message] || err.message
}
} }
return dataset
} }
const errorMessageMappings = {
"No datapackage.json at destination.":
`
<div>
<p style="color:red;"><b>No datapackage.json file in the data directory!</b></p>
<p >You need to add a datapackage.json file describing your dataset to the root folder.</p>
<p>For more information, see <a style="color:blue;" href="https://specs.frictionlessdata.io/tabular-data-package/#example">the documentation</a></p>
</div>
`,
"No directory provided.":
`
<div>
<p style="color:red;"><b>No data directory found!</b></p>
<p >You need to provide a data directory with CSV data, datapackage.json, and optionally a ReadMe file.</p>
<p>For more information, see <a style="color:blue;" href="https://specs.frictionlessdata.io/tabular-data-package/#example">the documentation</a></p>
</div>
`,
}

45
examples/dataset-frictionless/pages/index.js
View File

@@ -14,7 +14,7 @@ const datasetsDirectory = process.env.PORTAL_DATASET_PATH || path.join(process.c
export default function Home({ dataset, specs }) { export default function Home({ dataset, specs }) {
if (!dataset) { if (!dataset || dataset.hasError) {
return ( return (
<div className="container"> <div className="container">
<Head> <Head>
@@ -23,8 +23,10 @@ export default function Home({ dataset, specs }) {
<link rel="preconnect" href="https://fonts.gstatic.com" /> <link rel="preconnect" href="https://fonts.gstatic.com" />
<link href="https://fonts.googleapis.com/css2?family=Inconsolata&display=swap" rel="stylesheet" /> <link href="https://fonts.googleapis.com/css2?family=Inconsolata&display=swap" rel="stylesheet" />
</Head> </Head>
<h1 data-testid="datasetTitle" className="text-3xl font-bold mb-8"> <h1
No dataset found in path data-testid="datasetTitle"
className="m-10 text-center"
dangerouslySetInnerHTML={{ __html: dataset.errorMsg }}>
</h1> </h1>
</div> </div>
) )
@@ -48,11 +50,11 @@ export default function Home({ dataset, specs }) {
</Head> </Head>
<section name="key-info"> <section name="key-info">
<KeyInfo descriptor={descriptor} resources={resources} /> <KeyInfo descriptor={descriptor} resources={resources} />
</section> </section>
<section name="file-list"> <section name="file-list">
<ResourceInfo resources={resources} /> <ResourceInfo resources={resources} />
</section> </section>
@@ -85,10 +87,18 @@ export default function Home({ dataset, specs }) {
)} )}
</section> </section>
<section className="m-8" name="sample-table"> {
<h1 className="text-2xl font-bold mb-4">README</h1> dataset.readmeHtml != "" ? (
<ReadMe readme={dataset.readmeHtml} /> <section className="m-8" name="sample-table">
</section>
<h1 className="text-2xl font-bold mb-4">README</h1>
<ReadMe readme={dataset.readmeHtml} />
</section>
) : (
""
)
}
</div> </div>
) )
@@ -96,12 +106,17 @@ export default function Home({ dataset, specs }) {
export async function getStaticProps() { export async function getStaticProps() {
if (!datasetsDirectory) { const dataset = await getDataset(datasetsDirectory)
return { props: {} }
if (dataset.hasError == true) {
return {
props: {
dataset
}
}
} else {
const datasetWithViews = addView(dataset)
return datasetWithViews
} }
const dataset = await getDataset(datasetsDirectory)
const datasetWithViews = addView(dataset)
return datasetWithViews
} }