knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #642 from datopian/fix/dataset-loading

Browse Source 
Fix/dataset loading
This commit is contained in:
Anuar Ustayev 2022-02-01 10:26:19 +06:00 committed by GitHub
commit 02dd3dd2da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 89 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
examples/dataset-frictionless/lib/dataset.js
View File

@ -5,28 +5,66 @@ import toArray from 'stream-to-array'
export async function getDataset(directory) {
// get dataset descriptor and resources
const f11sDataset = await Dataset.load(directory)
const descriptor = f11sDataset.descriptor
try {
const resources = await Promise.all(f11sDataset.resources.map(async (resource) => {
let _tmp = resource.descriptor
let rowStream = await resource.rows({ keyed: true })
_tmp.sample = await toArray(rowStream)
_tmp.size = resource.size
return _tmp
}))
const readme = descriptor.readme
const processed = await remark()
.use(html)
.process(readme)
if (!directory) {
throw new Error('No directory provided.')
}
const readmeHtml = processed.toString()
const dataset = {
readme: readme,
readmeHtml: readmeHtml,
descriptor: descriptor,
resources: resources
const f11sDataset = await Dataset.load(directory)
const descriptor = f11sDataset.descriptor
const resources = await Promise.all(f11sDataset.resources.map(async (resource) => {
let _tmp = resource.descriptor
let rowStream = await resource.rows({ keyed: true })
_tmp.sample = await toArray(rowStream)
_tmp.size = resource.size
return _tmp
}))
const readme = descriptor.readme || ""
const processed = await remark()
.use(html)
.process(readme)
const readmeHtml = processed.toString()
const dataset = {
readme: readme,
readmeHtml: readmeHtml,
descriptor: descriptor,
resources: resources,
hasError: false,
errorMsg: ""
}
return dataset
} catch (err) {
console.log(err)
return {
hasError: true,
errorMsg: errorMessageMappings[err.message] || err.message
}
}
return dataset
}
const errorMessageMappings = {
"No datapackage.json at destination.":
`
<div>
<p style="color:red;"><b>No datapackage.json file in the data directory!</b></p>
<p >You need to add a datapackage.json file describing your dataset to the root folder.</p>
<p>For more information, see <a style="color:blue;" href="https://specs.frictionlessdata.io/tabular-data-package/#example">the documentation</a></p>
</div>
`,
"No directory provided.":
`
<div>
<p style="color:red;"><b>No data directory found!</b></p>
<p >You need to provide a data directory with CSV data, datapackage.json, and optionally a ReadMe file.</p>
<p>For more information, see <a style="color:blue;" href="https://specs.frictionlessdata.io/tabular-data-package/#example">the documentation</a></p>
</div>
`,
}

45
examples/dataset-frictionless/pages/index.js
View File

@ -14,7 +14,7 @@ const datasetsDirectory = process.env.PORTAL_DATASET_PATH || path.join(process.c
export default function Home({ dataset, specs }) {
if (!dataset) {
if (!dataset || dataset.hasError) {
return (
<div className="container">
<Head>
@ -23,8 +23,10 @@ export default function Home({ dataset, specs }) {
<link rel="preconnect" href="https://fonts.gstatic.com" />
<link href="https://fonts.googleapis.com/css2?family=Inconsolata&display=swap" rel="stylesheet" />
</Head>
<h1 data-testid="datasetTitle" className="text-3xl font-bold mb-8">
No dataset found in path
<h1
data-testid="datasetTitle"
className="m-10 text-center"
dangerouslySetInnerHTML={{ __html: dataset.errorMsg }}>
</h1>
</div>
)
@ -48,11 +50,11 @@ export default function Home({ dataset, specs }) {
</Head>
<section name="key-info">
<section name="key-info">
<KeyInfo descriptor={descriptor} resources={resources} />
</section>
<section name="file-list">
<section name="file-list">
<ResourceInfo resources={resources} />
</section>
@ -85,10 +87,18 @@ export default function Home({ dataset, specs }) {
)}
</section>
<section className="m-8" name="sample-table">
<h1 className="text-2xl font-bold mb-4">README</h1>
<ReadMe readme={dataset.readmeHtml} />
</section>
{
dataset.readmeHtml != "" ? (
<section className="m-8" name="sample-table">
<h1 className="text-2xl font-bold mb-4">README</h1>
<ReadMe readme={dataset.readmeHtml} />
</section>
) : (
""
)
}
</div>
)
@ -96,12 +106,17 @@ export default function Home({ dataset, specs }) {
export async function getStaticProps() {
if (!datasetsDirectory) {
return { props: {} }
const dataset = await getDataset(datasetsDirectory)
if (dataset.hasError == true) {
return {
props: {
dataset
}
}
} else {
const datasetWithViews = addView(dataset)
return datasetWithViews
}
const dataset = await getDataset(datasetsDirectory)
const datasetWithViews = addView(dataset)
return datasetWithViews
}