[#819,xlsx][m]: remove data-literate, excel and everything related to the xlsx dependency (#845)

This commit is contained in:
João Demenech 2023-05-04 13:42:13 -03:00 committed by GitHub
parent ad52721a38
commit 90b93e6819
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 2 additions and 646 deletions

View File

@ -1,186 +0,0 @@
import XLSX from 'xlsx';
import React from 'react';
function SheetJSApp() {
const [data, setData] = React.useState([]);
const [cols, setCols] = React.useState([]);
const handleFile = (file) => {
const reader = new FileReader();
const rABS = !!reader.readAsBinaryString;
reader.onload = (e) => {
/* Parse data */
const bstr = e.target.result;
const wb = XLSX.read(bstr, {type:rABS ? 'binary' : 'array'});
displayWorkbook(wb);
};
if(rABS) reader.readAsBinaryString(file); else reader.readAsArrayBuffer(file);
}
const handleUrl = (url) => {
let oReq = new XMLHttpRequest();
oReq.open("GET", url, true);
oReq.responseType = "arraybuffer";
oReq.onload = function (e) {
let arraybuffer = oReq.response;
/* not responseText!! */
/* convert data to binary string */
let data = new Uint8Array(arraybuffer);
let arr = new Array();
for (let i = 0; i != data.length; ++i) arr[i] = String.fromCharCode(data[i]);
let bstr = arr.join("");
/* Call XLSX */
let workbook = XLSX.read(bstr, {type: "binary"});
displayWorkbook(workbook);
};
oReq.send();
}
const displayWorkbook = (wb) => {
/* Get first worksheet */
const wsname = wb.SheetNames[0];
const ws = wb.Sheets[wsname];
/t Convert array of arrays */
const data = XLSX.utils.sheet_to_json(ws, {header:1});
/* Update state */
setData(data);
setCols(make_cols(ws['!ref']));
}
return (
<div>
<DragDropFile handleFile={handleFile}>
<h2>Drag or choose a spreadsheet file</h2>
<div className="">
<DataInput handleFile={handleFile} />
</div>
</DragDropFile>
<div className="mb-6">
<h3>Enter spreadsheet URL</h3>
<UrlInput handleUrl={handleUrl} />
</div>
<div className="row">
<OutTable data={data} cols={cols} />
</div>
</div>
);
}
if(typeof module !== 'undefined') module.exports = SheetJSApp
/* -------------------------------------------------------------------------- */
/*
Simple HTML5 file drag-and-drop wrapper
usage: <DragDropFile handleFile={handleFile}>...</DragDropFile>
handleFile(file:File):void;
*/
function DragDropFile({ handleFile, children }) {
const suppress = (e) => { e.stopPropagation(); e.preventDefault(); };
const handleDrop = (e) => { e.stopPropagation(); e.preventDefault();
const files = e.dataTransfer.files;
if(files && files[0]) handleFile(files[0]);
};
return (
<div
onDrop={handleDrop}
onDragEnter={suppress}
onDragOver={suppress}
>
{children}
</div>
);
}
function UrlInput({ handleUrl }) {
const handleChange = (e) => {
const url = e.target.value;
if(url) handleUrl(url);
};
return (
<form className="form-inline">
<div className="form-group">
<label htmlFor="url">Spreadsheet URL (with CORS enabled!)</label>
<br />
<small>Here is one: http://localhost:3000/_files/eight-centuries-of-global-real-interest-rates-r-g-and-the-suprasecular-decline-1311-2018-data.xlsx</small>
<br />
<input
type="text"
id="url"
className="border w-96"
accept={SheetJSFT}
onChange={handleChange}
/>
</div>
</form>
)
}
/*
Simple HTML5 file input wrapper
usage: <DataInput handleFile={callback} />
handleFile(file:File):void;
*/
function DataInput({ handleFile }) {
const handleChange = (e) => {
const files = e.target.files;
if(files && files[0]) handleFile(files[0]);
};
return (
<form className="form-inline">
<div className="form-group">
<label htmlFor="file">Select spreadsheet file</label>
<br />
<input
type="file"
className="form-control"
id="file"
accept={SheetJSFT}
onChange={handleChange}
/>
</div>
</form>
)
}
/*
Simple HTML Table
usage: <OutTable data={data} cols={cols} />
data:Array<Array<any> >;
cols:Array<{name:string, key:number|string}>;
*/
function OutTable({ data, cols }) {
return (
<div className="table-responsive">
<table className="table table-striped">
<thead>
<tr>{cols.map((c) => <th key={c.key}>{c.name}</th>)}</tr>
</thead>
<tbody>
{data.map((r,i) => <tr key={i}>
{cols.map(c => <td key={c.key}>{ r[c.key] }</td>)}
</tr>)}
</tbody>
</table>
</div>
);
}
/* list of supported file types */
const SheetJSFT = [
"xlsx", "xlsb", "xlsm", "xls", "xml", "csv", "txt", "ods", "fods", "uos", "sylk", "dif", "dbf", "prn", "qpw", "123", "wb*", "wq*", "html", "htm"
].map(x => `.${x}`).join(",");
/* generate an array of column objects */
const make_cols = refstr => {
let o = [], C = XLSX.utils.decode_range(refstr).e.c + 1;
for(var i = 0; i < C; ++i) o[i] = {name:XLSX.utils.encode_col(i), key:i}
return o;
};

View File

@ -1,46 +0,0 @@
import Layout from '../Layout'
import { MDXRemote } from 'next-mdx-remote'
import dynamic from 'next/dynamic'
import Head from 'next/head'
import CustomLink from '../CustomLink'
import { Vega, VegaLite } from 'react-vega'
// Custom components/renderers to pass to MDX.
// Since the MDX files aren't loaded by webpack, they have no knowledge of how
// to handle import statements. Instead, you must include components in scope
// here.
const components = {
a: CustomLink,
Table: dynamic(() => import('./Table')),
Excel: dynamic(() => import('./Excel')),
// TODO: try and make these dynamic ...
Vega: Vega,
VegaLite: VegaLite,
LineChart: dynamic(() => import('./LineChart')),
Head,
}
export default function DataLiterate({ source, frontMatter }) {
return (
<Layout title={frontMatter.title}>
<div className="prose mx-auto">
<header>
<div className="mb-6">
<h1>{frontMatter.title}</h1>
{frontMatter.author && (
<div className="-mt-6"><p className="opacity-60 pl-1">{frontMatter.author}</p></div>
)}
{frontMatter.description && (
<p className="description">{frontMatter.description}</p>
)}
</div>
</header>
<main>
<MDXRemote {...source} components={components} />
</main>
</div>
</Layout>
)
}

View File

@ -1,74 +0,0 @@
import axios from 'axios'
import XLSX from 'xlsx'
import React, { useEffect, useState } from 'react'
import Table from './Table'
export default function Excel ({ src='' }) {
const [data, setData] = React.useState([])
const [cols, setCols] = React.useState([])
const [workbook, setWorkbook] = React.useState(null)
const [error, setError] = React.useState('')
const [hasMounted, setHasMounted] = React.useState(0)
// so this is here so we re-render this in the browser
// and not just when we build the page statically in nextjs
useEffect(() => {
if (hasMounted==0) {
handleUrl(src)
}
setHasMounted(1)
})
function handleUrl(url) {
// if url is external may have CORS issue so we proxy it ...
if (url.startsWith('http')) {
const PROXY_URL = window.location.origin + '/api/proxy'
url = PROXY_URL + '?url=' + encodeURIComponent(url)
}
axios.get(url, {
responseType: 'arraybuffer'
}).then((res) => {
let out = new Uint8Array(res.data)
let workbook = XLSX.read(out, {type: "array"})
// Get first worksheet
const wsname = workbook.SheetNames[0]
const ws = workbook.Sheets[wsname]
// Convert array of arrays
const datatmp = XLSX.utils.sheet_to_json(ws, {header:1})
const colstmp = make_cols(ws['!ref'])
setData(datatmp)
setCols(colstmp)
setWorkbook(workbook)
}).catch((e) => {
setError(e.message)
})
}
return (
<>
{error &&
<div>
There was an error loading the excel file at {src}:
<p>{error}</p>
</div>
}
{workbook &&
<ul>
{workbook.SheetNames.map((value, index) => {
return <li key={index}>{value}</li>
})}
</ul>
}
<Table data={data} cols={cols} />
</>
)
}
/* generate an array of column objects */
const make_cols = refstr => {
let o = [], C = XLSX.utils.decode_range(refstr).e.c + 1
for(var i = 0; i < C; ++i) o[i] = {name:XLSX.utils.encode_col(i), key:i}
return o
}

View File

@ -1,33 +0,0 @@
import { Vega, VegaLite } from 'react-vega'
export default function LineChart( { data=[] }) {
var tmp = data
if (Array.isArray(data)) {
tmp = data.map((r,i) => {
return { x: r[0], y: r[1] }
})
}
const vegaData = { "table": tmp }
const spec = {
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
"mark": "line",
"data": {
"name": "table"
},
"encoding": {
"x": {
"field": "x",
"timeUnit": "year",
"type": "temporal"
},
"y": {
"field": "y",
"type": "quantitative"
}
}
}
return (
<VegaLite data={ vegaData } spec={ spec } />
)
}

View File

@ -1,83 +0,0 @@
import axios from 'axios'
import React, { useEffect, useState } from 'react'
const papa = require("papaparse")
/*
Simple HTML Table
usage: <OutTable data={data} cols={cols} />
data:Array<Array<any> >;
cols:Array<{name:string, key:number|string}>;
*/
export default function Table({ data=[], cols=[], csv='', url='' }) {
if (csv) {
const out = parseCsv(csv)
data = out.rows
cols = out.cols
}
const [ourdata, setData] = React.useState(data)
const [ourcols, setCols] = React.useState(cols)
const [error, setError] = React.useState('')
useEffect(() => {
if (url) {
loadUrl(url)
}
}, [url])
function loadUrl(path) {
// HACK: duplicate of Excel code - maybe refactor
// if url is external may have CORS issue so we proxy it ...
if (url.startsWith('http')) {
const PROXY_URL = window.location.origin + '/api/proxy'
url = PROXY_URL + '?url=' + encodeURIComponent(url)
}
axios.get(url).then((res) => {
const { rows, fields } = parseCsv(res.data)
setData(rows)
setCols(fields)
})
}
return (
<>
<SimpleTable data={ourdata} cols={ourcols} />
</>
)
}
/*
Simple HTML Table
usage: <OutTable data={data} cols={cols} />
data:Array<Array<any> >;
cols:Array<{name:string, key:number|string}>;
*/
function SimpleTable({ data=[], cols=[] }) {
return (
<div className="table-responsive">
<table className="table table-striped">
<thead>
<tr>{cols.map((c) => <th key={c.key}>{c.name}</th>)}</tr>
</thead>
<tbody>
{data.map((r,i) => <tr key={i}>
{cols.map(c => <td key={c.key}>{ r[c.key] }</td>)}
</tr>)}
</tbody>
</table>
</div>
)
}
function parseCsv(csv) {
csv = csv.trim()
const rawdata = papa.parse(csv, {header: true})
const cols = rawdata.meta.fields.map((r,i) => {
return { key: r, name: r }
})
return {
rows: rawdata.data,
fields: cols
}
}

View File

@ -1,35 +0,0 @@
// Used by Data Literate
import matter from 'gray-matter'
import toc from 'remark-toc'
import slug from 'remark-slug'
import gfm from 'remark-gfm'
import footnotes from 'remark-footnotes'
import { serialize } from 'next-mdx-remote/serialize'
/**
* Parse a markdown or MDX file to an MDX source form + front matter data
*
* @source: the contents of a markdown or mdx file
* @returns: { mdxSource: mdxSource, frontMatter: ...}
*/
const parse = async function(source) {
const { content, data } = matter(source)
const mdxSource = await serialize(content, {
// Optionally pass remark/rehype plugins
mdxOptions: {
remarkPlugins: [gfm, toc, slug, footnotes],
rehypePlugins: [],
},
scope: data,
})
return {
mdxSource: mdxSource,
frontMatter: data
}
}
export default parse

153
site/package-lock.json generated
View File

@ -44,8 +44,7 @@
"remark-smartypants": "^2.0.0",
"remark-toc": "^7.2.0",
"vega": "^5.20.2",
"vega-lite": "^5.1.0",
"xlsx": "^0.17.0"
"vega-lite": "^5.1.0"
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.9",
@ -2723,21 +2722,6 @@
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
}
},
"node_modules/adler-32": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.2.0.tgz",
"integrity": "sha1-aj5r8KY5ALoVZSgIyxXGgT0aXyU= sha512-/vUqU/UY4MVeFsg+SsK6c+/05RZXIHZMGJA+PX5JyWI0ZRcBpupnRuPLU/NXXoFwMYCPCoxIfElM2eS+DUXCqQ==",
"dependencies": {
"exit-on-epipe": "~1.0.1",
"printj": "~1.1.0"
},
"bin": {
"adler32": "bin/adler32.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
@ -3200,19 +3184,6 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/cfb": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.0.tgz",
"integrity": "sha512-sXMvHsKCICVR3Naq+J556K+ExBo9n50iKl6LGarlnvuA2035uMlGA/qVrc0wQtow5P1vJEw9UyrKLCbtIKz+TQ==",
"dependencies": {
"adler-32": "~1.2.0",
"crc-32": "~1.2.0",
"printj": "~1.1.2"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/character-entities": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz",
@ -3332,26 +3303,6 @@
"node": ">=6"
}
},
"node_modules/codepage": {
"version": "1.14.0",
"resolved": "https://registry.npmjs.org/codepage/-/codepage-1.14.0.tgz",
"integrity": "sha1-jL4lSBMjVZ19MHVxsP/5HnodL5k= sha512-iz3zJLhlrg37/gYRWgEPkaFTtzmnEv1h+r7NgZum2lFElYQPi0/5bnmuDfODHxfp0INEfnRqyfyeIJDbb7ahRw==",
"dependencies": {
"commander": "~2.14.1",
"exit-on-epipe": "~1.0.1"
},
"bin": {
"codepage": "bin/codepage.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/codepage/node_modules/commander": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/commander/-/commander-2.14.1.tgz",
"integrity": "sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw=="
},
"node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
@ -3438,21 +3389,6 @@
"layout-base": "^1.0.0"
}
},
"node_modules/crc-32": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz",
"integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==",
"dependencies": {
"exit-on-epipe": "~1.0.1",
"printj": "~1.1.0"
},
"bin": {
"crc32": "bin/crc32.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/cross-fetch": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
@ -4375,14 +4311,6 @@
"node": ">=0.8.x"
}
},
"node_modules/exit-on-epipe": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz",
"integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@ -4509,11 +4437,6 @@
"node": "^12.20 || >= 14.13"
}
},
"node_modules/fflate": {
"version": "0.3.11",
"resolved": "https://registry.npmjs.org/fflate/-/fflate-0.3.11.tgz",
"integrity": "sha512-Rr5QlUeGN1mbOHlaqcSYMKVpPbgLy0AWT/W0EHxA6NGI12yO1jpoui2zBBvU2G824ltM6Ut8BFgfHSBGfkmS0A=="
},
"node_modules/fill-range": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@ -4578,14 +4501,6 @@
"node": ">=12.20.0"
}
},
"node_modules/frac": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
"integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/fraction.js": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz",
@ -9745,17 +9660,6 @@
"url": "https://github.com/prettier/prettier?sponsor=1"
}
},
"node_modules/printj": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz",
"integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==",
"bin": {
"printj": "bin/printj.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/prism-react-renderer": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/prism-react-renderer/-/prism-react-renderer-1.3.5.tgz",
@ -12051,17 +11955,6 @@
}
}
},
"node_modules/ssf": {
"version": "0.11.2",
"resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
"integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
"dependencies": {
"frac": "~1.1.2"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/ssri": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz",
@ -13791,22 +13684,6 @@
"string-width": "^1.0.2 || 2 || 3 || 4"
}
},
"node_modules/wmf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
"integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/word": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
"integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
@ -13874,34 +13751,6 @@
}
}
},
"node_modules/xlsx": {
"version": "0.17.0",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.0.tgz",
"integrity": "sha512-bZ36FSACiAyjoldey1+7it50PMlDp1pcAJrZKcVZHzKd8BC/z6TQ/QAN8onuqcepifqSznR6uKnjPhaGt6ig9A==",
"dependencies": {
"adler-32": "~1.2.0",
"cfb": "^1.1.4",
"codepage": "~1.14.0",
"commander": "~2.17.1",
"crc-32": "~1.2.0",
"exit-on-epipe": "~1.0.1",
"fflate": "^0.3.8",
"ssf": "~0.11.2",
"wmf": "~1.0.1",
"word": "~0.3.0"
},
"bin": {
"xlsx": "bin/xlsx.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/xlsx/node_modules/commander": {
"version": "2.17.1",
"resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz",
"integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg=="
},
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",

View File

@ -46,8 +46,7 @@
"remark-smartypants": "^2.0.0",
"remark-toc": "^7.2.0",
"vega": "^5.20.2",
"vega-lite": "^5.1.0",
"xlsx": "^0.17.0"
"vega-lite": "^5.1.0"
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.9",

View File

@ -1,23 +0,0 @@
import fs from "fs";
import parse from "../../lib/data-literate/markdown";
import DataLiterate from "../../components/data-literate/DataLiterate";
export default function PostPage({ source, frontMatter }) {
return <DataLiterate source={source} frontMatter={frontMatter} />;
}
export const getStaticProps = async ({ params }) => {
const mdxPath = "content/data-literate/demo.mdx";
const source = fs.readFileSync(mdxPath);
const { mdxSource, frontMatter } = await parse(source);
return {
props: {
source: mdxSource,
frontMatter: frontMatter,
},
};
};

View File

@ -1,12 +0,0 @@
import Head from 'next/head'
import SheetJSApp from '../components/ExcelViewerApp.js'
import Layout from '../components/Layout'
export default function Index() {
return (
<Layout title='Excel Viewer'>
<h1>Excel Viewer</h1>
<SheetJSApp />
</Layout>
)
}