[packanges/ckan][xs] - fix type

.changeset/sharp-spoons-deny.md

@@ -0,0 +1,5 @@
+---
+'@portaljs/ckan': patch
+---
+
+remove optional from id in resource interface

package-lock.json

@@ -46942,7 +46942,7 @@
     },
     "packages/ckan": {
       "name": "@portaljs/ckan",
-      "version": "0.1.0",
+      "version": "0.0.3",
       "dependencies": {
         "formik": "^2.2.9",
         "swr": "^2.1.5",
@@ -47347,7 +47347,7 @@
     },
     "packages/components": {
       "name": "@portaljs/components",
-      "version": "0.3.2",
+      "version": "0.3.1",
       "dependencies": {
         "@githubocto/flat-ui": "^0.14.1",
         "@heroicons/react": "^2.0.17",
@@ -47828,7 +47828,7 @@
     },
     "packages/remark-wiki-link": {
       "name": "@portaljs/remark-wiki-link",
-      "version": "1.1.0",
+      "version": "1.0.4",
       "license": "MIT",
       "dependencies": {
         "mdast-util-to-markdown": "^1.5.0",

packages/ckan/CHANGELOG.md

@@ -1,15 +1,5 @@
 # @portaljs/ckan
 
-## 0.1.0
-
-### Minor Changes
-
-- [#1018](https://github.com/datopian/portaljs/pull/1018) [`50122cd0`](https://github.com/datopian/portaljs/commit/50122cd0cbbf68bdadc641341279b30b22538cfd) Thanks [@demenech](https://github.com/demenech)! - package_search method now supports custom headers and include_private parameter
-
-### Patch Changes
-
-- [#1016](https://github.com/datopian/portaljs/pull/1016) [`91217f32`](https://github.com/datopian/portaljs/commit/91217f325657e2f298b0e632793ae9bb8b08e870) Thanks [@luccasmmg](https://github.com/luccasmmg)! - remove optional from id in resource interface
-
 ## 0.0.5
 
 ### Patch Changes

packages/ckan/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@portaljs/ckan",
-  "version": "0.1.0",
+  "version": "0.0.5",
   "type": "module",
   "description": "https://portaljs.org",
   "keywords": [

packages/ckan/src/interfaces/dataset.interface.ts

@@ -69,7 +69,6 @@ export interface PackageSearchOptions {
   query?: string;
   resFormat?: Array<string>;
   sort?: string;
-  include_private?: boolean;
 }
 
 export interface Tag {

packages/ckan/src/lib/ckanapi.tsx

@@ -31,7 +31,11 @@ export default class CKAN {
 
   async getDatasetsListWithDetails(options: DatasetListQueryOptions) {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/current_package_list_with_resources?offset=${options.offset}&limit=${options.limit}`,
+      `${
+        this.DMS
+      }/api/3/action/current_package_list_with_resources?offset=${
+        options.offset
+      }&limit=${options.limit}`,
       3
     );
     const responseData = await response.json();
@@ -40,8 +44,7 @@ export default class CKAN {
   }
 
   async packageSearch(
-    options: PackageSearchOptions,
-    reqOptions: Partial<RequestInit> = {}
+    options: PackageSearchOptions
   ): Promise<{ datasets: Dataset[]; count: number }> {
     function buildGroupsQuery(groups: Array<string>) {
       if (groups.length > 0) {
@@ -96,18 +99,16 @@ export default class CKAN {
       options.groups,
       options?.resFormat
     );
-
-    let url = `${this.DMS}/api/3/action/package_search?`;
-    url += `start=${options.offset}`;
-    url += `&rows=${options.limit}`;
-    url += fq ? fq : '';
-    url += options.query ? '&q=' + options.query : '';
-    url += options.sort ? '&sort=' + options.sort : '';
-    url += options.include_private
-      ? '&include_private=' + options.include_private
-      : '';
-
-    const response = await fetchRetry(url, 3, reqOptions);
+    const response = await fetchRetry(
+      `${
+        this.DMS
+      }/api/3/action/package_search?start=${options.offset}&rows=${
+        options.limit
+      }${fq ? fq : ''}${options.query ? '&q=' + options.query : ''}${
+        options.sort ? '&sort=' + options.sort : ''
+      }`,
+      3
+    );
     const responseData = await response.json();
     const datasets: Array<Dataset> = responseData.result.results;
     return { datasets, count: responseData.result.count };
@@ -115,7 +116,9 @@ export default class CKAN {
 
   async getDatasetDetails(datasetName: string) {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/package_show?id=${datasetName}`,
+      `${
+        this.DMS
+      }/api/3/action/package_show?id=${datasetName}`,
       1
     );
     const responseData = await response.json();
@@ -128,7 +131,9 @@ export default class CKAN {
 
   async getDatasetActivityStream(datasetName: string) {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/package_activity_list?id=${datasetName}`,
+      `${
+        this.DMS
+      }/api/3/action/package_activity_list?id=${datasetName}`,
       3
     );
     const responseData = await response.json();
@@ -146,7 +151,9 @@ export default class CKAN {
   async getUser(userId: string) {
     try {
       const response = await fetchRetry(
-        `${this.DMS}/api/3/action/user_show?id=${userId}`,
+        `${
+          this.DMS
+        }/api/3/action/user_show?id=${userId}`,
         3
       );
       const responseData = await response.json();
@@ -159,7 +166,10 @@ export default class CKAN {
   }
 
   async getGroupList() {
-    const response = await fetchRetry(`${this.DMS}/api/3/action/group_list`, 3);
+    const response = await fetchRetry(
+      `${this.DMS}/api/3/action/group_list`,
+      3
+    );
     const responseData = await response.json();
     const groups: Array<string> = responseData.result;
     return groups;
@@ -167,7 +177,9 @@ export default class CKAN {
 
   async getGroupsWithDetails() {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/group_list?all_fields=True`,
+      `${
+        this.DMS
+      }/api/3/action/group_list?all_fields=True`,
       3
     );
     const responseData = await response.json();
@@ -177,7 +189,9 @@ export default class CKAN {
 
   async getGroupDetails(groupName: string) {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/group_show?id=${groupName}&include_datasets=True`,
+      `${
+        this.DMS
+      }/api/3/action/group_show?id=${groupName}&include_datasets=True`,
       3
     );
     const responseData = await response.json();
@@ -187,7 +201,9 @@ export default class CKAN {
 
   async getGroupActivityStream(groupName: string) {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/group_activity_list?id=${groupName}`,
+      `${
+        this.DMS
+      }/api/3/action/group_activity_list?id=${groupName}`,
       3
     );
     const responseData = await response.json();
@@ -214,7 +230,9 @@ export default class CKAN {
   async getOrgsWithDetails(accrossPages?: boolean) {
     if (!accrossPages) {
       const response = await fetchRetry(
-        `${this.DMS}/api/3/action/organization_list?all_fields=True`,
+        `${
+          this.DMS
+        }/api/3/action/organization_list?all_fields=True`,
         3
       );
       const responseData = await response.json();
@@ -233,7 +251,9 @@ export default class CKAN {
 
     for (let i = 0; i < pages; i++) {
       let allOrgListResponse = await fetchRetry(
-        `${this.DMS}/api/3/action/organization_list?all_fields=True&offset=${
+        `${
+          this.DMS
+        }/api/3/action/organization_list?all_fields=True&offset=${
           i * 25
         }&limit=25`,
         3
@@ -247,7 +267,9 @@ export default class CKAN {
 
   async getOrgDetails(orgName: string) {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/organization_show?id=${orgName}&include_datasets=True`,
+      `${
+        this.DMS
+      }/api/3/action/organization_show?id=${orgName}&include_datasets=True`,
       3
     );
     const responseData = await response.json();
@@ -257,7 +279,9 @@ export default class CKAN {
 
   async getOrgActivityStream(orgName: string) {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/organization_activity_list?id=${orgName}`,
+      `${
+        this.DMS
+      }/api/3/action/organization_activity_list?id=${orgName}`,
       3
     );
     const responseData = await response.json();
@@ -273,7 +297,9 @@ export default class CKAN {
 
   async getAllTags() {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/tag_list?all_fields=True`,
+      `${
+        this.DMS
+      }/api/3/action/tag_list?all_fields=True`,
       3
     );
     const responseData = await response.json();
@@ -282,41 +308,49 @@ export default class CKAN {
   }
 
   async getResourcesWithAliasList() {
-    const response = await fetch(`${this.DMS}/api/3/action/datastore_search`, {
-      method: 'POST',
-      headers: {
-        Accept: 'application/json',
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        id: '_table_metadata',
-        limit: '32000',
-      }),
-    });
+    const response = await fetch(
+      `${this.DMS}/api/3/action/datastore_search`,
+      {
+        method: 'POST',
+        headers: {
+          Accept: 'application/json',
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          id: '_table_metadata',
+          limit: '32000',
+        }),
+      }
+    );
     const responseData = await response.json();
     const tableMetadata: Array<TableMetadata> = responseData.result.records;
     return tableMetadata.filter((item) => item.alias_of);
   }
 
   async datastoreSearch(resourceId: string) {
-    const response = await fetch(`${this.DMS}/api/3/action/datastore_search`, {
-      method: 'POST',
-      headers: {
-        Accept: 'application/json',
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        id: resourceId,
-        limit: '32000',
-      }),
-    });
+    const response = await fetch(
+      `${this.DMS}/api/3/action/datastore_search`,
+      {
+        method: 'POST',
+        headers: {
+          Accept: 'application/json',
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          id: resourceId,
+          limit: '32000',
+        }),
+      }
+    );
     const responseData = await response.json();
     return responseData.result.records;
   }
 
   async getResourceMetadata(resourceId: string) {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/resource_show?id=${resourceId}`,
+      `${
+        this.DMS
+      }/api/3/action/resource_show?id=${resourceId}`,
       3
     );
     const responseData = await response.json();
@@ -325,14 +359,17 @@ export default class CKAN {
   }
 
   async getResourceInfo(resourceId: string) {
-    const response = await fetch(`${this.DMS}/api/3/action/datastore_info`, {
-      method: 'POST',
-      headers: {
-        Accept: 'application/json',
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({ resource_id: resourceId }),
-    });
+    const response = await fetch(
+      `${this.DMS}/api/3/action/datastore_info`,
+      {
+        method: 'POST',
+        headers: {
+          Accept: 'application/json',
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ resource_id: resourceId }),
+      }
+    );
     const responseData = await response.json();
     const resourceInfo: Array<ResourceInfo> = responseData.result;
     return resourceInfo;
@@ -340,7 +377,9 @@ export default class CKAN {
 
   async getFacetFields(field: 'res_format' | 'tags') {
     const response = await fetchRetry(
-      `${this.DMS}/api/3/action/package_search?facet.field=["${field}"]&rows=0`,
+      `${
+        this.DMS
+      }/api/3/action/package_search?facet.field=["${field}"]&rows=0`,
       3
     );
     const responseData = await response.json();

packages/ckan/src/lib/utils.ts

@@ -9,14 +9,10 @@ export function getDaysAgo(date: string) {
   return (+today - +createdOn) / msInDay;
 }
 
-export default async function fetchRetry(
-  url: string,
-  n: number,
-  opts: Partial<RequestInit> = {}
-): Promise<any> {
+export default async function fetchRetry(url: string, n: number): Promise<any> {
   const abortController = new AbortController();
   const id = setTimeout(() => abortController.abort(), 30000);
-  const res = await fetch(url, { signal: abortController.signal, ...opts });
+  const res = await fetch(url, { signal: abortController.signal });
   clearTimeout(id);
   if (!res.ok && n && n > 0) {
     return await fetchRetry(url, n - 1);
@@ -25,13 +21,13 @@ export default async function fetchRetry(
 }
 
 export function removeTag(tag?: string) {
-  if (tag === '{{description}}' || !tag) {
+  if (tag === "{{description}}" || !tag) {
     return undefined;
   }
-  if (typeof window !== 'undefined') {
-    const div = document.createElement('div');
+  if (typeof window !== "undefined") {
+    const div = document.createElement("div");
     div.innerHTML = tag;
-    return div.textContent || div.innerText || '';
+    return div.textContent || div.innerText || "";
   }
   return tag;
 }
@@ -42,10 +38,10 @@ export function convertFieldSchema(
 ) {
   function convertToGraphqlString(fieldName: string) {
     return fieldName
-      .replaceAll(' ', '_')
-      .replaceAll('(', '_')
-      .replaceAll(')', '_')
-      .replace(/[^\w\s]|(_)\1/gi, '_');
+      .replaceAll(" ", "_")
+      .replaceAll("(", "_")
+      .replaceAll(")", "_")
+      .replace(/[^\w\s]|(_)\1/gi, "_");
   }
   const entries = Object.entries(schema);
   return {

packages/components/CHANGELOG.md

@@ -1,11 +1,5 @@
 # @portaljs/components
 
-## 0.4.0
-
-### Minor Changes
-
-- [#1022](https://github.com/datopian/portaljs/pull/1022) [`83fd7727`](https://github.com/datopian/portaljs/commit/83fd7727bafb4902218777597e9848a3e3a71d87) Thanks [@luccasmmg](https://github.com/luccasmmg)! - FlatUiTables now accepts a bytes param and a parsingConfig param for CSV links
-
 ## 0.3.2
 
 ### Patch Changes

packages/components/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@portaljs/components",
-  "version": "0.4.0",
+  "version": "0.3.2",
   "type": "module",
   "description": "https://portaljs.org",
   "keywords": [

packages/components/src/components/FlatUiTable.tsx

@@ -5,20 +5,22 @@ import LoadingSpinner from './LoadingSpinner';
 
 const queryClient = new QueryClient();
 
-export async function getCsv(url: string, bytes) {
+export async function getCsv(url: string, corsProxy?: string) {
+  if (corsProxy) {
+    url = corsProxy + url;
+  }
   const response = await fetch(url, {
     headers: {
-      Range: `bytes=0-${bytes}`,
+      Range: 'bytes=0-5132288',
     },
   });
   const data = await response.text();
   return data;
 }
 
-export async function parseCsv(file: string, parsingConfig): Promise<any> {
+export async function parseCsv(file: string): Promise<any> {
   return new Promise((resolve, reject) => {
     Papa.parse(file, {
-      ...parsingConfig,
       header: true,
       dynamicTyping: true,
       skipEmptyLines: true,
@@ -39,28 +41,25 @@ export interface FlatUiTableProps {
   url?: string;
   data?: { [key: string]: number | string }[];
   rawCsv?: string;
+  corsProxy?: string;
   randomId?: number;
-  bytes: number;
-  parsingConfig: any;
 }
 export const FlatUiTable: React.FC<FlatUiTableProps> = ({
   url,
   data,
   rawCsv,
-  bytes = 5132288,
-  parsingConfig = {},
+  corsProxy,
 }) => {
   const randomId = Math.random();
   return (
     // Provide the client to your App
     <QueryClientProvider client={queryClient}>
       <TableInner
-        bytes={bytes}
+        corsProxy={corsProxy}
         url={url}
         data={data}
         rawCsv={rawCsv}
         randomId={randomId}
-        parsingConfig={parsingConfig}
       />
     </QueryClientProvider>
   );
@@ -70,9 +69,8 @@ const TableInner: React.FC<FlatUiTableProps> = ({
   url,
   data,
   rawCsv,
+  corsProxy,
   randomId,
-  bytes,
-  parsingConfig,
 }) => {
   if (data) {
     return (
@@ -83,16 +81,12 @@ const TableInner: React.FC<FlatUiTableProps> = ({
   }
   const { data: csvString, isLoading: isDownloadingCSV } = useQuery(
     ['dataCsv', url, randomId],
-    () => getCsv(url as string, bytes),
+    () => getCsv(url as string, corsProxy),
     { enabled: !!url }
   );
   const { data: parsedData, isLoading: isParsing } = useQuery(
     ['dataPreview', csvString, randomId],
-    () =>
-      parseCsv(
-        rawCsv ? (rawCsv as string) : (csvString as string),
-        parsingConfig
-      ),
+    () => parseCsv(rawCsv ? (rawCsv as string) : (csvString as string)),
     { enabled: rawCsv ? true : !!csvString }
   );
   if (isParsing || isDownloadingCSV)

packages/components/stories/FlatUiTable.stories.ts

@@ -9,24 +9,17 @@ const meta: Meta = {
   tags: ['autodocs'],
   argTypes: {
     data: {
-      description:
-        'Data to be displayed in the table, must be setup as an array of key value pairs',
+      description: "Data to be displayed in the table, must be setup as an array of key value pairs"
     },
     csv: {
-      description: 'CSV data as string.',
+      description: "CSV data as string.",
     },
     url: {
-      description:
-        'Fetch the data from a CSV file remotely. only the first 5MB of data will be displayed',
-    },
-    bytes: {
-      description:
-        'Fetch the data from a CSV file remotely. only the first <bytes> of data will be displayed',
-    },
-    parsingConfig: {
-      description:
-        'Configuration for parsing the CSV data. See https://www.papaparse.com/docs#config for more details',
+      description: "Fetch the data from a CSV file remotely. only the first 5MB of data will be displayed"
     },
+    corsProxy: {
+      description: "Optionally you cant set a CORS Proxy to which all your requests you be redirected"
+    }
   },
 };
 
@@ -36,7 +29,7 @@ type Story = StoryObj<FlatUiTableProps>;
 
 // More on writing stories with args: https://storybook.js.org/docs/react/writing-stories/args
 export const FromColumnsAndData: Story = {
-  name: 'Table data',
+  name: "Table data",
   args: {
     data: [
       { id: 1, lastName: 'Snow', firstName: 'Jon', age: 35 },
@@ -51,19 +44,20 @@ export const FromColumnsAndData: Story = {
 };
 
 export const FromRawCSV: Story = {
-  name: 'Table from raw CSV',
+  name: "Table from raw CSV",
   args: {
     rawCsv: `
     Year,Temp Anomaly
     1850,-0.418
     2020,0.923
-    `,
-  },
+    `
+  }
 };
 
 export const FromURL: Story = {
-  name: 'Table from URL',
+  name: "Table from URL",
   args: {
-    url: 'https://ckan-dev.sse.datopian.com/datastore/dump/601c9cf0-595e-46d8-88fc-d1ab2904e2db',
-  },
+    url: "https://raw.githubusercontent.com/datasets/finance-vix/main/data/vix-daily.csv"
+  }
 };
+

packages/remark-wiki-link/src/lib/fromMarkdown.ts

@@ -79,7 +79,7 @@ function fromMarkdown(opts: FromMarkdownOptions = {}) {
       data: { isEmbed, target, alias },
     } = wikiLink;
     // eslint-disable-next-line no-useless-escape
-    const wikiLinkWithHeadingPattern = /([\p{Letter}\d\s\/\.-_]*)(#.*)?/u;
+    const wikiLinkWithHeadingPattern = /([\w\s\/\.-]*)(#.*)?/;
     const [, path, heading = ""] = target.match(wikiLinkWithHeadingPattern);
 
     const possibleWikiLinkPermalinks = wikiLinkResolver(path);

site/content/docs/searching-datasets.md

@@ -101,7 +101,7 @@ List of available datasets:
 <Catalog datasets={datasets} facets={['group']}/>
 ```
 
-Rerun `npm run mddb`. You now have a filter in your page with all possible values automatically added to it.
+You now have a filter in your page with all possible values automatically added to it.
 
 ![Data catalog with facets built with PortalJS](https://i.imgur.com/p2miSdg.png)