datahub/examples/openspending/lib/project.ts

import { Octokit } from 'octokit';

export interface GithubProject {
  owner: string;
  repo: string;
  branch: string;
  files: string[];
  readme: string;
  description?: string;
  name?: string;
}

export async function getProjectReadme(
  owner: string,
  repo: string,
  branch: string = 'main',
  github_pat?: string
) {
  const octokit = new Octokit({ auth: github_pat });
  try {
    const response = await octokit.rest.repos.getContent({
      owner,
      repo,
      path: 'README.md',
      ref: branch,
    });
    const data = response.data as { content?: string };
    const fileContent = data.content ? data.content : '';
    if (fileContent === '') {
      return null;
    }
    const decodedContent = Buffer.from(fileContent, 'base64').toString();
    return decodedContent;
  } catch (error) {
    console.log(error);
    return null;
  }
}

export async function getLastUpdated(
  owner: string,
  repo: string,
  branch: string,
  readme: string,
  github_pat?: string
) {
  const octokit = new Octokit({ auth: github_pat });
  try {
    const response = await octokit.rest.repos.listCommits({
      owner,
      repo,
      path: readme,
      ref: branch,
    });
    return response.data[0].commit.committer.date;
  } catch (error) {
    console.log(error);
    return null;
  }
}
export async function getProjectMetadata(
  owner: string,
  repo: string,
  github_pat?: string
) {
  const octokit = new Octokit({ auth: github_pat });
  try {
    const response = await octokit.rest.repos.get({
      owner,
      repo,
    });
    return response.data;
  } catch (error) {
    console.log(error);
    return null;
  }
}

export async function getRepoContents(
  owner: string,
  repo: string,
  branch: string,
  files: string[],
  github_pat?: string
) {
  const octokit = new Octokit({ auth: github_pat });
  try {
    const contents = [];
    for (const path of files) {
      const response = await octokit.rest.repos.getContent({
        owner,
        repo,
        ref: branch,
        path: path,
      });
      const data = response.data as {
        download_url?: string;
        name: string;
        size: number;
      };
      contents.push({
        download_url: data.download_url,
        name: data.name,
        size: data.size,
      });
    }
    return contents;
  } catch (error) {
    console.log(error);
    return null;
  }
}

export async function getProject(project: GithubProject, github_pat?: string) {
  const projectMetadata = await getProjectMetadata(
    project.owner,
    project.repo,
    github_pat
  );
  if (!projectMetadata) {
    return null;
  }
  const projectReadme = await getProjectReadme(
    project.owner,
    project.repo,
    project.branch,
    github_pat
  );

  const projectData = await getRepoContents(
    project.owner,
    project.repo,
    project.branch,
    project.files,
    github_pat
  );
  if (!projectData) {
    return null;
  }

  let projectBase = '',
    last_updated = '';
  if (projectReadme) {
    projectBase =
      project.readme.split('/').length > 1
        ? project.readme.split('/').slice(0, -1).join('/')
        : '/';
    last_updated = await getLastUpdated(
      project.owner,
      project.repo,
      project.branch,
      projectBase,
      github_pat
    );
  }
  return {
    ...projectMetadata,
    files: projectData,
    readmeContent: projectReadme,
    last_updated,
    base_path: projectBase,
  };
}

export async function getProjectDataPackage(
  owner: string,
  repo: string,
  branch: string,
  github_pat?: string
) {
  const octokit = new Octokit({ auth: github_pat });
  try {
    const response = await octokit.rest.repos.getContent({
      owner,
      repo,
      path: 'datapackage.json',
      ref: branch,
    });
    const data = response.data as { content?: string };
    const fileContent = data.content ? data.content : '';
    if (fileContent === '') {
      return null;
    }
    const decodedContent = Buffer.from(fileContent, 'base64').toString();
    const datapackage = JSON.parse(decodedContent);

    return { ...datapackage, repo };
  } catch (error) {
    return null;
  }
}

export async function getAllProjectsFromOrg(
  org: string,
  branch?: string,
  github_pat?: string
) {
  const octokit = new Octokit({ auth: github_pat });
  const repos = await octokit.rest.repos.listForOrg({
    org,
    type: 'public',
    per_page: 100,
  });
  let failedProjects = [];
  const projects = await Promise.all(
    repos.data.map(async (_repo) => {
      const project = await getProjectDataPackage(
        org,
        _repo.name,
        branch ? branch : 'main',
        github_pat
      );
      if (!project) {
        failedProjects.push(_repo.name);
        return null;
      }

      return { datapackage: project, repo: _repo };
    })
  );
  return {
    results: projects.filter((item) => item !== null),
    failed: failedProjects,
  };
}