analyse github org npm dependencies

a-readme.md

First extract data with GitHub API.

The API code search is limited to 10 queries / minute so very slow.

# run package.json extraction
GITHUB_ORG=orgname GITHUB_TOKEN=token npx tsx extract-packages.ts

# show your packages 
find . -iname package.json -exec grep -Hn '"next":' {} \;

Run this with the previous output

const d = `[previous output]`

const versions = d.map((r) => ({
  path: r.split(":")[0],
  version: r.replace(/^.*"[\^~>=]*([\d.]+)",?$/, "$1"),
}));

const getByMajor = (major) =>
  versions
    .filter((v) => parseInt(v.version) === major)
    .sort((a, b) => parseFloat(a.version) - parseFloat(b.version))
    .map((v) => `${v.path}: ${v.version}`)
    .join("\n");

console.log("\n");
console.log("## Version ^14");
console.log(getByMajor(14));

console.log("\n");
console.log("## Version ^15");
console.log(getByMajor(15));

console.log("\n");
console.log("## Version ^16");
console.log(getByMajor(16));

extract-packages.ts

/*
run with:
GITHUB_ORG=orgname GITHUB_TOKEN=token npx tsx extract-packages.ts
*/

import * as fs from "fs";
import * as path from "path";
import * as https from "https";

interface GitHubRepo {
  name: string;
  full_name: string;
  default_branch: string;
}

interface GitHubContent {
  name: string;
  path: string;
  sha: string;
  size: number;
  url: string;
  html_url: string;
  git_url: string;
  download_url: string;
  type: string;
}

const wait = (delay = 10000) =>
  new Promise((resolve) => setTimeout(resolve, delay));

class GitHubPackageFetcher {
  private token: string;
  private org: string;
  private outputDir: string;

  constructor(token: string, org: string, outputDir: string = "./packages") {
    this.token = token;
    this.org = org;
    this.outputDir = outputDir;
  }

  private async makeRequest(url: string): Promise<any> {
    return new Promise((resolve, reject) => {
      const options = {
        headers: {
          "User-Agent": "Node.js",
          Authorization: `Bearer ${this.token}`,
          Accept: "application/vnd.github.v3+json",
        },
      };

      https
        .get(url, options, (res) => {
          let data = "";

          res.on("data", (chunk) => {
            data += chunk;
          });

          res.on("end", () => {
            if (res.statusCode === 200) {
              resolve(JSON.parse(data));
            } else {
              reject(new Error(`HTTP ${res.statusCode}: ${data}`));
            }
          });
        })
        .on("error", (err) => {
          reject(err);
        });
    });
  }

  private async getRepositories(): Promise<GitHubRepo[]> {
    console.log(`Fetching repositories for organization: ${this.org}`);
    let allRepos: GitHubRepo[] = [];
    let page = 1;
    const perPage = 100;

    while (true) {
      const url = `https://api.github.com/orgs/${this.org}/repos?page=${page}&per_page=${perPage}`;
      const repos = await this.makeRequest(url);

      if (repos.length === 0) break;

      allRepos = allRepos.concat(repos);
      console.log(`  Found ${repos.length} repositories on page ${page}`);

      if (repos.length < perPage) break;
      page++;
    }

    console.log(`Total repositories found: ${allRepos.length}\n`);
    return allRepos;
  }

  private async searchPackageJsonFiles(
    repo: GitHubRepo
  ): Promise<Map<string, string>> {
    const packageJsonFiles = new Map<string, string>();

    // Use GitHub Search API to find all package.json files in the repo
    try {
      const searchUrl = `https://api.github.com/search/code?q=filename:package.json+repo:${repo.full_name}`;
      const searchResults = await this.makeRequest(searchUrl);

      if (searchResults.items && searchResults.items.length > 0) {
        console.log(
          `  Found ${searchResults.items.length} package.json file(s)`
        );

        for (const item of searchResults.items) {
          try {
            // Get the actual content
            const content: GitHubContent = await this.makeRequest(item.url);

            if (content.download_url) {
              const packageJsonContent = await this.makeRequest(
                content.download_url
              );
              packageJsonFiles.set(
                item.path,
                JSON.stringify(packageJsonContent, null, 2)
              );
            }
          } catch (error: any) {
            console.error(`  ✗ Error fetching ${item.path}: ${error.message}`);
          }
        }
      }
    } catch (error: any) {
      if (error.message.includes("404")) {
        return packageJsonFiles; // No package.json found
      }
      throw error;
    }

    return packageJsonFiles;
  }

  private savePackageJson(
    repoName: string,
    packagePath: string,
    content: string
  ): void {
    if (!fs.existsSync(this.outputDir)) {
      fs.mkdirSync(this.outputDir, { recursive: true });
    }

    // Create a subdirectory for the repo
    const repoDir = path.join(this.outputDir, repoName);
    if (!fs.existsSync(repoDir)) {
      fs.mkdirSync(repoDir, { recursive: true });
    }

    // Preserve the directory structure from the repo
    const fullPath = path.join(repoDir, packagePath);
    const dir = path.dirname(fullPath);

    if (!fs.existsSync(dir)) {
      fs.mkdirSync(dir, { recursive: true });
    }

    fs.writeFileSync(fullPath, content, "utf8");
    console.log(`    ✓ Saved: ${path.relative(this.outputDir, fullPath)}`);
  }

  async fetchAll(): Promise<void> {
    console.log("Starting package.json fetch process...\n");

    const repos = await this.getRepositories();
    let successCount = 0;
    let notFoundCount = 0;

    for (const repo of repos) {
      console.log(`Processing: ${repo.name}`);
      try {
        const packageJsonFiles = await this.searchPackageJsonFiles(repo);

        if (packageJsonFiles.size > 0) {
          for (const [packagePath, content] of packageJsonFiles) {
            this.savePackageJson(repo.name, packagePath, content);
          }
          successCount += packageJsonFiles.size;
        } else {
          console.log(`  ✗ No package.json found`);
          notFoundCount++;
        }
      } catch (error: any) {
        console.error(`  ✗ Error: ${error.message}`);
      }
      await wait();
    }

    console.log(`\n=== Summary ===`);
    console.log(`Total repositories: ${repos.length}`);
    console.log(`Total package.json files found: ${successCount}`);
    console.log(`Repositories without package.json: ${notFoundCount}`);
    console.log(`Output directory: ${path.resolve(this.outputDir)}`);
  }
}

// Usage
async function main() {
  const GITHUB_TOKEN = process.env.GITHUB_TOKEN || "";
  const ORG_NAME = process.env.GITHUB_ORG || "";
  const OUTPUT_DIR = process.env.OUTPUT_DIR || "./packages";

  if (!GITHUB_TOKEN) {
    console.error("Error: GITHUB_TOKEN environment variable is required");
    console.error("Create a token at: https://github.com/settings/tokens");
    process.exit(1);
  }

  if (!ORG_NAME) {
    console.error("Error: GITHUB_ORG environment variable is required");
    process.exit(1);
  }

  const fetcher = new GitHubPackageFetcher(GITHUB_TOKEN, ORG_NAME, OUTPUT_DIR);
  await fetcher.fetchAll();
}

main().catch(console.error);