feat: enable multi page star fetching for private vercel instances (#2159)

* feat: enable multi-page stars' fetching for private vercel instances This commit enables multi-page stars' support from fetching on private Vercel instances. This feature can be disabled on the public Vercel instance by adding the `FETCH_SINGLE_PAGE_STARS=true` as an env variable in the public Vercel instance. This variable will not be present when people deploy their own Vercel instance, causing the code to fetch multiple star pages. * fix: improve stats multi-page fetching behavoir This commit makes sure that the GraphQL api is only called one time per 100 repositories. The old method added one unnecesairy GraphQL call. * docs: update documentation * style: improve code syntax Co-authored-by: Matteo Pierro <pierromatteo@gmail.com> * lol happy new year * docs: remove rate limit documentation for now Remove the `FETCH_SINGLE_PAGE_STARS` from documentation for now since it might confuse people. * fix: fix error in automatic merge * feat: make sure env variable is read Co-authored-by: Matteo Pierro <pierromatteo@gmail.com> Co-authored-by: Anurag <hazru.anurag@gmail.com>
2024-12-15 06:04:17 +08:00 · 2023-01-21 18:32:37 +01:00 · 2023-01-21 18:32:37 +01:00 · 60fae292a3
commit 60fae292a3
parent c1dc7b850c
4 changed files with 210 additions and 166 deletions
--- a/readme.md
+++ b/readme.md
@ -93,7 +93,7 @@ Visit <https://indiafightscorona.giveindia.org> and make a small donation to hel
    -   [Language Card Exclusive Options](#language-card-exclusive-options)
    -   [Wakatime Card Exclusive Option](#wakatime-card-exclusive-options)
 -   [Deploy Yourself](#deploy-on-your-own-vercel-instance)
-    - [Keep your fork up to date](#keep-your-fork-up-to-date)   
+    -   [Keep your fork up to date](#keep-your-fork-up-to-date)

 # GitHub Stats Card

@ -264,7 +264,7 @@ You can customize the appearance of your `Stats Card` or `Repo Card` however you
 -   `border_radius` - Corner rounding on the card. Default: `4.5`.

 > **Warning**
-> We use caching to decrease the load on our servers (see https://github.com/anuraghazra/github-readme-stats/issues/1471#issuecomment-1271551425). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours.
+> We use caching to decrease the load on our servers (see <https://github.com/anuraghazra/github-readme-stats/issues/1471#issuecomment-1271551425>). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours.

 ##### Gradient in bg_color

@ -354,7 +354,7 @@ Use [show_owner](#customization) variable to include the repo's owner username
 The top languages card shows a GitHub user's most frequently used top language.

 > **Note**
-> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats._
+> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats.

 ### Usage

@ -498,7 +498,7 @@ By default, GitHub does not lay out the cards side by side. To do that, you can

 ## Deploy on your own Vercel instance

-#### [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107)
+#### :film_projector: [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107)

 > **Warning**
 > If you are on the [hobby (i.e. free)](https://vercel.com/pricing) Vercel plan, please make sure you change the `maxDuration` parameter in the [vercel.json](https://github.com/anuraghazra/github-readme-stats/blob/master/vercel.json) file from `30` to `10` (see [#1416](https://github.com/anuraghazra/github-readme-stats/issues/1416#issuecomment-950275476) for more information).
--- a/src/fetchers/stats-fetcher.js
+++ b/src/fetchers/stats-fetcher.js
@ -1,5 +1,6 @@
 // @ts-check
 import axios from "axios";
+import * as dotenv from "dotenv";
 import githubUsernameRegex from "github-username-regex";
 import { calculateRank } from "../calculateRank.js";
 import { retryer } from "../common/retryer.js";
@ -11,46 +12,74 @@ import {
  wrapTextMultiline,
 } from "../common/utils.js";

+dotenv.config();
+
+// GraphQL queries.
+const GRAPHQL_REPOS_FIELD = `
+  repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) {
+    totalCount
+    nodes {
+      name
+      stargazers {
+        totalCount
+      }
+    }
+    pageInfo {
+      hasNextPage
+      endCursor
+    }
+  }
+`;
+
+const GRAPHQL_REPOS_QUERY = `
+  query userInfo($login: String!, $after: String) {
+    user(login: $login) {
+      ${GRAPHQL_REPOS_FIELD}
+    }
+  }
+`;
+
+const GRAPHQL_STATS_QUERY = `
+  query userInfo($login: String!, $after: String) {
+    user(login: $login) {
+      name
+      login
+      contributionsCollection {
+        totalCommitContributions
+        restrictedContributionsCount
+      }
+      repositoriesContributedTo(first: 1, contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) {
+        totalCount
+      }
+      pullRequests(first: 1) {
+        totalCount
+      }
+      openIssues: issues(states: OPEN) {
+        totalCount
+      }
+      closedIssues: issues(states: CLOSED) {
+        totalCount
+      }
+      followers {
+        totalCount
+      }
+      ${GRAPHQL_REPOS_FIELD}
+    }
+  }
+`;
+
 /**
 * Stats fetcher object.
 *
 * @param {import('axios').AxiosRequestHeaders} variables Fetcher variables.
 * @param {string} token GitHub token.
- * @returns {Promise<import('../common/types').StatsFetcherResponse>} Stats fetcher response.
+ * @returns {Promise<import('../common/types').Fetcher>} Stats fetcher response.
 */
 const fetcher = (variables, token) => {
+  const query = !variables.after ? GRAPHQL_STATS_QUERY : GRAPHQL_REPOS_QUERY;
  return request(
    {
-      query: `
-      query userInfo($login: String!) {
-        user(login: $login) {
-          name
-          login
-          contributionsCollection {
-            totalCommitContributions
-            restrictedContributionsCount
-          }
-          repositoriesContributedTo(contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) {
-            totalCount
-          }
-          pullRequests {
-            totalCount
-          }
-          openIssues: issues(states: OPEN) {
-            totalCount
-          }
-          closedIssues: issues(states: CLOSED) {
-            totalCount
-          }
-          followers {
-            totalCount
-          }
-          repositories(ownerAffiliations: OWNER) {
-            totalCount
-          }
-        }
-      }
-      `,
+      query,
      variables,
    },
    {
@ -60,39 +89,42 @@ const fetcher = (variables, token) => {
 };

 /**
- * Fetch first 100 repositories for a given username.
+ * Fetch stats information for a given username.
 *
- * @param {import('axios').AxiosRequestHeaders} variables Fetcher variables.
- * @param {string} token GitHub token.
- * @returns {Promise<import('../common/types').StatsFetcherResponse>} Repositories fetcher response.
+ * @param {string} username Github username.
+ * @returns {Promise<import('../common/types').StatsFetcher>} GraphQL Stats object.
+ *
+ * @description This function supports multi-page fetching if the 'FETCH_MULTI_PAGE_STARS' environment variable is set to true.
 */
-const repositoriesFetcher = (variables, token) => {
-  return request(
-    {
-      query: `
-      query userInfo($login: String!, $after: String) {
-        user(login: $login) {
-          repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) {
-            nodes {
-              name
-              stargazers {
-                totalCount
-              }
-            }
-            pageInfo {
-              hasNextPage
-              endCursor
-            }
-          }
-        }
-      }
-      `,
-      variables,
-    },
-    {
-      Authorization: `bearer ${token}`,
-    },
-  );
+const statsFetcher = async (username) => {
+  let stats;
+  let hasNextPage = true;
+  let endCursor = null;
+  while (hasNextPage) {
+    const variables = { login: username, first: 100, after: endCursor };
+    let res = await retryer(fetcher, variables);
+    if (res.data.errors) return res;
+
+    // Store stats data.
+    const repoNodes = res.data.data.user.repositories.nodes;
+    if (!stats) {
+      stats = res;
+    } else {
+      stats.data.data.user.repositories.nodes.push(...repoNodes);
+    }
+
+    // Disable multi page fetching on public Vercel instance due to rate limits.
+    const repoNodesWithStars = repoNodes.filter(
+      (node) => node.stargazers.totalCount !== 0,
+    );
+    hasNextPage =
+      process.env.FETCH_MULTI_PAGE_STARS === "true" &&
+      repoNodes.length === repoNodesWithStars.length &&
+      res.data.data.user.repositories.pageInfo.hasNextPage;
+    endCursor = res.data.data.user.repositories.pageInfo.endCursor;
+  }
+
+  return stats;
 };

 /**
@ -137,46 +169,6 @@ const totalCommitsFetcher = async (username) => {
  return 0;
 };

-/**
- * Fetch all the stars for all the repositories of a given username.
- *
- * @param {string} username GitHub username.
- * @param {array} repoToHide Repositories to hide.
- * @returns {Promise<number>} Total stars.
- */
-const totalStarsFetcher = async (username, repoToHide) => {
-  let nodes = [];
-  let hasNextPage = true;
-  let endCursor = null;
-  while (hasNextPage) {
-    const variables = { login: username, first: 100, after: endCursor };
-    let res = await retryer(repositoriesFetcher, variables);
-
-    if (res.data.errors) {
-      logger.error(res.data.errors);
-      throw new CustomError(
-        res.data.errors[0].message || "Could not fetch user",
-        CustomError.USER_NOT_FOUND,
-      );
-    }
-
-    const allNodes = res.data.data.user.repositories.nodes;
-    const nodesWithStars = allNodes.filter(
-      (node) => node.stargazers.totalCount !== 0,
-    );
-    nodes.push(...nodesWithStars);
-    // hasNextPage =
-    //   allNodes.length === nodesWithStars.length &&
-    //   res.data.data.user.repositories.pageInfo.hasNextPage;
-    hasNextPage = false; // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-    endCursor = res.data.data.user.repositories.pageInfo.endCursor;
-  }
-
-  return nodes
-    .filter((data) => !repoToHide[data.name])
-    .reduce((prev, curr) => prev + curr.stargazers.totalCount, 0);
-};
-
 /**
 * Fetch stats for a given username.
 *
@ -203,7 +195,7 @@ const fetchStats = async (
    rank: { level: "C", score: 0 },
  };

-  let res = await retryer(fetcher, { login: username });
+  let res = await statsFetcher(username);

  // Catch GraphQL errors.
  if (res.data.errors) {
@ -259,8 +251,15 @@ const fetchStats = async (
  stats.contributedTo = user.repositoriesContributedTo.totalCount;

  // Retrieve stars while filtering out repositories to be hidden
-  stats.totalStars = await totalStarsFetcher(username, repoToHide);
+  stats.totalStars = user.repositories.nodes
+    .filter((data) => {
+      return !repoToHide[data.name];
+    })
+    .reduce((prev, curr) => {
+      return prev + curr.stargazers.totalCount;
+    }, 0);

+  // @ts-ignore // TODO: Fix this.
  stats.rank = calculateRank({
    totalCommits: stats.totalCommits,
    totalRepos: user.repositories.totalCount,
--- a/tests/api.test.js
+++ b/tests/api.test.js
@ -25,7 +25,7 @@ stats.rank = calculateRank({
  issues: stats.totalIssues,
 });

-const data = {
+const data_stats = {
  data: {
    user: {
      name: stats.name,
@ -40,15 +40,6 @@ const data = {
      followers: { totalCount: 0 },
      repositories: {
        totalCount: 1,
-      },
-    },
-  },
-};
-
-const repositoriesData = {
-  data: {
-    user: {
-      repositories: {
        nodes: [{ stargazers: { totalCount: 100 } }],
        pageInfo: {
          hasNextPage: false,
@ -83,11 +74,7 @@ const faker = (query, data) => {
    setHeader: jest.fn(),
    send: jest.fn(),
  };
-  mock
-    .onPost("https://api.github.com/graphql")
-    .replyOnce(200, data)
-    .onPost("https://api.github.com/graphql")
-    .replyOnce(200, repositoriesData);
+  mock.onPost("https://api.github.com/graphql").replyOnce(200, data);

  return { req, res };
 };
@ -98,7 +85,7 @@ afterEach(() => {

 describe("Test /api/", () => {
  it("should test the request", async () => {
-    const { req, res } = faker({}, data);
+    const { req, res } = faker({}, data_stats);

    await api(req, res);

@ -133,7 +120,7 @@ describe("Test /api/", () => {
        text_color: "fff",
        bg_color: "fff",
      },
-      data,
+      data_stats,
    );

    await api(req, res);
@ -154,7 +141,7 @@ describe("Test /api/", () => {
  });

  it("should have proper cache", async () => {
-    const { req, res } = faker({}, data);
+    const { req, res } = faker({}, data_stats);

    await api(req, res);

@ -170,7 +157,7 @@ describe("Test /api/", () => {
  });

  it("should set proper cache", async () => {
-    const { req, res } = faker({ cache_seconds: 15000 }, data);
+    const { req, res } = faker({ cache_seconds: 15000 }, data_stats);
    await api(req, res);

    expect(res.setHeader.mock.calls).toEqual([
@ -196,7 +183,7 @@ describe("Test /api/", () => {

  it("should set proper cache with clamped values", async () => {
    {
-      let { req, res } = faker({ cache_seconds: 200000 }, data);
+      let { req, res } = faker({ cache_seconds: 200000 }, data_stats);
      await api(req, res);

      expect(res.setHeader.mock.calls).toEqual([
@ -212,7 +199,7 @@ describe("Test /api/", () => {

    // note i'm using block scoped vars
    {
-      let { req, res } = faker({ cache_seconds: 0 }, data);
+      let { req, res } = faker({ cache_seconds: 0 }, data_stats);
      await api(req, res);

      expect(res.setHeader.mock.calls).toEqual([
@ -227,7 +214,7 @@ describe("Test /api/", () => {
    }

    {
-      let { req, res } = faker({ cache_seconds: -10000 }, data);
+      let { req, res } = faker({ cache_seconds: -10000 }, data_stats);
      await api(req, res);

      expect(res.setHeader.mock.calls).toEqual([
@ -248,7 +235,7 @@ describe("Test /api/", () => {
        username: "anuraghazra",
        count_private: true,
      },
-      data,
+      data_stats,
    );

    await api(req, res);
@ -288,7 +275,7 @@ describe("Test /api/", () => {
        text_color: "fff",
        bg_color: "fff",
      },
-      data,
+      data_stats,
    );

    await api(req, res);
--- a/tests/fetchStats.test.js
+++ b/tests/fetchStats.test.js
@ -4,7 +4,8 @@ import MockAdapter from "axios-mock-adapter";
 import { calculateRank } from "../src/calculateRank.js";
 import { fetchStats } from "../src/fetchers/stats-fetcher.js";

-const data = {
+// Test parameters.
+const data_stats = {
  data: {
    user: {
      name: "Anurag Hazra",
@ -19,15 +20,6 @@ const data = {
      followers: { totalCount: 100 },
      repositories: {
        totalCount: 5,
-      },
-    },
-  },
-};
-
-const firstRepositoriesData = {
-  data: {
-    user: {
-      repositories: {
        nodes: [
          { name: "test-repo-1", stargazers: { totalCount: 100 } },
          { name: "test-repo-2", stargazers: { totalCount: 100 } },
@ -42,7 +34,7 @@ const firstRepositoriesData = {
  },
 };

-const secondRepositoriesData = {
+const data_repo = {
  data: {
    user: {
      repositories: {
@ -59,7 +51,7 @@ const secondRepositoriesData = {
  },
 };

-const repositoriesWithZeroStarsData = {
+const data_repo_zero_stars = {
  data: {
    user: {
      repositories: {
@ -93,13 +85,12 @@ const error = {
 const mock = new MockAdapter(axios);

 beforeEach(() => {
+  process.env.FETCH_MULTI_PAGE_STARS = "false"; // Set to `false` to fetch only one page of stars.
  mock
    .onPost("https://api.github.com/graphql")
-    .replyOnce(200, data)
+    .replyOnce(200, data_stats)
    .onPost("https://api.github.com/graphql")
-    .replyOnce(200, firstRepositoriesData);
-  // .onPost("https://api.github.com/graphql") // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-  // .replyOnce(200, secondRepositoriesData); // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+    .replyOnce(200, data_repo);
 });

 afterEach(() => {
@ -114,8 +105,7 @@ describe("Test fetchStats", () => {
      totalRepos: 5,
      followers: 100,
      contributions: 61,
-      // stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-      stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      stargazers: 300,
      prs: 300,
      issues: 200,
    });
@ -126,8 +116,7 @@ describe("Test fetchStats", () => {
      totalCommits: 100,
      totalIssues: 200,
      totalPRs: 300,
-      // totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-      totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      totalStars: 300,
      rank,
    });
  });
@ -136,9 +125,9 @@ describe("Test fetchStats", () => {
    mock.reset();
    mock
      .onPost("https://api.github.com/graphql")
-      .replyOnce(200, data)
+      .replyOnce(200, data_stats)
      .onPost("https://api.github.com/graphql")
-      .replyOnce(200, repositoriesWithZeroStarsData);
+      .replyOnce(200, data_repo_zero_stars);

    let stats = await fetchStats("anuraghazra");
    const rank = calculateRank({
@ -178,8 +167,7 @@ describe("Test fetchStats", () => {
      totalRepos: 5,
      followers: 100,
      contributions: 61,
-      // stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-      stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      stargazers: 300,
      prs: 300,
      issues: 200,
    });
@ -190,8 +178,7 @@ describe("Test fetchStats", () => {
      totalCommits: 150,
      totalIssues: 200,
      totalPRs: 300,
-      // totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-      totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      totalStars: 300,
      rank,
    });
  });
@ -207,8 +194,7 @@ describe("Test fetchStats", () => {
      totalRepos: 5,
      followers: 100,
      contributions: 61,
-      // stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-      stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      stargazers: 300,
      prs: 300,
      issues: 200,
    });
@ -219,8 +205,7 @@ describe("Test fetchStats", () => {
      totalCommits: 1050,
      totalIssues: 200,
      totalPRs: 300,
-      // totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-      totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      totalStars: 300,
      rank,
    });
  });
@ -236,8 +221,7 @@ describe("Test fetchStats", () => {
      totalRepos: 5,
      followers: 100,
      contributions: 61,
-      // stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-      stargazers: 200, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      stargazers: 200,
      prs: 300,
      issues: 200,
    });
@ -248,8 +232,82 @@ describe("Test fetchStats", () => {
      totalCommits: 1050,
      totalIssues: 200,
      totalPRs: 300,
-      // totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-      totalStars: 200, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      totalStars: 200,
+      rank,
+    });
+  });
+
+  it("should fetch two pages of stars if 'FETCH_MULTI_PAGE_STARS' env variable is set to `true`", async () => {
+    process.env.FETCH_MULTI_PAGE_STARS = true;
+
+    let stats = await fetchStats("anuraghazra");
+    const rank = calculateRank({
+      totalCommits: 100,
+      totalRepos: 5,
+      followers: 100,
+      contributions: 61,
+      stargazers: 400,
+      prs: 300,
+      issues: 200,
+    });
+
+    expect(stats).toStrictEqual({
+      contributedTo: 61,
+      name: "Anurag Hazra",
+      totalCommits: 100,
+      totalIssues: 200,
+      totalPRs: 300,
+      totalStars: 400,
+      rank,
+    });
+  });
+
+  it("should fetch one page of stars if 'FETCH_MULTI_PAGE_STARS' env variable is set to `false`", async () => {
+    process.env.FETCH_MULTI_PAGE_STARS = "false";
+
+    let stats = await fetchStats("anuraghazra");
+    const rank = calculateRank({
+      totalCommits: 100,
+      totalRepos: 5,
+      followers: 100,
+      contributions: 61,
+      stargazers: 300,
+      prs: 300,
+      issues: 200,
+    });
+
+    expect(stats).toStrictEqual({
+      contributedTo: 61,
+      name: "Anurag Hazra",
+      totalCommits: 100,
+      totalIssues: 200,
+      totalPRs: 300,
+      totalStars: 300,
+      rank,
+    });
+  });
+
+  it("should fetch one page of stars if 'FETCH_MULTI_PAGE_STARS' env variable is not set", async () => {
+    process.env.FETCH_MULTI_PAGE_STARS = undefined;
+
+    let stats = await fetchStats("anuraghazra");
+    const rank = calculateRank({
+      totalCommits: 100,
+      totalRepos: 5,
+      followers: 100,
+      contributions: 61,
+      stargazers: 300,
+      prs: 300,
+      issues: 200,
+    });
+
+    expect(stats).toStrictEqual({
+      contributedTo: 61,
+      name: "Anurag Hazra",
+      totalCommits: 100,
+      totalIssues: 200,
+      totalPRs: 300,
+      totalStars: 300,
      rank,
    });
  });