feat: enable multi page star fetching for private vercel instances (#2159)

* feat: enable multi-page stars' fetching for private vercel instances

This commit enables multi-page stars' support from fetching on private Vercel
instances. This feature can be disabled on the public Vercel instance by adding
the `FETCH_SINGLE_PAGE_STARS=true` as an env variable in the public Vercel
instance. This variable will not be present when people deploy their own Vercel
instance, causing the code to fetch multiple star pages.

* fix: improve stats multi-page fetching behavoir

This commit makes sure that the GraphQL api is only called one time per
100 repositories. The old method added one unnecesairy GraphQL call.

* docs: update documentation

* style: improve code syntax

Co-authored-by: Matteo Pierro <pierromatteo@gmail.com>

* lol happy new year

* docs: remove rate limit documentation for now

Remove the `FETCH_SINGLE_PAGE_STARS` from documentation for now since it
might confuse people.

* fix: fix error in automatic merge

* feat: make sure  env variable is read

Co-authored-by: Matteo Pierro <pierromatteo@gmail.com>
Co-authored-by: Anurag <hazru.anurag@gmail.com>

readme.md

@@ -93,7 +93,7 @@ Visit <https://indiafightscorona.giveindia.org> and make a small donation to hel
     -   [Language Card Exclusive Options](#language-card-exclusive-options)
     -   [Wakatime Card Exclusive Option](#wakatime-card-exclusive-options)
 -   [Deploy Yourself](#deploy-on-your-own-vercel-instance)
-    - [Keep your fork up to date](#keep-your-fork-up-to-date)   
+    -   [Keep your fork up to date](#keep-your-fork-up-to-date)
 
 # GitHub Stats Card
 
@@ -264,7 +264,7 @@ You can customize the appearance of your `Stats Card` or `Repo Card` however you
 -   `border_radius` - Corner rounding on the card. Default: `4.5`.
 
 > **Warning**
-> We use caching to decrease the load on our servers (see https://github.com/anuraghazra/github-readme-stats/issues/1471#issuecomment-1271551425). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours.
+> We use caching to decrease the load on our servers (see <https://github.com/anuraghazra/github-readme-stats/issues/1471#issuecomment-1271551425>). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours.
 
 ##### Gradient in bg_color
 
@@ -354,7 +354,7 @@ Use [show_owner](#customization) variable to include the repo's owner username
 The top languages card shows a GitHub user's most frequently used top language.
 
 > **Note**
-> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats._
+> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats.
 
 ### Usage
 
@@ -498,7 +498,7 @@ By default, GitHub does not lay out the cards side by side. To do that, you can
 
 ## Deploy on your own Vercel instance
 
-#### [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107)
+#### :film_projector: [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107)
 
 > **Warning**
 > If you are on the [hobby (i.e. free)](https://vercel.com/pricing) Vercel plan, please make sure you change the `maxDuration` parameter in the [vercel.json](https://github.com/anuraghazra/github-readme-stats/blob/master/vercel.json) file from `30` to `10` (see [#1416](https://github.com/anuraghazra/github-readme-stats/issues/1416#issuecomment-950275476) for more information).

src/fetchers/stats-fetcher.js

@@ -1,5 +1,6 @@
 // @ts-check
 import axios from "axios";
+import * as dotenv from "dotenv";
 import githubUsernameRegex from "github-username-regex";
 import { calculateRank } from "../calculateRank.js";
 import { retryer } from "../common/retryer.js";
@@ -11,46 +12,74 @@ import {
   wrapTextMultiline,
 } from "../common/utils.js";
 
+dotenv.config();
+
+// GraphQL queries.
+const GRAPHQL_REPOS_FIELD = `
+  repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) {
+    totalCount
+    nodes {
+      name
+      stargazers {
+        totalCount
+      }
+    }
+    pageInfo {
+      hasNextPage
+      endCursor
+    }
+  }
+`;
+
+const GRAPHQL_REPOS_QUERY = `
+  query userInfo($login: String!, $after: String) {
+    user(login: $login) {
+      ${GRAPHQL_REPOS_FIELD}
+    }
+  }
+`;
+
+const GRAPHQL_STATS_QUERY = `
+  query userInfo($login: String!, $after: String) {
+    user(login: $login) {
+      name
+      login
+      contributionsCollection {
+        totalCommitContributions
+        restrictedContributionsCount
+      }
+      repositoriesContributedTo(first: 1, contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) {
+        totalCount
+      }
+      pullRequests(first: 1) {
+        totalCount
+      }
+      openIssues: issues(states: OPEN) {
+        totalCount
+      }
+      closedIssues: issues(states: CLOSED) {
+        totalCount
+      }
+      followers {
+        totalCount
+      }
+      ${GRAPHQL_REPOS_FIELD}
+    }
+  }
+`;
+
 /**
  * Stats fetcher object.
  *
  * @param {import('axios').AxiosRequestHeaders} variables Fetcher variables.
  * @param {string} token GitHub token.
- * @returns {Promise<import('../common/types').StatsFetcherResponse>} Stats fetcher response.
+ * @returns {Promise<import('../common/types').Fetcher>} Stats fetcher response.
  */
 const fetcher = (variables, token) => {
+  const query = !variables.after ? GRAPHQL_STATS_QUERY : GRAPHQL_REPOS_QUERY;
   return request(
     {
-      query: `
+      query,
-      query userInfo($login: String!) {
-        user(login: $login) {
-          name
-          login
-          contributionsCollection {
-            totalCommitContributions
-            restrictedContributionsCount
-          }
-          repositoriesContributedTo(contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) {
-            totalCount
-          }
-          pullRequests {
-            totalCount
-          }
-          openIssues: issues(states: OPEN) {
-            totalCount
-          }
-          closedIssues: issues(states: CLOSED) {
-            totalCount
-          }
-          followers {
-            totalCount
-          }
-          repositories(ownerAffiliations: OWNER) {
-            totalCount
-          }
-        }
-      }
-      `,
       variables,
     },
     {
@@ -60,39 +89,42 @@ const fetcher = (variables, token) => {
 };
 
 /**
- * Fetch first 100 repositories for a given username.
+ * Fetch stats information for a given username.
  *
- * @param {import('axios').AxiosRequestHeaders} variables Fetcher variables.
+ * @param {string} username Github username.
- * @param {string} token GitHub token.
+ * @returns {Promise<import('../common/types').StatsFetcher>} GraphQL Stats object.
- * @returns {Promise<import('../common/types').StatsFetcherResponse>} Repositories fetcher response.
+ *
+ * @description This function supports multi-page fetching if the 'FETCH_MULTI_PAGE_STARS' environment variable is set to true.
  */
-const repositoriesFetcher = (variables, token) => {
+const statsFetcher = async (username) => {
-  return request(
+  let stats;
-    {
+  let hasNextPage = true;
-      query: `
+  let endCursor = null;
-      query userInfo($login: String!, $after: String) {
+  while (hasNextPage) {
-        user(login: $login) {
+    const variables = { login: username, first: 100, after: endCursor };
-          repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) {
+    let res = await retryer(fetcher, variables);
-            nodes {
+    if (res.data.errors) return res;
-              name
+
-              stargazers {
+    // Store stats data.
-                totalCount
+    const repoNodes = res.data.data.user.repositories.nodes;
-              }
+    if (!stats) {
-            }
+      stats = res;
-            pageInfo {
+    } else {
-              hasNextPage
+      stats.data.data.user.repositories.nodes.push(...repoNodes);
-              endCursor
+    }
-            }
+
-          }
+    // Disable multi page fetching on public Vercel instance due to rate limits.
-        }
+    const repoNodesWithStars = repoNodes.filter(
-      }
+      (node) => node.stargazers.totalCount !== 0,
-      `,
+    );
-      variables,
+    hasNextPage =
-    },
+      process.env.FETCH_MULTI_PAGE_STARS === "true" &&
-    {
+      repoNodes.length === repoNodesWithStars.length &&
-      Authorization: `bearer ${token}`,
+      res.data.data.user.repositories.pageInfo.hasNextPage;
-    },
+    endCursor = res.data.data.user.repositories.pageInfo.endCursor;
-  );
+  }
+
+  return stats;
 };
 
 /**
@@ -137,46 +169,6 @@ const totalCommitsFetcher = async (username) => {
   return 0;
 };
 
-/**
- * Fetch all the stars for all the repositories of a given username.
- *
- * @param {string} username GitHub username.
- * @param {array} repoToHide Repositories to hide.
- * @returns {Promise<number>} Total stars.
- */
-const totalStarsFetcher = async (username, repoToHide) => {
-  let nodes = [];
-  let hasNextPage = true;
-  let endCursor = null;
-  while (hasNextPage) {
-    const variables = { login: username, first: 100, after: endCursor };
-    let res = await retryer(repositoriesFetcher, variables);
-
-    if (res.data.errors) {
-      logger.error(res.data.errors);
-      throw new CustomError(
-        res.data.errors[0].message || "Could not fetch user",
-        CustomError.USER_NOT_FOUND,
-      );
-    }
-
-    const allNodes = res.data.data.user.repositories.nodes;
-    const nodesWithStars = allNodes.filter(
-      (node) => node.stargazers.totalCount !== 0,
-    );
-    nodes.push(...nodesWithStars);
-    // hasNextPage =
-    //   allNodes.length === nodesWithStars.length &&
-    //   res.data.data.user.repositories.pageInfo.hasNextPage;
-    hasNextPage = false; // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-    endCursor = res.data.data.user.repositories.pageInfo.endCursor;
-  }
-
-  return nodes
-    .filter((data) => !repoToHide[data.name])
-    .reduce((prev, curr) => prev + curr.stargazers.totalCount, 0);
-};
-
 /**
  * Fetch stats for a given username.
  *
@@ -203,7 +195,7 @@ const fetchStats = async (
     rank: { level: "C", score: 0 },
   };
 
-  let res = await retryer(fetcher, { login: username });
+  let res = await statsFetcher(username);
 
   // Catch GraphQL errors.
   if (res.data.errors) {
@@ -259,8 +251,15 @@ const fetchStats = async (
   stats.contributedTo = user.repositoriesContributedTo.totalCount;
 
   // Retrieve stars while filtering out repositories to be hidden
-  stats.totalStars = await totalStarsFetcher(username, repoToHide);
+  stats.totalStars = user.repositories.nodes
+    .filter((data) => {
+      return !repoToHide[data.name];
+    })
+    .reduce((prev, curr) => {
+      return prev + curr.stargazers.totalCount;
+    }, 0);
 
+  // @ts-ignore // TODO: Fix this.
   stats.rank = calculateRank({
     totalCommits: stats.totalCommits,
     totalRepos: user.repositories.totalCount,

tests/api.test.js

@@ -25,7 +25,7 @@ stats.rank = calculateRank({
   issues: stats.totalIssues,
 });
 
-const data = {
+const data_stats = {
   data: {
     user: {
       name: stats.name,
@@ -40,15 +40,6 @@ const data = {
       followers: { totalCount: 0 },
       repositories: {
         totalCount: 1,
-      },
-    },
-  },
-};
-
-const repositoriesData = {
-  data: {
-    user: {
-      repositories: {
         nodes: [{ stargazers: { totalCount: 100 } }],
         pageInfo: {
           hasNextPage: false,
@@ -83,11 +74,7 @@ const faker = (query, data) => {
     setHeader: jest.fn(),
     send: jest.fn(),
   };
-  mock
+  mock.onPost("https://api.github.com/graphql").replyOnce(200, data);
-    .onPost("https://api.github.com/graphql")
-    .replyOnce(200, data)
-    .onPost("https://api.github.com/graphql")
-    .replyOnce(200, repositoriesData);
 
   return { req, res };
 };
@@ -98,7 +85,7 @@ afterEach(() => {
 
 describe("Test /api/", () => {
   it("should test the request", async () => {
-    const { req, res } = faker({}, data);
+    const { req, res } = faker({}, data_stats);
 
     await api(req, res);
 
@@ -133,7 +120,7 @@ describe("Test /api/", () => {
         text_color: "fff",
         bg_color: "fff",
       },
-      data,
+      data_stats,
     );
 
     await api(req, res);
@@ -154,7 +141,7 @@ describe("Test /api/", () => {
   });
 
   it("should have proper cache", async () => {
-    const { req, res } = faker({}, data);
+    const { req, res } = faker({}, data_stats);
 
     await api(req, res);
 
@@ -170,7 +157,7 @@ describe("Test /api/", () => {
   });
 
   it("should set proper cache", async () => {
-    const { req, res } = faker({ cache_seconds: 15000 }, data);
+    const { req, res } = faker({ cache_seconds: 15000 }, data_stats);
     await api(req, res);
 
     expect(res.setHeader.mock.calls).toEqual([
@@ -196,7 +183,7 @@ describe("Test /api/", () => {
 
   it("should set proper cache with clamped values", async () => {
     {
-      let { req, res } = faker({ cache_seconds: 200000 }, data);
+      let { req, res } = faker({ cache_seconds: 200000 }, data_stats);
       await api(req, res);
 
       expect(res.setHeader.mock.calls).toEqual([
@@ -212,7 +199,7 @@ describe("Test /api/", () => {
 
     // note i'm using block scoped vars
     {
-      let { req, res } = faker({ cache_seconds: 0 }, data);
+      let { req, res } = faker({ cache_seconds: 0 }, data_stats);
       await api(req, res);
 
       expect(res.setHeader.mock.calls).toEqual([
@@ -227,7 +214,7 @@ describe("Test /api/", () => {
     }
 
     {
-      let { req, res } = faker({ cache_seconds: -10000 }, data);
+      let { req, res } = faker({ cache_seconds: -10000 }, data_stats);
       await api(req, res);
 
       expect(res.setHeader.mock.calls).toEqual([
@@ -248,7 +235,7 @@ describe("Test /api/", () => {
         username: "anuraghazra",
         count_private: true,
       },
-      data,
+      data_stats,
     );
 
     await api(req, res);
@@ -288,7 +275,7 @@ describe("Test /api/", () => {
         text_color: "fff",
         bg_color: "fff",
       },
-      data,
+      data_stats,
     );
 
     await api(req, res);

tests/fetchStats.test.js

@@ -4,7 +4,8 @@ import MockAdapter from "axios-mock-adapter";
 import { calculateRank } from "../src/calculateRank.js";
 import { fetchStats } from "../src/fetchers/stats-fetcher.js";
 
-const data = {
+// Test parameters.
+const data_stats = {
   data: {
     user: {
       name: "Anurag Hazra",
@@ -19,15 +20,6 @@ const data = {
       followers: { totalCount: 100 },
       repositories: {
         totalCount: 5,
-      },
-    },
-  },
-};
-
-const firstRepositoriesData = {
-  data: {
-    user: {
-      repositories: {
         nodes: [
           { name: "test-repo-1", stargazers: { totalCount: 100 } },
           { name: "test-repo-2", stargazers: { totalCount: 100 } },
@@ -42,7 +34,7 @@ const firstRepositoriesData = {
   },
 };
 
-const secondRepositoriesData = {
+const data_repo = {
   data: {
     user: {
       repositories: {
@@ -59,7 +51,7 @@ const secondRepositoriesData = {
   },
 };
 
-const repositoriesWithZeroStarsData = {
+const data_repo_zero_stars = {
   data: {
     user: {
       repositories: {
@@ -93,13 +85,12 @@ const error = {
 const mock = new MockAdapter(axios);
 
 beforeEach(() => {
+  process.env.FETCH_MULTI_PAGE_STARS = "false"; // Set to `false` to fetch only one page of stars.
   mock
     .onPost("https://api.github.com/graphql")
-    .replyOnce(200, data)
+    .replyOnce(200, data_stats)
     .onPost("https://api.github.com/graphql")
-    .replyOnce(200, firstRepositoriesData);
+    .replyOnce(200, data_repo);
-  // .onPost("https://api.github.com/graphql") // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-  // .replyOnce(200, secondRepositoriesData); // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
 });
 
 afterEach(() => {
@@ -114,8 +105,7 @@ describe("Test fetchStats", () => {
       totalRepos: 5,
       followers: 100,
       contributions: 61,
-      // stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      stargazers: 300,
-      stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
       prs: 300,
       issues: 200,
     });
@@ -126,8 +116,7 @@ describe("Test fetchStats", () => {
       totalCommits: 100,
       totalIssues: 200,
       totalPRs: 300,
-      // totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      totalStars: 300,
-      totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
       rank,
     });
   });
@@ -136,9 +125,9 @@ describe("Test fetchStats", () => {
     mock.reset();
     mock
       .onPost("https://api.github.com/graphql")
-      .replyOnce(200, data)
+      .replyOnce(200, data_stats)
       .onPost("https://api.github.com/graphql")
-      .replyOnce(200, repositoriesWithZeroStarsData);
+      .replyOnce(200, data_repo_zero_stars);
 
     let stats = await fetchStats("anuraghazra");
     const rank = calculateRank({
@@ -178,8 +167,7 @@ describe("Test fetchStats", () => {
       totalRepos: 5,
       followers: 100,
       contributions: 61,
-      // stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      stargazers: 300,
-      stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
       prs: 300,
       issues: 200,
     });
@@ -190,8 +178,7 @@ describe("Test fetchStats", () => {
       totalCommits: 150,
       totalIssues: 200,
       totalPRs: 300,
-      // totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      totalStars: 300,
-      totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
       rank,
     });
   });
@@ -207,8 +194,7 @@ describe("Test fetchStats", () => {
       totalRepos: 5,
       followers: 100,
       contributions: 61,
-      // stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      stargazers: 300,
-      stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
       prs: 300,
       issues: 200,
     });
@@ -219,8 +205,7 @@ describe("Test fetchStats", () => {
       totalCommits: 1050,
       totalIssues: 200,
       totalPRs: 300,
-      // totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      totalStars: 300,
-      totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
       rank,
     });
   });
@@ -236,8 +221,7 @@ describe("Test fetchStats", () => {
       totalRepos: 5,
       followers: 100,
       contributions: 61,
-      // stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      stargazers: 200,
-      stargazers: 200, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
       prs: 300,
       issues: 200,
     });
@@ -248,8 +232,82 @@ describe("Test fetchStats", () => {
       totalCommits: 1050,
       totalIssues: 200,
       totalPRs: 300,
-      // totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      totalStars: 200,
-      totalStars: 200, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
+      rank,
+    });
+  });
+
+  it("should fetch two pages of stars if 'FETCH_MULTI_PAGE_STARS' env variable is set to `true`", async () => {
+    process.env.FETCH_MULTI_PAGE_STARS = true;
+
+    let stats = await fetchStats("anuraghazra");
+    const rank = calculateRank({
+      totalCommits: 100,
+      totalRepos: 5,
+      followers: 100,
+      contributions: 61,
+      stargazers: 400,
+      prs: 300,
+      issues: 200,
+    });
+
+    expect(stats).toStrictEqual({
+      contributedTo: 61,
+      name: "Anurag Hazra",
+      totalCommits: 100,
+      totalIssues: 200,
+      totalPRs: 300,
+      totalStars: 400,
+      rank,
+    });
+  });
+
+  it("should fetch one page of stars if 'FETCH_MULTI_PAGE_STARS' env variable is set to `false`", async () => {
+    process.env.FETCH_MULTI_PAGE_STARS = "false";
+
+    let stats = await fetchStats("anuraghazra");
+    const rank = calculateRank({
+      totalCommits: 100,
+      totalRepos: 5,
+      followers: 100,
+      contributions: 61,
+      stargazers: 300,
+      prs: 300,
+      issues: 200,
+    });
+
+    expect(stats).toStrictEqual({
+      contributedTo: 61,
+      name: "Anurag Hazra",
+      totalCommits: 100,
+      totalIssues: 200,
+      totalPRs: 300,
+      totalStars: 300,
+      rank,
+    });
+  });
+
+  it("should fetch one page of stars if 'FETCH_MULTI_PAGE_STARS' env variable is not set", async () => {
+    process.env.FETCH_MULTI_PAGE_STARS = undefined;
+
+    let stats = await fetchStats("anuraghazra");
+    const rank = calculateRank({
+      totalCommits: 100,
+      totalRepos: 5,
+      followers: 100,
+      contributions: 61,
+      stargazers: 300,
+      prs: 300,
+      issues: 200,
+    });
+
+    expect(stats).toStrictEqual({
+      contributedTo: 61,
+      name: "Anurag Hazra",
+      totalCommits: 100,
+      totalIssues: 200,
+      totalPRs: 300,
+      totalStars: 300,
       rank,
     });
   });