From 60fae292a349cd3d92942dc56bf67778c213cb0d Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Sat, 21 Jan 2023 18:32:37 +0100 Subject: [PATCH] feat: enable multi page star fetching for private vercel instances (#2159) * feat: enable multi-page stars' fetching for private vercel instances This commit enables multi-page stars' support from fetching on private Vercel instances. This feature can be disabled on the public Vercel instance by adding the `FETCH_SINGLE_PAGE_STARS=true` as an env variable in the public Vercel instance. This variable will not be present when people deploy their own Vercel instance, causing the code to fetch multiple star pages. * fix: improve stats multi-page fetching behavoir This commit makes sure that the GraphQL api is only called one time per 100 repositories. The old method added one unnecesairy GraphQL call. * docs: update documentation * style: improve code syntax Co-authored-by: Matteo Pierro * lol happy new year * docs: remove rate limit documentation for now Remove the `FETCH_SINGLE_PAGE_STARS` from documentation for now since it might confuse people. * fix: fix error in automatic merge * feat: make sure env variable is read Co-authored-by: Matteo Pierro Co-authored-by: Anurag --- readme.md | 8 +- src/fetchers/stats-fetcher.js | 207 +++++++++++++++++----------------- tests/api.test.js | 35 ++---- tests/fetchStats.test.js | 126 +++++++++++++++------ 4 files changed, 210 insertions(+), 166 deletions(-) diff --git a/readme.md b/readme.md index 281c4877..678c5c0b 100644 --- a/readme.md +++ b/readme.md @@ -93,7 +93,7 @@ Visit and make a small donation to hel - [Language Card Exclusive Options](#language-card-exclusive-options) - [Wakatime Card Exclusive Option](#wakatime-card-exclusive-options) - [Deploy Yourself](#deploy-on-your-own-vercel-instance) - - [Keep your fork up to date](#keep-your-fork-up-to-date) + - [Keep your fork up to date](#keep-your-fork-up-to-date) # GitHub Stats Card @@ -264,7 +264,7 @@ You can customize the appearance of your `Stats Card` or `Repo Card` however you - `border_radius` - Corner rounding on the card. Default: `4.5`. > **Warning** -> We use caching to decrease the load on our servers (see https://github.com/anuraghazra/github-readme-stats/issues/1471#issuecomment-1271551425). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours. +> We use caching to decrease the load on our servers (see ). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours. ##### Gradient in bg_color @@ -354,7 +354,7 @@ Use [show_owner](#customization) variable to include the repo's owner username The top languages card shows a GitHub user's most frequently used top language. > **Note** -> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats._ +> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats. ### Usage @@ -498,7 +498,7 @@ By default, GitHub does not lay out the cards side by side. To do that, you can ## Deploy on your own Vercel instance -#### [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107) +#### :film_projector: [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107) > **Warning** > If you are on the [hobby (i.e. free)](https://vercel.com/pricing) Vercel plan, please make sure you change the `maxDuration` parameter in the [vercel.json](https://github.com/anuraghazra/github-readme-stats/blob/master/vercel.json) file from `30` to `10` (see [#1416](https://github.com/anuraghazra/github-readme-stats/issues/1416#issuecomment-950275476) for more information). diff --git a/src/fetchers/stats-fetcher.js b/src/fetchers/stats-fetcher.js index 7f6cb9e5..a7df1e50 100644 --- a/src/fetchers/stats-fetcher.js +++ b/src/fetchers/stats-fetcher.js @@ -1,5 +1,6 @@ // @ts-check import axios from "axios"; +import * as dotenv from "dotenv"; import githubUsernameRegex from "github-username-regex"; import { calculateRank } from "../calculateRank.js"; import { retryer } from "../common/retryer.js"; @@ -11,46 +12,74 @@ import { wrapTextMultiline, } from "../common/utils.js"; +dotenv.config(); + +// GraphQL queries. +const GRAPHQL_REPOS_FIELD = ` + repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) { + totalCount + nodes { + name + stargazers { + totalCount + } + } + pageInfo { + hasNextPage + endCursor + } + } +`; + +const GRAPHQL_REPOS_QUERY = ` + query userInfo($login: String!, $after: String) { + user(login: $login) { + ${GRAPHQL_REPOS_FIELD} + } + } +`; + +const GRAPHQL_STATS_QUERY = ` + query userInfo($login: String!, $after: String) { + user(login: $login) { + name + login + contributionsCollection { + totalCommitContributions + restrictedContributionsCount + } + repositoriesContributedTo(first: 1, contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) { + totalCount + } + pullRequests(first: 1) { + totalCount + } + openIssues: issues(states: OPEN) { + totalCount + } + closedIssues: issues(states: CLOSED) { + totalCount + } + followers { + totalCount + } + ${GRAPHQL_REPOS_FIELD} + } + } +`; + /** * Stats fetcher object. * * @param {import('axios').AxiosRequestHeaders} variables Fetcher variables. * @param {string} token GitHub token. - * @returns {Promise} Stats fetcher response. + * @returns {Promise} Stats fetcher response. */ const fetcher = (variables, token) => { + const query = !variables.after ? GRAPHQL_STATS_QUERY : GRAPHQL_REPOS_QUERY; return request( { - query: ` - query userInfo($login: String!) { - user(login: $login) { - name - login - contributionsCollection { - totalCommitContributions - restrictedContributionsCount - } - repositoriesContributedTo(contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) { - totalCount - } - pullRequests { - totalCount - } - openIssues: issues(states: OPEN) { - totalCount - } - closedIssues: issues(states: CLOSED) { - totalCount - } - followers { - totalCount - } - repositories(ownerAffiliations: OWNER) { - totalCount - } - } - } - `, + query, variables, }, { @@ -60,39 +89,42 @@ const fetcher = (variables, token) => { }; /** - * Fetch first 100 repositories for a given username. + * Fetch stats information for a given username. * - * @param {import('axios').AxiosRequestHeaders} variables Fetcher variables. - * @param {string} token GitHub token. - * @returns {Promise} Repositories fetcher response. + * @param {string} username Github username. + * @returns {Promise} GraphQL Stats object. + * + * @description This function supports multi-page fetching if the 'FETCH_MULTI_PAGE_STARS' environment variable is set to true. */ -const repositoriesFetcher = (variables, token) => { - return request( - { - query: ` - query userInfo($login: String!, $after: String) { - user(login: $login) { - repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) { - nodes { - name - stargazers { - totalCount - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - } - `, - variables, - }, - { - Authorization: `bearer ${token}`, - }, - ); +const statsFetcher = async (username) => { + let stats; + let hasNextPage = true; + let endCursor = null; + while (hasNextPage) { + const variables = { login: username, first: 100, after: endCursor }; + let res = await retryer(fetcher, variables); + if (res.data.errors) return res; + + // Store stats data. + const repoNodes = res.data.data.user.repositories.nodes; + if (!stats) { + stats = res; + } else { + stats.data.data.user.repositories.nodes.push(...repoNodes); + } + + // Disable multi page fetching on public Vercel instance due to rate limits. + const repoNodesWithStars = repoNodes.filter( + (node) => node.stargazers.totalCount !== 0, + ); + hasNextPage = + process.env.FETCH_MULTI_PAGE_STARS === "true" && + repoNodes.length === repoNodesWithStars.length && + res.data.data.user.repositories.pageInfo.hasNextPage; + endCursor = res.data.data.user.repositories.pageInfo.endCursor; + } + + return stats; }; /** @@ -137,46 +169,6 @@ const totalCommitsFetcher = async (username) => { return 0; }; -/** - * Fetch all the stars for all the repositories of a given username. - * - * @param {string} username GitHub username. - * @param {array} repoToHide Repositories to hide. - * @returns {Promise} Total stars. - */ -const totalStarsFetcher = async (username, repoToHide) => { - let nodes = []; - let hasNextPage = true; - let endCursor = null; - while (hasNextPage) { - const variables = { login: username, first: 100, after: endCursor }; - let res = await retryer(repositoriesFetcher, variables); - - if (res.data.errors) { - logger.error(res.data.errors); - throw new CustomError( - res.data.errors[0].message || "Could not fetch user", - CustomError.USER_NOT_FOUND, - ); - } - - const allNodes = res.data.data.user.repositories.nodes; - const nodesWithStars = allNodes.filter( - (node) => node.stargazers.totalCount !== 0, - ); - nodes.push(...nodesWithStars); - // hasNextPage = - // allNodes.length === nodesWithStars.length && - // res.data.data.user.repositories.pageInfo.hasNextPage; - hasNextPage = false; // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - endCursor = res.data.data.user.repositories.pageInfo.endCursor; - } - - return nodes - .filter((data) => !repoToHide[data.name]) - .reduce((prev, curr) => prev + curr.stargazers.totalCount, 0); -}; - /** * Fetch stats for a given username. * @@ -203,7 +195,7 @@ const fetchStats = async ( rank: { level: "C", score: 0 }, }; - let res = await retryer(fetcher, { login: username }); + let res = await statsFetcher(username); // Catch GraphQL errors. if (res.data.errors) { @@ -259,8 +251,15 @@ const fetchStats = async ( stats.contributedTo = user.repositoriesContributedTo.totalCount; // Retrieve stars while filtering out repositories to be hidden - stats.totalStars = await totalStarsFetcher(username, repoToHide); + stats.totalStars = user.repositories.nodes + .filter((data) => { + return !repoToHide[data.name]; + }) + .reduce((prev, curr) => { + return prev + curr.stargazers.totalCount; + }, 0); + // @ts-ignore // TODO: Fix this. stats.rank = calculateRank({ totalCommits: stats.totalCommits, totalRepos: user.repositories.totalCount, diff --git a/tests/api.test.js b/tests/api.test.js index 0037bcdb..461f3e18 100644 --- a/tests/api.test.js +++ b/tests/api.test.js @@ -25,7 +25,7 @@ stats.rank = calculateRank({ issues: stats.totalIssues, }); -const data = { +const data_stats = { data: { user: { name: stats.name, @@ -40,15 +40,6 @@ const data = { followers: { totalCount: 0 }, repositories: { totalCount: 1, - }, - }, - }, -}; - -const repositoriesData = { - data: { - user: { - repositories: { nodes: [{ stargazers: { totalCount: 100 } }], pageInfo: { hasNextPage: false, @@ -83,11 +74,7 @@ const faker = (query, data) => { setHeader: jest.fn(), send: jest.fn(), }; - mock - .onPost("https://api.github.com/graphql") - .replyOnce(200, data) - .onPost("https://api.github.com/graphql") - .replyOnce(200, repositoriesData); + mock.onPost("https://api.github.com/graphql").replyOnce(200, data); return { req, res }; }; @@ -98,7 +85,7 @@ afterEach(() => { describe("Test /api/", () => { it("should test the request", async () => { - const { req, res } = faker({}, data); + const { req, res } = faker({}, data_stats); await api(req, res); @@ -133,7 +120,7 @@ describe("Test /api/", () => { text_color: "fff", bg_color: "fff", }, - data, + data_stats, ); await api(req, res); @@ -154,7 +141,7 @@ describe("Test /api/", () => { }); it("should have proper cache", async () => { - const { req, res } = faker({}, data); + const { req, res } = faker({}, data_stats); await api(req, res); @@ -170,7 +157,7 @@ describe("Test /api/", () => { }); it("should set proper cache", async () => { - const { req, res } = faker({ cache_seconds: 15000 }, data); + const { req, res } = faker({ cache_seconds: 15000 }, data_stats); await api(req, res); expect(res.setHeader.mock.calls).toEqual([ @@ -196,7 +183,7 @@ describe("Test /api/", () => { it("should set proper cache with clamped values", async () => { { - let { req, res } = faker({ cache_seconds: 200000 }, data); + let { req, res } = faker({ cache_seconds: 200000 }, data_stats); await api(req, res); expect(res.setHeader.mock.calls).toEqual([ @@ -212,7 +199,7 @@ describe("Test /api/", () => { // note i'm using block scoped vars { - let { req, res } = faker({ cache_seconds: 0 }, data); + let { req, res } = faker({ cache_seconds: 0 }, data_stats); await api(req, res); expect(res.setHeader.mock.calls).toEqual([ @@ -227,7 +214,7 @@ describe("Test /api/", () => { } { - let { req, res } = faker({ cache_seconds: -10000 }, data); + let { req, res } = faker({ cache_seconds: -10000 }, data_stats); await api(req, res); expect(res.setHeader.mock.calls).toEqual([ @@ -248,7 +235,7 @@ describe("Test /api/", () => { username: "anuraghazra", count_private: true, }, - data, + data_stats, ); await api(req, res); @@ -288,7 +275,7 @@ describe("Test /api/", () => { text_color: "fff", bg_color: "fff", }, - data, + data_stats, ); await api(req, res); diff --git a/tests/fetchStats.test.js b/tests/fetchStats.test.js index 192146ea..04e943a7 100644 --- a/tests/fetchStats.test.js +++ b/tests/fetchStats.test.js @@ -4,7 +4,8 @@ import MockAdapter from "axios-mock-adapter"; import { calculateRank } from "../src/calculateRank.js"; import { fetchStats } from "../src/fetchers/stats-fetcher.js"; -const data = { +// Test parameters. +const data_stats = { data: { user: { name: "Anurag Hazra", @@ -19,15 +20,6 @@ const data = { followers: { totalCount: 100 }, repositories: { totalCount: 5, - }, - }, - }, -}; - -const firstRepositoriesData = { - data: { - user: { - repositories: { nodes: [ { name: "test-repo-1", stargazers: { totalCount: 100 } }, { name: "test-repo-2", stargazers: { totalCount: 100 } }, @@ -42,7 +34,7 @@ const firstRepositoriesData = { }, }; -const secondRepositoriesData = { +const data_repo = { data: { user: { repositories: { @@ -59,7 +51,7 @@ const secondRepositoriesData = { }, }; -const repositoriesWithZeroStarsData = { +const data_repo_zero_stars = { data: { user: { repositories: { @@ -93,13 +85,12 @@ const error = { const mock = new MockAdapter(axios); beforeEach(() => { + process.env.FETCH_MULTI_PAGE_STARS = "false"; // Set to `false` to fetch only one page of stars. mock .onPost("https://api.github.com/graphql") - .replyOnce(200, data) + .replyOnce(200, data_stats) .onPost("https://api.github.com/graphql") - .replyOnce(200, firstRepositoriesData); - // .onPost("https://api.github.com/graphql") // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - // .replyOnce(200, secondRepositoriesData); // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. + .replyOnce(200, data_repo); }); afterEach(() => { @@ -114,8 +105,7 @@ describe("Test fetchStats", () => { totalRepos: 5, followers: 100, contributions: 61, - // stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. + stargazers: 300, prs: 300, issues: 200, }); @@ -126,8 +116,7 @@ describe("Test fetchStats", () => { totalCommits: 100, totalIssues: 200, totalPRs: 300, - // totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. + totalStars: 300, rank, }); }); @@ -136,9 +125,9 @@ describe("Test fetchStats", () => { mock.reset(); mock .onPost("https://api.github.com/graphql") - .replyOnce(200, data) + .replyOnce(200, data_stats) .onPost("https://api.github.com/graphql") - .replyOnce(200, repositoriesWithZeroStarsData); + .replyOnce(200, data_repo_zero_stars); let stats = await fetchStats("anuraghazra"); const rank = calculateRank({ @@ -178,8 +167,7 @@ describe("Test fetchStats", () => { totalRepos: 5, followers: 100, contributions: 61, - // stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. + stargazers: 300, prs: 300, issues: 200, }); @@ -190,8 +178,7 @@ describe("Test fetchStats", () => { totalCommits: 150, totalIssues: 200, totalPRs: 300, - // totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. + totalStars: 300, rank, }); }); @@ -207,8 +194,7 @@ describe("Test fetchStats", () => { totalRepos: 5, followers: 100, contributions: 61, - // stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. + stargazers: 300, prs: 300, issues: 200, }); @@ -219,8 +205,7 @@ describe("Test fetchStats", () => { totalCommits: 1050, totalIssues: 200, totalPRs: 300, - // totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. + totalStars: 300, rank, }); }); @@ -236,8 +221,7 @@ describe("Test fetchStats", () => { totalRepos: 5, followers: 100, contributions: 61, - // stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - stargazers: 200, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. + stargazers: 200, prs: 300, issues: 200, }); @@ -248,8 +232,82 @@ describe("Test fetchStats", () => { totalCommits: 1050, totalIssues: 200, totalPRs: 300, - // totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. - totalStars: 200, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130. + totalStars: 200, + rank, + }); + }); + + it("should fetch two pages of stars if 'FETCH_MULTI_PAGE_STARS' env variable is set to `true`", async () => { + process.env.FETCH_MULTI_PAGE_STARS = true; + + let stats = await fetchStats("anuraghazra"); + const rank = calculateRank({ + totalCommits: 100, + totalRepos: 5, + followers: 100, + contributions: 61, + stargazers: 400, + prs: 300, + issues: 200, + }); + + expect(stats).toStrictEqual({ + contributedTo: 61, + name: "Anurag Hazra", + totalCommits: 100, + totalIssues: 200, + totalPRs: 300, + totalStars: 400, + rank, + }); + }); + + it("should fetch one page of stars if 'FETCH_MULTI_PAGE_STARS' env variable is set to `false`", async () => { + process.env.FETCH_MULTI_PAGE_STARS = "false"; + + let stats = await fetchStats("anuraghazra"); + const rank = calculateRank({ + totalCommits: 100, + totalRepos: 5, + followers: 100, + contributions: 61, + stargazers: 300, + prs: 300, + issues: 200, + }); + + expect(stats).toStrictEqual({ + contributedTo: 61, + name: "Anurag Hazra", + totalCommits: 100, + totalIssues: 200, + totalPRs: 300, + totalStars: 300, + rank, + }); + }); + + it("should fetch one page of stars if 'FETCH_MULTI_PAGE_STARS' env variable is not set", async () => { + process.env.FETCH_MULTI_PAGE_STARS = undefined; + + let stats = await fetchStats("anuraghazra"); + const rank = calculateRank({ + totalCommits: 100, + totalRepos: 5, + followers: 100, + contributions: 61, + stargazers: 300, + prs: 300, + issues: 200, + }); + + expect(stats).toStrictEqual({ + contributedTo: 61, + name: "Anurag Hazra", + totalCommits: 100, + totalIssues: 200, + totalPRs: 300, + totalStars: 300, rank, }); });