mirror/github-readme-stats
2
0
Fork 0
mirror of https://github.com/anuraghazra/github-readme-stats.git synced 2024-12-21 06:11:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: enable multi page star fetching for private vercel instances (#2159)

Browse Source 
* feat: enable multi-page stars' fetching for private vercel instances

This commit enables multi-page stars' support from fetching on private Vercel
instances. This feature can be disabled on the public Vercel instance by adding
the `FETCH_SINGLE_PAGE_STARS=true` as an env variable in the public Vercel
instance. This variable will not be present when people deploy their own Vercel
instance, causing the code to fetch multiple star pages.

* fix: improve stats multi-page fetching behavoir

This commit makes sure that the GraphQL api is only called one time per
100 repositories. The old method added one unnecesairy GraphQL call.

* docs: update documentation

* style: improve code syntax

Co-authored-by: Matteo Pierro <pierromatteo@gmail.com>

* lol happy new year

* docs: remove rate limit documentation for now

Remove the `FETCH_SINGLE_PAGE_STARS` from documentation for now since it
might confuse people.

* fix: fix error in automatic merge

* feat: make sure  env variable is read

Co-authored-by: Matteo Pierro <pierromatteo@gmail.com>
Co-authored-by: Anurag <hazru.anurag@gmail.com>
This commit is contained in:
Rick Staa 2023-01-21 18:32:37 +01:00 committed by GitHub
parent c1dc7b850c
commit 60fae292a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 210 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
readme.md
View File

@ -264,7 +264,7 @@ You can customize the appearance of your `Stats Card` or `Repo Card` however you
- `border_radius` - Corner rounding on the card. Default: `4.5`.
> **Warning**
> We use caching to decrease the load on our servers (see https://github.com/anuraghazra/github-readme-stats/issues/1471#issuecomment-1271551425). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours.
> We use caching to decrease the load on our servers (see <https://github.com/anuraghazra/github-readme-stats/issues/1471#issuecomment-1271551425>). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours.
##### Gradient in bg_color
@ -354,7 +354,7 @@ Use [show_owner](#customization) variable to include the repo's owner username
The top languages card shows a GitHub user's most frequently used top language.
> **Note**
> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats._
> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats.
### Usage
@ -498,7 +498,7 @@ By default, GitHub does not lay out the cards side by side. To do that, you can
## Deploy on your own Vercel instance
#### [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107)
#### :film_projector: [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107)
> **Warning**
> If you are on the [hobby (i.e. free)](https://vercel.com/pricing) Vercel plan, please make sure you change the `maxDuration` parameter in the [vercel.json](https://github.com/anuraghazra/github-readme-stats/blob/master/vercel.json) file from `30` to `10` (see [#1416](https://github.com/anuraghazra/github-readme-stats/issues/1416#issuecomment-950275476) for more information).

205
src/fetchers/stats-fetcher.js
View File

@ -1,5 +1,6 @@
// @ts-check
import axios from "axios";
import * as dotenv from "dotenv";
import githubUsernameRegex from "github-username-regex";
import { calculateRank } from "../calculateRank.js";
import { retryer } from "../common/retryer.js";
@ -11,68 +12,12 @@ import {
wrapTextMultiline,
} from "../common/utils.js";
/**
* Stats fetcher object.
*
* @param {import('axios').AxiosRequestHeaders} variables Fetcher variables.
* @param {string} token GitHub token.
* @returns {Promise<import('../common/types').StatsFetcherResponse>} Stats fetcher response.
*/
const fetcher = (variables, token) => {
return request(
{
query: `
query userInfo($login: String!) {
user(login: $login) {
name
login
contributionsCollection {
totalCommitContributions
restrictedContributionsCount
}
repositoriesContributedTo(contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) {
totalCount
}
pullRequests {
totalCount
}
openIssues: issues(states: OPEN) {
totalCount
}
closedIssues: issues(states: CLOSED) {
totalCount
}
followers {
totalCount
}
repositories(ownerAffiliations: OWNER) {
totalCount
}
}
}
`,
variables,
},
{
Authorization: `bearer ${token}`,
},
);
};
dotenv.config();
/**
* Fetch first 100 repositories for a given username.
*
* @param {import('axios').AxiosRequestHeaders} variables Fetcher variables.
* @param {string} token GitHub token.
* @returns {Promise<import('../common/types').StatsFetcherResponse>} Repositories fetcher response.
*/
const repositoriesFetcher = (variables, token) => {
return request(
{
query: `
query userInfo($login: String!, $after: String) {
user(login: $login) {
// GraphQL queries.
const GRAPHQL_REPOS_FIELD = `
repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) {
totalCount
nodes {
name
stargazers {
@ -84,9 +29,57 @@ const repositoriesFetcher = (variables, token) => {
endCursor
}
}
`;
const GRAPHQL_REPOS_QUERY = `
query userInfo($login: String!, $after: String) {
user(login: $login) {
${GRAPHQL_REPOS_FIELD}
}
}
`,
`;
const GRAPHQL_STATS_QUERY = `
query userInfo($login: String!, $after: String) {
user(login: $login) {
name
login
contributionsCollection {
totalCommitContributions
restrictedContributionsCount
}
repositoriesContributedTo(first: 1, contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) {
totalCount
}
pullRequests(first: 1) {
totalCount
}
openIssues: issues(states: OPEN) {
totalCount
}
closedIssues: issues(states: CLOSED) {
totalCount
}
followers {
totalCount
}
${GRAPHQL_REPOS_FIELD}
}
}
`;
/**
* Stats fetcher object.
*
* @param {import('axios').AxiosRequestHeaders} variables Fetcher variables.
* @param {string} token GitHub token.
* @returns {Promise<import('../common/types').Fetcher>} Stats fetcher response.
*/
const fetcher = (variables, token) => {
const query = !variables.after ? GRAPHQL_STATS_QUERY : GRAPHQL_REPOS_QUERY;
return request(
{
query,
variables,
},
{
@ -95,6 +88,45 @@ const repositoriesFetcher = (variables, token) => {
);
};
/**
* Fetch stats information for a given username.
*
* @param {string} username Github username.
* @returns {Promise<import('../common/types').StatsFetcher>} GraphQL Stats object.
*
* @description This function supports multi-page fetching if the 'FETCH_MULTI_PAGE_STARS' environment variable is set to true.
*/
const statsFetcher = async (username) => {
let stats;
let hasNextPage = true;
let endCursor = null;
while (hasNextPage) {
const variables = { login: username, first: 100, after: endCursor };
let res = await retryer(fetcher, variables);
if (res.data.errors) return res;
// Store stats data.
const repoNodes = res.data.data.user.repositories.nodes;
if (!stats) {
stats = res;
} else {
stats.data.data.user.repositories.nodes.push(...repoNodes);
}
// Disable multi page fetching on public Vercel instance due to rate limits.
const repoNodesWithStars = repoNodes.filter(
(node) => node.stargazers.totalCount !== 0,
);
hasNextPage =
process.env.FETCH_MULTI_PAGE_STARS === "true" &&
repoNodes.length === repoNodesWithStars.length &&
res.data.data.user.repositories.pageInfo.hasNextPage;
endCursor = res.data.data.user.repositories.pageInfo.endCursor;
}
return stats;
};
/**
* Fetch all the commits for all the repositories of a given username.
*
@ -137,46 +169,6 @@ const totalCommitsFetcher = async (username) => {
return 0;
};
/**
* Fetch all the stars for all the repositories of a given username.
*
* @param {string} username GitHub username.
* @param {array} repoToHide Repositories to hide.
* @returns {Promise<number>} Total stars.
*/
const totalStarsFetcher = async (username, repoToHide) => {
let nodes = [];
let hasNextPage = true;
let endCursor = null;
while (hasNextPage) {
const variables = { login: username, first: 100, after: endCursor };
let res = await retryer(repositoriesFetcher, variables);
if (res.data.errors) {
logger.error(res.data.errors);
throw new CustomError(
res.data.errors[0].message || "Could not fetch user",
CustomError.USER_NOT_FOUND,
);
}
const allNodes = res.data.data.user.repositories.nodes;
const nodesWithStars = allNodes.filter(
(node) => node.stargazers.totalCount !== 0,
);
nodes.push(...nodesWithStars);
// hasNextPage =
// allNodes.length === nodesWithStars.length &&
// res.data.data.user.repositories.pageInfo.hasNextPage;
hasNextPage = false; // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
endCursor = res.data.data.user.repositories.pageInfo.endCursor;
}
return nodes
.filter((data) => !repoToHide[data.name])
.reduce((prev, curr) => prev + curr.stargazers.totalCount, 0);
};
/**
* Fetch stats for a given username.
*
@ -203,7 +195,7 @@ const fetchStats = async (
rank: { level: "C", score: 0 },
};
let res = await retryer(fetcher, { login: username });
let res = await statsFetcher(username);
// Catch GraphQL errors.
if (res.data.errors) {
@ -259,8 +251,15 @@ const fetchStats = async (
stats.contributedTo = user.repositoriesContributedTo.totalCount;
// Retrieve stars while filtering out repositories to be hidden
stats.totalStars = await totalStarsFetcher(username, repoToHide);
stats.totalStars = user.repositories.nodes
.filter((data) => {
return !repoToHide[data.name];
})
.reduce((prev, curr) => {
return prev + curr.stargazers.totalCount;
}, 0);
// @ts-ignore // TODO: Fix this.
stats.rank = calculateRank({
totalCommits: stats.totalCommits,
totalRepos: user.repositories.totalCount,

35
tests/api.test.js
View File

@ -25,7 +25,7 @@ stats.rank = calculateRank({
issues: stats.totalIssues,
});
const data = {
const data_stats = {
data: {
user: {
name: stats.name,
@ -40,15 +40,6 @@ const data = {
followers: { totalCount: 0 },
repositories: {
totalCount: 1,
},
},
},
};
const repositoriesData = {
data: {
user: {
repositories: {
nodes: [{ stargazers: { totalCount: 100 } }],
pageInfo: {
hasNextPage: false,
@ -83,11 +74,7 @@ const faker = (query, data) => {
setHeader: jest.fn(),
send: jest.fn(),
};
mock
.onPost("https://api.github.com/graphql")
.replyOnce(200, data)
.onPost("https://api.github.com/graphql")
.replyOnce(200, repositoriesData);
mock.onPost("https://api.github.com/graphql").replyOnce(200, data);
return { req, res };
};
@ -98,7 +85,7 @@ afterEach(() => {
describe("Test /api/", () => {
it("should test the request", async () => {
const { req, res } = faker({}, data);
const { req, res } = faker({}, data_stats);
await api(req, res);
@ -133,7 +120,7 @@ describe("Test /api/", () => {
text_color: "fff",
bg_color: "fff",
},
data,
data_stats,
);
await api(req, res);
@ -154,7 +141,7 @@ describe("Test /api/", () => {
});
it("should have proper cache", async () => {
const { req, res } = faker({}, data);
const { req, res } = faker({}, data_stats);
await api(req, res);
@ -170,7 +157,7 @@ describe("Test /api/", () => {
});
it("should set proper cache", async () => {
const { req, res } = faker({ cache_seconds: 15000 }, data);
const { req, res } = faker({ cache_seconds: 15000 }, data_stats);
await api(req, res);
expect(res.setHeader.mock.calls).toEqual([
@ -196,7 +183,7 @@ describe("Test /api/", () => {
it("should set proper cache with clamped values", async () => {
{
let { req, res } = faker({ cache_seconds: 200000 }, data);
let { req, res } = faker({ cache_seconds: 200000 }, data_stats);
await api(req, res);
expect(res.setHeader.mock.calls).toEqual([
@ -212,7 +199,7 @@ describe("Test /api/", () => {
// note i'm using block scoped vars
{
let { req, res } = faker({ cache_seconds: 0 }, data);
let { req, res } = faker({ cache_seconds: 0 }, data_stats);
await api(req, res);
expect(res.setHeader.mock.calls).toEqual([
@ -227,7 +214,7 @@ describe("Test /api/", () => {
}
{
let { req, res } = faker({ cache_seconds: -10000 }, data);
let { req, res } = faker({ cache_seconds: -10000 }, data_stats);
await api(req, res);
expect(res.setHeader.mock.calls).toEqual([
@ -248,7 +235,7 @@ describe("Test /api/", () => {
username: "anuraghazra",
count_private: true,
},
data,
data_stats,
);
await api(req, res);
@ -288,7 +275,7 @@ describe("Test /api/", () => {
text_color: "fff",
bg_color: "fff",
},
data,
data_stats,
);
await api(req, res);

126
tests/fetchStats.test.js
View File

@ -4,7 +4,8 @@ import MockAdapter from "axios-mock-adapter";
import { calculateRank } from "../src/calculateRank.js";
import { fetchStats } from "../src/fetchers/stats-fetcher.js";
const data = {
// Test parameters.
const data_stats = {
data: {
user: {
name: "Anurag Hazra",
@ -19,15 +20,6 @@ const data = {
followers: { totalCount: 100 },
repositories: {
totalCount: 5,
},
},
},
};
const firstRepositoriesData = {
data: {
user: {
repositories: {
nodes: [
{ name: "test-repo-1", stargazers: { totalCount: 100 } },
{ name: "test-repo-2", stargazers: { totalCount: 100 } },
@ -42,7 +34,7 @@ const firstRepositoriesData = {
},
};
const secondRepositoriesData = {
const data_repo = {
data: {
user: {
repositories: {
@ -59,7 +51,7 @@ const secondRepositoriesData = {
},
};
const repositoriesWithZeroStarsData = {
const data_repo_zero_stars = {
data: {
user: {
repositories: {
@ -93,13 +85,12 @@ const error = {
const mock = new MockAdapter(axios);
beforeEach(() => {
process.env.FETCH_MULTI_PAGE_STARS = "false"; // Set to `false` to fetch only one page of stars.
mock
.onPost("https://api.github.com/graphql")
.replyOnce(200, data)
.replyOnce(200, data_stats)
.onPost("https://api.github.com/graphql")
.replyOnce(200, firstRepositoriesData);
// .onPost("https://api.github.com/graphql") // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
// .replyOnce(200, secondRepositoriesData); // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
.replyOnce(200, data_repo);
});
afterEach(() => {
@ -114,8 +105,7 @@ describe("Test fetchStats", () => {
totalRepos: 5,
followers: 100,
contributions: 61,
// stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
stargazers: 300,
prs: 300,
issues: 200,
});
@ -126,8 +116,7 @@ describe("Test fetchStats", () => {
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
// totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
totalStars: 300,
rank,
});
});
@ -136,9 +125,9 @@ describe("Test fetchStats", () => {
mock.reset();
mock
.onPost("https://api.github.com/graphql")
.replyOnce(200, data)
.replyOnce(200, data_stats)
.onPost("https://api.github.com/graphql")
.replyOnce(200, repositoriesWithZeroStarsData);
.replyOnce(200, data_repo_zero_stars);
let stats = await fetchStats("anuraghazra");
const rank = calculateRank({
@ -178,8 +167,7 @@ describe("Test fetchStats", () => {
totalRepos: 5,
followers: 100,
contributions: 61,
// stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
stargazers: 300,
prs: 300,
issues: 200,
});
@ -190,8 +178,7 @@ describe("Test fetchStats", () => {
totalCommits: 150,
totalIssues: 200,
totalPRs: 300,
// totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
totalStars: 300,
rank,
});
});
@ -207,8 +194,7 @@ describe("Test fetchStats", () => {
totalRepos: 5,
followers: 100,
contributions: 61,
// stargazers: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
stargazers: 300,
prs: 300,
issues: 200,
});
@ -219,8 +205,7 @@ describe("Test fetchStats", () => {
totalCommits: 1050,
totalIssues: 200,
totalPRs: 300,
// totalStars: 400, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
totalStars: 300,
rank,
});
});
@ -236,8 +221,7 @@ describe("Test fetchStats", () => {
totalRepos: 5,
followers: 100,
contributions: 61,
// stargazers: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
stargazers: 200, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
stargazers: 200,
prs: 300,
issues: 200,
});
@ -248,8 +232,82 @@ describe("Test fetchStats", () => {
totalCommits: 1050,
totalIssues: 200,
totalPRs: 300,
// totalStars: 300, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
totalStars: 200, // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
totalStars: 200,
rank,
});
});
it("should fetch two pages of stars if 'FETCH_MULTI_PAGE_STARS' env variable is set to `true`", async () => {
process.env.FETCH_MULTI_PAGE_STARS = true;
let stats = await fetchStats("anuraghazra");
const rank = calculateRank({
totalCommits: 100,
totalRepos: 5,
followers: 100,
contributions: 61,
stargazers: 400,
prs: 300,
issues: 200,
});
expect(stats).toStrictEqual({
contributedTo: 61,
name: "Anurag Hazra",
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
totalStars: 400,
rank,
});
});
it("should fetch one page of stars if 'FETCH_MULTI_PAGE_STARS' env variable is set to `false`", async () => {
process.env.FETCH_MULTI_PAGE_STARS = "false";
let stats = await fetchStats("anuraghazra");
const rank = calculateRank({
totalCommits: 100,
totalRepos: 5,
followers: 100,
contributions: 61,
stargazers: 300,
prs: 300,
issues: 200,
});
expect(stats).toStrictEqual({
contributedTo: 61,
name: "Anurag Hazra",
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
totalStars: 300,
rank,
});
});
it("should fetch one page of stars if 'FETCH_MULTI_PAGE_STARS' env variable is not set", async () => {
process.env.FETCH_MULTI_PAGE_STARS = undefined;
let stats = await fetchStats("anuraghazra");
const rank = calculateRank({
totalCommits: 100,
totalRepos: 5,
followers: 100,
contributions: 61,
stargazers: 300,
prs: 300,
issues: 200,
});
expect(stats).toStrictEqual({
contributedTo: 61,
name: "Anurag Hazra",
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
totalStars: 300,
rank,
});
});