Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature: fetch only requested data from GitHub GraphQL API to reduce load #3208

Merged
merged 6 commits into from
Oct 13, 2023
Merged

feature: fetch only requested data from GitHub GraphQL API to reduce load #3208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5cffa5b
feature: fetch only requested data from GitHub GraphQL API to reduce …
qwerty541 Sep 5, 2023
63edd91
dev
qwerty541 Sep 5, 2023
aaa1544
Merge remote-tracking branch 'upstream/master' into feature_fetch_onl…
qwerty541 Sep 9, 2023
5ceb4d8
Merge remote-tracking branch 'upstream/master' into feature_fetch_onl…
qwerty541 Sep 14, 2023
a292cd6
dev
qwerty541 Sep 14, 2023
5b4e3bc
Merge remote-tracking branch 'upstream/master' into feature_fetch_onl…
qwerty541 Oct 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion api/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,15 @@ export default async (req, res) => {
}

try {
const showStats = parseArray(show);
const stats = await fetchStats(
username,
parseBoolean(include_all_commits),
parseArray(exclude_repo),
showStats.includes("prs_merged") ||
showStats.includes("prs_merged_percentage"),
showStats.includes("discussions_started"),
showStats.includes("discussions_answered"),
);

let cacheSeconds = clampValue(
Expand Down Expand Up @@ -96,7 +101,7 @@ export default async (req, res) => {
locale: locale ? locale.toLowerCase() : null,
disable_animations: parseBoolean(disable_animations),
rank_icon,
show: parseArray(show),
show: showStats,
}),
);
} catch (err) {
Expand Down
60 changes: 47 additions & 13 deletions src/fetchers/stats-fetcher.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ const GRAPHQL_REPOS_QUERY = `
`;

const GRAPHQL_STATS_QUERY = `
query userInfo($login: String!, $after: String) {
query userInfo($login: String!, $after: String, $includeMergedPullRequests: Boolean!, $includeDiscussions: Boolean!, $includeDiscussionsAnswers: Boolean!) {
user(login: $login) {
name
login
Expand All @@ -54,7 +54,7 @@ const GRAPHQL_STATS_QUERY = `
pullRequests(first: 1) {
totalCount
}
mergedPullRequests: pullRequests(states: MERGED) {
mergedPullRequests: pullRequests(states: MERGED) @include(if: $includeMergedPullRequests) {
totalCount
}
openIssues: issues(states: OPEN) {
Expand All @@ -66,10 +66,10 @@ const GRAPHQL_STATS_QUERY = `
followers {
totalCount
}
repositoryDiscussions {
repositoryDiscussions @include(if: $includeDiscussions) {
totalCount
}
repositoryDiscussionComments(onlyAnswers: true) {
repositoryDiscussionComments(onlyAnswers: true) @include(if: $includeDiscussionsAnswers) {
totalCount
}
${GRAPHQL_REPOS_FIELD}
Expand Down Expand Up @@ -104,17 +104,33 @@ const fetcher = (variables, token) => {
/**
* Fetch stats information for a given username.
*
* @param {string} username Github username.
* @param {object} variables Fetcher variables.
* @param {string} variables.username Github username.
* @param {boolean} variables.includeMergedPullRequests Include merged pull requests.
* @param {boolean} variables.includeDiscussions Include discussions.
* @param {boolean} variables.includeDiscussionsAnswers Include discussions answers.
* @returns {Promise<AxiosResponse>} Axios response.
*
* @description This function supports multi-page fetching if the 'FETCH_MULTI_PAGE_STARS' environment variable is set to true.
*/
const statsFetcher = async (username) => {
const statsFetcher = async ({
username,
includeMergedPullRequests,
includeDiscussions,
includeDiscussionsAnswers,
}) => {
let stats;
let hasNextPage = true;
let endCursor = null;
while (hasNextPage) {
const variables = { login: username, first: 100, after: endCursor };
const variables = {
login: username,
first: 100,
after: endCursor,
includeMergedPullRequests,
includeDiscussions,
includeDiscussionsAnswers,
};
let res = await retryer(fetcher, variables);
if (res.data.errors) {
return res;
Expand Down Expand Up @@ -198,12 +214,18 @@ const totalCommitsFetcher = async (username) => {
* @param {string} username GitHub username.
* @param {boolean} include_all_commits Include all commits.
* @param {string[]} exclude_repo Repositories to exclude.
* @param {boolean} include_merged_pull_requests Include merged pull requests.
* @param {boolean} include_discussions Include discussions.
* @param {boolean} include_discussions_answers Include discussions answers.
* @returns {Promise<StatsData>} Stats data.
*/
const fetchStats = async (
username,
include_all_commits = false,
exclude_repo = [],
include_merged_pull_requests = false,
include_discussions = false,
include_discussions_answers = false,
) => {
if (!username) {
throw new MissingParamError(["username"]);
Expand All @@ -224,7 +246,12 @@ const fetchStats = async (
rank: { level: "C", percentile: 100 },
};

let res = await statsFetcher(username);
let res = await statsFetcher({
username,
includeMergedPullRequests: include_merged_pull_requests,
includeDiscussions: include_discussions,
includeDiscussionsAnswers: include_discussions_answers,
});

// Catch GraphQL errors.
if (res.data.errors) {
Expand Down Expand Up @@ -259,14 +286,21 @@ const fetchStats = async (
}

stats.totalPRs = user.pullRequests.totalCount;
stats.totalPRsMerged = user.mergedPullRequests.totalCount;
stats.mergedPRsPercentage =
(user.mergedPullRequests.totalCount / user.pullRequests.totalCount) * 100;
if (include_merged_pull_requests) {
stats.totalPRsMerged = user.mergedPullRequests.totalCount;
stats.mergedPRsPercentage =
(user.mergedPullRequests.totalCount / user.pullRequests.totalCount) * 100;
}
stats.totalReviews =
user.contributionsCollection.totalPullRequestReviewContributions;
stats.totalIssues = user.openIssues.totalCount + user.closedIssues.totalCount;
stats.totalDiscussionsStarted = user.repositoryDiscussions.totalCount;
stats.totalDiscussionsAnswered = user.repositoryDiscussionComments.totalCount;
if (include_discussions) {
stats.totalDiscussionsStarted = user.repositoryDiscussions.totalCount;
}
if (include_discussions_answers) {
stats.totalDiscussionsAnswered =
user.repositoryDiscussionComments.totalCount;
}
stats.contributedTo = user.repositoriesContributedTo.totalCount;

// Retrieve stars while filtering out repositories to be hidden.
Expand Down
106 changes: 82 additions & 24 deletions tests/fetchStats.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,12 +122,12 @@ describe("Test fetchStats", () => {
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
totalPRsMerged: 240,
mergedPRsPercentage: 80,
totalPRsMerged: 0,
mergedPRsPercentage: 0,
totalReviews: 50,
totalStars: 300,
totalDiscussionsStarted: 10,
totalDiscussionsAnswered: 40,
totalDiscussionsStarted: 0,
totalDiscussionsAnswered: 0,
rank,
});
});
Expand Down Expand Up @@ -158,12 +158,12 @@ describe("Test fetchStats", () => {
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
totalPRsMerged: 240,
mergedPRsPercentage: 80,
totalPRsMerged: 0,
mergedPRsPercentage: 0,
totalReviews: 50,
totalStars: 300,
totalDiscussionsStarted: 10,
totalDiscussionsAnswered: 40,
totalDiscussionsStarted: 0,
totalDiscussionsAnswered: 0,
rank,
});
});
Expand Down Expand Up @@ -200,12 +200,12 @@ describe("Test fetchStats", () => {
totalCommits: 1000,
totalIssues: 200,
totalPRs: 300,
totalPRsMerged: 240,
mergedPRsPercentage: 80,
totalPRsMerged: 0,
mergedPRsPercentage: 0,
totalReviews: 50,
totalStars: 300,
totalDiscussionsStarted: 10,
totalDiscussionsAnswered: 40,
totalDiscussionsStarted: 0,
totalDiscussionsAnswered: 0,
rank,
});
});
Expand Down Expand Up @@ -249,12 +249,12 @@ describe("Test fetchStats", () => {
totalCommits: 1000,
totalIssues: 200,
totalPRs: 300,
totalPRsMerged: 240,
mergedPRsPercentage: 80,
totalPRsMerged: 0,
mergedPRsPercentage: 0,
totalReviews: 50,
totalStars: 200,
totalDiscussionsStarted: 10,
totalDiscussionsAnswered: 40,
totalDiscussionsStarted: 0,
totalDiscussionsAnswered: 0,
rank,
});
});
Expand All @@ -280,12 +280,12 @@ describe("Test fetchStats", () => {
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
totalPRsMerged: 240,
mergedPRsPercentage: 80,
totalPRsMerged: 0,
mergedPRsPercentage: 0,
totalReviews: 50,
totalStars: 400,
totalDiscussionsStarted: 10,
totalDiscussionsAnswered: 40,
totalDiscussionsStarted: 0,
totalDiscussionsAnswered: 0,
rank,
});
});
Expand All @@ -311,12 +311,12 @@ describe("Test fetchStats", () => {
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
totalPRsMerged: 240,
mergedPRsPercentage: 80,
totalPRsMerged: 0,
mergedPRsPercentage: 0,
totalReviews: 50,
totalStars: 300,
totalDiscussionsStarted: 10,
totalDiscussionsAnswered: 40,
totalDiscussionsStarted: 0,
totalDiscussionsAnswered: 0,
rank,
});
});
Expand All @@ -336,6 +336,64 @@ describe("Test fetchStats", () => {
followers: 100,
});

expect(stats).toStrictEqual({
contributedTo: 61,
name: "Anurag Hazra",
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
totalPRsMerged: 0,
mergedPRsPercentage: 0,
totalReviews: 50,
totalStars: 300,
totalDiscussionsStarted: 0,
totalDiscussionsAnswered: 0,
rank,
});
});

it("should not fetch additional stats data when it not requested", async () => {
let stats = await fetchStats("anuraghazra");
const rank = calculateRank({
all_commits: false,
commits: 100,
prs: 300,
reviews: 50,
issues: 200,
repos: 5,
stars: 300,
followers: 100,
});

expect(stats).toStrictEqual({
contributedTo: 61,
name: "Anurag Hazra",
totalCommits: 100,
totalIssues: 200,
totalPRs: 300,
totalPRsMerged: 0,
mergedPRsPercentage: 0,
totalReviews: 50,
totalStars: 300,
totalDiscussionsStarted: 0,
totalDiscussionsAnswered: 0,
rank,
});
});

it("should fetch additional stats when it requested", async () => {
let stats = await fetchStats("anuraghazra", false, [], true, true, true);
const rank = calculateRank({
all_commits: false,
commits: 100,
prs: 300,
reviews: 50,
issues: 200,
repos: 5,
stars: 300,
followers: 100,
});

expect(stats).toStrictEqual({
contributedTo: 61,
name: "Anurag Hazra",
Expand Down
Loading