From f09af30c5f24767fef3ca2b202cc2902d16eccdf Mon Sep 17 00:00:00 2001 From: tc-imba Date: Thu, 8 Jul 2021 16:37:33 +0800 Subject: [PATCH 1/5] fix: fetch all repos for top languages --- src/fetchers/top-languages-fetcher.js | 34 +++++++++++++++++++++------ tests/fetchTopLanguages.test.js | 6 +++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/fetchers/top-languages-fetcher.js b/src/fetchers/top-languages-fetcher.js index 181375d9e66bc..78565ac7880d6 100644 --- a/src/fetchers/top-languages-fetcher.js +++ b/src/fetchers/top-languages-fetcher.js @@ -13,10 +13,16 @@ const fetcher = (variables, token) => { return request( { query: ` - query userInfo($login: String!) { + query userInfo($login: String!, $first: Int!, $after: String) { user(login: $login) { # fetch only owner repos & not forks - repositories(ownerAffiliations: OWNER, isFork: false, first: 100) { + repositories( + ownerAffiliations: OWNER, isFork: false, + first: $first, after: $after + ) { + edges { + cursor + } nodes { name languages(first: 10, orderBy: {field: SIZE, direction: DESC}) { @@ -49,14 +55,28 @@ const fetcher = (variables, token) => { async function fetchTopLanguages(username, exclude_repo = []) { if (!username) throw new MissingParamError(["username"]); - const res = await retryer(fetcher, { login: username }); + const page_size = 100; + let page_cursor = null; + + let repoNodes = []; + + while (true) { + const variables = { login: username, first: page_size, after: page_cursor }; + const res = await retryer(fetcher, variables); + + if (res.data.errors) { + logger.error(res.data.errors); + throw Error(res.data.errors[0].message || "Could not fetch user"); + } - if (res.data.errors) { - logger.error(res.data.errors); - throw Error(res.data.errors[0].message || "Could not fetch user"); + repoNodes = repoNodes.concat(res.data.data.user.repositories.nodes); + if (!res.data.data.user.repositories.edges || + res.data.data.user.repositories.edges.length < page_size) { + break; + } + page_cursor = res.data.data.user.repositories.edges[page_size - 1].cursor; } - let repoNodes = res.data.data.user.repositories.nodes; let repoToHide = {}; // populate repoToHide map for quick lookup diff --git a/tests/fetchTopLanguages.test.js b/tests/fetchTopLanguages.test.js index 24416cd294525..cfb11b385e8c7 100644 --- a/tests/fetchTopLanguages.test.js +++ b/tests/fetchTopLanguages.test.js @@ -13,6 +13,12 @@ const data_langs = { data: { user: { repositories: { + edges: [ + { cursor: "1" }, + { cursor: "2" }, + { cursor: "3" }, + { cursor: "4" }, + ], nodes: [ { name: "test-repo-1", From 93ec319dfa914d8652d40363ae0a6ec127ffe5c3 Mon Sep 17 00:00:00 2001 From: tc-imba Date: Thu, 8 Jul 2021 16:45:26 +0800 Subject: [PATCH 2/5] fix: fetch all repos for top languages (unify the variable names) --- src/fetchers/top-languages-fetcher.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fetchers/top-languages-fetcher.js b/src/fetchers/top-languages-fetcher.js index 78565ac7880d6..5f24d5cca46ea 100644 --- a/src/fetchers/top-languages-fetcher.js +++ b/src/fetchers/top-languages-fetcher.js @@ -55,13 +55,13 @@ const fetcher = (variables, token) => { async function fetchTopLanguages(username, exclude_repo = []) { if (!username) throw new MissingParamError(["username"]); - const page_size = 100; - let page_cursor = null; + const pageSize = 100; + let pageCursor = null; let repoNodes = []; while (true) { - const variables = { login: username, first: page_size, after: page_cursor }; + const variables = { login: username, first: pageSize, after: pageCursor }; const res = await retryer(fetcher, variables); if (res.data.errors) { @@ -71,10 +71,10 @@ async function fetchTopLanguages(username, exclude_repo = []) { repoNodes = repoNodes.concat(res.data.data.user.repositories.nodes); if (!res.data.data.user.repositories.edges || - res.data.data.user.repositories.edges.length < page_size) { + res.data.data.user.repositories.edges.length < pageSize) { break; } - page_cursor = res.data.data.user.repositories.edges[page_size - 1].cursor; + pageCursor = res.data.data.user.repositories.edges[pageSize - 1].cursor; } let repoToHide = {}; From 2f4a1f00dbacf2049e73b07517a47a0f2f15b3d7 Mon Sep 17 00:00:00 2001 From: Matteo Pierro Date: Tue, 4 Oct 2022 14:46:56 +0300 Subject: [PATCH 3/5] use cursor and add tests --- src/fetchers/top-languages-fetcher.js | 37 ++++++++++----------------- tests/fetchTopLanguages.test.js | 4 +++ tests/top-langs.test.js | 4 +++ 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/fetchers/top-languages-fetcher.js b/src/fetchers/top-languages-fetcher.js index 5f24d5cca46ea..077e191445131 100644 --- a/src/fetchers/top-languages-fetcher.js +++ b/src/fetchers/top-languages-fetcher.js @@ -13,16 +13,10 @@ const fetcher = (variables, token) => { return request( { query: ` - query userInfo($login: String!, $first: Int!, $after: String) { + query userInfo($login: String!, $after: String) { user(login: $login) { # fetch only owner repos & not forks - repositories( - ownerAffiliations: OWNER, isFork: false, - first: $first, after: $after - ) { - edges { - cursor - } + repositories(ownerAffiliations: OWNER, isFork: false, first: 100, after: $after) { nodes { name languages(first: 10, orderBy: {field: SIZE, direction: DESC}) { @@ -35,6 +29,10 @@ const fetcher = (variables, token) => { } } } + pageInfo { + hasNextPage + endCursor + } } } } @@ -55,13 +53,11 @@ const fetcher = (variables, token) => { async function fetchTopLanguages(username, exclude_repo = []) { if (!username) throw new MissingParamError(["username"]); - const pageSize = 100; - let pageCursor = null; - let repoNodes = []; - - while (true) { - const variables = { login: username, first: pageSize, after: pageCursor }; + let hasNextPage = true; + let endCursor = null; + while (hasNextPage) { + const variables = { login: username, first: 100, after: endCursor }; const res = await retryer(fetcher, variables); if (res.data.errors) { @@ -69,12 +65,9 @@ async function fetchTopLanguages(username, exclude_repo = []) { throw Error(res.data.errors[0].message || "Could not fetch user"); } - repoNodes = repoNodes.concat(res.data.data.user.repositories.nodes); - if (!res.data.data.user.repositories.edges || - res.data.data.user.repositories.edges.length < pageSize) { - break; - } - pageCursor = res.data.data.user.repositories.edges[pageSize - 1].cursor; + repoNodes.push(...res.data.data.user.repositories.nodes); + hasNextPage = res.data.data.user.repositories.pageInfo.hasNextPage + endCursor = res.data.data.user.repositories.pageInfo.endCursor } let repoToHide = {}; @@ -116,14 +109,12 @@ async function fetchTopLanguages(username, exclude_repo = []) { }; }, {}); - const topLangs = Object.keys(repoNodes) + return Object.keys(repoNodes) .sort((a, b) => repoNodes[b].size - repoNodes[a].size) .reduce((result, key) => { result[key] = repoNodes[key]; return result; }, {}); - - return topLangs; } export { fetchTopLanguages }; diff --git a/tests/fetchTopLanguages.test.js b/tests/fetchTopLanguages.test.js index cfb11b385e8c7..babf8520e7dd3 100644 --- a/tests/fetchTopLanguages.test.js +++ b/tests/fetchTopLanguages.test.js @@ -49,6 +49,10 @@ const data_langs = { }, }, ], + pageInfo: { + hasNextPage: false, + cursor: "cursor", + }, }, }, }, diff --git a/tests/top-langs.test.js b/tests/top-langs.test.js index b5232c29af3d5..a4414ba67d0ef 100644 --- a/tests/top-langs.test.js +++ b/tests/top-langs.test.js @@ -36,6 +36,10 @@ const data_langs = { }, }, ], + pageInfo: { + hasNextPage: false, + cursor: "cursor", + }, }, }, }, From a4b234760e125a832f1553d19d2293adfb318bd8 Mon Sep 17 00:00:00 2001 From: Matteo Pierro Date: Tue, 4 Oct 2022 15:02:50 +0300 Subject: [PATCH 4/5] fix prettier issues --- src/fetchers/top-languages-fetcher.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fetchers/top-languages-fetcher.js b/src/fetchers/top-languages-fetcher.js index 077e191445131..0a202d88d8db5 100644 --- a/src/fetchers/top-languages-fetcher.js +++ b/src/fetchers/top-languages-fetcher.js @@ -66,8 +66,8 @@ async function fetchTopLanguages(username, exclude_repo = []) { } repoNodes.push(...res.data.data.user.repositories.nodes); - hasNextPage = res.data.data.user.repositories.pageInfo.hasNextPage - endCursor = res.data.data.user.repositories.pageInfo.endCursor + hasNextPage = res.data.data.user.repositories.pageInfo.hasNextPage; + endCursor = res.data.data.user.repositories.pageInfo.endCursor; } let repoToHide = {}; From 37411579bb3506bad738343c6037e01ef8891268 Mon Sep 17 00:00:00 2001 From: Matteo Pierro Date: Sat, 8 Oct 2022 08:10:23 +0300 Subject: [PATCH 5/5] add test for two pages --- tests/top-langs.test.js | 77 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/top-langs.test.js b/tests/top-langs.test.js index a4414ba67d0ef..31769e4fa1f1d 100644 --- a/tests/top-langs.test.js +++ b/tests/top-langs.test.js @@ -45,6 +45,60 @@ const data_langs = { }, }; +const firstPage = { + data: { + user: { + repositories: { + nodes: [ + { + languages: { + edges: [{ size: 150, node: { color: "#0f0", name: "HTML" } }], + }, + }, + { + languages: { + edges: [{ size: 100, node: { color: "#0f0", name: "HTML" } }], + }, + }, + ], + pageInfo: { + hasNextPage: true, + cursor: "cursor", + }, + }, + }, + }, +}; + +const secondPage = { + data: { + user: { + repositories: { + nodes: [ + { + languages: { + edges: [ + { size: 100, node: { color: "#0ff", name: "javascript" } }, + ], + }, + }, + { + languages: { + edges: [ + { size: 100, node: { color: "#0ff", name: "javascript" } }, + ], + }, + }, + ], + pageInfo: { + hasNextPage: false, + cursor: "cursor", + }, + }, + }, + }, +}; + const error = { errors: [ { @@ -94,6 +148,29 @@ describe("Test /api/top-langs", () => { expect(res.send).toBeCalledWith(renderTopLanguages(langs)); }); + it("should test the request with two pages", async () => { + const req = { + query: { + username: "anuraghazra", + }, + }; + const res = { + setHeader: jest.fn(), + send: jest.fn(), + }; + + mock + .onPost("https://api.github.com/graphql") + .replyOnce(200, firstPage) + .onPost("https://api.github.com/graphql") + .replyOnce(200, secondPage); + + await topLangs(req, res); + + expect(res.setHeader).toBeCalledWith("Content-Type", "image/svg+xml"); + expect(res.send).toBeCalledWith(renderTopLanguages(langs)); + }); + it("should work with the query options", async () => { const req = { query: {