diff --git a/.github/workflows/repeat-recent-requests.yml b/.github/workflows/repeat-recent-requests.yml new file mode 100644 index 0000000000000..9f8c0fb4fed9f --- /dev/null +++ b/.github/workflows/repeat-recent-requests.yml @@ -0,0 +1,31 @@ +name: Trigger server to update cached data +on: + schedule: + # ┌───────────── minute (0 - 59) + # │ ┌───────────── hour (0 - 23) + # │ │ ┌───────────── day of the month (1 - 31) + # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) + # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) + # │ │ │ │ │ + # │ │ │ │ │ + # │ │ │ │ │ + # * * * * * + - cron: "45 * * * *" + workflow_dispatch: + +jobs: + triggerRepeatRecent: + name: Trigger server to update cached data. + runs-on: ubuntu-latest + steps: + - name: Make Request + id: myRequest + uses: fjogeleit/http-request-action@v1 + with: + url: "https://github-readme-stats-git-caching-martin-mfgs-projects.vercel.app/api/repeat-recent" + method: "POST" + timeout: "62000" + - name: Show Response + run: | + echo ${{ steps.myRequest.outputs.response }} + echo ${{ steps.myRequest.outputs.status }} diff --git a/.gitignore b/.gitignore index b1d9a017c5b80..c8683521459c7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,3 @@ vercel_token !.vscode/extensions.json !.vscode/settings.json *.code-workspace - -.vercel diff --git a/_dot_vercel_copy/output/config.json b/_dot_vercel_copy/output/config.json new file mode 100644 index 0000000000000..cd2f236b292fc --- /dev/null +++ b/_dot_vercel_copy/output/config.json @@ -0,0 +1,3 @@ +{ + "version": 3 +} diff --git a/_dot_vercel_copy/output/functions/api.func/.vc-config.json b/_dot_vercel_copy/output/functions/api.func/.vc-config.json new file mode 100644 index 0000000000000..605ee7c0d44cd --- /dev/null +++ b/_dot_vercel_copy/output/functions/api.func/.vc-config.json @@ -0,0 +1,5 @@ +{ + "runtime": "nodejs22.x", + "handler": "router.js", + "launcherType": "Nodejs" +} diff --git a/_dot_vercel_copy/output/functions/api.func/router.js b/_dot_vercel_copy/output/functions/api.func/router.js new file mode 100644 index 0000000000000..16c1ed8afa8b4 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api.func/router.js @@ -0,0 +1,55 @@ +import { default as api } from "./api-renamed/index.js"; +import { default as gist } from "./api-renamed/gist.js"; +import { default as pin } from "./api-renamed/pin.js"; +import { default as topLangs } from "./api-renamed/top-langs.js"; +import { default as wakatime } from "./api-renamed/wakatime.js"; +import { default as repeatRecent } from "./api-renamed/repeat-recent.js"; +import { default as patInfo } from "./api-renamed/status/pat-info.js"; +import { default as statusUp } from "./api-renamed/status/up.js"; + +export default async (req, res) => { + // remaining code expects express.js-like request and response objects + res.send = function (data) { + if (typeof data === "object") { + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify(data)); + } else if (typeof data === "string") { + res.end(data); + } else { + res.end(String(data)); + } + }; + const url = new URL(req.url, "https://localhost"); + req.query = Object.fromEntries(url.searchParams.entries()); + + switch (url.pathname) { + case "/api": + api(req, res); + break; + case "/api/gist": + gist(req, res); + break; + case "/api/pin": + pin(req, res); + break; + case "/api/top-langs": + topLangs(req, res); + break; + case "/api/wakatime": + wakatime(req, res); + break; + case "/api/repeat-recent": + repeatRecent(req, res); + break; + case "/api/status/pat-info": + patInfo(req, res); + break; + case "/api/status/up": + statusUp(req, res); + break; + default: + res.statusCode = 404; + res.end("Not Found"); + break; + } +}; diff --git a/_dot_vercel_copy/output/functions/api.prerender-config.json b/_dot_vercel_copy/output/functions/api.prerender-config.json new file mode 100644 index 0000000000000..d7747ff4f1778 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api.prerender-config.json @@ -0,0 +1,5 @@ +{ + "expiration": 39600, + "bypassToken": "r3fr3shT0k3n-r3fr3shT0k3n-r3fr3shT0k3n", + "passQuery": true +} \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/gist.func b/_dot_vercel_copy/output/functions/api/gist.func new file mode 120000 index 0000000000000..2e79e533501b3 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/gist.func @@ -0,0 +1 @@ +../api.func \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/gist.prerender-config.json b/_dot_vercel_copy/output/functions/api/gist.prerender-config.json new file mode 100644 index 0000000000000..d7747ff4f1778 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/gist.prerender-config.json @@ -0,0 +1,5 @@ +{ + "expiration": 39600, + "bypassToken": "r3fr3shT0k3n-r3fr3shT0k3n-r3fr3shT0k3n", + "passQuery": true +} \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/pin.func b/_dot_vercel_copy/output/functions/api/pin.func new file mode 120000 index 0000000000000..2e79e533501b3 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/pin.func @@ -0,0 +1 @@ +../api.func \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/pin.prerender-config.json b/_dot_vercel_copy/output/functions/api/pin.prerender-config.json new file mode 100644 index 0000000000000..d7747ff4f1778 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/pin.prerender-config.json @@ -0,0 +1,5 @@ +{ + "expiration": 39600, + "bypassToken": "r3fr3shT0k3n-r3fr3shT0k3n-r3fr3shT0k3n", + "passQuery": true +} \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/repeat-recent.func b/_dot_vercel_copy/output/functions/api/repeat-recent.func new file mode 120000 index 0000000000000..2e79e533501b3 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/repeat-recent.func @@ -0,0 +1 @@ +../api.func \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/status/pat-info.func b/_dot_vercel_copy/output/functions/api/status/pat-info.func new file mode 120000 index 0000000000000..0f4906ce46690 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/status/pat-info.func @@ -0,0 +1 @@ +../../api.func \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/status/up.func b/_dot_vercel_copy/output/functions/api/status/up.func new file mode 120000 index 0000000000000..0f4906ce46690 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/status/up.func @@ -0,0 +1 @@ +../../api.func \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/top-langs.func b/_dot_vercel_copy/output/functions/api/top-langs.func new file mode 120000 index 0000000000000..2e79e533501b3 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/top-langs.func @@ -0,0 +1 @@ +../api.func \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/top-langs.prerender-config.json b/_dot_vercel_copy/output/functions/api/top-langs.prerender-config.json new file mode 100644 index 0000000000000..d7747ff4f1778 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/top-langs.prerender-config.json @@ -0,0 +1,5 @@ +{ + "expiration": 39600, + "bypassToken": "r3fr3shT0k3n-r3fr3shT0k3n-r3fr3shT0k3n", + "passQuery": true +} \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/wakatime.func b/_dot_vercel_copy/output/functions/api/wakatime.func new file mode 120000 index 0000000000000..2e79e533501b3 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/wakatime.func @@ -0,0 +1 @@ +../api.func \ No newline at end of file diff --git a/_dot_vercel_copy/output/functions/api/wakatime.prerender-config.json b/_dot_vercel_copy/output/functions/api/wakatime.prerender-config.json new file mode 100644 index 0000000000000..d7747ff4f1778 --- /dev/null +++ b/_dot_vercel_copy/output/functions/api/wakatime.prerender-config.json @@ -0,0 +1,5 @@ +{ + "expiration": 39600, + "bypassToken": "r3fr3shT0k3n-r3fr3shT0k3n-r3fr3shT0k3n", + "passQuery": true +} \ No newline at end of file diff --git a/api/gist.js b/api-renamed/gist.js similarity index 92% rename from api/gist.js rename to api-renamed/gist.js index 5b03743ce9e6e..ed12e4d2f17ce 100644 --- a/api/gist.js +++ b/api-renamed/gist.js @@ -7,6 +7,7 @@ import { import { isLocaleAvailable } from "../src/translations.js"; import { renderGistCard } from "../src/cards/gist-card.js"; import { fetchGist } from "../src/fetchers/gist-fetcher.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { @@ -39,11 +40,12 @@ export default async (req, res) => { } try { + await storeRequest(req); const gistData = await fetchGist(id); let cacheSeconds = clampValue( - parseInt(cache_seconds || CONSTANTS.TWO_DAY, 10), - CONSTANTS.TWO_DAY, + parseInt(cache_seconds || CONSTANTS.TEN_HOURS, 10), + CONSTANTS.FOUR_HOURS, CONSTANTS.SIX_DAY, ); cacheSeconds = process.env.CACHE_SECONDS diff --git a/api/index.js b/api-renamed/index.js similarity index 97% rename from api/index.js rename to api-renamed/index.js index 6b14bf5839885..098dfa5b8b26e 100644 --- a/api/index.js +++ b/api-renamed/index.js @@ -9,6 +9,7 @@ import { } from "../src/common/utils.js"; import { fetchStats } from "../src/fetchers/stats-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { @@ -89,6 +90,7 @@ export default async (req, res) => { } try { + await storeRequest(req); const showStats = parseArray(show); const organizations = parseArray(owners); let repositories = parseArray(repos); @@ -115,7 +117,7 @@ export default async (req, res) => { let cacheSeconds = clampValue( parseInt(cache_seconds || CONSTANTS.CARD_CACHE_SECONDS, 10), - CONSTANTS.TWELVE_HOURS, + CONSTANTS.FOUR_HOURS, CONSTANTS.TWO_DAY, ); cacheSeconds = process.env.CACHE_SECONDS diff --git a/api/pin.js b/api-renamed/pin.js similarity index 96% rename from api/pin.js rename to api-renamed/pin.js index 2570eeb653838..4ac56bcce9819 100644 --- a/api/pin.js +++ b/api-renamed/pin.js @@ -9,6 +9,7 @@ import { } from "../src/common/utils.js"; import { fetchRepo } from "../src/fetchers/repo-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { @@ -81,6 +82,7 @@ export default async (req, res) => { } try { + await storeRequest(req); const showStats = parseArray(show); const repoData = await fetchRepo( username, @@ -94,7 +96,7 @@ export default async (req, res) => { let cacheSeconds = clampValue( parseInt(cache_seconds || CONSTANTS.PIN_CARD_CACHE_SECONDS, 10), - CONSTANTS.ONE_DAY, + CONSTANTS.FOUR_HOURS, CONSTANTS.TEN_DAY, ); cacheSeconds = process.env.CACHE_SECONDS diff --git a/api-renamed/repeat-recent.js b/api-renamed/repeat-recent.js new file mode 100644 index 0000000000000..0585b003d1a8d --- /dev/null +++ b/api-renamed/repeat-recent.js @@ -0,0 +1,18 @@ +import { repeatRecentRequests } from "../src/common/database.js"; + +export default async (req, res) => { + if (req.method !== "POST") { + res.statusCode = 405; + res.send({ error: "Method Not Allowed" }); + return; + } + try { + await repeatRecentRequests(); + res.statusCode = 200; + res.send({ message: "Recent requests repeated successfully." }); + } catch (error) { + console.error("Error repeating recent requests:", error); + res.statusCode = 500; + res.send({ error: error.message || "Internal Server Error" }); + } +}; diff --git a/api/status/pat-info.js b/api-renamed/status/pat-info.js similarity index 98% rename from api/status/pat-info.js rename to api-renamed/status/pat-info.js index 1f17bf65aadb9..8b15d76bd918d 100644 --- a/api/status/pat-info.js +++ b/api-renamed/status/pat-info.js @@ -2,11 +2,11 @@ * @file Contains a simple cloud function that can be used to check which PATs are no * longer working. It returns a list of valid PATs, expired PATs and PATs with errors. * - * @description This function is currently rate limited to 1 request per 5 minutes. + * @description This function is currently rate limited to 1 request per 3 minutes. */ import { logger, request, dateDiff } from "../../src/common/utils.js"; -export const RATE_LIMIT_SECONDS = 60 * 5; // 1 request per 5 minutes +export const RATE_LIMIT_SECONDS = 60 * 3; // 1 request per 3 minutes /** * @typedef {import('axios').AxiosRequestHeaders} AxiosRequestHeaders Axios request headers. diff --git a/api/status/up.js b/api-renamed/status/up.js similarity index 97% rename from api/status/up.js rename to api-renamed/status/up.js index 6ed7c37c04ea1..168930b798e7a 100644 --- a/api/status/up.js +++ b/api-renamed/status/up.js @@ -2,13 +2,13 @@ * @file Contains a simple cloud function that can be used to check if the PATs are still * functional. * - * @description This function is currently rate limited to 1 request per 5 minutes. + * @description This function is currently rate limited to 1 request per 3 minutes. */ import retryer from "../../src/common/retryer.js"; import { logger, request } from "../../src/common/utils.js"; -export const RATE_LIMIT_SECONDS = 60 * 5; // 1 request per 5 minutes +export const RATE_LIMIT_SECONDS = 60 * 3; // 1 request per 3 minutes /** * @typedef {import('axios').AxiosRequestHeaders} AxiosRequestHeaders Axios request headers. diff --git a/api/top-langs.js b/api-renamed/top-langs.js similarity index 97% rename from api/top-langs.js rename to api-renamed/top-langs.js index c5bed634c1eab..fc8aef9600499 100644 --- a/api/top-langs.js +++ b/api-renamed/top-langs.js @@ -8,6 +8,7 @@ import { } from "../src/common/utils.js"; import { fetchTopLanguages } from "../src/fetchers/top-languages-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { @@ -62,6 +63,7 @@ export default async (req, res) => { } try { + await storeRequest(req); const topLangs = await fetchTopLanguages( username, parseArray(exclude_repo), diff --git a/api/wakatime.js b/api-renamed/wakatime.js similarity index 95% rename from api/wakatime.js rename to api-renamed/wakatime.js index 73ef9986feeb7..196892d2d73bd 100644 --- a/api/wakatime.js +++ b/api-renamed/wakatime.js @@ -8,6 +8,7 @@ import { } from "../src/common/utils.js"; import { fetchWakatimeStats } from "../src/fetchers/wakatime-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { @@ -49,11 +50,12 @@ export default async (req, res) => { } try { + await storeRequest(req); const stats = await fetchWakatimeStats({ username, api_domain }); let cacheSeconds = clampValue( parseInt(cache_seconds || CONSTANTS.CARD_CACHE_SECONDS, 10), - CONSTANTS.SIX_HOURS, + CONSTANTS.FOUR_HOURS, CONSTANTS.TWO_DAY, ); cacheSeconds = process.env.CACHE_SECONDS diff --git a/jest.config.js b/jest.config.js index b4578cf5aaad5..8b2eb25869e97 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,6 +3,7 @@ export default { transform: {}, testEnvironment: "jsdom", coverageProvider: "v8", + setupFiles: ["/tests/setup.jest.js"], testPathIgnorePatterns: ["/node_modules/", "/tests/e2e/"], modulePathIgnorePatterns: ["/node_modules/", "/tests/e2e/"], coveragePathIgnorePatterns: [ diff --git a/package-lock.json b/package-lock.json index dc391590c9531..595a1a414a9c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "dotenv": "^16.5.0", "emoji-name-map": "^1.2.8", "github-username-regex": "^1.0.0", + "pg": "^8.16.2", "upgrade": "^1.1.0", "word-wrap": "^1.2.5" }, @@ -6526,6 +6527,104 @@ "node": ">=16" } }, + "node_modules/pg": { + "version": "8.16.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.2.tgz", + "integrity": "sha512-OtLWF0mKLmpxelOt9BqVq83QV6bTfsS0XLegIeAKqKjurRnRKie1Dc1iL89MugmSLhftxw6NNCyZhm1yQFLMEQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.9.1", + "pg-pool": "^3.10.1", + "pg-protocol": "^1.10.2", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.2.6" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.6.tgz", + "integrity": "sha512-uxmJAnmIgmYgnSFzgOf2cqGQBzwnRYcrEgXuFjJNEkpedEIPBSEzxY7ph4uA9k1mI+l/GR0HjPNS6FKNZe8SBQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", + "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", + "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.2.tgz", + "integrity": "sha512-Ci7jy8PbaWxfsck2dwZdERcDG2A0MG8JoQILs+uZNjABFuBuItAZCWUNz8sXRDMoui24rJw7WlXqgpMdBSN/vQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/pgpass/node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -6593,6 +6692,45 @@ "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", "dev": true }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -7858,6 +7996,15 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -12705,6 +12852,73 @@ "integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==", "dev": true }, + "pg": { + "version": "8.16.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.2.tgz", + "integrity": "sha512-OtLWF0mKLmpxelOt9BqVq83QV6bTfsS0XLegIeAKqKjurRnRKie1Dc1iL89MugmSLhftxw6NNCyZhm1yQFLMEQ==", + "requires": { + "pg-cloudflare": "^1.2.6", + "pg-connection-string": "^2.9.1", + "pg-pool": "^3.10.1", + "pg-protocol": "^1.10.2", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + } + }, + "pg-cloudflare": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.6.tgz", + "integrity": "sha512-uxmJAnmIgmYgnSFzgOf2cqGQBzwnRYcrEgXuFjJNEkpedEIPBSEzxY7ph4uA9k1mI+l/GR0HjPNS6FKNZe8SBQ==", + "optional": true + }, + "pg-connection-string": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", + "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-pool": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", + "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", + "requires": {} + }, + "pg-protocol": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.2.tgz", + "integrity": "sha512-Ci7jy8PbaWxfsck2dwZdERcDG2A0MG8JoQILs+uZNjABFuBuItAZCWUNz8sXRDMoui24rJw7WlXqgpMdBSN/vQ==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "requires": { + "split2": "^4.1.0" + }, + "dependencies": { + "split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" + } + } + }, "picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -12750,6 +12964,29 @@ "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", "dev": true }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==" + }, + "postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -13651,6 +13888,11 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index b70eac1e22e89..ef7e9261e47ca 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "format": "prettier --write .", "format:check": "prettier --check .", "prepare": "husky", - "lint": "npx eslint --max-warnings 0 \"./src/**/*.js\" \"./scripts/**/*.js\" \"./tests/**/*.js\" \"./api/**/*.js\" \"./themes/**/*.js\"", + "lint": "npx eslint --max-warnings 0 \"./src/**/*.js\" \"./scripts/**/*.js\" \"./tests/**/*.js\" \"./api-renamed/**/*.js\" \"./themes/**/*.js\"", "bench": "node --experimental-vm-modules node_modules/jest/bin/jest.js --config jest.bench.config.js" }, "author": "Anurag Hazra", @@ -65,6 +65,7 @@ "dotenv": "^16.5.0", "emoji-name-map": "^1.2.8", "github-username-regex": "^1.0.0", + "pg": "^8.16.2", "upgrade": "^1.1.0", "word-wrap": "^1.2.5" }, diff --git a/scripts/push-theme-readme.sh b/scripts/push-theme-readme.sh old mode 100755 new mode 100644 diff --git a/src/common/database.js b/src/common/database.js new file mode 100644 index 0000000000000..eca07cd0238c7 --- /dev/null +++ b/src/common/database.js @@ -0,0 +1,147 @@ +import axios from "axios"; +import pkg from "pg"; +const { Pool } = pkg; + +const pool = process.env.POSTGRES_URL + ? new Pool({ + connectionString: process.env.POSTGRES_URL, + }) + : null; + +/** + * Stores or updates a request in the database. + */ +export async function storeRequest(req) { + if (!pool) { + return; + } + + const isBypass = req.headers && req.headers["x-bypass-store"]; + const insertQuery = isBypass + ? ` + INSERT INTO requests (request, requested_at) + VALUES ($1, NOW()) + ON CONFLICT (request) + DO UPDATE SET requested_at = EXCLUDED.requested_at + ` + : ` + INSERT INTO requests (request, requested_at, user_requested_at) + VALUES ($1, NOW(), NOW()) + ON CONFLICT (request) + DO UPDATE SET requested_at = EXCLUDED.requested_at, user_requested_at = EXCLUDED.user_requested_at + `; + + try { + await pool.query(insertQuery, [req.url]); + } catch (err) { + // Check for undefined_table error (SQLSTATE 42P01) + if (err.code === "42P01") { + const createTableQuery = ` + CREATE TABLE IF NOT EXISTS requests ( + request TEXT PRIMARY KEY, + requested_at TIMESTAMP NOT NULL DEFAULT now(), + user_requested_at TIMESTAMP NOT NULL DEFAULT now() + ) + `; + await pool.query(createTableQuery); + // Retry the insert after creating the table + await pool.query(insertQuery, [req.url]); + } else { + throw err; // Re-throw if it's some other error + } + } +} + +/** + * Deletes all requests older than 8 days from the database. + */ +async function deleteOldRequests() { + if (!pool) { + return; + } + + const deleteQuery = ` + DELETE FROM requests + WHERE user_requested_at < NOW() - INTERVAL '8 days' + `; + const result = await pool.query(deleteQuery); + console.log(`Deleted ${result.rowCount} old requests.`); +} + +/** + * Fetches all requests which are between 11 hours and 8 days old. + * @returns {Promise} Array of all requests between 11 hours and 8 days old. + */ +async function getRecentRequests() { + if (!pool) { + return []; + } + + const query = ` + SELECT request + FROM requests + WHERE requested_at >= NOW() - INTERVAL '8 days' + AND requested_at < NOW() - INTERVAL '11 hours' + ORDER BY requested_at ASC + `; + const { rows } = await pool.query(query); + return rows.map((row) => row.request); +} + +/** + * Processes URLs with a thread pool of given size using axios.get. + * @param {string[]} urls An array of URLs to process. + * @param {number} poolSize The number of concurrent requests to process. + * @returns {Promise} A promise that resolves when all requests are processed. + */ +async function makeRequests(urls, poolSize) { + let current = 0; + + /** + * Worker function to process `urls`. + */ + async function worker() { + while (true) { + let idx = current++; + if (idx >= urls.length) { + break; + } + const url = "https://" + process.env.VERCEL_BRANCH_URL + urls[idx]; + try { + if (idx % 10 === 0) { + console.log(`Processing request ${idx + 1} out of ${urls.length}`); + } + await axios.get(url, { + timeout: 10000, + headers: { "x-bypass-store": "true" }, + }); + } catch (err) { + console.error(`Error fetching ${url}:`, err.message); + } + } + } + + const workers = []; + for (let i = 0; i < poolSize; i++) { + workers.push(worker()); + } + await Promise.all(workers); +} + +/** + * Repeats requests made in the last 8 days, excluding those made in the last 11 hours. + */ +export async function repeatRecentRequests() { + if (!pool) { + console.error("Postgres pool is not initialized."); + return; + } + + await deleteOldRequests(); + const urls = await getRecentRequests(); + if (urls.length === 0) { + console.log("No recent requests found."); + } else { + await makeRequests(urls, 5); + } +} diff --git a/src/common/utils.js b/src/common/utils.js index 81ebbfcc5cc73..474b44110961e 100644 --- a/src/common/utils.js +++ b/src/common/utils.js @@ -461,6 +461,7 @@ const TWO_HOURS = 7200; const FOUR_HOURS = 14400; const SIX_HOURS = 21600; const EIGHT_HOURS = 28800; +const TEN_HOURS = 36000; const TWELVE_HOURS = 43200; const ONE_DAY = 86400; const TWO_DAY = ONE_DAY * 2; @@ -477,14 +478,15 @@ const CONSTANTS = { FOUR_HOURS, SIX_HOURS, EIGHT_HOURS, + TEN_HOURS, TWELVE_HOURS, ONE_DAY, TWO_DAY, SIX_DAY, TEN_DAY, - CARD_CACHE_SECONDS: ONE_DAY, - TOP_LANGS_CACHE_SECONDS: SIX_DAY, - PIN_CARD_CACHE_SECONDS: TEN_DAY, + CARD_CACHE_SECONDS: TEN_HOURS, + TOP_LANGS_CACHE_SECONDS: TEN_HOURS, + PIN_CARD_CACHE_SECONDS: TEN_HOURS, ERROR_CACHE_SECONDS: TEN_MINUTES, }; diff --git a/tests/api.test.js b/tests/api.test.js index 48dc08462063d..8524026fa0b4e 100644 --- a/tests/api.test.js +++ b/tests/api.test.js @@ -1,7 +1,7 @@ import { jest } from "@jest/globals"; import axios from "axios"; import MockAdapter from "axios-mock-adapter"; -import api from "../api/index.js"; +import api from "../api-renamed/index.js"; import { calculateRank } from "../src/calculateRank.js"; import { renderStatsCard } from "../src/cards/stats-card.js"; import { CONSTANTS, renderError } from "../src/common/utils.js"; diff --git a/tests/gist.test.js b/tests/gist.test.js index bdf95156d5f44..f75093062cba2 100644 --- a/tests/gist.test.js +++ b/tests/gist.test.js @@ -5,7 +5,7 @@ import MockAdapter from "axios-mock-adapter"; import { expect, it, describe, afterEach } from "@jest/globals"; import { renderGistCard } from "../src/cards/gist-card.js"; import { renderError } from "../src/common/utils.js"; -import gist from "../api/gist.js"; +import gist from "../api-renamed/gist.js"; const gist_data = { data: { diff --git a/tests/pat-info.test.js b/tests/pat-info.test.js index 6c71d401c38f3..add3c869a16ad 100644 --- a/tests/pat-info.test.js +++ b/tests/pat-info.test.js @@ -7,7 +7,7 @@ dotenv.config(); import { jest } from "@jest/globals"; import axios from "axios"; import MockAdapter from "axios-mock-adapter"; -import patInfo, { RATE_LIMIT_SECONDS } from "../api/status/pat-info.js"; +import patInfo, { RATE_LIMIT_SECONDS } from "../api-renamed/status/pat-info.js"; import { expect, it, describe, afterEach, beforeAll } from "@jest/globals"; const mock = new MockAdapter(axios); diff --git a/tests/pin.test.js b/tests/pin.test.js index 2583ddfe9e9af..ff855aef51e98 100644 --- a/tests/pin.test.js +++ b/tests/pin.test.js @@ -2,7 +2,7 @@ import { jest } from "@jest/globals"; import "@testing-library/jest-dom"; import axios from "axios"; import MockAdapter from "axios-mock-adapter"; -import pin from "../api/pin.js"; +import pin from "../api-renamed/pin.js"; import { renderRepoCard } from "../src/cards/repo-card.js"; import { renderError } from "../src/common/utils.js"; import { expect, it, describe, afterEach } from "@jest/globals"; diff --git a/tests/setup.jest.js b/tests/setup.jest.js new file mode 100644 index 0000000000000..08947667a98ef --- /dev/null +++ b/tests/setup.jest.js @@ -0,0 +1,5 @@ +//https://stackoverflow.com/a/68468204/1643179 + +import { TextEncoder, TextDecoder } from "util"; + +Object.assign(global, { TextDecoder, TextEncoder }); diff --git a/tests/status.up.test.js b/tests/status.up.test.js index f15b96fb1eb37..7945d17de0221 100644 --- a/tests/status.up.test.js +++ b/tests/status.up.test.js @@ -4,7 +4,7 @@ import { jest } from "@jest/globals"; import axios from "axios"; import MockAdapter from "axios-mock-adapter"; -import up, { RATE_LIMIT_SECONDS } from "../api/status/up.js"; +import up, { RATE_LIMIT_SECONDS } from "../api-renamed/status/up.js"; import { expect, it, describe, afterEach } from "@jest/globals"; const mock = new MockAdapter(axios); diff --git a/tests/top-langs.test.js b/tests/top-langs.test.js index 1c8a45b7b9d48..87cfc7797a201 100644 --- a/tests/top-langs.test.js +++ b/tests/top-langs.test.js @@ -2,7 +2,7 @@ import { jest } from "@jest/globals"; import "@testing-library/jest-dom"; import axios from "axios"; import MockAdapter from "axios-mock-adapter"; -import topLangs from "../api/top-langs.js"; +import topLangs from "../api-renamed/top-langs.js"; import { renderTopLanguages } from "../src/cards/top-languages-card.js"; import { renderError } from "../src/common/utils.js"; import { expect, it, describe, afterEach } from "@jest/globals"; diff --git a/tests/wakatime.test.js b/tests/wakatime.test.js index 944c3e020dd38..977c21556f0b5 100644 --- a/tests/wakatime.test.js +++ b/tests/wakatime.test.js @@ -2,7 +2,7 @@ import { jest } from "@jest/globals"; import "@testing-library/jest-dom"; import axios from "axios"; import MockAdapter from "axios-mock-adapter"; -import wakatime from "../api/wakatime.js"; +import wakatime from "../api-renamed/wakatime.js"; import { renderWakatimeCard } from "../src/cards/wakatime-card.js"; import { expect, it, describe, afterEach } from "@jest/globals"; diff --git a/vercel.json b/vercel.json index f26984795d0b9..e3826c52cd0bc 100644 --- a/vercel.json +++ b/vercel.json @@ -1,10 +1,5 @@ { - "functions": { - "api/*.js": { - "memory": 128, - "maxDuration": 10 - } - }, + "buildCommand": "npm install && mkdir --parents .vercel/output/functions/api.func/ && cp --recursive ./* .vercel/output/functions/api.func/ && cp --recursive .vercel/output/functions/api.func/_dot_vercel_copy/output/ .vercel/", "redirects": [ { "source": "/",