add data loader layer and file-based cache system

Author: pettinarip

src/lib/utils/cacheAsyncFn.ts

@@ -0,0 +1,71 @@
+import fs from "fs"
+import path from "path"
+
+const CACHE_FILE_DIR = path.resolve(".next/cache")
+
+/**
+ * Caches the result of an asynchronous function to avoid multiple calls during build time.
+ * This helps prevent hitting external API rate limits by storing the result in a file.
+ *
+ * @param key A unique identifier for the cached function result
+ * @param fn The asynchronous function to be cached
+ * @param options Optional parameters to configure cache behavior
+ * @returns A new function that returns the cached result or executes the original function if the cache is invalid
+ *
+ * @example
+ * const cachedFetch = cacheAsyncFn('uniqueKey', fetchSomething);
+ *
+ * await cachedFetch(); // Fetches and caches the data
+ * await cachedFetch(); // Returns the cached data without re-fetching
+ *
+ * @note The cache is stored in the '.next/cache' directory and expires after the `cacheTimeout`
+ */
+
+export function cacheAsyncFn<T>(
+  key: string,
+  fn: () => Promise<T>,
+  options?: { cacheTimeout?: number }
+) {
+  const cacheFilePath = path.resolve(CACHE_FILE_DIR, `${key}.json`)
+
+  return async (): Promise<T> => {
+    let value: T | undefined
+
+    // Check if cache file exists
+    if (fs.existsSync(cacheFilePath)) {
+      const fileStats = fs.statSync(cacheFilePath)
+      const now = Date.now()
+      const cacheAge = now - new Date(fileStats.mtime).getTime()
+      const isCacheExpired =
+        options?.cacheTimeout && cacheAge > options.cacheTimeout
+
+      // Invalidate cache if it's too old
+      if (isCacheExpired) {
+        // Remove stale cache
+        fs.unlinkSync(cacheFilePath)
+        console.log("Stale cache removed", key)
+      } else {
+        // Cache hit
+        const cachedData = fs.readFileSync(cacheFilePath, "utf-8")
+        value = JSON.parse(cachedData)
+        console.log("Cache hit", key)
+      }
+    }
+
+    // If no cached data, fetch fresh data
+    if (!value) {
+      console.log("Running function for the first time", key)
+      value = await fn()
+      console.log("Function ran and cached", key)
+
+      // Ensure cache folder exists
+      fs.mkdirSync(CACHE_FILE_DIR, { recursive: true })
+
+      // Write data to cache file
+      fs.writeFileSync(cacheFilePath, JSON.stringify(value), "utf-8")
+      console.log("Function cached", key)
+    }
+
+    return value!
+  }
+}

src/lib/utils/dataLoader.ts

@@ -0,0 +1,41 @@
+import { cacheAsyncFn } from "./cacheAsyncFn"
+
+type DataLoaderFunction<T> = () => Promise<T>
+
+/**
+ * Loads data from multiple asynchronous functions and caches the results.
+ *
+ * @param loaders An array of tuples, each containing a unique identifier and the asynchronous function to be cached
+ * @returns A promise that resolves to an array of the results from the cached functions
+ *
+ * @example
+ * const [ethPrice, totalEthStaked, totalValueLocked] = await dataLoader([
+ *   ['ethPrice', fetchEthPrice],
+ *   ['totalEthStaked', fetchTotalEthStaked],
+ *   ['totalValueLocked', fetchTotalValueLocked],
+ * ]);
+ */
+
+export async function dataLoader<T extends unknown[]>(
+  loaders: {
+    [K in keyof T]: [string, DataLoaderFunction<T[K]>]
+  },
+  cacheTimeout: number = 1000 * 60 * 60 // 1 hour
+): Promise<T> {
+  const cachedLoaders = loaders.map(([key, loader]) => {
+    const cachedLoader = cacheAsyncFn(key, loader, {
+      cacheTimeout,
+    })
+    return async () => {
+      try {
+        return await cachedLoader()
+      } catch (error) {
+        console.error(`Error in dataLoader for key "${key}":`, error)
+        throw error
+      }
+    }
+  })
+
+  const results = await Promise.all(cachedLoaders.map((loader) => loader()))
+  return results as T
+}

src/lib/utils/runOnlyOnce.ts

@@ -28,10 +28,10 @@ import mdComponents from "@/components/MdComponents"
 import PageMetadata from "@/components/PageMetadata"
 
 import { getFileContributorInfo } from "@/lib/utils/contributors"
+import { dataLoader } from "@/lib/utils/dataLoader"
 import { dateToString } from "@/lib/utils/date"
 import { getLastDeployDate } from "@/lib/utils/getLastDeployDate"
 import { getContent, getContentBySlug } from "@/lib/utils/md"
-import { runOnlyOnce } from "@/lib/utils/runOnlyOnce"
 import { getLocaleTimestamp } from "@/lib/utils/time"
 import { remapTableOfContents } from "@/lib/utils/toc"
 import {
@@ -114,11 +114,6 @@ type Props = Omit<Parameters<LayoutMappingType[Layout]>[0], "children"> &
     gfissues: Awaited<ReturnType<typeof fetchGFIs>>
   }
 
-// Fetch external API data once to avoid hitting rate limit
-const gfIssuesDataFetch = runOnlyOnce(async () => {
-  return await fetchGFIs()
-})
-
 const commitHistoryCache: CommitHistory = {}
 
 export const getStaticProps = (async (context) => {
@@ -199,7 +194,7 @@ export const getStaticProps = (async (context) => {
     lastDeployDate
   )
 
-  const gfissues = await gfIssuesDataFetch()
+  const [gfissues] = await dataLoader([["gfissues", fetchGFIs]])
 
   return {
     props: {

src/pages/[...slug].tsx

@@ -21,9 +21,9 @@ import PageMetadata from "@/components/PageMetadata"
 import ProductCard from "@/components/ProductCard"
 import Translation from "@/components/Translation"
 
+import { dataLoader } from "@/lib/utils/dataLoader"
 import { existsNamespace } from "@/lib/utils/existsNamespace"
 import { getLastDeployDate } from "@/lib/utils/getLastDeployDate"
-import { runOnlyOnce } from "@/lib/utils/runOnlyOnce"
 import { getLocaleTimestamp } from "@/lib/utils/time"
 import { getRequiredNamespacesForPage } from "@/lib/utils/translations"
 
@@ -57,18 +57,16 @@ type Props = BasePageProps & {
   frameworksList: Framework[]
 }
 
-const cachedFetchLocalEnvironmentFrameworkData = runOnlyOnce(
-  getLocalEnvironmentFrameworkData
-)
-
 export const getStaticProps = (async ({ locale }) => {
   const requiredNamespaces = getRequiredNamespacesForPage(
     "/developers/local-environment"
   )
 
   const contentNotTranslated = !existsNamespace(locale!, requiredNamespaces[2])
 
-  const frameworksListData = await cachedFetchLocalEnvironmentFrameworkData()
+  const [frameworksListData] = await dataLoader([
+    ["frameworksListData", getLocalEnvironmentFrameworkData],
+  ])
 
   const lastDeployDate = getLastDeployDate()
   const lastDeployLocaleTimestamp = getLocaleTimestamp(

src/pages/developers/local-environment.tsx

@@ -56,12 +56,12 @@ import {
 import WindowBox from "@/components/WindowBox"
 
 import { cn } from "@/lib/utils/cn"
+import { dataLoader } from "@/lib/utils/dataLoader"
 import { isValidDate } from "@/lib/utils/date"
 import { existsNamespace } from "@/lib/utils/existsNamespace"
 import { getLastDeployDate } from "@/lib/utils/getLastDeployDate"
 import { trackCustomEvent } from "@/lib/utils/matomo"
 import { polishRSSList } from "@/lib/utils/rss"
-import { runOnlyOnce } from "@/lib/utils/runOnlyOnce"
 import { breakpointAsNumber } from "@/lib/utils/screen"
 import { getLocaleTimestamp } from "@/lib/utils/time"
 import { getRequiredNamespacesForPage } from "@/lib/utils/translations"
@@ -108,30 +108,49 @@ const Codeblock = lazy(() =>
 
 const StatsBoxGrid = lazy(() => import("@/components/StatsBoxGrid"))
 
-const cachedEthPrice = runOnlyOnce(fetchEthPrice)
-const cachedFetchTotalEthStaked = runOnlyOnce(fetchTotalEthStaked)
-const cachedFetchTotalValueLocked = runOnlyOnce(fetchTotalValueLocked)
-const cachedXmlBlogFeeds = runOnlyOnce(async () => await fetchRSS(BLOG_FEEDS))
-const cachedAttestantBlog = runOnlyOnce(fetchAttestantPosts)
-const cachedGrowThePieData = runOnlyOnce(fetchGrowThePie)
-const cachedFetchCommunityEvents = runOnlyOnce(fetchCommunityEvents)
+// API calls
+const fetchXmlBlogFeeds = async () => {
+  return await fetchRSS(BLOG_FEEDS)
+}
 
 type Props = BasePageProps & {
   metricResults: AllMetricData
   rssData: { rssItems: RSSItem[]; blogLinks: CommunityBlog[] }
 }
 
+// In seconds
+const REVALIDATE_TIME = BASE_TIME_UNIT * 24
+
 export const getStaticProps = (async ({ locale }) => {
-  const growThePieData = await cachedGrowThePieData()
+  const [
+    ethPrice,
+    totalEthStaked,
+    totalValueLocked,
+    growThePieData,
+    communityEvents,
+    attestantPosts,
+    xmlBlogs,
+  ] = await dataLoader(
+    [
+      ["ethPrice", fetchEthPrice],
+      ["totalEthStaked", fetchTotalEthStaked],
+      ["totalValueLocked", fetchTotalValueLocked],
+      ["growThePieData", fetchGrowThePie],
+      ["communityEvents", fetchCommunityEvents],
+      ["attestantPosts", fetchAttestantPosts],
+      ["rssData", fetchXmlBlogFeeds],
+    ],
+    REVALIDATE_TIME * 1000
+  )
+
   const metricResults: AllMetricData = {
-    ethPrice: await cachedEthPrice(),
-    totalEthStaked: await cachedFetchTotalEthStaked(),
-    totalValueLocked: await cachedFetchTotalValueLocked(),
+    ethPrice,
+    totalEthStaked,
+    totalValueLocked,
     txCount: growThePieData.txCount,
     txCostsMedianUsd: growThePieData.txCostsMedianUsd,
   }
 
-  const communityEvents = await cachedFetchCommunityEvents()
   const calendar = communityEvents.upcomingEventData
     .sort((a, b) => {
       const dateA = isValidDate(a.date) ? new Date(a.date).getTime() : -Infinity
@@ -153,10 +172,8 @@ export const getStaticProps = (async ({ locale }) => {
     lastDeployDate
   )
 
-  // load RSS feed items
-  const xmlBlogs = await cachedXmlBlogFeeds()
-  const attestantBlog = await cachedAttestantBlog()
-  const polishedRssItems = polishRSSList(attestantBlog, ...xmlBlogs)
+  // RSS feed items
+  const polishedRssItems = polishRSSList(attestantPosts, ...xmlBlogs)
   const rssItems = polishedRssItems.slice(0, RSS_DISPLAY_COUNT)
 
   const blogLinks = polishedRssItems.map(({ source, sourceUrl }) => ({
@@ -174,7 +191,7 @@ export const getStaticProps = (async ({ locale }) => {
       metricResults,
       rssData: { rssItems, blogLinks },
     },
-    revalidate: BASE_TIME_UNIT * 24,
+    revalidate: REVALIDATE_TIME,
   }
 }) satisfies GetStaticProps<Props>
 

src/pages/index.tsx

@@ -38,9 +38,9 @@ import Translation from "@/components/Translation"
 import { Divider } from "@/components/ui/divider"
 
 import { cn } from "@/lib/utils/cn"
+import { dataLoader } from "@/lib/utils/dataLoader"
 import { existsNamespace } from "@/lib/utils/existsNamespace"
 import { getLastDeployDate } from "@/lib/utils/getLastDeployDate"
-import { runOnlyOnce } from "@/lib/utils/runOnlyOnce"
 import { getLocaleTimestamp } from "@/lib/utils/time"
 import { getRequiredNamespacesForPage } from "@/lib/utils/translations"
 
@@ -90,10 +90,6 @@ type Props = BasePageProps & {
   marketsHasError: boolean
 }
 
-// Fetch external API data once to avoid hitting rate limit
-const ethereumEcosystemDataFetch = runOnlyOnce(fetchEthereumEcosystemData)
-const ethereumStablecoinsDataFetch = runOnlyOnce(fetchEthereumStablecoinsData)
-
 export const getStaticProps = (async ({ locale }) => {
   const lastDeployDate = getLastDeployDate()
   const lastDeployLocaleTimestamp = getLocaleTimestamp(
@@ -139,12 +135,12 @@ export const getStaticProps = (async ({ locale }) => {
   }
 
   try {
-    // Fetch token data in the Ethereum ecosystem
-    const ethereumEcosystemData: EthereumDataResponse =
-      await ethereumEcosystemDataFetch()
-    // Fetch token data for stablecoins
-    const stablecoinsData: StablecoinDataResponse =
-      await ethereumStablecoinsDataFetch()
+    const [ethereumEcosystemData, stablecoinsData] = await dataLoader<
+      [EthereumDataResponse, StablecoinDataResponse]
+    >([
+      ["ethereumEcosystemData", fetchEthereumEcosystemData],
+      ["ethereumStablecoinsData", fetchEthereumStablecoinsData],
+    ])
 
     // Get the intersection of stablecoins and Ethereum tokens to only have a list of data for stablecoins in the Ethereum ecosystem
     const ethereumStablecoinData = stablecoinsData.filter(

src/pages/stablecoins.tsx

@@ -34,9 +34,9 @@ import InlineLink from "@/components/ui/Link"
 import { ListItem, UnorderedList } from "@/components/ui/list"
 
 import { cn } from "@/lib/utils/cn"
+import { dataLoader } from "@/lib/utils/dataLoader"
 import { existsNamespace } from "@/lib/utils/existsNamespace"
 import { getLastDeployDate } from "@/lib/utils/getLastDeployDate"
-import { runOnlyOnce } from "@/lib/utils/runOnlyOnce"
 import { getLocaleTimestamp } from "@/lib/utils/time"
 import { getRequiredNamespacesForPage } from "@/lib/utils/translations"
 
@@ -149,8 +149,6 @@ const fetchBeaconchainData = async (): Promise<StakingStatsData> => {
   return { totalEthStaked, validatorscount, apr }
 }
 
-const cachedFetchBeaconchainData = runOnlyOnce(fetchBeaconchainData)
-
 type Props = BasePageProps & {
   data: StakingStatsData
 }
@@ -166,7 +164,7 @@ export const getStaticProps = (async ({ locale }) => {
 
   const contentNotTranslated = !existsNamespace(locale!, requiredNamespaces[2])
 
-  const data = await cachedFetchBeaconchainData()
+  const [data] = await dataLoader([["stakingStatsData", fetchBeaconchainData]])
 
   return {
     props: {