Leverage clickIdCache to reduce load on Tinybird

apps/web/app/(ee)/api/shopify/pixel/route.ts

@@ -36,7 +36,7 @@ export const POST = async (req: Request) => {
     if (clickId) {
       const clickEvent = await getClickEvent({ clickId });
 
-      if (!clickEvent || clickEvent.data.length === 0) {
+      if (!clickEvent) {
         clickId = null;
       }
     }

apps/web/app/(ee)/api/stripe/integration/webhook/checkout-session-completed.ts

@@ -69,9 +69,7 @@ export async function checkoutSessionCompleted(
   if (clientReferenceId?.startsWith("dub_id_")) {
     const dubClickId = clientReferenceId.split("dub_id_")[1];
 
-    clickEvent = await getClickEvent({ clickId: dubClickId }).then(
-      (res) => res.data[0],
-    );
+    clickEvent = await getClickEvent({ clickId: dubClickId });
 
     if (!clickEvent) {
       return `Click event with dub_id ${dubClickId} not found, skipping...`;

apps/web/app/(ee)/api/stripe/integration/webhook/utils/create-new-customer.ts

@@ -26,13 +26,11 @@ export async function createNewCustomer(event: Stripe.Event) {
   }
 
   // Find click
-  const clickEvent = await getClickEvent({ clickId });
-  if (!clickEvent || clickEvent.data.length === 0) {
+  const clickData = await getClickEvent({ clickId });
+  if (!clickData) {
     return `Click event with ID ${clickId} not found, skipping...`;
   }
 
-  const clickData = clickEvent.data[0];
-
   // Find link
   const linkId = clickData.link_id;
   const link = await prisma.link.findUnique({

apps/web/lib/api/conversions/track-lead.ts

@@ -6,7 +6,7 @@ import { createPartnerCommission } from "@/lib/partners/create-partner-commissio
 import { isStored, storage } from "@/lib/storage";
 import { getClickEvent, recordLead } from "@/lib/tinybird";
 import { logConversionEvent } from "@/lib/tinybird/log-conversion-events";
-import { ClickEventTB, WebhookPartner, WorkspaceProps } from "@/lib/types";
+import { WebhookPartner, WorkspaceProps } from "@/lib/types";
 import { redis } from "@/lib/upstash";
 import { sendWorkspaceWebhook } from "@/lib/webhook/publish";
 import { transformLeadEventData } from "@/lib/webhook/transform";
@@ -104,32 +104,9 @@ export const trackLead = async ({
   // we can proceed with the lead tracking process
   if (!isDuplicateEvent) {
     // First, we need to find the click event
-    let clickData: ClickEventTB | null = null;
-    const clickEvent = await getClickEvent({ clickId });
+    const clickData = await getClickEvent({ clickId });
 
-    if (clickEvent && clickEvent.data && clickEvent.data.length > 0) {
-      clickData = clickEvent.data[0];
-    }
-
-    // if there is no click data in Tinybird yet, check the clickIdCache
-    if (!clickData) {
-      const cachedClickData = await redis.get<ClickEventTB>(
-        `clickIdCache:${clickId}`,
-      );
-
-      if (cachedClickData) {
-        clickData = {
-          ...cachedClickData,
-          timestamp: cachedClickData.timestamp
-            .replace("T", " ")
-            .replace("Z", ""),
-          qr: cachedClickData.qr ? 1 : 0,
-          bot: cachedClickData.bot ? 1 : 0,
-        };
-      }
-    }
-
-    // if there is still no click data, throw an error
+    // if there is no click data, throw an error
     if (!clickData) {
       throw new DubApiError({
         code: "not_found",

apps/web/lib/api/conversions/track-sale.ts

@@ -151,32 +151,10 @@ export const trackSale = async ({
     }
 
     // Find the click event for the given clickId
-    const clickEvent = await getClickEvent({
+    const clickData = await getClickEvent({
       clickId,
     });
 
-    if (clickEvent && clickEvent.data && clickEvent.data.length > 0) {
-      clickData = clickEvent.data[0];
-    }
-
-    // If there is no click data in Tinybird yet, check the clickIdCache
-    if (!clickData) {
-      const cachedClickData = await redis.get<ClickEventTB>(
-        `clickIdCache:${clickId}`,
-      );
-
-      if (cachedClickData) {
-        clickData = {
-          ...cachedClickData,
-          timestamp: cachedClickData.timestamp
-            .replace("T", " ")
-            .replace("Z", ""),
-          qr: cachedClickData.qr ? 1 : 0,
-          bot: cachedClickData.bot ? 1 : 0,
-        };
-      }
-    }
-
     if (!clickData) {
       throw new DubApiError({
         code: "not_found",

apps/web/lib/integrations/shopify/create-lead.ts

@@ -1,4 +1,5 @@
 import { createId } from "@/lib/api/create-id";
+import { DubApiError } from "@/lib/api/errors";
 import { includeTags } from "@/lib/api/links/include-tags";
 import { generateRandomName } from "@/lib/names";
 import { getClickEvent, recordLead } from "@/lib/tinybird";
@@ -34,9 +35,15 @@ export async function createShopifyLead({
   const email = orderCustomer?.email;
 
   // find click
-  const clickEvent = await getClickEvent({ clickId });
+  const clickData = await getClickEvent({ clickId });
+
+  if (!clickData) {
+    throw new DubApiError({
+      code: "not_found",
+      message: `Click event not found for clickId: ${clickId}`,
+    });
+  }
 
-  const clickData = clickEvent.data[0];
   const { link_id: linkId, country, timestamp } = clickData;
 
   // create customer

apps/web/lib/tinybird/get-click-event.ts

@@ -1,11 +1,41 @@
+import { ClickEventTB } from "../types";
+import { redis } from "../upstash";
 import z from "../zod";
 import { clickEventSchemaTB } from "../zod/schemas/clicks";
 import { tb } from "./client";
 
-export const getClickEvent = tb.buildPipe({
+const getClickEventTB = tb.buildPipe({
   pipe: "get_click_event",
   parameters: z.object({
     clickId: z.string(),
   }),
   data: clickEventSchemaTB,
 });
+
+export const getClickEvent = async ({ clickId }: { clickId: string }) => {
+  // check Redis cache first
+  const cachedClickData = await redis.get<ClickEventTB>(
+    `clickIdCache:${clickId}`,
+  );
+
+  if (cachedClickData) {
+    return {
+      ...cachedClickData,
+      timestamp: cachedClickData.timestamp.replace("T", " ").replace("Z", ""),
+      qr: cachedClickData.qr ? 1 : 0,
+      bot: cachedClickData.bot ? 1 : 0,
+    };
+  }
+
+  try {
+    // fallback to Tinybird if Redis cache is not found
+    const { data } = await getClickEventTB({ clickId });
+    return data[0];
+  } catch (error) {
+    console.error(
+      `[getClickEvent] Error getting click event for clickId: ${clickId}`,
+      error,
+    );
+    return null;
+  }
+};

apps/web/lib/tinybird/record-click.ts

@@ -168,10 +168,10 @@ export async function recordClick({
   };
 
   if (shouldCacheClickId) {
-    // cache the click ID and its corresponding click data in Redis for 5 mins
+    // cache the click ID and its corresponding click data in Redis for 1 day
     // we're doing this because ingested click events are not available immediately in Tinybird
     await redis.set(`clickIdCache:${clickId}`, clickData, {
-      ex: 60 * 5,
+      ex: 60 * 60 * 24, // cache for 1 day
     });
   }
 

apps/web/lib/webhook/transform.ts

@@ -74,7 +74,7 @@ export const transformLeadEventData = (data: any) => {
     },
     // transformLink -> add shortLink, qrCode, workspaceId, etc.
     link: transformLink(lead.link as ExpandedLink),
-    metadata: lead.metadata ?? null,
+    metadata: lead.metadata || null,
   });
 };
 

apps/web/scripts/migrations/backfill-attribution.ts

@@ -36,16 +36,14 @@ async function main() {
     },
   });
 
-  const clickEvent = await getClickEvent({
+  const clickData = await getClickEvent({
     clickId,
   });
 
-  if (!clickEvent || clickEvent.data.length === 0) {
+  if (!clickData) {
     throw new Error("Click event not found");
   }
 
-  const clickData = clickEvent.data[0];
-
   const existingCustomer = await prisma.customer.findUnique({
     where: {
       stripeCustomerId: referredWorkspace.stripeId,

apps/web/scripts/tinybird/update-click-event.ts

@@ -7,8 +7,7 @@ async function main() {
   const columnName = "link_id";
   const columnValue = "link_1JRVCZWHWACS7R2KZB9ME6CJR";
 
-  const { data } = await getClickEvent({ clickId });
-  const oldData = data[0];
+  const oldData = await getClickEvent({ clickId });
   if (!oldData) {
     console.log("No data found");
     return;