Batch the URL deletion

Author: davidbarratt

workers/handler/src/index.ts

@@ -105,7 +105,7 @@ async function cachePurgeTag(batch: MessageBatch, env: Env) {
 
 		// Re-queue all of the tags as URLs.
 		// sendBatch only allows for a maximum of 100 messages.
-		const promises: ReturnType<typeof env.CACHE_PURGE_URL.sendBatch>[] = [];
+		const promises: ReturnType<typeof env.DB.batch>[] = [];
 		for (const urlChunk of chunks(results, 100)) {
 			promises.push(
 				env.CACHE_PURGE_URL.sendBatch(
@@ -118,23 +118,22 @@ async function cachePurgeTag(batch: MessageBatch, env: Env) {
 							contentType: "json",
 						}),
 					),
-				),
+				).then(() => {
+					const ids = urlChunk.map<string>(({ id }) => id);
+					return env.DB.batch([
+						env.DB.prepare(
+							`DELETE FROM tag WHERE url IN (${ids.map(() => "?").join(", ")})`,
+						).bind(...ids),
+						env.DB.prepare(
+							`DELETE FROM url WHERE id IN (${ids.map(() => "?").join(", ")})`,
+						).bind(...ids),
+					]);
+				}),
 			);
 		}
 
 		await Promise.all(promises);
 
-		const ids = results.map<string>(({ id }) => id);
-
-		await env.DB.batch([
-			env.DB.prepare(
-				`DELETE FROM tag WHERE url IN (${ids.map(() => "?").join(", ")})`,
-			).bind(...ids),
-			env.DB.prepare(
-				`DELETE FROM url WHERE id IN (${ids.map(() => "?").join(", ")})`,
-			).bind(...ids),
-		]);
-
 		for (const msg of msgs) {
 			msg.ack();
 		}

workers/handler/wrangler.toml

@@ -92,7 +92,7 @@ queue = "cache-capture"
 
 [[queues.consumers]]
 queue = "cache-purge-tag"
-max_batch_size = 90 # The queue limit is 100, but the D1 binding limit is 100 and sometimes we need a few more params.
+max_batch_size = 95 # The queue limit is 100, but the D1 binding limit is 100 and sometimes we need a few more (i.e. for the zone).
 
 [[queues.consumers]]
 queue = "cache-purge-url"