Consider disk usage of collDir instead of default /crawls

src/crawler.ts

@@ -1114,7 +1114,11 @@ self.__bx_behaviors.selectMainBehavior();
 
     if (this.params.diskUtilization) {
       // Check that disk usage isn't already or soon to be above threshold
-      const diskUtil = await checkDiskUtilization(this.params, size);
+      const diskUtil = await checkDiskUtilization(
+        this.collDir,
+        this.params,
+        size,
+      );
       if (diskUtil.stop === true) {
         interrupt = true;
       }

src/util/storage.ts

@@ -202,14 +202,15 @@ export async function getDirSize(dir: string) {
 }
 
 export async function checkDiskUtilization(
+  collDir: string,
   // TODO: Fix this the next time the file is edited.
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   params: Record<string, any>,
   archiveDirSize: number,
   dfOutput = null,
 ) {
   const diskUsage: Record<string, string> = await getDiskUsage(
-    "/crawls",
+    collDir,
     dfOutput,
   );
   const usedPercentage = parseInt(diskUsage["Use%"].slice(0, -1));

tests/storage.test.js

@@ -29,6 +29,7 @@ grpcfuse       1000000      285000    715000  28% /crawls`;
   // with combineWARC + generateWACZ, projected is 285k + 4 * 5k = 310k = 31%
   // does not exceed 90% threshold
   const returnValue = await checkDiskUtilization(
+    '/crawls',
     params,
     5000 * 1024,
     mockDfOutput,
@@ -55,6 +56,7 @@ grpcfuse       100000    85000     15000  85% /crawls`;
   // with generateWACZ, projected is 85k + 3k x 2 = 91k = 91%
   // exceeds 90% threshold
   const returnValue = await checkDiskUtilization(
+    '/crawls',
     params,
     3000 * 1024,
     mockDfOutput,