Makes document attachment usage consider external attachments

app/server/lib/ActiveDoc.ts

@@ -125,6 +125,7 @@ import {LogMethods} from 'app/server/lib/LogMethods';
 import {ISandboxOptions} from 'app/server/lib/NSandbox';
 import {NullSandbox, UnavailableSandboxMethodError} from 'app/server/lib/NullSandbox';
 import {DocRequests} from 'app/server/lib/Requests';
+import {SandboxError} from 'app/server/lib/sandboxUtil';
 import {
   getDocSessionAccess,
   getDocSessionAccessOrNull,
@@ -1054,6 +1055,8 @@ export class ActiveDoc extends EventEmitter {
       unused: 0,
     };
 
+    const sizesToUpdate = new Map<string, number>();
+
     await unpackTarArchive(tarFile, async (file) => {
       try {
         const fileIdent = archiveFilePathToAttachmentIdent(file.path);
@@ -1063,6 +1066,7 @@ export class ActiveDoc extends EventEmitter {
           fallbackStoreId,
         );
         if (isAdded) {
+          sizesToUpdate.set(fileIdent, file.size);
           results.added += 1;
         } else {
           results.unused += 1;
@@ -1076,6 +1080,11 @@ export class ActiveDoc extends EventEmitter {
       }
     });
 
+    // This updates _grist_Attachments.fileSize with the size of the uploaded files.
+    // This prevents a loophole where a user has altered `fileSize`, imported the altered document,
+    // then restored the originals.
+    await this._updateAttachmentFileSizesUsingFileIdent(docSession, sizesToUpdate);
+
     return results;
   }
 
@@ -2614,6 +2623,45 @@ export class ActiveDoc extends EventEmitter {
     }];
   }
 
+  private async _updateAttachmentFileSizesUsingFileIdent(docSession: OptDocSession,
+                                                         newFileSizesByFileIdent: Map<string, number>): Promise<void> {
+    const rowIdsToUpdate: number[] = [];
+    const newFileSizesForRows: CellValue[] = [];
+    const attachments = this.docData?.getMetaTable("_grist_Attachments").getRecords();
+    for (const attachmentRec of attachments ?? []) {
+      const newSize = newFileSizesByFileIdent.get(attachmentRec.fileIdent);
+      if (newSize && newSize !== attachmentRec.fileSize) {
+        rowIdsToUpdate.push(attachmentRec.id);
+        newFileSizesForRows.push(newSize);
+      }
+    }
+
+    if (rowIdsToUpdate.length === 0) {
+      return;
+    }
+
+    const action: BulkUpdateRecord = ['BulkUpdateRecord', '_grist_Attachments', rowIdsToUpdate, {
+      fileSize: newFileSizesForRows
+    }];
+
+    try {
+      await this._applyUserActionsWithExtendedOptions(
+        docSession,
+        [action],
+        {attachment: true},
+      );
+    } catch (e) {
+      if (e instanceof SandboxError) {
+        this._log.error(null, "Attachment sizes could not be updated due to a sandbox error: ", e);
+        throw new Error("Attachment sizes could not be updated due to a sandbox error", { cause: e });
+      }
+      throw e;
+    }
+
+    // Updates doc's overall attachment usage to reflect any changes to file sizes.
+    await this._updateAttachmentsSize();
+  }
+
   /**
    * If the software is newer than the document, migrate the document by fetching all tables, and
    * giving them to the sandbox so that it can produce migration actions.

app/server/lib/Archive.ts

@@ -122,6 +122,7 @@ export function create_tar_archive(
 export interface UnpackedFile {
   path: string;
   data: stream.Readable;
+  size: number;
 }
 
 export async function unpackTarArchive(
@@ -144,6 +145,8 @@ export async function unpackTarArchive(
       onFile({
         path: header.name,
         data: contentStream,
+        // Realistically this should never be undefined - it's mandatory for files in a .tar archive
+        size: header.size ?? 0,
       })
       // No sensible behaviour when an error is thrown by onFile - it's onFile's responsibility
       // to handle it.

app/server/lib/DocStorage.ts

@@ -1385,7 +1385,14 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
         -- to make LENGTH() quickly read the stored length instead of actually reading the blob data.
         -- We use LENGTH() in the first place instead of _grist_Attachments.fileSize because the latter can
         -- be changed by users.
-        SELECT MAX(LENGTH(files.data)) AS len
+        -- External attachments have no internal blob data, so we need to use fileSize for those.
+        -- To avoid tampering with fileSize in offline records, we update it whenever files are
+        -- uploaded or retrieved.
+        SELECT
+          CASE WHEN files.storageId IS NOT NULL
+            THEN MAX(meta.fileSize)
+            ELSE MAX(LENGTH(files.data))
+          END AS len
         FROM _gristsys_Files AS files
           JOIN _grist_Attachments AS meta
             ON meta.fileIdent = files.ident