Started to build function with more granular copy controls

Author: 3aa49ec6bfc910647fa1c5a013e48eef

worker/package-lock.json

@@ -20,6 +20,7 @@
     "sqlite3": "^5.1.6"
   },
   "devDependencies": {
+    "@types/ini": "^4.1.0",
     "@types/node": "^20.10.5",
     "@types/ping": "^0.4.4",
     "tsup": "^8.0.1",

worker/package.json

@@ -4,8 +4,9 @@ import fs from 'fs';
 import { executeBashCommand } from './bash.js';
 import { clearInterval, setInterval } from 'timers';
 import readLastLines from 'read-last-lines';
+import path from 'path';
 
-export const rcloneCopyWithProgress = async (path, rcloneConfig, destinationPath) => {
+export const rcloneCopyWithProgress = async (path: string, rcloneConfig: string, destinationPath: string) => {
     const intervalId = setInterval(() => {
         logWithTimestamp("Copying files, check rclone.log for status...");
         // Add something useful here, e.g. giving percentage complete, transfer speed, time remaining
@@ -18,6 +19,57 @@ export const rcloneCopyWithProgress = async (path, rcloneConfig, destinationPath
     }
 };
 
+// TODO: Move over to this new function (not tested yet)
+
+export const rCloneCopyWithProgressByFolder = async (basePath: string, rcloneConfig: string, destinationPath: string) => {
+    // Function to list subfolders
+    const listSubfolders = async (dir: string): Promise<string[]> => {
+        const subdirs = await fs.promises.readdir(dir);
+        const folders = await Promise.all(subdirs.map(async (subdir) => {
+            const res = path.resolve(dir, subdir);
+            return (await fs.promises.stat(res)).isDirectory() ? res : '';
+        }));
+        return folders.filter(Boolean);
+    };
+
+    // Function to get folder information (number of files and total size)
+    const getFolderInfo = async (folderPath: string): Promise<{ fileCount: number, totalSize: number }> => {
+        const files = await fs.promises.readdir(folderPath);
+        let totalSize = 0;
+        for (const file of files) {
+            const filePath = path.join(folderPath, file);
+            const stat = await fs.promises.stat(filePath);
+            if (stat.isFile()) totalSize += stat.size;
+        }
+        return { fileCount: files.length, totalSize };
+    };
+
+    // List and sort subfolders
+    const folders = await listSubfolders(basePath);
+    folders.sort().reverse();
+
+    for (const folder of folders) {
+        // Get folder information
+        const { fileCount, totalSize } = await getFolderInfo(folder);
+
+        // Log event
+        console.log(`Copying folder: ${folder}, Files: ${fileCount}, Total Size: ${Math.round(totalSize / 1024 / 1024)} bytes`);
+
+        const intervalId = setInterval(() => {
+            logWithTimestamp(`Still copying folder: ${folder}.  Check rclone.log for status...`);
+            // Add something useful here, e.g. giving percentage complete, transfer speed, time remaining
+        }, 60000); // 60000 ms = 1 minute
+    
+        try {
+            await rcloneCopy(folder, rcloneConfig, path.join(destinationPath, path.basename(folder)));
+        } finally {
+            clearInterval(intervalId);
+        }
+
+    }
+};
+
+
 
 export const getRcloneConfig = () => {
     const configPath = '/root/.config/rclone/rclone.conf';
@@ -30,7 +82,7 @@ export const getRcloneConfig = () => {
     return config['node-teslausb'];
 }
 
-export const rcloneCopy = async (sourcePath, rcloneConfig, destinationPath) => {
+export const rcloneCopy = async (sourcePath: string, rcloneConfig: string, destinationPath: string) => {
     if (fs.existsSync(sourcePath) === false) {
         logWithTimestamp(`Skipping rclone copy for ${sourcePath} (does not exist)`)
         return
@@ -40,7 +92,7 @@ export const rcloneCopy = async (sourcePath, rcloneConfig, destinationPath) => {
 
 }
 
-export const getLastLineAsObject = async (filePath) => {
+export const getLastLineAsObject = async (filePath: string) => {
     try {
         const lastLine = await readLastLines.read(filePath, 1);
         return JSON.parse(lastLine);