Merge branch 'release_0.4.1'

Author: melchor629

docs/actions/compress.md

@@ -165,6 +165,7 @@ This compression algorithm is fast and good-balanced in resources consumption. C
 | Name | Type | Description | Optional |
 |------|------|-------------|----------|
 | `compressionLevel` | `ìnt` | Compression level from 1 to 19 (default 3) | Yes |
+| `cpus` | `int` | Uses this amount of cores to compress the data | Yes |
 
 **Description**
 

docs/actions/encrypt.md

@@ -14,8 +14,9 @@
 |------|------|-------------|----------|
 | `passphrase` | `str` | Defines a passphrase that will be used to decrypt the data | Yes |
 | `recipients` | `List[str]` | List of emails that will be able to decrypt the data | Yes |
-| `cipherAlgorithm` | `str` | Changes the cypher algorithm | Yes |
-| `compressAlgorithm` | `str` | Changes the compress algorithm | Yes |
+| `cipherAlgorithm` | `str` | Changes the cipher algorithm | Yes |
+| `cypherAlgorithm` | `str` | Alias for `cypherAlgorithm` | Yes |
+| `compressAlgorithm` | `str` | Selects the compress algorithm, if not set, then will be uncompressed (default) | Yes |
 
 At least one recipient must be defined. If no recipients are defined, a passphrase must be provided. Both can be defined.
 

docs/configuration.md

@@ -46,9 +46,10 @@ This allows you to auto-complete with the elements available in the configuratio
     "cloud": {
       "compression": {
         "method": "gz|xz|bz2|br|zst",
-        "level": 8
+        "level": 8,
+        "cpus": 1
       },
-      "cypher": {
+      "encrypt": {
         "strategy": "gpg-keys|gpg-passphrase",
         "passphrase": "If using gpg-passphrase, this will be used as passphrase for the cypher",
         "keys": "If using gpg-keys, this will be used as recipients option for the gpg cypher (emails)",
@@ -114,8 +115,9 @@ cloud:
   compression:
     method: gz|xz|bz2|br|zst
     level: 8
+    cpus: 1
 
-  cypher:
+  encrypt:
     strategy: gpg-keys|gpg-passphrase
     passphrase: If using gpg-passphrase, this will be used as passphrase for the cypher
     keys: If using gpg-keys, this will be used as recipients option for the gpg cypher (emails)
@@ -217,7 +219,14 @@ In general, a lot of Linux distributions includes these commands, as well as in
 
 The compression level. Higher values indicates better but slower compressions. Values accepted for `gzip` are from 1 to 9. Values accepted for `xz` are from 0 to 9 (by default is 6, 7-9 are not recommended).
 
-### cypher
+#### cpus
+
+The number of cpus/threads to use when compressing. By default, will use only one thread to compress, but this can be changed to any number. This value is ignored by some compression algorithms.
+
+!!! Note "Threads and compression algorithms"
+    Currently, `xz` and `zst` supports `cpus` setting, the rest will always use 1 thread.
+
+### encrypt
 
 If defined, when backups are uploaded to a storage provider, folders will be encrypted using this configuration.
 

mdbackup/_commands/upload.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from typing import Iterable, List, Tuple
 
-from ..archive import archive_folder
+from ..archive import archive_file, archive_folder
 from ..config import Config
 from ..hooks import run_hook
 from ..storage import create_storage_instance
@@ -24,17 +24,29 @@ def _process_results(config: Config, backup: Path, items: Iterable[Tuple[Path, d
 
     # Compress directories
     for item, task in items:
-        # Compress if it is a directory
         if item.is_dir():
+            # Compress/encrypt if it is a directory
             filename = archive_folder(backup, item, config.cloud)
             final_items.append(backup / filename)
             items_to_remove.append(backup / filename)
             task['cloudResult'] = Path(filename)
         else:
-            final_items.append(item)
+            # Compress/encrypt if it is a file (if needed)
+            filename = archive_file(backup, item, task, config.cloud)
+            if filename is not None:
+                final_items.append(backup / filename)
+                items_to_remove.append(backup / filename)
+                task['cloudResult'] = Path(filename)
+            else:
+                final_items.append(item)
 
     # Add the manifest as well :)
-    final_items.append(backup / '.manifest.yaml')
+    manifest_filename = archive_file(backup, backup / '.manifest.yaml', {'actions': []}, config.cloud)
+    if manifest_filename is not None:
+        final_items.append(backup / manifest_filename)
+        items_to_remove.append(backup / manifest_filename)
+    else:
+        final_items.append(backup / '.manifest.yaml')
     return final_items, items_to_remove
 
 
@@ -119,7 +131,7 @@ def main_upload(config: Config, backup: Path, force: bool = False):
         raise FileNotFoundError(backup)
     if not backup.is_dir():
         raise NotADirectoryError(backup)
-    if not str(backup).startswith(str(config.backups_path)):
+    if not str(backup).startswith(str(config.backups_path.resolve())):
         raise ValueError(f'Backup path {backup} is not inside the backups path')
     manifest_path = backup / '.manifest.yaml'
     if not manifest_path.exists():
@@ -143,5 +155,5 @@ def main_upload(config: Config, backup: Path, force: bool = False):
     finally:
         # Remove compressed directories
         for item in items_to_remove:
-            logger.info(f'Removing file from compressed directory {item}')
+            logger.info(f'Removing temporary file {item}')
             item.unlink()

mdbackup/actions/builtin/database.py

@@ -102,6 +102,7 @@ def action_influxd_command(_, params: dict):
 
     if docker:
         params['image'] = params.get('image', 'influxdb:alpine')
+        params['user'] = None
         return action_docker(None, params)
     else:
         return action_command(None, params)

mdbackup/actions/builtin/encrypt.py

@@ -10,8 +10,8 @@
 def action_encrypt_opengpg(inp: InputDataStream, params) -> subprocess.Popen:
     passphrase = params.get('passphrase')
     recipients: list = params.get('recipients', [])
-    algorithm = params.get('cipherAlgorithm')
-    compress = params.get('compressAlgorithm', False)
+    algorithm = params.get('cipherAlgorithm', params.get('cypherAlgorithm'))
+    compress = params.get('compressAlgorithm', None)
 
     raise_if_type_is_incorrect(passphrase, str, 'passphrase must be a string')
     raise_if_type_is_incorrect(recipients, list, 'recipients must be a list of strings')

mdbackup/actions/builtin/file.py

@@ -102,7 +102,7 @@ def _checks(params: dict) -> Path:
 
 def _file_has_changed(entry, old_file: Path) -> bool:
     mod_time_current = entry.st_mtime_ns
-    mod_time_prev = old_file.stat().st_mtime_ns
+    mod_time_prev = old_file.lstat().st_mtime_ns
     return mod_time_current != mod_time_prev
 
 
@@ -148,7 +148,7 @@ def action_copy_file(_, params: dict):
     logger = logging.getLogger(__name__).getChild('action_copy_file')
     orig_path = Path(params['from']) if params.get('from') is not None else None
     orig_stream = params.get('_stream')
-    orig_stat = orig_path.stat() if orig_path is not None else params['_stat']
+    orig_stat = orig_path.lstat() if orig_path is not None else params['_stat']
     dest_path = _checks(params)
     in_path = Path(params['to'])
     preserve_stats = params.get('preserveStats', 'utime')
@@ -200,7 +200,7 @@ def action_reverse_copy_file(_, params: dict):
     dest_path = Path(params['from'])
     orig_path = _checks(params) if 'to' in params or 'path' in params else None
     orig_stream = params.get('_stream')
-    orig_stat = orig_path.stat() if orig_path is not None else params['_stat']
+    orig_stat = orig_path.lstat() if orig_path is not None else params['_stat']
     preserve_stats = params.get('preserveStats', 'utime')
 
     raise_if_type_is_incorrect(preserve_stats, (str, bool), 'preserveStats must be a string or a boolean')
@@ -241,7 +241,7 @@ def action_clone_file(_, params: dict):
             # https://stackoverflow.com/questions/52766388/how-can-i-use-the-copy-on-write-of-a-btrfs-from-c-code
             # https://github.com/coreutils/coreutils/blob/master/src/copy.c#L370
             if not dest_path.exists():
-                dest_path.touch(orig_path.stat().st_mode)
+                dest_path.touch(orig_path.lstat().st_mode)
             src_fd = os.open(orig_path, flags=os.O_RDONLY)
             dst_fd = os.open(dest_path, flags=os.O_WRONLY | os.O_TRUNC)
             try:
@@ -257,7 +257,7 @@ def action_clone_file(_, params: dict):
 
             if preserve_stats:
                 xattrs = _read_xattrs(orig_path)
-                _preserve_stats(dest_path, orig_path.stat(), xattrs, preserve_stats)
+                _preserve_stats(dest_path, orig_path.lstat(), xattrs, preserve_stats)
         else:
             cow_failed = True
 

mdbackup/actions/ds.py

@@ -27,7 +27,7 @@ def __init__(self, _type: str, path: Path, stats, **kwargs):
 
     @staticmethod
     def from_real_path(path: Path, root_path: Optional[Path] = None, **kwargs) -> 'DirEntry':
-        stats = path.stat()
+        stats = path.lstat()
         rel_path = path.relative_to(root_path) if root_path is not None else path
         try:
             xattrs = _read_xattrs(path)

mdbackup/archive.py

@@ -1,5 +1,6 @@
 import logging
 from pathlib import Path
+from typing import Optional
 
 from mdbackup.actions.runner import run_task_actions
 from mdbackup.config import CloudConfig
@@ -16,7 +17,7 @@ def archive_folder(backup_path: Path, folder: Path, cloud_config: CloudConfig) -
 
     The returned value is the file name for the archived folder.
     """
-    logger = logging.getLogger(__name__)
+    logger = logging.getLogger(__name__).getChild('archive_folder')
     filename = str(folder.relative_to(backup_path)) + '.tar'
     actions = [
         {'from-directory': str(folder)},
@@ -27,12 +28,13 @@ def archive_folder(backup_path: Path, folder: Path, cloud_config: CloudConfig) -
         actions.append({
             f'compress-{cloud_config.compression_strategy}': {
                 'level': cloud_config.compression_level,
+                'cpus': cloud_config.compression_cpus,
             },
         })
         filename += f'.{cloud_config.compression_strategy}'
     if cloud_config.cypher_strategy is not None:
         actions.append({
-            f'encrypt-gpg': {
+            'encrypt-gpg': {
                 'passphrase': cloud_config.cypher_params.get('passphrase'),
                 'recipients': cloud_config.cypher_params.get('keys', []),
                 'algorithm': cloud_config.cypher_params.get('algorithm'),
@@ -42,8 +44,53 @@ def archive_folder(backup_path: Path, folder: Path, cloud_config: CloudConfig) -
 
     actions.append({'to-file': {'_backup_path': backup_path, 'to': filename}})
 
-    # Do the compression
-    logger.info(f'Compressing directory {folder} into {filename}')
+    # Do the magic
+    logger.info(f'Compressing/encrypting directory {folder} into {filename}')
     run_task_actions('archive-folder', actions)
 
     return filename
+
+
+def archive_file(backup_path: Path, file_path: Path, task: dict, cloud_config: CloudConfig) -> Optional[str]:
+    logger = logging.getLogger(__name__).getChild('archive_file')
+
+    # Gets the actions used in this file and check if it is compressed or encrypted using any action
+    actions = [next(iter(item.keys())) for item in task['actions']]
+    has_compress_action = len([item for item in actions if 'compress-' in item]) != 0
+    has_encrypt_action = len([item for item in actions if 'encrypt-' in item]) != 0
+
+    filename = str(file_path.relative_to(backup_path))
+    archive_actions = [
+        {'from-file': str(file_path)},
+    ]
+
+    if cloud_config.compression_strategy is not None and not has_compress_action:
+        archive_actions.append({
+            f'compress-{cloud_config.compression_strategy}': {
+                'level': cloud_config.compression_level,
+                'cpus': cloud_config.compression_cpus,
+            },
+        })
+        filename += f'.{cloud_config.compression_strategy}'
+    if cloud_config.cypher_strategy is not None and not has_encrypt_action:
+        archive_actions.append({
+            'encrypt-gpg': {
+                'passphrase': cloud_config.cypher_params.get('passphrase'),
+                'recipients': cloud_config.cypher_params.get('keys', []),
+                'algorithm': cloud_config.cypher_params.get('algorithm'),
+            },
+        })
+        filename += '.asc'
+
+    # If no extra actions are required, then just return and do nothing
+    if len(archive_actions) == 1:
+        logger.info(f'File {file_path} will not compress/encrypt because it is not required')
+        return None
+
+    archive_actions.append({'to-file': {'_backup_path': backup_path, 'to': filename}})
+
+    # Do the magic
+    logger.info(f'Compressing/encrypting file {file_path} into {filename}')
+    run_task_actions('archive-file', archive_actions)
+
+    return filename

mdbackup/config/cloud.py

@@ -10,9 +10,11 @@ def __init__(self, conf: Dict[str, str]):
         if 'compression' in conf:
             self.__compression_level = conf['compression'].get('level', 6)
             self.__compression_strategy = conf['compression']['method']
+            self.__compression_cpus = conf['compression'].get('cpus', None)
         else:
             self.__compression_level = None
             self.__compression_strategy = None
+            self.__compression_cpus = None
         if 'encrypt' in conf:
             self.__cypher_strategy = conf['encrypt']['strategy']
             self.__cypher_params = change_keys(conf['encrypt'])
@@ -42,6 +44,13 @@ def compression_level(self) -> Optional[int]:
         """
         return self.__compression_level
 
+    @property
+    def compression_cpus(self) -> Optional[int]:
+        """
+        :return: The threads/cpus to use during compression
+        """
+        return self.__compression_cpus
+
     @property
     def cypher_strategy(self) -> Optional[str]:
         """

mdbackup/json-schemas/cloud.compression.schema.json

@@ -22,9 +22,14 @@
     "level": {
       "$id": "#/properties/level",
       "type": "number",
-      "title": "Selects the compression level",
-      "default": 6,
-      "enum": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ]
+      "title": "Selects the compression level to use to compress files (value depends on compression algorithm)",
+      "default": 6
+    },
+    "cpus": {
+      "$id": "#/properties/cpus",
+      "type": "number",
+      "title": "Selects the number of threads to use to compress files (may be ignored by some algorithms)",
+      "default": 1
     }
   }
 }

setup.py

@@ -2,7 +2,7 @@
 
 import setuptools
 
-version = '0.4.0'
+version = '0.4.1'
 
 with open("README.md", "r") as fh:
     long_description = fh.read()