Adjust Archive handling to save (original) mtime of files and always apply default checksum calculation when requested through the Schedule Tasks background jobs.

git-svn-id: svn+ssh://svn.code.sf.net/p/migrid/code/trunk@6045 b75ad72c-e7d7-11dd-a971-7dbc132099af

Author: jonasbardino

mig/shared/freezefunctions.py

@@ -182,17 +182,24 @@ def build_freezeitem_object(configuration, freeze_dict, summary=False,
                 'title': 'Show archive file %(name)s' % file_item,
                 'text': ''
             }
+            # NOTE: we have file creation time in archive as timestamp
             int_timestamp = int(file_item['timestamp'])
+            # NOTE: we may have preserved file modification time as file_mtime
+            int_origtime = int(file_item.get('file_mtime', int_timestamp))
             # NOTE: datetime is not json-serializable so we force to string
             dt_timestamp = datetime.datetime.fromtimestamp(int_timestamp)
+            dt_origtime = datetime.datetime.fromtimestamp(int_origtime)
             str_timestamp = "%s" % dt_timestamp
+            if dt_timestamp != dt_origtime:
+                str_timestamp += " / %s" % dt_origtime
             # NOTE: xmlrpc is limited to 32-bit ints so force size to string
             str_size = "%(size)s" % file_item
             entry = {
                 'object_type': 'frozenfile',
                 'name': file_item['name'],
                 'showfile_link': showfile_link,
                 'timestamp': int_timestamp,
+                'origtime': int_origtime,
                 'date': str_timestamp,
                 'size': str_size,
             }
@@ -713,14 +720,16 @@ def get_frozen_files(client_id, freeze_id, configuration,
             entry = file_map.get(rel_path, {})
             if not entry or needs_update:
                 entry['name'] = entry.get('name', rel_path)
-                file_ctime, file_size = -1, -1
+                file_ctime, file_mtime, file_size = -1, -1, -1
                 try:
                     file_ctime = os.path.getctime(frozen_path)
+                    file_mtime = os.path.getmtime(frozen_path)
                     file_size = os.path.getsize(frozen_path)
                 except Exception as err:
                     _logger.warning("failed to update cached %s stats: %s" %
                                     (frozen_path, err))
-                entry['timestamp'] = file_ctime
+                entry['timestamp'] = entry['file_ctime'] = file_ctime
+                entry['file_mtime'] = file_mtime
                 entry['size'] = file_size
                 updates += 1
 

mig/shared/functionality/addfreezedata.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # addfreezedata - back end for adding data to existing freeze archive
-# Copyright (C) 2003-2018  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -26,10 +26,11 @@
 #
 
 """Add a file or folder to existing freeze archive e.g. in background job"""
+
 from __future__ import absolute_import
 
 from mig.shared.functionality.createfreeze import main as freeze_main, \
-    signature as freeze_signature
+    signature as freeze_signature, default_algo
 
 
 def main(client_id, user_arguments_dict):
@@ -39,4 +40,6 @@ def main(client_id, user_arguments_dict):
     args_dict = freeze_signature()[1]
     args_dict.update(user_arguments_dict)
     args_dict['flavor'] = ['freeze']
+    # Always checksum when running in background
+    args_dict['checksum_list'] = [default_algo]
     return freeze_main(client_id, args_dict)

mig/shared/functionality/createbackup.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # createbackup - back end for one-shot backup archive creation
-# Copyright (C) 2003-2018  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -29,7 +29,7 @@
 from __future__ import absolute_import
 
 from mig.shared.functionality.createfreeze import main as createfreeze_main, \
-    signature as createfreeze_signature, keyword_final
+    signature as createfreeze_signature, keyword_final, default_algo
 
 
 def main(client_id, user_arguments_dict):
@@ -39,5 +39,7 @@ def main(client_id, user_arguments_dict):
     args_dict = createfreeze_signature()[1]
     args_dict.update(user_arguments_dict)
     args_dict['flavor'] = ['backup']
+    # Always checksum when running in background
+    args_dict['checksum_list'] = [default_algo]
     args_dict['freeze_state'] = [keyword_final]
     return createfreeze_main(client_id, args_dict)

mig/shared/functionality/createfreeze.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # createfreeze - back end for freezing archives
-# Copyright (C) 2003-2023  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -38,7 +38,7 @@
     keyword_auto, keyword_pending, keyword_final
 from mig.shared.fileio import strip_dir, walk
 from mig.shared.freezefunctions import create_frozen_archive, published_url, \
-    is_frozen_archive
+    is_frozen_archive, get_frozen_files, default_algo
 from mig.shared.functional import validate_input_and_cert, REJECT_UNSET
 from mig.shared.handlers import safe_handler, get_csrf_limit, make_csrf_token
 from mig.shared.html import man_base_js, man_base_html, html_post_helper
@@ -62,6 +62,7 @@ def signature():
         'freeze_department': [''],
         'freeze_organization': [''],
         'freeze_state': [keyword_auto],
+        'checksum_list': [],
     }
     return ['text', defaults]
 
@@ -243,6 +244,7 @@ def main(client_id, user_arguments_dict):
     freeze_department = accepted['freeze_department'][-1].strip()
     freeze_organization = accepted['freeze_organization'][-1].strip()
     freeze_publish = accepted['freeze_publish'][-1].strip()
+    chksums = accepted['checksum_list']
     do_publish = (freeze_publish.lower() in ('on', 'true', 'yes', '1'))
 
     # Share init of base meta with lookup of default state in freeze_flavors
@@ -336,6 +338,13 @@ def main(client_id, user_arguments_dict):
     freeze_id = freeze_meta['ID']
     logger.info("%s: successful for '%s': %s" % (op_name,
                                                  freeze_id, client_id))
+    # Calculate / update checksums if requested
+    if chksums:
+        (chksum_status, chksum_res) = get_frozen_files(client_id, freeze_id,
+                                                       configuration,
+                                                       checksum_list=chksums,
+                                                       force_refresh=True)
+
     # Return simple status mainly for use in scripting
     output_objects.append({'object_type': 'freezestatus', 'freeze_id': freeze_id,
                            'flavor': flavor, 'freeze_state': freeze_state})

mig/shared/output.py

@@ -1619,7 +1619,7 @@ def html_format(configuration, ret_val, ret_msg, out_obj):
     <tr>
         <th>Name</th>
         <th class="icon">Action<!-- Open, Delete --></th>
-        <th>Date</th>
+        <th>Date added / original</th>
         <th>Size in bytes</th>'''
             for algo in sorted_hash_algos:
                 checksum_field = '%ssum' % algo