Merge pull request #185 from kieranjol/copyit

Copyit

Author: kieranjol

README.md

@@ -61,11 +61,12 @@ Note: Documentation template has been copied from [mediamicroservices](https://g
 ## Arrangement ##
 
 ### sipcreator.py ###
-* Accepts one or more directories as input and wraps them up in a directory structure in line with IFI procedures using `moveit.py`.
-* Folders will be stored in an objects directory. Directory structure is a parent directory named with a UUID, with three child directories (objects, logs metadata):
-* Checksums are stored for the package and metadata is extracted for the AV material in the objects. A log records the major events in the process.
+* Accepts one or more files or directories as input and wraps them up in a directory structure in line with IFI procedures using `copyit.py`.
+* Source objects will be stored in an /objects directory. Directory structure is: parent directory named with a UUID, with three child directories (objects, logs metadata):
+* Metadata is extracted for the AV material and MD5 checksums are stored for the entire package. A log records the major events in the process.
 * Usage for one directory - `sipcreator.py -i /path/to/directory_name -o /path/to/output_folder`
 * Usage for more than one directory - `sipcreator.py -i /path/to/directory_name1 /path/to/directory_name2 -o /path/to/output_folder`
+* Run `sipcreator.py -h` for all options.
 
 ## Transcodes ##
 
@@ -87,10 +88,11 @@ Note: Documentation template has been copied from [mediamicroservices](https://g
 * This script has many extra options, such as deinterlacing, quality settings, rescaling. Use `prores.py -h` to see all options
 
 ### concat.py ###
-* Concatenate/join video files together using ffmpeg stream copy into a single Matroska container. As the streams are losslessly copied, the speed is quite fast.
+* Concatenate/join video files together using ffmpeg stream copy into a single Matroska container. Each source clip will have its own chapter marker. As the streams are copied, the speed is quite fast.
 * Usage: `concat.py -i /path/to/filename1.mov /path/to/filename2.mov -o /path/to/destination_folder`
+* A lossless verification process will also run, which takes stream level checksums of all streams and compares the values. This is not very reliable at the moment.
 * Warning - video files must have the same technical attributes such as codec, width, height, fps. Some characters in filenames will cause the script to fail. Some of these include quotes. The script will ask the user if quotes should be renamed with underscores. Also, a temporary concatenation textfile will be stored in your temp folder. Currently only tested on Ubuntu.
-
+* Dependencies: mkvpropedit, ffmpeg.
 ## Digital Cinema Package Scripts ##
 
 ### dcpaccess.py ###
@@ -112,8 +114,8 @@ Note: Documentation template has been copied from [mediamicroservices](https://g
 
 ## Fixity Scripts ##
 
-### moveit.py ###
-* Copies a directory, creating a md5 manifest at source and destination and comparing the two. Skips hidden files and directories.
+### copyit.py ###
+* Copies a file or directory, creating a md5 manifest at source and destination and comparing the two. Skips hidden files and directories.
 * Usage: ` moveit.py source_dir destination_dir`
 * Dependencies:  OSX requires gcp - `brew install coreutils`
 

concat.py

@@ -6,9 +6,11 @@
 import subprocess
 import os
 import argparse
+import time
+import shutil
+import sipcreator
 import ififuncs
 
-
 def parse_args(args_):
     '''
     Parse command line arguments.
@@ -29,6 +31,17 @@ def parse_args(args_):
         '-r', '-recursive',
         help='recursively process all files in subdirectories. This could be potentially a disaster - so use with caution or with XDCAM', action='store_true'
     )
+    parser.add_argument(
+        '--no-sip',
+        help='Do not run sipcreator.py on the resulting file.', action='store_true'
+    )
+    parser.add_argument(
+        '-user',
+        help='Declare who you are. If this is not set, you will be prompted.')
+    parser.add_argument(
+        '-oe',
+        help='Enter the Object Entry number for the representation.SIP will be placed in a folder with this name.'
+    )
     parsed_args = parser.parse_args(args_)
     return parsed_args
 
@@ -37,17 +50,22 @@ def ffmpeg_concat(concat_file, args, uuid):
     '''
     Launch the actual ffmpeg concatenation command
     '''
+    fmd5_logfile = os.path.join(args.o, '%s_concat.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
+    fmd5_env_dict = ififuncs.set_environment(fmd5_logfile)
+    print fmd5_logfile
+    print fmd5_env_dict
     cmd = [
-        'ffmpeg', '-f', 'concat', '-safe', '0',
+        'ffmpeg', '-report', '-f', 'concat', '-safe', '0',
         '-i', concat_file,
-        '-c', 'copy', '-map', '0:a?', '-map', '0:v',
-        os.path.join(args.o, '%s.mkv' % uuid)
+        '-c', 'copy', '-map', '0:v', '-map', '0:a?',
+        os.path.join(args.o, '%s.mkv' % uuid),
+        '-f', 'md5', '-map', '0:v', '-map', '0:a?','-c', 'copy',  '-'
     ]
     print cmd
-    subprocess.call(
-        cmd
+    source_bitstream_md5 = subprocess.check_output(
+        cmd, env=fmd5_env_dict
     )
-
+    return source_bitstream_md5.rstrip(), fmd5_logfile.replace('\\\\', '\\').replace('\:', ':')
 
 def recursive_file_list(video_files):
     '''
@@ -67,6 +85,25 @@ def recursive_file_list(video_files):
     print recursive_list
     return recursive_list
 
+def make_chapters(video_files):
+    millis = ififuncs.get_milliseconds(video_files[0])
+    timestamp = ififuncs.convert_millis(int(millis))
+    chapter_list = [['00:00:00.000', os.path.basename(video_files[0])], [timestamp, os.path.basename(video_files[1])]]
+    count = 2
+    for video in video_files[1:]:
+        millis += ififuncs.get_milliseconds(video)
+        timestamp = ififuncs.convert_millis(int(millis))
+        if count == len(video_files):
+            continue
+        else:
+            chapter_list.append([timestamp, os.path.basename(video_files[count])])
+        count+=1
+    chapter_counter = 1
+    # uh use a real path/filename.
+    with open('chapters.txt', 'wb') as fo:
+        for i in chapter_list:
+            fo.write('CHAPTER%s=%s\nCHAPTER%sNAME=%s\n' % (str(chapter_counter).zfill(2), i[0], str(chapter_counter).zfill(2), i[1]))
+            chapter_counter += 1
 
 
 def main(args_):
@@ -75,13 +112,115 @@ def main(args_):
     '''
     uuid = ififuncs.create_uuid()
     args = parse_args(args_)
+    print args
+    log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
+    ififuncs.generate_log(log_name_source, 'concat.py started.')
+    ififuncs.generate_log(
+        log_name_source,
+        'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py'))
+    ififuncs.generate_log(
+        log_name_source,
+        'Command line arguments: %s' % args
+    )
+    if args.user:
+        user = args.user
+    else:
+        user = ififuncs.get_user()
+    if args.oe:
+        if args.oe[:2] != 'oe':
+            print 'First two characters must be \'oe\' and last four characters must be four digits'
+            object_entry = ififuncs.get_object_entry()
+        elif len(args.oe[2:]) != 4:
+            print 'First two characters must be \'oe\' and last four characters must be four digits'
+            object_entry = ififuncs.get_object_entry()
+        elif not args.oe[2:].isdigit():
+           object_entry = ififuncs.get_object_entry()
+           print 'First two characters must be \'oe\' and last four characters must be four digits'
+        else:
+            object_entry = args.oe
+    else:
+        object_entry = ififuncs.get_object_entry()
+    ififuncs.generate_log(
+        log_name_source,
+        'EVENT = agentName=%s' % user
+    )
+    source_uuid_check = ififuncs.check_for_uuid(args)
+    if source_uuid_check == False:
+        source_uuid = ififuncs.get_source_uuid()
+    else: source_uuid = source_uuid_check
+    ififuncs.generate_log(
+        log_name_source,
+        'Relationship, derivation, has source=%s' % source_uuid
+    )
     video_files = args.i
     concat_file = ififuncs.get_temp_concat('concat_stuff')
+    ififuncs.generate_log(
+        log_name_source,
+        'concatenation file=%s' % concat_file)
     if args.r:
         video_files = recursive_file_list(video_files)
     video_files = ififuncs.sanitise_filenames(video_files)
+    for source_files in video_files:
+        ififuncs.generate_log(
+            log_name_source,
+            'source_files = %s' % source_files)
+    make_chapters(video_files)
+    ififuncs.concat_textfile(video_files, concat_file)
+    ififuncs.generate_log(
+        log_name_source,
+        'EVENT = Concatenation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.mkv' % uuid))
+    source_bitstream_md5, fmd5_logfile = ffmpeg_concat(concat_file, args, uuid)
+    output_file = os.path.join(args.o, '%s.mkv' % uuid)
+    ififuncs.generate_log(
+        log_name_source,
+        'EVENT = Concatenation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.mkv' % uuid))
+    ififuncs.generate_log(
+        log_name_source,
+        'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
+    validation_logfile = os.path.join(args.o, '%s_validation.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
+    validation_env_dict = ififuncs.set_environment(validation_logfile)
+    output_bitstream_md5 = subprocess.check_output([
+        'ffmpeg', '-report',
+        '-i', output_file,
+        '-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-'
+    ], env=validation_env_dict).rstrip()
+    ififuncs.generate_log(
+        log_name_source,
+        'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
+    if source_bitstream_md5 == output_bitstream_md5:
+        print 'process appears to be lossless'
+        print source_bitstream_md5, output_bitstream_md5
+        ififuncs.generate_log(
+        log_name_source,
+        'EVENT = losslessness verification, eventOutcome=pass')
+    else:
+        print 'something went wrong - not lossless!'
+        print source_bitstream_md5,output_bitstream_md5
+        ififuncs.generate_log(
+        log_name_source,
+        'EVENT = losslessness verification, eventOutcome=fail')
+    subprocess.call(['mkvpropedit', output_file, '-c', 'chapters.txt'])
+    ififuncs.generate_log(
+        log_name_source,
+        'EVENT = eventType=modification, agentName=mkvpropedit, eventDetail=Chapters added to file detailing start point of source clips.')
     ififuncs.concat_textfile(video_files, concat_file)
-    ffmpeg_concat(concat_file, args, uuid)
+    with open(log_name_source, 'r') as concat_log:
+        concat_lines = concat_log.readlines()
+    if not args.no_sip:
+        sipcreator_log, sipcreator_manifest = sipcreator.main(['-i', output_file, '-u', uuid, '-oe', object_entry, '-user', user, '-o', args.o])
+        shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log))
+        shutil.move(validation_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log))
+        logs_dir = os.path.dirname(sipcreator_log)
+        ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir, os.path.basename(fmd5_logfile)))
+        ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir,(os.path.basename(validation_logfile.replace('\\\\', '\\').replace('\:', ':')))))
+        with open(sipcreator_log, 'r') as sipcreator_log_object:
+            sipcreator_lines = sipcreator_log_object.readlines()
+        with open(sipcreator_log, 'wb') as fo:
+            for lines in concat_lines:
+                fo.write(lines)
+            for remaining_lines in sipcreator_lines:
+                fo.write(remaining_lines)
+        ififuncs.checksum_replace(sipcreator_manifest, sipcreator_log)
 
 if __name__ == '__main__':
     main(sys.argv[1:])