Merge pull request #39 from lukewys/main

Upgrade command-line to support saving synthesis parameters. Bump up version to 0.2.0.

Author: lukewys

midi_ddsp/midi_ddsp_synthesize.py

@@ -34,6 +34,7 @@
 from midi_ddsp.utils.audio_io import save_wav
 from midi_ddsp.utils.training_utils import get_hp
 from midi_ddsp.utils.inference_utils import ensure_same_length
+from midi_ddsp.utils.file_utils import pickle_dump
 from midi_ddsp.hparams_synthesis_generator import hparams as hp
 from midi_ddsp.modules.get_synthesis_generator import get_synthesis_generator, \
   get_fake_data_synthesis_generator
@@ -92,7 +93,8 @@ def synthesize_midi(synthesis_generator, expression_generator, midi_file,
                     use_fluidsynth=False,
                     sf2_path='/usr/share/sounds/sf2/FluidR3_GM.sf2',
                     display_progressbar=True,
-                    skip_existing_files=False):
+                    skip_existing_files=False,
+                    save_synth_params=False):
   """
   Synthesize a midi file using MIDI-DDSP.
   Args:
@@ -101,13 +103,15 @@ def synthesize_midi(synthesis_generator, expression_generator, midi_file,
       midi_file: The path to the MIDI file.
       pitch_offset: Pitch in semitone to transpose.
       speed_rate: The speed to synthesize the MIDI file.
-      output_dir: The directory for output audio.
+      output_dir: The directory for saving outputs.
+        If output_dir is None, outputs will not be written to disk.
       use_fluidsynth: Whether to use FluidSynth for synthesizing instruments
         that are not available in MIDI-DDSP.
       sf2_path: The path to a sf2 soundfont file used for FluidSynth.
       display_progressbar: Whether to display progress bar.
       skip_existing_files: Skip synthesizing MIDI files if already exist
         output folders.
+      save_synth_params: also save synthesis parameters.
 
   Returns: A dict of output:
           'mix_audio': mix audio,
@@ -134,6 +138,8 @@ def synthesize_midi(synthesis_generator, expression_generator, midi_file,
   conditioning_df_all = []
   part_synth_by_model = []
   midi_audio_all = {}
+  midi_synth_params_all = {}
+  midi_control_params_all = {}
 
   # For each part, predict expressions using MIDI-DDSP,
   # or synthesize using FluidSynth.
@@ -173,12 +179,27 @@ def synthesize_midi(synthesis_generator, expression_generator, midi_file,
         instrument_id_all,
         display_progressbar=display_progressbar)
 
+    midi_synth_params = midi_synth_params['inputs']
     for i in range(midi_audio.shape[0]):
       part_number = part_synth_by_model[i]
+
+      # align audio with part number
       midi_audio_all[part_number] = midi_audio[i].numpy()
-  else:
-    midi_control_params = None
-    midi_synth_params = None
+
+      # align synthesis parameters with part number
+      # get the midi synth parameters
+      midi_synth_params_all[part_number] = {
+        k: v[i].numpy() for k, v in midi_synth_params.items()}
+
+      # align control parameters with part number
+      # (yusongwu) sorry for mis-aligned variable names between
+      # synth_params and control_params due to historical issues
+      midi_control_params_all[part_number] = {
+        'amplitudes': midi_control_params[1][i].numpy(),
+        'harmonic_distribution': midi_control_params[2][i].numpy(),
+        'noise_magnitudes': midi_control_params[3][i].numpy(),
+        'f0_hz': midi_control_params[0][i].numpy(),
+      }
 
   # Sorting out and save the wav.
   if output_dir is not None:
@@ -205,17 +226,20 @@ def synthesize_midi(synthesis_generator, expression_generator, midi_file,
   if midi_audio_all:
     midi_audio_mix = np.sum(
       np.stack(ensure_same_length(
-        [a.astype(np.float) for a in midi_audio_all.values()], axis=0),
+        [a.astype(np.float64) for a in midi_audio_all.values()], axis=0),
         axis=-1),
       axis=-1)
     if output_dir is not None:
       save_wav(midi_audio_mix, os.path.join(output_dir, 'mix.wav'), 16000)
+      if save_synth_params:
+        pickle_dump(midi_synth_params_all,
+                    os.path.join(output_dir, 'synth_params.pickle'))
     output = {
       'mix_audio': midi_audio_mix,
       'stem_audio': midi_audio_all,
       'part_synth_by_model': part_synth_by_model,
-      'midi_control_params': midi_control_params,
-      'midi_synth_params': midi_synth_params,
+      'midi_control_params': midi_control_params_all,
+      'midi_synth_params': midi_synth_params_all,
       'conditioning_df': conditioning_df_all,
     }
   else:
@@ -259,6 +283,10 @@ def main():
                       action='store_true',
                       help='Skip synthesizing MIDI files if already exist '
                            'output folders.')
+  parser.add_argument('--save_synth_params', dest='save_synth_params',
+                      action='store_true',
+                      help='Save synthesis parameters generated by MIDI-DDSP.')
+
   args = parser.parse_args()
 
   synthesis_generator, expression_generator = load_pretrained_model(
@@ -304,7 +332,8 @@ def main():
       sf2_path=args.sf2_path,
       use_fluidsynth=args.use_fluidsynth,
       display_progressbar=True,
-      skip_existing_files=args.skip_existing_files
+      skip_existing_files=args.skip_existing_files,
+      save_synth_params=args.save_synth_params
     )
 
 

setup.py

@@ -21,7 +21,7 @@
 
 setuptools.setup(
   name='midi-ddsp',
-  version='0.1.5',
+  version='0.2.0',
   description='Synthesis of MIDI with DDSP',
   long_description=long_description,
   long_description_content_type='text/markdown',