Release/4.2.0 (#28)

* Release 4.2.0

* release 4.2.0

* fixing workflow

Author: drache42

.github/workflows/dotnet-build.yml

@@ -39,9 +39,6 @@ jobs:
     - name: Net Publish Tests ${{ matrix.config }}
       if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }}
       run: dotnet publish --no-build -p:DebugType=None -p:DebugSymbols=false -c ${{ matrix.config }} -f ${{ vars.PYTHON_NETPACKAGE_FRAMEWORK }} -o './dist/tests/' './tests/Tableau.Migration.Tests/Tableau.Migration.Tests.csproj'
-    - name: Net Publish TestComponents ${{ matrix.config }}
-      if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }}
-      run: dotnet publish --no-build -p:DebugType=None -p:DebugSymbols=false -c ${{ matrix.config }} -f ${{ vars.PYTHON_NETPACKAGE_FRAMEWORK }} -o './dist/testcomponents/' './tests/Tableau.Migration.TestComponents/Tableau.Migration.TestComponents.csproj'
     - name: Upload Published Artifacts 
       uses: actions/upload-artifact@v4
       if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }}
@@ -54,12 +51,6 @@ jobs:
       with:
         name: tests-published-${{ matrix.config }}
         path: './dist/tests/**'
-    - name: Upload TestComponents Artifacts 
-      uses: actions/upload-artifact@v4
-      if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }}
-      with:
-        name: testcomponents-published-${{ matrix.config }}
-        path: './dist/testcomponents/**'
     - name: Upload Nupkg Artifact 
       uses: actions/upload-artifact@v4
       if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }}

.github/workflows/python-test.yml

@@ -34,18 +34,9 @@ jobs:
         with:
           name: tests-published-${{ matrix.config }}
           path: ./src/Python/src/tableau_migration/bin/
-      - uses: actions/download-artifact@v4
-        with:
-          name: testcomponents-published-${{ matrix.config }}
-          path: ./tests/Python.TestApplication/bin/
       - name: Lint with ruff
         run: python -m hatch run lint:lint
 
       - name: Test with pytest
         run: |
           python -m hatch --data-dir=.hatch --cache-dir=.hatch_cache run test:testcov
-          
-      - name: Test TestApplication with pytest
-        run: |
-          python -m hatch --data-dir=.hatch --cache-dir=.hatch_cache run test:test
-        working-directory: ./tests/Python.TestApplication

.github/workflows/sdk-workflow.yml

@@ -7,6 +7,8 @@ on:
     branches:
       - main
       - 'release/**'
+      - 'staging/**'
+      - 'feature/**' 
     paths-ignore:
       - README.md
       - '**/README.md'

.gitignore

@@ -5,6 +5,7 @@
 *.csproj.user
 .idea/
 /.vscode
+*.pyproj.user
 
 # Python stuff 
 # Source: https://github.com/github/gitignore/blob/main/Python.gitignore

Directory.Build.props

@@ -4,7 +4,7 @@
 		<Nullable>enable</Nullable>
 		<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
 		<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
-		<Version>4.1.0</Version>
+		<Version>4.2.0</Version>
 		<Authors>Salesforce, Inc.</Authors>
 		<Company>Salesforce, Inc.</Company>
 		<Copyright>Copyright (c) 2024, Salesforce, Inc. and its licensors</Copyright>

Migration SDK.sln

@@ -37,16 +37,12 @@ Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "Python", "src\Python\Python
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tableau.Migration.TestApplication", "tests\Tableau.Migration.TestApplication\Tableau.Migration.TestApplication.csproj", "{9B20E6B0-E733-4280-9D63-0BAFA0A23276}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tableau.Migration.TestComponents", "tests\Tableau.Migration.TestComponents\Tableau.Migration.TestComponents.csproj", "{0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}"
-EndProject
 Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "Python.TestApplication", "tests\Python.TestApplication\Python.TestApplication.pyproj", "{BCA30617-0CAF-405C-A26B-522CF20DB027}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Examples", "Examples", "{00B78B56-BF89-4F26-AF75-435A4B88D43F}"
 EndProject
 Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "Python.ExampleApplication", "examples\Python.ExampleApplication\Python.ExampleApplication.pyproj", "{9C94FBC9-AE67-4A26-BDDA-EB2CE9FE5C25}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tableau.Migration.TestComponents.Tests", "tests\Tableau.Migration.TestComponents.Tests\Tableau.Migration.TestComponents.Tests.csproj", "{A00DC50C-4184-4658-A437-00AD2A07412C}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Csharp.ExampleApplication", "examples\Csharp.ExampleApplication\Csharp.ExampleApplication.csproj", "{B0D16449-BC58-4873-8B95-F62E805EE39C}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{92044843-B4D7-4062-B1D5-DE7596072E33}"
@@ -85,7 +81,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "setup-dotnet", "setup-dotne
 		.github\actions\setup-dotnet\action.yml = .github\actions\setup-dotnet\action.yml
 	EndProjectSection
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tableau.Migration.PythonGenerator", "src\Tableau.Migration.PythonGenerator\Tableau.Migration.PythonGenerator.csproj", "{F20029C7-4514-4668-8941-B2C3BC245CCB}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tableau.Migration.PythonGenerator", "src\Tableau.Migration.PythonGenerator\Tableau.Migration.PythonGenerator.csproj", "{F20029C7-4514-4668-8941-B2C3BC245CCB}"
+EndProject
+Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "Python.ExampleApplication.Tests", "tests\Python.ExampleApplication.Tests\Python.ExampleApplication.Tests.pyproj", "{B1884017-8E25-4A26-8C89-D9D880CFA392}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -109,18 +107,10 @@ Global
 		{9B20E6B0-E733-4280-9D63-0BAFA0A23276}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{9B20E6B0-E733-4280-9D63-0BAFA0A23276}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9B20E6B0-E733-4280-9D63-0BAFA0A23276}.Release|Any CPU.Build.0 = Release|Any CPU
-		{0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}.Release|Any CPU.Build.0 = Release|Any CPU
 		{BCA30617-0CAF-405C-A26B-522CF20DB027}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{BCA30617-0CAF-405C-A26B-522CF20DB027}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9C94FBC9-AE67-4A26-BDDA-EB2CE9FE5C25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{9C94FBC9-AE67-4A26-BDDA-EB2CE9FE5C25}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A00DC50C-4184-4658-A437-00AD2A07412C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A00DC50C-4184-4658-A437-00AD2A07412C}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A00DC50C-4184-4658-A437-00AD2A07412C}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A00DC50C-4184-4658-A437-00AD2A07412C}.Release|Any CPU.Build.0 = Release|Any CPU
 		{B0D16449-BC58-4873-8B95-F62E805EE39C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{B0D16449-BC58-4873-8B95-F62E805EE39C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B0D16449-BC58-4873-8B95-F62E805EE39C}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -133,6 +123,8 @@ Global
 		{F20029C7-4514-4668-8941-B2C3BC245CCB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F20029C7-4514-4668-8941-B2C3BC245CCB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F20029C7-4514-4668-8941-B2C3BC245CCB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B1884017-8E25-4A26-8C89-D9D880CFA392}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B1884017-8E25-4A26-8C89-D9D880CFA392}.Release|Any CPU.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -141,17 +133,16 @@ Global
 		{E71B1084-8907-4070-AD26-2189C80A2C34} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11}
 		{1D53209C-9F3D-4F64-ABDF-FEA30F0C3D5D} = {C5BD8316-60C9-4C96-9B15-820E8BA4DF7F}
 		{9B20E6B0-E733-4280-9D63-0BAFA0A23276} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11}
-		{0454E8DE-D973-4E8F-95C7-9F6F02EF26FD} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11}
 		{BCA30617-0CAF-405C-A26B-522CF20DB027} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11}
 		{9C94FBC9-AE67-4A26-BDDA-EB2CE9FE5C25} = {00B78B56-BF89-4F26-AF75-435A4B88D43F}
-		{A00DC50C-4184-4658-A437-00AD2A07412C} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11}
 		{B0D16449-BC58-4873-8B95-F62E805EE39C} = {00B78B56-BF89-4F26-AF75-435A4B88D43F}
 		{92044843-B4D7-4062-B1D5-DE7596072E33} = {C5BD8316-60C9-4C96-9B15-820E8BA4DF7F}
 		{99DA12FB-BB16-4EE1-9C9C-047755210255} = {00B78B56-BF89-4F26-AF75-435A4B88D43F}
 		{90102C4B-EC3B-4279-A6C6-A6CFDFCD4DB4} = {C5BD8316-60C9-4C96-9B15-820E8BA4DF7F}
 		{6B735E6E-1FFB-4C37-8CF6-BD979B4F8D9B} = {90102C4B-EC3B-4279-A6C6-A6CFDFCD4DB4}
 		{EDD52CC0-7289-4167-A01E-E88B015FC67F} = {90102C4B-EC3B-4279-A6C6-A6CFDFCD4DB4}
 		{454EF272-D967-4668-A20D-AD6B3EE96C1A} = {6B735E6E-1FFB-4C37-8CF6-BD979B4F8D9B}
+		{B1884017-8E25-4A26-8C89-D9D880CFA392} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {2C9E9FF4-E825-47A4-90BE-5499D5EDF3CC}

examples/Csharp.ExampleApplication/MyMigrationApplication.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
+using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
 using Csharp.ExampleApplication.Config;
@@ -16,6 +18,7 @@
 using Tableau.Migration;
 using Tableau.Migration.Content;
 using Tableau.Migration.Content.Schedules.Cloud;
+using Tableau.Migration.Engine.Manifest;
 using Tableau.Migration.Engine.Pipelines;
 
 #region namespace
@@ -30,13 +33,15 @@ internal sealed class MyMigrationApplication : IHostedService
         private readonly IMigrator _migrator;
         private readonly MyMigrationApplicationOptions _options;
         private readonly ILogger<MyMigrationApplication> _logger;
+        private readonly MigrationManifestSerializer _manifestSerializer;
 
         public MyMigrationApplication(
             IHostApplicationLifetime appLifetime,
             IMigrationPlanBuilder planBuilder,
             IMigrator migrator,
             IOptions<MyMigrationApplicationOptions> options,
-            ILogger<MyMigrationApplication> logger)
+            ILogger<MyMigrationApplication> logger,
+            MigrationManifestSerializer manifestSerializer)
         {
             _timer = new Stopwatch();
 
@@ -49,10 +54,19 @@ public MyMigrationApplication(
             _migrator = migrator;
             _options = options.Value;
             _logger = logger;
+            _manifestSerializer = manifestSerializer;
         }
 
         public async Task StartAsync(CancellationToken cancel)
         {
+            var executablePath = Assembly.GetExecutingAssembly().Location;
+            var currentFolder = Path.GetDirectoryName(executablePath);
+            if (currentFolder is null)
+            {
+                throw new Exception("Could not get the current folder path.");
+            }
+            var manifestPath = $"{currentFolder}/manifest.json";
+
             var startTime = DateTime.UtcNow;
             _timer.Start();
 
@@ -130,7 +144,7 @@ public async Task StartAsync(CancellationToken cancel)
             _planBuilder.Hooks.Add<LogMigrationActionsHook>();
             #endregion
 
-            //Add batch migration completed hooks
+            // Add batch migration completed hooks
             #region LogMigrationBatchesHook-Registration
             _planBuilder.Hooks.Add<LogMigrationBatchesHook<IUser>>();
             _planBuilder.Hooks.Add<LogMigrationBatchesHook<IProject>>();
@@ -139,14 +153,20 @@ public async Task StartAsync(CancellationToken cancel)
             _planBuilder.Hooks.Add<LogMigrationBatchesHook<ICloudExtractRefreshTask>>();
             #endregion
 
+            // Load the previous manifest if possible
+            var prevManifest = await LoadManifest(manifestPath, cancel);
+
             // Build the plan
             var plan = _planBuilder.Build();
 
             // Execute the migration
-            var result = await _migrator.ExecuteAsync(plan, cancel);
+            var result = await _migrator.ExecuteAsync(plan, prevManifest, cancel);
 
             _timer.Stop();
 
+            // Save the manifest
+            await _manifestSerializer.SaveAsync(result.Manifest, manifestPath);
+
             PrintResult(result);
 
             _logger.LogInformation($"Migration Started: {startTime}");
@@ -206,6 +226,31 @@ private void PrintResult(MigrationResult result)
                 }
             }
         }
+
+        private async Task<MigrationManifest?> LoadManifest(string manifestFilepath, CancellationToken cancel)
+        {
+            var manifest = await _manifestSerializer.LoadAsync(manifestFilepath, cancel);
+            if (manifest is not null)
+            {
+                ConsoleKey key;
+                do
+                {
+                    Console.Write($"Existing Manifest found at {manifestFilepath}. Should it be used? [Y/n] ");
+                    key = Console.ReadKey().Key;
+                    Console.WriteLine(); // make Console logs prettier
+                } while (key is not ConsoleKey.Enter && key is not ConsoleKey.Y && key is not ConsoleKey.N);
+
+                if (key is ConsoleKey.N)
+                {
+                    return null;
+                }
+
+                _logger.LogInformation($"Using previous manifest from {manifestFilepath}");
+                return manifest;
+            }
+
+            return null;
+        }
     }
 }
 

examples/Python.ExampleApplication/Python.ExampleApplication.py

@@ -2,6 +2,10 @@
 # By default all supported content will be migrated, but can be modified to your specific needs.
 # The application assumes you have already installed the Tableau Migration SDK Python package.
 
+from dotenv import load_dotenv
+
+load_dotenv()
+
 import configparser          # configuration parser
 import os                    # environment variables
 import sys                   # system utility
@@ -10,9 +14,20 @@
 
 from threading import Thread # threading
 
+from tableau_migration import (
+    MigrationManifestSerializer,
+    MigrationManifest
+    )
+
+serializer = MigrationManifestSerializer()
+
 def migrate():
     """Performs a migration using Tableau Migration SDK."""
 
+    # Get the absolute path of the current file
+    current_file_path = os.path.abspath(__file__)
+    manifest_path = os.path.join(os.path.dirname(current_file_path), 'manifest.json')
+
     plan_builder = tableau_migration.MigrationPlanBuilder()
     migration = tableau_migration.Migrator()
 
@@ -37,6 +52,10 @@ def migrate():
 
     # TODO: add filters, mappings, transformers, etc. here.
 
+
+    # Load the previous manifest file if it exists.
+    prev_manifest = load_manifest(f'{manifest_path}')
+
     # Validate the migration plan.
     validation_result = plan_builder.validate()
 
@@ -45,13 +64,31 @@ def migrate():
     plan = plan_builder.build()
 
     # Run the migration.
-    results = migration.execute(plan)
+    results = migration.execute(plan, prev_manifest)
 
+    # Save the manifest file.
+    serializer.save(results.manifest, f'{manifest_path}')
+
     # TODO: Handle results here.
     print_result(results)
 
     print("All done.")
 
+def load_manifest(manifest_path: str) -> MigrationManifest | None:
+        """Loads a manifest if requested."""
+        manifest = serializer.load(manifest_path)
+    
+        if manifest is not None:
+            while True:
+                answer = input(f'Existing Manifest found at {manifest_path}. Should it be used? [Y/n] ').upper()
+
+                if answer == 'N':
+                    return None
+                elif answer == 'Y' or answer == '':
+                    return manifest
+                
+        return None
+    
 
 if __name__ == '__main__':
 

examples/Python.ExampleApplication/requirements.txt

@@ -5,3 +5,4 @@ cffi==1.16.0
 pycparser==2.22
 pythonnet==3.0.3
 typing_extensions==4.12.2
+python-dotenv==1.0.1