Merge pull request #23 from tableau/release/4.0.0

Release 4.0.0

Author: drache42

.github/workflows/python-test.yml

@@ -44,10 +44,6 @@ jobs:
       - name: Test with pytest
         run: |
           python -m hatch --data-dir=.hatch --cache-dir=.hatch_cache run test:testcov
-          python -m hatch --data-dir=.hatch --cache-dir=.hatch_cache run test:pytest tests/test_migrations_engine_hooks_lifetime.py
-          python -m hatch --data-dir=.hatch --cache-dir=.hatch_cache run test:pytest tests/test_migrations_engine_hooks_filters_lifetime.py
-          python -m hatch --data-dir=.hatch --cache-dir=.hatch_cache run test:pytest tests/test_migrations_engine_hooks_mappings_lifetime.py
-          python -m hatch --data-dir=.hatch --cache-dir=.hatch_cache run test:pytest tests/test_migrations_engine_hooks_transformers_lifetime.py
           
       - name: Test TestApplication with pytest
         run: |

.gitignore

@@ -176,8 +176,8 @@ appsettings.Development.json
 clean-server-settings.dev.json
 launchSettings.json
 UpgradeLog.htm
-*.DEV.ini
-*.DEV.json
+*.*.ini
+*.*.json
 /src/Python/Documentation/_build
 /src/Python/Documentation/_static
 /src/Python/Python.pyproj.user

Directory.Build.props

@@ -4,9 +4,10 @@
 		<Nullable>enable</Nullable>
 		<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
 		<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
-		<Version>3.0.1</Version>
+		<Version>4.0.0</Version>
 		<Authors>Salesforce, Inc.</Authors>
 		<Company>Salesforce, Inc.</Company>
 		<Copyright>Copyright (c) 2024, Salesforce, Inc. and its licensors</Copyright>
+	    <PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
 	</PropertyGroup>
 </Project>

LICENSE.txt

@@ -1,6 +1,6 @@
 Apache License Version 2.0
 
-Copyright (c) 2023 Salesforce, Inc.
+Copyright (c) 2024 Salesforce, Inc.
 All rights reserved.
 
 Apache License

Migration SDK.sln

@@ -18,6 +18,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		CONTRIBUTING.md = CONTRIBUTING.md
 		Directory.Build.props = Directory.Build.props
 		global.json = global.json
+		LICENSE.txt = LICENSE.txt
 		README.md = README.md
 		SECURITY.md = SECURITY.md
 	EndProjectSection
@@ -82,6 +83,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "setup-dotnet", "setup-dotne
 		.github\actions\setup-dotnet\action.yml = .github\actions\setup-dotnet\action.yml
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tableau.Migration.PythonGenerator", "src\Tableau.Migration.PythonGenerator\Tableau.Migration.PythonGenerator.csproj", "{F20029C7-4514-4668-8941-B2C3BC245CCB}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -124,6 +127,10 @@ Global
 		{99DA12FB-BB16-4EE1-9C9C-047755210255}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{99DA12FB-BB16-4EE1-9C9C-047755210255}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{99DA12FB-BB16-4EE1-9C9C-047755210255}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F20029C7-4514-4668-8941-B2C3BC245CCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F20029C7-4514-4668-8941-B2C3BC245CCB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F20029C7-4514-4668-8941-B2C3BC245CCB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F20029C7-4514-4668-8941-B2C3BC245CCB}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

examples/Csharp.ExampleApplication/Hooks/Mappings/ChangeProjectMapping.cs

@@ -5,31 +5,32 @@
 using Tableau.Migration;
 using Tableau.Migration.Content;
 using Tableau.Migration.Engine.Hooks.Mappings;
+using Tableau.Migration.Resources;
 
 namespace Csharp.ExampleApplication.Hooks.Mappings
 {
     #region class
-    public class ChangeProjectMapping : IContentMapping<IDataSource>, IContentMapping<IWorkbook>
+    public class ChangeProjectMapping<T> : ContentMappingBase<T>
+        where T : IContentReference, IMappableContainerContent
     {
         private static readonly StringComparer StringComparer = StringComparer.OrdinalIgnoreCase;
 
-        private readonly ILogger<ChangeProjectMapping> _logger;
+        private readonly ILogger<IContentMapping<T>>? _logger;
 
-        public ChangeProjectMapping(ILogger<ChangeProjectMapping> logger)
+        public ChangeProjectMapping(ISharedResourcesLocalizer? localizer, ILogger<IContentMapping<T>>? logger) : base(localizer, logger)
         {
             _logger = logger;
         }
 
-        private async Task<ContentMappingContext<T>?> ExecuteAsync<T>(ContentMappingContext<T> ctx)
-            where T : IContentReference, IMappableContainerContent
+        public override Task<ContentMappingContext<T>?> MapAsync(ContentMappingContext<T> ctx, CancellationToken cancel)
         {
             // Get the container (project) location for the content item.
             var containerLocation = ctx.ContentItem.Location.Parent();
 
             // We only want to map content items whose project name is "Test".
             if (!StringComparer.Equals("Test", containerLocation.Name))
             {
-                return ctx;
+                return ctx.ToTask();
             }
 
             // Build the new project location.
@@ -41,20 +42,20 @@ public ChangeProjectMapping(ILogger<ChangeProjectMapping> logger)
             // Map the new content item location.
             ctx = ctx.MapTo(newLocation);
 
-            _logger.LogInformation(
+            _logger?.LogInformation(
                 "{ContentType} mapped from {OldLocation} to {NewLocation}.",
                 typeof(T).Name,
                 ctx.ContentItem.Location,
                 ctx.MappedLocation);
 
-            return await ctx.ToTask();
+            return ctx.ToTask();
         }
 
-        public async Task<ContentMappingContext<IDataSource>?> ExecuteAsync(ContentMappingContext<IDataSource> ctx, CancellationToken cancel)
-            => await ExecuteAsync(ctx);
+        public async Task<ContentMappingContext<IDataSource>?> MapAsync(ContentMappingContext<IDataSource> ctx, CancellationToken cancel)
+            => await MapAsync(ctx, cancel);
 
-        public async Task<ContentMappingContext<IWorkbook>?> ExecuteAsync(ContentMappingContext<IWorkbook> ctx, CancellationToken cancel)
-            => await ExecuteAsync(ctx);
+        public async Task<ContentMappingContext<IWorkbook>?> MapAsync(ContentMappingContext<IWorkbook> ctx, CancellationToken cancel)
+            => await MapAsync(ctx, cancel);
     }
     #endregion
 }

examples/Csharp.ExampleApplication/Hooks/Mappings/EmailDomainMapping.cs

@@ -42,11 +42,12 @@ public EmailDomainMapping(
         public override Task<ContentMappingContext<IUser>?> MapAsync(ContentMappingContext<IUser> userMappingContext, CancellationToken cancel)
         {
             var domain = userMappingContext.MappedLocation.Parent();
+
             // Re-use an existing email if it already exists.
             if (!string.IsNullOrEmpty(userMappingContext.ContentItem.Email))
                 return userMappingContext.MapTo(domain.Append(userMappingContext.ContentItem.Email)).ToTask();
 
-            // Takes the existing username and appends the default domain to build the email
+            // Takes the existing username and appends the domain to build the email
             var testEmail = $"{userMappingContext.ContentItem.Name}@{_domain}";
             return userMappingContext.MapTo(domain.Append(testEmail)).ToTask();
         }

examples/Csharp.ExampleApplication/Hooks/Transformers/EncryptExtractTransformer.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Tableau.Migration;
+using Tableau.Migration.Content;
+using Tableau.Migration.Engine.Hooks.Transformers;
+using Tableau.Migration.Resources;
+
+namespace Csharp.ExampleApplication.Hooks.Transformers
+{
+    #region class
+    public class EncryptExtractsTransformer<T> : ContentTransformerBase<T> where T : IContentReference, IFileContent, IExtractContent
+    {
+        private readonly ILogger<IContentTransformer<T>>? _logger;
+
+        public EncryptExtractsTransformer(ISharedResourcesLocalizer? localizer, ILogger<IContentTransformer<T>>? logger) : base(localizer, logger)
+        {
+            _logger = logger;
+        }
+
+        public override async Task<T?> TransformAsync(T itemToTransform, CancellationToken cancel)
+        {
+            itemToTransform.EncryptExtracts = true;
+
+            _logger?.LogInformation(
+                @"Setting encrypt extract to true for {ContentType} {ContentLocation}",
+                typeof(T).Name,
+                itemToTransform.Location);
+
+            return await Task.FromResult(itemToTransform);
+        }
+
+        public async Task<IPublishableWorkbook?> TransformAsync(IPublishableWorkbook ctx, CancellationToken cancel)
+            => await TransformAsync(ctx, cancel);
+
+        public async Task<IPublishableDataSource?> TransformAsync(IPublishableDataSource ctx, CancellationToken cancel)
+            => await TransformAsync(ctx, cancel);
+    }
+    #endregion
+}

examples/Csharp.ExampleApplication/Hooks/Transformers/MigratedTagTransformer.cs

@@ -6,41 +6,41 @@
 using Tableau.Migration;
 using Tableau.Migration.Content;
 using Tableau.Migration.Engine.Hooks.Transformers;
+using Tableau.Migration.Resources;
 
 namespace Csharp.ExampleApplication.Hooks.Transformers
 {
     #region class
-    public class MigratedTagTransformer : IContentTransformer<IPublishableWorkbook>, IContentTransformer<IPublishableDataSource>
+    public class MigratedTagTransformer<T> : ContentTransformerBase<T> where T : IContentReference, IWithTags
     {
-        private readonly ILogger<MigratedTagTransformer> _logger;
+        private readonly ILogger<IContentTransformer<T>>? _logger;
 
-        public MigratedTagTransformer(ILogger<MigratedTagTransformer> logger)
+        public MigratedTagTransformer(ISharedResourcesLocalizer? localizer, ILogger<IContentTransformer<T>>? logger) : base(localizer, logger)
         {
             _logger = logger;
         }
 
-        protected async Task<T?> ExecuteAsync<T>(T ctx)
-            where T : IContentReference, IWithTags
+        public override async Task<T?> TransformAsync(T itemToTransform, CancellationToken cancel)
         {
             var tag = "Migrated";
 
             // Add the tag to the content item.
-            ctx.Tags.Add(new Tag(tag));
+            itemToTransform.Tags.Add(new Tag(tag));
 
-            _logger.LogInformation(
+            _logger?.LogInformation(
                 @"Added ""{Tag}"" tag to {ContentType} {ContentLocation}.",
                 tag,
                 typeof(T).Name,
-                ctx.Location);
+                itemToTransform.Location);
 
-            return await Task.FromResult(ctx);
+            return await Task.FromResult(itemToTransform);
         }
 
-        public async Task<IPublishableWorkbook?> ExecuteAsync(IPublishableWorkbook ctx, CancellationToken cancel)
-            => await ExecuteAsync(ctx);
+        public async Task<IPublishableWorkbook?> TransformAsync(IPublishableWorkbook ctx, CancellationToken cancel)
+            => await TransformAsync(ctx, cancel);
 
-        public async Task<IPublishableDataSource?> ExecuteAsync(IPublishableDataSource ctx, CancellationToken cancel)
-            => await ExecuteAsync(ctx);
+        public async Task<IPublishableDataSource?> TransformAsync(IPublishableDataSource ctx, CancellationToken cancel)
+            => await TransformAsync(ctx, cancel);
     }
     #endregion
 }

examples/Csharp.ExampleApplication/MyMigrationApplication.cs

@@ -86,8 +86,8 @@ public async Task StartAsync(CancellationToken cancel)
             #endregion
 
             #region ChangeProjectMapping-Registration
-            _planBuilder.Mappings.Add<ChangeProjectMapping, IDataSource>();
-            _planBuilder.Mappings.Add<ChangeProjectMapping, IWorkbook>();
+            _planBuilder.Mappings.Add<ChangeProjectMapping<IDataSource>, IDataSource>();
+            _planBuilder.Mappings.Add<ChangeProjectMapping<IWorkbook>, IWorkbook>();
             #endregion
 
             // Add filters
@@ -111,8 +111,13 @@ public async Task StartAsync(CancellationToken cancel)
 
             // Add transformers
             #region MigratedTagTransformer-Registration
-            _planBuilder.Transformers.Add<MigratedTagTransformer, IPublishableDataSource>();
-            _planBuilder.Transformers.Add<MigratedTagTransformer, IPublishableWorkbook>();
+            _planBuilder.Transformers.Add<MigratedTagTransformer<IPublishableDataSource>, IPublishableDataSource>();
+            _planBuilder.Transformers.Add<MigratedTagTransformer<IPublishableWorkbook>, IPublishableWorkbook>();
+            #endregion
+
+            #region EncryptExtractTransformer-Registration
+            _planBuilder.Transformers.Add<EncryptExtractsTransformer<IPublishableDataSource>, IPublishableDataSource>();
+            _planBuilder.Transformers.Add<EncryptExtractsTransformer<IPublishableWorkbook>, IPublishableWorkbook>();
             #endregion
 
             // Add migration action completed hooks

examples/Csharp.ExampleApplication/Program.cs

@@ -9,6 +9,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Tableau.Migration;
+using Tableau.Migration.Content;
 
 #region namespace
 namespace Csharp.ExampleApplication
@@ -56,7 +57,8 @@ public static IServiceCollection AddCustomizations(this IServiceCollection servi
             #endregion
 
             #region ChangeProjectMapping-DI
-            services.AddScoped<ChangeProjectMapping>();
+            services.AddScoped<ChangeProjectMapping<IWorkbook>>();
+            services.AddScoped<ChangeProjectMapping<IDataSource>>();
             #endregion
 
             #region DefaultProjectsFilter-DI
@@ -76,7 +78,13 @@ public static IServiceCollection AddCustomizations(this IServiceCollection servi
             #endregion
 
             #region MigratedTagTransformer-DI
-            services.AddScoped<MigratedTagTransformer>();
+            services.AddScoped<MigratedTagTransformer<IPublishableDataSource>>();
+            services.AddScoped<MigratedTagTransformer<IPublishableWorkbook>>();
+            #endregion
+
+            #region EncryptExtractTransformer-DI
+            services.AddScoped<EncryptExtractsTransformer<IPublishableDataSource>>();
+            services.AddScoped<EncryptExtractsTransformer<IPublishableWorkbook>>();
             #endregion
 
             #region LogMigrationActionsHook-DI

examples/Python.ExampleApplication/Hooks/Filters/default_project_filter.py

@@ -0,0 +1,11 @@
+from tableau_migration import (
+    IProject,
+    ContentMigrationItem,
+    ContentFilterBase)
+    
+
+class DefaultProjectFilter(ContentFilterBase[IProject]):
+    def should_migrate(self, item: ContentMigrationItem[IProject]) -> bool:
+        if item.source_item.name.casefold() == 'Default'.casefold():
+            return False
+        return True

examples/Python.ExampleApplication/Hooks/Filters/unlicensed_user_filter.py

@@ -0,0 +1,12 @@
+from tableau_migration import (
+    IUser,
+    ContentMigrationItem,
+    ContentFilterBase,
+    SiteRoles)
+    
+
+class UnlicensedUserFilter(ContentFilterBase[IUser]):
+    def should_migrate(self, item: ContentMigrationItem[IUser]) -> bool:
+        if item.source_item.license_level.casefold() == SiteRoles.UNLICENSED.casefold():
+            return False
+        return True

examples/Python.ExampleApplication/Hooks/batch_migration_completed/log_migration_batches_hook.py

@@ -0,0 +1,34 @@
+import logging
+from typing import TypeVar
+from tableau_migration import(
+    ContentBatchMigrationCompletedHookBase,
+    IContentBatchMigrationResult,
+    IUser
+    )
+
+T = TypeVar("T")
+
+class LogMigrationBatchesHook(ContentBatchMigrationCompletedHookBase[T]):
+    def __init__(self) -> None:
+        super().__init__()
+        self._logger = logging.getLogger(__name__)
+        
+    def execute(self, ctx: IContentBatchMigrationResult[T]) -> IContentBatchMigrationResult[T]:
+        
+        item_status = ""
+        for item in ctx.item_results:
+            item_status += "%s: %s".format(item.manifest_entry.source.location, item.manifest_entry.status)
+                
+        self._logger.info("%s batch of %d item(s) completed:\n%s", ctx._content_type, ctx.item_results.count, item_status)
+        
+        pass
+    
+class LogMigrationBatchesHookForUsers(ContentBatchMigrationCompletedHookBase[IUser]):
+    def __init__(self) -> None:
+        super().__init__()
+        self._content_type = "User";
+
+class LogMigrationBatchesHookForGoups(ContentBatchMigrationCompletedHookBase[IUser]):
+    def __init__(self) -> None:
+        super().__init__()
+        self._content_type = "Group";