Skip to content

Commit 37e300b

Browse files
authored
Added the CLI command for migrating DLT pipelines (#3579)
## Changes Added cli command for migration DLT pipelines with the flags for limiting the pipelines ids ### Linked issues Introduces #3107 ### Functionality - [x] added new CLI command ### Tests - [x] manually tested - [ ] added unit tests - [ ] added integration tests - [ ] verified on staging environment (screenshot attached)
1 parent 9cea6a6 commit 37e300b

File tree

3 files changed

+32
-3
lines changed

3 files changed

+32
-3
lines changed

labs.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,3 +368,11 @@ commands:
368368

369369
- name: assign-owner-group
370370
description: Assign owner group to the workspace. This group will be assigned as an owner to all migrated tables and views.
371+
372+
- name: migrate-dlt-pipelines
373+
description: Migrate DLT pipelines from HMS to UC using the DLT Migration API
374+
flags:
375+
- name: include-pipeline-ids
376+
description: (Optional) Comma separated list of pipeline IDs to include in the migration
377+
- name: exclude-pipeline-ids
378+
description: (Optional) Comma separated list of pipeline IDs to exclude from the migration

src/databricks/labs/ucx/cli.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -911,12 +911,18 @@ def migrate_dlt_pipelines(
911911
ctx: WorkspaceContext | None = None,
912912
run_as_collection: bool = False,
913913
a: AccountClient | None = None,
914+
**named_parameters,
914915
) -> None:
915-
"""Migrate DLT pipelines to UC"""
916+
"""
917+
Migrate DLT pipelines to UC that were crawled in the assessment workflow
918+
Flags
919+
-- include-pipeline-ids: Comma separated list of pipeline ids to migrate
920+
-- exclude-pipeline-ids: Comma separated list of pipeline ids to exclude from migration
921+
"""
916922
if ctx:
917923
workspace_contexts = [ctx]
918924
else:
919-
workspace_contexts = _get_workspace_contexts(w, a, run_as_collection)
925+
workspace_contexts = _get_workspace_contexts(w, a, run_as_collection, **named_parameters)
920926

921927
for workspace_context in workspace_contexts:
922928
workspace_context.pipelines_migrator.migrate_pipelines()

src/databricks/labs/ucx/contexts/application.py

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,8 +374,23 @@ def table_ownership_grant_loader(self) -> TableOwnershipGrantLoader:
374374

375375
@cached_property
376376
def pipelines_migrator(self) -> PipelinesMigrator:
377+
include_pipeline_ids = (
378+
self.named_parameters.get('include_pipeline_ids', '').split(',')
379+
if 'include_pipeline_ids' in self.named_parameters
380+
else None
381+
)
382+
exclude_pipeline_ids = (
383+
self.named_parameters.get('exclude_pipeline_ids', '').split(',')
384+
if 'exclude_pipeline_ids' in self.named_parameters
385+
else None
386+
)
377387
return PipelinesMigrator(
378-
self.workspace_client, self.pipelines_crawler, self.jobs_crawler, self.config.ucx_catalog
388+
self.workspace_client,
389+
self.pipelines_crawler,
390+
self.jobs_crawler,
391+
self.config.ucx_catalog,
392+
include_pipeline_ids=include_pipeline_ids,
393+
exclude_pipeline_ids=exclude_pipeline_ids,
379394
)
380395

381396
@cached_property

0 commit comments

Comments
 (0)