From df3f74366323f635ce1a913d454d0cbcd32a4558 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 29 Jun 2025 12:04:21 -0500 Subject: [PATCH 01/13] Restructured "Is Migration Assistant Right for you?" page. Signed-off-by: Brian Presley --- .../getting-started-data-migration.md | 17 +------- _migration-assistant/overview/index.md | 6 +-- .../is-migration-assistant-right-for-you.md | 43 ++++++++++--------- 3 files changed, 27 insertions(+), 39 deletions(-) diff --git a/_migration-assistant/deploying-migration-assistant/getting-started-data-migration.md b/_migration-assistant/deploying-migration-assistant/getting-started-data-migration.md index 856d2e53aa5..5e0ac5dcf59 100644 --- a/_migration-assistant/deploying-migration-assistant/getting-started-data-migration.md +++ b/_migration-assistant/deploying-migration-assistant/getting-started-data-migration.md @@ -12,22 +12,7 @@ redirect_from: This quickstart outlines how to deploy Migration Assistant for OpenSearch and execute an existing data migration using `Reindex-from-Snapshot` (RFS). It uses AWS for illustrative purposes. However, the steps can be modified for use with other cloud providers. -## Prerequisites and assumptions - -Before using this quickstart, make sure you fulfill the following prerequisites: - -* Verify that your migration path [is supported]({{site.url}}{{site.baseurl}}/migration-assistant/overview/is-migration-assistant-right-for-you/#supported-migration-paths). Note that we test with the exact versions specified, but you should be able to migrate data on alternative minor versions as long as the major version is supported. -* The source cluster must be deployed Amazon Simple Storage Service (Amazon S3) plugin. -* The target cluster must be deployed. -* Verify that the `CDKToolkit` stack exists and is set to `CREATE_COMPLETE`. For more information about how to bootstrap your AWS account in the required AWS Region, see [the CDKToolkit documentation](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html). - -The steps in this guide assume the following: - -* In this guide, a snapshot will be taken and stored in Amazon S3; the following assumptions are made about this snapshot: - * The `_source` flag is enabled on all indexes to be migrated. - * The snapshot includes the global cluster state (`include_global_state` is `true`). - * Shard sizes of up to approximately 80 GB are supported. Larger shards cannot be migrated. If this presents challenges for your migration, contact the [migration team](https://opensearch.slack.com/archives/C054JQ6UJFK). -* Migration Assistant will be installed in the same AWS Region and have access to both the source snapshot and target cluster. +Before using this quickstart, make sure you review ["Is Migration Assistant right for you?"]({{site.url}}{{site.baseurl}}/migration-assistant/overview/is-migration-assistant-right-for-you/#supported-migration-paths). --- diff --git a/_migration-assistant/overview/index.md b/_migration-assistant/overview/index.md index 7962fe5e85d..11c7d716ef5 100644 --- a/_migration-assistant/overview/index.md +++ b/_migration-assistant/overview/index.md @@ -6,15 +6,15 @@ has_children: true has_toc: false permalink: /migration-assistant/overview/ items: + - heading: "Is Migration Assistant right for you?" + description: "Evaluate whether Migration Assistant is right for your use case." + link: "/migration-assistant/overview/is-migration-assistant-right-for-you/" - heading: "Key components" description: "Get familiar with the key components of Migration Assistant." link: "/migration-assistant/overview/key-components/" - heading: "Architecture" description: "Understand how Migration Assistant integrates into your infrastructure." link: "/migration-assistant/overview/architecture/" - - heading: "Is Migration Assistant right for you?" - description: "Evaluate whether Migration Assistant is right for your use case." - link: "/migration-assistant/overview/is-migration-assistant-right-for-you/" --- # Migration Assistant overview diff --git a/_migration-assistant/overview/is-migration-assistant-right-for-you.md b/_migration-assistant/overview/is-migration-assistant-right-for-you.md index 2d9a4a20508..308e461de3c 100644 --- a/_migration-assistant/overview/is-migration-assistant-right-for-you.md +++ b/_migration-assistant/overview/is-migration-assistant-right-for-you.md @@ -9,13 +9,26 @@ redirect_from: # Is Migration Assistant right for you? -Deciding whether to use Migration Assistant depends on your specific upgrade path, infrastructure complexity, and operational goals. This page will help you evaluate whether Migration Assistant is right for your use case—or whether another tool might be a better fit. +Deciding whether to use Migration Assistant depends on your specific upgrade path, infrastructure complexity, and operational goals. This page will help you evaluate whether Migration Assistant is right for your case. -Migration Assistant was built to fill important gaps in common migration strategies. For example, if you're upgrading across multiple major versions—such as from Elasticsearch 6.8 to OpenSearch 2.19—Migration Assistant lets you do this in a single step. Other methods, like rolling upgrades or snapshot restores, require you to upgrade through each major version, often reindexing your data at every step. +Migration Assistant was built to address limitations in common migration strategies. For example, if you're upgrading across multiple major versions—such as from Elasticsearch 6.8 to OpenSearch 2.19—Migration Assistant lets you do this in a single step. Other methods, like rolling upgrades or snapshot restores, require you to upgrade through each major version, often reindexing your data at every step. -Migration Assistant also supports live traffic replication, allowing for zero-downtime migrations. This makes it a strong choice for production environments, where minimizing service disruption is critical. +Migration Assistant also supports live traffic replication, allowing for zero-downtime migrations. This makes it a strong choice for environments where minimizing service disruption is critical. -If your migration is limited to a static cluster configuration (like index templates and aliases), or if you're not concerned about downtime, simpler tools may be sufficient. But for complex migrations involving real-time traffic or major version jumps, Migration Assistant offers robust, flexible capabilities. +## Migration Assistant Assumptions and Limitations + +Before using Migration Assistant, review the following assumptions and limitations. If a limitation only, applies to a specific component (i.e., Capture-and-Replay or Reindex-from-Snapshot) it is specified. + +* If deploying to AWS, the Idendity used to deploy Migration Assistant must have permissions to install all resources used by Migration Assistant. For a full list of resource refer to the AWS documentation, [AWS Services in this Solution](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/architecture-details.html#aws-services-in-this-solution). +* *Reindex-from-Snapshot only.* The source cluster must be deployed Amazon Simple Storage Service (Amazon S3) plugin for Reindex-from-Snapshot. +* The target cluster must be deployed and reachable by Migration Assistant by Reindex-from-Snapshot resources for backfill and the Traffic Replayer for live capture. +* *Capture-and-Replay.* The Traffic Capture Proxy must be deployed to intercept client traffic relevant for migration. The proxy must. +* *Reindex-from-Snapshot.* The snapshot includes the global cluster state (`include_global_state` is `true`). +* By default shards up to 80GB are supported but larger shards are permitted if configured. There is, however, a hard limitation of 80GB shards if the source is in an AWS GovCloud region. +* *Capture-and-Replay.* Live capture should be liitated to 4TB/day of network traffic. +* *Capture-and-Replay.* Currently, for index requests, automatically generated document IDs will not be preserved during the migration. Clients must specify document IDs when writing or updating indecing if using Capture-and-Replay. +* The `_source` flag is enabled on all indexes to be migrated. Refer to [Source documentation]({{site.url}}{{site.baseurl}}/field-types/metadata-fields/source/) for details. +* If deploying to AWS, verify `CDKToolkit` stack exists and is set to `CREATE_COMPLETE`. For more information about how to bootstrap your AWS account in the required AWS Region, see [the CDKToolkit documentation](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html). ## Supported migration paths @@ -34,7 +47,7 @@ The following matrix shows which source versions can be directly migrated to whi - + {% for target in unique_targets %} {% endfor %} @@ -59,28 +72,18 @@ The following matrix shows which source versions can be directly migrated to whi **Source and target platforms** - Self-managed (on premises or hosted by a cloud provider) -- Amazon OpenSearch Service - - -**AWS Regions** - -Migration Assistant is supported in the following AWS Regions: - -- US East (N. Virginia, Ohio) -- US West (Oregon, N. California) -- Europe (Frankfurt, Ireland, London) -- Asia Pacific (Tokyo, Singapore, Sydney) -- AWS GovCloud (US-East, US-West)[^1] +- Amazon OpenSearch Service. Amazon OpenSearch Serverless Collections are not supported. -[^1]: In AWS GovCloud (US), `reindex-from-snapshot` (RFS) is limited to shard sizes of 80 GiB or smaller. +**Supported AWS Regions** +Refer to [AWS Supported Regions](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/plan-your-deployment.html#supported-aws-regions) for the official list of supported regions. -## Supported components +## Supported features Before starting a migration, consider the scope of the components involved. The following table outlines components that should potentially be migrated, indicates whether they are supported by Migration Assistant, and provides recommendations. -| Component | Supported | Recommendations | +| Feature | Supported | Recommendations | | :--- |:--- | :--- | | **Documents** | Yes | Migrate existing data with RFS and live traffic with capture and replay. | | **Index settings** | Yes | Migrate with the `Metadata-Migration-Tool`. | From c62887a1e3bb9d11a0f687de68c1f39ab26ab460 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 29 Jun 2025 12:38:06 -0500 Subject: [PATCH 02/13] Broke down limitation by component. Minor editorial changes. Signed-off-by: Brian Presley --- .../is-migration-assistant-right-for-you.md | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/_migration-assistant/overview/is-migration-assistant-right-for-you.md b/_migration-assistant/overview/is-migration-assistant-right-for-you.md index 308e461de3c..6e530a9dfc9 100644 --- a/_migration-assistant/overview/is-migration-assistant-right-for-you.md +++ b/_migration-assistant/overview/is-migration-assistant-right-for-you.md @@ -9,30 +9,41 @@ redirect_from: # Is Migration Assistant right for you? -Deciding whether to use Migration Assistant depends on your specific upgrade path, infrastructure complexity, and operational goals. This page will help you evaluate whether Migration Assistant is right for your case. +Whether Migration Assistant is right for you depends on your upgrade path, infrastructure complexity, and operational goals. This page will help you evaluate whether Migration Assistant fits your use case. -Migration Assistant was built to address limitations in common migration strategies. For example, if you're upgrading across multiple major versions—such as from Elasticsearch 6.8 to OpenSearch 2.19—Migration Assistant lets you do this in a single step. Other methods, like rolling upgrades or snapshot restores, require you to upgrade through each major version, often reindexing your data at every step. +Migration Assistant addresses key limitations in traditional migration approaches. For example, if you're upgrading across multiple major versions—such as from Elasticsearch 6.8 to OpenSearch 2.19—Migration Assistant enables you to complete the process in a single step. Other methods, like rolling upgrades or snapshot restores, require upgrading through each major version, often with reindexing at every stage. -Migration Assistant also supports live traffic replication, allowing for zero-downtime migrations. This makes it a strong choice for environments where minimizing service disruption is critical. +Migration Assistant also supports live traffic replication, enabling zero-downtime migrations. This makes it a strong fit for environments where minimizing service disruption is critical. -## Migration Assistant Assumptions and Limitations +## Migration Assistant assumptions and limitations -Before using Migration Assistant, review the following assumptions and limitations. If a limitation only, applies to a specific component (i.e., Capture-and-Replay or Reindex-from-Snapshot) it is specified. +Before using Migration Assistant, review the following assumptions and limitations. They are grouped by scope to clarify which apply generally and which are specific to components. -* If deploying to AWS, the Idendity used to deploy Migration Assistant must have permissions to install all resources used by Migration Assistant. For a full list of resource refer to the AWS documentation, [AWS Services in this Solution](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/architecture-details.html#aws-services-in-this-solution). -* *Reindex-from-Snapshot only.* The source cluster must be deployed Amazon Simple Storage Service (Amazon S3) plugin for Reindex-from-Snapshot. -* The target cluster must be deployed and reachable by Migration Assistant by Reindex-from-Snapshot resources for backfill and the Traffic Replayer for live capture. -* *Capture-and-Replay.* The Traffic Capture Proxy must be deployed to intercept client traffic relevant for migration. The proxy must. -* *Reindex-from-Snapshot.* The snapshot includes the global cluster state (`include_global_state` is `true`). -* By default shards up to 80GB are supported but larger shards are permitted if configured. There is, however, a hard limitation of 80GB shards if the source is in an AWS GovCloud region. -* *Capture-and-Replay.* Live capture should be liitated to 4TB/day of network traffic. -* *Capture-and-Replay.* Currently, for index requests, automatically generated document IDs will not be preserved during the migration. Clients must specify document IDs when writing or updating indecing if using Capture-and-Replay. -* The `_source` flag is enabled on all indexes to be migrated. Refer to [Source documentation]({{site.url}}{{site.baseurl}}/field-types/metadata-fields/source/) for details. -* If deploying to AWS, verify `CDKToolkit` stack exists and is set to `CREATE_COMPLETE`. For more information about how to bootstrap your AWS account in the required AWS Region, see [the CDKToolkit documentation](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html). +### General + +- If deploying on AWS, the identity used to deploy Migration Assistant must have permission to install all required resources. For a full list of services, see [AWS Services in this Solution](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/architecture-details.html#aws-services-in-this-solution). +- The target cluster must be deployed and reachable by Migration Assistant components, including the Migration Console, Reindex-from-Snapshot, and Traffic Replayer, depending on which features are used. +- The `_source` field must be enabled on all indexes to be migrated. See [Source documentation]({{site.url}}{{site.baseurl}}/field-types/metadata-fields/source/) for more information. +- If deploying to AWS, ensure the `CDKToolkit` stack exists and is in the `CREATE_COMPLETE` state. For setup instructions, see the [CDK Toolkit documentation](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html). + +### Reindex-from-Snapshot + +- The source cluster must have the Amazon S3 plugin installed. +- Snapshots must include global cluster state (`include_global_state` is `true`). +- Shards up to 80 GiB are supported by default. Larger shards can be configured, except in AWS GovCloud, where 80 GiB is the maximum supported size. + +### Capture-and-Replay + +- The Traffic Capture Proxy must be deployed to intercept relevant client traffic. +- Live capture is recommended only for environments with less than 4 TB/day of incoming traffic to the source cluster. +- Automatically generated document IDs are not preserved for index requests. Clients must explicitly provide document IDs when indexing or updating documents. + +--- ## Supported migration paths -The following matrix shows which source versions can be directly migrated to which OpenSearch target versions: +The matrix below shows which source versions can be directly migrated to which OpenSearch target versions: + {% comment %}First, collect all unique target versions{% endcomment %} @@ -72,16 +83,15 @@ The following matrix shows which source versions can be directly migrated to whi **Source and target platforms** - Self-managed (on premises or hosted by a cloud provider) -- Amazon OpenSearch Service. Amazon OpenSearch Serverless Collections are not supported. - +- Amazon OpenSearch Service (OpenSearch Serverless Collections are not supported) **Supported AWS Regions** -Refer to [AWS Supported Regions](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/plan-your-deployment.html#supported-aws-regions) for the official list of supported regions. +Refer to [AWS Supported Regions](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/plan-your-deployment.html#supported-aws-regions) for the for the full list of supported regions. ## Supported features -Before starting a migration, consider the scope of the components involved. The following table outlines components that should potentially be migrated, indicates whether they are supported by Migration Assistant, and provides recommendations. +Before starting an upgrade or migration, consider the cluster feature to be included. The table below lists what can be migrated using Migration Assistant, whether it is currently supported, and recommendations for how to handle each component. | Feature | Supported | Recommendations | | :--- |:--- | :--- | From b58adc45081075915648f9f8f18f4434a7f30e14 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 29 Jun 2025 12:58:32 -0500 Subject: [PATCH 03/13] Removed "Chossing migration approach" since it will be revised and included in a seperate migration patterns page. Updated checklist. Signed-off-by: Brian Presley --- .../is-migration-assistant-right-for-you.md | 58 +++---------------- 1 file changed, 9 insertions(+), 49 deletions(-) diff --git a/_migration-assistant/overview/is-migration-assistant-right-for-you.md b/_migration-assistant/overview/is-migration-assistant-right-for-you.md index 6e530a9dfc9..11d2a3cf10e 100644 --- a/_migration-assistant/overview/is-migration-assistant-right-for-you.md +++ b/_migration-assistant/overview/is-migration-assistant-right-for-you.md @@ -108,59 +108,19 @@ Before starting an upgrade or migration, consider the cluster feature to be incl [^2]: Support for Kibana 5.0 through 7.10.2 migration paths to OpenSearch Dashboards will be added in a future version. Kibana 8 and later are not supported. For more information, see [issue #944](https://github.com/opensearch-project/opensearch-migrations/issues/944). -## Choosing your migration approach - -Use the following checklist to determine which Migration Assistant components best fit your use case. - -### Metadata migration - -Use [metadata migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/) if: - -- You need to migrate while mitigating breaking changes between the source and target clusters, such as differences in mappings, settings, aliases, or component templates. -- You want a relatively consistent configuration between the source and target clusters. - -### Backfill migration - -Use [backfill migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/) if: - -- You need to move historical data without disrupting live traffic. -- You want to backfill indexes from a specific point in time without impacting the source cluster. -- You want to verify historical data in the target cluster before switching over. -- You want to backfill using an existing or incremental snapshot. -- You need the fastest backfill option that includes reindexing. -- You want the ability to pause and resume migration. - -### RFS - -Use [RFS]({{site.url}}{{site.baseurl}}/migration-assistant/deploying-migration-assistant/getting-started-data-migration/) if: - -- You already use OpenSearch snapshots for backups. -- You need to migrate documents at scale in parallel, such as with Amazon Elastic Container Service (Amazon ECS). -- You require a data migration path as part of a zero-downtime migration. -- Your AWS Region supports RFS and your shard sizes are within supported limits. - -### Combination of all three - -Use a combination of all three migration types if: - -- You're performing a complex, multi-version migration. -- You require zero downtime and full validation of the target environment. -- You want end-to-end tooling for metadata, data movement, and cluster behavior comparison. -- You're cloning an existing cluster and changing the source's configuration. -- You're setting up disaster recovery. ## Checklist -Use this checklist to decide whether Migration Assistant is right for you: - -- Are you migrating across one or more major versions? - -- Do you need to maintain service availability with zero downtime? - -- Do you need to validate a new OpenSearch cluster before switching over? +Use this checklist to determine whether Migration Assistant is the right fit for your migration: +- Are you migrating across one or more major versions—for example, from Elasticsearch 5 to OpenSearch 3—in a single step? +- Are you upgrading but want the ability to safely back out, reducing the risk of data loss or service disruption? +- Do you need to maintain high service availability with minimal or zero downtime? +- Do you need to validate a new OpenSearch cluster before switching over, with rollback capabilities? - Is your environment self-managed or running on Amazon OpenSearch Service? - -- Are you looking for tooling that can automate metadata migration and performance comparison? +- Are you looking for tooling to migrate index settings and other metadata? +- Do you need to reconfigure your target cluster—for example, by changing the sharding strategy and reindexing? +- Are you migrating across regions, from on-premises, or from another cloud provider? +- Do you need a high-performance backfill solution that can reliably reindex documents—with support for pause, resume, or checkpoint recovery? If you answered "yes" to most of these questions, Migration Assistant is likely the right solution for your migration. From 84a25c1a6b883cf8367ec8c05f6a6223e4baa448 Mon Sep 17 00:00:00 2001 From: Archer Date: Tue, 1 Jul 2025 06:58:03 -0500 Subject: [PATCH 04/13] Rename Migration Assistant section Signed-off-by: Archer --- _config.yml | 8 +- _migration-assistant/index.md | 44 ------ .../assets/css/breaking-changes-selector.css | 0 .../assets/js/breaking-changes-data.js | 0 .../assets/js/breaking-changes-filter.js | 0 .../assets/js/breaking-changes-index.js | 0 .../assets/js/breaking-changes-module.js | 0 .../assets/js/breaking-changes-ui.js | 0 .../configuration-options.md | 1 + .../getting-started-data-migration.md | 1 + ...d-security-groups-for-existing-clusters.md | 1 + .../deploying-migration-assistant/index.md | 0 _migration-upgrade/index.md | 132 ++++++++++++++++++ .../accessing-the-migration-console.md | 1 + .../migration-console/index.md | 0 .../migration-console-commands-references.md | 1 + .../migration-phases/backfill.md | 1 + .../migration-phases/index.md | 0 .../live-traffic-migration/index.md | 1 + ...itching-traffic-from-the-source-cluster.md | 1 + .../using-traffic-replayer.md | 1 + .../migration-phases/migrating-metadata.md | 1 + .../assessing-your-cluster-for-migration.md | 1 + .../handling-field-type-breaking-changes.md | 1 + .../handling-type-mapping-deprecation.md | 1 + .../planning-your-migration/index.md | 1 + .../verifying-migration-tools.md | 1 + .../removing-migration-infrastructure.md | 1 + .../overview/architecture.md | 1 + .../overview/index.md | 0 .../is-migration-assistant-right-for-you.md | 1 + .../overview/key-components.md | 1 + 32 files changed, 155 insertions(+), 48 deletions(-) delete mode 100644 _migration-assistant/index.md rename {_migration-assistant => _migration-upgrade}/assets/css/breaking-changes-selector.css (100%) rename {_migration-assistant => _migration-upgrade}/assets/js/breaking-changes-data.js (100%) rename {_migration-assistant => _migration-upgrade}/assets/js/breaking-changes-filter.js (100%) rename {_migration-assistant => _migration-upgrade}/assets/js/breaking-changes-index.js (100%) rename {_migration-assistant => _migration-upgrade}/assets/js/breaking-changes-module.js (100%) rename {_migration-assistant => _migration-upgrade}/assets/js/breaking-changes-ui.js (100%) rename {_migration-assistant => _migration-upgrade}/deploying-migration-assistant/configuration-options.md (99%) rename {_migration-assistant => _migration-upgrade}/deploying-migration-assistant/getting-started-data-migration.md (99%) rename {_migration-assistant => _migration-upgrade}/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md (97%) rename {_migration-assistant => _migration-upgrade}/deploying-migration-assistant/index.md (100%) create mode 100644 _migration-upgrade/index.md rename {_migration-assistant => _migration-upgrade}/migration-console/accessing-the-migration-console.md (95%) rename {_migration-assistant => _migration-upgrade}/migration-console/index.md (100%) rename {_migration-assistant => _migration-upgrade}/migration-console/migration-console-commands-references.md (97%) rename {_migration-assistant => _migration-upgrade}/migration-phases/backfill.md (99%) rename {_migration-assistant => _migration-upgrade}/migration-phases/index.md (100%) rename {_migration-assistant => _migration-upgrade}/migration-phases/live-traffic-migration/index.md (96%) rename {_migration-assistant => _migration-upgrade}/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md (96%) rename {_migration-assistant => _migration-upgrade}/migration-phases/live-traffic-migration/using-traffic-replayer.md (99%) rename {_migration-assistant => _migration-upgrade}/migration-phases/migrating-metadata.md (99%) rename {_migration-assistant => _migration-upgrade}/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md (97%) rename {_migration-assistant => _migration-upgrade}/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md (97%) rename {_migration-assistant => _migration-upgrade}/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md (99%) rename {_migration-assistant => _migration-upgrade}/migration-phases/planning-your-migration/index.md (93%) rename {_migration-assistant => _migration-upgrade}/migration-phases/planning-your-migration/verifying-migration-tools.md (99%) rename {_migration-assistant => _migration-upgrade}/migration-phases/removing-migration-infrastructure.md (95%) rename {_migration-assistant => _migration-upgrade}/overview/architecture.md (96%) rename {_migration-assistant => _migration-upgrade}/overview/index.md (100%) rename {_migration-assistant => _migration-upgrade}/overview/is-migration-assistant-right-for-you.md (99%) rename {_migration-assistant => _migration-upgrade}/overview/key-components.md (97%) diff --git a/_config.yml b/_config.yml index 5c2ef7212c6..baac4a1c7c6 100644 --- a/_config.yml +++ b/_config.yml @@ -89,7 +89,7 @@ collections: data-prepper: permalink: /:collection/:path/ output: true - migration-assistant: + migration-upgrade: permalink: /:collection/:path/ output: true tools: @@ -215,10 +215,10 @@ clients_collection: name: Clients nav_fold: true -migration_assistant_collection: +migration_upgrade_collection: collections: - migration-assistant: - name: Migration Assistant + migration-upgrade: + name: Migration and upgrade nav_fold: true benchmark_collection: diff --git a/_migration-assistant/index.md b/_migration-assistant/index.md deleted file mode 100644 index 2ce5f0d8d74..00000000000 --- a/_migration-assistant/index.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -layout: default -title: Migration Assistant for OpenSearch -nav_order: 1 -has_children: false -nav_exclude: true -has_toc: false -permalink: /migration-assistant/ -redirect_from: - - /migration-assistant/index/ - - /upgrade-to/index/ - - /upgrade-to/ - - /upgrade-to/upgrade-to/ -tutorial_cards: - - heading: "Overview" - description: "Get familiar with the key components of Migration Assistant and evaluate your use case." - link: "/migration-assistant/overview/" - - heading: "Deploying Migration Assistant" - description: "Follow step-by-step instructions to deploy Migration Assistant and prepare data for migration." - link: "/deploying-migration-assistant/" - - heading: "Migration phases" - description: "Execute your migration in phases—metadata, backfill, and traffic replay—for a controlled and validated transition." - link: "/migration-phases/" - - heading: "Migration console" - description: "Use CLI commands provided by the migration console to orchestrate and monitor your migration process." - link: "/migration-console/" ---- - -# Migration Assistant for OpenSearch - -Migration Assistant for OpenSearch aids you in successfully performing an end-to-end, zero-downtime migration to OpenSearch from other search providers. It helps with the following scenarios: - -- **Metadata migration**: Migrating cluster metadata, such as index settings, aliases, and templates. -- **Backfill migration**: Migrating existing or historical data from a source to a target cluster. -- **Live traffic migration**: Replicating live ongoing traffic from a source to a target cluster. -- **Comparative tooling**: Comparing the performance and behaviors of an existing cluster with a prospective new one. - -This user guide focuses on conducting a comprehensive migration involving both existing and live data with zero downtime and the option to back out of a migration. - -It's crucial to note that migration strategies are not universally applicable. This guide provides a detailed methodology, based on certain assumptions detailed throughout, emphasizing the importance of robust engineering practices to ensure a successful migration. -{: .tip } - -{% include cards.html cards=page.tutorial_cards %} - diff --git a/_migration-assistant/assets/css/breaking-changes-selector.css b/_migration-upgrade/assets/css/breaking-changes-selector.css similarity index 100% rename from _migration-assistant/assets/css/breaking-changes-selector.css rename to _migration-upgrade/assets/css/breaking-changes-selector.css diff --git a/_migration-assistant/assets/js/breaking-changes-data.js b/_migration-upgrade/assets/js/breaking-changes-data.js similarity index 100% rename from _migration-assistant/assets/js/breaking-changes-data.js rename to _migration-upgrade/assets/js/breaking-changes-data.js diff --git a/_migration-assistant/assets/js/breaking-changes-filter.js b/_migration-upgrade/assets/js/breaking-changes-filter.js similarity index 100% rename from _migration-assistant/assets/js/breaking-changes-filter.js rename to _migration-upgrade/assets/js/breaking-changes-filter.js diff --git a/_migration-assistant/assets/js/breaking-changes-index.js b/_migration-upgrade/assets/js/breaking-changes-index.js similarity index 100% rename from _migration-assistant/assets/js/breaking-changes-index.js rename to _migration-upgrade/assets/js/breaking-changes-index.js diff --git a/_migration-assistant/assets/js/breaking-changes-module.js b/_migration-upgrade/assets/js/breaking-changes-module.js similarity index 100% rename from _migration-assistant/assets/js/breaking-changes-module.js rename to _migration-upgrade/assets/js/breaking-changes-module.js diff --git a/_migration-assistant/assets/js/breaking-changes-ui.js b/_migration-upgrade/assets/js/breaking-changes-ui.js similarity index 100% rename from _migration-assistant/assets/js/breaking-changes-ui.js rename to _migration-upgrade/assets/js/breaking-changes-ui.js diff --git a/_migration-assistant/deploying-migration-assistant/configuration-options.md b/_migration-upgrade/deploying-migration-assistant/configuration-options.md similarity index 99% rename from _migration-assistant/deploying-migration-assistant/configuration-options.md rename to _migration-upgrade/deploying-migration-assistant/configuration-options.md index 5fa28e44c43..5032dcb757e 100644 --- a/_migration-assistant/deploying-migration-assistant/configuration-options.md +++ b/_migration-upgrade/deploying-migration-assistant/configuration-options.md @@ -3,6 +3,7 @@ layout: default title: Configuration options nav_order: 15 parent: Deploying Migration Assistant +permalink: /migration-assistant/deploying-migration-assistant/configuration-options/ --- # Configuration options diff --git a/_migration-assistant/deploying-migration-assistant/getting-started-data-migration.md b/_migration-upgrade/deploying-migration-assistant/getting-started-data-migration.md similarity index 99% rename from _migration-assistant/deploying-migration-assistant/getting-started-data-migration.md rename to _migration-upgrade/deploying-migration-assistant/getting-started-data-migration.md index 856d2e53aa5..6fb28102840 100644 --- a/_migration-assistant/deploying-migration-assistant/getting-started-data-migration.md +++ b/_migration-upgrade/deploying-migration-assistant/getting-started-data-migration.md @@ -3,6 +3,7 @@ layout: default title: Getting started with data migration parent: Deploying Migration Assistant nav_order: 10 +permalink: /migration-assistant/deploying-migration-assistant/getting-started-with-data-migration/ redirect_from: - /upgrade-to/snapshot-migrate/ - /migration-assistant/getting-started-with-data-migration/ diff --git a/_migration-assistant/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md b/_migration-upgrade/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md similarity index 97% rename from _migration-assistant/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md rename to _migration-upgrade/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md index 291c2cba6a5..5ed62887838 100644 --- a/_migration-assistant/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md +++ b/_migration-upgrade/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md @@ -3,6 +3,7 @@ layout: default title: IAM and security groups for existing clusters nav_order: 20 parent: Deploying Migration Assistant +permalink: /migration-assistant/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters/ --- # IAM and security groups for existing clusters diff --git a/_migration-assistant/deploying-migration-assistant/index.md b/_migration-upgrade/deploying-migration-assistant/index.md similarity index 100% rename from _migration-assistant/deploying-migration-assistant/index.md rename to _migration-upgrade/deploying-migration-assistant/index.md diff --git a/_migration-upgrade/index.md b/_migration-upgrade/index.md new file mode 100644 index 00000000000..c44a1b5855d --- /dev/null +++ b/_migration-upgrade/index.md @@ -0,0 +1,132 @@ +--- +layout: default +title: Migration Assistant for OpenSearch +nav_order: 1 +has_children: false +nav_exclude: true +has_toc: false +permalink: /migration-assistant/ +redirect_from: + - /migration-assistant/index/ + - /upgrade-to/index/ + - /upgrade-to/ + - /upgrade-to/upgrade-to/ +tutorial_cards: + - heading: "Overview" + description: "Get familiar with the key components of Migration Assistant and evaluate your use case." + link: "/migration-assistant/overview/" + - heading: "Deploying Migration Assistant" + description: "Follow step-by-step instructions to deploy Migration Assistant and prepare data for migration." + link: "/deploying-migration-assistant/" + - heading: "Migration phases" + description: "Execute your migration in phases—metadata, backfill, and traffic replay—for a controlled and validated transition." + link: "/migration-phases/" + - heading: "Migration console" + description: "Use CLI commands provided by the migration console to orchestrate and monitor your migration process." + link: "/migration-console/" +--- + +# Migration and upgrade options + +Upgrading or migrating OpenSearch is essential for maintaining optimal performance, security, and access to the latest features. Whether you're upgrading an existing OpenSearch deployment or migrating from another system such as Elasticsearch OSS, choosing the right approach is critical to a successful transition. + +This page outlines four primary methods for upgrading or migrating OpenSearch clusters—each with distinct benefits and trade-offs. These methods include rolling upgrades, snapshot and restore, Migration Assistant, and remote reindexing. Use this guide to evaluate which option best fits your data size, infrastructure, and operational requirements. + +## Migration Assistant + +The Migration Assistant is a comprehensive tool designed to simplify upgrades by automating the process and supporting live data capture. + +**Pros:** +- Supports multi-version upgrades in a single migration. +- Enables near-zero downtime using live data capture. +- Includes rollback support to revert changes if issues occur. +- Integrates with existing OpenSearch tooling for a streamlined experience. + +**Cons:** +- Requires additional setup and infrastructure. + +### Why choose Migration Assistant? + +Migration Assistant offers the most automated and resilient path for OpenSearch upgrades, especially for complex environments and large version gaps. + +- **Multi-version hops:** Avoids sequential upgrades by migrating across multiple versions in one step. +- **Live data capture:** Maintains service continuity by syncing changes during the migration. +- **Reversion support:** Provides a fallback option in case of errors or issues. +- **Integrated automation:** Designed to fit into OpenSearch workflows for a guided upgrade experience. + +### Getting started with Migration Assistant + +To get started with Migration Assistant, determine which of the following scenarios best suits your needs: + +- **Metadata migration**: Migrating cluster metadata, such as index settings, aliases, and templates. +- **Backfill migration**: Migrating existing or historical data from a source to a target cluster. +- **Live traffic migration**: Replicating live ongoing traffic from a source to a target cluster. +- **Comparative tooling**: Comparing the performance and behaviors of an existing cluster with a prospective new one. + +It's crucial to note that migration strategies are not universally applicable. This guide provides a detailed methodology, based on certain assumptions detailed throughout, emphasizing the importance of robust engineering practices to ensure a successful migration. +{: .tip } + +{% include cards.html cards=page.tutorial_cards %} + +## Rolling upgrades + +Rolling upgrades allow you to upgrade one node at a time, keeping the cluster operational throughout the process. + +**Pros:** +- Minimal downtime; the cluster remains available during the upgrade. +- No new infrastructure is required. + +**Cons:** +- Only supports adjacent major version upgrades. +- Incompatibilities may still arise, requiring a snapshot restore that can be complex and risky. +- Multiple upgrade cycles are needed for large version jumps. +- Manual reindexing may be required to enable newer features. + + +## Snapshot and restore + +This method involves taking a snapshot of your existing OpenSearch or Elasticsearch OSS cluster and restoring it to a new cluster running the target version. + +**Pros:** +- Original cluster remains untouched, allowing rollback if needed (note: may result in data loss without a change data capture (CDC) solution). +- Scales well for large data volumes, including cold storage and datasets up to 1 PB. + +**Cons:** +- Requires downtime or an external CDC solution. +- Requires provisioning a new cluster. +- Manual reindexing may be necessary for full feature support. + +## Remote reindexing + +This method reindexes data from your current cluster into a new OpenSearch cluster, typically running in parallel. + +**Pros:** +- No downtime; clusters operate concurrently. +- Supports upgrades across multiple versions. + +**Cons:** +- Slow and resource-intensive, not recommended for data sets over 1 GB. +- May impact performance of the source cluster. +- Requires careful configuration and a compatible target cluster. + +## Why choose Migration Assistant? + +Migration Assistant offers the most automated and resilient path for OpenSearch upgrades, especially for complex environments and large version gaps. + +- **Multi-version hops:** Avoids sequential upgrades by migrating across multiple versions in one step. +- **Live data capture:** Maintains service continuity by syncing changes during the migration. +- **Reversion support:** Provides a fallback option in case of errors or issues. +- **Integrated automation:** Designed to fit into OpenSearch workflows for a guided upgrade experience. + +## Before you begin + +Before choosing a method, make sure that your OpenSearch clients and plugins are compatible with the target version. For example, tools like Logstash OSS and Filebeat OSS may enforce version checks that impact upgrade paths. + + +## Next steps + +After you've determined which upgrade or migration path works best for you, take one of the following next steps: + +- Review the [Migration Assistant Overview]({{site.url}}{{site.baseurl}}/migration-assistant/overview/) +- Perform a [rolling upgrade]({{site.url}}{{site.baseurl}}/install-and-configure/upgrade-opensearch/rolling-upgrade/) + diff --git a/_migration-assistant/migration-console/accessing-the-migration-console.md b/_migration-upgrade/migration-console/accessing-the-migration-console.md similarity index 95% rename from _migration-assistant/migration-console/accessing-the-migration-console.md rename to _migration-upgrade/migration-console/accessing-the-migration-console.md index ea66f5c04c9..cf530e3ccd4 100644 --- a/_migration-assistant/migration-console/accessing-the-migration-console.md +++ b/_migration-upgrade/migration-console/accessing-the-migration-console.md @@ -3,6 +3,7 @@ layout: default title: Accessing the migration console nav_order: 35 parent: Migration console +permalink: /migration-assistant/migration-console/accessing-the-migration-console/ --- # Accessing the migration console diff --git a/_migration-assistant/migration-console/index.md b/_migration-upgrade/migration-console/index.md similarity index 100% rename from _migration-assistant/migration-console/index.md rename to _migration-upgrade/migration-console/index.md diff --git a/_migration-assistant/migration-console/migration-console-commands-references.md b/_migration-upgrade/migration-console/migration-console-commands-references.md similarity index 97% rename from _migration-assistant/migration-console/migration-console-commands-references.md rename to _migration-upgrade/migration-console/migration-console-commands-references.md index 21d793b3f3f..a07439e32b8 100644 --- a/_migration-assistant/migration-console/migration-console-commands-references.md +++ b/_migration-upgrade/migration-console/migration-console-commands-references.md @@ -3,6 +3,7 @@ layout: default title: Command reference nav_order: 40 parent: Migration console +permalink: /migration-assistant/migration-console/migration-console-commands-reference/ --- # Migration console command reference diff --git a/_migration-assistant/migration-phases/backfill.md b/_migration-upgrade/migration-phases/backfill.md similarity index 99% rename from _migration-assistant/migration-phases/backfill.md rename to _migration-upgrade/migration-phases/backfill.md index e4b621c16ec..8a98412675b 100644 --- a/_migration-assistant/migration-phases/backfill.md +++ b/_migration-upgrade/migration-phases/backfill.md @@ -3,6 +3,7 @@ layout: default title: Backfill nav_order: 90 parent: Migration phases +permalink: /migration-assistant/migration-phases/backfill/ --- # Backfill diff --git a/_migration-assistant/migration-phases/index.md b/_migration-upgrade/migration-phases/index.md similarity index 100% rename from _migration-assistant/migration-phases/index.md rename to _migration-upgrade/migration-phases/index.md diff --git a/_migration-assistant/migration-phases/live-traffic-migration/index.md b/_migration-upgrade/migration-phases/live-traffic-migration/index.md similarity index 96% rename from _migration-assistant/migration-phases/live-traffic-migration/index.md rename to _migration-upgrade/migration-phases/live-traffic-migration/index.md index f9e3d9f71f0..7cdf7c466d1 100644 --- a/_migration-assistant/migration-phases/live-traffic-migration/index.md +++ b/_migration-upgrade/migration-phases/live-traffic-migration/index.md @@ -3,6 +3,7 @@ layout: default title: Live traffic migration nav_order: 99 parent: Migration phases +permalink: /live-traffic-migration/ has_toc: false has_children: true --- diff --git a/_migration-assistant/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md b/_migration-upgrade/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md similarity index 96% rename from _migration-assistant/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md rename to _migration-upgrade/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md index e9e477654ab..8ec7f1d929b 100644 --- a/_migration-assistant/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md +++ b/_migration-upgrade/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md @@ -4,6 +4,7 @@ title: Switching traffic from the source cluster nav_order: 110 grand_parent: Migration phases parent: Live traffic migration +permalink: /migration-assistant/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster/ redirect_from: - /migration-assistant/migration-phases/switching-traffic-from-the-source-cluster/ --- diff --git a/_migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer.md b/_migration-upgrade/migration-phases/live-traffic-migration/using-traffic-replayer.md similarity index 99% rename from _migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer.md rename to _migration-upgrade/migration-phases/live-traffic-migration/using-traffic-replayer.md index a1b54d49417..6ebedf0d91a 100644 --- a/_migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer.md +++ b/_migration-upgrade/migration-phases/live-traffic-migration/using-traffic-replayer.md @@ -4,6 +4,7 @@ title: Using Traffic Replayer nav_order: 100 grand_parent: Migration phases parent: Live traffic migration +permalink: /migration-assistant/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster/ redirect_from: - /migration-assistant/migration-phases/using-traffic-replayer/ --- diff --git a/_migration-assistant/migration-phases/migrating-metadata.md b/_migration-upgrade/migration-phases/migrating-metadata.md similarity index 99% rename from _migration-assistant/migration-phases/migrating-metadata.md rename to _migration-upgrade/migration-phases/migrating-metadata.md index 249a2ca4d0c..71e3da9776d 100644 --- a/_migration-assistant/migration-phases/migrating-metadata.md +++ b/_migration-upgrade/migration-phases/migrating-metadata.md @@ -3,6 +3,7 @@ layout: default title: Migrating metadata nav_order: 85 parent: Migration phases +permalink: /migration-assistant/migration-phases/migrating-metadata/ --- # Migrating metadata diff --git a/_migration-assistant/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md b/_migration-upgrade/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md similarity index 97% rename from _migration-assistant/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md rename to _migration-upgrade/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md index 1687da15893..a86f9f5db5e 100644 --- a/_migration-assistant/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md +++ b/_migration-upgrade/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md @@ -4,6 +4,7 @@ title: Assessing your cluster for migration nav_order: 60 parent: Planning your migration grand_parent: Migration phases +permalink: /migration-assistant/migration-phases/planning-your-migration/assessing-your-cluster-for-migration/ redirect_from: - /migration-assistant/migration-phases/assessing-your-cluster-for-migration/ --- diff --git a/_migration-assistant/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md b/_migration-upgrade/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md similarity index 97% rename from _migration-assistant/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md rename to _migration-upgrade/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md index b6a653f96ae..e09da59fd00 100644 --- a/_migration-assistant/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md +++ b/_migration-upgrade/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md @@ -3,6 +3,7 @@ layout: default title: Handling breaking changes in field types nav_order: 60 parent: Planning your migration +permalink: /migration-assistant/migration-phases/planning-your-migration/handling-breaking-changes-in-field-types/ grand_parent: Migration phases --- diff --git a/_migration-assistant/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md b/_migration-upgrade/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md similarity index 99% rename from _migration-assistant/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md rename to _migration-upgrade/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md index edd81612dac..a39ac1fdda9 100644 --- a/_migration-assistant/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md +++ b/_migration-upgrade/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md @@ -4,6 +4,7 @@ title: Managing type mapping deprecation nav_order: 60 parent: Planning your migration grand_parent: Migration phases +permalink: /migration-assistant/migration-phases/planning-your-migration/handling-type-mapping-deprecation/ --- # Managing type mapping deprecation diff --git a/_migration-assistant/migration-phases/planning-your-migration/index.md b/_migration-upgrade/migration-phases/planning-your-migration/index.md similarity index 93% rename from _migration-assistant/migration-phases/planning-your-migration/index.md rename to _migration-upgrade/migration-phases/planning-your-migration/index.md index 4f0d264e93e..9c6e7fee0bc 100644 --- a/_migration-assistant/migration-phases/planning-your-migration/index.md +++ b/_migration-upgrade/migration-phases/planning-your-migration/index.md @@ -3,6 +3,7 @@ layout: default title: Planning your migration nav_order: 59 parent: Migration phases +permalink: /planning-your-migration/ has_toc: false has_children: true --- diff --git a/_migration-assistant/migration-phases/planning-your-migration/verifying-migration-tools.md b/_migration-upgrade/migration-phases/planning-your-migration/verifying-migration-tools.md similarity index 99% rename from _migration-assistant/migration-phases/planning-your-migration/verifying-migration-tools.md rename to _migration-upgrade/migration-phases/planning-your-migration/verifying-migration-tools.md index 6cfe762c85b..42bde888a1a 100644 --- a/_migration-assistant/migration-phases/planning-your-migration/verifying-migration-tools.md +++ b/_migration-upgrade/migration-phases/planning-your-migration/verifying-migration-tools.md @@ -4,6 +4,7 @@ title: Verifying migration tools nav_order: 70 parent: Planning your migration grand_parent: Migration phases +permalink: /migration-assistant/migration-phases/planning-your-migration/verifying-migration-tools/ redirect_from: - /migration-assistant/migration-phases/verifying-migration-tools/ --- diff --git a/_migration-assistant/migration-phases/removing-migration-infrastructure.md b/_migration-upgrade/migration-phases/removing-migration-infrastructure.md similarity index 95% rename from _migration-assistant/migration-phases/removing-migration-infrastructure.md rename to _migration-upgrade/migration-phases/removing-migration-infrastructure.md index adc2a4b7776..7d677908822 100644 --- a/_migration-assistant/migration-phases/removing-migration-infrastructure.md +++ b/_migration-upgrade/migration-phases/removing-migration-infrastructure.md @@ -3,6 +3,7 @@ layout: default title: Removing migration infrastructure nav_order: 120 parent: Migration phases +permalink: /migration-assistant/migration-phases/removing-migration-infrastructure/ --- # Removing migration infrastructure diff --git a/_migration-assistant/overview/architecture.md b/_migration-upgrade/overview/architecture.md similarity index 96% rename from _migration-assistant/overview/architecture.md rename to _migration-upgrade/overview/architecture.md index 309261e4915..48b3c8fbcfc 100644 --- a/_migration-assistant/overview/architecture.md +++ b/_migration-upgrade/overview/architecture.md @@ -3,6 +3,7 @@ layout: default title: Architecture nav_order: 15 parent: Overview +permalink: /migration-assistant/overview/architecture/ --- # Architecture diff --git a/_migration-assistant/overview/index.md b/_migration-upgrade/overview/index.md similarity index 100% rename from _migration-assistant/overview/index.md rename to _migration-upgrade/overview/index.md diff --git a/_migration-assistant/overview/is-migration-assistant-right-for-you.md b/_migration-upgrade/overview/is-migration-assistant-right-for-you.md similarity index 99% rename from _migration-assistant/overview/is-migration-assistant-right-for-you.md rename to _migration-upgrade/overview/is-migration-assistant-right-for-you.md index 2d9a4a20508..1e7fc287974 100644 --- a/_migration-assistant/overview/is-migration-assistant-right-for-you.md +++ b/_migration-upgrade/overview/is-migration-assistant-right-for-you.md @@ -3,6 +3,7 @@ layout: default title: Is Migration Assistant right for you? nav_order: 10 parent: Overview +permalink: /migration-assistant/overview/is-migration-assistant-right-for-you/ redirect_from: - /migration-assistant/is-migration-assistant-right-for-you/ --- diff --git a/_migration-assistant/overview/key-components.md b/_migration-upgrade/overview/key-components.md similarity index 97% rename from _migration-assistant/overview/key-components.md rename to _migration-upgrade/overview/key-components.md index ca09c7f29e8..cfa69e70311 100644 --- a/_migration-assistant/overview/key-components.md +++ b/_migration-upgrade/overview/key-components.md @@ -3,6 +3,7 @@ layout: default title: Key components nav_order: 10 parent: Overview +permalink: /migration-assistant/overview/key-components/ --- # Key components From a28ad150fb517547b21fc5f3fb4d0f4f8709f098 Mon Sep 17 00:00:00 2001 From: Archer Date: Tue, 1 Jul 2025 07:35:30 -0500 Subject: [PATCH 05/13] Fix broken links Signed-off-by: Archer --- .../getting-started-data-migration.md | 2 +- _migration-upgrade/migration-phases/index.md | 2 +- .../live-traffic-migration/using-traffic-replayer.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/_migration-upgrade/deploying-migration-assistant/getting-started-data-migration.md b/_migration-upgrade/deploying-migration-assistant/getting-started-data-migration.md index 6fb28102840..d0592e2dd24 100644 --- a/_migration-upgrade/deploying-migration-assistant/getting-started-data-migration.md +++ b/_migration-upgrade/deploying-migration-assistant/getting-started-data-migration.md @@ -3,7 +3,7 @@ layout: default title: Getting started with data migration parent: Deploying Migration Assistant nav_order: 10 -permalink: /migration-assistant/deploying-migration-assistant/getting-started-with-data-migration/ +permalink: /migration-assistant/deploying-migration-assistant/getting-started-data-migration/ redirect_from: - /upgrade-to/snapshot-migrate/ - /migration-assistant/getting-started-with-data-migration/ diff --git a/_migration-upgrade/migration-phases/index.md b/_migration-upgrade/migration-phases/index.md index 37d835135fd..a330b9e598e 100644 --- a/_migration-upgrade/migration-phases/index.md +++ b/_migration-upgrade/migration-phases/index.md @@ -11,7 +11,7 @@ redirect_from: # Migration phases -This page details how to conduct a migration with Migration Assistant. It shows you how to [plan for your migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/planning-your-migration/index/) and encompasses a variety of migration scenarios, including: +This page details how to conduct a migration with Migration Assistant. It shows you how to [plan for your migration]({{site.url}}{{site.baseurl}}/planning-your-migration/) and encompasses a variety of migration scenarios, including: - [**Metadata migration**]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/): Migrating cluster metadata, such as index settings, aliases, and templates. - [**Backfill migration**]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/): Migrating existing or historical data from a source to a target cluster. diff --git a/_migration-upgrade/migration-phases/live-traffic-migration/using-traffic-replayer.md b/_migration-upgrade/migration-phases/live-traffic-migration/using-traffic-replayer.md index 6ebedf0d91a..1454f7f04a3 100644 --- a/_migration-upgrade/migration-phases/live-traffic-migration/using-traffic-replayer.md +++ b/_migration-upgrade/migration-phases/live-traffic-migration/using-traffic-replayer.md @@ -4,7 +4,7 @@ title: Using Traffic Replayer nav_order: 100 grand_parent: Migration phases parent: Live traffic migration -permalink: /migration-assistant/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster/ +permalink: /migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer/ redirect_from: - /migration-assistant/migration-phases/using-traffic-replayer/ --- From 9d68c5a221bfccd590bc20025a14f34735429567 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Tue, 1 Jul 2025 21:27:26 -0500 Subject: [PATCH 06/13] Add addition upgrade options section Signed-off-by: Brian Presley --- _migration-upgrade/index.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/_migration-upgrade/index.md b/_migration-upgrade/index.md index c44a1b5855d..5d17cba894c 100644 --- a/_migration-upgrade/index.md +++ b/_migration-upgrade/index.md @@ -70,7 +70,7 @@ It's crucial to note that migration strategies are not universally applicable. T ## Rolling upgrades -Rolling upgrades allow you to upgrade one node at a time, keeping the cluster operational throughout the process. +Rolling upgrades allow you to upgrade one node at a time, keeping the cluster operational throughout the process. Note that this is only an upgrade option. **Pros:** - Minimal downtime; the cluster remains available during the upgrade. @@ -100,6 +100,10 @@ This method involves taking a snapshot of your existing OpenSearch or Elasticsea This method reindexes data from your current cluster into a new OpenSearch cluster, typically running in parallel. +## Other migration options + +Additional migration strategies not covered in detail in this documentation include rebuilding your target cluster from source systems and using traffic replication to mirror production traffic during migration. + **Pros:** - No downtime; clusters operate concurrently. - Supports upgrades across multiple versions. From b6b1220ce7ee34da2843dcd69331f049337f697a Mon Sep 17 00:00:00 2001 From: Archer Date: Wed, 2 Jul 2025 09:24:05 -0500 Subject: [PATCH 07/13] Fix home cards page Signed-off-by: Archer --- _includes/home_cards.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_includes/home_cards.html b/_includes/home_cards.html index 098472d4c85..1509d5402ea 100644 --- a/_includes/home_cards.html +++ b/_includes/home_cards.html @@ -61,9 +61,9 @@
- -

Migration Assistant

-

Migrate to OpenSearch.

+ +

Migrate and upgrade

+

Migrate or upgrade to OpenSearch.

Documentation →

From 3e364faf9eb3fab5221ab06b9e53b1d7cc1e9fe0 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Wed, 2 Jul 2025 18:22:32 -0500 Subject: [PATCH 08/13] Updated requirements around VPC connectivity. Signed-off-by: Brian Presley --- .../is-migration-assistant-right-for-you.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/_migration-assistant/overview/is-migration-assistant-right-for-you.md b/_migration-assistant/overview/is-migration-assistant-right-for-you.md index 11d2a3cf10e..0cda220dc7b 100644 --- a/_migration-assistant/overview/is-migration-assistant-right-for-you.md +++ b/_migration-assistant/overview/is-migration-assistant-right-for-you.md @@ -21,10 +21,25 @@ Before using Migration Assistant, review the following assumptions and limitatio ### General + - If deploying on AWS, the identity used to deploy Migration Assistant must have permission to install all required resources. For a full list of services, see [AWS Services in this Solution](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/architecture-details.html#aws-services-in-this-solution). - The target cluster must be deployed and reachable by Migration Assistant components, including the Migration Console, Reindex-from-Snapshot, and Traffic Replayer, depending on which features are used. - The `_source` field must be enabled on all indexes to be migrated. See [Source documentation]({{site.url}}{{site.baseurl}}/field-types/metadata-fields/source/) for more information. - If deploying to AWS, ensure the `CDKToolkit` stack exists and is in the `CREATE_COMPLETE` state. For setup instructions, see the [CDK Toolkit documentation](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html). +- If deploying Migration Assistant into an existing AWS VPC, you must configure VPC interface endpoints and IAM permissions for the following AWS services: + - **Amazon CloudWatch Logs** – for log ingestion from ECS tasks. + - **Amazon CloudWatch** – for metrics published during migration + - **Amazon Elastic Container Registry (ECR)** – for pulling container images. + - **Amazon Elastic Container Service (ECS)** – for task orchestration and migration. + - **Elastic Load Balancing (ELB)** – for routing traffic to Capture Proxy. + - **AWS Secrets Manager** – for storing credentials. + - **AWS Systems Manager Parameter Store** – for storing and accessing parameter values. + - **AWS Systems Manager Session Manager** – for secure shell access to EC2 (i.e., bootstrap instance). + - **Amazon EC2** – for launching the bootstrap instance responsible for build and deployment. + - **Amazon Elastic Block Store (EBS)** – for disk storage during migration. + - **Amazon Virtual Private Cloud (VPC)** – for private networking and VPC endpoints. + - **AWS X-Ray** – for distributed tracing across components. + - **Amazon Elastic File System (EFS)** – for persistent logging. ### Reindex-from-Snapshot From abdfa960c1945ac11fc3fbfefb6c49389882a045 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Wed, 2 Jul 2025 21:29:36 -0500 Subject: [PATCH 09/13] Restructured migration journey and added migration phases. Signed-off-by: Brian Presley --- _config.yml | 18 +- _includes/home_cards.html | 6 +- .../assets/css/breaking-changes-selector.css | 0 .../assets/js/breaking-changes-data.js | 0 .../assets/js/breaking-changes-filter.js | 0 .../assets/js/breaking-changes-index.js | 0 .../assets/js/breaking-changes-module.js | 0 .../assets/js/breaking-changes-ui.js | 0 _migration-assistant/index.md | 35 ++ .../accessing-the-migration-console.md | 42 ++ .../migration-phases/assessment.md | 76 ++++ .../migration-phases/create-snapshot.md | 244 ++++++++++++ .../configuration-options.md | 233 ++++++++++++ .../getting-started-data-migration.md | 360 ++++++++++++++++++ ...d-security-groups-for-existing-clusters.md | 93 +++++ .../deploying-migration-assistant/index.md | 13 + .../migration-phases/deployment.md | 341 +++++++++++++++++ .../migration-phases/index.md | 87 +++++ .../migration-phases/migrate-metadata.md | 249 ++++++++++++ .../replay-captured-traffic.md | 319 ++++++++++++++++ .../reroute-source-to-proxy.md | 102 +++++ ...te-traffic-from-capture-proxy-to-target.md | 54 +++ .../migration-phases/teardown.md | 30 ++ .../verify-backfill-components.md | 191 ++++++++++ .../verify-live-capture-components.md | 157 ++++++++ _migration-assistant/overview/architecture.md | 25 ++ _migration-assistant/overview/index.md | 25 ++ .../is-migration-assistant-right-for-you.md | 154 ++++++++ .../overview/key-components.md | 39 ++ .../handling-field-type-breaking-changes.md | 158 ++++++++ .../handling-type-mapping-deprecation.md | 318 ++++++++++++++++ .../planning-your-migration/index.md | 15 + _migration-upgrade/index.md | 36 +- .../migration-phases/backfill.md | 97 +---- _migration-upgrade/migration-phases/index.md | 72 +++- 35 files changed, 3456 insertions(+), 133 deletions(-) rename {_migration-upgrade => _migration-assistant}/assets/css/breaking-changes-selector.css (100%) rename {_migration-upgrade => _migration-assistant}/assets/js/breaking-changes-data.js (100%) rename {_migration-upgrade => _migration-assistant}/assets/js/breaking-changes-filter.js (100%) rename {_migration-upgrade => _migration-assistant}/assets/js/breaking-changes-index.js (100%) rename {_migration-upgrade => _migration-assistant}/assets/js/breaking-changes-module.js (100%) rename {_migration-upgrade => _migration-assistant}/assets/js/breaking-changes-ui.js (100%) create mode 100644 _migration-assistant/index.md create mode 100644 _migration-assistant/migration-phases/accessing-the-migration-console.md create mode 100644 _migration-assistant/migration-phases/assessment.md create mode 100644 _migration-assistant/migration-phases/create-snapshot.md create mode 100644 _migration-assistant/migration-phases/deploying-migration-assistant/configuration-options.md create mode 100644 _migration-assistant/migration-phases/deploying-migration-assistant/getting-started-data-migration.md create mode 100644 _migration-assistant/migration-phases/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md create mode 100644 _migration-assistant/migration-phases/deploying-migration-assistant/index.md create mode 100644 _migration-assistant/migration-phases/deployment.md create mode 100644 _migration-assistant/migration-phases/index.md create mode 100644 _migration-assistant/migration-phases/migrate-metadata.md create mode 100644 _migration-assistant/migration-phases/replay-captured-traffic.md create mode 100644 _migration-assistant/migration-phases/reroute-source-to-proxy.md create mode 100644 _migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target.md create mode 100644 _migration-assistant/migration-phases/teardown.md create mode 100644 _migration-assistant/migration-phases/verify-backfill-components.md create mode 100644 _migration-assistant/migration-phases/verify-live-capture-components.md create mode 100644 _migration-assistant/overview/architecture.md create mode 100644 _migration-assistant/overview/index.md create mode 100644 _migration-assistant/overview/is-migration-assistant-right-for-you.md create mode 100644 _migration-assistant/overview/key-components.md create mode 100644 _migration-assistant/planning-your-migration/handling-field-type-breaking-changes.md create mode 100644 _migration-assistant/planning-your-migration/handling-type-mapping-deprecation.md create mode 100644 _migration-assistant/planning-your-migration/index.md diff --git a/_config.yml b/_config.yml index baac4a1c7c6..27f751e4010 100644 --- a/_config.yml +++ b/_config.yml @@ -92,6 +92,9 @@ collections: migration-upgrade: permalink: /:collection/:path/ output: true + migration-assistant: + permalink: /:collection/:path/ + output: true tools: permalink: /:collection/:path/ output: true @@ -221,6 +224,12 @@ migration_upgrade_collection: name: Migration and upgrade nav_fold: true +migration_assistant_collection: + collections: + migration-assistant: + name: Migration Assistant + nav_fold: true + benchmark_collection: collections: benchmark: @@ -262,10 +271,10 @@ defaults: section-name: "Benchmark" - scope: - path: "_migration-assistant" + path: "_migrations-and-upgrades" values: - section: "migration-assistant" - section-name: "Migration Assistant" + section: "migrations-and-upgrades" + section-name: "Migrations and Upgrades" # Enable or disable the site search # By default, just-the-docs enables its JSON file-based search. We also have an OpenSearch-driven search functionality. @@ -326,6 +335,7 @@ plugins: - jekyll-redirect-from - jekyll-sitemap - jekyll-spec-insert + - jekyll-include-cache # This format has to conform to RFC822 last-modified-at: @@ -352,4 +362,4 @@ exclude: - .bundle/ - _site/ - spec-insert - - release-notes \ No newline at end of file + - release-notes diff --git a/_includes/home_cards.html b/_includes/home_cards.html index 1509d5402ea..8be45489317 100644 --- a/_includes/home_cards.html +++ b/_includes/home_cards.html @@ -61,9 +61,9 @@
- -

Migrate and upgrade

-

Migrate or upgrade to OpenSearch.

+ +

Modernize Workloads

+

Migration Assistant for OpenSearch and other upgrade and migration options

Documentation →

diff --git a/_migration-upgrade/assets/css/breaking-changes-selector.css b/_migration-assistant/assets/css/breaking-changes-selector.css similarity index 100% rename from _migration-upgrade/assets/css/breaking-changes-selector.css rename to _migration-assistant/assets/css/breaking-changes-selector.css diff --git a/_migration-upgrade/assets/js/breaking-changes-data.js b/_migration-assistant/assets/js/breaking-changes-data.js similarity index 100% rename from _migration-upgrade/assets/js/breaking-changes-data.js rename to _migration-assistant/assets/js/breaking-changes-data.js diff --git a/_migration-upgrade/assets/js/breaking-changes-filter.js b/_migration-assistant/assets/js/breaking-changes-filter.js similarity index 100% rename from _migration-upgrade/assets/js/breaking-changes-filter.js rename to _migration-assistant/assets/js/breaking-changes-filter.js diff --git a/_migration-upgrade/assets/js/breaking-changes-index.js b/_migration-assistant/assets/js/breaking-changes-index.js similarity index 100% rename from _migration-upgrade/assets/js/breaking-changes-index.js rename to _migration-assistant/assets/js/breaking-changes-index.js diff --git a/_migration-upgrade/assets/js/breaking-changes-module.js b/_migration-assistant/assets/js/breaking-changes-module.js similarity index 100% rename from _migration-upgrade/assets/js/breaking-changes-module.js rename to _migration-assistant/assets/js/breaking-changes-module.js diff --git a/_migration-upgrade/assets/js/breaking-changes-ui.js b/_migration-assistant/assets/js/breaking-changes-ui.js similarity index 100% rename from _migration-upgrade/assets/js/breaking-changes-ui.js rename to _migration-assistant/assets/js/breaking-changes-ui.js diff --git a/_migration-assistant/index.md b/_migration-assistant/index.md new file mode 100644 index 00000000000..1d7f8bb1db5 --- /dev/null +++ b/_migration-assistant/index.md @@ -0,0 +1,35 @@ +--- +layout: default +title: Migration Assistant for OpenSearch +nav_order: 1 +has_children: false +nav_exclude: true +has_toc: false +permalink: /migration-assistant/ +redirect_from: + - /migration-assistant/index/ + - /upgrade-to/index/ + - /upgrade-to/ + - /upgrade-to/upgrade-to/ +tutorial_cards: + - heading: "Overview" + description: "Get familiar with the key components of Migration Assistant and evaluate your use case." + link: "/migration-assistant/overview/" + - heading: "Migration phases" + description: "Execute your migration in phases—metadata, backfill, and traffic replay—for a controlled and validated transition." + link: "/migration-assistant/migration-phases/" +--- + +# Migration Assistant for OpenSearch + +Migration Assistant for OpenSearch aids you in successfully performing an end-to-end, zero-downtime migration to OpenSearch from other search providers. It helps with the following scenarios: + +- **Metadata migration**: Migrating cluster metadata, such as index settings, aliases, and templates. +- **Backfill migration**: Migrating existing or historical data from a source to a target cluster. +- **Live traffic migration**: Replicating live ongoing traffic from a source to a target cluster. +- **Comparative tooling**: Comparing the performance and behaviors of an existing cluster with a prospective new one. + +This user guide focuses on conducting a comprehensive migration involving both existing and live data with zero downtime and the option to back out of a migration. + +{% include cards.html cards=page.tutorial_cards %} + diff --git a/_migration-assistant/migration-phases/accessing-the-migration-console.md b/_migration-assistant/migration-phases/accessing-the-migration-console.md new file mode 100644 index 00000000000..fc86d61ca2b --- /dev/null +++ b/_migration-assistant/migration-phases/accessing-the-migration-console.md @@ -0,0 +1,42 @@ +--- +layout: default +title: Accessing the Migration Console +parent: Deploying Migration Assistant +nav_order: 10 +redirect_from: +--- +# Accessing the Migration Console + +The Migrations Assistant deployment includes an ECS task that hosts tools to run different phases of the migration and check the progress or results of the migration. + +## SSH into the Migration Console + +Following the AWS Solutions deployment, the bootstrap box contains a script that simplifies access to the migration console through that instance. + +To access the Migration Console, use the following commands: + +```sh +export STAGE=dev +export AWS_REGION=us-west-2 +/opensearch-migrations/deployment/cdk/opensearch-service-migration/accessContainer.sh migration-console ${STAGE} ${AWS_REGION} +``` +When opening the console a message will appear above the command prompt, Welcome to the Migration Assistant Console. +
+ + +SSH from any machine into Migration Console + + +On a machine with the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) ↗ and the [AWS Session Manager Plugin](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html) ↗, you can directly connect to the migration console. Ensure you've run `aws configure` with credentials that have access to the environment. + +Use the following commands: + +```shell +export STAGE=dev +export SERVICE_NAME=migration-console +export TASK_ARN=$(aws ecs list-tasks --cluster migration-${STAGE}-ecs-cluster --family "migration-${STAGE}-${SERVICE_NAME}" | jq --raw-output '.taskArns[0]') +aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${TASK_ARN}" --container "${SERVICE_NAME}" --interactive --command "/bin/bash" +``` +
+ +> **Note:** Typically, `STAGE` is `dev`, but this may vary based on what the user specified during deployment. \ No newline at end of file diff --git a/_migration-assistant/migration-phases/assessment.md b/_migration-assistant/migration-phases/assessment.md new file mode 100644 index 00000000000..d5210674e66 --- /dev/null +++ b/_migration-assistant/migration-phases/assessment.md @@ -0,0 +1,76 @@ +--- +layout: default +title: Assess +nav_order: 1 +parent: Migration phases +redirect_from: + - /migration-assistant/migration-phases/assessing-your-cluster-for-migration/ +--- + +# Assessing your cluster for migration + +The goal of the Migration Assistant is to streamline the process of migrating from one location or version of Elasticsearch/OpenSearch to another. However, completing a migration sometimes requires resolving client compatibility issues before they can communicate directly with the target cluster. + + +## Understanding breaking changes + +Before performing any upgrade or migration, you should review any breaking changes that might exist between version because there may be changes required in order for clients to connect to the new cluster. Use the following tool by selecting the versions in your migration path. The tool will respond with any breaking changes you should note when migrating: + + + +
+

Find a list of breaking changes for your migration path

+ +
+ + + + + +
+ +
+
+ + +
+ +
+
+ + + + + +## Impact of data transformations + +Any time you apply a transformation to your data, such as: + +- Changing index names +- Modifying field names or field mappings +- Splitting indices with type mappings + +These changes might need to be reflected in your client configurations. For example, if your clients are reliant on specific index or field names, you must ensure that their queries are updated accordingly. + +We recommend running production-like queries against the target cluster before switching over actual production traffic. This helps verify that the client can: + +- Communicate with the target cluster +- Locate the necessary indices and fields +- Retrieve the expected results + +For complex migrations involving multiple transformations or breaking changes, we highly recommend performing a trial migration with representative, non-production data (e.g., in a staging environment) to fully test client compatibility with the target cluster. + +## Supported transformations + +The following [transformations]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer/#transformations) are included in Migration Assistant. They can be enabled, combined, and configured to customize a migration for your use case. To request additional Migration Assistant transformations , create a GitHub issue [in the OpenSearch migrations repository](https://github.com/opensearch-project/opensearch-migrations/issues). + +- [Type mapping deprecation]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/planning-your-migration/handling-type-mapping-deprecation/) diff --git a/_migration-assistant/migration-phases/create-snapshot.md b/_migration-assistant/migration-phases/create-snapshot.md new file mode 100644 index 00000000000..6b24ae78b6b --- /dev/null +++ b/_migration-assistant/migration-phases/create-snapshot.md @@ -0,0 +1,244 @@ +--- +layout: default +title: Snapshot +nav_order: 4 +parent: Migration phases +--- + +## Creating snapshot + +TODO: Add Bring your own snapshot + +Creating a snapshot of the source cluster captures all the metadata and documents to be migrated to a new target cluster. + +Create the initial snapshot of the source cluster using the following command: + +```shell +console snapshot create +``` +{% include copy.html %} + +To check the progress of the snapshot in real time, use the following command: + +```shell +console snapshot status --deep-check +``` +{% include copy.html %} + +You should receive the following response when the snapshot is created: + +```shell +SUCCESS +Snapshot is SUCCESS. +Percent completed: 100.00% +Data GiB done: 29.211/29.211 +Total shards: 40 +Successful shards: 40 +Failed shards: 0 +Start time: 2024-07-22 18:21:42 +Duration: 0h 13m 4s +Anticipated duration remaining: 0h 0m 0s +Throughput: 38.13 MiB/sec +``` + +### Managing slow snapshot speeds + +Depending on the size of the data in the source cluster and the bandwidth allocated for snapshots, the process can take some time. Adjust the maximum rate at which the source cluster's nodes create the snapshot using the `--max-snapshot-rate-mb-per-node` option. Increasing the snapshot rate will consume more node resources, which may affect the cluster's ability to handle normal traffic. + +## Command arguments + +For the following commands, to identify all valid arguments, please run with `--help`. + +```shell +console metadata evaluate --help +``` +{% include copy.html %} + +```shell +console metadata migrate --help +``` +{% include copy.html %} + +Based on the migration console deployment options, a number of commands will be pre-populated. To view them, run console with verbosity: + +```shell +console -v metadata migrate --help +``` +{% include copy.html %} + +You should receive a response similar to the following: + +```shell +(.venv) bash-5.2# console -v metadata migrate --help +INFO:console_link.cli:Logging set to INFO +. +. +. +INFO:console_link.models.metadata:Migrating metadata with command: /root/metadataMigration/bin/MetadataMigration --otel-collector-endpoint http://otel-collector:4317 migrate --snapshot-name snapshot_2023_01_01 --target-host https://opensearchtarget:9200 --min-replicas 0 --file-system-repo-path /snapshot/test-console --target-username admin --target-password ******** --target-insecure --help +. +. +. +``` + + +## Using the `evaluate` command + +By scanning the contents of the source cluster, applying filtering, and applying modifications a list of all items that will be migrated will be created. Any items not seen in this output will not be migrated onto the target cluster if the migrate command was to be run. This is a safety check before making modifications on the target cluster. + +```shell +console metadata evaluate [...] +``` +{% include copy.html %} + +You should receive a response similar to the following: + +```bash +Starting Metadata Evaluation +Clusters: + Source: + Remote Cluster: OpenSearch 1.3.16 ConnectionContext(uri=http://localhost:33039, protocol=HTTP, insecure=false, compressionSupported=false) + + Target: + Remote Cluster: OpenSearch 2.14.0 ConnectionContext(uri=http://localhost:33037, protocol=HTTP, insecure=false, compressionSupported=false) + + +Migration Candidates: + Index Templates: + simple_index_template + + Component Templates: + simple_component_template + + Indexes: + blog_2023, movies_2023 + + Aliases: + alias1, movies-alias + + +Results: + 0 issue(s) detected +``` + + +## Using the migrate command + +Running through the same data as the evaluate command all of the migrated items will be applied onto the target cluster. If re-run multiple times items that were previously migrated will not be recreated. If any items do need to be re-migrated, please delete them from the target cluster and then rerun the evaluate then migrate commands to ensure the desired changes are made. + +```shell +console metadata migrate [...] +``` +{% include copy.html %} + +You should receive a response similar to the following: + +```shell +Starting Metadata Migration + +Clusters: + Source: + Snapshot: OpenSearch 1.3.16 FileSystemRepo(repoRootDir=/tmp/junit10626813752669559861) + + Target: + Remote Cluster: OpenSearch 2.14.0 ConnectionContext(uri=http://localhost:33042, protocol=HTTP, insecure=false, compressionSupported=false) + + +Migrated Items: + Index Templates: + simple_index_template + + Component Templates: + simple_component_template + + Indexes: + blog_2023, movies_2023 + + Aliases: + alias1, movies-alias + + +Results: + 0 issue(s) detected +``` + + +## Metadata verification process + +Before moving on to additional migration steps, it is recommended to confirm details of your cluster. Depending on your configuration, this could be checking the sharding strategy or making sure index mappings are correctly defined by ingesting a test document. + +## Troubleshooting + +Use these instructions to help troubleshoot the following issues. + +### Accessing detailed logs + +Metadata migration creates a detailed log file that includes low level tracing information for troubleshooting. For each execution of the program a log file is created inside a shared volume on the migration console named `shared-logs-output` the following command will list all log files, one for each run of the command. + +```shell +ls -al /shared-logs-output/migration-console-default/*/metadata/ +``` +{% include copy.html %} + +To inspect the file within the console `cat`, `tail` and `grep` commands line tools. By looking for warnings, errors and exceptions in this log file can help understand the source of failures, or at the very least be useful for creating issues in this project. + +```shell +tail /shared-logs-output/migration-console-default/*/metadata/*.log +``` +{% include copy.html %} + +### Warnings and errors + +When encountering `WARN` or `ERROR` elements in the response, they will be accompanied by a short message, such as `WARN - my_index already exists`. More information can be found in the detailed logs associated with the warning or error. + +### OpenSearch running in compatibility mode + +There might be an error about being unable to update an ES 7.10.2 cluster, this can occur when compatibility mode has been enabled on an OpenSearch cluster disable it to continue, see [Enable compatibility mode](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/rename.html#rename-upgrade). + + +### Breaking change compatibility + +Metadata migration requires modifying data from the source to the target versions to recreate items. Sometimes these features are no longer supported and have been removed from the target version. Sometimes these features are not available in the target version, which is especially true when downgrading. While this tool is meant to make this process easier, it is not exhaustive in its support. When encountering a compatibility issue or an important feature gap for your migration, [search the issues and comment on the existing issue](https://github.com/opensearch-project/opensearch-migrations/issues) or [create a new](https://github.com/opensearch-project/opensearch-migrations/issues/new/choose) issue if one cannot be found. + +#### Deprecation of Mapping Types + +In Elasticsearch 6.8 the mapping types feature was discontinued in Elasticsearch 7.0+ which has created complexity in migrating to newer versions of Elasticsearch and OpenSearch, [learn more](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html) ↗. + +As Metadata migration supports migrating from ES 6.8 on to the latest versions of OpenSearch this scenario is handled by removing the type mapping types and restructuring the template or index properties. Note that, at the time of this writing multiple type mappings are not supported, [tracking task](https://opensearch.atlassian.net/browse/MIGRATIONS-1778) ↗. + + +**Example starting state with mapping type foo (ES 6):** + +```json +{ + "mappings": [ + { + "foo": { + "properties": { + "field1": { "type": "text" }, + "field2": { "type": "keyword" } + } + } + } + ] +} +``` +{% include copy.html %} + +**Example ending state with foo removed (ES 7):** + +```json +{ + "mappings": { + "properties": { + "field1": { "type": "text" }, + "field2": { "type": "keyword" }, + } + } +} +``` +{% include copy.html %} + +For additional technical details, [view the mapping type removal source code](https://github.com/opensearch-project/opensearch-migrations/blob/main/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java). + +TODO: Add troublshooting +
  • Verify Backfill Components
    • \ No newline at end of file diff --git a/_migration-assistant/migration-phases/deploying-migration-assistant/configuration-options.md b/_migration-assistant/migration-phases/deploying-migration-assistant/configuration-options.md new file mode 100644 index 00000000000..5032dcb757e --- /dev/null +++ b/_migration-assistant/migration-phases/deploying-migration-assistant/configuration-options.md @@ -0,0 +1,233 @@ +--- +layout: default +title: Configuration options +nav_order: 15 +parent: Deploying Migration Assistant +permalink: /migration-assistant/deploying-migration-assistant/configuration-options/ +--- + +# Configuration options + +This page outlines the configuration options for three key migrations scenarios: + +1. **Metadata migration** +2. **Backfill migration with `Reindex-from-Snapshot` (RFS)** +3. **Live capture migration with Capture and Replay (C&R)** + +Each of these migrations depends on either a snapshot or a capture proxy. The following example `cdk.context.json` configurations are used by AWS Cloud Development Kit (AWS CDK) to deploy and configure Migration Assistant for OpenSearch, shown as separate blocks for each migration type. If you are performing a migration applicable to multiple scenarios, these options can be combined. + + +For a complete list of configuration options, see [opensearch-migrations-options.md](https://github.com/opensearch-project/opensearch-migrations/blob/main/deployment/cdk/opensearch-service-migration/options.md). If you need a configuration option that is not found on this page, create an issue in the [OpenSearch Migrations repository](https://github.com/opensearch-project/opensearch-migrations/issues). +{: .tip } + +Options for the source cluster endpoint, target cluster endpoint, and existing virtual private cloud (VPC) should be configured in order for the migration tools to function effectively. + +## Shared configuration options + +Each migration configuration shares the following options. + + +| Name | Example | Description | +| :--- | :--- | :--- | +| `sourceClusterEndpoint` | `"https://source-cluster.elb.us-east-1.endpoint.com"` | The endpoint for the source cluster. | +| `targetClusterEndpoint` | `"https://vpc-demo-opensearch-cluster-cv6hggdb66ybpk4kxssqt6zdhu.us-west-2.es.amazonaws.com:443"` | The endpoint for the target cluster. Required if using an existing target cluster for the migration instead of creating a new one. | +| `vpcId` | `"vpc-123456789abcdefgh"` | The ID of the existing VPC in which the migration resources will be stored. The VPC must have at least two private subnets that span two Availability Zones. | + + +## Backfill migration using RFS + +The following CDK performs a backfill migrations using RFS: + +```json +{ + "backfill-migration": { + "stage": "dev", + "vpcId": , + "sourceCluster": { + "endpoint": , + "version": "ES 7.10", + "auth": {"type": "none"} + }, + "targetCluster": { + "endpoint": , + "auth": { + "type": "basic", + "username": , + "passwordFromSecretArn": + } + }, + "reindexFromSnapshotServiceEnabled": true, + "reindexFromSnapshotExtraArgs": "", + "artifactBucketRemovalPolicy": "DESTROY" + } +} +``` +{% include copy.html %} + +Performing an RFS backfill migration requires an existing snapshot. + + +The RFS configuration uses the following options. All options are optional. + +| Name | Example | Description | +| :--- | :--- | :--- | +| `reindexFromSnapshotServiceEnabled` | `true` | Enables deployment and configuration of the RFS ECS service. | +| `reindexFromSnapshotExtraArgs` | `"--target-aws-region us-east-1 --target-aws-service-signing-name es"` | Extra arguments for the Document Migration command, with space separation. See [RFS Extra Arguments](https://github.com/opensearch-project/opensearch-migrations/blob/main/DocumentsFromSnapshotMigration/README.md#arguments) for more information. You can pass `--no-insecure` to remove the `--insecure` flag. | + +To view all available arguments for `reindexFromSnapshotExtraArgs`, see [Snapshot migrations README](https://github.com/opensearch-project/opensearch-migrations/blob/main/DocumentsFromSnapshotMigration/README.md#arguments). At a minimum, no extra arguments may be needed. + +## Live capture migration with C&R + +The following sample CDK performs a live capture migration with C&R: + +```json +{ + "live-capture-migration": { + "stage": "dev", + "vpcId": , + "sourceCluster": { + "endpoint": , + "version": "ES 7.10", + "auth": {"type": "none"} + }, + "targetCluster": { + "endpoint": , + "auth": { + "type": "basic", + "username": , + "passwordFromSecretArn": + } + }, + "captureProxyServiceEnabled": true, + "captureProxyExtraArgs": "", + "trafficReplayerServiceEnabled": true, + "trafficReplayerExtraArgs": "--speedup-factor 2.0", + "targetClusterProxyServiceEnabled": true + } +} +``` +{% include copy.html %} + +Performing a live capture migration requires that a Capture Proxy be configured to capture incoming traffic and send it to the target cluster using the Traffic Replayer service. For arguments available in `captureProxyExtraArgs`, refer to the `@Parameter` fields [here](https://github.com/opensearch-project/opensearch-migrations/blob/main/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java). For `trafficReplayerExtraArgs`, refer to the `@Parameter` fields [here](https://github.com/opensearch-project/opensearch-migrations/blob/main/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java). At a minimum, no extra arguments may be needed. + + +| Name | Example | Description | +| :--- | :--- | :--- | +| `captureProxyServiceEnabled` | `true` | Enables the Capture Proxy service deployment using an AWS CloudFormation stack. | +| `captureProxyExtraArgs` | `"--suppressCaptureForHeaderMatch user-agent .*elastic-java/7.17.0.*"` | Extra arguments for the Capture Proxy command, including options specified by the [Capture Proxy](https://github.com/opensearch-project/opensearch-migrations/blob/main/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java). | +| `trafficReplayerServiceEnabled` | `true` | Enables the Traffic Replayer service deployment using a CloudFormation stack. | +| `trafficReplayerExtraArgs` | `"--sigv4-auth-header-service-region es,us-east-1 --speedup-factor 5"` | Extra arguments for the Traffic Replayer command, including options for auth headers and other parameters specified by the [Traffic Replayer](https://github.com/opensearch-project/opensearch-migrations/blob/main/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java). | +| `targetClusterProxyServiceEnabled` | `true` | Enables the target cluster proxy service deployment using a CloudFormation stack. | + +For arguments available in `captureProxyExtraArgs`, see the `@Parameter` fields in [`CaptureProxy.java`](https://github.com/opensearch-project/opensearch-migrations/blob/main/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java). For `trafficReplayerExtraArgs`, see the `@Parameter` fields in [`TrafficReplayer.java`](https://github.com/opensearch-project/opensearch-migrations/blob/main/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java). + + +## Cluster authentication options + +Both the source and target cluster can use no authentication, authentication limited to VPC, basic authentication with a username and password, or AWS Signature Version 4 scoped to a user or role. + +### No authentication + +```json + "sourceCluster": { + "endpoint": , + "version": "ES 7.10", + "auth": {"type": "none"} + } +``` +{% include copy.html %} + +### Basic authentication + +```json + "sourceCluster": { + "endpoint": , + "version": "ES 7.10", + "auth": { + "type": "basic", + "username": , + "passwordFromSecretArn": + } + } +``` +{% include copy.html %} + +### AWS Signature Version 4 authentication + +```json + "sourceCluster": { + "endpoint": , + "version": "ES 7.10", + "auth": { + "type": "sigv4", + "region": "us-east-1", + "serviceSigningName": "es" + } + } +``` +{% include copy.html %} + +The `serviceSigningName` can be `es` for an Elasticsearch or OpenSearch domain. + +All of these authentication options apply to both source and target clusters. + +## Snapshot options + +The following configuration options customize the process of migrating from snapshots. + +### Snapshot of a managed service source + +If your source cluster is on Amazon OpenSearch Service, you need to set up an additional AWS Identity and Access Management (IAM) role and pass it with the snapshot creation call, as described in the [AWS documentation](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-snapshots.html). Migration Assistant can automatically manage this process. OpenSearch Service snapshots are only compatible with AWS Signature Version 4 authentication. The following parameter ensures that the additional IAM role is created and passed. + +| Name | Example | Description | +| :--- | :--- | :--- | +| `managedServiceSourceSnapshotEnabled` | `true` | Creates the necessary roles and trust relationships for taking a snapshot of an OpenSearch Service source cluster. This is only compatible with AWS Signature Version 4 authentication.| + +### Bring your own snapshot + +You can use an existing Amazon Simple Storage Service (Amazon S3) snapshot to perform [metadata]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/) and [backfill]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/) migrations instead of using Migration Assistant to create a snapshot: + +```json + "snapshot": { + "snapshotName": "my-snapshot-name", + "snapshotRepoName": "my-snapshot-repo", + "s3Uri": "s3://my-s3-bucket-name/my-bucket-path-to-snapshot-repo", + "s3Region": "us-east-2" + } +``` +{% include copy.html %} + +By default, Amazon S3 buckets automatically allow roles in the same AWS account (with the appropriate `s3:*` permissions) to access the S3 bucket, regardless of the bucket's AWS Region. If the external S3 bucket is in the same AWS account as the Migration Assistant deployment, no further IAM configuration is required to access the bucket. + +If you use a custom permission model with Amazon S3, any access control list (ACL) or custom bucket policy should allow the Migration Assistant task roles for RFS and the migration console to read from the S3 bucket. + +If the S3 bucket is in a separate AWS account from the Migration Assistant deployment, you need a custom bucket policy similar to the following to allow access to Migration Assistant: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowExternalAccountReadAccessToBucket", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::root" + }, + "Action": [ + "s3:GetObject", + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::my-s3-bucket-name", + "arn:aws:s3:::my-s3-bucket-name/*" + ] + } + ] +} +``` +{% include copy.html %} + +## Network configuration + +The migration tooling expects the source cluster, target cluster, and migration resources to exist in the same VPC. If this is not the case, manual networking setup outside of this documentation is likely required. diff --git a/_migration-assistant/migration-phases/deploying-migration-assistant/getting-started-data-migration.md b/_migration-assistant/migration-phases/deploying-migration-assistant/getting-started-data-migration.md new file mode 100644 index 00000000000..d0592e2dd24 --- /dev/null +++ b/_migration-assistant/migration-phases/deploying-migration-assistant/getting-started-data-migration.md @@ -0,0 +1,360 @@ +--- +layout: default +title: Getting started with data migration +parent: Deploying Migration Assistant +nav_order: 10 +permalink: /migration-assistant/deploying-migration-assistant/getting-started-data-migration/ +redirect_from: + - /upgrade-to/snapshot-migrate/ + - /migration-assistant/getting-started-with-data-migration/ +--- + +# Getting started with data migration + +This quickstart outlines how to deploy Migration Assistant for OpenSearch and execute an existing data migration using `Reindex-from-Snapshot` (RFS). It uses AWS for illustrative purposes. However, the steps can be modified for use with other cloud providers. + +## Prerequisites and assumptions + +Before using this quickstart, make sure you fulfill the following prerequisites: + +* Verify that your migration path [is supported]({{site.url}}{{site.baseurl}}/migration-assistant/overview/is-migration-assistant-right-for-you/#supported-migration-paths). Note that we test with the exact versions specified, but you should be able to migrate data on alternative minor versions as long as the major version is supported. +* The source cluster must be deployed Amazon Simple Storage Service (Amazon S3) plugin. +* The target cluster must be deployed. +* Verify that the `CDKToolkit` stack exists and is set to `CREATE_COMPLETE`. For more information about how to bootstrap your AWS account in the required AWS Region, see [the CDKToolkit documentation](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html). + +The steps in this guide assume the following: + +* In this guide, a snapshot will be taken and stored in Amazon S3; the following assumptions are made about this snapshot: + * The `_source` flag is enabled on all indexes to be migrated. + * The snapshot includes the global cluster state (`include_global_state` is `true`). + * Shard sizes of up to approximately 80 GB are supported. Larger shards cannot be migrated. If this presents challenges for your migration, contact the [migration team](https://opensearch.slack.com/archives/C054JQ6UJFK). +* Migration Assistant will be installed in the same AWS Region and have access to both the source snapshot and target cluster. + +--- + +## Step 1: Install Bootstrap on an Amazon EC2 instance (~10 minutes) + +To begin your migration, use the following steps to install a `bootstrap` box on an Amazon Elastic Compute Cloud (Amazon EC2) instance. The instance uses AWS CloudFormation to create and manage the stack. + +1. Log in to the target AWS account in which you want to deploy Migration Assistant. +2. From the browser where you are logged in to your target AWS account, right-click [here](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?templateURL=https://solutions-reference.s3.amazonaws.com/migration-assistant-for-amazon-opensearch-service/latest/migration-assistant-for-amazon-opensearch-service.template&redirectId=SolutionWeb) to load the CloudFormation template from a new browser tab. +3. Follow the CloudFormation stack wizard: + * **Stack Name:** `MigrationBootstrap` + * **Stage Name:** `dev` + * Choose **Next** after each step > **Acknowledge** > **Submit**. +4. Verify that the Bootstrap stack exists and is set to `CREATE_COMPLETE`. This process takes around 10 minutes to complete. + +--- + +## Step 2: Set up Bootstrap instance access (~5 minutes) + +Use the following steps to set up Bootstrap instance access: + +1. After deployment, find the EC2 instance ID for the `bootstrap-dev-instance`. +2. Create an AWS Identity and Access Management (IAM) policy using the following snippet, replacing ``, ``, ``, and `` with your information: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ssm:StartSession", + "Resource": [ + "arn:aws:ec2:::instance/", + "arn:aws:ssm:::document/BootstrapShellDoc--" + ] + } + ] + } + ``` + {% include copy.html %} + +3. Name the policy, for example, `SSM-OSMigrationBootstrapAccess`, and then create the policy by selecting **Create policy**. +4. Attach the newly created policy to your EC2 instance's IAM role. + +--- + +## Step 3: Log in to Bootstrap and building Migration Assistant (~15 minutes) + +Next, log in to Bootstrap and build Migration Assistant using the following steps. + +### Prerequisites + +To use these steps, make sure you fulfill the following prerequisites: + +* The AWS Command Line Interface (AWS CLI) and AWS Session Manager plugin are installed on your instance. +* The AWS credentials are configured (`aws configure`) for your instance. + +### Steps + +1. Load AWS credentials into your terminal. +2. Log in to the instance using the following command, replacing `` and `` with your instance ID and Region: + + ```bash + aws ssm start-session --document-name BootstrapShellDoc-- --target --region [--profile ] + ``` + {% include copy.html %} + +3. Once logged in, run the following command from the shell of the Bootstrap instance in the `/opensearch-migrations` directory: + + ```bash + ./initBootstrap.sh && cd deployment/cdk/opensearch-service-migration + ``` + {% include copy.html %} + +4. After a successful build, note the path for infrastructure deployment, which will be used in the next step. + +--- + +## Step 4: Configure and deploy RFS (~20 minutes) + +To deploy Migration Assistant with RFS, the following stacks must be deployed: + +These commands deploy the following stacks: + +* `Migration Assistant network` stack +* `RFS` stack +* `Migration console` stack + +Use the following steps to configure and deploy RFS, deploy Migration Assistant, and verify installation of the required stacks: + +1. Add the source and target cluster password as separate **Secrets** in [AWS Secrets Manager](https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html) as an unstructured string. Be sure to copy the secret Amazon Resource Name (ARN) for use during deployment. +2. From the same shell as the Bootstrap instance, modify the `cdk.context.json` file located in the `/opensearch-migrations/deployment/cdk/opensearch-service-migration` directory and configure the following settings: + + ```json + { + "default": { + "stage": "dev", + "vpcId": "", + "targetCluster": { + "endpoint": "", + "auth": { + "type": "basic", + "username": "", + "passwordFromSecretArn": "" + } + }, + "sourceCluster": { + "endpoint": "", + "version": "", + "auth": { + "type": "basic", + "username": "", + "passwordFromSecretArn": "" + } + }, + "reindexFromSnapshotExtraArgs": "", + "reindexFromSnapshotMaxShardSizeGiB": 80, + "otelCollectorEnabled": true, + "migrationConsoleServiceEnabled": true + } + } + ``` + {% include copy.html %} + + The source and target cluster authorization can be configured to have no authorization, `basic` with a username and password, or `sigv4`. + +3. After the `cdk.context.json` file is fully configured, bootstrap the account and deploy the required stacks using the following command: + + ```bash + cdk bootstrap --c contextId=default --require-approval never + ``` + {% include copy.html %} + +4. Deploy Migration Assistant using the following command: + + ```bash + cdk deploy "*" --c contextId=default --require-approval never --concurrency 5 + ``` + {% include copy.html %} + +5. From the same Bootstrap instance shell, verify that all CloudFormation stacks were installed successfully: + + ```bash + aws cloudformation list-stacks --query "StackSummaries[?StackStatus!='DELETE_COMPLETE'].[StackName,StackStatus]" --output table + ``` + {% include copy.html %} + +You should receive a similar output for your Region: + +```bash +------------------------------------------------------------------------ +| ListStacks | ++--------------------------------------------------+-------------------+ +| OSMigrations-dev-us-east-1-MigrationConsole | CREATE_COMPLETE | +| OSMigrations-dev-us-east-1-ReindexFromSnapshot | CREATE_COMPLETE | +| OSMigrations-dev-us-east-1-MigrationInfra | CREATE_COMPLETE | +| OSMigrations-dev-us-east-1-default-NetworkInfra | CREATE_COMPLETE | +| MigrationBootstrap | CREATE_COMPLETE | +| CDKToolkit | CREATE_COMPLETE | ++--------------------------------------------------+-------------------+ +``` + +### RFS parameters + +If you're creating a snapshot using migration tooling, these parameters are automatically configured. If you're using an existing snapshot, modify the `reindexFromSnapshotExtraArgs` setting with the following values: + +```bash + "reindexFromSnapshotExtraArgs": "--s3-repo-uri s3:/// --s3-region --snapshot-name " +``` + +You will also need to give the `migrationconsole` and `reindexFromSnapshot` TaskRoles permissions to the S3 bucket. + +--- + +## Step 5: Access the migration console + +Run the following command to access the migration console: + +```bash +./accessContainer.sh migration-console dev +``` +{% include copy.html %} + + +`accessContainer.sh` is located in `/opensearch-migrations/deployment/cdk/opensearch-service-migration/` on the Bootstrap instance. To learn more, see [Accessing the migration console]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/). +{: .note} + +--- + +## Step 6: Verify the connection to the source and target clusters + +To verify the connection to the clusters, run the following command: + +```bash +console clusters connection-check +``` +{% include copy.html %} + +You should receive the following output: + +```bash +SOURCE CLUSTER +ConnectionResult(connection_message='Successfully connected!', connection_established=True, cluster_version='') +TARGET CLUSTER +ConnectionResult(connection_message='Successfully connected!', connection_established=True, cluster_version='') +``` + +To learn more about migration console commands, see [Migration commands]. + +--- + +## Step 7: Create a snapshot + +Run the following command to initiate snapshot creation from the source cluster: + +```bash +console snapshot create [...] +``` +{% include copy.html %} + +To check the snapshot creation status, run the following command: + +```bash +console snapshot status [...] +``` +{% include copy.html %} + +To learn more information about the snapshot, run the following command: + +```bash +console snapshot status --deep-check [...] +``` +{% include copy.html %} + +Wait for snapshot creation to complete before moving to step 9. + +To learn more about snapshot creation, see [Snapshot Creation]. + +--- + +## Step 8: Migrate metadata + +Run the following command to migrate metadata: + +```bash +console metadata migrate [...] +``` +{% include copy.html %} + +For more information, see [Migrating metadata]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/). + +--- + +## Step 9: Migrate documents with RFS + +You can now use RFS to migrate documents from your original cluster: + +1. To start the migration from RFS, start a `backfill` using the following command: + + ```bash + console backfill start + ``` + {% include copy.html %} + +2. _(Optional)_ To speed up the migration, increase the number of documents processed at a simultaneously by using the following command: + + ```bash + console backfill scale + ``` + {% include copy.html %} + +3. To check the status of the documentation backfill, use the following command: + + ```bash + console backfill status + ``` + {% include copy.html %} + +4. If you need to stop the backfill process, use the following command: + + ```bash + console backfill stop + ``` + {% include copy.html %} + +For more information, see [Backfill]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/). + +--- + +## Step 10: Backfill monitoring + +Use the following command for detailed monitoring of the backfill process: + +```bash +console backfill status --deep-check +``` +{% include copy.html %} + +You should receive the following output: + +```json +BackfillStatus.RUNNING +Running=9 +Pending=1 +Desired=10 +Shards total: 62 +Shards completed: 46 +Shards incomplete: 16 +Shards in progress: 11 +Shards unclaimed: 5 +``` + +Logs and metrics are available in Amazon CloudWatch in the `OpenSearchMigrations` log group. + +--- + +## Step 11: Verify that all documents were migrated + +Use the following query in CloudWatch Logs Insights to identify failed documents: + +```bash +fields @message +| filter @message like "Bulk request succeeded, but some operations failed." +| sort @timestamp desc +| limit 10000 +``` +{% include copy.html %} + +If any failed documents are identified, you can index the failed documents directly as opposed to using RFS. diff --git a/_migration-assistant/migration-phases/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md b/_migration-assistant/migration-phases/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md new file mode 100644 index 00000000000..5ed62887838 --- /dev/null +++ b/_migration-assistant/migration-phases/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md @@ -0,0 +1,93 @@ +--- +layout: default +title: IAM and security groups for existing clusters +nav_order: 20 +parent: Deploying Migration Assistant +permalink: /migration-assistant/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters/ +--- + +# IAM and security groups for existing clusters + +This page outlines security scenarios for using the migration tools with existing clusters, including any necessary configuration changes to ensure proper communication between them. + +## Importing an Amazon OpenSearch Service or Amazon OpenSearch Serverless target cluster + +Use the following scenarios for Amazon OpenSearch Service or Amazon OpenSearch Serverless target clusters. + +### OpenSearch Service + +For an OpenSearch Domain, two main configurations are typically required to ensure proper functioning of the migration solution: + +1. **Security Group Configuration** + + The domain should have a security group that allows communication from the applicable migration services (Traffic Replayer, Migration Console, `Reindex-from-Snapshot`). The CDK automatically creates an `osClusterAccessSG` security group, which is applied to the migration services. The user should then add this security group to their existing domain to allow access. + +2. **Access Policy Configuration** should be one of the following: + - An open access policy that allows all access. + - Configured to allow at least the AWS Identity and Access Management (IAM) task roles for the applicable migration services (Traffic Replayer, Migration Console, `Reindex-from-Snapshot`) to access the domain. + +### Managed service role mapping (Cross-managed-cluster migrations) + +When migrating between two managed clusters, for example, when both domains were created using Amazon OpenSearch Service, provide Migration Assistant components with sufficient permissions to modify both the source and target clusters. + +Use the following steps to grant the required permissions: + +1. In the AWS Management Console, navigate to **CloudFormation** > **Stacks**. +2. Locate the stack that starts with `OSMigrations--` (created during CDK deployment). +3. Go to the **Resources** tab and locate the following IAM roles: + + ```bash + arn:aws:iam::****:role/OSMigrations---MigrationServiceTaskRoleC- + arn:aws:iam::****:role/OSMigrations---reindexfromsnapshotTaskRo- + arn:aws:iam::****:role/OSMigrations---trafficreplayerdefaultTas- + ``` + +4. In both the source and target clusters, map users to each Amazon Resource Name (ARN) using the following steps: + A. Access OpenSearch Dashboards. If you're using Elasticsearch, access Kibana. + B. Navigate to **Security -> Roles -> all_access**. + C. In the "Mapped users" section, add each ARN as a backend role. + D. Save your changes. + +### OpenSearch Serverless + +For an OpenSearch Serverless Collection, you will need to configure both network and data access policies: + +1. **Network Policy Configuration**: + The Collection should have a network policy that uses the `VPC` access type. This requires creating a VPC endpoint on the VPC used for the solution. The VPC endpoint should be configured for the private subnets of the VPC and should attach the `osClusterAccessSG` security group. + +2. **Data Access Policy Configuration**: + The data access policy should grant permission to perform all [index operations](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-data-access.html#serverless-data-supported-permissions) (`aoss:*`) for all indexes in the Collection. The IAM task roles of the applicable Migration services (Traffic Replayer, migration console, `Reindex-from-Snapshot`) should be used as the principals for this data access policy. + +## Capture Proxy on Coordinator Nodes of Source Cluster + +Although the CDK does not automatically set up the Capture Proxy on source cluster nodes (except in the demo solution), the Capture Proxy instances must communicate with the resources deployed by the CDK, such as Kafka. This section outlines the necessary steps to set up communication. + +Before [setting up Capture Proxy instances](https://github.com/opensearch-project/opensearch-migrations/tree/main/TrafficCapture/trafficCaptureProxyServer#installing-capture-proxy-on-coordinator-nodes) on the source cluster, ensure the following configurations are in place: + +1. **Security Group Configuration**: + The coordinator nodes should add the `trafficStreamSourceSG` security group to allow sending captured traffic to Kafka. + +2. **IAM Policy Configuration**: + The IAM role used by the coordinator nodes should have permissions to publish captured traffic to Kafka. You can add the following template policy through the AWS Console (IAM Role → Add permissions → Create inline policy → JSON view): + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "kafka-cluster:Connect", + "Resource": "arn:aws:kafka:::cluster/migration-msk-cluster-/*", + "Effect": "Allow" + }, + { + "Action": [ + "kafka-cluster:CreateTopic", + "kafka-cluster:DescribeTopic", + "kafka-cluster:WriteData" + ], + "Resource": "arn:aws:kafka:::topic/migration-msk-cluster-/*", + "Effect": "Allow" + } + ] +} +``` diff --git a/_migration-assistant/migration-phases/deploying-migration-assistant/index.md b/_migration-assistant/migration-phases/deploying-migration-assistant/index.md new file mode 100644 index 00000000000..1c559a81b1c --- /dev/null +++ b/_migration-assistant/migration-phases/deploying-migration-assistant/index.md @@ -0,0 +1,13 @@ +--- +layout: default +title: Deploying Migration Assistant +nav_order: 15 +has_children: true +permalink: /deploying-migration-assistant/ +redirect-from: + - /deploying-migration-assistant/index/ +--- + +# Deploying Migration Assistant + +This section provides information about the available options for deploying Migration Assistant. diff --git a/_migration-assistant/migration-phases/deployment.md b/_migration-assistant/migration-phases/deployment.md new file mode 100644 index 00000000000..2c4da875663 --- /dev/null +++ b/_migration-assistant/migration-phases/deployment.md @@ -0,0 +1,341 @@ +--- +layout: default +title: Deploy +parent: Migration phases +nav_order: 2 +redirect_from: + - /upgrade-to/snapshot-migrate/ + - /migration-assistant/getting-started-with-data-migration/ +--- + +# Deploying Migration Assistant +This document assumes you have performed assessment. + +--- + +## Step 1: Install Bootstrap on an Amazon EC2 instance (~10 minutes) + +To begin your migration, use the following steps to install a `bootstrap` box on an Amazon Elastic Compute Cloud (Amazon EC2) instance. The instance uses AWS CloudFormation to create and manage the stack. + +1. Log in to the target AWS account in which you want to deploy Migration Assistant. +2. From the browser where you are logged in to your target AWS account, right-click [here](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?templateURL=https://solutions-reference.s3.amazonaws.com/migration-assistant-for-amazon-opensearch-service/latest/migration-assistant-for-amazon-opensearch-service.template&redirectId=SolutionWeb) to load the CloudFormation template from a new browser tab. +3. Follow the CloudFormation stack wizard: + * **Stack Name:** `MigrationBootstrap` + * **Stage Name:** `dev` + * Choose **Next** after each step > **Acknowledge** > **Submit**. +4. Verify that the Bootstrap stack exists and is set to `CREATE_COMPLETE`. This process takes around 10 minutes to complete. + +--- + +## Step 2: Set up Bootstrap instance access (~5 minutes) + +Use the following steps to set up Bootstrap instance access: + +1. After deployment, find the EC2 instance ID for the `bootstrap-dev-instance`. +2. Create an AWS Identity and Access Management (IAM) policy using the following snippet, replacing ``, ``, ``, and `` with your information: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ssm:StartSession", + "Resource": [ + "arn:aws:ec2:::instance/", + "arn:aws:ssm:::document/BootstrapShellDoc--" + ] + } + ] + } + ``` + {% include copy.html %} + +3. Name the policy, for example, `SSM-OSMigrationBootstrapAccess`, and then create the policy by selecting **Create policy**. +4. Attach the newly created policy to your EC2 instance's IAM role. + +--- + +## Step 3: Log in to Bootstrap and building Migration Assistant (~15 minutes) + +Next, log in to Bootstrap and build Migration Assistant using the following steps. + +### Prerequisites + +To use these steps, make sure you fulfill the following prerequisites: + +* The AWS Command Line Interface (AWS CLI) and AWS Session Manager plugin are installed on your instance. +* The AWS credentials are configured (`aws configure`) for your instance. + +### Steps + +1. Load AWS credentials into your terminal. +2. Log in to the instance using the following command, replacing `` and `` with your instance ID and Region: + + ```bash + aws ssm start-session --document-name BootstrapShellDoc-- --target --region [--profile ] + ``` + {% include copy.html %} + +3. Once logged in, run the following command from the shell of the Bootstrap instance in the `/opensearch-migrations` directory: + + ```bash + ./initBootstrap.sh && cd deployment/cdk/opensearch-service-migration + ``` + {% include copy.html %} + +4. After a successful build, note the path for infrastructure deployment, which will be used in the next step. + +--- + +## Step 4: Configure and deploy RFS (~20 minutes) + +To deploy Migration Assistant with RFS, the following stacks must be deployed: + +These commands deploy the following stacks: + +* `Migration Assistant network` stack +* `RFS` stack +* `Migration console` stack + +Use the following steps to configure and deploy RFS, deploy Migration Assistant, and verify installation of the required stacks: + +1. Add the source and target cluster password as separate **Secrets** in [AWS Secrets Manager](https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html) as an unstructured string. Be sure to copy the secret Amazon Resource Name (ARN) for use during deployment. +2. From the same shell as the Bootstrap instance, modify the `cdk.context.json` file located in the `/opensearch-migrations/deployment/cdk/opensearch-service-migration` directory and configure the following settings: + + ```json + { + "default": { + "stage": "dev", + "vpcId": "", + "targetCluster": { + "endpoint": "", + "auth": { + "type": "basic", + "username": "", + "passwordFromSecretArn": "" + } + }, + "sourceCluster": { + "endpoint": "", + "version": "", + "auth": { + "type": "basic", + "username": "", + "passwordFromSecretArn": "" + } + }, + "reindexFromSnapshotExtraArgs": "", + "reindexFromSnapshotMaxShardSizeGiB": 80, + "otelCollectorEnabled": true, + "migrationConsoleServiceEnabled": true + } + } + ``` + {% include copy.html %} + + The source and target cluster authorization can be configured to have no authorization, `basic` with a username and password, or `sigv4`. + +3. After the `cdk.context.json` file is fully configured, bootstrap the account and deploy the required stacks using the following command: + + ```bash + cdk bootstrap --c contextId=default --require-approval never + ``` + {% include copy.html %} + +4. Deploy Migration Assistant using the following command: + + ```bash + cdk deploy "*" --c contextId=default --require-approval never --concurrency 5 + ``` + {% include copy.html %} + +5. From the same Bootstrap instance shell, verify that all CloudFormation stacks were installed successfully: + + ```bash + aws cloudformation list-stacks --query "StackSummaries[?StackStatus!='DELETE_COMPLETE'].[StackName,StackStatus]" --output table + ``` + {% include copy.html %} + +You should receive a similar output for your Region: + +```bash +------------------------------------------------------------------------ +| ListStacks | ++--------------------------------------------------+-------------------+ +| OSMigrations-dev-us-east-1-MigrationConsole | CREATE_COMPLETE | +| OSMigrations-dev-us-east-1-ReindexFromSnapshot | CREATE_COMPLETE | +| OSMigrations-dev-us-east-1-MigrationInfra | CREATE_COMPLETE | +| OSMigrations-dev-us-east-1-default-NetworkInfra | CREATE_COMPLETE | +| MigrationBootstrap | CREATE_COMPLETE | +| CDKToolkit | CREATE_COMPLETE | ++--------------------------------------------------+-------------------+ +``` + +### RFS parameters + +If you're creating a snapshot using migration tooling, these parameters are automatically configured. If you're using an existing snapshot, modify the `reindexFromSnapshotExtraArgs` setting with the following values: + +```bash + "reindexFromSnapshotExtraArgs": "--s3-repo-uri s3:/// --s3-region --snapshot-name " +``` + +You will also need to give the `migrationconsole` and `reindexFromSnapshot` TaskRoles permissions to the S3 bucket. + +--- + +## Step 5: Access the migration console + +Run the following command to access the migration console: + +```bash +./accessContainer.sh migration-console dev +``` +{% include copy.html %} + + +`accessContainer.sh` is located in `/opensearch-migrations/deployment/cdk/opensearch-service-migration/` on the Bootstrap instance. To learn more, see [Accessing the migration console]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/). +{: .note} + +--- + +## Step 6: Verify the connection to the source and target clusters + +To verify the connection to the clusters, run the following command: + +```bash +console clusters connection-check +``` +{% include copy.html %} + +You should receive the following output: + +```bash +SOURCE CLUSTER +ConnectionResult(connection_message='Successfully connected!', connection_established=True, cluster_version='') +TARGET CLUSTER +ConnectionResult(connection_message='Successfully connected!', connection_established=True, cluster_version='') +``` + +To learn more about migration console commands, see [Migration console command reference](https://docs.opensearch.org/docs/latest/migration-assistant/migration-console/migration-console-commands-references/). + +--- + +## Step 7: Create a snapshot + +Run the following command to initiate snapshot creation from the source cluster: + +```bash +console snapshot create [...] +``` +{% include copy.html %} + +To check the snapshot creation status, run the following command: + +```bash +console snapshot status [...] +``` +{% include copy.html %} + +To learn more information about the snapshot, run the following command: + +```bash +console snapshot status --deep-check [...] +``` +{% include copy.html %} + +Wait for snapshot creation to complete before moving to step 9. + +To learn more about snapshot creation, see [Snapshot Creation]. + +--- + +## Step 8: Migrate metadata + +Run the following command to migrate metadata: + +```bash +console metadata migrate [...] +``` +{% include copy.html %} + +For more information, see [Migrating metadata]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/). + +--- + +## Step 9: Migrate documents with RFS + +You can now use RFS to migrate documents from your original cluster: + +1. To start the migration from RFS, start a `backfill` using the following command: + + ```bash + console backfill start + ``` + {% include copy.html %} + +2. _(Optional)_ To speed up the migration, increase the number of documents processed at a simultaneously by using the following command: + + ```bash + console backfill scale + ``` + {% include copy.html %} + +3. To check the status of the documentation backfill, use the following command: + + ```bash + console backfill status + ``` + {% include copy.html %} + +4. If you need to stop the backfill process, use the following command: + + ```bash + console backfill stop + ``` + {% include copy.html %} + +For more information, see [Backfill]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/). + +--- + +## Step 10: Backfill monitoring + +Use the following command for detailed monitoring of the backfill process: + +```bash +console backfill status --deep-check +``` +{% include copy.html %} + +You should receive the following output: + +```json +BackfillStatus.RUNNING +Running=9 +Pending=1 +Desired=10 +Shards total: 62 +Shards completed: 46 +Shards incomplete: 16 +Shards in progress: 11 +Shards unclaimed: 5 +``` + +Logs and metrics are available in Amazon CloudWatch in the `OpenSearchMigrations` log group. + +--- + +## Step 11: Verify that all documents were migrated + +Use the following query in CloudWatch Logs Insights to identify failed documents: + +```bash +fields @message +| filter @message like "Bulk request succeeded, but some operations failed." +| sort @timestamp desc +| limit 10000 +``` +{% include copy.html %} + +If any failed documents are identified, you can index the failed documents directly as opposed to using RFS. diff --git a/_migration-assistant/migration-phases/index.md b/_migration-assistant/migration-phases/index.md new file mode 100644 index 00000000000..8a7dcc95ab7 --- /dev/null +++ b/_migration-assistant/migration-phases/index.md @@ -0,0 +1,87 @@ +--- +layout: default +title: Migration phases +parent: Migration Assistant for OpenSearch +nav_order: 30 +has_children: false +has_toc: true +permalink: /migration-assistant/overview/ +redirect-from: /migration-assistant/overview/index/ +--- + +# Migration phases + +This page outlines the core phases of migrating with Migration Assistant. Each scenario below consists of a sequence of common steps. Expand a scenario to explore the detailed flow. + +--- + + + +
    +Scenario 1 – Backfill Only + +
      +
    1. Assessment
      2. +
    2. Deployment
      3. + +
    3. Create Snapshot
      4. +
    4. Migrate Metadata
      5. +
    5. Migrate Data
      6. +
    6. Teardown
      7. +
    + +
    + +
    +Scenario 2 – Live Capture Only + +
      +
    1. Assessment
      2. +
    2. Deployment
      3. +
    3. Verify Backfill Components
      4. +
    4. Reroute Traffic from Source to Capture Proxy
      5. +
    5. Migrate Metadata
      6. +
    6. Verify Live Capture Components
      7. +
    7. Replay Captured Traffic
      8. +
    8. Reroute Traffic from Capture Proxy to Target
      9. +
    9. Teardown
      10. +
    + +
    + +
    +Scenario 3 – Live Capture with Backfill + +
      +
    1. Assessment
      2. +
    2. Deployment
      3. +
    3. Reroute Traffic from Source to Capture Proxy
      4. +
    4. Create Snapshot
      5. +
    5. Migrate Metadata
      6. +
    6. Migrate Data
      7. +
    7. Replay Traffic
      8. +
    8. Reroute Traffic from Capture Proxy to Target
      9. +
    9. Teardown
      10. +
    + +
    + +--- + +💡 *Need help getting started? Begin with [Planning your migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/planning-your-migration/index/).* diff --git a/_migration-assistant/migration-phases/migrate-metadata.md b/_migration-assistant/migration-phases/migrate-metadata.md new file mode 100644 index 00000000000..39c1d7d1149 --- /dev/null +++ b/_migration-assistant/migration-phases/migrate-metadata.md @@ -0,0 +1,249 @@ +--- +layout: default +title: Migrate Metadata +nav_order: 5 +parent: Migration phases +redirect_from: + - /migration-assistant/migration-phases/migrating-metadata +--- + +# Migrate metadata + +Metadata migration involves creating a snapshot of your cluster and then migrating the metadata from the snapshot using the migration console. + +This tool gathers information from a source cluster through a snapshot or through HTTP requests against the source cluster. These snapshots are fully compatible with the backfill process for `Reindex-From-Snapshot` (RFS) scenarios. + +After collecting information on the source cluster, comparisons are made against the target cluster. If running a migration, any metadata items that do not already exist will be created on the target cluster. + +## Creating the snapshot + +Creating a snapshot of the source cluster captures all the metadata and documents to be migrated to a new target cluster. + +Create the initial snapshot of the source cluster using the following command: + +```shell +console snapshot create +``` +{% include copy.html %} + +To check the progress of the snapshot in real time, use the following command: + +```shell +console snapshot status --deep-check +``` +{% include copy.html %} + +You should receive the following response when the snapshot is created: + +```shell +SUCCESS +Snapshot is SUCCESS. +Percent completed: 100.00% +Data GiB done: 29.211/29.211 +Total shards: 40 +Successful shards: 40 +Failed shards: 0 +Start time: 2024-07-22 18:21:42 +Duration: 0h 13m 4s +Anticipated duration remaining: 0h 0m 0s +Throughput: 38.13 MiB/sec +``` + +### Managing slow snapshot speeds + +Depending on the size of the data in the source cluster and the bandwidth allocated for snapshots, the process can take some time. Adjust the maximum rate at which the source cluster's nodes create the snapshot using the `--max-snapshot-rate-mb-per-node` option. Increasing the snapshot rate will consume more node resources, which may affect the cluster's ability to handle normal traffic. + +## Command arguments + +For the following commands, to identify all valid arguments, please run with `--help`. + +```shell +console metadata evaluate --help +``` +{% include copy.html %} + +```shell +console metadata migrate --help +``` +{% include copy.html %} + +Based on the migration console deployment options, a number of commands will be pre-populated. To view them, run console with verbosity: + +```shell +console -v metadata migrate --help +``` +{% include copy.html %} + +You should receive a response similar to the following: + +```shell +(.venv) bash-5.2# console -v metadata migrate --help +INFO:console_link.cli:Logging set to INFO +. +. +. +INFO:console_link.models.metadata:Migrating metadata with command: /root/metadataMigration/bin/MetadataMigration --otel-collector-endpoint http://otel-collector:4317 migrate --snapshot-name snapshot_2023_01_01 --target-host https://opensearchtarget:9200 --min-replicas 0 --file-system-repo-path /snapshot/test-console --target-username admin --target-password ******** --target-insecure --help +. +. +. +``` + + +## Using the `evaluate` command + +By scanning the contents of the source cluster, applying filtering, and applying modifications a list of all items that will be migrated will be created. Any items not seen in this output will not be migrated onto the target cluster if the migrate command was to be run. This is a safety check before making modifications on the target cluster. + +```shell +console metadata evaluate [...] +``` +{% include copy.html %} + +You should receive a response similar to the following: + +```bash +Starting Metadata Evaluation +Clusters: + Source: + Remote Cluster: OpenSearch 1.3.16 ConnectionContext(uri=http://localhost:33039, protocol=HTTP, insecure=false, compressionSupported=false) + + Target: + Remote Cluster: OpenSearch 2.14.0 ConnectionContext(uri=http://localhost:33037, protocol=HTTP, insecure=false, compressionSupported=false) + + +Migration Candidates: + Index Templates: + simple_index_template + + Component Templates: + simple_component_template + + Indexes: + blog_2023, movies_2023 + + Aliases: + alias1, movies-alias + + +Results: + 0 issue(s) detected +``` + + +## Using the migrate command + +Running through the same data as the evaluate command all of the migrated items will be applied onto the target cluster. If re-run multiple times items that were previously migrated will not be recreated. If any items do need to be re-migrated, please delete them from the target cluster and then rerun the evaluate then migrate commands to ensure the desired changes are made. + +```shell +console metadata migrate [...] +``` +{% include copy.html %} + +You should receive a response similar to the following: + +```shell +Starting Metadata Migration + +Clusters: + Source: + Snapshot: OpenSearch 1.3.16 FileSystemRepo(repoRootDir=/tmp/junit10626813752669559861) + + Target: + Remote Cluster: OpenSearch 2.14.0 ConnectionContext(uri=http://localhost:33042, protocol=HTTP, insecure=false, compressionSupported=false) + + +Migrated Items: + Index Templates: + simple_index_template + + Component Templates: + simple_component_template + + Indexes: + blog_2023, movies_2023 + + Aliases: + alias1, movies-alias + + +Results: + 0 issue(s) detected +``` + + +## Metadata verification process + +Before moving on to additional migration steps, it is recommended to confirm details of your cluster. Depending on your configuration, this could be checking the sharding strategy or making sure index mappings are correctly defined by ingesting a test document. + +## Troubleshooting + +Use these instructions to help troubleshoot the following issues. + +### Accessing detailed logs + +Metadata migration creates a detailed log file that includes low level tracing information for troubleshooting. For each execution of the program a log file is created inside a shared volume on the migration console named `shared-logs-output` the following command will list all log files, one for each run of the command. + +```shell +ls -al /shared-logs-output/migration-console-default/*/metadata/ +``` +{% include copy.html %} + +To inspect the file within the console `cat`, `tail` and `grep` commands line tools. By looking for warnings, errors and exceptions in this log file can help understand the source of failures, or at the very least be useful for creating issues in this project. + +```shell +tail /shared-logs-output/migration-console-default/*/metadata/*.log +``` +{% include copy.html %} + +### Warnings and errors + +When encountering `WARN` or `ERROR` elements in the response, they will be accompanied by a short message, such as `WARN - my_index already exists`. More information can be found in the detailed logs associated with the warning or error. + +### OpenSearch running in compatibility mode + +There might be an error about being unable to update an ES 7.10.2 cluster, this can occur when compatibility mode has been enabled on an OpenSearch cluster disable it to continue, see [Enable compatibility mode](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/rename.html#rename-upgrade). + + +### Breaking change compatibility + +Metadata migration requires modifying data from the source to the target versions to recreate items. Sometimes these features are no longer supported and have been removed from the target version. Sometimes these features are not available in the target version, which is especially true when downgrading. While this tool is meant to make this process easier, it is not exhaustive in its support. When encountering a compatibility issue or an important feature gap for your migration, [search the issues and comment on the existing issue](https://github.com/opensearch-project/opensearch-migrations/issues) or [create a new](https://github.com/opensearch-project/opensearch-migrations/issues/new/choose) issue if one cannot be found. + +#### Deprecation of Mapping Types + +In Elasticsearch 6.8 the mapping types feature was discontinued in Elasticsearch 7.0+ which has created complexity in migrating to newer versions of Elasticsearch and OpenSearch, [learn more](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html) ↗. + +As Metadata migration supports migrating from ES 6.8 on to the latest versions of OpenSearch this scenario is handled by removing the type mapping types and restructuring the template or index properties. Note that, at the time of this writing multiple type mappings are not supported, [tracking task](https://opensearch.atlassian.net/browse/MIGRATIONS-1778) ↗. + + +**Example starting state with mapping type foo (ES 6):** + +```json +{ + "mappings": [ + { + "foo": { + "properties": { + "field1": { "type": "text" }, + "field2": { "type": "keyword" } + } + } + } + ] +} +``` +{% include copy.html %} + +**Example ending state with foo removed (ES 7):** + +```json +{ + "mappings": { + "properties": { + "field1": { "type": "text" }, + "field2": { "type": "keyword" }, + } + } +} +``` +{% include copy.html %} + +For additional technical details, [view the mapping type removal source code](https://github.com/opensearch-project/opensearch-migrations/blob/main/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java). diff --git a/_migration-assistant/migration-phases/replay-captured-traffic.md b/_migration-assistant/migration-phases/replay-captured-traffic.md new file mode 100644 index 00000000000..0f4d1d6ea2f --- /dev/null +++ b/_migration-assistant/migration-phases/replay-captured-traffic.md @@ -0,0 +1,319 @@ +--- +layout: default +title: Replay +nav_order: 7 +grand_parent: Migration phases +parent: Live traffic migration +redirect_from: + - /migration-assistant/migration-phases/using-traffic-replayer/ +--- + +# Using Traffic Replayer + +This guide covers how to use Traffic Replayer to replay captured traffic from a source cluster to a target cluster during the migration process. Traffic Replayer allows you to verify that the target cluster can handle requests in the same way as the source cluster and catch up to real-time traffic for a smooth migration. + +## When to run Traffic Replayer + +After deploying Migration Assistant, Traffic Replayer does not run by default. It should be started only after all metadata and documents have been migrated to ensure that recent changes to the source cluster are properly reflected in the target cluster. + +For example, if a document was deleted after a snapshot was taken, starting Traffic Replayer before the document migration is complete may cause the deletion request to execute before the document is added to the target. Running Traffic Replayer after all other migration processes ensures that the target cluster will be consistent with the source cluster. + +## Configuration options + +[Traffic Replayer settings]({{site.url}}{{site.baseurl}}/migration-assistant/deploying-migration-assistant/configuration-options/) are configured during the deployment of Migration Assistant. Make sure to set the authentication mode for Traffic Replayer so that it can properly communicate with the target cluster. + +## Using Traffic Replayer + +To manage Traffic Replayer, use the `console replay` command. The following examples show the available commands. + +### Start Traffic Replayer + +The following command starts Traffic Replayer with the options specified at deployment: + +```bash +console replay start +``` + +When starting Traffic Replayer, you should receive an output similar to the following: + +```bash +root@ip-10-0-2-66:~# console replay start +Replayer started successfully. +Service migration-dev-traffic-replayer-default set to 1 desired count. Currently 0 running and 0 pending. +``` + +## Check the status of Traffic Replayer + +Use the following command to show the status of Traffic Replayer: + +```bash +console replay status +``` + +Replay will return one of the following statuses: + +- `Running` shows how many container instances are actively running. +- `Pending` indicates how many instances are being provisione.d +- `Desired` shows the total number of instances that should be running. + +You should receive an output similar to the following: + +```bash +root@ip-10-0-2-66:~# console replay status +(, 'Running=0\nPending=0\nDesired=0') +``` + +## Stop Traffic Replayer + +The following command stops Traffic Replayer: + +```bash +console replay stop +``` + +You should receive an output similar to the following: + +```bash +root@ip-10-0-2-66:~# console replay stop +Replayer stopped successfully. +Service migration-dev-traffic-replayer-default set to 0 desired count. Currently 0 running and 0 pending. +``` + + + +### Delivery guarantees + +Traffic Replayer retrieves traffic from Kafka and updates its commit cursor after sending requests to the target cluster. This provides an "at least once" delivery guarantee; however, success isn't always guaranteed. Therefore, you should monitor metrics and tuple outputs or perform external validation to ensure that the target cluster is functioning as expected. + +## Time scaling + +Traffic Replayer sends requests in the same order that they were received from each connection to the source. However, relative timing between different connections is not guaranteed. For example: + +- **Scenario**: Two connections exist:one sends a PUT request every minute, and the other sends a GET request every second. +- **Behavior**: Traffic Replayer will maintain the sequence within each connection, but the relative timing between the connections (PUTs and GETs) is not preserved. + +Assume that a source cluster responds to requests (GETs and PUTs) within 100 ms: + +- With a **speedup factor of 1**, the target will experience the same request rates and idle periods as the source. +- With a **speedup factor of 2**, requests will be sent twice as fast, with GETs sent every 500 ms and PUTs every 30 seconds. +- With a **speedup factor of 10**, requests will be sent 10x faster, and as long as the target responds quickly, Traffic Replayer can maintain the pace. + +If the target cannot respond fast enough, Traffic Replayer will wait for the previous request to complete before sending the next one. This may cause delays and affect global relative ordering. + +## Transformations + +During migrations, some requests may need to be transformed between versions. For example, Elasticsearch previously supported multiple type mappings in indexes, but this is no longer the case in OpenSearch. Clients may need to be adjusted accordingly by splitting documents into multiple indexes or transforming request data. + +Traffic Replayer automatically rewrites host and authentication headers, but for more complex transformations, custom transformation rules can be specified using the `--transformer-config` option. For more information, see the [Traffic Replayer README](https://github.com/opensearch-project/opensearch-migrations/blob/c3d25958a44ec2e7505892b4ea30e5fbfad4c71b/TrafficCapture/trafficReplayer/README.md#transformations). + +### Example transformation + +Suppose that a source request contains a `tagToExcise` element that needs to be removed and its children promoted and that the URI path includes `extraThingToRemove`, which should also be removed. The following Jolt script handles this transformation: + +```json +[{ "JsonJoltTransformerProvider": +[ + { + "script": { + "operation": "shift", + "spec": { + "payload": { + "inlinedJsonBody": { + "top": { + "tagToExcise": { + "*": "payload.inlinedJsonBody.top.&" + }, + "*": "payload.inlinedJsonBody.top.&" + }, + "*": "payload.inlinedJsonBody.&" + }, + "*": "payload.&" + }, + "*": "&" + } + } + }, + { + "script": { + "operation": "modify-overwrite-beta", + "spec": { + "URI": "=split('/extraThingToRemove',@(1,&))" + } + } + }, + { + "script": { + "operation": "modify-overwrite-beta", + "spec": { + "URI": "=join('',@(1,&))" + } + } + } +] +}] +``` + +The resulting request sent to the target will appear similar to the following: + +```bash +PUT /oldStyleIndex/moreStuff HTTP/1.0 +host: testhostname + +{"top":{"properties":{"field1":{"type":"text"},"field2":{"type":"keyword"}}}} +``` +{% include copy.html %} + +You can pass Base64-encoded transformation scripts using `--transformer-config-base64`. + +## Result logs + +HTTP transactions from the source capture and those resent to the target cluster are logged in files located at `/shared-logs-output/traffic-replayer-default/*/tuples/tuples.log`. The `/shared-logs-output` directory is shared across containers, including the migration console. You can access these files from the migration console using the same path. Previous runs are also available in a `gzipped` format. + +Each log entry is a newline-delimited JSON object, containing information about the source and target requests/responses along with other transaction details, such as response times. + +These logs contain the contents of all requests, including authorization headers and the contents of all HTTP messages. Ensure that access to the migration environment is restricted, as these logs serve as a source of truth for determining what happened in both the source and target clusters. Response times for the source refer to the amount of time between the proxy sending the end of a request and receiving the response. While response times for the target are recorded in the same manner, keep in mind that the locations of the capture proxy, Traffic Replayer, and target may differ and that these logs do not account for the client's location. +{: .note} + + +### Example log entry + +The following example log entry shows a `/_cat/indices?v` request sent to both the source and target clusters: + +```json +{ + "sourceRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": {"keepAliveDefault": true}, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 59, + "content-type": "text/plain; charset=UTF-8", + "content-length": "214", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICAgICB..." + }, + "targetRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "targetResponses": [{ + "HTTP-Version": {"keepAliveDefault": true}, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 721, + "content-type": "text/plain; charset=UTF-8", + "content-length": "484", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICAgICB..." + }], + "connectionId": "0242acfffe13000a-0000000a-00000005-1eb087a9beb83f3e-a32794b4.0", + "numRequests": 1, + "numErrors": 0 +} +``` +{% include copy.html %} + + +### Decoding log content + +The contents of HTTP message bodies are Base64 encoded in order to handle various types of traffic, including compressed data. To view the logs in a more human-readable format, use the console library `tuples show`. Running the script as follows will produce a `readable-tuples.log` in the home directory: + +```shell +console tuples show --in /shared-logs-output/traffic-replayer-default/d3a4b31e1af4/tuples/tuples.log > readable-tuples.log +``` + +The `readable-tuples.log` should appear similar to the following: + +```json +{ + "sourceRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": {"keepAliveDefault": true}, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 59, + "content-type": "text/plain; charset=UTF-8", + "content-length": "214", + "body": "health status index uuid ..." + }, + "targetRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "targetResponses": [{ + "HTTP-Version": {"keepAliveDefault": true}, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 721, + "content-type": "text/plain; charset=UTF-8", + "content-length": "484", + "body": "health status index uuid ..." + }], + "connectionId": "0242acfffe13000a-0000000a-00000005-1eb087a9beb83f3e-a32794b4.0", + "numRequests": 1, + "numErrors": 0 +} +``` + + +## Amazon CloudWatch metrics and dashboard +Migration Assistant creates an Amazon CloudWatch dashboard named `MigrationAssistant_ReindexFromSnapshot_Dashboard` to visualize the health and performance of the backfill process. This dashboard combines metrics for the backfill workers and migration to Amazon OpenSearch Service, providing insights into the performance and health of the Capture Proxy and Traffic Replayer components, including metrics such as: + +- The number of bytes read and written. +- The number of active connections. +- The replay speed multiplier. + +You can find the Capture and Replay dashboard in the AWS Management Console for CloudWatch Dashboards in the AWS Region where you deployed Migration Assistant. + +Traffic Replayer emits various OpenTelemetry metrics to Amazon CloudWatch, and traces are sent through AWS X-Ray. The following are some useful metrics that can help evaluate migration performance. + +### `sourceStatusCode` + +This metric tracks the HTTP status codes for both the source and target clusters, with dimensions for the HTTP verb, such as `GET` or `POST`, and the status code families (200--299). These dimensions can help quickly identify discrepancies between the source and target, such as when `DELETE 200s` becomes `4xx` or `GET 4xx` errors turn into `5xx` errors. + +### `lagBetweenSourceAndTargetRequests` + +This metric shows the delay between requests hitting the source and target clusters. With a speedup factor greater than 1 and a target cluster that can handle requests efficiently, this value should decrease as the replay progresses, indicating a reduction in replay lag. + +### Additional metrics + +The following metrics are also reported: + +- **Throughput**: `bytesWrittenToTarget` and `bytesReadFromTarget` indicate the throughput to and from the cluster. +- **Retries**: `numRetriedRequests` tracks the number of requests retried due to status code mismatches between the source and target. +- **Event counts**: Various `(*)Count` metrics track the number of completed events. +- **Durations**: `(*)Duration` metrics measure the duration of each step in the process. +- **Exceptions**: `(*)ExceptionCount` shows the number of exceptions encountered during each processing phase. + + +## CloudWatch considerations + +Metrics and dashboards pushed to CloudWatch may experience a visibility lag of around 5 minutes. CloudWatch also retains higher-resolution data for a shorter period than lower-resolution data. For more information, see [Amazon CloudWatch concepts](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html). \ No newline at end of file diff --git a/_migration-assistant/migration-phases/reroute-source-to-proxy.md b/_migration-assistant/migration-phases/reroute-source-to-proxy.md new file mode 100644 index 00000000000..9bfa70bfd2b --- /dev/null +++ b/_migration-assistant/migration-phases/reroute-source-to-proxy.md @@ -0,0 +1,102 @@ +--- +layout: default +title: Reroute Client Traffic +nav_order: 3 +parent: Migration phases +redirect_from: + - /migration-assistant/migration-phases/resource-source-to-proxy +--- + +## Capture proxy data replication + +If you're interested in capturing live traffic during your migration, Migration Assistant includes an Application Load Balancer for routing traffic to the capture proxy and the target cluster. Upstream client traffic must be routed through the capture proxy in order to replay the requests later. Before using the capture proxy, remember the following: + +* The layer upstream from the Application Load Balancer is compatible with the certificate on the Application Load Balancer listener, whether it's for clients or a Network Load Balancer. The `albAcmCertArn` in the `cdk.context.json` may need to be provided to ensure that clients trust the Application Load Balancer certificate. +* If a Network Load Balancer is used directly upstream of the Application Load Balancer, it must use a TLS listener. +* Upstream resources and security groups must allow network access to the Migration Assistant Application Load Balancer. + +To set up the capture proxy, go to the AWS Management Console and navigate to **EC2 > Load Balancers > Migration Assistant Application Load Balancer**. Copy the Application Load Balancer URL. With the URL copied, you can use one of the following options. + + + +### If you are using **Network Load Balancer → Application Load Balancer → Cluster** + +1. Ensure that ingress is provided directly to the Application Load Balancer for the capture proxy. +2. Create a target group for the Migration Assistant Application Load Balancer on port `9200`, and set the health check to `HTTPS`. +3. Associate this target group with your existing Network Load Balancer on a new listener for testing. +4. Verify that the health check is successful, and perform smoke testing with some clients through the new listener port. +5. Once you are ready to migrate all clients, detach the Migration Assistant Application Load Balancer target group from the testing Network Load Balancer listener and modify the existing Network Load Balancer listener to direct traffic to this target group. +6. Now client requests will be routed through the proxy (once they establish a new connection). Verify the application metrics. + +### If you are using **Network Load Balancer → Cluster** + +If you do not want to modify application logic, add an Application Load Balancer in front of your cluster and follow the **Network Load Balancer → Application Load Balancer → Cluster** steps. Otherwise: + +1. Create a target group for the Application Load Balancer on port `9200` and set the health check to `HTTPS`. +2. Associate this target group with your existing Network Load Balancer on a new listener. +3. Verify that the health check is successful, and perform smoke testing with some clients through the new listener port. +4. Once you are ready to migrate all clients, deploy a change so that clients hit the new listener. + + +### If you are **not using an Network Load Balancer** + +If you're only using backfill as your migration technique, make a client/DNS change to route clients to the Migration Assistant Application Load Balancer on port `9200`. + + +### Kafka connection + +After you have routed the client based on your use case, test adding records against HTTP requests using the following steps: + +In the migration console, run the following command: + +```bash +console kafka describe-topic-records +``` +{% include copy.html %} + +Note the records in the logging topic. + +After a short period, execute the same command again and compare the increased number of records against the expected HTTP requests. + +## Creating a snapshot + +Create a snapshot for your backfill using the following command: + +```bash +console snapshot create +``` +{% include copy.html %} + +To check the progress of your snapshot, use the following command: + +```bash +console snapshot status --deep-check +``` +{% include copy.html %} + +Depending on the size of the data in the source cluster and the bandwidth allocated for snapshots, the process can take some time. Adjust the maximum rate at which the source cluster's nodes create the snapshot using the `--max-snapshot-rate-mb-per-node` option. Increasing the snapshot rate will consume more node resources, which may affect the cluster's ability to handle normal traffic. + +## Backfilling documents to the source cluster + +From the snapshot you created of your source cluster, you can begin backfilling documents into the target cluster. Once you have started this process, a fleet of workers will spin up to read the snapshot and reindex documents into the target cluster. This fleet of workers can be scaled to increased the speed at which documents are reindexed into the target cluster. + +### Checking the starting state of the clusters + +You can check the indexes and document counts of the source and target clusters by running the `cat-indices` command. This can be used to monitor the difference between the source and target for any migration scenario. Check the indexes of both clusters using the following command: + +```shell +console clusters cat-indices +``` +{% include copy.html %} + +You should receive the following response: + +```shell +SOURCE CLUSTER +health status index uuid pri rep docs.count docs.deleted store.size pri.store.size +green open my-index WJPVdHNyQ1KMKol84Cy72Q 1 0 8 0 44.7kb 44.7kb + +TARGET CLUSTER +health status index uuid pri rep docs.count docs.deleted store.size pri.store.size +green open .opendistro_security N3uy88FGT9eAO7FTbLqqqA 1 0 10 0 78.3kb 78.3kb +``` diff --git a/_migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target.md b/_migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target.md new file mode 100644 index 00000000000..314cc9a8647 --- /dev/null +++ b/_migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target.md @@ -0,0 +1,54 @@ +--- +layout: default +title: Cutover +nav_order: 7 +parent: Migration phases +redirect_from: + - /migration-assistant/migration-phases/switching-traffic-from-the-source-cluster/ +--- + +# Switching traffic from the source cluster + +After the source and target clusters are synchronized, traffic needs to be switched to the target cluster so that the source cluster can be taken offline. + +## Assumptions + +This page assumes that the following has occurred before making the switch: + +- All client traffic is being routed through a switchover listener in the [MigrationAssistant Application Load Balancer]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/). +- Client traffic has been verified as compatible with the target cluster. +- The target cluster is in a good state to accept client traffic. +- The target proxy service is deployed. + +## Switching traffic + +Use the following steps to switch traffic from the source cluster to the target cluster: + +1. In the AWS Management Console, navigate to **ECS** > **Migration Assistant Cluster**. Note the desired count of the capture proxy, which should be greater than 1. + +2. Update the **ECS Service** of the target proxy to be at least as large as the traffic capture proxy. Wait for tasks to start up, and verify that all targets are healthy in the target proxy service's **Load balancer health** section. + +3. Navigate to **EC2** > **Load Balancers** > **Migration Assistant ALB**. + +4. Navigate to **ALB Metrics** and examine any useful information, specifically looking at **Active Connection Count** and **New Connection Count**. Note any large discrepancies, which can indicate reused connections affecting traffic switchover. + +5. Navigate to **Capture Proxy Target Group** (`ALBSourceProxy--TG`) > **Monitoring**. + +6. Examine the **Metrics Requests**, **Target (2XX, 3XX, 4XX, 5XX)**, and **Target Response Time** metrics. Verify that this appears as expected and includes all traffic expected to be included in the switchover. Note details that could help identify anomalies during the switchover, including the expected response time and response code rate. + +7. Navigate back to **ALB Metrics** and choose **Target Proxy Target Group** (`ALBTargetProxy--TG`). Verify that all expected targets are healthy and that none are in a draining state. + +8. Navigate back to **ALB Metrics** and to the **Listener** on port `9200`. + +9. Choose the **Default rule** and **Edit**. + +10. Modify the weights of the targets to switch the desired traffic to the target proxy. To perform a full switchover, modify the **Target Proxy** weight to `1` and the **Source Proxy** weight to `0`. + +11. Choose **Save Changes**. + +12. Navigate to both **SourceProxy** and **TargetProxy TG Monitoring** metrics and verify that traffic is switching over as expected. If connections are being reused by clients, perform any necessary actions to terminate them. Monitor these metrics until **SourceProxy TG** shows 0 requests when all clients have switched over. + + +## Fallback + +If you need to fall back to the source cluster at any point during the switchover, revert the **Default rule** so that the Application Load Balancer routes to the **SourceProxy Target Group**. \ No newline at end of file diff --git a/_migration-assistant/migration-phases/teardown.md b/_migration-assistant/migration-phases/teardown.md new file mode 100644 index 00000000000..99e975c93b6 --- /dev/null +++ b/_migration-assistant/migration-phases/teardown.md @@ -0,0 +1,30 @@ +--- +layout: default +title: Remove Migration Infrastructure +nav_order: 8 +parent: Migration phases +redirect_from: + - /migration-assistant/migration-phases/removing-migration-infrastructure +--- + +# Removing migration infrastructure + +After a migration is complete, you should remove all resources except for the target cluster and, optionally, your Amazon CloudWatch logs and Traffic Replayer logs. + +To remove the AWS Cloud Development Kit (AWS CDK) stack(s) created during a deployment, run the following command within the CDK directory: + +```bash +cd deployment/cdk/opensearch-service-migration +cdk destroy "*" --c contextId= +``` +{% include copy.html %} + +Follow the instructions on the command line to remove the deployed resources from your AWS account. + +You can also use the AWS Management Console to remove Migration Assistant resources and confirm that they are no longer present in the account. + +## Uninstalling Migration Assistant for Amazon OpenSearch Service + +You can uninstall Migration Assistant for Amazon OpenSearch Service from the AWS Management Console or by using the AWS Command Line Interface (AWS CLI). Manually remove the contents of the Amazon Simple Storage Service (Amazon S3) bucket that matches the syntax `cdk--assets--`, the bucket created by Migration Assistant. Migration Assistant for Amazon OpenSearch Service does not automatically delete Amazon S3 buckets. + +To delete the stored data and the AWS CloudFormation stacks created by Migration Assistant, see [Uninstall the solution](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/uninstall-the-solution.html) in the Amazon OpenSearch Service documentation. diff --git a/_migration-assistant/migration-phases/verify-backfill-components.md b/_migration-assistant/migration-phases/verify-backfill-components.md new file mode 100644 index 00000000000..e8f99d7d88b --- /dev/null +++ b/_migration-assistant/migration-phases/verify-backfill-components.md @@ -0,0 +1,191 @@ +--- +layout: default +title: Backfill Setup Verification +nav_order: 3 +redirect_from: + - /migration-assistant/migration-phases/verifying-migration-tools/ +--- + +# Verifying Migration Assistant Backfill Components + +Before using the Migration Assistant, take the following steps to verify that your cluster is ready for migration. + +## Verifying snapshot creation + +Verify that a snapshot can be created of your source cluster and used for metadata and backfill scenarios. + +### Installing the Elasticsearch S3 Repository plugin + +The snapshot needs to be stored in a location that Migration Assistant can access. This guide uses Amazon Simple Storage Service (Amazon S3). By default, Migration Assistant creates an S3 bucket for storage. Therefore, it is necessary to install the [Elasticsearch S3 repository plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/7.10/repository-s3.html) on your source nodes (https://www.elastic.co/guide/en/elasticsearch/plugins/7.10/repository-s3.html). + +Additionally, make sure that the plugin has been configured with AWS credentials that allow it to read and write to Amazon S3. If your Elasticsearch cluster is running on Amazon Elastic Compute Cloud (Amazon EC2) or Amazon Elastic Container Service (Amazon ECS) instances with an AWS Identity and Access Management (IAM) execution role, include the necessary S3 permissions. Alternatively, you can store the credentials in the [Elasticsearch keystore](https://www.elastic.co/guide/en/elasticsearch/plugins/7.10/repository-s3-client.html). + +### Verifying the S3 repository plugin configuration + +You can verify that the S3 repository plugin is configured correctly by creating a test snapshot. + +Create an S3 bucket for the snapshot using the following AWS Command Line Interface (AWS CLI) command: + +```shell +aws s3api create-bucket --bucket --region +``` +{% include copy.html %} + +Register a new S3 snapshot repository on your source cluster using the following cURL command: + +```shell +curl -X PUT "http://:9200/_snapshot/test_s3_repository" -H "Content-Type: application/json" -d '{ + "type": "s3", + "settings": { + "bucket": "", + "region": "" + } +}' +``` +{% include copy.html %} + +Next, create a test snapshot that captures only the cluster's metadata: + +```shell +curl -X PUT "http://:9200/_snapshot/test_s3_repository/test_snapshot_1" -H "Content-Type: application/json" -d '{ + "indices": "", + "ignore_unavailable": true, + "include_global_state": true +}' +``` +{% include copy.html %} + +Check the AWS Management Console to confirm that your bucket contains the snapshot. + +### Removing test snapshots after verification + +To remove the resources created during verification, you can use the following deletion commands: + +**Test snapshot** + +```shell +curl -X DELETE "http://:9200/_snapshot/test_s3_repository/test_snapshot_1?pretty" +``` +{% include copy.html %} + +**Test snapshot repository** + +```shell +curl -X DELETE "http://:9200/_snapshot/test_s3_repository?pretty" +``` +{% include copy.html %} + +**S3 bucket** + +```shell +aws s3 rm s3:// --recursive +aws s3api delete-bucket --bucket --region +``` +{% include copy.html %} + +### Troubleshooting + +Use this guidance to troubleshoot any of the following snapshot verification issues. + +#### Access denied error (403) + +If you encounter an error like `AccessDenied (Service: Amazon S3; Status Code: 403)`, verify the following: + +- Make sure you're using the S3 bucket created by Migration Assistant. +- If you're using a custom S3 bucket, verify that: + - The IAM role assigned to your Elasticsearch cluster has the necessary S3 permissions. + - The bucket name and AWS Region provided in the snapshot configuration match the actual S3 bucket you created. + +#### Older versions of Elasticsearch + +Older versions of the Elasticsearch S3 repository plugin may have trouble reading IAM role credentials embedded in Amazon EC2 and Amazon ECS instances. This is because the copy of the AWS SDK shipped with them is too old to read the new standard way of retrieving those credentials, as shown in [the Instance Metadata Service v2 (IMDSv2) specification](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html). This can result in snapshot creation failures, with an error message similar to the following: + +```json +{"error":{"root_cause":[{"type":"repository_verification_exception","reason":"[migration_assistant_repo] path [rfs-snapshot-repo] is not accessible on master node"}],"type":"repository_verification_exception","reason":"[migration_assistant_repo] path [rfs-snapshot-repo] is not accessible on master node","caused_by":{"type":"i_o_exception","reason":"Unable to upload object [rfs-snapshot-repo/tests-s8TvZ3CcRoO8bvyXcyV2Yg/master.dat] using a single upload","caused_by":{"type":"amazon_service_exception","reason":"Unauthorized (Service: null; Status Code: 401; Error Code: null; Request ID: null)"}}},"status":500} +``` + +If you encounter this issue, you can resolve it by temporarily enabling IMDSv1 on the instances in your source cluster for the duration of the snapshot. There is a toggle for this available in the AWS Management Console as well as in the AWS CLI. Switching this toggle will turn on the older access model and enable the Elasticsearch S3 repository plugin to work as normal. For more information about IMDSv1, see [Modify instance metadata options for existing instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-IMDS-existing-instances.html). + +## Switching over client traffic + +The Migration Assistant Application Load Balancer is deployed with a listener that shifts traffic between the source and target clusters through proxy services. The Application Load Balancer should start in **Source Passthrough** mode. + +### Verifying that the traffic switchover is complete + +Use the following steps to verify that the traffic switchover is complete: + +1. In the AWS Management Console, navigate to **EC2 > Load Balancers**. +2. Select the **MigrationAssistant ALB**. +3. Examine the listener on port `9200` and verify that 100% of the traffic is directed to the **Source Proxy**. +4. Navigate to the **Migration ECS Cluster** in the AWS Management Console. +5. Select the **Target Proxy Service**. +6. Verify that the desired count for the service is running: + * If the desired count is not met, update the service to increase it to at least 1 and wait for the service to start. +7. On the **Health and Metrics** tab under **Load balancer health**, verify that all targets are reporting as healthy: + * This confirms that the Application Load Balancer can connect to the target cluster through the target proxy. +8. (Reset) Update the desired count for the **Target Proxy Service** back to its original value in Amazon ECS. + +### Fixing unidentified traffic patterns + +When switching over traffic to the target cluster, you might encounter unidentified traffic patterns. To help identify the cause of these patterns, use the following steps: +* Verify that the target cluster allows traffic ingress from the **Target Proxy Security Group**. +* Navigate to **Target Proxy ECS Tasks** to investigate any failing tasks. +Set the **Filter desired status** to **Any desired status** to view all tasks, then navigate to the logs for any stopped tasks. + + +## Verifying replication + +Use the following steps to verify that replication is working once the traffic capture proxy is deployed: + + +1. Navigate to the **Migration ECS Cluster** in the AWS Management Console. +2. Navigate to **Capture Proxy Service**. +3. Verify that the capture proxy is running with the desired proxy count. If it is not, update the service to increase it to at least 1 and wait for startup. +4. Under **Health and Metrics** > **Load balancer health**, verify that all targets are healthy. This means that the Application Load Balancer is able to connect to the source cluster through the capture proxy. +5. Navigate to the **Migration Console Terminal**. +6. Run `console kafka describe-topic-records`. Wait 30 seconds for another Application Load Balancer health check. +7. Run `console kafka describe-topic-records` again and verify that the number of RECORDS increased between runs. +8. Run `console replay start` to start Traffic Replayer. +9. Run `tail -f /shared-logs-output/traffic-replayer-default/*/tuples/tuples.log | jq '.targetResponses[]."Status-Code"'` to confirm that the Kafka requests were sent to the target and that it responded as expected. If the responses don't appear: + * Check that the migration console can access the target cluster by running `./catIndices.sh`, which should show the indexes in the source and target. + * Confirm that messages are still being recorded to Kafka. + * Check for errors in the Traffic Replayer logs (`/migration/STAGE/default/traffic-replayer-default`) using CloudWatch. +10. (Reset) Update the desired count for the **Capture Proxy Service** back to its original value in Amazon ECS. + +### Troubleshooting + +Use this guidance to troubleshoot any of the following replication verification issues. + +### Health check responses with 401/403 status code + +If the source cluster is configured to require authentication, the capture proxy will not be able to verify replication beyond receiving a 401/403 status code for Application Load Balancer health checks. For more information, see [Failure Modes](https://github.com/opensearch-project/opensearch-migrations/blob/main/TrafficCapture/trafficCaptureProxyServer/README.md#failure-modes). + +### Traffic does not reach the source cluster + +Verify that the source cluster allows traffic ingress from the Capture Proxy Security Group. + +Look for failing tasks by navigating to **Traffic Capture Proxy ECS**. Change **Filter desired status** to **Any desired status** in order to see all tasks and navigate to the logs for stopped tasks. + +### Snapshot and S3 bucket issues + +When using the CDK deployment for Migration Assistant, you might encounter the following errors during snapshot creation and deletion. + +#### Bucket permissions + +To make sure that you can delete snapshots as well as create them during the CDK deployment process, confirm that the `OSMigrations-dev--CustomS3AutoDeleteObjects` stack has S3 object deletion rights. Then, verify that `OSMigrations-dev--default-SnapshotRole` has the following S3 permissions: + + - List bucket contents + - Read/Write/Delete objects + +#### Snapshot conflicts + +To prevent snapshot conflicts, use the `console snapshot delete` command from the migration console. If you delete snapshots or snapshot repositories in a location other than the migration console, you might encounter "already exists" errors. + +## Resetting before migration + +After all verifications are complete, reset all resources before using Migration Assistant for an actual migration. + +The following steps outline how to reset resources with Migration Assistant before executing the actual migration. At this point all verifications are expected to have been completed. These steps can be performed after [Accessing the Migration Console]({{site.url}}{{site.baseurl}}/migration-assistant/migration-console/accessing-the-migration-console/). + + +{% include copy.html %} diff --git a/_migration-assistant/migration-phases/verify-live-capture-components.md b/_migration-assistant/migration-phases/verify-live-capture-components.md new file mode 100644 index 00000000000..90666b80bc8 --- /dev/null +++ b/_migration-assistant/migration-phases/verify-live-capture-components.md @@ -0,0 +1,157 @@ +--- +layout: default +title: Live Capture Setup Verification +nav_order: 9 +redirect_from: + - /migration-assistant/migration-phases/verifying-migration-tools/ +--- + +# Verifying migration tools + +Before using the Migration Assistant, take the following steps to verify that your cluster is ready for migration. + +### Traffic Replayer + +To stop running Traffic Replayer, use the following command: + +```bash +console replay stop +``` +{% include copy.html %} + +### Kafka + +To clear all captured traffic from the Kafka topic, you can run the following command. + +This command will result in the loss of any traffic data captured by the capture proxy up to this point and thus should be used with caution. +{: .warning} + +```bash +console kafka delete-topic +``` +{% include copy.html %} + +### Target cluster + +To clear non-system indexes from the target cluster that may have been created as a result of testing, you can run the following command: + +This command will result in the loss of all data in the target cluster and should be used with caution. +{: .warning} + +```bash +console clusters clear-indices --cluster target +``` + +## Switching over client traffic + +The Migration Assistant Application Load Balancer is deployed with a listener that shifts traffic between the source and target clusters through proxy services. The Application Load Balancer should start in **Source Passthrough** mode. + +### Verifying that the traffic switchover is complete + +Use the following steps to verify that the traffic switchover is complete: + +1. In the AWS Management Console, navigate to **EC2 > Load Balancers**. +2. Select the **MigrationAssistant ALB**. +3. Examine the listener on port `9200` and verify that 100% of the traffic is directed to the **Source Proxy**. +4. Navigate to the **Migration ECS Cluster** in the AWS Management Console. +5. Select the **Target Proxy Service**. +6. Verify that the desired count for the service is running: + * If the desired count is not met, update the service to increase it to at least 1 and wait for the service to start. +7. On the **Health and Metrics** tab under **Load balancer health**, verify that all targets are reporting as healthy: + * This confirms that the Application Load Balancer can connect to the target cluster through the target proxy. +8. (Reset) Update the desired count for the **Target Proxy Service** back to its original value in Amazon ECS. + +### Fixing unidentified traffic patterns + +When switching over traffic to the target cluster, you might encounter unidentified traffic patterns. To help identify the cause of these patterns, use the following steps: +* Verify that the target cluster allows traffic ingress from the **Target Proxy Security Group**. +* Navigate to **Target Proxy ECS Tasks** to investigate any failing tasks. +Set the **Filter desired status** to **Any desired status** to view all tasks, then navigate to the logs for any stopped tasks. + + +## Verifying replication + +Use the following steps to verify that replication is working once the traffic capture proxy is deployed: + + +1. Navigate to the **Migration ECS Cluster** in the AWS Management Console. +2. Navigate to **Capture Proxy Service**. +3. Verify that the capture proxy is running with the desired proxy count. If it is not, update the service to increase it to at least 1 and wait for startup. +4. Under **Health and Metrics** > **Load balancer health**, verify that all targets are healthy. This means that the Application Load Balancer is able to connect to the source cluster through the capture proxy. +5. Navigate to the **Migration Console Terminal**. +6. Run `console kafka describe-topic-records`. Wait 30 seconds for another Application Load Balancer health check. +7. Run `console kafka describe-topic-records` again and verify that the number of RECORDS increased between runs. +8. Run `console replay start` to start Traffic Replayer. +9. Run `tail -f /shared-logs-output/traffic-replayer-default/*/tuples/tuples.log | jq '.targetResponses[]."Status-Code"'` to confirm that the Kafka requests were sent to the target and that it responded as expected. If the responses don't appear: + * Check that the migration console can access the target cluster by running `./catIndices.sh`, which should show the indexes in the source and target. + * Confirm that messages are still being recorded to Kafka. + * Check for errors in the Traffic Replayer logs (`/migration/STAGE/default/traffic-replayer-default`) using CloudWatch. +10. (Reset) Update the desired count for the **Capture Proxy Service** back to its original value in Amazon ECS. + +### Troubleshooting + +Use this guidance to troubleshoot any of the following replication verification issues. + +### Health check responses with 401/403 status code + +If the source cluster is configured to require authentication, the capture proxy will not be able to verify replication beyond receiving a 401/403 status code for Application Load Balancer health checks. For more information, see [Failure Modes](https://github.com/opensearch-project/opensearch-migrations/blob/main/TrafficCapture/trafficCaptureProxyServer/README.md#failure-modes). + +### Traffic does not reach the source cluster + +Verify that the source cluster allows traffic ingress from the Capture Proxy Security Group. + +Look for failing tasks by navigating to **Traffic Capture Proxy ECS**. Change **Filter desired status** to **Any desired status** in order to see all tasks and navigate to the logs for stopped tasks. + +### Snapshot and S3 bucket issues + +When using the CDK deployment for Migration Assistant, you might encounter the following errors during snapshot creation and deletion. + +#### Bucket permissions + +To make sure that you can delete snapshots as well as create them during the CDK deployment process, confirm that the `OSMigrations-dev--CustomS3AutoDeleteObjects` stack has S3 object deletion rights. Then, verify that `OSMigrations-dev--default-SnapshotRole` has the following S3 permissions: + + - List bucket contents + - Read/Write/Delete objects + +#### Snapshot conflicts + +To prevent snapshot conflicts, use the `console snapshot delete` command from the migration console. If you delete snapshots or snapshot repositories in a location other than the migration console, you might encounter "already exists" errors. + +## Resetting before migration + +After all verifications are complete, reset all resources before using Migration Assistant for an actual migration. + +The following steps outline how to reset resources with Migration Assistant before executing the actual migration. At this point all verifications are expected to have been completed. These steps can be performed after [Accessing the Migration Console]({{site.url}}{{site.baseurl}}/migration-assistant/migration-console/accessing-the-migration-console/). + +### Traffic Replayer + +To stop running Traffic Replayer, use the following command: + +```bash +console replay stop +``` +{% include copy.html %} + +### Kafka + +To clear all captured traffic from the Kafka topic, you can run the following command. + +This command will result in the loss of any traffic data captured by the capture proxy up to this point and thus should be used with caution. +{: .warning} + +```bash +console kafka delete-topic +``` +{% include copy.html %} + +### Target cluster + +To clear non-system indexes from the target cluster that may have been created as a result of testing, you can run the following command: + +This command will result in the loss of all data in the target cluster and should be used with caution. +{: .warning} + +```bash +console clusters clear-indices --cluster target +``` +{% include copy.html %} diff --git a/_migration-assistant/overview/architecture.md b/_migration-assistant/overview/architecture.md new file mode 100644 index 00000000000..48b3c8fbcfc --- /dev/null +++ b/_migration-assistant/overview/architecture.md @@ -0,0 +1,25 @@ +--- +layout: default +title: Architecture +nav_order: 15 +parent: Overview +permalink: /migration-assistant/overview/architecture/ +--- + +# Architecture + +The Migration Assistant architecture is based on the use of an AWS Cloud infrastructure, but most tools are designed to be cloud independent. A local containerized version of this solution is also available. + +The design deployed on AWS uses the following architecture. + +![Migration architecture overview]({{site.url}}{{site.baseurl}}/images/migrations/migrations-architecture-overview.png) + +Each node in the diagram correlates to the following steps in the migration process: + +1. Client traffic is directed to the existing cluster. +2. An Application Load Balancer with capture proxies relays traffic to a source while replicating data to Amazon Managed Streaming for Apache Kafka (Amazon MSK). +3. Using the migration console, you can initiate metadata migration to establish indexes, templates, component templates, and aliases on the target cluster. +4. With continuous traffic capture in place, you can use a `reindex-from-snapshot` process to capture data from your current index. +5. Once `Reindex-from-Snapshot` is complete, captured traffic is replayed from Amazon MSK to the target cluster by [Traffic Replayer](https://docs.opensearch.org/docs/latest/migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer/). +6. Performance and behavior of traffic sent to the source and target clusters are compared by reviewing logs and metrics. +7. After confirming that the target cluster's functionality meets expectations, clients are redirected to the new target. \ No newline at end of file diff --git a/_migration-assistant/overview/index.md b/_migration-assistant/overview/index.md new file mode 100644 index 00000000000..7962fe5e85d --- /dev/null +++ b/_migration-assistant/overview/index.md @@ -0,0 +1,25 @@ +--- +layout: default +title: Overview +nav_order: 2 +has_children: true +has_toc: false +permalink: /migration-assistant/overview/ +items: + - heading: "Key components" + description: "Get familiar with the key components of Migration Assistant." + link: "/migration-assistant/overview/key-components/" + - heading: "Architecture" + description: "Understand how Migration Assistant integrates into your infrastructure." + link: "/migration-assistant/overview/architecture/" + - heading: "Is Migration Assistant right for you?" + description: "Evaluate whether Migration Assistant is right for your use case." + link: "/migration-assistant/overview/is-migration-assistant-right-for-you/" +--- + +# Migration Assistant overview + +Use this section to get familiar with the key concepts and structure of Migration Assistant before diving into setup or execution. These pages provide the architecture, key components, and guidance to help you determine whether Migration Assistant is right for your use case. + +{% include list.html list_items=page.items%} + diff --git a/_migration-assistant/overview/is-migration-assistant-right-for-you.md b/_migration-assistant/overview/is-migration-assistant-right-for-you.md new file mode 100644 index 00000000000..1e7fc287974 --- /dev/null +++ b/_migration-assistant/overview/is-migration-assistant-right-for-you.md @@ -0,0 +1,154 @@ +--- +layout: default +title: Is Migration Assistant right for you? +nav_order: 10 +parent: Overview +permalink: /migration-assistant/overview/is-migration-assistant-right-for-you/ +redirect_from: + - /migration-assistant/is-migration-assistant-right-for-you/ +--- + +# Is Migration Assistant right for you? + +Deciding whether to use Migration Assistant depends on your specific upgrade path, infrastructure complexity, and operational goals. This page will help you evaluate whether Migration Assistant is right for your use case—or whether another tool might be a better fit. + +Migration Assistant was built to fill important gaps in common migration strategies. For example, if you're upgrading across multiple major versions—such as from Elasticsearch 6.8 to OpenSearch 2.19—Migration Assistant lets you do this in a single step. Other methods, like rolling upgrades or snapshot restores, require you to upgrade through each major version, often reindexing your data at every step. + +Migration Assistant also supports live traffic replication, allowing for zero-downtime migrations. This makes it a strong choice for production environments, where minimizing service disruption is critical. + +If your migration is limited to a static cluster configuration (like index templates and aliases), or if you're not concerned about downtime, simpler tools may be sufficient. But for complex migrations involving real-time traffic or major version jumps, Migration Assistant offers robust, flexible capabilities. + +## Supported migration paths + +The following matrix shows which source versions can be directly migrated to which OpenSearch target versions: + + +{% comment %}First, collect all unique target versions{% endcomment %} +{% assign all_targets = "" | split: "" %} +{% for path in site.data.migration-assistant.valid_migrations.migration_paths %} + {% for target in path.targets %} + {% assign all_targets = all_targets | push: target %} + {% endfor %} +{% endfor %} +{% assign unique_targets = all_targets | uniq | sort %} + +
    Source Version{{ target }}
    + + + + {% for target in unique_targets %} + + {% endfor %} + + + + {% for path in site.data.migration-assistant.valid_migrations.migration_paths %} + + + {% for target_version in unique_targets %} + + {% endfor %} + + {% endfor %} + +
    {{ target }}
    {{ path.source }} + {% if path.targets contains target_version %}✓{% endif %} +
    + +## Supported platforms + +**Source and target platforms** + +- Self-managed (on premises or hosted by a cloud provider) +- Amazon OpenSearch Service + + +**AWS Regions** + +Migration Assistant is supported in the following AWS Regions: + +- US East (N. Virginia, Ohio) +- US West (Oregon, N. California) +- Europe (Frankfurt, Ireland, London) +- Asia Pacific (Tokyo, Singapore, Sydney) +- AWS GovCloud (US-East, US-West)[^1] + +[^1]: In AWS GovCloud (US), `reindex-from-snapshot` (RFS) is limited to shard sizes of 80 GiB or smaller. + + + +## Supported components + +Before starting a migration, consider the scope of the components involved. The following table outlines components that should potentially be migrated, indicates whether they are supported by Migration Assistant, and provides recommendations. + +| Component | Supported | Recommendations | +| :--- |:--- | :--- | +| **Documents** | Yes | Migrate existing data with RFS and live traffic with capture and replay. | +| **Index settings** | Yes | Migrate with the `Metadata-Migration-Tool`. | +| **Index mappings** | Yes | Migrate with the `Metadata-Migration-Tool`. | +| **Index templates** | Yes | Migrate with the `Metadata-Migration-Tool`. | +| **Component templates** | Yes | Migrate with the `Metadata-Migration-Tool`. | +| **Aliases** | Yes | Migrate with the `Metadata-Migration-Tool`. | +| **Index State Management (ISM) policies** | Expected in 2025 | Manually migrate using an API. For more information about ISM support, see [issue #944](https://github.com/opensearch-project/opensearch-migrations/issues/944). | +| **Elasticsearch Kibana[^2] dashboards** | Expected in 2025 | This tool is only needed when migrating from Elasticsearch Kibana dashboards to OpenSearch Dashboards. Start by exporting JSON files from Kibana and importing them into OpenSearch Dashboards. For Elasticsearch versions 7.10.2 to 7.17, use the [`dashboardsSanitizer`](https://github.com/opensearch-project/opensearch-migrations/tree/main/dashboardsSanitizer) tool before importing X-Pack visualizations like Canvas and Lens in Kibana dashboards, as they may require recreation for compatibility with OpenSearch.| +| **Security constructs** | No | Configure roles and permissions based on cloud provider recommendations. For example, if using AWS, leverage AWS Identity and Access Management (IAM) for enhanced security management. | +| **Plugins** | No | Check plugin compatibility; some Elasticsearch plugins may not have direct OpenSearch equivalents. | + +[^2]: Support for Kibana 5.0 through 7.10.2 migration paths to OpenSearch Dashboards will be added in a future version. Kibana 8 and later are not supported. For more information, see [issue #944](https://github.com/opensearch-project/opensearch-migrations/issues/944). + +## Choosing your migration approach + +Use the following checklist to determine which Migration Assistant components best fit your use case. + +### Metadata migration + +Use [metadata migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/) if: + +- You need to migrate while mitigating breaking changes between the source and target clusters, such as differences in mappings, settings, aliases, or component templates. +- You want a relatively consistent configuration between the source and target clusters. + +### Backfill migration + +Use [backfill migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/) if: + +- You need to move historical data without disrupting live traffic. +- You want to backfill indexes from a specific point in time without impacting the source cluster. +- You want to verify historical data in the target cluster before switching over. +- You want to backfill using an existing or incremental snapshot. +- You need the fastest backfill option that includes reindexing. +- You want the ability to pause and resume migration. + +### RFS + +Use [RFS]({{site.url}}{{site.baseurl}}/migration-assistant/deploying-migration-assistant/getting-started-data-migration/) if: + +- You already use OpenSearch snapshots for backups. +- You need to migrate documents at scale in parallel, such as with Amazon Elastic Container Service (Amazon ECS). +- You require a data migration path as part of a zero-downtime migration. +- Your AWS Region supports RFS and your shard sizes are within supported limits. + +### Combination of all three + +Use a combination of all three migration types if: + +- You're performing a complex, multi-version migration. +- You require zero downtime and full validation of the target environment. +- You want end-to-end tooling for metadata, data movement, and cluster behavior comparison. +- You're cloning an existing cluster and changing the source's configuration. +- You're setting up disaster recovery. + +## Checklist + +Use this checklist to decide whether Migration Assistant is right for you: + +- Are you migrating across one or more major versions? + +- Do you need to maintain service availability with zero downtime? + +- Do you need to validate a new OpenSearch cluster before switching over? + +- Is your environment self-managed or running on Amazon OpenSearch Service? + +- Are you looking for tooling that can automate metadata migration and performance comparison? + +If you answered "yes" to most of these questions, Migration Assistant is likely the right solution for your migration. diff --git a/_migration-assistant/overview/key-components.md b/_migration-assistant/overview/key-components.md new file mode 100644 index 00000000000..cfa69e70311 --- /dev/null +++ b/_migration-assistant/overview/key-components.md @@ -0,0 +1,39 @@ +--- +layout: default +title: Key components +nav_order: 10 +parent: Overview +permalink: /migration-assistant/overview/key-components/ +--- + +# Key components + +The following are the key components of Migration Assistant. + +## Elasticsearch/OpenSearch source + +In this solution, your source cluster operates on either Elasticsearch or OpenSearch and is hosted on Amazon Elastic Compute Cloud (Amazon EC2) instances or in a similar computing environment. A proxy is set up to interact with this source cluster, either positioned in front of or directly on the coordinating nodes of the cluster. + +## Migration console + +The migration console provides a migration-specific CLI and offers a variety of tools for streamlining the migration process. Everything necessary for completing a migration, other than cleaning up the migration resources, can be performed through this console. + +## Traffic capture proxy + +This component is designed for HTTP RESTful traffic. It forwards traffic to the source cluster and also splits and channels this traffic to a stream processing service for later playback. + +## Traffic Replayer + +Acting as a traffic simulation tool, [Traffic Replayer](https://docs.opensearch.org/docs/latest/migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer/) replays recorded request traffic to a target cluster, mirroring source traffic patterns. It links original requests and their responses to those directed at the target cluster, facilitating comparative analysis. + +## Metadata migration tool + +The metadata migration tool integrated into the Migration CLI can be used independently to migrate cluster metadata, including index mappings, index configuration settings, templates, component templates, and aliases. + +## Reindex-from-Snapshot + +`Reindex-from-Snapshot` (RFS) reindexes data from an existing snapshot. Amazon Elastic Container Service (Amazon ECS) workers coordinate the migration of documents from an existing snapshot, reindexing the documents in parallel to a target cluster. + +## Target cluster + +The target cluster is the destination cluster for migration or comparison in an A/B test. \ No newline at end of file diff --git a/_migration-assistant/planning-your-migration/handling-field-type-breaking-changes.md b/_migration-assistant/planning-your-migration/handling-field-type-breaking-changes.md new file mode 100644 index 00000000000..b6a653f96ae --- /dev/null +++ b/_migration-assistant/planning-your-migration/handling-field-type-breaking-changes.md @@ -0,0 +1,158 @@ +--- +layout: default +title: Handling breaking changes in field types +nav_order: 60 +parent: Planning your migration +grand_parent: Migration phases +--- + +# Handling breaking changes in field types + +This guide explains how to use Migration Assistant to transform field types that are deprecated or incompatible during a migration to OpenSearch. + +Field types define how data is stored and queried in an index. Each field in a document is mapped to a data type, which determines how it is indexed and what operations can be performed on it. + +For example, the following index mapping for a library's book collection defines three fields, each with a different type: + +```json +GET /library-books/_mappings +{ + "library-books": { + "mappings": { + "properties": { + "title": { "type": "text" }, + "publishedDate": { "type": "date" }, + "pageCount": { "type": "integer" } + } + } + } +} +``` + +For more information, see [Mappings and field types]({{site.url}}{{site.baseurl}}/field-types/). + +## Configure item transformations + +You can customize how field types are transformed during metadata and data migrations by supplying a transformation configuration file using the following steps: + +1. Open the Migration Assistant console. +2. Create a JavaScript file to define your transformation logic using the following command: + + ```bash + vim /shared-logs-output/field-type-converter.js + ``` + {% include copy.html %} + +3. Write any JavaScript rules that perform the desired field type conversions. For an example of how the rules can be implemented, see the [example `field-type-converter.js` implementation](#example-field-type-converterjs-implementation). +4. Create a transformation descriptor file using the following command: + + ```bash + vim /shared-logs-output/transformation.json + ``` + {% include copy.html %} + +5. Add a reference to your JavaScript file in `transformation.json`. +6. Run the metadata migration and supply the transformation configuration using a command similar to the following: + + ```bash + console metadata migrate \ + --transformer-config-file /shared-logs-output/transformation.json + ``` + {% include copy.html %} + +### Example `field-type-converter.js` implementation + +The following script demonstrates how to perform common field type conversions, including: + +* Replacing the deprecated `string` type with `text`. +* Converting `flattened` to `flat_object` and removing the `index` property if present. + +```javascript +function main(context) { + const rules = [ + { + when: { type: "string" }, + set: { type: "text" } + }, + { + when: { type: "flattened" }, + set: { type: "flat_object" }, + remove: ["index"] + } + ]; + + function applyRules(node, rules) { + if (Array.isArray(node)) { + node.forEach((child) => applyRules(child, rules)); + } else if (node instanceof Map) { + for (const { when, set, remove = [] } of rules) { + const matches = Object.entries(when).every(([k, v]) => node.get(k) === v); + if (matches) { + Object.entries(set).every(([k, v]) => node.set(k, v)); + remove.forEach((key) => node.delete(key)); + } + } + for (const child of node.values()) { + applyRules(child, rules); + } + } else if (node && typeof node === "object") { + for (const { when, set, remove = [] } of rules) { + const matches = Object.entries(when).every(([k, v]) => node[k] === v); + if (matches) { + Object.assign(node, set); + remove.forEach((key) => delete node[key]); + } + } + Object.values(node).forEach((child) => applyRules(child, rules)); + } + } + + return (doc) => { + if (doc && doc.type && doc.name && doc.body) { + applyRules(doc, rules); + } + return doc; + }; +} +(() => main)(); +``` +{% include copy.html %} + +The script contains the following elements: + +1. The `rules` array defines transformation logic: + + * `when`: Key-value conditions to match on a node + * `set`: Key-value pairs to apply when the `when` clause matches + * `remove` (optional): Keys to delete from the node when matched + +2. The `applyRules` function recursively traverses the input: + + * Arrays are recursively processed element by element. + * `Map` objects are matched and mutated using the defined rules. + * Plain objects are checked for matches and transformed accordingly. + +3. The `main` function returns a transformation function that: + + * Applies the rules to each document. + * Returns the modified document for migration or replay. + +### Example `transformation.json` + +The following JSON file references your transformation script and initializes the JavaScript engine with your custom rules: + +```json +[ + { + "JsonJSTransformerProvider": { + "initializationScriptFile": "/shared-logs-output/field-type-converter.js", + "bindingsObject": "{}" + } + } +] +``` +{% include copy.html %} + +## Summary + +By using a transformation configuration, you can rewrite deprecated or incompatible field types during metadata migration or data replay. This ensures that your target OpenSearch cluster only receives compatible mappings—even if the source cluster includes outdated types like `string` or features like `flattened` that need conversion. diff --git a/_migration-assistant/planning-your-migration/handling-type-mapping-deprecation.md b/_migration-assistant/planning-your-migration/handling-type-mapping-deprecation.md new file mode 100644 index 00000000000..edd81612dac --- /dev/null +++ b/_migration-assistant/planning-your-migration/handling-type-mapping-deprecation.md @@ -0,0 +1,318 @@ +--- +layout: default +title: Managing type mapping deprecation +nav_order: 60 +parent: Planning your migration +grand_parent: Migration phases +--- + +# Managing type mapping deprecation + +This guide provides solutions for managing the deprecation of the type mapping functionality when migrating from Elasticsearch 6.x or earlier to OpenSearch. + +In versions of Elasticsearch prior to 6.x, an index could contain multiple types, each with its own mapping. These types allowed you to store and query different kinds of documents—such as books and movies—in a single index. For example, both `book` and `movie` types could have a shared field like `title`, while each had additional fields specific to that type. + +Newer versions of Elasticsearch and OpenSearch no longer support multiple mapping types. Each index now supports only a single mapping type. During migration, you must define how to transform or restructure data that used multiple types. The following example shows multiple mapping types: + + +```JSON +GET /library/_mappings +{ + "library": { + "mappings": { + "book": { + "properties": { + "title": { "type": "text" }, + "pageCount": { "type": "integer" } + } + }, + "movie": { + "properties": { + "title": { "type": "text" }, + "runTime": { "type": "integer" } + } + } + } + } +} +``` + +For more information, see the [official Elasticsearch documentation on the removal of mapping types](https://www.elastic.co/guide/en/elasticsearch/reference/7.10/removal-of-types.html). + +## Using the type mapping transformer + +To address type mapping deprecation, use the `TypeMappingsSanitizationTransformer`. This transformer can modify data, including metadata, documents, and requests, so that the previously mapped data can be used in OpenSearch. To use the mapping transformer: + +1. Navigate to the bootstrap box and open the `cdk.context.json` file with Vim. +2. Add or update the key `reindexFromSnapshotExtraArgs` to include `--doc-transformer-config-file /shared-logs-output/transformation.json`. +3. Add or update the key `trafficReplayerExtraArgs` to include `--transformer-config-file /shared-logs-output/transformation.json`. +4. Deploy Migration Assistant. +5. Navigate to the Migration Assistant console. +6. Create a file named `/shared-logs-output/transformation.json`. +7. Add your transformation configuration to the file. For configuration options, see [Configuration options](#configuration-options). +8. When running the metadata migration, run the configuration with the transformer using the command `console metadata migrate --transformer-config-file /shared-logs-output/transformation.json`. + +Whenever the transformation configuration is updated, the backfill and replayer tools need to be stopped and restarted in order to apply the changes. Any previously migrated data and metadata may need to be cleared in order to avoid an inconsistent state. + +### Configuration options + +The `TypeMappingsSanitizationTransformer` supports several strategies for managing type mappings: + +1. **Route different types to separate indexes**: Split different types into their own indexes. +2. **Merge all types into one index**: Combine multiple types into a single index. +3. **Drop specific types**: Selectively migrate only specific types. +4. **Keep the original structure**: Maintain the same index name while conforming to the new type standards. + +### Type mapping transformer configuration schema + +The type mapping transformer uses the following configuration options. + +| **Field** | **Type** | **Required** | **Description** | +| :--- | :--- | :--- | :--- | +| `staticMappings` | `object` | No | A map of `{ indexName: { typeName: targetIndex } }` used to **statically** route specific types.

    For any **index** listed on this page, types **not** included in its object are **dropped** (no data or requests are migrated for those omitted types). | +| `regexMappings` | `array` | No | A list of **regex-based** rules for **dynamic** routing of source index/type names to a target index.

    Each element in this array is itself an object with `sourceIndexPattern`, `sourceTypePattern`, and `targetIndexPattern` fields.

    For information about the **default value**, see [Defaults](#Defaults). | +| `sourceProperties` | `object` | Yes | Additional **metadata** about the source (for example, its Elasticsearch/OpenSearch version). Must include at least `"version"` with `"major"` and `"minor"` fields. | + +The following example JSON configuration provides a transformation schema: + +
    +Example JSON Configuration + +```JSON +{ + "TypeMappingSanitizationTransformerProvider": { + "staticMappings": { + "{index-name-1}": { + "{type-name-1}": "{target-index-name-1}", + "{type-name-2}": "{target-index-name-2}" + } + }, + "regexMappings": [ + { + "sourceIndexPattern": "{source-index-pattern}", + "sourceTypePattern": "{source-type-pattern}", + "targetIndexPattern": "{target-index-pattern}" + } + ], + "sourceProperties": { + "version": { + "major": "NUMBER", + "minor": "NUMBER" + } + } + } +} +``` +{% include copy.html %} + +
    + +## Example configurations + +The following example configurations show you how to use the transformer for different mapping type scenarios. + +### Route different types to separate indexes + +If you have an index `activity` with types `user` and `post` that you want to split into separate indexes, use the following configuration: + +```json +[ + { + "TypeMappingSanitizationTransformerProvider": { + "staticMappings": { + "activity": { + "user": "new_users", + "post": "new_posts" + } + }, + "sourceProperties": { + "version": { + "major": 6, + "minor": 8 + } + } + } + } +] +{% include copy.html %} +``` + +This transformer will perform the following: + +- Route documents with type `user` to the `new_users` index. +- Route documents with type `post` to the `new_posts` index. + +### Merge all types into one index + +To merge all types into one index, use the following configuration: + +```json +[ + { + "TypeMappingSanitizationTransformerProvider": { + "staticMappings": { + "activity": { + "user": "activity", + "post": "activity" + } + }, + "sourceProperties": { + "version": { + "major": 6, + "minor": 8 + } + } + } + } +] +``` +{% include copy.html %} + +### Drop specific types + +To migrate only the `user` type within the `activity` index and drop all documents/requests with types not directly specified, use the following configuration: + +```json +[ + { + "TypeMappingSanitizationTransformerProvider": { + "staticMappings": { + "activity": { + "user": "users_only" + } + }, + "sourceProperties": { + "version": { + "major": 6, + "minor": 8 + } + } + } + } +] +``` +{% include copy.html %} + +This configuration only migrates documents of type `user` and ignores other document types in the `activity` index. + +### Keep the original structure + +To migrate only specific types and keep the original structure, use the following configuration: + +```JSON +[ + { + "TypeMappingSanitizationTransformerProvider": { + "regexMappings": [ + { + "sourceIndexPattern": "(.*)", + "sourceTypePattern": ".*", + "targetIndexPattern": "$1" + } + ], + "sourceProperties": { + "version": { + "major": 6, + "minor": 8 + } + } + } + } +] +``` +{% include copy.html %} + +This is equivalent to the strategy of merging all types into one index but also uses a pattern-based routing strategy. + +### Combining multiple strategies + +You can combine both static and regex-based mappings to manage different indexes or patterns in a single migration. For example, you might have one index that must use `staticMappings` and another that uses `regexMappings` to route all types by pattern. + +For each document, request, or metadata item (processed individually for bulk requests), the following steps are performed: + +1. The index is checked to determine whether it matches an entry in the static mappings. + - If matched, the type is checked against the index component of the static mappings entry. + - If the type matches, the mapping is applied, and the resulting index includes the value of the type key. + - If the type doesn't match, the request/document/metadata is dropped and not migrated. +2. If the index is not matched in the static mappings, the index-type combination is checked against each item in the regex mappings list, in order from first to last. If a match is found, the mapping is applied, the resulting index includes the value of the type key, and no further regex matching is performed. +3. Any request, document, or metadata that doesn't match the preceding cases is dropped, and the documents they contain are not migrated. + +The following example demonstrates how to combine static and regex-based mappings for different indexes: + +```json +[ + { + "TypeMappingSanitizationTransformerProvider": { + "staticMappings": { + "activity": { + "user": "users_activity", + "post": "posts_activity" + }, + "logs": { + "error": "logs_error", + "info": "logs_info" + } + }, + "regexMappings": [ + { + "sourceIndexPattern": "orders.*", + "sourceTypePattern": ".*", + "targetIndexPattern": "all_orders" + } + ], + "sourceProperties": { + "version": { + "major": 6, + "minor": 8 + } + } + } + } +] +``` +{% include copy.html %} + +### Defaults + +When the `regexMappings` key is missing from the transformation configuration, `regexMappings` will default to the following: + +```JSON +{ + "regexMappings": [ + { + "sourceIndexPattern": "(.+)", + "sourceTypePattern": "_doc", + "targetIndexPattern": "$1" + }, + { + "sourceIndexPattern": "(.+)", + "sourceTypePattern": "(.+)", + "targetIndexPattern": "$1_$2" + } + ] +} +``` +{% include copy.html %} + +This has the effect of retaining the index name for indexes created in Elasticsearch 6.x or later while combining the type and index name for indexes created in Elasticsearch 5.x. If you want to retain the index name for indexes created in Elasticsearch 5.x, use the `staticMappings` option or override the type mappings using the `regexMappings` option. + +## Limitations + +When using the transformer, remember the following limitations. +When using the transformer, remember the following limitations. +### Traffic Replayer + +For the Traffic Replayer, **only a subset** of requests that include types is supported. These requests are listed in the following table. + +| **Operation** | **HTTP method(s)** | **Endpoint** | **Description** | +| :--- | :--- | :--- | :--- | +| **Index (by ID)** | PUT/POST | `/{index}/{type}/{id}` | Create or update a single document with an explicit ID. | +| **Index (auto ID)** | PUT/POST | `/{index}/{type}/` | Create a single document for which the ID is automatically generated. | +| **Get Document** | GET | `/{index}/{type}/{id}` | Retrieve a document by ID. | +| **Bulk Index/Update/Delete** | PUT/POST | `/_bulk` | Perform multiple create/update/delete operations in a single request. | +| **Bulk Index/Update/Delete** | PUT/POST | `/{index}/_bulk` | Perform multiple create/update/delete operations in a single request with default index assignment. | +| **Bulk Index/Update/Delete** | PUT/POST | `/{index}/{type}/_bulk` | Perform multiple create/update/delete operations in a single request with default index and type assignment. | +| **Create/Update Index** | PUT/POST | `/{index}` | Create or update an index.

    **Split** behavior is not supported in the Traffic Replayer. See [this GitHub issue](https://github.com/opensearch-project/opensearch-migrations/issues/1305) to provide feedback or to vote on this feature. | + +### Reindex-From_Shapshot +For `Reindex-From-Snapshot,` indexes created in Elasticsearch 6.x or later will use `_doc` as the type for all documents, even if a different type was specified in Elasticsearch 6. diff --git a/_migration-assistant/planning-your-migration/index.md b/_migration-assistant/planning-your-migration/index.md new file mode 100644 index 00000000000..4f0d264e93e --- /dev/null +++ b/_migration-assistant/planning-your-migration/index.md @@ -0,0 +1,15 @@ +--- +layout: default +title: Planning your migration +nav_order: 59 +parent: Migration phases +has_toc: false +has_children: true +--- + +# Planning your migration + +This section describes how to plan for your migration to OpenSearch by: + +- [Assessing your current cluster for migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/planning-your-migration/assessing-your-cluster-for-migration/). +- [Verifying that you have the tools for migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/planning-your-migration/verifying-migration-tools/). \ No newline at end of file diff --git a/_migration-upgrade/index.md b/_migration-upgrade/index.md index 5d17cba894c..7aba907619d 100644 --- a/_migration-upgrade/index.md +++ b/_migration-upgrade/index.md @@ -5,9 +5,8 @@ nav_order: 1 has_children: false nav_exclude: true has_toc: false -permalink: /migration-assistant/ +permalink: /migrations-and-upgrades/ redirect_from: - - /migration-assistant/index/ - /upgrade-to/index/ - /upgrade-to/ - /upgrade-to/upgrade-to/ @@ -15,15 +14,9 @@ tutorial_cards: - heading: "Overview" description: "Get familiar with the key components of Migration Assistant and evaluate your use case." link: "/migration-assistant/overview/" - - heading: "Deploying Migration Assistant" - description: "Follow step-by-step instructions to deploy Migration Assistant and prepare data for migration." - link: "/deploying-migration-assistant/" - heading: "Migration phases" description: "Execute your migration in phases—metadata, backfill, and traffic replay—for a controlled and validated transition." - link: "/migration-phases/" - - heading: "Migration console" - description: "Use CLI commands provided by the migration console to orchestrate and monitor your migration process." - link: "/migration-console/" + link: "/migration-assistant/migration-phases/" --- # Migration and upgrade options @@ -54,19 +47,10 @@ Migration Assistant offers the most automated and resilient path for OpenSearch - **Reversion support:** Provides a fallback option in case of errors or issues. - **Integrated automation:** Designed to fit into OpenSearch workflows for a guided upgrade experience. -### Getting started with Migration Assistant - -To get started with Migration Assistant, determine which of the following scenarios best suits your needs: - -- **Metadata migration**: Migrating cluster metadata, such as index settings, aliases, and templates. -- **Backfill migration**: Migrating existing or historical data from a source to a target cluster. -- **Live traffic migration**: Replicating live ongoing traffic from a source to a target cluster. -- **Comparative tooling**: Comparing the performance and behaviors of an existing cluster with a prospective new one. -It's crucial to note that migration strategies are not universally applicable. This guide provides a detailed methodology, based on certain assumptions detailed throughout, emphasizing the importance of robust engineering practices to ensure a successful migration. -{: .tip } +### Getting started with Migration Assistant -{% include cards.html cards=page.tutorial_cards %} +- Review the [Migration Assistant]({{site.url}}{{site.baseurl}}/migration-assistant/) ## Rolling upgrades @@ -82,6 +66,11 @@ Rolling upgrades allow you to upgrade one node at a time, keeping the cluster op - Multiple upgrade cycles are needed for large version jumps. - Manual reindexing may be required to enable newer features. +### Getting started with rolling upgrades + +- Perform a [rolling upgrade]({{site.url}}{{site.baseurl}}/install-and-configure/upgrade-opensearch/rolling-upgrade/) + + ## Snapshot and restore @@ -127,10 +116,3 @@ Migration Assistant offers the most automated and resilient path for OpenSearch Before choosing a method, make sure that your OpenSearch clients and plugins are compatible with the target version. For example, tools like Logstash OSS and Filebeat OSS may enforce version checks that impact upgrade paths. -## Next steps - -After you've determined which upgrade or migration path works best for you, take one of the following next steps: - -- Review the [Migration Assistant Overview]({{site.url}}{{site.baseurl}}/migration-assistant/overview/) -- Perform a [rolling upgrade]({{site.url}}{{site.baseurl}}/install-and-configure/upgrade-opensearch/rolling-upgrade/) - diff --git a/_migration-upgrade/migration-phases/backfill.md b/_migration-upgrade/migration-phases/backfill.md index 8a98412675b..f2d879c0f7a 100644 --- a/_migration-upgrade/migration-phases/backfill.md +++ b/_migration-upgrade/migration-phases/backfill.md @@ -1,7 +1,7 @@ --- layout: default -title: Backfill -nav_order: 90 +title: Migrate Data +nav_order: 6 parent: Migration phases permalink: /migration-assistant/migration-phases/backfill/ --- @@ -10,99 +10,6 @@ permalink: /migration-assistant/migration-phases/backfill/ After the [metadata]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/) for your cluster has been migrated, you can use capture proxy data replication and snapshots to backfill your data into the next cluster. -## Capture proxy data replication - -If you're interested in capturing live traffic during your migration, Migration Assistant includes an Application Load Balancer for routing traffic to the capture proxy and the target cluster. Upstream client traffic must be routed through the capture proxy in order to replay the requests later. Before using the capture proxy, remember the following: - -* The layer upstream from the Application Load Balancer is compatible with the certificate on the Application Load Balancer listener, whether it's for clients or a Network Load Balancer. The `albAcmCertArn` in the `cdk.context.json` may need to be provided to ensure that clients trust the Application Load Balancer certificate. -* If a Network Load Balancer is used directly upstream of the Application Load Balancer, it must use a TLS listener. -* Upstream resources and security groups must allow network access to the Migration Assistant Application Load Balancer. - -To set up the capture proxy, go to the AWS Management Console and navigate to **EC2 > Load Balancers > Migration Assistant Application Load Balancer**. Copy the Application Load Balancer URL. With the URL copied, you can use one of the following options. - - -### If you are using **Network Load Balancer → Application Load Balancer → Cluster** - -1. Ensure that ingress is provided directly to the Application Load Balancer for the capture proxy. -2. Create a target group for the Migration Assistant Application Load Balancer on port `9200`, and set the health check to `HTTPS`. -3. Associate this target group with your existing Network Load Balancer on a new listener for testing. -4. Verify that the health check is successful, and perform smoke testing with some clients through the new listener port. -5. Once you are ready to migrate all clients, detach the Migration Assistant Application Load Balancer target group from the testing Network Load Balancer listener and modify the existing Network Load Balancer listener to direct traffic to this target group. -6. Now client requests will be routed through the proxy (once they establish a new connection). Verify the application metrics. - -### If you are using **Network Load Balancer → Cluster** - -If you do not want to modify application logic, add an Application Load Balancer in front of your cluster and follow the **Network Load Balancer → Application Load Balancer → Cluster** steps. Otherwise: - -1. Create a target group for the Application Load Balancer on port `9200` and set the health check to `HTTPS`. -2. Associate this target group with your existing Network Load Balancer on a new listener. -3. Verify that the health check is successful, and perform smoke testing with some clients through the new listener port. -4. Once you are ready to migrate all clients, deploy a change so that clients hit the new listener. - - -### If you are **not using an Network Load Balancer** - -If you're only using backfill as your migration technique, make a client/DNS change to route clients to the Migration Assistant Application Load Balancer on port `9200`. - - -### Kafka connection - -After you have routed the client based on your use case, test adding records against HTTP requests using the following steps: - -In the migration console, run the following command: - -```bash -console kafka describe-topic-records -``` -{% include copy.html %} - -Note the records in the logging topic. - -After a short period, execute the same command again and compare the increased number of records against the expected HTTP requests. - -## Creating a snapshot - -Create a snapshot for your backfill using the following command: - -```bash -console snapshot create -``` -{% include copy.html %} - -To check the progress of your snapshot, use the following command: - -```bash -console snapshot status --deep-check -``` -{% include copy.html %} - -Depending on the size of the data in the source cluster and the bandwidth allocated for snapshots, the process can take some time. Adjust the maximum rate at which the source cluster's nodes create the snapshot using the `--max-snapshot-rate-mb-per-node` option. Increasing the snapshot rate will consume more node resources, which may affect the cluster's ability to handle normal traffic. - -## Backfilling documents to the source cluster - -From the snapshot you created of your source cluster, you can begin backfilling documents into the target cluster. Once you have started this process, a fleet of workers will spin up to read the snapshot and reindex documents into the target cluster. This fleet of workers can be scaled to increased the speed at which documents are reindexed into the target cluster. - -### Checking the starting state of the clusters - -You can check the indexes and document counts of the source and target clusters by running the `cat-indices` command. This can be used to monitor the difference between the source and target for any migration scenario. Check the indexes of both clusters using the following command: - -```shell -console clusters cat-indices -``` -{% include copy.html %} - -You should receive the following response: - -```shell -SOURCE CLUSTER -health status index uuid pri rep docs.count docs.deleted store.size pri.store.size -green open my-index WJPVdHNyQ1KMKol84Cy72Q 1 0 8 0 44.7kb 44.7kb - -TARGET CLUSTER -health status index uuid pri rep docs.count docs.deleted store.size pri.store.size -green open .opendistro_security N3uy88FGT9eAO7FTbLqqqA 1 0 10 0 78.3kb 78.3kb -``` - ### Starting the backfill Use the following command to start the backfill and deploy the workers: diff --git a/_migration-upgrade/migration-phases/index.md b/_migration-upgrade/migration-phases/index.md index a330b9e598e..d8f314d6b75 100644 --- a/_migration-upgrade/migration-phases/index.md +++ b/_migration-upgrade/migration-phases/index.md @@ -11,13 +11,77 @@ redirect_from: # Migration phases -This page details how to conduct a migration with Migration Assistant. It shows you how to [plan for your migration]({{site.url}}{{site.baseurl}}/planning-your-migration/) and encompasses a variety of migration scenarios, including: +This page outlines the core phases of migrating with Migration Assistant. Each scenario below consists of a sequence of common steps. Expand a scenario to explore the detailed flow. -- [**Metadata migration**]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/): Migrating cluster metadata, such as index settings, aliases, and templates. -- [**Backfill migration**]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/): Migrating existing or historical data from a source to a target cluster. -- [**Live traffic migration**]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/using-traffic-replayer/): Replicating live ongoing traffic from [a source cluster]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/switching-traffic-from-the-source-cluster/) to a target cluster. +--- + + + +
    +Scenario 1 – Backfill Only + +
      +
    1. Assessment
      2. +
    2. Deployment
      3. + +
    3. Create Snapshot
      4. +
    4. Migrate Metadata
      5. +
    5. Migrate Data
      6. +
    6. Teardown
      7. +
    +
    +
    +Scenario 2 – Live Capture Only +
      +
    1. Assessment
      2. +
    2. Deployment
      3. +
    3. Verify Backfill Components
      4. +
    4. Reroute Traffic from Source to Capture Proxy
      5. +
    5. Migrate Metadata
      6. +
    6. Verify Live Capture Components
      7. +
    7. Replay Captured Traffic
      8. +
    8. Reroute Traffic from Capture Proxy to Target
      9. +
    9. Teardown
      10. +
    +
    + +
    +Scenario 3 – Live Capture with Backfill + +
      +
    1. Assessment
      2. +
    2. Deployment
      3. +
    3. Reroute Traffic from Source to Capture Proxy
      4. +
    4. Create Snapshot
      5. +
    5. Migrate Metadata
      6. +
    6. Migrate Data
      7. +
    7. Replay Traffic
      8. +
    8. Reroute Traffic from Capture Proxy to Target
      9. +
    9. Teardown
      10. +
    + +
    + +--- +💡 *Need help getting started? Begin with [Planning your migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/planning-your-migration/index/).* From 49e03abc6e5aa35570838c6834a3fe1426fb3ded Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Wed, 2 Jul 2025 21:50:01 -0500 Subject: [PATCH 10/13] Remove 3.x source support Signed-off-by: Brian Presley --- _data/migration-assistant/valid_migrations.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/_data/migration-assistant/valid_migrations.yml b/_data/migration-assistant/valid_migrations.yml index 772c3bc981a..60031815539 100644 --- a/_data/migration-assistant/valid_migrations.yml +++ b/_data/migration-assistant/valid_migrations.yml @@ -26,7 +26,4 @@ migration_paths: - source: "OpenSearch 2.x" targets: - "OpenSearch 2.x" - - "OpenSearch 3.x" - - source: "OpenSearch 3.x" - targets: - - "OpenSearch 3.x" + - "OpenSearch 3.x" \ No newline at end of file From a4a178caccf43621181adca3c2cd556dc76c4b4d Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Wed, 2 Jul 2025 23:14:52 -0500 Subject: [PATCH 11/13] Modifications, fixes, and clarity for migration phases. Signed-off-by: Brian Presley --- _migration-assistant/index.md | 23 ++++-- .../migration-phases/index.md | 8 +- _migration-assistant/overview/architecture.md | 9 +- _migration-assistant/overview/index.md | 25 ------ .../is-migration-assistant-right-for-you.md | 82 +++++++++---------- .../overview/key-components.md | 10 +-- _migration-upgrade/index.md | 11 ++- 7 files changed, 75 insertions(+), 93 deletions(-) delete mode 100644 _migration-assistant/overview/index.md diff --git a/_migration-assistant/index.md b/_migration-assistant/index.md index 1d7f8bb1db5..e4147b7dcff 100644 --- a/_migration-assistant/index.md +++ b/_migration-assistant/index.md @@ -8,16 +8,23 @@ has_toc: false permalink: /migration-assistant/ redirect_from: - /migration-assistant/index/ + - /migration-assistant/overview/ - /upgrade-to/index/ - /upgrade-to/ - /upgrade-to/upgrade-to/ -tutorial_cards: - - heading: "Overview" - description: "Get familiar with the key components of Migration Assistant and evaluate your use case." - link: "/migration-assistant/overview/" - - heading: "Migration phases" - description: "Execute your migration in phases—metadata, backfill, and traffic replay—for a controlled and validated transition." - link: "/migration-assistant/migration-phases/" +items: + - heading: "Is Migration Assistant right for you?" + description: "Evaluate whether Migration Assistant is right for your use case." + link: "/migration-assistant/overview/is-migration-assistant-right-for-you/" + - heading: "Key components" + description: "Get familiar with the key components of Migration Assistant." + link: "/migration-assistant/overview/key-components/" + - heading: "Architecture" + description: "Understand how Migration Assistant integrates into your infrastructure." + link: "/migration-assistant/overview/architecture/" + - heading: "Execute your migration in phases" + description: "A step-by-step guide for performing a migration." + link: "/migration-assistant/migration-phases" --- # Migration Assistant for OpenSearch @@ -31,5 +38,5 @@ Migration Assistant for OpenSearch aids you in successfully performing an end-to This user guide focuses on conducting a comprehensive migration involving both existing and live data with zero downtime and the option to back out of a migration. -{% include cards.html cards=page.tutorial_cards %} +{% include list.html list_items=page.items%} diff --git a/_migration-assistant/migration-phases/index.md b/_migration-assistant/migration-phases/index.md index 8a7dcc95ab7..6db56d90d2e 100644 --- a/_migration-assistant/migration-phases/index.md +++ b/_migration-assistant/migration-phases/index.md @@ -1,12 +1,12 @@ --- layout: default title: Migration phases -parent: Migration Assistant for OpenSearch +parent: Migration Assistant nav_order: 30 has_children: false -has_toc: true -permalink: /migration-assistant/overview/ -redirect-from: /migration-assistant/overview/index/ +has_toc: false +permalink: /migration-assistant/migration-phases/ +redirect-from: /migration-assistant/migration-phases/index/ --- # Migration phases diff --git a/_migration-assistant/overview/architecture.md b/_migration-assistant/overview/architecture.md index 48b3c8fbcfc..f768d3fa646 100644 --- a/_migration-assistant/overview/architecture.md +++ b/_migration-assistant/overview/architecture.md @@ -18,8 +18,7 @@ Each node in the diagram correlates to the following steps in the migration proc 1. Client traffic is directed to the existing cluster. 2. An Application Load Balancer with capture proxies relays traffic to a source while replicating data to Amazon Managed Streaming for Apache Kafka (Amazon MSK). -3. Using the migration console, you can initiate metadata migration to establish indexes, templates, component templates, and aliases on the target cluster. -4. With continuous traffic capture in place, you can use a `reindex-from-snapshot` process to capture data from your current index. -5. Once `Reindex-from-Snapshot` is complete, captured traffic is replayed from Amazon MSK to the target cluster by [Traffic Replayer](https://docs.opensearch.org/docs/latest/migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer/). -6. Performance and behavior of traffic sent to the source and target clusters are compared by reviewing logs and metrics. -7. After confirming that the target cluster's functionality meets expectations, clients are redirected to the new target. \ No newline at end of file +3. Using the migration console, a point-in-time snapshot is taken. Once the snapshot completes, `Metadata-Migration-Tool` is used to establish indexes, templates, component templates, and aliases on the target cluster. With continuous traffic capture in place, `Reindex-from-Snapshot` process to migrate data from source. +4. Once `Reindex-from-Snapshot` is complete, captured traffic is replayed from Amazon MSK to the target cluster by `Traffic-Capture-Replayer`. +5. Performance and behavior of traffic sent to the source and target clusters are compared by reviewing logs and metrics. +6. After confirming that the target cluster's functionality meets expectations, clients are redirected to the new target. \ No newline at end of file diff --git a/_migration-assistant/overview/index.md b/_migration-assistant/overview/index.md deleted file mode 100644 index 11c7d716ef5..00000000000 --- a/_migration-assistant/overview/index.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -layout: default -title: Overview -nav_order: 2 -has_children: true -has_toc: false -permalink: /migration-assistant/overview/ -items: - - heading: "Is Migration Assistant right for you?" - description: "Evaluate whether Migration Assistant is right for your use case." - link: "/migration-assistant/overview/is-migration-assistant-right-for-you/" - - heading: "Key components" - description: "Get familiar with the key components of Migration Assistant." - link: "/migration-assistant/overview/key-components/" - - heading: "Architecture" - description: "Understand how Migration Assistant integrates into your infrastructure." - link: "/migration-assistant/overview/architecture/" ---- - -# Migration Assistant overview - -Use this section to get familiar with the key concepts and structure of Migration Assistant before diving into setup or execution. These pages provide the architecture, key components, and guidance to help you determine whether Migration Assistant is right for your use case. - -{% include list.html list_items=page.items%} - diff --git a/_migration-assistant/overview/is-migration-assistant-right-for-you.md b/_migration-assistant/overview/is-migration-assistant-right-for-you.md index 9a5580d54ed..dcca20f553f 100644 --- a/_migration-assistant/overview/is-migration-assistant-right-for-you.md +++ b/_migration-assistant/overview/is-migration-assistant-right-for-you.md @@ -16,45 +16,6 @@ Migration Assistant addresses key limitations in traditional migration approache Migration Assistant also supports live traffic replication, enabling zero-downtime migrations. This makes it a strong fit for environments where minimizing service disruption is critical. -## Migration Assistant assumptions and limitations - -Before using Migration Assistant, review the following assumptions and limitations. They are grouped by scope to clarify which apply generally and which are specific to components. - -### General - - -- If deploying on AWS, the identity used to deploy Migration Assistant must have permission to install all required resources. For a full list of services, see [AWS Services in this Solution](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/architecture-details.html#aws-services-in-this-solution). -- The target cluster must be deployed and reachable by Migration Assistant components, including the Migration Console, Reindex-from-Snapshot, and Traffic Replayer, depending on which features are used. -- The `_source` field must be enabled on all indexes to be migrated. See [Source documentation]({{site.url}}{{site.baseurl}}/field-types/metadata-fields/source/) for more information. -- If deploying to AWS, ensure the `CDKToolkit` stack exists and is in the `CREATE_COMPLETE` state. For setup instructions, see the [CDK Toolkit documentation](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html). -- If deploying Migration Assistant into an existing AWS VPC, you must configure VPC interface endpoints and IAM permissions for the following AWS services: - - **Amazon CloudWatch Logs** – for log ingestion from ECS tasks. - - **Amazon CloudWatch** – for metrics published during migration - - **Amazon Elastic Container Registry (ECR)** – for pulling container images. - - **Amazon Elastic Container Service (ECS)** – for task orchestration and migration. - - **Elastic Load Balancing (ELB)** – for routing traffic to Capture Proxy. - - **AWS Secrets Manager** – for storing credentials. - - **AWS Systems Manager Parameter Store** – for storing and accessing parameter values. - - **AWS Systems Manager Session Manager** – for secure shell access to EC2 (i.e., bootstrap instance). - - **Amazon EC2** – for launching the bootstrap instance responsible for build and deployment. - - **Amazon Elastic Block Store (EBS)** – for disk storage during migration. - - **Amazon Virtual Private Cloud (VPC)** – for private networking and VPC endpoints. - - **AWS X-Ray** – for distributed tracing across components. - - **Amazon Elastic File System (EFS)** – for persistent logging. - -### Reindex-from-Snapshot - -- The source cluster must have the Amazon S3 plugin installed. -- Snapshots must include global cluster state (`include_global_state` is `true`). -- Shards up to 80 GiB are supported by default. Larger shards can be configured, except in AWS GovCloud, where 80 GiB is the maximum supported size. - -### Capture-and-Replay - -- The Traffic Capture Proxy must be deployed to intercept relevant client traffic. -- Live capture is recommended only for environments with less than 4 TB/day of incoming traffic to the source cluster. -- Automatically generated document IDs are not preserved for index requests. Clients must explicitly provide document IDs when indexing or updating documents. - ---- ## Supported migration paths @@ -122,9 +83,6 @@ Before starting an upgrade or migration, consider the cluster feature to be incl | **Security constructs** | No | Configure roles and permissions based on cloud provider recommendations. For example, if using AWS, leverage AWS Identity and Access Management (IAM) for enhanced security management. | | **Plugins** | No | Check plugin compatibility; some Elasticsearch plugins may not have direct OpenSearch equivalents. | -[^2]: Support for Kibana 5.0 through 7.10.2 migration paths to OpenSearch Dashboards will be added in a future version. Kibana 8 and later are not supported. For more information, see [issue #944](https://github.com/opensearch-project/opensearch-migrations/issues/944). - - ## Checklist Use this checklist to determine whether Migration Assistant is the right fit for your migration: @@ -140,3 +98,43 @@ Use this checklist to determine whether Migration Assistant is the right fit for - Do you need a high-performance backfill solution that can reliably reindex documents—with support for pause, resume, or checkpoint recovery? If you answered "yes" to most of these questions, Migration Assistant is likely the right solution for your migration. + +## Migration Assistant assumptions and limitations + +Before using Migration Assistant, review the following assumptions and limitations. They are grouped by scope to clarify which apply generally and which are specific to components. + +### General + + +- If deploying on AWS, the identity used to deploy Migration Assistant must have permission to install all required resources. For a full list of services, see [AWS Services in this Solution](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/architecture-details.html#aws-services-in-this-solution). +- The target cluster must be deployed and reachable by Migration Assistant components, including the Migration Console, Reindex-from-Snapshot, and Traffic Replayer, depending on which features are used. +- The `_source` field must be enabled on all indexes to be migrated. See [Source documentation]({{site.url}}{{site.baseurl}}/field-types/metadata-fields/source/) for more information. +- If deploying to AWS, ensure the `CDKToolkit` stack exists and is in the `CREATE_COMPLETE` state. For setup instructions, see the [CDK Toolkit documentation](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html). +- If deploying Migration Assistant into an existing AWS VPC, you must configure VPC interface endpoints and IAM permissions for the following AWS services: + - **Amazon CloudWatch Logs** – for log ingestion from ECS tasks. + - **Amazon CloudWatch** – for metrics published during migration + - **Amazon Elastic Container Registry (ECR)** – for pulling container images. + - **Amazon Elastic Container Service (ECS)** – for task orchestration and migration. + - **Elastic Load Balancing (ELB)** – for routing traffic to Capture Proxy. + - **AWS Secrets Manager** – for storing credentials. + - **AWS Systems Manager Parameter Store** – for storing and accessing parameter values. + - **AWS Systems Manager Session Manager** – for secure shell access to EC2 (i.e., bootstrap instance). + - **Amazon EC2** – for launching the bootstrap instance responsible for build and deployment. + - **Amazon Elastic Block Store (EBS)** – for disk storage during migration. + - **Amazon Virtual Private Cloud (VPC)** – for private networking and VPC endpoints. + - **AWS X-Ray** – for distributed tracing across components. + - **Amazon Elastic File System (EFS)** – for persistent logging. + +### Reindex-from-Snapshot + +- The source cluster must have the Amazon S3 plugin installed. +- Snapshots must include global cluster state (`include_global_state` is `true`). +- Shards up to 80 GiB are supported by default. Larger shards can be configured, except in AWS GovCloud, where 80 GiB is the maximum supported size. + +### Capture-and-Replay + +- The Traffic Capture Proxy must be deployed to intercept relevant client traffic. +- Live capture is recommended only for environments with less than 4 TB/day of incoming traffic to the source cluster. +- Automatically generated document IDs are not preserved for index requests. Clients must explicitly provide document IDs when indexing or updating documents. + +--- \ No newline at end of file diff --git a/_migration-assistant/overview/key-components.md b/_migration-assistant/overview/key-components.md index cfa69e70311..6e5aa34db44 100644 --- a/_migration-assistant/overview/key-components.md +++ b/_migration-assistant/overview/key-components.md @@ -12,13 +12,13 @@ The following are the key components of Migration Assistant. ## Elasticsearch/OpenSearch source -In this solution, your source cluster operates on either Elasticsearch or OpenSearch and is hosted on Amazon Elastic Compute Cloud (Amazon EC2) instances or in a similar computing environment. A proxy is set up to interact with this source cluster, either positioned in front of or directly on the coordinating nodes of the cluster. +In this solution, your source cluster operates on either Elasticsearch or OpenSearch and is hosted on Amazon Elastic Compute Cloud (Amazon EC2) instances or in a similar computing environment. Traffic is rerouted from the source cluster to a traffic capture proxy and replayed to a target typically on a later version of OpenSearch. -## Migration console +## Migration Console The migration console provides a migration-specific CLI and offers a variety of tools for streamlining the migration process. Everything necessary for completing a migration, other than cleaning up the migration resources, can be performed through this console. -## Traffic capture proxy +## Traffic Capture Proxy This component is designed for HTTP RESTful traffic. It forwards traffic to the source cluster and also splits and channels this traffic to a stream processing service for later playback. @@ -26,7 +26,7 @@ This component is designed for HTTP RESTful traffic. It forwards traffic to the Acting as a traffic simulation tool, [Traffic Replayer](https://docs.opensearch.org/docs/latest/migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer/) replays recorded request traffic to a target cluster, mirroring source traffic patterns. It links original requests and their responses to those directed at the target cluster, facilitating comparative analysis. -## Metadata migration tool +## Metadata-Migration-Tool The metadata migration tool integrated into the Migration CLI can be used independently to migrate cluster metadata, including index mappings, index configuration settings, templates, component templates, and aliases. @@ -36,4 +36,4 @@ The metadata migration tool integrated into the Migration CLI can be used indepe ## Target cluster -The target cluster is the destination cluster for migration or comparison in an A/B test. \ No newline at end of file +The target cluster is the destination cluster for migration or comparison cluster in an A/B test. \ No newline at end of file diff --git a/_migration-upgrade/index.md b/_migration-upgrade/index.md index 7aba907619d..bd0f7319ee0 100644 --- a/_migration-upgrade/index.md +++ b/_migration-upgrade/index.md @@ -50,7 +50,7 @@ Migration Assistant offers the most automated and resilient path for OpenSearch ### Getting started with Migration Assistant -- Review the [Migration Assistant]({{site.url}}{{site.baseurl}}/migration-assistant/) +- Upgrade or migrate with [Migration Assistant for OpenSearch]({{site.url}}{{site.baseurl}}/migration-assistant/) ## Rolling upgrades @@ -68,8 +68,7 @@ Rolling upgrades allow you to upgrade one node at a time, keeping the cluster op ### Getting started with rolling upgrades -- Perform a [rolling upgrade]({{site.url}}{{site.baseurl}}/install-and-configure/upgrade-opensearch/rolling-upgrade/) - +- Perform a [rolling upgrade]({{site.url}}{{site.baseurl}}/install-and-configure/upgrade-opensearch/rolling-upgrade/). ## Snapshot and restore @@ -77,7 +76,7 @@ Rolling upgrades allow you to upgrade one node at a time, keeping the cluster op This method involves taking a snapshot of your existing OpenSearch or Elasticsearch OSS cluster and restoring it to a new cluster running the target version. **Pros:** -- Original cluster remains untouched, allowing rollback if needed (note: may result in data loss without a change data capture (CDC) solution). +- Original cluster remains untouched, allowing rollback if needed (note: may result in data loss without a change data capture (CDC) solution. In some cases, one may choose to use `Capture-and-Replay` from Migration Assistant in combination with Snapshot and Restore). - Scales well for large data volumes, including cold storage and datasets up to 1 PB. **Cons:** @@ -85,6 +84,10 @@ This method involves taking a snapshot of your existing OpenSearch or Elasticsea - Requires provisioning a new cluster. - Manual reindexing may be necessary for full feature support. +### Get started with Snapshot and Restore + +Upgrade or Migration with [Snapshot and Restore](https://docs.aws.amazon.com/solutions/latest/ tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore/) + ## Remote reindexing This method reindexes data from your current cluster into a new OpenSearch cluster, typically running in parallel. From 198c4487bdbe2cdc1b560bec0976bd62cc189c09 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Thu, 3 Jul 2025 13:07:16 -0500 Subject: [PATCH 12/13] Moved assets from _migration-upgrade to _migration-assistant Signed-off-by: Brian Presley --- .../configuration-options.md | 0 .../getting-started-data-migration.md | 0 ...d-security-groups-for-existing-clusters.md | 0 .../deploying-migration-assistant/index.md | 0 _migration-assistant/index.md | 1 + .../accessing-the-migration-console.md | 0 .../migration-console/index.md | 0 .../migration-console-commands-references.md | 0 .../migration-phases/backfill.md | 0 .../migration-phases/deployment.md | 2 +- .../migration-phases/index.md | 2 +- .../live-traffic-migration/index.md | 0 ...itching-traffic-from-the-source-cluster.md | 0 .../using-traffic-replayer.md | 0 .../migration-phases/migrating-metadata.md | 0 .../assessing-your-cluster-for-migration.md | 0 .../handling-field-type-breaking-changes.md | 0 .../handling-type-mapping-deprecation.md | 0 .../planning-your-migration/index.md | 0 .../verifying-migration-tools.md | 0 .../removing-migration-infrastructure.md | 0 .../overview/index.md | 0 .../deploying-migration-assistant/index.md | 13 -- _migration-upgrade/migration-phases/index.md | 87 ---------- _migration-upgrade/overview/architecture.md | 25 --- .../is-migration-assistant-right-for-you.md | 154 ------------------ _migration-upgrade/overview/key-components.md | 39 ----- 27 files changed, 3 insertions(+), 320 deletions(-) rename {_migration-upgrade => _migration-assistant}/deploying-migration-assistant/configuration-options.md (100%) rename {_migration-upgrade => _migration-assistant}/deploying-migration-assistant/getting-started-data-migration.md (100%) rename {_migration-upgrade => _migration-assistant}/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md (100%) rename _migration-assistant/{migration-phases => }/deploying-migration-assistant/index.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-console/accessing-the-migration-console.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-console/index.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-console/migration-console-commands-references.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/backfill.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/live-traffic-migration/index.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/live-traffic-migration/using-traffic-replayer.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/migrating-metadata.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/planning-your-migration/index.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/planning-your-migration/verifying-migration-tools.md (100%) rename {_migration-upgrade => _migration-assistant}/migration-phases/removing-migration-infrastructure.md (100%) rename {_migration-upgrade => _migration-assistant}/overview/index.md (100%) delete mode 100644 _migration-upgrade/deploying-migration-assistant/index.md delete mode 100644 _migration-upgrade/migration-phases/index.md delete mode 100644 _migration-upgrade/overview/architecture.md delete mode 100644 _migration-upgrade/overview/is-migration-assistant-right-for-you.md delete mode 100644 _migration-upgrade/overview/key-components.md diff --git a/_migration-upgrade/deploying-migration-assistant/configuration-options.md b/_migration-assistant/deploying-migration-assistant/configuration-options.md similarity index 100% rename from _migration-upgrade/deploying-migration-assistant/configuration-options.md rename to _migration-assistant/deploying-migration-assistant/configuration-options.md diff --git a/_migration-upgrade/deploying-migration-assistant/getting-started-data-migration.md b/_migration-assistant/deploying-migration-assistant/getting-started-data-migration.md similarity index 100% rename from _migration-upgrade/deploying-migration-assistant/getting-started-data-migration.md rename to _migration-assistant/deploying-migration-assistant/getting-started-data-migration.md diff --git a/_migration-upgrade/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md b/_migration-assistant/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md similarity index 100% rename from _migration-upgrade/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md rename to _migration-assistant/deploying-migration-assistant/iam-and-security-groups-for-existing-clusters.md diff --git a/_migration-assistant/migration-phases/deploying-migration-assistant/index.md b/_migration-assistant/deploying-migration-assistant/index.md similarity index 100% rename from _migration-assistant/migration-phases/deploying-migration-assistant/index.md rename to _migration-assistant/deploying-migration-assistant/index.md diff --git a/_migration-assistant/index.md b/_migration-assistant/index.md index e4147b7dcff..cf3a006ce32 100644 --- a/_migration-assistant/index.md +++ b/_migration-assistant/index.md @@ -12,6 +12,7 @@ redirect_from: - /upgrade-to/index/ - /upgrade-to/ - /upgrade-to/upgrade-to/ + - /upgrade-to/snapshot-migrate/ items: - heading: "Is Migration Assistant right for you?" description: "Evaluate whether Migration Assistant is right for your use case." diff --git a/_migration-upgrade/migration-console/accessing-the-migration-console.md b/_migration-assistant/migration-console/accessing-the-migration-console.md similarity index 100% rename from _migration-upgrade/migration-console/accessing-the-migration-console.md rename to _migration-assistant/migration-console/accessing-the-migration-console.md diff --git a/_migration-upgrade/migration-console/index.md b/_migration-assistant/migration-console/index.md similarity index 100% rename from _migration-upgrade/migration-console/index.md rename to _migration-assistant/migration-console/index.md diff --git a/_migration-upgrade/migration-console/migration-console-commands-references.md b/_migration-assistant/migration-console/migration-console-commands-references.md similarity index 100% rename from _migration-upgrade/migration-console/migration-console-commands-references.md rename to _migration-assistant/migration-console/migration-console-commands-references.md diff --git a/_migration-upgrade/migration-phases/backfill.md b/_migration-assistant/migration-phases/backfill.md similarity index 100% rename from _migration-upgrade/migration-phases/backfill.md rename to _migration-assistant/migration-phases/backfill.md diff --git a/_migration-assistant/migration-phases/deployment.md b/_migration-assistant/migration-phases/deployment.md index 2c4da875663..40e3e6244ee 100644 --- a/_migration-assistant/migration-phases/deployment.md +++ b/_migration-assistant/migration-phases/deployment.md @@ -4,8 +4,8 @@ title: Deploy parent: Migration phases nav_order: 2 redirect_from: - - /upgrade-to/snapshot-migrate/ - /migration-assistant/getting-started-with-data-migration/ + - /deploying-migration-assistant/ --- # Deploying Migration Assistant diff --git a/_migration-assistant/migration-phases/index.md b/_migration-assistant/migration-phases/index.md index 6db56d90d2e..c8b76a34dda 100644 --- a/_migration-assistant/migration-phases/index.md +++ b/_migration-assistant/migration-phases/index.md @@ -3,7 +3,7 @@ layout: default title: Migration phases parent: Migration Assistant nav_order: 30 -has_children: false +has_children: true has_toc: false permalink: /migration-assistant/migration-phases/ redirect-from: /migration-assistant/migration-phases/index/ diff --git a/_migration-upgrade/migration-phases/live-traffic-migration/index.md b/_migration-assistant/migration-phases/live-traffic-migration/index.md similarity index 100% rename from _migration-upgrade/migration-phases/live-traffic-migration/index.md rename to _migration-assistant/migration-phases/live-traffic-migration/index.md diff --git a/_migration-upgrade/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md b/_migration-assistant/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md similarity index 100% rename from _migration-upgrade/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md rename to _migration-assistant/migration-phases/live-traffic-migration/switching-traffic-from-the-source-cluster.md diff --git a/_migration-upgrade/migration-phases/live-traffic-migration/using-traffic-replayer.md b/_migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer.md similarity index 100% rename from _migration-upgrade/migration-phases/live-traffic-migration/using-traffic-replayer.md rename to _migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer.md diff --git a/_migration-upgrade/migration-phases/migrating-metadata.md b/_migration-assistant/migration-phases/migrating-metadata.md similarity index 100% rename from _migration-upgrade/migration-phases/migrating-metadata.md rename to _migration-assistant/migration-phases/migrating-metadata.md diff --git a/_migration-upgrade/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md b/_migration-assistant/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md similarity index 100% rename from _migration-upgrade/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md rename to _migration-assistant/migration-phases/planning-your-migration/assessing-your-cluster-for-migration.md diff --git a/_migration-upgrade/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md b/_migration-assistant/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md similarity index 100% rename from _migration-upgrade/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md rename to _migration-assistant/migration-phases/planning-your-migration/handling-field-type-breaking-changes.md diff --git a/_migration-upgrade/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md b/_migration-assistant/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md similarity index 100% rename from _migration-upgrade/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md rename to _migration-assistant/migration-phases/planning-your-migration/handling-type-mapping-deprecation.md diff --git a/_migration-upgrade/migration-phases/planning-your-migration/index.md b/_migration-assistant/migration-phases/planning-your-migration/index.md similarity index 100% rename from _migration-upgrade/migration-phases/planning-your-migration/index.md rename to _migration-assistant/migration-phases/planning-your-migration/index.md diff --git a/_migration-upgrade/migration-phases/planning-your-migration/verifying-migration-tools.md b/_migration-assistant/migration-phases/planning-your-migration/verifying-migration-tools.md similarity index 100% rename from _migration-upgrade/migration-phases/planning-your-migration/verifying-migration-tools.md rename to _migration-assistant/migration-phases/planning-your-migration/verifying-migration-tools.md diff --git a/_migration-upgrade/migration-phases/removing-migration-infrastructure.md b/_migration-assistant/migration-phases/removing-migration-infrastructure.md similarity index 100% rename from _migration-upgrade/migration-phases/removing-migration-infrastructure.md rename to _migration-assistant/migration-phases/removing-migration-infrastructure.md diff --git a/_migration-upgrade/overview/index.md b/_migration-assistant/overview/index.md similarity index 100% rename from _migration-upgrade/overview/index.md rename to _migration-assistant/overview/index.md diff --git a/_migration-upgrade/deploying-migration-assistant/index.md b/_migration-upgrade/deploying-migration-assistant/index.md deleted file mode 100644 index 1c559a81b1c..00000000000 --- a/_migration-upgrade/deploying-migration-assistant/index.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -layout: default -title: Deploying Migration Assistant -nav_order: 15 -has_children: true -permalink: /deploying-migration-assistant/ -redirect-from: - - /deploying-migration-assistant/index/ ---- - -# Deploying Migration Assistant - -This section provides information about the available options for deploying Migration Assistant. diff --git a/_migration-upgrade/migration-phases/index.md b/_migration-upgrade/migration-phases/index.md deleted file mode 100644 index d8f314d6b75..00000000000 --- a/_migration-upgrade/migration-phases/index.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -layout: default -title: Migration phases -nav_order: 30 -has_children: true -has_toc: false -permalink: /migration-phases/ -redirect_from: - - /migration-phases/index/ ---- - -# Migration phases - -This page outlines the core phases of migrating with Migration Assistant. Each scenario below consists of a sequence of common steps. Expand a scenario to explore the detailed flow. - ---- - - - -
    -Scenario 1 – Backfill Only - -
      -
    1. Assessment
      2. -
    2. Deployment
      3. - -
    3. Create Snapshot
      4. -
    4. Migrate Metadata
      5. -
    5. Migrate Data
      6. -
    6. Teardown
      7. -
    - -
    - -
    -Scenario 2 – Live Capture Only - -
      -
    1. Assessment
      2. -
    2. Deployment
      3. -
    3. Verify Backfill Components
      4. -
    4. Reroute Traffic from Source to Capture Proxy
      5. -
    5. Migrate Metadata
      6. -
    6. Verify Live Capture Components
      7. -
    7. Replay Captured Traffic
      8. -
    8. Reroute Traffic from Capture Proxy to Target
      9. -
    9. Teardown
      10. -
    - -
    - -
    -Scenario 3 – Live Capture with Backfill - -
      -
    1. Assessment
      2. -
    2. Deployment
      3. -
    3. Reroute Traffic from Source to Capture Proxy
      4. -
    4. Create Snapshot
      5. -
    5. Migrate Metadata
      6. -
    6. Migrate Data
      7. -
    7. Replay Traffic
      8. -
    8. Reroute Traffic from Capture Proxy to Target
      9. -
    9. Teardown
      10. -
    - -
    - ---- - -💡 *Need help getting started? Begin with [Planning your migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/planning-your-migration/index/).* diff --git a/_migration-upgrade/overview/architecture.md b/_migration-upgrade/overview/architecture.md deleted file mode 100644 index 48b3c8fbcfc..00000000000 --- a/_migration-upgrade/overview/architecture.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -layout: default -title: Architecture -nav_order: 15 -parent: Overview -permalink: /migration-assistant/overview/architecture/ ---- - -# Architecture - -The Migration Assistant architecture is based on the use of an AWS Cloud infrastructure, but most tools are designed to be cloud independent. A local containerized version of this solution is also available. - -The design deployed on AWS uses the following architecture. - -![Migration architecture overview]({{site.url}}{{site.baseurl}}/images/migrations/migrations-architecture-overview.png) - -Each node in the diagram correlates to the following steps in the migration process: - -1. Client traffic is directed to the existing cluster. -2. An Application Load Balancer with capture proxies relays traffic to a source while replicating data to Amazon Managed Streaming for Apache Kafka (Amazon MSK). -3. Using the migration console, you can initiate metadata migration to establish indexes, templates, component templates, and aliases on the target cluster. -4. With continuous traffic capture in place, you can use a `reindex-from-snapshot` process to capture data from your current index. -5. Once `Reindex-from-Snapshot` is complete, captured traffic is replayed from Amazon MSK to the target cluster by [Traffic Replayer](https://docs.opensearch.org/docs/latest/migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer/). -6. Performance and behavior of traffic sent to the source and target clusters are compared by reviewing logs and metrics. -7. After confirming that the target cluster's functionality meets expectations, clients are redirected to the new target. \ No newline at end of file diff --git a/_migration-upgrade/overview/is-migration-assistant-right-for-you.md b/_migration-upgrade/overview/is-migration-assistant-right-for-you.md deleted file mode 100644 index 1e7fc287974..00000000000 --- a/_migration-upgrade/overview/is-migration-assistant-right-for-you.md +++ /dev/null @@ -1,154 +0,0 @@ ---- -layout: default -title: Is Migration Assistant right for you? -nav_order: 10 -parent: Overview -permalink: /migration-assistant/overview/is-migration-assistant-right-for-you/ -redirect_from: - - /migration-assistant/is-migration-assistant-right-for-you/ ---- - -# Is Migration Assistant right for you? - -Deciding whether to use Migration Assistant depends on your specific upgrade path, infrastructure complexity, and operational goals. This page will help you evaluate whether Migration Assistant is right for your use case—or whether another tool might be a better fit. - -Migration Assistant was built to fill important gaps in common migration strategies. For example, if you're upgrading across multiple major versions—such as from Elasticsearch 6.8 to OpenSearch 2.19—Migration Assistant lets you do this in a single step. Other methods, like rolling upgrades or snapshot restores, require you to upgrade through each major version, often reindexing your data at every step. - -Migration Assistant also supports live traffic replication, allowing for zero-downtime migrations. This makes it a strong choice for production environments, where minimizing service disruption is critical. - -If your migration is limited to a static cluster configuration (like index templates and aliases), or if you're not concerned about downtime, simpler tools may be sufficient. But for complex migrations involving real-time traffic or major version jumps, Migration Assistant offers robust, flexible capabilities. - -## Supported migration paths - -The following matrix shows which source versions can be directly migrated to which OpenSearch target versions: - - -{% comment %}First, collect all unique target versions{% endcomment %} -{% assign all_targets = "" | split: "" %} -{% for path in site.data.migration-assistant.valid_migrations.migration_paths %} - {% for target in path.targets %} - {% assign all_targets = all_targets | push: target %} - {% endfor %} -{% endfor %} -{% assign unique_targets = all_targets | uniq | sort %} - - - - - - {% for target in unique_targets %} - - {% endfor %} - - - - {% for path in site.data.migration-assistant.valid_migrations.migration_paths %} - - - {% for target_version in unique_targets %} - - {% endfor %} - - {% endfor %} - -
    {{ target }}
    {{ path.source }} - {% if path.targets contains target_version %}✓{% endif %} -
    - -## Supported platforms - -**Source and target platforms** - -- Self-managed (on premises or hosted by a cloud provider) -- Amazon OpenSearch Service - - -**AWS Regions** - -Migration Assistant is supported in the following AWS Regions: - -- US East (N. Virginia, Ohio) -- US West (Oregon, N. California) -- Europe (Frankfurt, Ireland, London) -- Asia Pacific (Tokyo, Singapore, Sydney) -- AWS GovCloud (US-East, US-West)[^1] - -[^1]: In AWS GovCloud (US), `reindex-from-snapshot` (RFS) is limited to shard sizes of 80 GiB or smaller. - - - -## Supported components - -Before starting a migration, consider the scope of the components involved. The following table outlines components that should potentially be migrated, indicates whether they are supported by Migration Assistant, and provides recommendations. - -| Component | Supported | Recommendations | -| :--- |:--- | :--- | -| **Documents** | Yes | Migrate existing data with RFS and live traffic with capture and replay. | -| **Index settings** | Yes | Migrate with the `Metadata-Migration-Tool`. | -| **Index mappings** | Yes | Migrate with the `Metadata-Migration-Tool`. | -| **Index templates** | Yes | Migrate with the `Metadata-Migration-Tool`. | -| **Component templates** | Yes | Migrate with the `Metadata-Migration-Tool`. | -| **Aliases** | Yes | Migrate with the `Metadata-Migration-Tool`. | -| **Index State Management (ISM) policies** | Expected in 2025 | Manually migrate using an API. For more information about ISM support, see [issue #944](https://github.com/opensearch-project/opensearch-migrations/issues/944). | -| **Elasticsearch Kibana[^2] dashboards** | Expected in 2025 | This tool is only needed when migrating from Elasticsearch Kibana dashboards to OpenSearch Dashboards. Start by exporting JSON files from Kibana and importing them into OpenSearch Dashboards. For Elasticsearch versions 7.10.2 to 7.17, use the [`dashboardsSanitizer`](https://github.com/opensearch-project/opensearch-migrations/tree/main/dashboardsSanitizer) tool before importing X-Pack visualizations like Canvas and Lens in Kibana dashboards, as they may require recreation for compatibility with OpenSearch.| -| **Security constructs** | No | Configure roles and permissions based on cloud provider recommendations. For example, if using AWS, leverage AWS Identity and Access Management (IAM) for enhanced security management. | -| **Plugins** | No | Check plugin compatibility; some Elasticsearch plugins may not have direct OpenSearch equivalents. | - -[^2]: Support for Kibana 5.0 through 7.10.2 migration paths to OpenSearch Dashboards will be added in a future version. Kibana 8 and later are not supported. For more information, see [issue #944](https://github.com/opensearch-project/opensearch-migrations/issues/944). - -## Choosing your migration approach - -Use the following checklist to determine which Migration Assistant components best fit your use case. - -### Metadata migration - -Use [metadata migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrating-metadata/) if: - -- You need to migrate while mitigating breaking changes between the source and target clusters, such as differences in mappings, settings, aliases, or component templates. -- You want a relatively consistent configuration between the source and target clusters. - -### Backfill migration - -Use [backfill migration]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/) if: - -- You need to move historical data without disrupting live traffic. -- You want to backfill indexes from a specific point in time without impacting the source cluster. -- You want to verify historical data in the target cluster before switching over. -- You want to backfill using an existing or incremental snapshot. -- You need the fastest backfill option that includes reindexing. -- You want the ability to pause and resume migration. - -### RFS - -Use [RFS]({{site.url}}{{site.baseurl}}/migration-assistant/deploying-migration-assistant/getting-started-data-migration/) if: - -- You already use OpenSearch snapshots for backups. -- You need to migrate documents at scale in parallel, such as with Amazon Elastic Container Service (Amazon ECS). -- You require a data migration path as part of a zero-downtime migration. -- Your AWS Region supports RFS and your shard sizes are within supported limits. - -### Combination of all three - -Use a combination of all three migration types if: - -- You're performing a complex, multi-version migration. -- You require zero downtime and full validation of the target environment. -- You want end-to-end tooling for metadata, data movement, and cluster behavior comparison. -- You're cloning an existing cluster and changing the source's configuration. -- You're setting up disaster recovery. - -## Checklist - -Use this checklist to decide whether Migration Assistant is right for you: - -- Are you migrating across one or more major versions? - -- Do you need to maintain service availability with zero downtime? - -- Do you need to validate a new OpenSearch cluster before switching over? - -- Is your environment self-managed or running on Amazon OpenSearch Service? - -- Are you looking for tooling that can automate metadata migration and performance comparison? - -If you answered "yes" to most of these questions, Migration Assistant is likely the right solution for your migration. diff --git a/_migration-upgrade/overview/key-components.md b/_migration-upgrade/overview/key-components.md deleted file mode 100644 index cfa69e70311..00000000000 --- a/_migration-upgrade/overview/key-components.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -layout: default -title: Key components -nav_order: 10 -parent: Overview -permalink: /migration-assistant/overview/key-components/ ---- - -# Key components - -The following are the key components of Migration Assistant. - -## Elasticsearch/OpenSearch source - -In this solution, your source cluster operates on either Elasticsearch or OpenSearch and is hosted on Amazon Elastic Compute Cloud (Amazon EC2) instances or in a similar computing environment. A proxy is set up to interact with this source cluster, either positioned in front of or directly on the coordinating nodes of the cluster. - -## Migration console - -The migration console provides a migration-specific CLI and offers a variety of tools for streamlining the migration process. Everything necessary for completing a migration, other than cleaning up the migration resources, can be performed through this console. - -## Traffic capture proxy - -This component is designed for HTTP RESTful traffic. It forwards traffic to the source cluster and also splits and channels this traffic to a stream processing service for later playback. - -## Traffic Replayer - -Acting as a traffic simulation tool, [Traffic Replayer](https://docs.opensearch.org/docs/latest/migration-assistant/migration-phases/live-traffic-migration/using-traffic-replayer/) replays recorded request traffic to a target cluster, mirroring source traffic patterns. It links original requests and their responses to those directed at the target cluster, facilitating comparative analysis. - -## Metadata migration tool - -The metadata migration tool integrated into the Migration CLI can be used independently to migrate cluster metadata, including index mappings, index configuration settings, templates, component templates, and aliases. - -## Reindex-from-Snapshot - -`Reindex-from-Snapshot` (RFS) reindexes data from an existing snapshot. Amazon Elastic Container Service (Amazon ECS) workers coordinate the migration of documents from an existing snapshot, reindexing the documents in parallel to a target cluster. - -## Target cluster - -The target cluster is the destination cluster for migration or comparison in an A/B test. \ No newline at end of file From 1f597805a63a569353344454d91e10d592ad4a9d Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Thu, 3 Jul 2025 13:09:33 -0500 Subject: [PATCH 13/13] Add Migration Assistant to _config.yml Signed-off-by: Brian Presley --- _config.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/_config.yml b/_config.yml index 27f751e4010..a1d9d2a0464 100644 --- a/_config.yml +++ b/_config.yml @@ -218,10 +218,10 @@ clients_collection: name: Clients nav_fold: true -migration_upgrade_collection: +migrations_and_upgrades_collection: collections: - migration-upgrade: - name: Migration and upgrade + migrations-and-upgrades: + name: Migrations and upgrades nav_fold: true migration_assistant_collection: @@ -274,7 +274,13 @@ defaults: path: "_migrations-and-upgrades" values: section: "migrations-and-upgrades" - section-name: "Migrations and Upgrades" + section-name: "Migrations and upgrades" + - + scope: + path: "_migration-assistant" + values: + section: "migrations-assistant" + section-name: "Migration Assistant" # Enable or disable the site search # By default, just-the-docs enables its JSON file-based search. We also have an OpenSearch-driven search functionality.