diff --git a/backend/dataall/core/environment/__init__.py b/backend/dataall/core/environment/__init__.py index 7007ae59a..eda70ff55 100644 --- a/backend/dataall/core/environment/__init__.py +++ b/backend/dataall/core/environment/__init__.py @@ -1,3 +1,3 @@ """The central package of the application to work with the environment""" -from dataall.core.environment import api, cdk, tasks +from dataall.core.environment import api, db, cdk, tasks diff --git a/backend/dataall/core/environment/api/input_types.py b/backend/dataall/core/environment/api/input_types.py index 076416043..6add6346b 100644 --- a/backend/dataall/core/environment/api/input_types.py +++ b/backend/dataall/core/environment/api/input_types.py @@ -1,6 +1,6 @@ from dataall.base.api import gql from dataall.base.api.constants import GraphQLEnumMapper, SortDirection - +from dataall.core.environment.db.environment_enums import PolicyManagementOptions AwsEnvironmentInput = gql.InputType( name='AwsEnvironmentInput', @@ -101,7 +101,7 @@ class EnvironmentSortField(GraphQLEnumMapper): gql.Argument('groupUri', gql.NonNullableType(gql.String)), gql.Argument('IAMRoleArn', gql.NonNullableType(gql.String)), gql.Argument('environmentUri', gql.NonNullableType(gql.String)), - gql.Argument('dataallManaged', gql.NonNullableType(gql.Boolean)), + gql.Argument('dataallManaged', gql.NonNullableType(PolicyManagementOptions.toGraphQLEnum())), ], ) @@ -120,5 +120,6 @@ class EnvironmentSortField(GraphQLEnumMapper): arguments=[ gql.Argument('consumptionRoleName', gql.String), gql.Argument('groupUri', gql.String), + gql.Argument('dataallManaged', gql.NonNullableType(PolicyManagementOptions.toGraphQLEnum())), ], ) diff --git a/backend/dataall/core/environment/api/types.py b/backend/dataall/core/environment/api/types.py index 4479b7bab..487965889 100644 --- a/backend/dataall/core/environment/api/types.py +++ b/backend/dataall/core/environment/api/types.py @@ -8,7 +8,7 @@ resolve_user_role, ) from dataall.core.environment.api.enums import EnvironmentPermission - +from dataall.core.environment.db.environment_enums import PolicyManagementOptions EnvironmentUserPermission = gql.ObjectType( name='EnvironmentUserPermission', @@ -163,7 +163,7 @@ gql.Field(name='policy_name', type=gql.String), gql.Field(name='policy_type', type=gql.String), gql.Field(name='exists', type=gql.Boolean), - gql.Field(name='attached', type=gql.Boolean), + gql.Field(name='attached', type=gql.String), ], ) @@ -176,7 +176,7 @@ gql.Field(name='environmentUri', type=gql.String), gql.Field(name='IAMRoleArn', type=gql.String), gql.Field(name='IAMRoleName', type=gql.String), - gql.Field(name='dataallManaged', type=gql.Boolean), + gql.Field(name='dataallManaged', type=gql.NonNullableType(PolicyManagementOptions.toGraphQLEnum())), gql.Field(name='created', type=gql.String), gql.Field(name='updated', type=gql.String), gql.Field(name='deleted', type=gql.String), diff --git a/backend/dataall/core/environment/cdk/environment_stack.py b/backend/dataall/core/environment/cdk/environment_stack.py index c154bbd52..fd174975c 100644 --- a/backend/dataall/core/environment/cdk/environment_stack.py +++ b/backend/dataall/core/environment/cdk/environment_stack.py @@ -451,32 +451,33 @@ def create_group_environment_role(self, group: EnvironmentGroup, id: str): permissions=group_permissions, ).generate_policies() - external_managed_policies = [] - policy_manager = PolicyManager( - environmentUri=self._environment.environmentUri, - resource_prefix=self._environment.resourcePrefix, - role_name=group.environmentIAMRoleName, - account=self._environment.AwsAccountId, - region=self._environment.region, - ) - try: - managed_policies = policy_manager.get_all_policies() - except Exception as e: - logger.info(f'Not adding any managed policy because of exception: {e}') - # Known exception raised in first deployment because pivot role does not exist and cannot be assumed - managed_policies = [] - for policy in managed_policies: - # If there is a managed policy that exist it should be attached to the IAM role - if policy.get('exists', False): - external_managed_policies.append( - iam.ManagedPolicy.from_managed_policy_name( - self, - id=f'{self._environment.resourcePrefix}-managed-policy-{policy.get("policy_name")}', - managed_policy_name=policy.get('policy_name'), + with self.engine.scoped_session() as session: + external_managed_policies = [] + policy_manager = PolicyManager( + session=session, + account=self._environment.AwsAccountId, + region=self._environment.region, + environmentUri=self._environment.environmentUri, + resource_prefix=self._environment.resourcePrefix, + role_name=group.environmentIAMRoleName, + ) + try: + managed_policies = policy_manager.get_all_policies() + except Exception as e: + logger.info(f'Not adding any managed policy because of exception: {e}') + # Known exception raised in first deployment because pivot role does not exist and cannot be assumed + managed_policies = [] + for policy in managed_policies: + # If there is a managed policy that exist it should be attached to the IAM role + if policy.get('exists', False): + external_managed_policies.append( + iam.ManagedPolicy.from_managed_policy_name( + self, + id=f'{self._environment.resourcePrefix}-managed-policy-{policy.get("policy_name")}', + managed_policy_name=policy.get('policy_name'), + ) ) - ) - with self.engine.scoped_session() as session: data_policy = S3Policy( stack=self, tag_key='Team', diff --git a/backend/dataall/core/environment/db/__init__.py b/backend/dataall/core/environment/db/__init__.py index e69de29bb..8036795be 100644 --- a/backend/dataall/core/environment/db/__init__.py +++ b/backend/dataall/core/environment/db/__init__.py @@ -0,0 +1 @@ +from . import environment_enums, environment_models, environment_repositories diff --git a/backend/dataall/core/environment/db/environment_enums.py b/backend/dataall/core/environment/db/environment_enums.py new file mode 100644 index 000000000..8ac3a3193 --- /dev/null +++ b/backend/dataall/core/environment/db/environment_enums.py @@ -0,0 +1,7 @@ +from dataall.base.api import GraphQLEnumMapper + + +class PolicyManagementOptions(GraphQLEnumMapper): + FULLY_MANAGED = 'Fully-Managed' + PARTIALLY_MANAGED = 'Partially-Managed' + EXTERNALLY_MANAGED = 'Externally-Managed' diff --git a/backend/dataall/core/environment/db/environment_models.py b/backend/dataall/core/environment/db/environment_models.py index 7f5e9a122..15bd19c55 100644 --- a/backend/dataall/core/environment/db/environment_models.py +++ b/backend/dataall/core/environment/db/environment_models.py @@ -107,7 +107,7 @@ class ConsumptionRole(Base): groupUri = Column(String, nullable=False) IAMRoleName = Column(String, nullable=False) IAMRoleArn = Column(String, nullable=False) - dataallManaged = Column(Boolean, nullable=False, default=True) + dataallManaged = Column(String, nullable=False) created = Column(DateTime, default=datetime.datetime.now) updated = Column(DateTime, onupdate=datetime.datetime.now) deleted = Column(DateTime) diff --git a/backend/dataall/core/environment/db/environment_repositories.py b/backend/dataall/core/environment/db/environment_repositories.py index 5d787f7a3..1801b7142 100644 --- a/backend/dataall/core/environment/db/environment_repositories.py +++ b/backend/dataall/core/environment/db/environment_repositories.py @@ -160,7 +160,6 @@ def query_user_environment_consumption_roles(session, groups, uri, filter) -> Qu ) ) if filter and filter.get('groupUri'): - print('filter group') group = filter['groupUri'] query = query.filter( or_( diff --git a/backend/dataall/core/environment/services/environment_service.py b/backend/dataall/core/environment/services/environment_service.py index b064d759f..9645943fb 100644 --- a/backend/dataall/core/environment/services/environment_service.py +++ b/backend/dataall/core/environment/services/environment_service.py @@ -13,6 +13,7 @@ from dataall.base.aws.sts import SessionHelper from dataall.base.context import get_context from dataall.base.db.exceptions import AWSResourceNotFound +from dataall.core.environment.db.environment_enums import PolicyManagementOptions from dataall.core.organizations.db.organization_repositories import OrganizationRepository from dataall.core.permissions.services.environment_permissions import ( ENABLE_ENVIRONMENT_SUBSCRIPTIONS, @@ -46,6 +47,7 @@ from dataall.core.permissions.services.tenant_permissions import MANAGE_ENVIRONMENTS from dataall.core.stacks.db.stack_repositories import StackRepository from dataall.core.vpc.db.vpc_repositories import VpcRepository +from dataall.modules.shares_base.services.shares_enums import PrincipalType log = logging.getLogger(__name__) @@ -397,15 +399,21 @@ def invite_group(uri, data=None) -> (Environment, EnvironmentGroup): env_group_iam_role_arn = f'arn:aws:iam::{environment.AwsAccountId}:role/{env_group_iam_role_name}' env_role_imported = False - # If environment role is imported, then data.all should attach the policies at import time - # If environment role is created in environment stack, then data.all should attach the policies in the env stack + # If environment role is imported, then data.all should attach the policies at import time ( Fully Managed ) + # If environment role is created in environment stack, then data.all should attach the policies in the env stack ( Partially Managed - Here policy will be created but won't be attached ) + policy_management: str = ( + PolicyManagementOptions.FULLY_MANAGED.value + if env_role_imported is True + else PolicyManagementOptions.PARTIALLY_MANAGED.value + ) PolicyManager( + session=session, role_name=env_group_iam_role_name, environmentUri=environment.environmentUri, account=environment.AwsAccountId, region=environment.region, resource_prefix=environment.resourcePrefix, - ).create_all_policies(managed=env_role_imported) + ).create_all_policies(policy_management=policy_management) athena_workgroup = NamingConventionService( target_uri=environment.environmentUri, @@ -470,6 +478,7 @@ def remove_group(uri, group): group_membership = EnvironmentService.find_environment_group(session, group, environment.environmentUri) PolicyManager( + session=session, role_name=group_membership.environmentIAMRoleName, environmentUri=environment.environmentUri, account=environment.AwsAccountId, @@ -590,16 +599,17 @@ def add_consumption_role(uri, data=None) -> (Environment, EnvironmentGroup): groupUri=group, IAMRoleArn=IAMRoleArn, IAMRoleName=IAMRoleArn.split('/')[-1], - dataallManaged=data.get('dataallManaged', True), + dataallManaged=data.get('dataallManaged'), ) PolicyManager( + session=session, role_name=consumption_role.IAMRoleName, environmentUri=environment.environmentUri, account=environment.AwsAccountId, region=environment.region, resource_prefix=environment.resourcePrefix, - ).create_all_policies(managed=consumption_role.dataallManaged) + ).create_all_policies(policy_management=consumption_role.dataallManaged) session.add(consumption_role) session.commit() @@ -630,6 +640,7 @@ def remove_consumption_role(uri, env_uri): if consumption_role: PolicyManager( + session=session, role_name=consumption_role.IAMRoleName, environmentUri=environment.environmentUri, account=environment.AwsAccountId, @@ -668,6 +679,26 @@ def update_consumption_role(uri, env_uri, input): for key, value in input.items(): setattr(consumption_role, key, value) session.commit() + + # If the input consumption role is not Fully-Managed then attach the share policy if it exists + if consumption_role.dataallManaged == PolicyManagementOptions.FULLY_MANAGED.value: + environment: Environment = EnvironmentService.get_environment_by_uri(session, env_uri) + share_policy_manager = PolicyManager( + session=session, + account=environment.AwsAccountId, + region=environment.region, + environmentUri=environment.environmentUri, + resource_prefix=environment.resourcePrefix, + role_name=consumption_role.IAMRoleName, + ) + for policy_manager in [ + Policy + for Policy in share_policy_manager.initializedPolicies + if Policy.policy_type == 'SharePolicy' + ]: + managed_policy_list = policy_manager.get_policies_unattached_to_role() + policy_manager.attach_policies(managed_policy_list) + return consumption_role @staticmethod @@ -808,6 +839,15 @@ def paginated_all_environment_consumption_roles(uri, data=None) -> dict: def get_consumption_role(session, uri) -> Query: return EnvironmentRepository.get_consumption_role(session, uri) + @staticmethod + def get_role_policy_management_type(principal_type: str, principal_id: str): + with get_context().db_engine.scoped_session() as session: + if principal_type == PrincipalType.ConsumptionRole.value: + consumption_role: ConsumptionRole = EnvironmentService.get_consumption_role(session, uri=principal_id) + return consumption_role.dataallManaged + + return PolicyManagementOptions.FULLY_MANAGED.value + @staticmethod @ResourcePolicyService.has_resource_permission(environment_permissions.LIST_ENVIRONMENT_NETWORKS) def paginated_environment_networks(uri, data=None) -> dict: @@ -893,6 +933,7 @@ def delete_environment(uri): StackStatus.DELETE_COMPLETE.value, ]: PolicyManager( + session=session, role_name=environment.EnvironmentDefaultIAMRoleName, environmentUri=environment.environmentUri, account=environment.AwsAccountId, @@ -1111,16 +1152,12 @@ def get_template_from_resource_bucket(uri, template_name): @ResourcePolicyService.has_resource_permission(environment_permissions.GET_ENVIRONMENT) def resolve_consumption_role_policies(uri, IAMRoleName): environment = EnvironmentService.find_environment_by_uri(uri=uri) - return PolicyManager( - role_name=IAMRoleName, - environmentUri=uri, - account=environment.AwsAccountId, - region=environment.region, - resource_prefix=environment.resourcePrefix, - ).get_all_policies() - - @staticmethod - @ResourcePolicyService.has_resource_permission(environment_permissions.GET_ENVIRONMENT) - def get_consumption_role_by_name(uri, IAMRoleName): with get_context().db_engine.scoped_session() as session: - return EnvironmentRepository.get_environment_consumption_role_by_name(session, uri, IAMRoleName) + return PolicyManager( + session=session, + account=environment.AwsAccountId, + region=environment.region, + environmentUri=uri, + resource_prefix=environment.resourcePrefix, + role_name=IAMRoleName, + ).get_all_policies() diff --git a/backend/dataall/core/environment/services/managed_iam_policies.py b/backend/dataall/core/environment/services/managed_iam_policies.py index ebf7a90ce..434974a0e 100644 --- a/backend/dataall/core/environment/services/managed_iam_policies.py +++ b/backend/dataall/core/environment/services/managed_iam_policies.py @@ -3,6 +3,8 @@ import json from abc import ABC, abstractmethod from dataall.base.aws.iam import IAM +from dataall.core.environment.db.environment_enums import PolicyManagementOptions +from dataall.core.environment.db.environment_repositories import EnvironmentRepository logger = logging.getLogger(__name__) @@ -93,14 +95,8 @@ def _get_policy_names(self, base_policy_name): class PolicyManager(object): - def __init__( - self, - role_name, - account, - region, - environmentUri, - resource_prefix, - ): + def __init__(self, session, account, region, environmentUri, resource_prefix, role_name): + self.session = session self.role_name = role_name self.account = account self.region = region @@ -119,7 +115,7 @@ def _initialize_policy(self, managedPolicy): resource_prefix=self.resource_prefix, ) - def create_all_policies(self, managed) -> bool: + def create_all_policies(self, policy_management: str) -> bool: """ Manager that registers and calls all policies created by data.all modules and that need to be created for consumption roles and team roles @@ -136,7 +132,7 @@ def create_all_policies(self, managed) -> bool: policy=json.dumps(empty_policy), ) - if managed: + if policy_management == PolicyManagementOptions.FULLY_MANAGED.value: IAM.attach_role_policy( account_id=self.account, region=self.region, @@ -199,12 +195,27 @@ def get_all_policies(self) -> List[dict]: if policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name): policy_name_list.append(old_managed_policy_name) + # Check if the role_name is registered as a consumption role. + # If its a consumption role with a "Externally Managed" policy management then 'attached' will be marked as 'N/A' + externally_managed_role: bool = False + role_arn = f'arn:aws:iam::{self.account}:role/{self.role_name}' + consumption_role_details = EnvironmentRepository.find_consumption_roles_by_IAMArn( + session=self.session, uri=self.environmentUri, arn=role_arn + ) + if ( + consumption_role_details + and consumption_role_details.dataallManaged == PolicyManagementOptions.EXTERNALLY_MANAGED.value + ): + externally_managed_role = True + for policy_name in policy_name_list: policy_dict = { 'policy_name': policy_name, 'policy_type': policy_manager.policy_type, 'exists': policy_manager.check_if_policy_exists(policy_name=policy_name), - 'attached': policy_manager.check_if_policy_attached(policy_name=policy_name), + 'attached': 'N/A' + if externally_managed_role + else policy_manager.check_if_policy_attached(policy_name=policy_name), } all_policies.append(policy_dict) logger.info(f'All policies currently added to role: {self.role_name} are: {str(all_policies)}') diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py index 644591fec..e5dcf3e10 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py @@ -1,5 +1,6 @@ from dataall.base.db.exceptions import UnauthorizedOperation, InvalidInput from dataall.base.aws.iam import IAM +from dataall.core.environment.db.environment_enums import PolicyManagementOptions from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.environment.db.environment_models import EnvironmentGroup, ConsumptionRole from dataall.core.environment.services.managed_iam_policies import PolicyManager @@ -105,7 +106,7 @@ def _validate_iam_role_and_policy( session, principal_id, environment.environmentUri ) principal_role_name = consumption_role.IAMRoleName - managed = consumption_role.dataallManaged + managed = consumption_role.dataallManaged == PolicyManagementOptions.FULLY_MANAGED.value else: env_group: EnvironmentGroup = EnvironmentService.get_environment_group( @@ -125,6 +126,7 @@ def _validate_iam_role_and_policy( log.info('Verifying data.all managed share IAM policy is attached to IAM role...') share_policy_manager = PolicyManager( + session=session, role_name=principal_role_name, environmentUri=environment.environmentUri, account=environment.AwsAccountId, @@ -156,10 +158,6 @@ def _validate_iam_role_and_policy( # End of backwards compatibility unattached = policy_manager.get_policies_unattached_to_role() - if unattached and not managed and not attachMissingPolicies: - raise Exception( - f'Required customer managed policies {policy_manager.get_policies_unattached_to_role()} are not attached to role {principal_role_name}' - ) - elif unattached: + if unattached and (managed or attachMissingPolicies): managed_policy_list = policy_manager.get_policies_unattached_to_role() policy_manager.attach_policies(managed_policy_list) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index 791960ff6..9f6e8f825 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -6,6 +6,9 @@ from warnings import warn from dataall.base.db.exceptions import AWSServiceQuotaExceeded +from dataall.core.environment.db.environment_enums import PolicyManagementOptions +from dataall.core.environment.db.environment_models import ConsumptionRole +from dataall.core.environment.db.environment_repositories import EnvironmentRepository from dataall.core.environment.services.environment_service import EnvironmentService from dataall.base.db import utils from dataall.base.aws.sts import SessionHelper @@ -164,6 +167,15 @@ def check_target_role_access_policy(self) -> None: """ logger.info(f'Check target role {self.target_requester_IAMRoleName} access policy') + is_managed_role: bool = True + if self.share.principalType == PrincipalType.ConsumptionRole.value: + # When principalType is a consumptionRole type then principalId contains the uri of the consumption role + consumption_role: ConsumptionRole = EnvironmentRepository.get_consumption_role( + self.session, self.share.principalId + ) + if consumption_role.dataallManaged == PolicyManagementOptions.EXTERNALLY_MANAGED.value: + is_managed_role = False + key_alias = f'alias/{self.dataset.KmsAlias}' kms_client = KmsClient(self.dataset_account_id, self.source_environment.region) kms_key_id = kms_client.get_key_id(key_alias) @@ -201,13 +213,14 @@ def check_target_role_access_policy(self) -> None: self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return - unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() - if len(unattached_policies) > 0: - logger.info( - f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' - ) - self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) - return + if is_managed_role: + unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() + if len(unattached_policies) > 0: + logger.info( + f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' + ) + self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) + return s3_target_resources = [ f'arn:aws:s3:::{self.bucket_name}', @@ -399,7 +412,7 @@ def grant_target_role_access_policy(self): consumption_role = EnvironmentService.get_consumption_role( session=self.session, uri=self.share.principalId ) - if consumption_role.dataallManaged: + if consumption_role.dataallManaged == PolicyManagementOptions.FULLY_MANAGED.value: share_managed_policies = share_policy_service.get_managed_policies() share_policy_service.attach_policies(share_managed_policies) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index aac6fea78..e44aff335 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -6,7 +6,9 @@ from dataall.base.aws.iam import IAM from dataall.base.aws.sts import SessionHelper from dataall.base.db.exceptions import AWSServiceQuotaExceeded -from dataall.core.environment.db.environment_models import Environment +from dataall.core.environment.db.environment_enums import PolicyManagementOptions +from dataall.core.environment.db.environment_models import Environment, ConsumptionRole +from dataall.core.environment.db.environment_repositories import EnvironmentRepository from dataall.core.environment.services.environment_service import EnvironmentService from dataall.modules.s3_datasets.db.dataset_models import DatasetBucket from dataall.modules.s3_datasets_shares.aws.kms_client import ( @@ -74,6 +76,15 @@ def check_s3_iam_access(self) -> None: """ logger.info(f'Check target role {self.target_requester_IAMRoleName} access policy') + is_managed_role: bool = True + if self.share.principalType == PrincipalType.ConsumptionRole.value: + # When principalType is a consumptionRole type then principalId contains the uri of the consumption role + consumption_role: ConsumptionRole = EnvironmentRepository.get_consumption_role( + self.session, self.share.principalId + ) + if consumption_role.dataallManaged == PolicyManagementOptions.EXTERNALLY_MANAGED.value: + is_managed_role = False + key_alias = f'alias/{self.target_bucket.KmsAlias}' kms_client = KmsClient(self.source_account_id, self.source_environment.region) kms_key_id = kms_client.get_key_id(key_alias) @@ -113,13 +124,14 @@ def check_s3_iam_access(self) -> None: self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return - unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() - if len(unattached_policies) > 0: - logger.info( - f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' - ) - self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) - return + if is_managed_role: + unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() + if len(unattached_policies) > 0: + logger.info( + f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' + ) + self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) + return s3_target_resources = [f'arn:aws:s3:::{self.bucket_name}', f'arn:aws:s3:::{self.bucket_name}/*'] @@ -304,7 +316,7 @@ def grant_s3_iam_access(self): consumption_role = EnvironmentService.get_consumption_role( session=self.session, uri=self.share.principalId ) - if consumption_role.dataallManaged: + if consumption_role.dataallManaged == PolicyManagementOptions.FULLY_MANAGED.value: share_managed_policies = share_policy_service.get_managed_policies() share_policy_service.attach_policies(share_managed_policies) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_processors/s3_access_point_share_processor.py b/backend/dataall/modules/s3_datasets_shares/services/share_processors/s3_access_point_share_processor.py index d522bb79d..d28679c67 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_processors/s3_access_point_share_processor.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_processors/s3_access_point_share_processor.py @@ -11,6 +11,7 @@ ShareItemStatus, ShareObjectActions, ShareItemActions, + PrincipalType, ) from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation from dataall.modules.shares_base.db.share_object_repositories import ShareObjectRepository diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_processors/s3_bucket_share_processor.py b/backend/dataall/modules/s3_datasets_shares/services/share_processors/s3_bucket_share_processor.py index 447988156..af8e9f4ab 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_processors/s3_bucket_share_processor.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_processors/s3_bucket_share_processor.py @@ -1,6 +1,5 @@ import logging from datetime import datetime -from logging import exception from typing import List from dataall.modules.shares_base.services.share_exceptions import PrincipalRoleNotFound @@ -11,6 +10,7 @@ ShareItemStatus, ShareObjectActions, ShareItemActions, + PrincipalType, ) from dataall.modules.shares_base.db.share_object_repositories import ShareObjectRepository from dataall.modules.shares_base.db.share_state_machines_repositories import ShareStatusRepository diff --git a/backend/dataall/modules/shares_base/api/resolvers.py b/backend/dataall/modules/shares_base/api/resolvers.py index 285326e46..c8fc29749 100644 --- a/backend/dataall/modules/shares_base/api/resolvers.py +++ b/backend/dataall/modules/shares_base/api/resolvers.py @@ -288,6 +288,12 @@ def resolve_existing_shared_items(context: Context, source: ShareObject, **kwarg return ShareItemService.check_existing_shared_items(source) +def resolve_role_policy_management(context, source: ShareObject): + if not source: + return 'Not Available' + return EnvironmentService.get_role_policy_management_type(source.principalType, source.principalId) + + def list_shareable_objects(context: Context, source: ShareObject, filter: dict = None): if not source: return None diff --git a/backend/dataall/modules/shares_base/api/types.py b/backend/dataall/modules/shares_base/api/types.py index a6f92d00d..570564ca4 100644 --- a/backend/dataall/modules/shares_base/api/types.py +++ b/backend/dataall/modules/shares_base/api/types.py @@ -1,4 +1,5 @@ from dataall.base.api import gql +from dataall.core.environment.db.environment_enums import PolicyManagementOptions from dataall.modules.shares_base.services.shares_enums import ( ShareableType, PrincipalType, @@ -14,6 +15,7 @@ list_shareable_objects, resolve_user_role, resolve_can_view_logs, + resolve_role_policy_management, ) from dataall.core.environment.api.resolvers import resolve_environment @@ -171,6 +173,11 @@ resolver=resolve_user_role, ), gql.Field('permissions', gql.ArrayType(ShareObjectDataPermission.toGraphQLEnum())), + gql.Field( + name='policyManagement', + type=gql.NonNullableType(PolicyManagementOptions.toGraphQLEnum()), + resolver=resolve_role_policy_management, + ), ], ) diff --git a/backend/migrations/versions/0d1653ee4dc3_merging_disjoint_heads.py b/backend/migrations/versions/0d1653ee4dc3_merging_disjoint_heads.py new file mode 100644 index 000000000..918e71ebb --- /dev/null +++ b/backend/migrations/versions/0d1653ee4dc3_merging_disjoint_heads.py @@ -0,0 +1,24 @@ +"""merging disjoint heads + +Revision ID: 0d1653ee4dc3 +Revises: 77c3f1b2bec8, ba2da94739ab +Create Date: 2025-05-08 14:07:07.096840 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0d1653ee4dc3' +down_revision = ('77c3f1b2bec8', 'ba2da94739ab') +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/backend/migrations/versions/77c3f1b2bec8_consumption_role_column_update.py b/backend/migrations/versions/77c3f1b2bec8_consumption_role_column_update.py new file mode 100644 index 000000000..977e123f2 --- /dev/null +++ b/backend/migrations/versions/77c3f1b2bec8_consumption_role_column_update.py @@ -0,0 +1,79 @@ +"""Consumption role schema change and backfilling + +Revision ID: 77c3f1b2bec8 +Revises: af2e1362d4cb +Create Date: 2025-02-05 11:05:55.782419 + +""" + +from typing import List, Dict + +from alembic import op +import sqlalchemy as sa +from sqlalchemy import orm + +from dataall.core.environment.db.environment_enums import PolicyManagementOptions +from dataall.core.environment.db.environment_models import ConsumptionRole + +# revision identifiers, used by Alembic. +revision = '77c3f1b2bec8' +down_revision = 'af2e1362d4cb' +branch_labels = None +depends_on = None + + +def get_session(): + bind = op.get_bind() + session = orm.Session(bind=bind) + return session + + +def upgrade(): + # Update the column type to String and also remove the DEFAULT True + op.alter_column( + table_name='consumptionrole', + column_name='dataallManaged', + nullable=False, + existing_type=sa.Boolean(), + type_=sa.String(), + server_default=None, + ) + + session = get_session() + + # For all consumption roles, set the dataallManaged column value with the PolicyManagementOptions types + consumption_roles: List[ConsumptionRole] = session.query(ConsumptionRole).all() + for consumption_role in consumption_roles: + if consumption_role.dataallManaged == 'true': + consumption_role.dataallManaged = PolicyManagementOptions.FULLY_MANAGED.value + else: + consumption_role.dataallManaged = PolicyManagementOptions.PARTIALLY_MANAGED.value + session.add_all(consumption_roles) + session.commit() + + +def downgrade(): + session = get_session() + consumption_roles: List[ConsumptionRole] = session.query(ConsumptionRole).all() + # For each consumption role, get the policy management options and set value to True if FullyManaged else False + consumption_role_policy_mgmt_map: Dict[str, str] = { + consumption_role.consumptionRoleUri: True + if consumption_role.dataallManaged == PolicyManagementOptions.FULLY_MANAGED.value + else False + for consumption_role in consumption_roles + } + + op.drop_column(table_name='consumptionrole', column_name='dataallManaged') + op.add_column( + 'consumptionrole', + sa.Column('dataallManaged', sa.Boolean(), nullable=False, server_default=sa.sql.expression.true()), + ) + + # Update all the consumption.dataallManaged column with boolean values by using consumption_role_policy_mgmt_map map + consumption_roles: List[ConsumptionRole] = session.query(ConsumptionRole).all() + for consumption_role in consumption_roles: + consumption_role.dataallManaged = consumption_role_policy_mgmt_map.get( + consumption_role.consumptionRoleUri, True + ) + session.add_all(consumption_roles) + session.commit() diff --git a/frontend/src/design/components/InfoIconWithToolTip.js b/frontend/src/design/components/InfoIconWithToolTip.js new file mode 100644 index 000000000..e2ce78beb --- /dev/null +++ b/frontend/src/design/components/InfoIconWithToolTip.js @@ -0,0 +1,25 @@ +import { Tooltip } from '@mui/material'; +import InfoIcon from '@mui/icons-material/Info'; +import PropTypes from 'prop-types'; + +export const InfoIconWithToolTip = (props) => { + const { title, size, placement } = props; + + return ( + + + + ); +}; + +InfoIconWithToolTip.propTypes = { + title: PropTypes.any, + size: PropTypes.number, + placement: PropTypes.string +}; + +InfoIconWithToolTip.defaultProps = { + title: '', + size: 1, + placement: 'bottom' +}; diff --git a/frontend/src/design/components/index.js b/frontend/src/design/components/index.js index 724562cd1..ffa320b85 100644 --- a/frontend/src/design/components/index.js +++ b/frontend/src/design/components/index.js @@ -32,3 +32,4 @@ export * from './popovers'; export * from './SanitizedHTML'; export * from './NoAccessMaintenanceWindow'; export * from './UserModal'; +export * from './InfoIconWithToolTip'; diff --git a/frontend/src/modules/Catalog/components/RequestAccessModal.js b/frontend/src/modules/Catalog/components/RequestAccessModal.js index 316df29a2..573ff8f84 100644 --- a/frontend/src/modules/Catalog/components/RequestAccessModal.js +++ b/frontend/src/modules/Catalog/components/RequestAccessModal.js @@ -720,7 +720,9 @@ export const RequestAccessModal = (props) => { )} {!values.consumptionRole || - values.consumptionRole.dataallManaged || + values.consumptionRole.dataallManaged === 'Fully-Managed' || + values.consumptionRole.dataallManaged === + 'Externally-Managed' || isSharePolicyAttached ? ( ) : ( @@ -742,25 +744,24 @@ export const RequestAccessModal = (props) => { color="textSecondary" component="p" variant="caption" - > - {values.consumptionRole && - !( - values.consumptionRole.dataallManaged || - isSharePolicyAttached || - values.attachMissingPolicies - ) ? ( - - Selected consumption role is managed by - customer, but the share policy{' '} - {unAttachedPolicyNames} is - not attached. -
- Please attach it or let Data.all attach it for - you. -
- ) : ( - '' - )} + > + {values.consumptionRole && + values.consumptionRole.dataallManaged === + 'Partially-Managed' && + !isSharePolicyAttached ? ( + + Selected consumption role is partially + managed by customer and the share policy{' '} + {unAttachedPolicyNames} is + not attached. +
+ Please attach it or let Data.all attach it + for you. +
+ ) : ( + '' + )} + } /> diff --git a/frontend/src/modules/Environments/components/EnvironmentRoleAddForm.js b/frontend/src/modules/Environments/components/EnvironmentRoleAddForm.js index 8926de4dc..5a105ada6 100644 --- a/frontend/src/modules/Environments/components/EnvironmentRoleAddForm.js +++ b/frontend/src/modules/Environments/components/EnvironmentRoleAddForm.js @@ -1,27 +1,34 @@ import { GroupAddOutlined } from '@mui/icons-material'; import { LoadingButton } from '@mui/lab'; import { + Alert, Autocomplete, Box, CardContent, CircularProgress, Dialog, - FormControlLabel, - Switch, TextField, Typography } from '@mui/material'; import { Formik } from 'formik'; import { useSnackbar } from 'notistack'; import PropTypes from 'prop-types'; -import React from 'react'; import * as Yup from 'yup'; import { SET_ERROR, useDispatch } from 'globalErrors'; import { useClient, useFetchGroups } from 'services'; import { addConsumptionRoleToEnvironment } from '../services'; +import { policyManagementInfoMap } from '../../constants'; +import { InfoIconWithToolTip } from '../../../design'; export const EnvironmentRoleAddForm = (props) => { - const { environment, onClose, open, reloadRoles, ...other } = props; + const { + environment, + onClose, + open, + reloadRoles, + policyManagementOptions, + ...other + } = props; const { enqueueSnackbar } = useSnackbar(); const dispatch = useDispatch(); const client = useClient(); @@ -95,7 +102,7 @@ export const EnvironmentRoleAddForm = (props) => { { consumptionRoleName: Yup.string() .max(255) .required('*Consumption Role Name is required'), - IAMRoleArn: Yup.string().required('*IAM Role Arn is required') + IAMRoleArn: Yup.string().required('*IAM Role Arn is required'), + dataallManaged: Yup.string() + .required( + 'Policy Management option required. Please select a valid option' + ) + .oneOf(policyManagementOptions.map((obj) => obj.key)) })} onSubmit={async ( values, @@ -181,30 +193,68 @@ export const EnvironmentRoleAddForm = (props) => { /> - { + if (value && value.key) { + setFieldValue('dataallManaged', value.key); + } else { + setFieldValue('dataallManaged', ''); + } + }} + renderOption={(props, option) => { + const { key, ...propOptions } = props; + return ( + + {option.label} + + {policyManagementInfoMap[option.key] != null + ? policyManagementInfoMap[option.key] + : 'Invalid Option for policy management.'} + + } + placement={'right-start'} + size={1} + /> + + ); + }} + renderInput={(params) => ( + - } - label={ -
- Data.all managed - - Allow Data.all to attach IAM policies to this role - -
- } + )} />
+ {values.dataallManaged === 'EXTERNALLY_MANAGED' ? ( + + + With "Externally-Managed" policy management, you are + completely responsible for attaching / giving your + consumption role appropriate permissions. Please select + "Externally-Managed" if you know that your role has some + super-user permissions or if you are completely managing + the role and its policies. + + + ) : ( +
+ )} { const dispatch = useDispatch(); const { enqueueSnackbar } = useSnackbar(); const client = useClient(); + const [policyAttachStatus, setPolicyAttachStatus] = useState('Not Attached'); useEffect(() => { if (client) { @@ -318,6 +322,11 @@ export const IAMRolePolicyDataGridCell = ({ environmentUri, IAMRoleName }) => { ); if (!response.errors) { setManagedPolicyDetails(response.data.getConsumptionRolePolicies); + setPolicyAttachStatus( + getIAMPolicyAttachementStatus( + response.data.getConsumptionRolePolicies + ) + ); } else { dispatch({ type: SET_ERROR, error: response.errors[0].message }); } @@ -328,27 +337,37 @@ export const IAMRolePolicyDataGridCell = ({ environmentUri, IAMRoleName }) => { } }; + const getIAMPolicyTagColour = () => { + if (policyAttachStatus === 'N/A') return 'warning'; + if (policyAttachStatus === 'Not Attached') return 'error'; + if (policyAttachStatus === 'No Policies Present') return 'error'; + return 'success'; + }; + + const getIAMPolicyAttachementStatus = (managedPolicyDetails) => { + if (managedPolicyDetails.length === 0) { + return 'No Policies Present'; + } + const is_policy_attach = managedPolicyDetails.map( + (policy) => policy.attached + ); + if (is_policy_attach.every((policy) => policy === 'N/A')) { + return 'N/A'; + } + if (is_policy_attach.some((policy) => policy === 'false')) { + return 'Not Attached'; + } + return 'Attached'; + }; + return ( {isLoading ? ( ) : ( -