Add optional Ranger HDFS plugin and updates to the policy

Varun Rao Bhamidimarri · Varun Rao Bhamidimarri · commit 109a35f2c534 · 2023-01-18T13:34:10.000-08:00
diff --git a/aws_emr_blog_v3/cloudformation/ec2-win-ad.template b/aws_emr_blog_v3/cloudformation/ec2-win-ad.template
@@ -290,6 +290,22 @@ Resources:
                   - !Ref 'DefaultADUserPassword'
                   - "' -Force)\n"
                   - "Enable-ADAccount -Identity \"analyst2\"\n"
+                  - "New-ADUser -Name \"tina\" -OtherAttributes @{'title'=\"tina\";'mail'=\"tina@"
+                  - !Ref 'DomainDNSName'
+                  - "\"}\n"
+                  - "Enable-ADAccount -Identity \"tina\"\n"
+                  - "Set-ADAccountPassword -Identity 'tina' -Reset -NewPassword (ConvertTo-SecureString -AsPlainText '"
+                  - !Ref 'DefaultADUserPassword'
+                  - "' -Force)\n"
+                  - "Enable-ADAccount -Identity \"tina\"\n"
+                  - "New-ADUser -Name \"alex\" -OtherAttributes @{'title'=\"alex\";'mail'=\"alex@"
+                  - !Ref 'DomainDNSName'
+                  - "\"}\n"
+                  - "Enable-ADAccount -Identity \"alex\"\n"
+                  - "Set-ADAccountPassword -Identity 'alex' -Reset -NewPassword (ConvertTo-SecureString -AsPlainText '"
+                  - !Ref 'DefaultADUserPassword'
+                  - "' -Force)\n"
+                  - "Enable-ADAccount -Identity \"alex\"\n"
           services:
             windows:
               cfn-hup:
diff --git a/aws_emr_blog_v3/cloudformation/step2_ranger-rds-emr.template b/aws_emr_blog_v3/cloudformation/step2_ranger-rds-emr.template
@@ -311,6 +311,11 @@ Parameters:
     Default: false
     Type: String
     AllowedValues: [ true, false ]
+  InstallRangerHDFSPlugin:
+    Description: Flag to control if the Ranger HDFS plugin will be added
+    Default: false
+    Type: String
+    AllowedValues: [ true, false ]
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -417,6 +422,7 @@ Resources:
         VPC: !Ref VPC
         emrReleaseLabel: !Ref emrReleaseLabel
         CreateNonEMRResources: !Ref CreateNonEMRResources
+        InstallRangerHDFSPlugin: !Ref InstallRangerHDFSPlugin
         ClusterSubnetID: !If [ InstallEMRRangerinPublicSubnet, !Ref PublicSubnet1AID, !Ref PrivateSubnet1AID ]
         EMRLogDir: !Ref EMRLogDir
         MasterInstanceCount: !Ref 'MasterInstanceCount'
diff --git a/aws_emr_blog_v3/code/launch-cluster/cremr.py b/aws_emr_blog_v3/code/launch-cluster/cremr.py
@@ -62,18 +62,6 @@ def create(event, context):
                               ],
                               'Applications': applist,
                               'Steps': [
-                                  {
-                                      "Name": "InstallHiveHDFSRangerPlugin",
-                                      "ActionOnFailure": "CONTINUE",
-                                      "HadoopJarStep": {
-                                          "Jar": scriptRunnerJar,
-                                          "Args": [
-                                              "/mnt/tmp/aws-blog-emr-ranger/scripts/emr-steps/install-hdfs-ranger-plugin.sh",
-                                              event["ResourceProperties"]["RangerHostname"],
-                                              event["ResourceProperties"]["StackRegion"]
-                                          ]
-                                      }
-                                  },
                                   {
                                       "Name": "CreateDefaultHiveTables",
                                       "ActionOnFailure": "CONTINUE",
@@ -317,6 +305,19 @@ def create(event, context):
                 }
             })
 
+        if event["ResourceProperties"]["InstallRangerHDFSPlugin"] == "true":
+            cluster_parameters['Steps'].append({
+                "Name": "InstallHiveHDFSRangerPlugin",
+                "ActionOnFailure": "CONTINUE",
+                "HadoopJarStep": {
+                    "Jar": scriptRunnerJar,
+                    "Args": [
+                        "/mnt/tmp/aws-blog-emr-ranger/scripts/emr-steps/install-hdfs-ranger-plugin.sh",
+                        event["ResourceProperties"]["RangerHostname"],
+                        event["ResourceProperties"]["StackRegion"]
+                    ]
+                }
+            },)
         # Set the default hive properties
         if event["ResourceProperties"]["EnableGlueSupport"] == "true":
             hive_site_properties = {
diff --git a/aws_emr_blog_v3/inputdata/ranger-hdfs-policy-hdfs-home-dir.json b/aws_emr_blog_v3/inputdata/ranger-hdfs-policy-hdfs-home-dir.json
@@ -0,0 +1,48 @@
+{
+ "service": "hadoopdev",
+ "name": "Access to /user for home dir",
+ "policyType": 0,
+ "policyPriority": 0,
+ "description": "",
+ "isAuditEnabled": true,
+ "resources": {
+  "path": {
+   "values": [
+    "/user"
+   ],
+   "isExcludes": false,
+   "isRecursive": false
+  }
+ },
+ "policyItems": [
+  {
+   "accesses": [
+    {
+     "type": "write",
+     "isAllowed": true
+    }
+   ],
+   "users": [
+    "{USER}"
+   ],
+   "groups": [],
+   "roles": [],
+   "conditions": [],
+   "delegateAdmin": false
+  }
+ ],
+ "denyPolicyItems": [],
+ "allowExceptions": [],
+ "denyExceptions": [],
+ "dataMaskPolicyItems": [],
+ "rowFilterPolicyItems": [],
+ "serviceType": "hdfs",
+ "options": {},
+ "validitySchedules": [],
+ "policyLabels": [],
+ "zoneName": "",
+ "isDenyAllElse": false,
+ "id": 108,
+ "isEnabled": true,
+ "version": 1
+}
diff --git a/aws_emr_blog_v3/inputdata/ranger-hdfs-policy-user-home-dir.json b/aws_emr_blog_v3/inputdata/ranger-hdfs-policy-user-home-dir.json
@@ -1,6 +1,6 @@
 {
  "service": "hadoopdev",
- "name": "User path",
+ "name": "User home dir in HDFS",
  "policyType": 0,
  "policyPriority": 0,
  "description": "",
diff --git a/aws_emr_blog_v3/scripts/emr-steps/install-hdfs-ranger-plugin.sh b/aws_emr_blog_v3/scripts/emr-steps/install-hdfs-ranger-plugin.sh
@@ -76,7 +76,7 @@ aws secretsmanager get-secret-value --secret-id emr/rangerGAagentkey --version-s
 
 openssl pkcs12 -export -in ${ranger_agents_certs_path}/certificateChain.pem -inkey ${ranger_agents_certs_path}/privateKey.pem -chain -CAfile ${ranger_agents_certs_path}/certificateChain.pem -name ${keystore_alias} -out ${ranger_agents_certs_path}/keystore.p12 -password pass:${keystore_password}
 keytool -delete -alias ${keystore_alias} -keystore ${keystore_location} -storepass ${keystore_password} -noprompt || true
-sudo keytool -importkeystore -deststorepass ${keystore_password} -destkeystore ${keystore_location} -srckeystore ${ranger_agents_certs_path}/keystore.p12 -srcstoretype PKCS12 -srcstorepass ${keystore_password}
+sudo keytool -importkeystore -deststorepass ${keystore_password} -destkeystore ${keystore_location} -srckeystore ${ranger_agents_certs_path}/keystore.p12 -srcstoretype PKCS12 -srcstorepass ${keystore_password} -noprompt || true
 sudo chmod 444 ${keystore_location}
 # -----
 
@@ -101,17 +101,12 @@ cd $installpath/$ranger_hdfs_plugin
 
 ## Updates for new Ranger
 mkdir -p /usr/lib/ranger/hadoop/etc
-sudo ln -s /etc/hadoop /usr/lib/ranger/hadoop/etc/
+sudo ln -s /etc/hadoop /usr/lib/ranger/hadoop/etc/ || true
 sudo ln -s /usr/lib/ranger/hadoop/etc/hadoop/conf/hdfs-site.xml /usr/lib/ranger/hadoop/etc/hadoop/hdfs-site.xml || true
 sudo cp -r $installpath/$ranger_hdfs_plugin/lib/* /usr/lib/hadoop-hdfs/lib/
 sudo cp /usr/lib/hadoop-hdfs/lib/ranger-hdfs-plugin-impl/*.jar /usr/lib/hadoop-hdfs/lib/ || true
-sudo ln -s /etc/hadoop/ /usr/lib/ranger/hadoop/
+sudo ln -s /etc/hadoop/ /usr/lib/ranger/hadoop/ || true
 
-## Copy the keystore and strustone information
-sudo cp /etc/hive/conf/ranger-plugin-keystore.jks /etc/hadoop/conf/
-sudo cp /etc/hive/conf/ranger-keystore-creds.jceks /etc/hadoop/conf/
-sudo cp /etc/hive/conf/ranger-plugin-truststore.jks /etc/hadoop/conf/
-sudo cp /etc/hive/conf/ranger-truststore-creds.jceks /etc/hadoop/conf/
 #SSL configs
 sudo sed -i "s|POLICY_MGR_URL=.*|POLICY_MGR_URL=$RANGER_HTTP_URL|g" install.properties
 sudo sed -i "s|SSL_TRUSTSTORE_FILE_PATH=.*|SSL_TRUSTSTORE_FILE_PATH=${truststore_location}|g" install.properties
diff --git a/aws_emr_blog_v3/scripts/emr-steps/loadDataIntoHDFS.sh b/aws_emr_blog_v3/scripts/emr-steps/loadDataIntoHDFS.sh
@@ -5,17 +5,17 @@ set -x
 # Creates dummy HDFS data for testing
 #================================================================
 #% SYNOPSIS
-#+    createHiveTables.sh args ...
+#+    loadDataIntoHDFS.sh args ...
 #%
 #% DESCRIPTION
 #%    Uses Hadoop commands to create dummy HDFS data for testing
 #%
 #% EXAMPLES
-#%    createHiveTables.sh args ..
+#%    loadDataIntoHDFS.sh args ..
 #%
 #================================================================
 #- IMPLEMENTATION
-#-    version         createHiveTables.sh 1.0
+#-    version         loadDataIntoHDFS.sh 1.0
 #-    author          Varun Bhamidimarri
 #-    license         MIT license
 #-
@@ -32,7 +32,11 @@ sudo aws s3 cp $hdfs_data_location/football_coach.tsv . --region us-east-1
 sudo aws s3 cp $hdfs_data_location/football_coach_position.tsv . --region us-east-1
 sudo -u hdfs hadoop fs -mkdir -p /user/analyst1
 sudo -u hdfs hadoop fs -mkdir -p /user/analyst2
+sudo -u hdfs hadoop fs -mkdir -p /user/tina
+sudo -u hdfs hadoop fs -mkdir -p /user/alex
 sudo -u hdfs hadoop fs -put -f football_coach.tsv /user/analyst1
 sudo -u hdfs hadoop fs -put -f football_coach_position.tsv /user/analyst2
 sudo -u hdfs hadoop fs -chown -R analyst1:analyst1 /user/analyst1
 sudo -u hdfs hadoop fs -chown -R analyst2:analyst2 /user/analyst2
+sudo -u hdfs hadoop fs -chown -R tina:tina /user/tina
+sudo -u hdfs hadoop fs -chown -R alex:alex /user/alex

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"service": "hadoopdev",`
`3`		`- "name": "User path",`
	`3`	`+ "name": "User home dir in HDFS",`
`4`	`4`	`"policyType": 0,`
`5`	`5`	`"policyPriority": 0,`
`6`	`6`	`"description": "",`