Removed deployment_type, changed to secure_cluster. This now allows independent control of HDFS HA and Secure Cluster options during deployment.

smithzc · smithzc · commit f31231ab2e3b · 2019-09-17T16:12:13.000Z
diff --git a/README.md b/README.md
@@ -40,11 +40,11 @@ This automation supports using a local KDC deployed on the Cloudera Manager inst
 
 Also - for cluster management, you will need to manually create at a minimum the HDFS Superuser Principal as [detailed here](https://www.cloudera.com/documentation/enterprise/latest/topics/cm_sg_using_cm_sec_config.html#create-hdfs-superuser) after deployment.
 
-Enabling Kerberos is managed using a terraform metadata tag "deployment_type".   Setting this value to "secure" will enable cluster security as part of the setup process.  Changing this to "simple" will deploy an unsecured cluster.  By default this value is set to "simple" for speed of deployment and ease of use for those not familiar with secure cluster operation.
-
+Enabling Kerberos is managed using a terraform metadata tag "secure_cluster"   Setting this value to "True" will enable cluster security as part of the setup process.  Changing this to "False" will deploy an unsecured cluster.
+  
 ## High Availability
 
-High Availability is also offered as part of the deployment process.  When secure cluster operation is chosen this is enabled by default.  It can be disabled by either changing the deployment_type to "simple", or modifying the [deploy_on_oci.py](https://github.com/oracle/oci-quickstart-cloudera/blob/master/scripts/deploy_on_oci.py#L60) script and changing the value for "hdfs_ha".
+High Availability for HDFS services is also offered as part of the deployment process.  This can be toggled during the installation process by setting the value to "True".
 
 ## Metadata and MySQL
 
diff --git a/compute.tf b/compute.tf
@@ -15,7 +15,6 @@ module "bastion" {
 	cloudera_manager = "cdh-utility-1.public${var.availability_domain}.${module.network.vcn-dn}"
 	cm_version = "${var.cm_version}"
 	cdh_version = "${var.cdh_version}"
-	deployment_type = "${var.deployment_type}"
 }
 
 module "utility" {
@@ -40,8 +39,8 @@ module "utility" {
 	worker_shape = "${var.worker_instance_shape}"
 	block_volume_count = "${var.block_volumes_per_worker}"
 	AD = "${var.availability_domain}"
-	deployment_type = "${var.deployment_type}"
 	hdfs_ha = "${var.hdfs_ha}"
+	secure_cluster = "${var.secure_cluster}"
 	cluster_name = "${var.cluster_name}"
 }
 
@@ -62,7 +61,6 @@ module "master" {
         cloudera_manager = "cdh-utility-1.public${var.availability_domain}.${module.network.vcn-dn}"
         cm_version = "${var.cm_version}"
         cdh_version = "${var.cdh_version}"
-	deployment_type = "${var.deployment_type}"
 }
 
 module "worker" {
@@ -85,5 +83,4 @@ module "worker" {
         cm_version = "${var.cm_version}"
         cdh_version = "${var.cdh_version}"
 	block_volume_count = "${var.block_volumes_per_worker}"
-	deployment_type = "${var.deployment_type}"
 }
diff --git a/modules/bastion/main.tf b/modules/bastion/main.tf
@@ -17,7 +17,6 @@ resource "oci_core_instance" "Bastion" {
     cloudera_manager    = "${var.cloudera_manager}"
     cdh_version         = "${var.cdh_version}"
     cm_version          = "${var.cm_version}"
-    deployment_type     = "${var.deployment_type}" 
   }
 
   timeouts {
diff --git a/modules/bastion/variables.tf b/modules/bastion/variables.tf
@@ -15,7 +15,6 @@ variable "image_ocid" {}
 variable "cm_version" {}
 variable "cdh_version" {}
 variable "cloudera_manager" {}
-variable "deployment_type" {}
 
 # ---------------------------------------------------------------------------------------------------------------------
 # Optional variables
diff --git a/modules/master/main.tf b/modules/master/main.tf
@@ -19,7 +19,6 @@ resource "oci_core_instance" "Master" {
     cloudera_manager    = "${var.cloudera_manager}"
     cdh_version         = "${var.cdh_version}"
     cm_version          = "${var.cm_version}"
-    deployment_type     = "${var.deployment_type}" 
   }
 
   timeouts {
diff --git a/modules/master/variables.tf b/modules/master/variables.tf
@@ -15,7 +15,6 @@ variable "image_ocid" {}
 variable "cm_version" {}
 variable "cdh_version" {}
 variable "cloudera_manager" {}
-variable "deployment_type" {}
 
 # ---------------------------------------------------------------------------------------------------------------------
 # Optional variables
diff --git a/modules/utility/main.tf b/modules/utility/main.tf
@@ -21,7 +21,7 @@ resource "oci_core_instance" "Utility" {
     worker_shape        = "${var.worker_shape}"
     block_volume_count  = "${var.block_volume_count}"	
     availability_domain = "${var.AD}"
-    deployment_type     = "${var.deployment_type}"
+    secure_cluster      = "${var.secure_cluster}"
     hdfs_ha		= "${var.hdfs_ha}"
     cluster_name	= "${var.cluster_name}"
   }
diff --git a/modules/utility/variables.tf b/modules/utility/variables.tf
@@ -20,7 +20,7 @@ variable "AD" {}
 variable "cloudera_manager" {}
 variable "cm_install" {}
 variable "deploy_on_oci" {}
-variable "deployment_type" {}
+variable "secure_cluster" {}
 variable "hdfs_ha" {} 
 variable "cluster_name" {}
 
diff --git a/modules/worker/main.tf b/modules/worker/main.tf
@@ -20,7 +20,6 @@ resource "oci_core_instance" "Worker" {
     cdh_version         = "${var.cdh_version}"
     cm_version          = "${var.cm_version}" 
     block_volume_count  = "${var.block_volume_count}"
-    deployment_type     = "${var.deployment_type}"
   }
 
   timeouts {
diff --git a/modules/worker/variables.tf b/modules/worker/variables.tf
@@ -16,7 +16,6 @@ variable "cm_version" {}
 variable "cdh_version" {}
 variable "cloudera_manager" {}
 variable "block_volume_count" {}
-variable "deployment_type" {}
 
 # ---------------------------------------------------------------------------------------------------------------------
 # Optional variables
diff --git a/schema.yaml b/schema.yaml
@@ -13,9 +13,11 @@ groupings:
   - ${cm_version}
   - ${cdh_version}
   - ${cluster_name}
-  - ${deployment_type}
   - ${log_volume_size_in_gbs}
   - ${cloudera_volume_size_in_gbs}
+- title: "Security"
+  variables:
+  - ${secure_cluster}
 - title: "High Availability"
   variables:
   - ${hdfs_ha}
@@ -102,12 +104,13 @@ variables:
     maximum: 3
     required: true
 
-  deployment_type:
+  secure_cluster:
     type: enum
     enum:
-    - "simple"
-    - "secure"
-    title: "Deployment Type"
+    - "True"
+    - "False"
+    title: "Kerberos Security"
+    description: "Enable Kerberos for Secure Cluster?"
     required: true
 
   hdfs_ha:
@@ -116,7 +119,7 @@ variables:
     - "True"
     - "False"
     title: "High Availability"
-    description: "Enable High Availability?"
+    description: "Enable HDFS High Availability?"
     required: true
 
   worker_instance_shape:
diff --git a/scripts/boot.sh b/scripts/boot.sh
@@ -9,7 +9,6 @@ cdh_major_version=`echo $cdh_version | cut -d '.' -f1`
 cm_version=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cm_version`
 cm_major_version=`echo  $cm_version | cut -d '.' -f1`
 block_volume_count=`curl -L http://169.254.169.254/opc/v1/instance/metadata/block_volume_count`
-deployment_type=`curl -L http://169.254.169.254/opc/v1/instance/metadata/`
 EXECNAME="TUNING"
 log "->TUNING START"
 sed -i.bak 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
diff --git a/scripts/cms_mysql.sh b/scripts/cms_mysql.sh
@@ -13,7 +13,7 @@ cm_major_version=`echo  $cm_version | cut -d '.' -f1`
 availability_domain=`curl -L http://169.254.169.254/opc/v1/instance/metadata/availability_domain`
 worker_shape=`curl -L http://169.254.169.254/opc/v1/instance/metadata/worker_shape`
 worker_disk_count=`curl -L http://169.254.169.254/opc/v1/instance/metadata/block_volume_count`
-deployment_type=`curl -L http://169.254.169.254/opc/v1/instance/metadata/deployment_type`
+secure_cluster=`curl -L http://169.254.169.254/opc/v1/instance/metadata/secure_cluster`
 hdfs_ha=`curl -L http://169.254.169.254/opc/v1/instance/metadata/hdfs_ha`
 cluster_name=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cluster_name`
 EXECNAME="TUNING"
@@ -407,11 +407,21 @@ for w in `seq 1 $num_workers`; do
 done;
 log "-->Host List: ${fqdn_list}"
 log "-->Cluster Build"
-if [ $hdfs_ha = "True" ]; then 
-	log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -D ${deployment_type} -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name}"
-	python /var/lib/cloud/instance/scripts/deploy_on_oci.py -D ${deployment_type} -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name} 2>&1 1>> $LOG_FILE	
+if [ $secure_cluster = "True" ]; then 
+	if [ $hdfs_ha = "True" ]; then 
+		log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -S -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name}"
+		python /var/lib/cloud/instance/scripts/deploy_on_oci.py -S -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name} 2>&1 1>> $LOG_FILE	
+	else
+		log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -S -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name}"
+		python /var/lib/cloud/instance/scripts/deploy_on_oci.py -S -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name} 2>&1 1>> $LOG_FILE
+	fi
 else
-	log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -D ${deployment_type} -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name}"
-	python /var/lib/cloud/instance/scripts/deploy_on_oci.py -D ${deployment_type} -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name} 2>&1 1>> $LOG_FILE
+        if [ $hdfs_ha = "True" ]; then
+                log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name}"
+                python /var/lib/cloud/instance/scripts/deploy_on_oci.py -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name} 2>&1 1>> $LOG_FILE
+        else
+                log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name}"
+                python /var/lib/cloud/instance/scripts/deploy_on_oci.py -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cdh_version} -ad ${availability_domain} -N ${cluster_name} 2>&1 1>> $LOG_FILE
+	fi
 fi
 log "->DONE"
diff --git a/scripts/deploy_on_oci.py b/scripts/deploy_on_oci.py
@@ -29,7 +29,6 @@
 host_fqdn_list = []
 data_tiering = 'False'
 nvme_disks = 0
-deployment_type = 'simple'  # type: str
 availability_domain = 'None'
 # Custom Global Parameters - Customize below here
 debug = 'False'  # type: str
@@ -42,7 +41,7 @@
 cdh_version = ' '  # type: str
 cluster_primary_version = ' '
 kafka_parcel_url = ' '
-secure_cluster = 'True'  # type: bool
+secure_cluster = 'False'  # type: bool
 hdfs_ha = 'False'  # type: bool
 # These should match what is in the Cloudera Manager CloudInit bootstrap file and instance boot files
 realm = 'HADOOP.COM'
@@ -2016,11 +2015,10 @@ def options_parser(args=None):
     global objects
     global remote_parcel_url
     global cdh_version
-    parser = argparse.ArgumentParser(prog='python deploy_on_oci.py', description='Deploy a Cloudera EDH %s Cluster on '
+    parser = argparse.ArgumentParser(prog='python deploy_on_oci.py', description='Deploy a Cloudera EDH Cluster on '
                                                                                  'OCI using cm_client with Cloudera '
-                                                                                 'Manager API %s' % (cdh_version,
-                                                                                                     api_version))
-    parser.add_argument('-D', '--deployment_type', metavar='deployment_type', help='simple, no Kerberos at deployment, or secure to enable')
+                                                                                 'Manager API')
+    parser.add_argument('-S', '--secure_cluster', action='store_true')
     parser.add_argument('-H', '--hdfs_ha', action='store_true')
     parser.add_argument('-m', '--cm_server', metavar='cm_server', required='True',
                         help='Cloudera Manager IP to connect API using cm_client')
@@ -2084,7 +2082,7 @@ def options_parser(args=None):
             sys.exit()
 
     return (options.cm_server, options.input_host_list, options.disk_count, options.license_file, options.worker_shape,
-            options.num_workers, options.deployment_type, options.hdfs_ha, options.cdh_version, options.availability_domain, 
+            options.num_workers, options.secure_cluster, options.hdfs_ha, options.cdh_version, options.availability_domain, 
             options.cluster_name, cluster_primary_version, kafka_parcel_url)
 
 #
@@ -2277,7 +2275,7 @@ def enable_kerberos():
 #
 
 if __name__ == '__main__':
-    cm_server, input_host_list, disk_count, license_file, worker_shape, num_workers, deployment_type, hdfs_ha, \
+    cm_server, input_host_list, disk_count, license_file, worker_shape, num_workers, secure_cluster, hdfs_ha, \
     cdh_version, availability_domain, cluster_name, cluster_primary_version, kafka_parcel_url =\
     options_parser(sys.argv[1:])
     if debug == 'True':
@@ -2287,11 +2285,8 @@ def enable_kerberos():
         print('license_file = %s' % license_file)
         print('worker_shape = %s' % worker_shape)
         print('cluster_name = %s' % cluster_name)
-        print('deployment_type = %s' % deployment_type)
-    if deployment_type == 'simple':
-        secure_cluster = 'False'
-    else:
-	pass
+        print('secure_cluster = %s' % secure_cluster)
+        print('hdfs_ha = %s' % hdfs_ha)
     user_name = 'admin'
     password = 'admin'
     print('->Building API Endpoints')
@@ -2317,30 +2312,21 @@ def enable_kerberos():
             time.sleep(30)
             wait_status = wait_status + '*'
 
-    if deployment_type == 'simple':
-        print('Simple Deployment Selected')
-        print('Cluster Security and High Availabilty are DISABLED')
-    else:
-        print('Cluster Deployment options - HA: %s - Kerberos: %s' % (hdfs_ha, secure_cluster))
-
+    print('Cluster Deployment options - High Availability: %s - Kerberos: %s' % (hdfs_ha, secure_cluster))
     build_cloudera_cluster(cluster_primary_version)
-    if deployment_type == 'simple':
-        exit(0)
+    if hdfs_ha is True:
+        hdfs_ha_deployment_start = time.time()
+        print('->Enabling HDFS HA')
+        hdfs_enable_nn_ha(snn_host_id)
+        wait_for_active_service_commands('\tEnable HDFS HA', 'HDFS')
+        global hdfs_ha_deployment_time
+        hdfs_ha_deployment_time = time.time() - hdfs_ha_deployment_start
     else:
-        if hdfs_ha is True:
-            hdfs_ha_deployment_start = time.time()
-            print('->Enabling HDFS HA')
-            hdfs_enable_nn_ha(snn_host_id)
-            wait_for_active_service_commands('\tEnable HDFS HA', 'HDFS')
-            global hdfs_ha_deployment_time
-            hdfs_ha_deployment_time = time.time() - hdfs_ha_deployment_start
-        else:
-            pass
-        if secure_cluster == 'True':
-            print('->Enable Kerberos')
-            enable_kerberos()
-        else:
-            pass
-
+        pass
+    if secure_cluster is True:
+        print('->Enable Kerberos')
+        enable_kerberos()
+    else:
+        pass
     print('-> CLUSTER DEPLOYMENT COMPLETE <-')
-
+    exit(0)
diff --git a/variables.tf b/variables.tf
@@ -19,9 +19,8 @@ variable "cm_version" {
 variable "cdh_version" { 
     default = "6.2.0" 
 }
-# Deployment type - set this to "secure" to enable Kerberos and High Availability
-variable "deployment_type" { 
-    default = "simple" 
+variable "secure_cluster" { 
+    default = "True" 
 }
 
 variable "hdfs_ha" {

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,6 @@ resource "oci_core_instance" "Bastion" {`
`17`	`17`	`cloudera_manager = "${var.cloudera_manager}"`
`18`	`18`	`cdh_version = "${var.cdh_version}"`
`19`	`19`	`cm_version = "${var.cm_version}"`
`20`		`- deployment_type = "${var.deployment_type}"`
`21`	`20`	`}`
`22`	`21`
`23`	`22`	`timeouts {`
Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,6 @@ resource "oci_core_instance" "Master" {`
`19`	`19`	`cloudera_manager = "${var.cloudera_manager}"`
`20`	`20`	`cdh_version = "${var.cdh_version}"`
`21`	`21`	`cm_version = "${var.cm_version}"`
`22`		`- deployment_type = "${var.deployment_type}"`
`23`	`22`	`}`
`24`	`23`
`25`	`24`	`timeouts {`
Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,7 @@ resource "oci_core_instance" "Utility" {`
`21`	`21`	`worker_shape = "${var.worker_shape}"`
`22`	`22`	`block_volume_count = "${var.block_volume_count}"`
`23`	`23`	`availability_domain = "${var.AD}"`
`24`		`- deployment_type = "${var.deployment_type}"`
	`24`	`+ secure_cluster = "${var.secure_cluster}"`
`25`	`25`	`hdfs_ha = "${var.hdfs_ha}"`
`26`	`26`	`cluster_name = "${var.cluster_name}"`
`27`	`27`	`}`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,6 @@ resource "oci_core_instance" "Worker" {`
`20`	`20`	`cdh_version = "${var.cdh_version}"`
`21`	`21`	`cm_version = "${var.cm_version}"`
`22`	`22`	`block_volume_count = "${var.block_volume_count}"`
`23`		`- deployment_type = "${var.deployment_type}"`
`24`	`23`	`}`
`25`	`24`
`26`	`25`	`timeouts {`