Skip to content
This repository was archived by the owner on Apr 18, 2024. It is now read-only.

Commit b4234b5

Browse files
smithzcsmithzc
committed
Added logic to deployment, uses a metadata flag to determine type of cluster install (simple or not).
Added sanity check to UUID output to fstab Updates to python script output, fixed boolean variables to work properly
1 parent ebf6e9d commit b4234b5

File tree

13 files changed

+93
-57
lines changed

13 files changed

+93
-57
lines changed

‎scripts/boot.sh

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,9 @@ block_data_mount () {
221221
mkdir -p /data$dcount
222222
mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /data$dcount
223223
UUID=`lsblk -no UUID /dev/oracleoci/$disk`
224-
echo "UUID=$UUID /data$dcount ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab
224+
if [ ! -z $UUID ]; then
225+
echo "UUID=$UUID /data$dcount ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab
226+
fi
225227
}
226228
EXECNAME="DISK SETUP"
227229
log "->Checking for disks..."
@@ -247,7 +249,9 @@ for i in `seq 1 ${#iqn[@]}`; do
247249
mkdir -p /var/log/cloudera
248250
mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /var/log/cloudera
249251
UUID=`lsblk -no UUID /dev/oracleoci/$disk`
250-
echo "UUID=$UUID /var/log/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab
252+
if [ ! -z $UUID ]; then
253+
echo "UUID=$UUID /var/log/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab
254+
fi
251255
mkdir -p /var/log/cloudera/cloudera-scm-agent
252256
ln -s /var/log/cloudera/cloudera-scm-agent /var/log/cloudera-scm-agent
253257
;;
@@ -257,7 +261,9 @@ for i in `seq 1 ${#iqn[@]}`; do
257261
mkdir -p /opt/cloudera
258262
mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /opt/cloudera
259263
UUID=`lsblk -no UUID /dev/oracleoci/$disk`
260-
echo "UUID=$UUID /opt/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab
264+
if [ ! -z $UUID ]; then
265+
echo "UUID=$UUID /opt/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab
266+
fi
261267
;;
262268
*)
263269
mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/oracleoci/$disk
@@ -266,6 +272,7 @@ for i in `seq 1 ${#iqn[@]}`; do
266272
;;
267273
esac
268274
/sbin/tune2fs -i0 -c0 /dev/oracleoci/$disk
275+
unset UUID
269276
dsetup="1"
270277
else
271278
log "--->${disk} not found, running ISCSI again."

‎scripts/deploy_on_oci.py

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
data_tiering = 'False'
3434
nvme_disks = 0
3535
cluster_version = '6.2.0' # type: str
36-
SIMPLE = 'None'
36+
simple_deployment = 'False' # type: bool
3737

3838
#
3939
# Custom Global Parameters - Customize below here
@@ -51,12 +51,12 @@
5151
cluster_name = 'TestCluster' # type: str
5252

5353
# Set this to 'True' (default) to enable secure cluster (Kerberos) functionality
54-
# Set this to 'False" to deploy an insecure cluster - This is automatically off for SIMPLE deployment
55-
secure_cluster = 'True'
54+
# Set this to 'False" to deploy an insecure cluster - This is automatically off for simple_deployment
55+
secure_cluster = 'True' # type: bool
5656

5757
# Set this to 'False' if you do not want HDFS HA - useful for Development or if you want to save some setup time
58-
# This is automatically off for SIMPLE deployment
59-
hdfs_ha = 'True'
58+
# This is automatically off for simple_deployment
59+
hdfs_ha = 'True' # type: bool
6060

6161
# They should match what is in the Cloudera Manager CloudInit bootstrap file and instance boot files
6262
realm = 'HADOOP.COM'
@@ -2054,7 +2054,7 @@ def options_parser(args=None):
20542054
'OCI using cm_client with Cloudera '
20552055
'Manager API %s' % (cluster_version,
20562056
api_version))
2057-
parser.add_argument('-S', '--SIMPLE', action='store_true', help='Simple, no HA or Kerberos at deployment')
2057+
parser.add_argument('-S', '--simple_deployment', action='store_true', help='Simple, no HA or Kerberos at deployment')
20582058
parser.add_argument('-m', '--cm_server', metavar='cm_server', required='True',
20592059
help='Cloudera Manager IP to connect API using cm_client')
20602060
parser.add_argument('-i', '--input_host_list', metavar='input_host_list',
@@ -2098,7 +2098,7 @@ def options_parser(args=None):
20982098
sys.exit()
20992099

21002100
return (options.cm_server, options.input_host_list, options.disk_count, options.license_file, options.worker_shape,
2101-
options.num_workers, options.SIMPLE, options.cdh_version, options.availability_domain)
2101+
options.num_workers, options.simple_deployment, options.cdh_version, options.availability_domain)
21022102

21032103
#
21042104
# MAIN FUNCTION FOR CLUSTER DEPLOYMENT
@@ -2269,7 +2269,7 @@ def enable_kerberos():
22692269
#
22702270

22712271
if __name__ == '__main__':
2272-
cm_server, input_host_list, disk_count, license_file, worker_shape, num_workers, SIMPLE, cdh_version, cms_version =\
2272+
cm_server, input_host_list, disk_count, license_file, worker_shape, num_workers, simple_deployment, cdh_version, cms_version =\
22732273
options_parser(sys.argv[1:])
22742274
if debug == 'True':
22752275
print('cm_server = %s' % cm_server)
@@ -2302,21 +2302,30 @@ def enable_kerberos():
23022302
time.sleep(30)
23032303
wait_status = wait_status + '*'
23042304

2305+
if simple_deployment is True:
2306+
print('Simple Deployment Selected')
2307+
print('Cluster Security and High Availabilty is DISABLED')
2308+
else:
2309+
print('Cluster Deployment options - HA: %s - Kerberos: %s' % hdfs_ha, secure_cluster)
2310+
23052311
build_cloudera_cluster()
2306-
if SIMPLE == 'True':
2307-
pass
2312+
if simple_deployment is True:
2313+
exit(0)
23082314
else:
2309-
if hdfs_ha == 'True':
2315+
if hdfs_ha is True:
23102316
hdfs_ha_deployment_start = time.time()
23112317
print('->Enabling HDFS HA')
23122318
hdfs_enable_nn_ha(snn_host_id)
23132319
wait_for_active_service_commands('\tEnable HDFS HA', 'HDFS')
23142320
global hdfs_ha_deployment_time
23152321
hdfs_ha_deployment_time = time.time() - hdfs_ha_deployment_start
2316-
2317-
if secure_cluster == 'True':
2322+
else:
2323+
pass
2324+
if secure_cluster is True:
23182325
print('->Enable Kerberos')
23192326
enable_kerberos()
2327+
else:
2328+
pass
23202329

23212330
print('Access Cloudera Manager: http://%s:%s/cmf/' % (cm_server, cm_port))
23222331

‎terraform/compute.tf

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ module "bastion" {
1717
cloudera_manager = "cdh-utility-1.public${var.availability_domain}.${module.network.vcn-dn}"
1818
cm_version = "${var.cm_version}"
1919
cdh_version = "${var.cdh_version}"
20+
deployment_type = "${var.deployment_type}"
2021
}
2122

2223
module "utility" {
@@ -43,6 +44,7 @@ module "utility" {
4344
worker_shape = "${var.worker_instance_shape}"
4445
block_volume_count = "${var.block_volume_count}"
4546
AD = "${var.availability_domain}"
47+
deployment_type = "${var.deployment_type}"
4648
}
4749

4850
module "master" {
@@ -64,6 +66,7 @@ module "master" {
6466
cloudera_manager = "cdh-utility-1.public${var.availability_domain}.${module.network.vcn-dn}"
6567
cm_version = "${var.cm_version}"
6668
cdh_version = "${var.cdh_version}"
69+
deployment_type = "${var.deployment_type}"
6770
}
6871

6972
module "worker" {
@@ -88,4 +91,5 @@ module "worker" {
8891
cm_version = "${var.cm_version}"
8992
cdh_version = "${var.cdh_version}"
9093
block_volume_count = "${var.block_volumes_per_worker}"
94+
deployment_type = "${var.deployment_type}"
9195
}

‎terraform/modules/bastion/main.tf

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ resource "oci_core_instance" "Bastion" {
1616
user_data = "${var.user_data}"
1717
cloudera_manager = "${var.cloudera_manager}"
1818
cdh_version = "${var.cdh_version}"
19-
cm_version = "${var.cm_version}"
19+
cm_version = "${var.cm_version}"
20+
deployment_type = "${var.deployment_type}"
2021
}
2122

2223
timeouts {

‎terraform/modules/bastion/variables.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ variable "image_ocid" {}
1616
variable "cm_version" {}
1717
variable "cdh_version" {}
1818
variable "cloudera_manager" {}
19+
variable "deployment_type" {}
1920

2021
# ---------------------------------------------------------------------------------------------------------------------
2122
# Optional variables

‎terraform/modules/master/main.tf

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ resource "oci_core_instance" "Master" {
1818
user_data = "${var.user_data}"
1919
cloudera_manager = "${var.cloudera_manager}"
2020
cdh_version = "${var.cdh_version}"
21-
cm_version = "${var.cm_version}"
21+
cm_version = "${var.cm_version}"
22+
deployment_type = "${var.deployment_type}"
2223
}
2324

2425
timeouts {

‎terraform/modules/master/variables.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ variable "image_ocid" {}
1616
variable "cm_version" {}
1717
variable "cdh_version" {}
1818
variable "cloudera_manager" {}
19+
variable "deployment_type" {}
1920

2021
# ---------------------------------------------------------------------------------------------------------------------
2122
# Optional variables

‎terraform/modules/utility/main.tf

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,10 @@ resource "oci_core_instance" "Utility" {
1818
cloudera_manager = "${var.cloudera_manager}"
1919
cdh_version = "${var.cdh_version}"
2020
cm_version = "${var.cm_version}"
21-
worker_shape = "${var.worker_shape}"
22-
block_volume_count = "${var.block_volume_count}"
21+
worker_shape = "${var.worker_shape}"
22+
block_volume_count = "${var.block_volume_count}"
2323
availability_domain = "${var.AD}"
24+
deployment_type = "${var.deployment_type}"
2425
}
2526

2627
extended_metadata {

‎terraform/modules/utility/variables.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ variable "AD" {}
2121
variable "cloudera_manager" {}
2222
variable "cm_install" {}
2323
variable "deploy_on_oci" {}
24+
variable "deployment_type" {}
2425

2526
# ---------------------------------------------------------------------------------------------------------------------
2627
# Optional variables

‎terraform/modules/worker/main.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ resource "oci_core_instance" "Worker" {
2020
cdh_version = "${var.cdh_version}"
2121
cm_version = "${var.cm_version}"
2222
block_volume_count = "${var.block_volume_count}"
23+
deployment_type = "${var.deployment_type}"
2324
}
2425

2526
timeouts {

0 commit comments

Comments
 (0)