From ee3575b3d8d3898fa8f67d472b1cd80106318035 Mon Sep 17 00:00:00 2001 From: Devdatta Kulkarni Date: Fri, 9 Feb 2018 19:17:43 -0600 Subject: [PATCH 1/2] Support for using GCP service account --- client/fmcmds/setup_gcloud.py | 96 +++++++++++++------ server/server_plugins/gcloud/coe/gke.py | 64 ++++++++----- .../server_plugins/gcloud/coe/gke_app_base.py | 2 + .../gcloud/coe/gke_single_container.py | 7 +- server/server_plugins/gcloud/gcloud_helper.py | 25 +++++ server/server_plugins/gcloud/resource/gcr.py | 18 +++- 6 files changed, 153 insertions(+), 59 deletions(-) diff --git a/client/fmcmds/setup_gcloud.py b/client/fmcmds/setup_gcloud.py index 93d6da0..e3559e8 100644 --- a/client/fmcmds/setup_gcloud.py +++ b/client/fmcmds/setup_gcloud.py @@ -23,8 +23,73 @@ def _execute_cmd(self, cmd): print(e) return err, output + def _setup_zone(self): + zones = ['us-central1-a', 'us-central1-b', 'us-central1-c', 'us-central1-f', + 'us-west1-a', 'us-west1-b', 'us-west1-c', + 'us-east1-b', 'us-east1-c', 'us-east1-d', + 'us-east4-a', 'us-east4-b', 'us-east4-c', + 'southamerica-east1-a', 'southamerica-east1-b', 'southamerica-east1-c', + 'europe-west1-b', 'europe-west1-c', 'europe-west1-d', + 'europe-west3-a', 'europe-west3-b', 'europe-west3-c', + 'europe-west2-a', 'europe-west2-b', 'europe-west2-c', + 'asia-southeast1-a', 'asia-southeast1-b' + 'asia-east1-a', 'asia-east1-b', 'asia-east1-c', + 'asia-northeast1-a', 'asia-northeast1-b', 'asia-northeast1-c', + 'asia-south1-a', 'asia-south1-b', 'asia-south1-c', + 'australia-southeast1-a', 'australia-southeast1-b', 'australia-southeast1-c', + ] + for z in zones: + print(z) + + zone = '' + while True: + zone = raw_input("Enter zone:" ) + if zone in zones: + break + else: + print("Incorrect region specified. Please choose one of above.") + return zone + + def _get_gcp_account(self): + account = raw_input("Enter email address associated with your GCP account>") + return account def take_action(self, parsed_args): + json_key_file_path = raw_input("Enter path of service account json key file>") + + cloudark_google_setup_details_path = APP_STORE_PATH + "/gcp-service-account-key.json" + if not os.path.exists(APP_STORE_PATH): + mkdir_command = ("mkdir -p {app_store_path}").format(app_store_path=APP_STORE_PATH) + os.system(mkdir_command) + + if os.path.exists(json_key_file_path): + fp1 = open(json_key_file_path, "r") + input_lines = fp1.readlines() + fp = open(cloudark_google_setup_details_path, "w") + fp.writelines(input_lines) + fp.close() + fp1.close() + else: + print("File not found.") + exit() + + print("Setting up GCP account information.") + account = self._get_gcp_account() + + print("Setting up zone information.") + zone = self._setup_zone() + + cloudark_google_setup_details_path = APP_STORE_PATH + "/google-creds-cloudark" + if not os.path.exists(APP_STORE_PATH): + mkdir_command = ("mkdir -p {app_store_path}").format(app_store_path=APP_STORE_PATH) + os.system(mkdir_command) + + fp = open(cloudark_google_setup_details_path, "w") + fp.write("zone:%s\n" % zone.strip()) + fp.write("account:%s" % account.strip()) + fp.close() + + def take_action_prev(self, parsed_args): gcloud_df = "Dockerfile.gcloudsetup" df = ("FROM lmecld/clis:gcloud \n" @@ -57,37 +122,6 @@ def take_action(self, parsed_args): print("Error occurred in setting up Google cloud. Exiting.") exit() - zones = ['us-central1-a', 'us-central1-b', 'us-central1-c', 'us-central1-f', - 'us-west1-a', 'us-west1-b', 'us-west1-c', - 'us-east1-b', 'us-east1-c', 'us-east1-d', - 'us-east4-a', 'us-east4-b', 'us-east4-c', - 'southamerica-east1-a', 'southamerica-east1-b', 'southamerica-east1-c', - 'europe-west1-b', 'europe-west1-c', 'europe-west1-d', - 'europe-west3-a', 'europe-west3-b', 'europe-west3-c', - 'europe-west2-a', 'europe-west2-b', 'europe-west2-c', - 'asia-southeast1-a', 'asia-southeast1-b' - 'asia-east1-a', 'asia-east1-b', 'asia-east1-c', - 'asia-northeast1-a', 'asia-northeast1-b', 'asia-northeast1-c', - 'asia-south1-a', 'asia-south1-b', 'asia-south1-c', - 'australia-southeast1-a', 'australia-southeast1-b', 'australia-southeast1-c', - ] - for z in zones: - print(z) - while True: - zone = raw_input("Enter zone:" ) - if zone in zones: - break - else: - print("Incorrect region specified. Please choose one of above.") - - cloudark_google_setup_details_path = APP_STORE_PATH + "/google-creds-cloudark" - if not os.path.exists(APP_STORE_PATH): - mkdir_command = ("mkdir -p {app_store_path}").format(app_store_path=APP_STORE_PATH) - os.system(mkdir_command) - - fp = open(cloudark_google_setup_details_path, "w") - fp.write("zone:%s" % zone.strip()) - fp.close() os.system("./google-cloud-sdk/install.sh --quiet") os.system("./google-cloud-sdk/bin/gcloud auth login") diff --git a/server/server_plugins/gcloud/coe/gke.py b/server/server_plugins/gcloud/coe/gke.py index 96e2fe1..aaab4f2 100644 --- a/server/server_plugins/gcloud/coe/gke.py +++ b/server/server_plugins/gcloud/coe/gke.py @@ -10,6 +10,8 @@ from googleapiclient import discovery from oauth2client.client import GoogleCredentials +from google.oauth2 import service_account + from server.common import constants from server.common import common_functions from server.common import docker_lib @@ -26,6 +28,8 @@ APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir) +SERVICE_ACCOUNT_FILE = APP_AND_ENV_STORE_PATH + "gcp-service-account-key.json" + fmlogger = fm_logger.Logging() GCR = "us.gcr.io" @@ -49,7 +53,8 @@ class GKEHandler(coe_base.COEBase): def __init__(self): - credentials = GoogleCredentials.get_application_default() + #credentials = GoogleCredentials.get_application_default() + credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE) self.gke_service = discovery.build('container', 'v1', credentials=credentials) self.compute_service = discovery.build('compute', 'v1', @@ -253,17 +258,22 @@ def create_cluster(self, env_id, env_info): if 'instance_type' in env_details['environment']['app_deployment']: instance_type = env_details['environment']['app_deployment']['instance_type'] - try: - self._create_network(env_id, project, cluster_name) - except Exception as e: - fmlogger.error(e) - return + network = '' + if 'network' in env_details['environment']['app_deployment']: + network = env_details['environment']['app_deployment']['network'] + if not network or network != 'default': + network = cluster_name + try: + self._create_network(env_id, project, cluster_name) + except Exception as e: + fmlogger.error(e) + return - try: - self._create_firewall_rule(env_id, project, cluster_name) - except Exception as e: - fmlogger.error(e) - return + try: + self._create_firewall_rule(env_id, project, cluster_name) + except Exception as e: + fmlogger.error(e) + return resp = '' try: @@ -275,7 +285,7 @@ def create_cluster(self, env_id, env_info): "nodeConfig": { "oauthScopes": "https://www.googleapis.com/auth/devstorage.read_only", "machineType": instance_type}, - "network": cluster_name}} + "network": network}} ).execute() fmlogger.debug(resp) except Exception as e: @@ -344,17 +354,25 @@ def delete_cluster(self, env_id, env_info, resource_obj): project = filtered_description['project'] zone = filtered_description['zone'] - # Network delete is not working for some reason. So temporarily - # commenting it out. - #try: - # self._delete_network(project, cluster_name) - #except Exception as e: - # fmlogger.error("Exception deleting network %s " % str(e)) - # env_update = {} - # env_update['output_config'] = str({'error': str(e)}) - # env_db.Environment().update(env_id, env_update) - - self._delete_firewall_rule(project, cluster_name) + env_obj = env_db.Environment().get(env_id) + env_name = env_obj.name + env_details = ast.literal_eval(env_obj.env_definition) + + network = '' + if 'network' in env_details['environment']['app_deployment']: + network = env_details['environment']['app_deployment']['network'] + if not network or network != 'default': + # Network delete is not working for some reason. So temporarily + # commenting it out. + #try: + # self._delete_network(project, cluster_name) + #except Exception as e: + # fmlogger.error("Exception deleting network %s " % str(e)) + # env_update = {} + # env_update['output_config'] = str({'error': str(e)}) + # env_db.Environment().update(env_id, env_update) + + self._delete_firewall_rule(project, cluster_name) try: resp = self.gke_service.projects().zones().clusters().delete( diff --git a/server/server_plugins/gcloud/coe/gke_app_base.py b/server/server_plugins/gcloud/coe/gke_app_base.py index cf68df1..11d51db 100644 --- a/server/server_plugins/gcloud/coe/gke_app_base.py +++ b/server/server_plugins/gcloud/coe/gke_app_base.py @@ -24,6 +24,8 @@ APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir) +SERVICE_ACCOUNT_FILE = APP_AND_ENV_STORE_PATH + "gcp-service-account-key.json" + fmlogger = fm_logger.Logging() GCR = "us.gcr.io" diff --git a/server/server_plugins/gcloud/coe/gke_single_container.py b/server/server_plugins/gcloud/coe/gke_single_container.py index cbf74f9..e03ec73 100644 --- a/server/server_plugins/gcloud/coe/gke_single_container.py +++ b/server/server_plugins/gcloud/coe/gke_single_container.py @@ -9,6 +9,8 @@ from googleapiclient import discovery from oauth2client.client import GoogleCredentials +from google.oauth2 import service_account + from server.common import constants from server.common import common_functions from server.common import docker_lib @@ -24,6 +26,8 @@ APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir) +SERVICE_ACCOUNT_FILE = APP_AND_ENV_STORE_PATH + "gcp-service-account-key.json" + fmlogger = fm_logger.Logging() GCR = "us.gcr.io" @@ -34,7 +38,8 @@ class GKESingleContainer(gke_app_base.GKEAppBase): gcloudhelper = gcloud_helper.GCloudHelper() def __init__(self): - credentials = GoogleCredentials.get_application_default() + #credentials = GoogleCredentials.get_application_default() + credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE) self.gke_service = discovery.build('container', 'v1', credentials=credentials) self.compute_service = discovery.build('compute', 'v1', diff --git a/server/server_plugins/gcloud/gcloud_helper.py b/server/server_plugins/gcloud/gcloud_helper.py index 5168b23..27082bf 100644 --- a/server/server_plugins/gcloud/gcloud_helper.py +++ b/server/server_plugins/gcloud/gcloud_helper.py @@ -23,6 +23,8 @@ APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir) +GOOGLE_CREDS_FILE = APP_AND_ENV_STORE_PATH + "google-creds-cloudark" + fmlogger = fm_logger.Logging() GCR = "us.gcr.io" @@ -75,6 +77,29 @@ def get_access_token(self, df_dir, df, cont_name): return access_token def get_deployment_details(self, env_id): + project = '' + zone = '' + account = '' + + if 'app_deployment' in env_details['environment']: + project = env_details['environment']['app_deployment']['project'] + zone = env_details['environment']['app_deployment']['zone'] + else: + project = env_details['environment']['resources']['gcloud'][0]['resource']['project'] + zone = env_details['environment']['resources']['gcloud'][0]['resource']['zone'] + + fp = open(GOOGLE_CREDS_FILE, "r") + lines = fp.readlines() + for line in lines: + parts = line.split(":") + if parts[0].strip() == 'zone': + zone = parts[1].strip() + if parts[0].strip() == 'account': + account = parts[1].strip() + + return user_account, project, zone + + def get_deployment_details_bak(self, env_id): env_obj = env_db.Environment().get(env_id) env_details = ast.literal_eval(env_obj.env_definition) project = '' diff --git a/server/server_plugins/gcloud/resource/gcr.py b/server/server_plugins/gcloud/resource/gcr.py index 6d61858..335ee0c 100644 --- a/server/server_plugins/gcloud/resource/gcr.py +++ b/server/server_plugins/gcloud/resource/gcr.py @@ -55,10 +55,13 @@ def _build_container(self, cont_info, tag=''): def _push_container(self, cont_info, tagged_image): access_token = self._get_access_token(cont_info) - self.docker_handler.docker_login("oauth2accesstoken", - access_token, "https://" + GCR) + err, output = self.docker_handler.docker_login("oauth2accesstoken", + access_token, "https://" + GCR) - self.docker_handler.push_container(tagged_image) + if not err: + err, output = self.docker_handler.push_container(tagged_image) + + return err, output def _delete_container(self, tagged_image, cont_info): err, output = self.docker_handler.remove_container_image(tagged_image) @@ -118,14 +121,21 @@ def create(self, cont_name, cont_info): cont_data['status'] = 'pushing-cont-to-gcr-repository' cont_db.Container().update(cont_name, cont_data) + err = '' try: - self._push_container(cont_info, tagged_image) + err, output = self._push_container(cont_info, tagged_image) except Exception as e: fmlogger.error("Exception encountered in pushing container to gcr %s" % e) cont_data['status'] = 'error-in-container-push-to-gcr:' + str(e) cont_db.Container().update(cont_name, cont_data) return + if err: + fmlogger.error("Exception encountered in pushing container to gcr %s" % err) + cont_data['status'] = 'error-in-container-push-to-gcr:' + str(err) + cont_db.Container().update(cont_name, cont_data) + return + cont_data['status'] = 'container-ready' cont_db.Container().update(cont_name, cont_data) From f43802dae8e8d37d2119622cb04665da77bf2ba7 Mon Sep 17 00:00:00 2001 From: Devdatta Kulkarni Date: Fri, 9 Feb 2018 19:17:43 -0600 Subject: [PATCH 2/2] Support for using GCP service account --- client/fmcmds/setup_gcloud.py | 96 +++++++++++++------ server/server_plugins/gcloud/coe/gke.py | 64 ++++++++----- .../server_plugins/gcloud/coe/gke_app_base.py | 2 + .../gcloud/coe/gke_single_container.py | 7 +- server/server_plugins/gcloud/gcloud_helper.py | 25 +++++ server/server_plugins/gcloud/resource/gcr.py | 18 +++- 6 files changed, 153 insertions(+), 59 deletions(-) diff --git a/client/fmcmds/setup_gcloud.py b/client/fmcmds/setup_gcloud.py index 93d6da0..e3559e8 100644 --- a/client/fmcmds/setup_gcloud.py +++ b/client/fmcmds/setup_gcloud.py @@ -23,8 +23,73 @@ def _execute_cmd(self, cmd): print(e) return err, output + def _setup_zone(self): + zones = ['us-central1-a', 'us-central1-b', 'us-central1-c', 'us-central1-f', + 'us-west1-a', 'us-west1-b', 'us-west1-c', + 'us-east1-b', 'us-east1-c', 'us-east1-d', + 'us-east4-a', 'us-east4-b', 'us-east4-c', + 'southamerica-east1-a', 'southamerica-east1-b', 'southamerica-east1-c', + 'europe-west1-b', 'europe-west1-c', 'europe-west1-d', + 'europe-west3-a', 'europe-west3-b', 'europe-west3-c', + 'europe-west2-a', 'europe-west2-b', 'europe-west2-c', + 'asia-southeast1-a', 'asia-southeast1-b' + 'asia-east1-a', 'asia-east1-b', 'asia-east1-c', + 'asia-northeast1-a', 'asia-northeast1-b', 'asia-northeast1-c', + 'asia-south1-a', 'asia-south1-b', 'asia-south1-c', + 'australia-southeast1-a', 'australia-southeast1-b', 'australia-southeast1-c', + ] + for z in zones: + print(z) + + zone = '' + while True: + zone = raw_input("Enter zone:" ) + if zone in zones: + break + else: + print("Incorrect region specified. Please choose one of above.") + return zone + + def _get_gcp_account(self): + account = raw_input("Enter email address associated with your GCP account>") + return account def take_action(self, parsed_args): + json_key_file_path = raw_input("Enter path of service account json key file>") + + cloudark_google_setup_details_path = APP_STORE_PATH + "/gcp-service-account-key.json" + if not os.path.exists(APP_STORE_PATH): + mkdir_command = ("mkdir -p {app_store_path}").format(app_store_path=APP_STORE_PATH) + os.system(mkdir_command) + + if os.path.exists(json_key_file_path): + fp1 = open(json_key_file_path, "r") + input_lines = fp1.readlines() + fp = open(cloudark_google_setup_details_path, "w") + fp.writelines(input_lines) + fp.close() + fp1.close() + else: + print("File not found.") + exit() + + print("Setting up GCP account information.") + account = self._get_gcp_account() + + print("Setting up zone information.") + zone = self._setup_zone() + + cloudark_google_setup_details_path = APP_STORE_PATH + "/google-creds-cloudark" + if not os.path.exists(APP_STORE_PATH): + mkdir_command = ("mkdir -p {app_store_path}").format(app_store_path=APP_STORE_PATH) + os.system(mkdir_command) + + fp = open(cloudark_google_setup_details_path, "w") + fp.write("zone:%s\n" % zone.strip()) + fp.write("account:%s" % account.strip()) + fp.close() + + def take_action_prev(self, parsed_args): gcloud_df = "Dockerfile.gcloudsetup" df = ("FROM lmecld/clis:gcloud \n" @@ -57,37 +122,6 @@ def take_action(self, parsed_args): print("Error occurred in setting up Google cloud. Exiting.") exit() - zones = ['us-central1-a', 'us-central1-b', 'us-central1-c', 'us-central1-f', - 'us-west1-a', 'us-west1-b', 'us-west1-c', - 'us-east1-b', 'us-east1-c', 'us-east1-d', - 'us-east4-a', 'us-east4-b', 'us-east4-c', - 'southamerica-east1-a', 'southamerica-east1-b', 'southamerica-east1-c', - 'europe-west1-b', 'europe-west1-c', 'europe-west1-d', - 'europe-west3-a', 'europe-west3-b', 'europe-west3-c', - 'europe-west2-a', 'europe-west2-b', 'europe-west2-c', - 'asia-southeast1-a', 'asia-southeast1-b' - 'asia-east1-a', 'asia-east1-b', 'asia-east1-c', - 'asia-northeast1-a', 'asia-northeast1-b', 'asia-northeast1-c', - 'asia-south1-a', 'asia-south1-b', 'asia-south1-c', - 'australia-southeast1-a', 'australia-southeast1-b', 'australia-southeast1-c', - ] - for z in zones: - print(z) - while True: - zone = raw_input("Enter zone:" ) - if zone in zones: - break - else: - print("Incorrect region specified. Please choose one of above.") - - cloudark_google_setup_details_path = APP_STORE_PATH + "/google-creds-cloudark" - if not os.path.exists(APP_STORE_PATH): - mkdir_command = ("mkdir -p {app_store_path}").format(app_store_path=APP_STORE_PATH) - os.system(mkdir_command) - - fp = open(cloudark_google_setup_details_path, "w") - fp.write("zone:%s" % zone.strip()) - fp.close() os.system("./google-cloud-sdk/install.sh --quiet") os.system("./google-cloud-sdk/bin/gcloud auth login") diff --git a/server/server_plugins/gcloud/coe/gke.py b/server/server_plugins/gcloud/coe/gke.py index 96e2fe1..aaab4f2 100644 --- a/server/server_plugins/gcloud/coe/gke.py +++ b/server/server_plugins/gcloud/coe/gke.py @@ -10,6 +10,8 @@ from googleapiclient import discovery from oauth2client.client import GoogleCredentials +from google.oauth2 import service_account + from server.common import constants from server.common import common_functions from server.common import docker_lib @@ -26,6 +28,8 @@ APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir) +SERVICE_ACCOUNT_FILE = APP_AND_ENV_STORE_PATH + "gcp-service-account-key.json" + fmlogger = fm_logger.Logging() GCR = "us.gcr.io" @@ -49,7 +53,8 @@ class GKEHandler(coe_base.COEBase): def __init__(self): - credentials = GoogleCredentials.get_application_default() + #credentials = GoogleCredentials.get_application_default() + credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE) self.gke_service = discovery.build('container', 'v1', credentials=credentials) self.compute_service = discovery.build('compute', 'v1', @@ -253,17 +258,22 @@ def create_cluster(self, env_id, env_info): if 'instance_type' in env_details['environment']['app_deployment']: instance_type = env_details['environment']['app_deployment']['instance_type'] - try: - self._create_network(env_id, project, cluster_name) - except Exception as e: - fmlogger.error(e) - return + network = '' + if 'network' in env_details['environment']['app_deployment']: + network = env_details['environment']['app_deployment']['network'] + if not network or network != 'default': + network = cluster_name + try: + self._create_network(env_id, project, cluster_name) + except Exception as e: + fmlogger.error(e) + return - try: - self._create_firewall_rule(env_id, project, cluster_name) - except Exception as e: - fmlogger.error(e) - return + try: + self._create_firewall_rule(env_id, project, cluster_name) + except Exception as e: + fmlogger.error(e) + return resp = '' try: @@ -275,7 +285,7 @@ def create_cluster(self, env_id, env_info): "nodeConfig": { "oauthScopes": "https://www.googleapis.com/auth/devstorage.read_only", "machineType": instance_type}, - "network": cluster_name}} + "network": network}} ).execute() fmlogger.debug(resp) except Exception as e: @@ -344,17 +354,25 @@ def delete_cluster(self, env_id, env_info, resource_obj): project = filtered_description['project'] zone = filtered_description['zone'] - # Network delete is not working for some reason. So temporarily - # commenting it out. - #try: - # self._delete_network(project, cluster_name) - #except Exception as e: - # fmlogger.error("Exception deleting network %s " % str(e)) - # env_update = {} - # env_update['output_config'] = str({'error': str(e)}) - # env_db.Environment().update(env_id, env_update) - - self._delete_firewall_rule(project, cluster_name) + env_obj = env_db.Environment().get(env_id) + env_name = env_obj.name + env_details = ast.literal_eval(env_obj.env_definition) + + network = '' + if 'network' in env_details['environment']['app_deployment']: + network = env_details['environment']['app_deployment']['network'] + if not network or network != 'default': + # Network delete is not working for some reason. So temporarily + # commenting it out. + #try: + # self._delete_network(project, cluster_name) + #except Exception as e: + # fmlogger.error("Exception deleting network %s " % str(e)) + # env_update = {} + # env_update['output_config'] = str({'error': str(e)}) + # env_db.Environment().update(env_id, env_update) + + self._delete_firewall_rule(project, cluster_name) try: resp = self.gke_service.projects().zones().clusters().delete( diff --git a/server/server_plugins/gcloud/coe/gke_app_base.py b/server/server_plugins/gcloud/coe/gke_app_base.py index cf68df1..11d51db 100644 --- a/server/server_plugins/gcloud/coe/gke_app_base.py +++ b/server/server_plugins/gcloud/coe/gke_app_base.py @@ -24,6 +24,8 @@ APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir) +SERVICE_ACCOUNT_FILE = APP_AND_ENV_STORE_PATH + "gcp-service-account-key.json" + fmlogger = fm_logger.Logging() GCR = "us.gcr.io" diff --git a/server/server_plugins/gcloud/coe/gke_single_container.py b/server/server_plugins/gcloud/coe/gke_single_container.py index cbf74f9..e03ec73 100644 --- a/server/server_plugins/gcloud/coe/gke_single_container.py +++ b/server/server_plugins/gcloud/coe/gke_single_container.py @@ -9,6 +9,8 @@ from googleapiclient import discovery from oauth2client.client import GoogleCredentials +from google.oauth2 import service_account + from server.common import constants from server.common import common_functions from server.common import docker_lib @@ -24,6 +26,8 @@ APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir) +SERVICE_ACCOUNT_FILE = APP_AND_ENV_STORE_PATH + "gcp-service-account-key.json" + fmlogger = fm_logger.Logging() GCR = "us.gcr.io" @@ -34,7 +38,8 @@ class GKESingleContainer(gke_app_base.GKEAppBase): gcloudhelper = gcloud_helper.GCloudHelper() def __init__(self): - credentials = GoogleCredentials.get_application_default() + #credentials = GoogleCredentials.get_application_default() + credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE) self.gke_service = discovery.build('container', 'v1', credentials=credentials) self.compute_service = discovery.build('compute', 'v1', diff --git a/server/server_plugins/gcloud/gcloud_helper.py b/server/server_plugins/gcloud/gcloud_helper.py index 5168b23..9973a79 100644 --- a/server/server_plugins/gcloud/gcloud_helper.py +++ b/server/server_plugins/gcloud/gcloud_helper.py @@ -23,6 +23,8 @@ APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir) +GOOGLE_CREDS_FILE = APP_AND_ENV_STORE_PATH + "google-creds-cloudark" + fmlogger = fm_logger.Logging() GCR = "us.gcr.io" @@ -75,6 +77,29 @@ def get_access_token(self, df_dir, df, cont_name): return access_token def get_deployment_details(self, env_id): + project = '' + zone = '' + account = '' + + if 'app_deployment' in env_details['environment']: + project = env_details['environment']['app_deployment']['project'] + zone = env_details['environment']['app_deployment']['zone'] + else: + project = env_details['environment']['resources']['gcloud'][0]['resource']['project'] + zone = env_details['environment']['resources']['gcloud'][0]['resource']['zone'] + + fp = open(GOOGLE_CREDS_FILE, "r") + lines = fp.readlines() + for line in lines: + parts = line.split(":") + if parts[0].strip() == 'zone': + zone = parts[1].strip() + if parts[0].strip() == 'account': + account = parts[1].strip() + + return account, project, zone + + def get_deployment_details_bak(self, env_id): env_obj = env_db.Environment().get(env_id) env_details = ast.literal_eval(env_obj.env_definition) project = '' diff --git a/server/server_plugins/gcloud/resource/gcr.py b/server/server_plugins/gcloud/resource/gcr.py index 6d61858..335ee0c 100644 --- a/server/server_plugins/gcloud/resource/gcr.py +++ b/server/server_plugins/gcloud/resource/gcr.py @@ -55,10 +55,13 @@ def _build_container(self, cont_info, tag=''): def _push_container(self, cont_info, tagged_image): access_token = self._get_access_token(cont_info) - self.docker_handler.docker_login("oauth2accesstoken", - access_token, "https://" + GCR) + err, output = self.docker_handler.docker_login("oauth2accesstoken", + access_token, "https://" + GCR) - self.docker_handler.push_container(tagged_image) + if not err: + err, output = self.docker_handler.push_container(tagged_image) + + return err, output def _delete_container(self, tagged_image, cont_info): err, output = self.docker_handler.remove_container_image(tagged_image) @@ -118,14 +121,21 @@ def create(self, cont_name, cont_info): cont_data['status'] = 'pushing-cont-to-gcr-repository' cont_db.Container().update(cont_name, cont_data) + err = '' try: - self._push_container(cont_info, tagged_image) + err, output = self._push_container(cont_info, tagged_image) except Exception as e: fmlogger.error("Exception encountered in pushing container to gcr %s" % e) cont_data['status'] = 'error-in-container-push-to-gcr:' + str(e) cont_db.Container().update(cont_name, cont_data) return + if err: + fmlogger.error("Exception encountered in pushing container to gcr %s" % err) + cont_data['status'] = 'error-in-container-push-to-gcr:' + str(err) + cont_db.Container().update(cont_name, cont_data) + return + cont_data['status'] = 'container-ready' cont_db.Container().update(cont_name, cont_data)