Support for using GCP service account

client/fmcmds/setup_gcloud.py

@@ -23,8 +23,73 @@ def _execute_cmd(self, cmd):
             print(e)
         return err, output
 
+    def _setup_zone(self):
+        zones = ['us-central1-a', 'us-central1-b', 'us-central1-c', 'us-central1-f',
+                 'us-west1-a', 'us-west1-b', 'us-west1-c',
+                 'us-east1-b', 'us-east1-c', 'us-east1-d',
+                 'us-east4-a', 'us-east4-b', 'us-east4-c',
+                 'southamerica-east1-a', 'southamerica-east1-b', 'southamerica-east1-c',
+                 'europe-west1-b', 'europe-west1-c', 'europe-west1-d',
+                 'europe-west3-a', 'europe-west3-b', 'europe-west3-c',
+                 'europe-west2-a', 'europe-west2-b', 'europe-west2-c',
+                 'asia-southeast1-a', 'asia-southeast1-b'
+                 'asia-east1-a', 'asia-east1-b', 'asia-east1-c',
+                 'asia-northeast1-a', 'asia-northeast1-b', 'asia-northeast1-c',
+                 'asia-south1-a', 'asia-south1-b', 'asia-south1-c',
+                 'australia-southeast1-a', 'australia-southeast1-b', 'australia-southeast1-c',
+                ]
+        for z in zones:
+            print(z)
+
+        zone = ''
+        while True:
+            zone = raw_input("Enter zone:" )
+            if zone in zones:
+                break
+            else:
+                print("Incorrect region specified. Please choose one of above.")
+        return zone
+
+    def _get_gcp_account(self):
+        account = raw_input("Enter email address associated with your GCP account>")
+        return account
 
     def take_action(self, parsed_args):
+        json_key_file_path = raw_input("Enter path of service account json key file>")
+
+        cloudark_google_setup_details_path = APP_STORE_PATH + "/gcp-service-account-key.json"
+        if not os.path.exists(APP_STORE_PATH):
+            mkdir_command = ("mkdir -p {app_store_path}").format(app_store_path=APP_STORE_PATH)
+            os.system(mkdir_command)
+
+        if os.path.exists(json_key_file_path):
+            fp1 = open(json_key_file_path, "r")
+            input_lines = fp1.readlines()
+            fp = open(cloudark_google_setup_details_path, "w")
+            fp.writelines(input_lines)
+            fp.close()
+            fp1.close()
+        else:
+            print("File not found.")
+            exit()
+
+        print("Setting up GCP account information.")
+        account = self._get_gcp_account()
+
+        print("Setting up zone information.")
+        zone = self._setup_zone()
+
+        cloudark_google_setup_details_path = APP_STORE_PATH + "/google-creds-cloudark"
+        if not os.path.exists(APP_STORE_PATH):
+            mkdir_command = ("mkdir -p {app_store_path}").format(app_store_path=APP_STORE_PATH)
+            os.system(mkdir_command)
+
+        fp = open(cloudark_google_setup_details_path, "w")
+        fp.write("zone:%s\n" % zone.strip())
+        fp.write("account:%s" % account.strip())
+        fp.close()
+
+    def take_action_prev(self, parsed_args):
 
         gcloud_df = "Dockerfile.gcloudsetup"
         df = ("FROM lmecld/clis:gcloud \n"
@@ -57,37 +122,6 @@ def take_action(self, parsed_args):
                 print("Error occurred in setting up Google cloud. Exiting.")
                 exit()
 
-        zones = ['us-central1-a', 'us-central1-b', 'us-central1-c', 'us-central1-f',
-                 'us-west1-a', 'us-west1-b', 'us-west1-c',
-                 'us-east1-b', 'us-east1-c', 'us-east1-d',
-                 'us-east4-a', 'us-east4-b', 'us-east4-c',
-                 'southamerica-east1-a', 'southamerica-east1-b', 'southamerica-east1-c',
-                 'europe-west1-b', 'europe-west1-c', 'europe-west1-d',
-                 'europe-west3-a', 'europe-west3-b', 'europe-west3-c',
-                 'europe-west2-a', 'europe-west2-b', 'europe-west2-c',
-                 'asia-southeast1-a', 'asia-southeast1-b'
-                 'asia-east1-a', 'asia-east1-b', 'asia-east1-c',
-                 'asia-northeast1-a', 'asia-northeast1-b', 'asia-northeast1-c',
-                 'asia-south1-a', 'asia-south1-b', 'asia-south1-c',
-                 'australia-southeast1-a', 'australia-southeast1-b', 'australia-southeast1-c',
-                ]
-        for z in zones:
-            print(z)
-        while True:
-            zone = raw_input("Enter zone:" )
-            if zone in zones:
-                break
-            else:
-                print("Incorrect region specified. Please choose one of above.")
-
-        cloudark_google_setup_details_path = APP_STORE_PATH + "/google-creds-cloudark"
-        if not os.path.exists(APP_STORE_PATH):
-            mkdir_command = ("mkdir -p {app_store_path}").format(app_store_path=APP_STORE_PATH)
-            os.system(mkdir_command)
-
-        fp = open(cloudark_google_setup_details_path, "w")
-        fp.write("zone:%s" % zone.strip())
-        fp.close()
 
         os.system("./google-cloud-sdk/install.sh --quiet")
         os.system("./google-cloud-sdk/bin/gcloud auth login")

server/server_plugins/gcloud/coe/gke.py

@@ -10,6 +10,8 @@
 from googleapiclient import discovery
 from oauth2client.client import GoogleCredentials
 
+from google.oauth2 import service_account
+
 from server.common import constants
 from server.common import common_functions
 from server.common import docker_lib
@@ -26,6 +28,8 @@
 
 APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir)
 
+SERVICE_ACCOUNT_FILE = APP_AND_ENV_STORE_PATH + "gcp-service-account-key.json"
+
 fmlogger = fm_logger.Logging()
 
 GCR = "us.gcr.io"
@@ -49,7 +53,8 @@ class GKEHandler(coe_base.COEBase):
 
 
     def __init__(self):
-        credentials = GoogleCredentials.get_application_default()
+        #credentials = GoogleCredentials.get_application_default()
+        credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE)
         self.gke_service = discovery.build('container', 'v1',
                                            credentials=credentials)
         self.compute_service = discovery.build('compute', 'v1',
@@ -253,17 +258,22 @@ def create_cluster(self, env_id, env_info):
         if 'instance_type' in env_details['environment']['app_deployment']:
             instance_type = env_details['environment']['app_deployment']['instance_type']
 
-        try:
-            self._create_network(env_id, project, cluster_name)
-        except Exception as e:
-            fmlogger.error(e)
-            return
+        network = ''
+        if 'network' in env_details['environment']['app_deployment']:
+            network = env_details['environment']['app_deployment']['network']
+        if not network or network != 'default':
+            network = cluster_name
+            try:
+                self._create_network(env_id, project, cluster_name)
+            except Exception as e:
+                fmlogger.error(e)
+                return
 
-        try:
-            self._create_firewall_rule(env_id, project, cluster_name)
-        except Exception as e:
-            fmlogger.error(e)
-            return
+            try:
+                self._create_firewall_rule(env_id, project, cluster_name)
+            except Exception as e:
+                fmlogger.error(e)
+                return
 
         resp = ''
         try:
@@ -275,7 +285,7 @@ def create_cluster(self, env_id, env_info):
                                   "nodeConfig": {
                                       "oauthScopes": "https://www.googleapis.com/auth/devstorage.read_only",
                                       "machineType": instance_type},
-                                  "network": cluster_name}}
+                                  "network": network}}
             ).execute()
             fmlogger.debug(resp)
         except Exception as e:
@@ -344,17 +354,25 @@ def delete_cluster(self, env_id, env_info, resource_obj):
             project = filtered_description['project']
             zone = filtered_description['zone']
 
-            # Network delete is not working for some reason. So temporarily
-            # commenting it out.
-            #try:
-            #    self._delete_network(project, cluster_name)
-            #except Exception as e:
-            #    fmlogger.error("Exception deleting network %s " % str(e))
-            #    env_update = {}
-            #    env_update['output_config'] = str({'error': str(e)})
-            #    env_db.Environment().update(env_id, env_update)
-
-            self._delete_firewall_rule(project, cluster_name)
+            env_obj = env_db.Environment().get(env_id)
+            env_name = env_obj.name
+            env_details = ast.literal_eval(env_obj.env_definition)
+
+            network = ''
+            if 'network' in env_details['environment']['app_deployment']:
+                network = env_details['environment']['app_deployment']['network']
+            if not network or network != 'default':
+                # Network delete is not working for some reason. So temporarily
+                # commenting it out.
+                #try:
+                #    self._delete_network(project, cluster_name)
+                #except Exception as e:
+                #    fmlogger.error("Exception deleting network %s " % str(e))
+                #    env_update = {}
+                #    env_update['output_config'] = str({'error': str(e)})
+                #    env_db.Environment().update(env_id, env_update)
+
+                self._delete_firewall_rule(project, cluster_name)
 
             try:
                 resp = self.gke_service.projects().zones().clusters().delete(

server/server_plugins/gcloud/coe/gke_app_base.py

@@ -24,6 +24,8 @@
 
 APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir)
 
+SERVICE_ACCOUNT_FILE = APP_AND_ENV_STORE_PATH + "gcp-service-account-key.json"
+
 fmlogger = fm_logger.Logging()
 
 GCR = "us.gcr.io"

server/server_plugins/gcloud/coe/gke_single_container.py

@@ -9,6 +9,8 @@
 from googleapiclient import discovery
 from oauth2client.client import GoogleCredentials
 
+from google.oauth2 import service_account
+
 from server.common import constants
 from server.common import common_functions
 from server.common import docker_lib
@@ -24,6 +26,8 @@
 
 APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir)
 
+SERVICE_ACCOUNT_FILE = APP_AND_ENV_STORE_PATH + "gcp-service-account-key.json"
+
 fmlogger = fm_logger.Logging()
 
 GCR = "us.gcr.io"
@@ -34,7 +38,8 @@ class GKESingleContainer(gke_app_base.GKEAppBase):
     gcloudhelper = gcloud_helper.GCloudHelper()
 
     def __init__(self):
-        credentials = GoogleCredentials.get_application_default()
+        #credentials = GoogleCredentials.get_application_default()
+        credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE)
         self.gke_service = discovery.build('container', 'v1',
                                            credentials=credentials)
         self.compute_service = discovery.build('compute', 'v1',

server/server_plugins/gcloud/gcloud_helper.py

@@ -23,6 +23,8 @@
 
 APP_AND_ENV_STORE_PATH = ("{home_dir}/.cld/data/deployments/").format(home_dir=home_dir)
 
+GOOGLE_CREDS_FILE = APP_AND_ENV_STORE_PATH + "google-creds-cloudark"
+
 fmlogger = fm_logger.Logging()
 
 GCR = "us.gcr.io"
@@ -75,6 +77,29 @@ def get_access_token(self, df_dir, df, cont_name):
         return access_token
 
     def get_deployment_details(self, env_id):
+        project = ''
+        zone = ''
+        account = ''
+
+        if 'app_deployment' in env_details['environment']:
+            project = env_details['environment']['app_deployment']['project']
+            zone = env_details['environment']['app_deployment']['zone']
+        else:
+            project = env_details['environment']['resources']['gcloud'][0]['resource']['project']
+            zone = env_details['environment']['resources']['gcloud'][0]['resource']['zone']
+
+        fp = open(GOOGLE_CREDS_FILE, "r")
+        lines = fp.readlines()
+        for line in lines:
+            parts = line.split(":")
+            if parts[0].strip() == 'zone':
+                zone = parts[1].strip()
+            if parts[0].strip() == 'account':
+                account = parts[1].strip()
+
+        return user_account, project, zone
+
+    def get_deployment_details_bak(self, env_id):
         env_obj = env_db.Environment().get(env_id)
         env_details = ast.literal_eval(env_obj.env_definition)
         project = ''

server/server_plugins/gcloud/resource/gcr.py

@@ -55,10 +55,13 @@ def _build_container(self, cont_info, tag=''):
     def _push_container(self, cont_info, tagged_image):
         access_token = self._get_access_token(cont_info)
 
-        self.docker_handler.docker_login("oauth2accesstoken",
-                                         access_token, "https://" + GCR)
+        err, output = self.docker_handler.docker_login("oauth2accesstoken",
+                                                       access_token, "https://" + GCR)
 
-        self.docker_handler.push_container(tagged_image)
+        if not err:
+            err, output = self.docker_handler.push_container(tagged_image)
+
+        return err, output
 
     def _delete_container(self, tagged_image, cont_info):
         err, output = self.docker_handler.remove_container_image(tagged_image)
@@ -118,14 +121,21 @@ def create(self, cont_name, cont_info):
         cont_data['status'] = 'pushing-cont-to-gcr-repository'
 
         cont_db.Container().update(cont_name, cont_data)
+        err = ''
         try:
-            self._push_container(cont_info, tagged_image)
+            err, output = self._push_container(cont_info, tagged_image)
         except Exception as e:
             fmlogger.error("Exception encountered in pushing container to gcr %s" % e)
             cont_data['status'] = 'error-in-container-push-to-gcr:' + str(e)
             cont_db.Container().update(cont_name, cont_data)
             return
 
+        if err:
+            fmlogger.error("Exception encountered in pushing container to gcr %s" % err)
+            cont_data['status'] = 'error-in-container-push-to-gcr:' + str(err)
+            cont_db.Container().update(cont_name, cont_data)
+            return
+
         cont_data['status'] = 'container-ready'
         cont_db.Container().update(cont_name, cont_data)