Merge pull request #19 from TerraHubCorp/dev

Another set of fixes and improvements

Author: euliancom

README.md

@@ -6,10 +6,10 @@ corresponding terraform configurations:
 
 | GCP Resource | Terraform Resource | Link to TerraHub Config |
 |-----------------------|--------------------|-------------------------|
-| Cloud Function | google_cloudfunctions_function | https://github.com/TerraHubCorp/demo-terraform-automation-gcp/blob/master/google_function/.terrahub.yml#L24 |
-| Cloud Storage | google_storage_bucket | https://github.com/TerraHubCorp/demo-terraform-automation-gcp/blob/master/google_storage/.terrahub.yml#L7 |
-| Static Website | google_storage_bucket_object | https://github.com/TerraHubCorp/demo-terraform-automation-gcp/blob/master/google_function/.terrahub.yml#L19 |
-| IAM Policy | iam_member_object_viewer | https://github.com/TerraHubCorp/demo-terraform-automation-gcp/blob/master/google_storage/.terrahub.yml#L7 |
+| Cloud Function | google_cloudfunctions_function | [google_function/.terrahub.yml#L24](https://github.com/TerraHubCorp/demo-terraform-automation-gcp/blob/master/google_function/.terrahub.yml#L24) |
+| Cloud Storage | google_storage_bucket | [google_storage/.terrahub.yml#L7](https://github.com/TerraHubCorp/demo-terraform-automation-gcp/blob/master/google_storage/.terrahub.yml#L7) |
+| Static Website | google_storage_bucket_object | [static_website/.terrahub.yml#L8](https://github.com/TerraHubCorp/demo-terraform-automation-gcp/blob/master/static_website/.terrahub.yml#L8) |
+| IAM Policy | google_storage_bucket_iam_member | [iam_object_viewer/.terrahub.yml#L9](https://github.com/TerraHubCorp/demo-terraform-automation-gcp/blob/master/iam_object_viewer/.terrahub.yml#L9) |
 
 Follow below instructions to try this out in your own Google Cloud account.
 
@@ -193,7 +193,7 @@ Your output should be similar to the one below:
 The source code of Google Function is stored in `index.js`:
 
 ```shell
-cat 'exports.helloGET = (req, res) => { res.send("Hello World!\n"); };' > index.js
+echo 'exports.helloGET = (req, res) => { res.send("Hello World!\n"); };' > index.js
 ```
 
 The source code of Static Website is cloned from another public repository:
@@ -212,8 +212,8 @@ Run the following command in terminal:
 ```shell
 terrahub component -t google_storage_bucket -n google_storage \
 && terrahub component -t google_cloudfunctions_function -n google_function -o ../google_storage \
-&& terrahub component -t google_storage_bucket -n google_static_website -d ./www/.terrahub \
-&& terrahub component -t google_storage_bucket_iam_member -n iam_member_object_viewer -d ./www/.terrahub -o ../google_static_website
+&& terrahub component -t google_storage_bucket -n static_website \
+&& terrahub component -t google_storage_bucket_iam_member -n iam_object_viewer -o ../static_website
 ```
 
 Your output should be similar to the one below:
@@ -295,34 +295,34 @@ Your output should be similar to the one below:
 
 Run the following commands in terminal:
 ```shell
-terrahub configure -i google_static_website -c component.template.terraform.backend.local.path='/tmp/.terrahub/local_backend/google_static_website/terraform.tfstate'
-terrahub configure -i google_static_website -c component.template.resource.google_storage_bucket.google_static_website.name="${STORAGE_BUCKET}_website"
-terrahub configure -i google_static_website -c component.template.resource.google_storage_bucket.google_static_website.location='US'
-terrahub configure -i google_static_website -c component.template.resource.google_storage_bucket.google_static_website.force_destroy='true'
-terrahub configure -i google_static_website -c component.template.resource.google_storage_bucket.google_static_website.project='${local.google_project_id}'
-terrahub configure -i google_static_website -c component.template.resource.google_storage_bucket.google_static_website.website.main_page_suffix='index.html'
-terrahub configure -i google_static_website -c component.template.resource.google_storage_bucket.google_static_website.website.not_found_page='/404.html'
-terrahub configure -i google_static_website -c component.template.variable -D -y
-terrahub configure -i google_static_website -c build.env.variables.THUB_ENV='dev'
-terrahub configure -i google_static_website -c build.env.variables.THUB_INDEX_FILE='www.txt'
-terrahub configure -i google_static_website -c build.env.variables.THUB_S3_PATH="gs://${STORAGE_BUCKET}_website"
-terrahub configure -i google_static_website -c build.env.variables.THUB_ROBOTS='../../robots.dev.txt'
-terrahub configure -i google_static_website -c build.env.variables.THUB_BUILD_PATH='../../build'
-terrahub configure -i google_static_website -c build.env.variables.THUB_SOURCE_PATH='../../assets ../../static/fonts ../../static/img ../../views'
-terrahub configure -i google_static_website -c build.env.variables.THUB_BUILD_OK='false'
-terrahub configure -i google_static_website -c build.env.variables.THUB_MAX_AGE='600'
-terrahub configure -i google_static_website -c build.phases.pre_build.commands[0]='echo "BUILD: Running pre_build step"'
-terrahub configure -i google_static_website -c build.phases.pre_build.commands[1]='./scripts/download.sh $THUB_INDEX_FILE $THUB_S3_PATH/$THUB_INDEX_FILE'
-terrahub configure -i google_static_website -c build.phases.pre_build.commands[2]='./scripts/compare.sh $THUB_INDEX_FILE $THUB_SOURCE_PATH'
-terrahub configure -i google_static_website -c build.phases.pre_build.finally[0]='echo "BUILD: pre_build step successful"'
-terrahub configure -i google_static_website -c build.phases.build.commands[0]='echo "BUILD: Running build step"'
-terrahub configure -i google_static_website -c build.phases.build.commands[1]='../../bin/compile.sh'
-terrahub configure -i google_static_website -c build.phases.build.finally[0]='echo "BUILD: build step successful"'
-terrahub configure -i google_static_website -c build.phases.post_build.commands[0]='echo "BUILD: Running post_build step"'
-terrahub configure -i google_static_website -c build.phases.post_build.commands[1]='./scripts/shasum.sh $THUB_BUILD_PATH/$THUB_INDEX_FILE'
-terrahub configure -i google_static_website -c build.phases.post_build.commands[2]='./scripts/upload.sh $THUB_BUILD_PATH $THUB_S3_PATH --cache-control max-age=$THUB_MAX_AGE'
-terrahub configure -i google_static_website -c build.phases.post_build.commands[3]='rm -f .terrahub_build.env $THUB_INDEX_FILE'
-terrahub configure -i google_static_website -c build.phases.post_build.finally[0]='echo "BUILD: post_build step successful"'
+terrahub configure -i static_website -c component.template.terraform.backend.local.path='/tmp/.terrahub/local_backend/static_website/terraform.tfstate'
+terrahub configure -i static_website -c component.template.resource.google_storage_bucket.static_website.name="${STORAGE_BUCKET}_website"
+terrahub configure -i static_website -c component.template.resource.google_storage_bucket.static_website.location='US'
+terrahub configure -i static_website -c component.template.resource.google_storage_bucket.static_website.force_destroy='true'
+terrahub configure -i static_website -c component.template.resource.google_storage_bucket.static_website.project='${local.google_project_id}'
+terrahub configure -i static_website -c component.template.resource.google_storage_bucket.static_website.website.main_page_suffix='index.html'
+terrahub configure -i static_website -c component.template.resource.google_storage_bucket.static_website.website.not_found_page='/404.html'
+terrahub configure -i static_website -c component.template.variable -D -y
+terrahub configure -i static_website -c build.env.variables.THUB_ENV='dev'
+terrahub configure -i static_website -c build.env.variables.THUB_INDEX_FILE='www.txt'
+terrahub configure -i static_website -c build.env.variables.THUB_GS_PATH="gs://${STORAGE_BUCKET}_website"
+terrahub configure -i static_website -c build.env.variables.THUB_ROBOTS='../www/robots.dev.txt'
+terrahub configure -i static_website -c build.env.variables.THUB_BUILD_PATH='../www/build'
+terrahub configure -i static_website -c build.env.variables.THUB_SOURCE_PATH='../www/assets ../www/static/fonts ../www/static/img ../www/views'
+terrahub configure -i static_website -c build.env.variables.THUB_BUILD_OK='false'
+terrahub configure -i static_website -c build.env.variables.THUB_MAX_AGE='600'
+terrahub configure -i static_website -c build.phases.pre_build.commands[0]='echo "BUILD: Running pre_build step"'
+terrahub configure -i static_website -c build.phases.pre_build.commands[1]='./scripts/download.sh $THUB_INDEX_FILE $THUB_GS_PATH/$THUB_INDEX_FILE'
+terrahub configure -i static_website -c build.phases.pre_build.commands[2]='./scripts/compare.sh $THUB_INDEX_FILE $THUB_SOURCE_PATH'
+terrahub configure -i static_website -c build.phases.pre_build.finally[0]='echo "BUILD: pre_build step successful"'
+terrahub configure -i static_website -c build.phases.build.commands[0]='echo "BUILD: Running build step"'
+terrahub configure -i static_website -c build.phases.build.commands[1]='../www/bin/compile.sh'
+terrahub configure -i static_website -c build.phases.build.finally[0]='echo "BUILD: build step successful"'
+terrahub configure -i static_website -c build.phases.post_build.commands[0]='echo "BUILD: Running post_build step"'
+terrahub configure -i static_website -c build.phases.post_build.commands[1]='./scripts/shasum.sh $THUB_BUILD_PATH/$THUB_INDEX_FILE'
+terrahub configure -i static_website -c build.phases.post_build.commands[2]='./scripts/upload.sh $THUB_BUILD_PATH $THUB_GS_PATH --cache-control max-age=$THUB_MAX_AGE'
+terrahub configure -i static_website -c build.phases.post_build.commands[3]='rm -f .terrahub_build.env $THUB_INDEX_FILE'
+terrahub configure -i static_website -c build.phases.post_build.finally[0]='echo "BUILD: post_build step successful"'
 ```
 
 Your output should be similar to the one below:
@@ -334,13 +334,13 @@ Your output should be similar to the one below:
 
 Run the following commands in terminal:
 ```shell
-terrahub configure -i iam_member_object_viewer -c component.template.terraform.backend.local.path='/tmp/.terrahub/local_backend/iam_member_object_viewer/terraform.tfstate'
-terrahub configure -i iam_member_object_viewer -c component.template.data.terraform_remote_state.storage.backend='local'
-terrahub configure -i iam_member_object_viewer -c component.template.data.terraform_remote_state.storage.config.path='/tmp/.terrahub/local_backend/google_static_website/terraform.tfstate'
-terrahub configure -i iam_member_object_viewer -c component.template.resource.google_storage_bucket_iam_member.iam_member_object_viewer.bucket="${STORAGE_BUCKET}_website"
-terrahub configure -i iam_member_object_viewer -c component.template.resource.google_storage_bucket_iam_member.iam_member_object_viewer.role="roles/storage.objectViewer"
-terrahub configure -i iam_member_object_viewer -c component.template.resource.google_storage_bucket_iam_member.iam_member_object_viewer.member="allUsers"
-terrahub configure -i iam_member_object_viewer -c component.template.variable -D -y
+terrahub configure -i iam_object_viewer -c component.template.terraform.backend.local.path='/tmp/.terrahub/local_backend/iam_object_viewer/terraform.tfstate'
+terrahub configure -i iam_object_viewer -c component.template.data.terraform_remote_state.storage.backend='local'
+terrahub configure -i iam_object_viewer -c component.template.data.terraform_remote_state.storage.config.path='/tmp/.terrahub/local_backend/static_website/terraform.tfstate'
+terrahub configure -i iam_object_viewer -c component.template.resource.google_storage_bucket_iam_member.iam_object_viewer.bucket="${STORAGE_BUCKET}_website"
+terrahub configure -i iam_object_viewer -c component.template.resource.google_storage_bucket_iam_member.iam_object_viewer.role="roles/storage.objectViewer"
+terrahub configure -i iam_object_viewer -c component.template.resource.google_storage_bucket_iam_member.iam_object_viewer.member="allUsers"
+terrahub configure -i iam_object_viewer -c component.template.variable -D -y
 ```
 
 Your output should be similar to the one below:
@@ -360,8 +360,8 @@ Your output should be similar to the one below:
 Project: demo-terraform-automation-gcp
  ├─ google_storage [path: ./google_storage]
  │  └─ google_function [path: ./google_function]
- └─ google_static_website [path: ./www/.terrahub/google_static_website]
-    └─ iam_member_object_viewer [path: ./www/.terrahub/iam_member_object_viewer]
+ └─ static_website [path: ./static_website]
+    └─ iam_object_viewer [path: ./iam_object_viewer]
 ```
 
 ## Run TerraHub Automation
@@ -370,7 +370,7 @@ Project: demo-terraform-automation-gcp
 
 Run the following command in terminal:
 ```shell
-terrahub run -y -a -i google_storage,google_static_website
+terrahub run -y -a -i google_storage,static_website
 ```
 
 Your output should be similar to the one below:
@@ -382,13 +382,17 @@ Your output should be similar to the one below:
 Run the following command in terminal:
 
 ```shell
-terrahub build -i google_function,google_static_website
+terrahub build -i google_function,static_website
 ```
 
 Your output should be similar to the one below:
 ```
 ```
 
+> NOTE: If you get an error saying `ServiceException: 401 Anonymous caller
+does not have storage.objects.get access to [***]`, run `gsutil config` and
+setup default project id.
+
 ### Run TerraHub Automation
 
 Run the following command in terminal:

bin/deploy.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+gcloud --version > /dev/null 2>&1 || { echo >&2 'gcloud is missing. aborting...'; exit 1; }
+npm --version > /dev/null 2>&1 || { echo >&2 "npm is missing. aborting..."; exit 1; }
+export NODE_PATH="$(npm root -g)"
+
+if [ -z "${BRANCH_FROM}" ]; then BRANCH_FROM = "dev"; fi
+if [ -z "${BRANCH_TO}" ]; then BRANCH_TO = "dev"; fi
+if [ "${BRANCH_TO}" != "dev" ]; then THUB_ENV="-e ${BRANCH_TO}"; fi
+if [ "${THUB_STATE}" == "approved" ]; then THUB_APPLY="-a"; fi
+
+git --version > /dev/null 2>&1 || { echo >&2 "git is missing. aborting..."; exit 1; }
+git checkout $BRANCH_TO
+git checkout $BRANCH_FROM
+
+jq --version > /dev/null 2>&1 || { echo >&2 'jq is missing. aborting...'; exit 1; }
+GOOGLE_CLOUD_PROJECT="$(gcloud config list --format=json | jq '.core.project')"
+BILLING_ID="$(gcloud beta billing accounts list --format=json | jq '.[0].name[16:]')"
+GOOGLE_APPLICATION_CREDENTIALS="${HOME}/.config/gcloud/${GOOGLE_CLOUD_PROJECT}.json"
+SERVICE_ACCOUNT="terraform@${GOOGLE_CLOUD_PROJECT}.iam.gserviceaccount.com"
+gcloud iam service-accounts keys create ${GOOGLE_APPLICATION_CREDENTIALS} --iam-account=${SERVICE_ACCOUNT}
+
+terrahub --version > /dev/null 2>&1 || { echo >&2 "terrahub is missing. aborting..."; exit 1; }
+terrahub configure -c template.locals.google_project_id="${GOOGLE_CLOUD_PROJECT}"
+terrahub configure -c template.locals.google_billing_account="${BILLING_ID}"
+
+terrahub run -y -b ${THUB_APPLY} ${THUB_ENV}
+echo "Execution successful: from ${BRANCH_FROM} into ${BRANCH_TO}"

google_function/.terrahub.yml

@@ -5,27 +5,12 @@ component:
   dependsOn:
     - ../google_storage
   template:
-    terraform:
-      backend:
-        local:
-          path: /tmp/.terrahub/local_backend/google_function/terraform.tfstate
-    data:
-      terraform_remote_state:
-        storage:
-          backend: local
-          config:
-            path: /tmp/.terrahub/local_backend/google_storage/terraform.tfstate
     resource:
-      google_storage_bucket_object:
-        google_storage_object:
-          name: demo.zip
-          bucket: '${data.terraform_remote_state.storage.thub_id}'
-          source: ./demo.zip
       google_cloudfunctions_function:
         google_function:
+          name: 'demofunction${local.project["code"]}'
           depends_on:
             - google_storage_bucket_object.google_storage_object
-          name: 'demofunction${local.project["code"]}'
           region: us-central1
           runtime: nodejs8
           description: My demo function
@@ -35,6 +20,21 @@ component:
           trigger_http: true
           timeout: 60
           entry_point: helloGET
+      google_storage_bucket_object:
+        google_storage_object:
+          name: demo.zip
+          bucket: '${data.terraform_remote_state.storage.thub_id}'
+          source: ./demo.zip
+    terraform:
+      backend:
+        local:
+          path: /tmp/.terrahub/local_backend/google_function/terraform.tfstate
+    data:
+      terraform_remote_state:
+        storage:
+          backend: local
+          config:
+            path: /tmp/.terrahub/local_backend/google_storage/terraform.tfstate
     output:
       id:
         value: '${google_cloudfunctions_function.google_function.id}'

google_function/scripts/build.sh

@@ -1,20 +1,20 @@
 #!/bin/bash
 
-## Component name
-COMPONENT_NAME=${1}
-if [ -z "${COMPONENT_NAME}" ]; then
-  echo >&2 'ERROR: COMPONENT_NAME variable is empty. Aborting...'
+## TerraHub Component
+THUB_COMPONENT=${1}
+if [ -z "${THUB_COMPONENT}" ]; then
+  echo >&2 'ERROR: THUB_COMPONENT variable is empty. Aborting...'
   exit 1
 fi
 
-## Object name
-OBJECT_NAME=${2}
-if [ -z "${OBJECT_NAME}" ]; then
-  echo >&2 'ERROR: OBJECT_NAME variable is empty. Aborting...'
+## Google Storage object name
+THUB_OBJECT=${2}
+if [ -z "${THUB_OBJECT}" ]; then
+  echo >&2 'ERROR: THUB_OBJECT variable is empty. Aborting...'
   exit 1
 fi
 
-## Bucket key
+## Google Storage bucket key
 THUB_BUCKET_KEY=${3}
 if [ -z "${THUB_BUCKET_KEY}" ]; then
   echo >&2 'ERROR: THUB_BUCKET_KEY variable is empty. Aborting...'
@@ -26,8 +26,9 @@ fi
 
 ## Checking if THUB_BUILD_OK is true
 if [ "$THUB_BUILD_OK" == "true" ]; then
-  OBJECT_IN_COMPONENT='component.template.resource.google_storage_bucket_object.'${OBJECT_NAME}'.name'
-  terrahub configure -i ${COMPONENT_NAME} -c ${OBJECT_IN_COMPONENT}=${THUB_BUCKET_KEY}$(date +%s).zip
+  THUB_OBJECT_KEY='component.template.resource.google_storage_bucket_object.'${THUB_OBJECT}'.name'
+  terrahub --version > /dev/null 2>&1 || { echo >&2 'terrahub is missing. Aborting...'; exit 1; }
+  terrahub configure -i ${THUB_COMPONENT} -c ${THUB_OBJECT_KEY}=${THUB_BUCKET_KEY}$(date +%s).zip
   echo 'Build was executed'
 else
   echo 'Build was NOT executed'

google_function/scripts/compare.sh

@@ -35,7 +35,7 @@ fi
 
 ## Checking if the project requires to be built
 THUB_COMPARE=$(head -n 1 ${THUB_SRC})
-echo "INFO: S3 Object SHA256 => ${THUB_COMPARE}"
+echo "INFO: Google Storage Object SHA256 => ${THUB_COMPARE}"
 if [ "${THUB_SHA}" != "${THUB_COMPARE}" ]; then
   echo 'Build is required!'
   echo 'export THUB_BUILD_OK="true"' >> .terrahub_build.env

google_function/scripts/download.sh

@@ -7,29 +7,29 @@ if [ -z "${THUB_SRC}" ]; then
   exit 1
 fi
 
-## S3 bucket name
-THUB_S3_PATH=${2-${THUB_S3_PATH}}
-if [ -z "${THUB_S3_PATH}" ]; then
-  echo >&2 'ERROR: THUB_S3_PATH variable is empty. Aborting...'
+## Google Storage bucket name
+THUB_GS_PATH=${2-${THUB_GS_PATH}}
+if [ -z "${THUB_GS_PATH}" ]; then
+  echo >&2 'ERROR: THUB_GS_PATH variable is empty. Aborting...'
   exit 1
 fi
 
 ## Clean environmental variables
 > .terrahub_build.env
 
-## Checking if THUB_S3_PATH file exists in S3
+## Checking if THUB_GS_PATH file exists in Google Storage
 gsutil --version > /dev/null 2>&1 || { echo >&2 'gsutil is missing. Aborting...'; exit 1; }
-THUB_CHECK_TYPE=$(gsutil ls ${THUB_S3_PATH} || echo "")
+THUB_CHECK_TYPE=$(gsutil ls ${THUB_GS_PATH} || echo "")
 if [ -z "${THUB_CHECK_TYPE}" ]; then
-  echo "INFO: ${THUB_S3_PATH} does NOT exist ==> First execution."
+  echo "INFO: ${THUB_GS_PATH} does NOT exist ==> First execution."
   echo 'export THUB_BUILD_OK="true"' >> .terrahub_build.env
   exit 0
 fi
 
-## Downloading from S3
-echo 'Downloading THUB_SRC from THUB_S3_PATH'
+## Downloading from Google Storage
+echo 'Downloading THUB_SRC from THUB_GS_PATH'
 if [[ $THUB_CHECK_TYPE = *" PRE "* ]]; then
-  gsutil rsync ${THUB_S3_PATH} ${THUB_SRC}
+  gsutil rsync ${THUB_GS_PATH} ${THUB_SRC}
 else
-  gsutil cp ${THUB_S3_PATH} ${THUB_SRC}
+  gsutil cp ${THUB_GS_PATH} ${THUB_SRC}
 fi