Merge branch 'dev'

Author: Tereius

cloudRun.tf

@@ -39,8 +39,8 @@ resource "google_cloud_run_v2_service" "autoscaler" {
         value = local.projectId
       }
       env {
-        name  = "ZONE"
-        value = local.zone
+        name  = "ZONES"
+        value = join(",", local.zones)
       }
       env {
         name  = "TASK_QUEUE"

iam.tf

@@ -62,12 +62,15 @@ resource "google_project_iam_custom_role" "read_secret_version" {
 
 // ---- autoscaler-sa roles member ----
 resource "google_project_iam_member" "manage_vm_instances_member" {
+
+  count = length(local.zones)
+
   project = local.projectId
   member  = "serviceAccount:${google_service_account.autoscaler_sa.email}"
   role    = google_project_iam_custom_role.manage_vm_instances.id
   condition {
-    title       = "VM instance administration with a fix prefix ${var.github_runner_prefix}"
-    expression  = "resource.name.startsWith('projects/${local.projectId}/zones/${local.zone}/instances/${var.github_runner_prefix}-')"
+    title       = "VM instance administration with a fix prefix ${var.github_runner_prefix} in zone ${local.zones[count.index]}"
+    expression  = "resource.name.startsWith('projects/${local.projectId}/zones/${local.zones[count.index]}/instances/${var.github_runner_prefix}-')"
   }
 }
 
@@ -103,12 +106,15 @@ resource "google_project_iam_member" "create_vm_from_instance_template_member" {
 }
 
 resource "google_project_iam_member" "create_disk_member" {
+
+  count = length(local.zones)
+
   project = local.projectId
   member  = "serviceAccount:${google_service_account.autoscaler_sa.email}"
   role    = google_project_iam_custom_role.create_disk.id
   condition {
-    title      = "Create disk with a fix prefix ${var.github_runner_prefix}"
-    expression = "resource.name.startsWith('projects/${local.projectId}/zones/${local.zone}/disks/${var.github_runner_prefix}-')"
+    title      = "Create disk with a fix prefix ${var.github_runner_prefix} in zone ${local.zones[count.index]}"
+    expression = "resource.name.startsWith('projects/${local.projectId}/zones/${local.zones[count.index]}/disks/${var.github_runner_prefix}-')"
   }
 }
 

main.tf

@@ -18,7 +18,7 @@ locals {
   projectId                   = data.google_client_config.current.project
   projectNumber               = data.google_project.current.number
   region                      = data.google_client_config.current.region
-  zone                        = data.google_client_config.current.zone
+  zones                       = distinct(concat(var.machine_zones, data.google_client_config.current.zone != null ? [data.google_client_config.current.zone] : [] ))
   runnerLabel                 = join(",", var.github_runner_labels)
   runnerLabelInstanceTemplate = length(var.github_runner_labels) == 0 ? "" : format("--no-default-labels --labels '%s'", local.runnerLabel)
   hasEnterprise               = length(var.github_enterprise) > 0

outputs.tf

@@ -13,3 +13,8 @@ output "github_pat_secret_name" {
   value       = google_secret_manager_secret.github_pat_token.secret_id
   description = "The name of the secret in gcp Secret Manager where the GitHub Fine-grained personal access token (classic) has to be saved"
 }
+
+output "machine_zones" {
+  value       = local.zones
+  description = "The zones where VM instances will be created in"
+}

runner-autoscaler/README.md

@@ -32,7 +32,7 @@ The scaler is configured via the following environment variables:
 | ROUTE_DELETE_VM         | "/delete_vm"                           | The Cloud Run callback path invoked by Cloud Task when a VM instance should be **deleted**. The payload contains the name of the "to be deleted" VM instance.                                                                                       |
 | ROUTE_CREATE_VM         | "/create_vm"                           | The Cloud Run callback path invoked by Cloud Task when a VM instance should be **created**. The payload contains the name of the "to be created" VM instance.                                                                                       |
 | PROJECT_ID              | ""                                     | The Google Cloud Project Id.                                                                                                                                                                                                                        |
-| ZONE                    | ""                                     | The Google Cloud zone where the VM instance will be created.                                                                                                                                                                                        |
+| ZONES                   | "" *(comma separated list)*            | One or multiple Google Cloud zones where the VM instances will be created in. The zone is selected at random for each instance.                                                                                                                     |
 | TASK_QUEUE              | ""                                     | The relative resource name of the Cloud Task queue.                                                                                                                                                                                                 |
 | TASK_DISPATCH_TIMEOUT   | "180"                                  | The timeout in seconds for the Cloud Task callback (should be longer than it takes to create/delete a VM instance)                                                                                                                                  |
 | CREATE_VM_DELAY         | "10"                                   | The delay in seconds to wait before the VM is created. Useful for skipping the VM creation if the workflow job is canceled by the user shortly afterwards.                                                                                          |

runner-autoscaler/main.go

@@ -68,7 +68,7 @@ func main() {
 		RouteDeleteVm:     getEnvDefault("ROUTE_DELETE_VM", "/delete_vm"),
 		RouteCreateVm:     getEnvDefault("ROUTE_CREATE_VM", "/create_vm"),
 		ProjectId:         mustGetEnv("PROJECT_ID"),
-		Zone:              mustGetEnv("ZONE"),
+		Zones:             strings.Split(mustGetEnv("ZONES"), ","),
 		TaskQueue:         mustGetEnv("TASK_QUEUE"),
 		TaskTimeout:       getEnvDefaultInt64("TASK_DISPATCH_TIMEOUT", 180),
 		InstanceTemplate:  mustGetEnv("INSTANCE_TEMPLATE"),

runner-autoscaler/pkg/srv.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/hmac"
 	"crypto/sha256"
+	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -230,6 +231,15 @@ func CalcSigHex(secret []byte, data []byte) string {
 	return hex.EncodeToString(sig.Sum(nil))
 }
 
+// depending on an arbitrary input string a zone is selected
+// the same input string leads to the same zone
+func (s *Autoscaler) PickRandomZone(seed string) string {
+
+	hash := sha256.Sum256([]byte(seed))
+	index := binary.BigEndian.Uint64(hash[:8]) % uint64(len(s.conf.Zones))
+	return s.conf.Zones[index]
+}
+
 // returns http body, "src" query, error
 func (s *Autoscaler) verifySignature(ctx *gin.Context) ([]byte, Source, error) {
 
@@ -262,6 +272,7 @@ func (s *Autoscaler) verifySignature(ctx *gin.Context) ([]byte, Source, error) {
 	}
 }
 
+/*
 func (s *Autoscaler) GetInstanceState(ctx context.Context, instanceName string) (State, error) {
 
 	client := newComputeClient(ctx)
@@ -334,36 +345,40 @@ func (s *Autoscaler) StopInstance(ctx context.Context, instanceName string) erro
 	}
 	return nil
 }
+*/
 
 // blocking until the instance is deleted or the deletion fails
 func (s *Autoscaler) DeleteInstance(ctx context.Context, instanceName string) error {
 
 	if s.conf.Simulate {
-		log.Debugf("(SIMULATE) About to delete instance: %s", instanceName)
+		log.Debugf("(SIMULATE) About to delete instance %s", instanceName)
 		time.Sleep(30 * time.Second)
-		log.Infof("(SIMULATE) Deleted instance: %s", instanceName)
+		log.Infof("(SIMULATE) Deleted instance %s", instanceName)
 	} else {
-		log.Debugf("About to delete instance: %s", instanceName)
+
+		zone := s.PickRandomZone(instanceName)
+
+		log.Debugf("About to delete instance %s (%s)", instanceName, zone)
 		client := newComputeClient(ctx)
 		defer client.Close()
 		if res, err := client.Delete(ctx, &computepb.DeleteInstanceRequest{
 			Project:  s.conf.ProjectId,
-			Zone:     s.conf.Zone,
+			Zone:     zone,
 			Instance: instanceName,
 		}); err != nil {
 			if apiErr, ok := err.(*apierror.APIError); ok && apiErr.HTTPCode() == 404 {
 				// We ignore this error because the instance may no longer exist, as it may have been terminated prematurely
-				log.Infof("Instance already gone: %s", instanceName)
+				log.Infof("Instance %s (%s) already gone", instanceName, zone)
 			} else {
-				log.Errorf("Could not delete instance: %s - %s", instanceName, err.Error())
+				log.Errorf("Could not delete instance %s (%s): %s", instanceName, zone, err.Error())
 				return err
 			}
 		} else {
 			if err := res.Wait(ctx); err != nil {
-				log.Errorf("Failed to wait for instance to be deleted: %s", err.Error())
+				log.Errorf("Failed to wait for instance %s (%s) to be deleted: %s", instanceName, zone, err.Error())
 				return err
 			} else {
-				log.Infof("Deleted instance: %s", instanceName)
+				log.Infof("Deleted instance %s (%s)", instanceName, zone)
 			}
 		}
 	}
@@ -378,18 +393,21 @@ func (s *Autoscaler) CreateInstanceFromTemplate(ctx context.Context, instanceNam
 		time.Sleep(1 * time.Minute)
 		log.Infof("(SIMULATE) Created instance from template: %s", instanceName)
 	} else {
-		log.Debugf("About to create instance %s from template", instanceName)
+
+		zone := s.PickRandomZone(instanceName)
+
+		log.Debugf("About to create instance %s (%s) from template", instanceName, zone)
 		computeClient := newComputeClient(ctx)
 		defer computeClient.Close()
 
 		var machine *string = nil
 		if machineType != nil {
-			machine = proto.String(fmt.Sprintf("zones/%s/machineTypes/%s", s.conf.Zone, *machineType))
+			machine = proto.String(fmt.Sprintf("zones/%s/machineTypes/%s", zone, *machineType))
 		}
 
 		if res, err := computeClient.Insert(ctx, &computepb.InsertInstanceRequest{
 			Project: s.conf.ProjectId,
-			Zone:    s.conf.Zone,
+			Zone:    zone,
 			InstanceResource: &computepb.Instance{
 				Name:        proto.String(instanceName),
 				MachineType: machine,
@@ -399,14 +417,14 @@ func (s *Autoscaler) CreateInstanceFromTemplate(ctx context.Context, instanceNam
 			},
 			SourceInstanceTemplate: &s.conf.InstanceTemplate,
 		}); err != nil {
-			log.Errorf("Could not create instance %s from template: %s - %s", instanceName, s.conf.InstanceTemplate, err.Error())
+			log.Errorf("Could not create instance %s (%s) from template %s: %s", instanceName, zone, s.conf.InstanceTemplate, err.Error())
 			return err
 		} else {
 			if err := res.Wait(ctx); err != nil {
-				log.Errorf("Failed to wait for instance to be created from template: %s", err.Error())
+				log.Errorf("Failed to wait for instance %s (%s) to be created from template: %s", instanceName, zone, err.Error())
 				return err
 			} else {
-				log.Infof("Created instance from template: %s", instanceName)
+				log.Infof("Created instance %s (%s) from template", instanceName, zone)
 			}
 		}
 	}
@@ -706,7 +724,7 @@ type AutoscalerConfig struct {
 	RouteCreateVm     string
 	RouteDeleteVm     string
 	ProjectId         string
-	Zone              string
+	Zones             []string
 	TaskQueue         string
 	TaskTimeout       int64
 	InstanceTemplate  string

runner-autoscaler/test/main_test.go

@@ -37,7 +37,7 @@ func init() {
 		RouteCreateVm:    "/create",
 		RouteDeleteVm:    "/delete",
 		ProjectId:        PROJECT_ID,
-		Zone:             ZONE,
+		Zones:            []string{ZONE},
 		TaskQueue:        "projects/" + PROJECT_ID + "/locations/" + REGION + "/queues/autoscaler-callback-queue",
 		InstanceTemplate: "projects/" + PROJECT_ID + "/global/instanceTemplates/ephemeral-github-runner",
 		SecretVersion:    "projects/" + PROJECT_ID + "/secrets/github-pat-token/versions/latest",

variables.tf

@@ -50,6 +50,12 @@ variable "machine_timeout" {
   default     = 14400 // 4 h
 }
 
+variable "machine_zones" {
+  type        = list(string)
+  description = "One or multiple Google Cloud zones where the VM instances will be created in. The zone is selected at random for each instance."
+  default     = []
+}
+
 variable "autoscaler_timeout" {
   type        = number
   description = "The timeout of the autoscaler in seconds. Should be greater than the time required to create/delete a VM instance."

version.tf

@@ -1,4 +1,4 @@
 # used to tag the docker image of autoscaler. Should be incremented every time the autoscaler code is changed.
 locals {
-  autoscaler_version = "1.3"
+  autoscaler_version = "1.4-snapshot"
 }