Autoscale crater instances

Crater no longer requires each instance to be uniquely identified for
assigned crates to not be duplicated across instances, which means that we can
relatively directly just copy the stateless configuration into more instances.
This commit adds a CPU-based autoscaler to the GCP managed instance group, and
configures a systemd timer via the startup script which will update the
crater-agent on the instances.

This continues building upon the startup scripts rather than pursuing Ansible
automation (perhaps backed by Packer or so) because they're both pretty simple
and it doesn't seem worth the fairly high complexity cost Ansible and Packer
would bring at this time.

The current autoscaling is just so that in theory we shutdown instances during
idle periods (if we have them), but in practice that's likely more of an
opportunistic nice to have -- eventually it might allow us to burst up to finish
work much faster and then go idle, but in practice it's not clear whether we
want slow, but constant, or fast, but sometimes unavailable until the next
month. Something to continue thinking about.

Author: Mark-Simulacrum

terraform/crater/agent.tf

@@ -159,6 +159,10 @@ resource "google_compute_instance_template" "agent" {
       role       = aws_iam_role.agent.arn,
       docker_url = module.ecr.url
     })
+    update-script = templatefile("update.sh", {
+      role       = aws_iam_role.agent.arn,
+      docker_url = module.ecr.url
+    })
   }
 
   service_account {
@@ -171,6 +175,23 @@ resource "google_compute_instance_template" "agent" {
   }
 }
 
+resource "google_compute_region_autoscaler" "agents" {
+  name   = "crater-autoscaler"
+  target = google_compute_region_instance_group_manager.agents.id
+
+  autoscaling_policy {
+    max_replicas    = 3
+    min_replicas    = 1
+    cooldown_period = 120
+    // This is pretty low, but in practice we want to scale out to the max
+    // unless we're entirely idle: crater is either all up or all down.
+    cpu_utilization {
+      target = 0.1
+    }
+  }
+}
+
+
 resource "google_compute_region_instance_group_manager" "agents" {
   name = "crater-agents"
 
@@ -180,8 +201,6 @@ resource "google_compute_region_instance_group_manager" "agents" {
     instance_template = google_compute_instance_template.agent.id
   }
 
-  target_size = 1
-
   auto_healing_policies {
     health_check      = google_compute_health_check.tcp_health.id
     initial_delay_sec = 600

terraform/crater/startup-script.sh

@@ -37,6 +37,15 @@ docker pull ${docker_url}
 
 mkdir -p /var/lib/crater-agent-workspace
 
+curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/update-script \
+    -o /opt/update.sh \
+    -H "Metadata-Flavor: Google"
+
+chmod +x /opt/update.sh
+
+# Run update task every 5 minutes
+sudo systemd-run --unit crater-agent-update --on-calendar='*:0/5' /opt/update.sh
+
 systemd-run \
     --unit crater-agent \
     docker run --init --rm --name crater-agent \

terraform/crater/update.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -euo pipefail
+
+aws sts assume-role-with-web-identity \
+    --role-arn ${role} \
+    --role-session-name $(hostname) \
+    --duration-seconds 900 \
+    --web-identity-token $(curl \
+        -H "Metadata-Flavor: Google" \
+        'http://metadata/computeMetadata/v1/instance/service-accounts/default/identity?audience=aws') \
+> credentials
+
+export AWS_ACCESS_KEY_ID="$(jq -r .Credentials.AccessKeyId credentials)"
+export AWS_SECRET_ACCESS_KEY="$(jq -r .Credentials.SecretAccessKey credentials)"
+export AWS_SESSION_TOKEN="$(jq -r .Credentials.SessionToken credentials)"
+
+rm credentials # Remove the raw file on disk, no need for that to exist
+
+AGENT_TOKEN=$(aws --region us-west-1 \
+    --output text --query Parameter.Value \
+    ssm get-parameter \
+    --name /prod/ansible/crater-gcp-2/crater-token \
+    --with-decryption)
+
+eval $(aws ecr get-login --no-include-email --region us-west-1)
+
+old_id="$(docker images --format "{{.ID}}" "${docker_url}")"
+docker pull "${docker_url}"
+new_id="$(docker images --format "{{.ID}}" "${docker_url}")"
+
+if [[ "$old_id" != "$new_id" ]]; then
+    echo "restarting container..."
+    sudo systemctl restart crater-agent
+fi