Clean up orphaned droplets

.github/workflows/golangci-lint.yml

@@ -21,4 +21,4 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v8
         with:
-          version: v2.1
+          version: v2.4

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/golangci/golangci-lint
-    rev: v2.1.6
+    rev: v2.4.0
     hooks:
       - id: golangci-lint
         name: golangci-lint

README.md

@@ -35,6 +35,7 @@ check "hashistack-allocated-cpu" {
   # ...
   target "do-droplets" {
     create_reserved_addresses                    = "true"
+    init_grace_period                            = "10m"
     ipv6                                         = "true"
     name                                         = "hashi-worker"
     node_class                                   = "hashistack"
@@ -56,6 +57,8 @@ check "hashistack-allocated-cpu" {
 
 - `name` `(string: <required>)` - A logical name of a Droplet "group". Every managed Droplet will be tagged with this value and its name is this value with a random suffix
 
+- `init_grace_period` `(duration: "0m")` Any droplets tagged with the `name` value which are older than this duration and still not recognised as a nomad client are considered to be orphans, and will be deleted at a subsequent scale-in event. Setting this to zero will disable this feature.
+
 - `region` `(string: <required>)` - The region to start in.
 
 - `vpc_uuid` `(string: <required>)` - The ID of the VPC where the Droplet will be located.
@@ -64,7 +67,8 @@ check "hashistack-allocated-cpu" {
 
 - `snapshot_id` `(string: <required>)` - The Droplet image ID.
 
-- `user_data` `(string: "")` - A string of the desired User Data for the Droplet or a path to a file containing the User Data
+- `user_data` `(string: "")` - A (raw or base64-encoded) string of the desired User Data for the Droplet,
+  or a path to a file containing the User Data
 
 - `ssh_keys` `(string: "")` - A comma-separated list of SSH fingerprints to enable
 
@@ -95,7 +99,7 @@ check "hashistack-allocated-cpu" {
 
 - `secure_introduction_wrapped_secret_validity` `(duration: <required if approle is defined>)` The duration the request wrapper for the SecretID is valid for, from the time it is generated.
 
-- `secure_introduction_filename` `(string: <required if approle is defined>)` The filename to store the unwrapped SecretID in
+- `secure_introduction_directory` `(string: "/run/vault-agent/")` The directory to store the unwrapped SecretID in, along with the RoleID
 
 ### Secure Introduction
 

TODO.md

@@ -0,0 +1,2 @@
+- replace quartz with synctime
+- use github.com/avast/retry-go

demo/image/packer/scripts/client.sh

@@ -16,15 +16,15 @@ telemetry {
 EOF
 
 hashi-up consul install \
-  --version 1.21.2 \
+  --version 1.21.4 \
   --local \
   --client-addr 0.0.0.0 \
   --advertise-addr "{{ GetInterfaceIP \"eth1\" }}" \
   --connect \
   --retry-join "provider=digitalocean region=${REGION} tag_name=${TAG_NAME} api_token=${API_TOKEN}"
 
 hashi-up nomad install \
-  --version 1.10.2 \
+  --version 1.10.4 \
   --local \
   --client \
   --datacenter "${DATACENTER}" \

demo/image/packer/scripts/server.sh

@@ -16,7 +16,7 @@ telemetry {
 EOF
 
 hashi-up consul install \
-  --version 1.21.2 \
+  --version 1.21.4 \
   --local \
   --server \
   --bootstrap-expect ${NR_OF_SERVERS} \
@@ -26,7 +26,7 @@ hashi-up consul install \
   --retry-join "provider=digitalocean region=${REGION} tag_name=${TAG_NAME} api_token=${API_TOKEN}"
 
 hashi-up nomad install \
-  --version 1.10.2 \
+  --version 1.10.4 \
   --local \
   --server \
   --bootstrap-expect ${NR_OF_SERVERS} \

demo/image/packer/scripts/setup.sh

@@ -18,12 +18,12 @@ curl -sSL https://github.com/containernetworking/plugins/releases/download/v1.7.
 curl -sL get.hashi-up.dev | sh
 
 hashi-up consul install \
-  --version 1.21.2 \
+  --version 1.21.4 \
   --local \
   --skip-enable
 
 hashi-up nomad install \
-  --version 1.10.2 \
+  --version 1.10.4 \
   --local \
   --skip-enable
 

demo/jobs/templates/autoscaler.nomad

@@ -11,7 +11,7 @@ job "autoscaler" {
       driver = "docker"
 
       artifact {
-        source      = "https://github.com/Aiven-Open/nomad-droplets-autoscaler/releases/download/v0.0.25/nomad-droplets-autoscaler_Linux_x86_64.tar.gz"
+        source      = "https://github.com/Aiven-Open/nomad-droplets-autoscaler/releases/download/v0.1.9/nomad-droplets-autoscaler_Linux_x86_64.tar.gz"
         destination = "local/plugins/"
       }
 
@@ -87,21 +87,20 @@ scaling "batch" {
     }
 
     target "do-droplets" {
-      name = "hashi-batch"
-      region = "${region}"
-      size = "s-1vcpu-1gb"
-      snapshot_id = ${snapshot_id}
-      user_data = "local/batch-startup.sh"
-      tags = "hashi-stack"
-
-      datacenter             = "batch_workers"
-      node_drain_deadline    = "1h"
-      node_selector_strategy = "empty_ignore_system"
-
-      reserve_ipv4_addresses = "false"
-      reserve_ipv6_addresses = "false"
-      ipv6 = "true"
       create_reserved_addresses = "false"
+      datacenter                = "batch_workers"
+      init_grace_period         = "10m"
+      ipv6                      = "true"
+      name                      = "hashi-batch"
+      node_drain_deadline       = "1h"
+      node_selector_strategy    = "empty_ignore_system"
+      region                    = "${region}"
+      reserve_ipv4_addresses    = "false"
+      reserve_ipv6_addresses    = "false"
+      size                      = "s-1vcpu-1gb"
+      snapshot_id               = ${snapshot_id}
+      tags                      = "hashi-stack"
+      user_data                 = "local/batch-startup.sh"
     }
   }
 }
@@ -156,6 +155,8 @@ EOF
 server:
   http_listen_port: {{ env "NOMAD_PORT_promtail" }}
   grpc_listen_port: 0
+  log_level: "debug"
+
 
 positions:
   filename: /tmp/positions.yaml

go.mod

@@ -1,18 +1,18 @@
 module github.com/Aiven-Open/nomad-droplets-autoscaler
 
-go 1.24
+go 1.25.0
 
 require (
 	github.com/coder/quartz v0.2.1
-	github.com/digitalocean/godo v1.159.0
-	github.com/goccy/go-yaml v1.18.0
+	github.com/digitalocean/godo v1.165.1
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-hclog v1.6.3
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/hashicorp/nomad-autoscaler v0.4.7
-	github.com/hashicorp/nomad/api v0.0.0-20250721135329-36b4aa79df33
+	github.com/hashicorp/nomad/api v0.0.0-20251003131842-48863bda8a9b
 	github.com/hashicorp/vault-client-go v0.4.3
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/stretchr/testify v1.10.0
+	github.com/stretchr/testify v1.11.1
 )
 
 require (
@@ -25,7 +25,7 @@ require (
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
-	github.com/hashicorp/cronexpr v1.1.2 // indirect
+	github.com/hashicorp/cronexpr v1.1.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect

go.sum

@@ -11,8 +11,8 @@ github.com/coder/quartz v0.2.1/go.mod h1:vsiCc+AHViMKH2CQpGIpFgdHIEQsxwm8yCscqKm
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/digitalocean/godo v1.159.0 h1:GQLfVueriDHYpwLzDcbydHs6nBvQBO8/r8r9imPC434=
-github.com/digitalocean/godo v1.159.0/go.mod h1:tYeiWY5ZXVpU48YaFv0M5irUFHXGorZpDNm7zzdWMzM=
+github.com/digitalocean/godo v1.165.1 h1:H37+W7TaGFOVH+HpMW4ZeW/hrq3AGNxg+B/K8/dZ9mQ=
+github.com/digitalocean/godo v1.165.1/go.mod h1:xQsWpVCCbkDrWisHA72hPzPlnC+4W5w/McZY5ij9uvU=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -26,8 +26,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
 github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
-github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
-github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -39,8 +37,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A=
-github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
+github.com/hashicorp/cronexpr v1.1.3 h1:rl5IkxXN2m681EfivTlccqIryzYJSXRGRNa0xeG7NA4=
+github.com/hashicorp/cronexpr v1.1.3/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -58,12 +56,14 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl/v2 v2.23.0 h1:Fphj1/gCylPxHutVSEOf2fBOh1VE4AuLV7+kbJf3qos=
 github.com/hashicorp/hcl/v2 v2.23.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA=
 github.com/hashicorp/nomad-autoscaler v0.4.7 h1:rjWGNeTdVJBo/iSrf+Oc4ecbFRas7vZrYl5CJ68X2lw=
 github.com/hashicorp/nomad-autoscaler v0.4.7/go.mod h1:jYBuDvQEgGHp9Pip7NyolV03Ae6/MCJ2NsX9MLqOdfA=
-github.com/hashicorp/nomad/api v0.0.0-20250721135329-36b4aa79df33 h1:AK4QkmjlDDAmUnfNBYu1csQc2kKF/lLXL4g8ZyO0tkA=
-github.com/hashicorp/nomad/api v0.0.0-20250721135329-36b4aa79df33/go.mod h1:y4olHzVXiQolzyk6QD/gqJxQTnnchlTf/QtczFFKwOI=
+github.com/hashicorp/nomad/api v0.0.0-20251003131842-48863bda8a9b h1:lwbKo7Jf1W81jALYSVH+Qcv0Sw4zh1Hbadx35Vyo29E=
+github.com/hashicorp/nomad/api v0.0.0-20251003131842-48863bda8a9b/go.mod h1:sldFTIgs+FsUeKU3LwVjviAIuksxD8TzDOn02MYwslE=
 github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
 github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
 github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
@@ -95,12 +95,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
-github.com/shoenig/test v1.12.1 h1:mLHfnMv7gmhhP44WrvT+nKSxKkPDiNkIuHGdIGI9RLU=
-github.com/shoenig/test v1.12.1/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI=
+github.com/shoenig/test v1.12.2 h1:ZVT8NeIUwGWpZcKaepPmFMoNQ3sVpxvqUh/MAqwFiJI=
+github.com/shoenig/test v1.12.2/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0=
 github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0=
 github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=

main.go

@@ -15,5 +15,10 @@ func main() {
 }
 
 func factory(log hclog.Logger) interface{} {
-	return plugin.NewDODropletsPlugin(context.Background(), log, plugin.Must(plugin.NewVault()))
+	ctx := context.Background()
+	v, err := plugin.NewVault(ctx, log)
+	if err != nil {
+		log.Error("cannot create vault client", "error", err)
+	}
+	return plugin.NewDODropletsPlugin(ctx, log, v)
 }

plugin/cloudinit.go

@@ -1,98 +1,24 @@
 package plugin
 
 import (
-	"errors"
 	"fmt"
-	"regexp"
 	"strings"
-
-	"github.com/goccy/go-yaml"
 )
 
-type CloudConfigPart struct {
-	Type    string `json:"type"`
-	Content string `json:"content"`
-}
-
-type CloudConfigArchive struct {
-	Parts []CloudConfigPart `json:"cloud-config-archive"`
-}
-
-func NewCloudConfigArchive(parts ...CloudConfigPart) *CloudConfigArchive {
-	result := &CloudConfigArchive{Parts: make([]CloudConfigPart, 0, 3)}
-	result.Parts = append(result.Parts, parts...)
-	return result
-}
-
-func ParseCloudConfigArchive(data string) (*CloudConfigArchive, error) {
-	result := &CloudConfigArchive{}
-	err := yaml.Unmarshal(
-		[]byte(
-			"---\ncloud-config-archive:\n  "+strings.ReplaceAll(
-				data,
-				"\n",
-				"\n  ",
-			),
-		),
-		result,
-	)
-	return result, err
-}
-
-func (c *CloudConfigArchive) String() string {
-	blankLine := regexp.MustCompile(`(?m)^\s*$`)
-	var result strings.Builder
-	_, _ = result.WriteString("#cloud-config-archive\n")
-	for _, part := range c.Parts {
-		_, _ = result.WriteString(fmt.Sprintf("- type: %v\n", part.Type))
-		_, _ = result.WriteString(
-			strings.TrimRight(blankLine.ReplaceAllLiteralString(
-				fmt.Sprintf(
-					"  content: |\n    %v\n",
-					strings.ReplaceAll(part.Content, "\n", "\n    "),
-				),
-				"",
-			), "\n") + "\n",
-		)
-	}
-	return result.String()
-}
-
 // PrependShellScriptToUserData will prepend a cloud-boothook section to the
 // existing user data, which may be empty, a
 // bare shell command, or using the cloud-config-archive format
 func PrependShellScriptToUserData(originalUserData, script string) (string, error) {
 	originalUserData = strings.TrimSpace(originalUserData)
-	cca := NewCloudConfigArchive(CloudConfigPart{Type: "text/x-shellscript", Content: script})
-
-	// empty original data
-	if len(originalUserData) == 0 {
-		return cca.String(), nil
+	if originalUserData != "" && !strings.HasPrefix(originalUserData, "#!/bin/bash\n") {
+		return "", fmt.Errorf("only bash scripts are supported, not ones starting with %v", originalUserData[:20])
 	}
-
-	// MIME multipart
-	if strings.HasPrefix(originalUserData, "Content-Type:") {
-		return "", errors.New("MIME multipart is not supported")
-	}
-
-	// raw shell script, so append to cloud config archive
-	if strings.HasPrefix(originalUserData, "#!") {
-		cca.Parts = append(
-			cca.Parts,
-			CloudConfigPart{Type: "text/x-shellscript", Content: originalUserData},
-		)
-		return cca.String(), nil
+	script = strings.TrimSpace(script)
+	if !strings.HasPrefix(script, "#!/bin/bash\n") {
+		return "", fmt.Errorf("only bash scripts are supported in the injected script, not ones starting with %v", script[:20])
 	}
 
-	// cloud config archive, so just prepend another script
-	if strings.HasPrefix(originalUserData, "#cloud-config-archive\n") {
-		sections := strings.SplitN(originalUserData, "\n", 2)
-		originalCca, err := ParseCloudConfigArchive(sections[1])
-		if err != nil {
-			return "", fmt.Errorf("unable to parse original cloud-config-archive: %w", err)
-		}
-		cca.Parts = append(cca.Parts, originalCca.Parts...)
-		return cca.String(), nil
-	}
-	return "", errors.New("unrecognised user data format")
+	// the shebang only is treated as a non-comment on the first line, so simply concatenating
+	// these files is good enough
+	return fmt.Sprintf("%v\n%v\n", script, originalUserData), nil
 }