Allow for configuring a specific CML version (#255)

* got test -update
* Simplify semver handling

Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com>

Author: dacbd

go.mod

@@ -18,9 +18,11 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.18.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.7.2
 	github.com/aws/smithy-go v1.9.0
+	github.com/blang/semver/v4 v4.0.0
 	github.com/brianvoe/gofakeit/v6 v6.9.0
 	github.com/cloudflare/gokey v0.1.0
 	github.com/gobwas/glob v0.2.3
+	github.com/google/go-github/v42 v42.0.0
 	github.com/hashicorp/hcl/v2 v2.8.0 // indirect
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.8.0
 	github.com/rclone/rclone v1.57.0

go.sum

@@ -221,10 +221,14 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/billziss-gh/cgofuse v1.5.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
+github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
+github.com/bradleyfalzon/ghinstallation/v2 v2.0.3/go.mod h1:tlgi+JWCXnKFx/Y4WtnDbZEINo31N5bcvnCoqieefmk=
 github.com/brianvoe/gofakeit/v6 v6.9.0 h1:UCGhPCKLiqBc910TKS7LcOGf74NozftibFCbGIS6GZQ=
 github.com/brianvoe/gofakeit/v6 v6.9.0/go.mod h1:palrJUk4Fyw38zIFB/uBZqsgzW5VsNllhHKKwAebzew=
 github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
@@ -389,6 +393,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -447,7 +452,11 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-github/v39 v39.0.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
+github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
+github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=

iterative/resource_runner.go

@@ -51,6 +51,12 @@ func resourceRunner() *schema.Resource {
 				Required: true,
 				ForceNew: true,
 			},
+			"cml_version": &schema.Schema{
+				Type:     schema.TypeString,
+				Optional: true,
+				ForceNew: true,
+				Default:  "",
+			},
 			"labels": &schema.Schema{
 				Type:     schema.TypeString,
 				Optional: true,
@@ -303,7 +309,7 @@ sudo systemctl is-enabled cml.service && return 0
 
 {{- if not .container}}
 {{- if .setup}}{{.setup}}{{- end}}
-sudo npm config set user 0 && sudo npm install --global @dvcorg/cml
+{{.setupCML}}
 {{- end}}
 
 {{- if not .container}}
@@ -335,7 +341,7 @@ export KUBERNETES_CONFIGURATION={{escape .KUBERNETES_CONFIGURATION}}
 {{.runner_startup_script}}
 {{- end}}
 
-HOME="$(mktemp -d)" exec cml-runner \
+HOME="$(mktemp -d)" exec $(which cml-runner || echo "cml runner") \
   {{if .name}} --name {{escape .name}}{{end}} \
   {{if .labels}} --labels {{escape .labels}}{{end}} \
   {{if .idle_timeout}} --idle-timeout {{escape .idle_timeout}}{{end}} \
@@ -453,6 +459,7 @@ func provisionerCode(d *schema.ResourceData) (string, error) {
 	data["KUBERNETES_CONFIGURATION"] = os.Getenv("KUBERNETES_CONFIGURATION")
 	data["container"] = isContainerAvailable(d.Get("cloud").(string))
 	data["setup"] = strings.Replace(string(setup[:]), "#/bin/sh", "", 1)
+	data["setupCML"] = utils.GetCML(d.Get("cml_version").(string))
 
 	return renderScript(data)
 }

iterative/resource_runner_test.go

@@ -65,6 +65,7 @@ func generateSchemaTestData(cloud string, t *testing.T) *schema.ResourceData {
 		"driver":                "15 value with \"quotes\" and spaces",
 		"labels":                "16 value with \"quotes\" and spaces",
 		"instance_gpu":          "17 value with \"quotes\" and spaces",
+		"cml_version":           "18 value with \"quotes\" and spaces",
 		"runner_startup_script": "echo \"custom startup script\"",
 	})
 }

iterative/testdata/script_template_cloud_aws.golden

@@ -37,14 +37,14 @@ if [ ! -f "$FILE" ]; then
   echo OK | sudo tee "$FILE"
 fi
 
-sudo npm config set user 0 && sudo npm install --global @dvcorg/cml
+sudo npm config set user 0 && sudo npm install --global 18 value with "quotes" and spaces
 sudo tee /usr/bin/cml.sh << 'EOF'
 #!/bin/sh
 export AWS_SECRET_ACCESS_KEY='0 value with "quotes" and spaces'
 export AWS_ACCESS_KEY_ID='1 value with "quotes" and spaces'
 export AWS_SESSION_TOKEN='2 value with "quotes" and spaces'
 
-HOME="$(mktemp -d)" exec cml-runner \
+HOME="$(mktemp -d)" exec $(which cml-runner || echo "cml runner") \
    --name '10 value with "quotes" and spaces' \
    --labels '16 value with "quotes" and spaces' \
    --idle-timeout 11 \

iterative/testdata/script_template_cloud_azure.golden

@@ -37,15 +37,15 @@ if [ ! -f "$FILE" ]; then
   echo OK | sudo tee "$FILE"
 fi
 
-sudo npm config set user 0 && sudo npm install --global @dvcorg/cml
+sudo npm config set user 0 && sudo npm install --global 18 value with "quotes" and spaces
 sudo tee /usr/bin/cml.sh << 'EOF'
 #!/bin/sh
 export AZURE_CLIENT_ID='3 value with "quotes" and spaces'
 export AZURE_CLIENT_SECRET='4 value with "quotes" and spaces'
 export AZURE_SUBSCRIPTION_ID='5 value with "quotes" and spaces'
 export AZURE_TENANT_ID='6 value with "quotes" and spaces'
 
-HOME="$(mktemp -d)" exec cml-runner \
+HOME="$(mktemp -d)" exec $(which cml-runner || echo "cml runner") \
    --name '10 value with "quotes" and spaces' \
    --labels '16 value with "quotes" and spaces' \
    --idle-timeout 11 \

iterative/testdata/script_template_cloud_gcp.golden

@@ -37,12 +37,12 @@ if [ ! -f "$FILE" ]; then
   echo OK | sudo tee "$FILE"
 fi
 
-sudo npm config set user 0 && sudo npm install --global @dvcorg/cml
+sudo npm config set user 0 && sudo npm install --global 18 value with "quotes" and spaces
 sudo tee /usr/bin/cml.sh << 'EOF'
 #!/bin/sh
 export GOOGLE_APPLICATION_CREDENTIALS_DATA='7 value with "quotes" and spaces'
 
-HOME="$(mktemp -d)" exec cml-runner \
+HOME="$(mktemp -d)" exec $(which cml-runner || echo "cml runner") \
    --name '10 value with "quotes" and spaces' \
    --labels '16 value with "quotes" and spaces' \
    --idle-timeout 11 \

iterative/testdata/script_template_cloud_invalid.golden

@@ -37,11 +37,11 @@ if [ ! -f "$FILE" ]; then
   echo OK | sudo tee "$FILE"
 fi
 
-sudo npm config set user 0 && sudo npm install --global @dvcorg/cml
+sudo npm config set user 0 && sudo npm install --global 18 value with "quotes" and spaces
 sudo tee /usr/bin/cml.sh << 'EOF'
 #!/bin/sh
 
-HOME="$(mktemp -d)" exec cml-runner \
+HOME="$(mktemp -d)" exec $(which cml-runner || echo "cml runner") \
    --name '10 value with "quotes" and spaces' \
    --labels '16 value with "quotes" and spaces' \
    --idle-timeout 11 \

iterative/testdata/script_template_cloud_kubernetes.golden

@@ -2,7 +2,7 @@
 sudo systemctl is-enabled cml.service && return 0
 export KUBERNETES_CONFIGURATION='8 value with "quotes" and spaces'
 
-HOME="$(mktemp -d)" exec cml-runner \
+HOME="$(mktemp -d)" exec $(which cml-runner || echo "cml runner") \
    --name '10 value with "quotes" and spaces' \
    --labels '16 value with "quotes" and spaces' \
    --idle-timeout 11 \

iterative/utils/helpers.go

@@ -1,12 +1,74 @@
 package utils
 
 import (
+	"context"
+	"fmt"
 	"os"
+	"strings"
 
 	"github.com/aohorodnyk/uid"
+	"github.com/blang/semver/v4"
+	"github.com/google/go-github/v42/github"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 )
 
+func GetCML(version string) string {
+	// default latest if unset
+	if version == "" {
+		client := github.NewClient(nil)
+		release, _, err := client.Repositories.GetLatestRelease(context.Background(), "iterative", "cml")
+		if err != nil {
+			// GitHub API failed
+			return getNPMCML("@dvcorg/cml")
+		}
+		for _, asset := range release.Assets {
+			if *asset.Name == "cml-linux" {
+				return getGHCML(*asset.BrowserDownloadURL)
+			}
+		}
+	}
+	// handle semver
+	ver, err := semver.Make(strings.TrimPrefix(version, "v"))
+	if err == nil {
+		return getSemverCML(ver)
+	}
+	// user must know best, npm install <string>
+	if version != "" {
+		return getNPMCML(version)
+	}
+	// original fallback, some error has forced this
+	return getNPMCML("@dvcorg/cml")
+}
+
+func getGHCML(v string) string {
+	return fmt.Sprintf(`sudo mkdir -p /opt/cml/
+sudo curl --location --url %s --output /opt/cml/cml-linux
+sudo chmod +x /opt/cml/cml-linux
+sudo ln -s /opt/cml/cml-linux /usr/bin/cml`, v)
+}
+
+func getNPMCML(v string) string {
+	npmCML := "sudo npm config set user 0 && sudo npm install --global %s"
+	return fmt.Sprintf(npmCML, v)
+}
+
+func getSemverCML(sv semver.Version) string {
+	directDownloadVersion, _ := semver.ParseRange(">=0.10.0")
+	if directDownloadVersion(sv) {
+		client := github.NewClient(nil)
+		release, _, err := client.Repositories.GetReleaseByTag(context.Background(), "iterative", "cml", "v"+sv.String())
+		if err == nil {
+			for _, asset := range release.Assets {
+				if *asset.Name == "cml-linux" {
+					return getGHCML(*asset.BrowserDownloadURL)
+				}
+			}
+		}
+	}
+	// npm install
+	return getNPMCML("@dvcorg/cml@v" + sv.String())
+}
+
 func MachinePrefix(d *schema.ResourceData) string {
 	prefix := ""
 	if _, hasMachine := d.GetOk("machine"); hasMachine {