Skip to content

Commit d0638a6

Browse files
DavidGOrtega0x2b3bfa0
DavidGOrtega
and
0x2b3bfa0
authored
Setup machine if no image present (#79)
* Setup machine if no image present * regions-one-source-truth * Add eu-west-2 region as per #76 Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com>
1 parent db994d2 commit d0638a6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+18558
-62
lines changed

‎cml/clone-amis.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import (
66
"log"
77
"os"
88

9+
awsprovider "terraform-provider-iterative/iterative/aws"
10+
911
"github.com/aws/aws-sdk-go/aws"
1012
"github.com/aws/aws-sdk-go/aws/session"
1113
"github.com/aws/aws-sdk-go/service/ec2"
@@ -15,7 +17,7 @@ import (
1517
func main() {
1618
region := "us-west-1"
1719
amiName := "iterative-cml"
18-
regions := []string{"us-east-1", "us-east-2", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2"}
20+
regions := awsprovider.ImageRegions
1921

2022
sess, sessError := session.NewSession(&aws.Config{
2123
Region: aws.String(region)},

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ require (
1111
github.com/aws/aws-sdk-go v1.34.13
1212
github.com/hashicorp/terraform v0.14.2 // indirect
1313
github.com/hashicorp/terraform-plugin-sdk/v2 v2.4.0
14+
github.com/stretchr/testify v1.6.1
1415
github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf
1516
github.com/zclconf/go-cty v1.7.0 // indirect
1617
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0

‎go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
148148
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
149149
github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
150150
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
151+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
151152
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
152153
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM=
153154
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -464,6 +465,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
464465
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
465466
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
466467
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
468+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
467469
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
468470
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
469471
github.com/posener/complete v1.2.1/go.mod h1:6gapUrK/U1TAN7ciCoNRIdVC5sbdBTUh1DKN0g6uH7E=
@@ -503,6 +505,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
503505
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
504506
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
505507
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
508+
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
506509
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
507510
github.com/svanharmelen/jsonapi v0.0.0-20180618144545-0c0828c3f16d/go.mod h1:BSTlc8jOjh0niykqEGVXOLXdi9o0r0kR8tCYiMvjFgw=
508511
github.com/tencentcloud/tencentcloud-sdk-go v3.0.82+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
@@ -868,6 +871,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
868871
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
869872
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
870873
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
874+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
871875
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
872876
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
873877
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

‎iterative/aws/provider.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,9 @@ func awsClient(region string) (*ec2.EC2, error) {
306306
return svc, err
307307
}
308308

309+
//ImageRegions provider available image regions
310+
var ImageRegions = []string{"us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-north-1", "eu-west-1", "eu-west-2"}
311+
309312
func getRegion(region string) string {
310313
instanceRegions := make(map[string]string)
311314
instanceRegions["us-east"] = "us-east-1"

‎iterative/azure/provider.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,9 @@ func getVMClient(subscriptionID string) (compute.VirtualMachinesClient, error) {
348348
return client, err
349349
}
350350

351+
//ImageRegions provider available image regions
352+
var ImageRegions = []string{}
353+
351354
func getRegion(region string) string {
352355
instanceRegions := make(map[string]string)
353356
instanceRegions["us-east"] = "eastus"

‎iterative/resource_runner.go

Lines changed: 84 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import (
1010
"strconv"
1111
"text/template"
1212

13+
"terraform-provider-iterative/iterative/aws"
14+
"terraform-provider-iterative/iterative/azure"
1315
"terraform-provider-iterative/iterative/utils"
1416

1517
"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
@@ -179,69 +181,13 @@ func resourceRunnerDelete(ctx context.Context, d *schema.ResourceData, m interfa
179181
return resourceMachineDelete(ctx, d, m)
180182
}
181183

182-
func provisionerCode(d *schema.ResourceData) (string, error) {
183-
var code string
184-
185-
tfResource := ResourceType{
186-
Mode: "managed",
187-
Type: "iterative_cml_runner",
188-
Name: "runner",
189-
Provider: "provider[\"registry.terraform.io/iterative/iterative\"]",
190-
Instances: InstancesType{
191-
InstanceType{
192-
SchemaVersion: 0,
193-
Attributes: AttributesType{
194-
ID: d.Id(),
195-
Cloud: d.Get("cloud").(string),
196-
Region: d.Get("region").(string),
197-
Name: d.Get("name").(string),
198-
Labels: "",
199-
IdleTimeout: d.Get("idle_timeout").(int),
200-
Repo: "",
201-
Token: "",
202-
Driver: "",
203-
AwsSecurityGroup: "",
204-
CustomData: "",
205-
Image: "",
206-
InstanceGpu: "",
207-
InstanceHddSize: d.Get("instance_hdd_size").(int),
208-
InstanceIP: "",
209-
InstanceLaunchTime: "",
210-
InstanceType: "",
211-
SSHName: "",
212-
SSHPrivate: "",
213-
SSHPublic: "",
214-
},
215-
},
216-
},
217-
}
218-
jsonResource, err := json.Marshal(tfResource)
219-
if err != nil {
220-
return code, err
221-
}
222-
223-
data := make(map[string]string)
224-
data["cloud"] = d.Get("cloud").(string)
225-
data["token"] = d.Get("token").(string)
226-
data["repo"] = d.Get("repo").(string)
227-
data["driver"] = d.Get("driver").(string)
228-
data["labels"] = d.Get("labels").(string)
229-
data["idle_timeout"] = strconv.Itoa(d.Get("idle_timeout").(int))
230-
data["name"] = d.Get("name").(string)
231-
data["tf_resource"] = base64.StdEncoding.EncodeToString(jsonResource)
232-
data["instance_gpu"] = d.Get("instance_gpu").(string)
233-
data["AWS_SECRET_ACCESS_KEY"] = os.Getenv("AWS_SECRET_ACCESS_KEY")
234-
data["AWS_ACCESS_KEY_ID"] = os.Getenv("AWS_ACCESS_KEY_ID")
235-
data["AWS_SESSION_TOKEN"] = os.Getenv("AWS_SESSION_TOKEN")
236-
data["AZURE_CLIENT_ID"] = os.Getenv("AZURE_CLIENT_ID")
237-
data["AZURE_CLIENT_SECRET"] = os.Getenv("AZURE_CLIENT_SECRET")
238-
data["AZURE_SUBSCRIPTION_ID"] = os.Getenv("AZURE_SUBSCRIPTION_ID")
239-
data["AZURE_TENANT_ID"] = os.Getenv("AZURE_TENANT_ID")
184+
func renderScript(data map[string]interface{}) (string, error) {
185+
var script string
240186

241187
tmpl, err := template.New("deploy").Parse(`#!/bin/sh
242188
export DEBIAN_FRONTEND=noninteractive
243189
244-
{{if eq .cloud "azure"}}
190+
{{if not .ami }}
245191
echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes
246192
247193
sudo apt remove unattended-upgrades
@@ -312,10 +258,87 @@ sudo systemctl enable cml.service --now
312258
err = tmpl.Execute(&customDataBuffer, data)
313259

314260
if err == nil {
315-
code = customDataBuffer.String()
261+
script = customDataBuffer.String()
262+
}
263+
264+
return script, err
265+
}
266+
267+
func provisionerCode(d *schema.ResourceData) (string, error) {
268+
var code string
269+
270+
tfResource := ResourceType{
271+
Mode: "managed",
272+
Type: "iterative_cml_runner",
273+
Name: "runner",
274+
Provider: "provider[\"registry.terraform.io/iterative/iterative\"]",
275+
Instances: InstancesType{
276+
InstanceType{
277+
SchemaVersion: 0,
278+
Attributes: AttributesType{
279+
ID: d.Id(),
280+
Cloud: d.Get("cloud").(string),
281+
Region: d.Get("region").(string),
282+
Name: d.Get("name").(string),
283+
Labels: "",
284+
IdleTimeout: d.Get("idle_timeout").(int),
285+
Repo: "",
286+
Token: "",
287+
Driver: "",
288+
AwsSecurityGroup: "",
289+
CustomData: "",
290+
Image: "",
291+
InstanceGpu: "",
292+
InstanceHddSize: d.Get("instance_hdd_size").(int),
293+
InstanceIP: "",
294+
InstanceLaunchTime: "",
295+
InstanceType: "",
296+
SSHName: "",
297+
SSHPrivate: "",
298+
SSHPublic: "",
299+
},
300+
},
301+
},
302+
}
303+
jsonResource, err := json.Marshal(tfResource)
304+
if err != nil {
305+
return code, err
306+
}
307+
308+
data := make(map[string]interface{})
309+
data["token"] = d.Get("token").(string)
310+
data["repo"] = d.Get("repo").(string)
311+
data["driver"] = d.Get("driver").(string)
312+
data["labels"] = d.Get("labels").(string)
313+
data["idle_timeout"] = strconv.Itoa(d.Get("idle_timeout").(int))
314+
data["name"] = d.Get("name").(string)
315+
data["tf_resource"] = base64.StdEncoding.EncodeToString(jsonResource)
316+
data["instance_gpu"] = d.Get("instance_gpu").(string)
317+
data["AWS_SECRET_ACCESS_KEY"] = os.Getenv("AWS_SECRET_ACCESS_KEY")
318+
data["AWS_ACCESS_KEY_ID"] = os.Getenv("AWS_ACCESS_KEY_ID")
319+
data["AWS_SESSION_TOKEN"] = os.Getenv("AWS_SESSION_TOKEN")
320+
data["AZURE_CLIENT_ID"] = os.Getenv("AZURE_CLIENT_ID")
321+
data["AZURE_CLIENT_SECRET"] = os.Getenv("AZURE_CLIENT_SECRET")
322+
data["AZURE_SUBSCRIPTION_ID"] = os.Getenv("AZURE_SUBSCRIPTION_ID")
323+
data["AZURE_TENANT_ID"] = os.Getenv("AZURE_TENANT_ID")
324+
data["ami"] = isAMIAvailable(d.Get("cloud").(string), d.Get("region").(string))
325+
326+
return renderScript(data)
327+
}
328+
329+
func isAMIAvailable(cloud string, region string) bool {
330+
regions := aws.ImageRegions
331+
if cloud == "azure" {
332+
regions = azure.ImageRegions
333+
}
334+
335+
for _, item := range regions {
336+
if item == region {
337+
return true
338+
}
316339
}
317340

318-
return code, nil
341+
return false
319342
}
320343

321344
type AttributesType struct {

‎iterative/resource_runner_test.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package iterative
2+
3+
import (
4+
"strings"
5+
"testing"
6+
7+
"github.com/stretchr/testify/assert"
8+
)
9+
10+
func TestScript(t *testing.T) {
11+
t.Run("AWS known region should not add the NVIDA drivers", func(t *testing.T) {
12+
data := make(map[string]interface{})
13+
data["ami"] = isAMIAvailable("aws", "us-east-1")
14+
15+
script, _ := renderScript(data)
16+
assert.Equal(t, strings.Contains(script, "sudo ubuntu-drivers autoinstall"), false)
17+
})
18+
19+
t.Run("AWS unknown region should add the NVIDA drivers", func(t *testing.T) {
20+
data := make(map[string]interface{})
21+
data["ami"] = isAMIAvailable("aws", "us-east-99")
22+
23+
script, _ := renderScript(data)
24+
assert.Equal(t, strings.Contains(script, "sudo ubuntu-drivers autoinstall"), true)
25+
})
26+
27+
t.Run("Azure known region should add the NVIDA drivers", func(t *testing.T) {
28+
data := make(map[string]interface{})
29+
data["ami"] = isAMIAvailable("azure", "westus")
30+
31+
script, _ := renderScript(data)
32+
assert.Equal(t, strings.Contains(script, "sudo ubuntu-drivers autoinstall"), true)
33+
})
34+
35+
t.Run("Azure unknown region should add the NVIDA drivers", func(t *testing.T) {
36+
data := make(map[string]interface{})
37+
data["ami"] = isAMIAvailable("azure", "us-east-99")
38+
39+
script, _ := renderScript(data)
40+
assert.Equal(t, strings.Contains(script, "sudo ubuntu-drivers autoinstall"), true)
41+
})
42+
}

‎terraform-provider-iterative

35.5 MB
Binary file not shown.

‎vendor/github.com/davecgh/go-spew/LICENSE

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)