Skip to content

Commit 9e751fe

Browse files
DavidGOrtegaDavidGOrtega
committed
wip runner
1 parent 6802196 commit 9e751fe

File tree

754 files changed

+256713
-18915
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

754 files changed

+256713
-18915
lines changed

‎cml/ami.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
},
1212
"region" : "us-west-1",
1313
"ami_name" : "iterative-cml",
14-
"ami_description" : "CML (Continous Machine Learning)",
14+
"ami_description" : "CML (Continous Machine Learning). Ubuntu 18.04",
1515
"ami_groups": ["all"],
1616
"force_deregister": "true",
1717
"force_delete_snapshot": "true",

‎cml/azure.json

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{
2+
"variables": {
3+
"rg": "IterativeCMLRG"
4+
},
5+
6+
"builders": [{
7+
"type": "azure-arm",
8+
9+
"client_id": "8c7fa34f-8cb7-4b32-b007-4cbd4e317371",
10+
"client_secret": "L-17k_d.-tvWG2QeaWwi2-smVcmN7v9WBt",
11+
"tenant_id": "7d111986-d84c-4bef-817e-c4ab3addf85b",
12+
"subscription_id": "cee76754-ef49-49f7-b371-f6841fa82182",
13+
14+
"managed_image_resource_group_name": "IterativeCMLRG",
15+
"managed_image_name": "iterative-cml",
16+
"shared_image_gallery_destination": {
17+
"resource_group": "{{user `rg`}}",
18+
"gallery_name": "iterative",
19+
"image_name": "iterative-cml",
20+
"image_version": "1.0.0",
21+
"replication_regions": ["US East"]
22+
},
23+
"shared_image_gallery_timeout": "2h5m2s",
24+
25+
"os_type": "Linux",
26+
"image_publisher": "Canonical",
27+
"image_offer": "UbuntuServer",
28+
"image_sku": "18.04-LTS",
29+
30+
"azure_tags": {
31+
"Author" : "iterative"
32+
},
33+
34+
"location": "East US",
35+
"vm_size": "Standard_DS2_v2"
36+
}],
37+
"provisioners": [
38+
{
39+
"type" : "shell",
40+
"environment_vars": ["DEBIAN_FRONTEND=noninteractive"],
41+
"script" : "./setup.sh"
42+
},
43+
{
44+
"type": "shell",
45+
"inline": [
46+
"sudo shutdown -r now",
47+
"sleep 60"
48+
],
49+
"start_retry_timeout": "10m",
50+
"expect_disconnect": true
51+
}
52+
]
53+
}

‎cml/hello.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Hello world

‎cml/local.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
//usr/bin/env go run $0 "$@"; exit
2+
package main
3+
4+
import (
5+
"fmt"
6+
7+
localexec "github.com/hashicorp/terraform/builtin/provisioners/local-exec"
8+
"github.com/hashicorp/terraform/terraform"
9+
)
10+
11+
func main() {
12+
c := terraform.NewResourceConfigRaw(map[string]interface{}{
13+
"command": "echo 'Hello world' > hello.txt",
14+
})
15+
p := localexec.Provisioner()
16+
if err := p.Apply(new(terraform.MockUIOutput), nil, c); err != nil {
17+
fmt.Println(err)
18+
} else {
19+
fmt.Println("nice")
20+
}
21+
}

‎cml/setup.sh

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,24 @@
11
#/bin/sh
22

3+
DEBIAN_FRONTEND=noninteractive
34
echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes
45

5-
curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh && \
6-
sudo usermod -aG docker \${USER}
7-
sudo setfacl --modify user:\${USER}:rw /var/run/docker.sock
6+
sudo apt update
7+
sudo curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh && \
8+
sudo usermod -aG docker ubuntu
9+
sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock
810

11+
curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
12+
sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
13+
sudo apt update && sudo apt-get install -y terraform
14+
15+
curl -sL https://deb.nodesource.com/setup_12.x | sudo bash
16+
sudo apt update && sudo apt-get install -y nodejs
17+
18+
sudo apt install -y ubuntu-drivers-common git
19+
sudo ubuntu-drivers autoinstall
920
curl -s -L https://nvidia.GitHub.io/nvidia-docker/gpgkey | sudo apt-key add - && \
10-
curl -s -L https://nvidia.GitHub.io/nvidia-docker/ubuntu18.04/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list && \
11-
sudo apt update && sudo apt install -y ubuntu-drivers-common && \
12-
sudo ubuntu-drivers autoinstall && \
13-
sudo apt install -y nvidia-container-toolkit
21+
curl -s -L https://nvidia.GitHub.io/nvidia-docker/ubuntu18.04/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
22+
sudo apt update && sudo apt install -y nvidia-container-toolkit
23+
24+
sudo npm install -g git+https://github.com/iterative/cml.git#cml-runner

‎examples/main.tf

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,31 +9,54 @@ terraform {
99

1010
provider "iterative" {}
1111

12+
/*
13+
resource "iterative_machine" "machine-az" {
14+
cloud = "azure"
15+
region = "us-east"
16+
instance_type = "m"
17+
}
18+
*/
1219

13-
resource "iterative_machine" "machine-aws" {
14-
cloud = "aws"
15-
region = "us-west"
16-
instance_type = "t2.micro" //fallback to known instance type
20+
resource "iterative_runner" "runner-az" {
21+
token = "arszDpb3xtNdKaXmQ6vN"
22+
repo = "https://gitlab.com/DavidGOrtega/3_tensorboard"
23+
driver = "gitlab"
24+
labels = "tf"
25+
machine {
26+
cloud = "azure"
27+
region = "us-west"
28+
instance_type = "m"
29+
}
1730
}
1831

32+
/*
33+
resource "iterative_machine" "machine-aws" {
34+
cloud = "aws"
35+
region = "us-west"
36+
instance_type = "t2.micro" //fallback to known instance type
37+
}
38+
*/
1939

20-
/* resource "iterative_machine" "machine-azure" {
40+
41+
/*
42+
resource "iterative_machine" "machine-az" {
2143
cloud = "azure"
22-
region = "us-west"
44+
region = "us-east"
2345
instance_type = "m"
2446
2547
provisioner "remote-exec" {
26-
inline = [
27-
"echo 'hello azure'"
48+
scripts = [
49+
"provision.sh",
2850
]
2951
3052
connection {
3153
user = "ubuntu"
32-
private_key = "${self.key_private}"
54+
private_key = "${self.ssh_private}"
3355
host = "${self.instance_ip}"
3456
}
3557
}
36-
} */
58+
}
59+
*/
3760

3861

3962
/* resource "iterative_machine" "machine-azure-gpu" {

‎examples/provision.sh

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
3+
DEBIAN_FRONTEND=noninteractive
4+
echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes
5+
6+
curl -sL https://deb.nodesource.com/setup_12.x | sudo bash
7+
curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
8+
sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
9+
sudo apt update && sudo apt-get install -y terraform nodejs
10+
sudo npm install -g git+https://github.com/iterative/cml.git#cml-runner
11+
12+
echo 'launching runner'
13+
14+
nohup cml-runner \
15+
--name furia4 \
16+
--workspace ~/runner \
17+
--labels tf \
18+
--idle-timeout 180 \
19+
--repo https://gitlab.com/DavidGOrtega/3_tensorboard \
20+
--token arszDpb3xtNdKaXmQ6vN < /dev/null > std.out 2> std.err &
21+
22+
sleep 5
23+
echo 'Finished'

‎go.mod

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ require (
99
github.com/Azure/go-autorest/autorest/azure/auth v0.5.3
1010
github.com/Azure/go-autorest/autorest/to v0.4.0
1111
github.com/aws/aws-sdk-go v1.34.13
12-
github.com/hashicorp/terraform-plugin-sdk/v2 v2.0.1
12+
github.com/hashicorp/terraform v0.14.2
13+
github.com/hashicorp/terraform-plugin-sdk/v2 v2.4.0
1314
github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf
15+
github.com/zclconf/go-cty v1.7.0 // indirect
1416
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
1517
)

0 commit comments

Comments
 (0)