Skip to content

Commit 2b16f77

Browse files
DavidGOrtegaDavidGOrtega
authored
Release v0.5.0 (#6)
* releaser * no error keypar * no keypar delete * no keypar delete * packer * endline * credentials * fix typo * sudo * no packer * dollar sign * public and no vpc * image * local provider and reset machine * SubnetId and packer regions * v0.3.3 * clone amis * mapping and name * release/v0.5.0
1 parent a01c40a commit 2b16f77

File tree

10 files changed

+411
-91
lines changed

10 files changed

+411
-91
lines changed

‎.github/workflows/build-cml-ami.yml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: cml-ami
2+
on:
3+
push:
4+
tags:
5+
- 'ami*'
6+
7+
jobs:
8+
build-ami:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- name: Checkout
12+
uses: actions/checkout@v2
13+
14+
- name: Packer
15+
env:
16+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
17+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
18+
AWS_INSTANCE_TYPE: g2.2xlarge
19+
run: |
20+
sudo apt install unzip
21+
22+
cd packer
23+
packer validate ami.json
24+
packer build ami.json
25+
./clone-amis.go

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
.DS_Store
2-
terraform-provider-hashicups
2+
examples
33
bin
44

55
# Local .terraform directories

‎Makefile

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,6 @@ default: install
1313
build:
1414
go build -o ${BINARY}
1515

16-
# release:
17-
# GOOS=darwin GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_darwin_amd64
18-
# GOOS=freebsd GOARCH=386 go build -o ./bin/${BINARY}_${VERSION}_freebsd_386
19-
# GOOS=freebsd GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_freebsd_amd64
20-
# GOOS=freebsd GOARCH=arm go build -o ./bin/${BINARY}_${VERSION}_freebsd_arm
21-
# GOOS=linux GOARCH=386 go build -o ./bin/${BINARY}_${VERSION}_linux_386
22-
# GOOS=linux GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_linux_amd64
23-
# GOOS=linux GOARCH=arm go build -o ./bin/${BINARY}_${VERSION}_linux_arm
24-
# GOOS=openbsd GOARCH=386 go build -o ./bin/${BINARY}_${VERSION}_openbsd_386
25-
# GOOS=openbsd GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_openbsd_amd64
26-
# GOOS=solaris GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_solaris_amd64
27-
# GOOS=windows GOARCH=386 go build -o ./bin/${BINARY}_${VERSION}_windows_386
28-
# GOOS=windows GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_windows_amd64
29-
3016
install: build
3117
mkdir -p ~/.terraform.d/plugins/${HOSTNAME}/${NAMESPACE}/${NAME}/${VERSION}/${OS_ARCH}
3218
mv ${BINARY} ~/.terraform.d/plugins/${HOSTNAME}/${NAMESPACE}/${NAME}/${VERSION}/${OS_ARCH}

‎README.md

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,67 @@
1-
# cml-terraform-provider
2-
Terraform provider for CML
1+
![Terraform Provider Iterative](https://user-images.githubusercontent.com/414967/98701372-7f60d700-2379-11eb-90d0-47b5eeb22658.png)
32

4-
```sh
5-
go mod vendor
6-
make install
3+
# Terraform Provider Iterative
4+
5+
The Terraform Iterative provider is a plugin for Terraform that allows for the full lifecycle management of GPU or non GPU cloud resources with your favourite [vendor](#supported-vendors). The provider offers a simple and homogeneous way to deploy a GPU or a cluster of them reducing the complexity.
6+
7+
# Usage
8+
9+
```tf
10+
terraform {
11+
required_providers {
12+
iterative = {
13+
source = "iterative/iterative"
14+
version = "0.5.0"
15+
}
16+
}
17+
}
18+
19+
provider "iterative" {}
20+
21+
resource "iterative_machine" "machine" {
22+
region = "us-west"
23+
instance_name = "machine"
24+
instance_hdd_size = "20"
25+
instance_type = "m"
26+
instance_gpu = "tesla"
27+
}
728
```
29+
30+
## Argument reference
31+
32+
| Variable | Values | Default | |
33+
| ------- | ------ | -------- | ------------- |
34+
| ```region``` | ```us-west``` ```us-east``` ```eu-west``` ```eu-north``` | ```us-west``` | Sets the collocation region |
35+
| ```instance_name``` | | cml_{UID} | Sets the instance name and related resources like AWS key pair. |
36+
| ```instance_hdd_size``` | | 10 | Sets the instance hard disk size in gb |
37+
| ```instance_type``` | ```m```, ```l```, ```xl``` | ```m``` | Sets thee instance computing size. You can also specify vendor specific machines in AWS i.e. ```t2.micro``` |
38+
| ```instance_gpu``` | ``` ```, ```testla```, ```k80``` | ``` ``` | Sets the desired GPU if the ```instance_type``` is one of our types. |
39+
| ```key_public``` | | | Set up ssh access with your OpenSSH public key. If not provided one be automatically generated and returned in terraform.tfstate |
40+
| aws_security_group | | ```cml``` | AWS specific variable to setup an specific security group. If specified the instance will be launched in with that sg within the vpc managed by the specified sg. If not a new sg called ```cml``` will be created under the default vpc |
41+
42+
43+
# Supported vendors
44+
45+
- AWS
46+
47+
### AWS instance equivalences.
48+
The instance type in AWS is calculated joining the ```instance_type``` and ```instance_gpu```
49+
50+
| type | gpu | aws |
51+
| ------- | ------ | -------- |
52+
| m | | m5.2xlarge |
53+
| l | | m5.8xlarge |
54+
| xl | | m5.16xlarge |
55+
| m | k80 | p2.xlarge |
56+
| l | k80 | p2.8xlarge |
57+
| xl | k80 | p2.16xlarge |
58+
| m | tesla | p3.xlarge |
59+
| l | tesla | p3.8xlarge |
60+
| xl | tesla | p3.16xlarge |
61+
62+
| region | aws |
63+
| ------- | ------ |
64+
| us-west | us-west-1 |
65+
| us-east | us-east-1 |
66+
| eu-north | us-north-1 |
67+
| eu-west | us-west-1 |

‎cml/ami-test.json

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"variables" : {
3+
"instance_type" : "{{env `AWS_INSTANCE_TYPE`}}"
4+
},
5+
"builders" : [
6+
{
7+
"type" : "amazon-ebs",
8+
"assume_role": {
9+
"role_arn": "arn:aws:iam::260760892802:role/dvc-cml-packer",
10+
"session_name": "cml-packer-session"
11+
},
12+
"region" : "us-west-1",
13+
"ami_name" : "iterative-cml-test",
14+
"ami_description" : "CML (Continous Machine Learning)",
15+
"ami_groups": ["all"],
16+
"force_deregister": "true",
17+
"force_delete_snapshot": "true",
18+
"ssh_username" : "ubuntu",
19+
"instance_type" : "g2.2xlarge",
20+
"source_ami_filter": {
21+
"filters": {
22+
"virtualization-type": "hvm",
23+
"name": "iterative-cml",
24+
"root-device-type": "ebs"
25+
},
26+
"owners": ["260760892802"],
27+
"most_recent": true
28+
},
29+
"run_tags" : {
30+
"Author" : "iterative"
31+
}
32+
}
33+
],
34+
"provisioners" : [
35+
{
36+
"type": "shell",
37+
"inline": [
38+
"nvidia-smi"
39+
],
40+
"start_retry_timeout": "10m",
41+
"expect_disconnect": true
42+
}
43+
]
44+
}

‎cml/ami.json

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"variables" : {
3+
"instance_type" : "{{env `AWS_INSTANCE_TYPE`}}"
4+
},
5+
"builders" : [
6+
{
7+
"type" : "amazon-ebs",
8+
"assume_role": {
9+
"role_arn": "arn:aws:iam::260760892802:role/dvc-cml-packer",
10+
"session_name": "cml-packer-session"
11+
},
12+
"region" : "us-west-1",
13+
"ami_name" : "iterative-cml",
14+
"ami_description" : "CML (Continous Machine Learning)",
15+
"ami_groups": ["all"],
16+
"force_deregister": "true",
17+
"force_delete_snapshot": "true",
18+
"ssh_username" : "ubuntu",
19+
"instance_type" : "g2.2xlarge",
20+
"source_ami_filter": {
21+
"filters": {
22+
"virtualization-type": "hvm",
23+
"name": "ubuntu/images/*ubuntu-*-18.04-amd64-server-*",
24+
"root-device-type": "ebs"
25+
},
26+
"owners": ["099720109477"],
27+
"most_recent": true
28+
},
29+
"run_tags" : {
30+
"Author" : "iterative"
31+
}
32+
}
33+
],
34+
"provisioners" : [
35+
{
36+
"type" : "shell",
37+
"environment_vars": ["DEBIAN_FRONTEND=noninteractive"],
38+
"script" : "./setup.sh"
39+
},
40+
{
41+
"type": "shell",
42+
"inline": [
43+
"sudo shutdown -r now",
44+
"sleep 60"
45+
],
46+
"start_retry_timeout": "10m",
47+
"expect_disconnect": true
48+
}
49+
]
50+
}

‎cml/clone-amis.go

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
//usr/bin/env go run $0 "$@"; exit
2+
package main
3+
4+
import (
5+
"fmt"
6+
"log"
7+
"os"
8+
9+
"github.com/aws/aws-sdk-go/aws"
10+
"github.com/aws/aws-sdk-go/aws/session"
11+
"github.com/aws/aws-sdk-go/service/ec2"
12+
"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
13+
)
14+
15+
func main() {
16+
region := "us-west-1"
17+
amiName := "iterative-cml"
18+
regions := []string{"us-east-1", "us-east-2", "us-west-2", "eu-central-1", "eu-west-1"}
19+
20+
sess, sessError := session.NewSession(&aws.Config{
21+
Region: aws.String(region)},
22+
)
23+
if sessError != nil {
24+
log.Printf("[ERROR] %s", sessError)
25+
os.Exit(1)
26+
}
27+
28+
svc := ec2.New(sess)
29+
30+
amiParams := &ec2.DescribeImagesInput{
31+
Filters: []*ec2.Filter{
32+
{
33+
Name: aws.String("name"),
34+
Values: []*string{aws.String(amiName)},
35+
},
36+
{
37+
Name: aws.String("architecture"),
38+
Values: []*string{aws.String("x86_64")},
39+
},
40+
},
41+
}
42+
imagesRes, imagesErr := svc.DescribeImages(amiParams)
43+
if imagesErr != nil {
44+
diag.FromErr(imagesErr)
45+
}
46+
if len(imagesRes.Images) == 0 {
47+
log.Printf("[ERROR] ami %s not found", amiName)
48+
os.Exit(1)
49+
}
50+
51+
ami := imagesRes.Images[0]
52+
amiID := *ami.ImageId
53+
amiDesc := *ami.Description
54+
55+
for _, value := range regions {
56+
fmt.Println("Cloning", value)
57+
58+
sess, _ := session.NewSession(&aws.Config{
59+
Region: aws.String(value)},
60+
)
61+
62+
svc := ec2.New(sess)
63+
64+
copyResult, err := svc.CopyImage(&ec2.CopyImageInput{
65+
SourceImageId: aws.String(amiID),
66+
SourceRegion: aws.String(region),
67+
Name: aws.String(amiName),
68+
Description: aws.String(amiDesc),
69+
})
70+
if err != nil {
71+
fmt.Println(err)
72+
}
73+
74+
svc.WaitUntilImageExists(&ec2.DescribeImagesInput{
75+
ImageIds: []*string{aws.String(*copyResult.ImageId)},
76+
Filters: []*ec2.Filter{
77+
{
78+
Name: aws.String("state"),
79+
Values: []*string{aws.String("available")},
80+
},
81+
},
82+
})
83+
84+
_, modifyErr := svc.ModifyImageAttribute(&ec2.ModifyImageAttributeInput{
85+
ImageId: aws.String(*copyResult.ImageId),
86+
LaunchPermission: &ec2.LaunchPermissionModifications{
87+
Add: []*ec2.LaunchPermission{
88+
{
89+
Group: aws.String("all"),
90+
},
91+
},
92+
},
93+
})
94+
if modifyErr != nil {
95+
fmt.Println(modifyErr)
96+
}
97+
}
98+
}

‎cml/setup.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#/bin/sh
2+
3+
echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes
4+
5+
curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh && \
6+
sudo usermod -aG docker \${USER}
7+
sudo setfacl --modify user:\${USER}:rw /var/run/docker.sock
8+
9+
curl -s -L https://nvidia.GitHub.io/nvidia-docker/gpgkey | sudo apt-key add - && \
10+
curl -s -L https://nvidia.GitHub.io/nvidia-docker/ubuntu18.04/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list && \
11+
sudo apt update && sudo apt install -y ubuntu-drivers-common && \
12+
sudo ubuntu-drivers autoinstall && \
13+
sudo apt install -y nvidia-container-toolkit

‎examples/main.tf

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
1-
21
terraform {
32
required_providers {
43
iterative = {
5-
source = "DavidGOrtega/iterative"
6-
version = "0.4.0"
4+
#source = "DavidGOrtega/iterative"
5+
#version = "0.4.0"
6+
versions = ["0.3"]
7+
source = "github.com/davidgortega/iterative"
78
}
89
}
910
}
1011

1112
provider "iterative" {}
1213

1314
resource "iterative_machine" "machine" {
14-
region = "us-east-1"
15-
16-
17-
18-
}
15+
region = "us-west"
16+
instance_type = "t2.micro"
17+
instance_gpu = "tesla"
18+
}

0 commit comments

Comments
 (0)