Azure provider (#18)

* wip

* wip

* wip

* wip

* wipp

* wip

* wipp

* code

* pre-release

* wip runner

* json

* wait

* base64

* clooud

* credentials

* remove log

* setname

* aws delete by tag

* same id

* install gpu

* aws az runner

* iterative_cml_runner and cleanup

* remove azure

Author: DavidGOrtega

.gitignore

@@ -1,5 +1,5 @@
 .DS_Store
-examples
+examples/.terraform
 bin
 
 # Local .terraform directories

Makefile

@@ -1,12 +1,11 @@
 TEST?=$$(go list ./... | grep -v 'vendor')
 HOSTNAME=github.com
-NAMESPACE=DavidGOrtega
+NAMESPACE=iterative
 NAME=iterative
-VERSION=0.3
+VERSION=0.6
 #OS_ARCH=linux_amd64
 OS_ARCH=darwin_amd64
 BINARY=terraform-provider-${NAME}
-VERSION2=0.3.3
 
 default: install
 

cml/ami.json

@@ -11,7 +11,7 @@
             },
             "region" : "us-west-1",
             "ami_name" : "iterative-cml",
-            "ami_description" : "CML (Continous Machine Learning)",
+            "ami_description" : "CML (Continous Machine Learning). Ubuntu 18.04",
             "ami_groups": ["all"],
             "force_deregister": "true",
             "force_delete_snapshot": "true",

cml/setup.sh

@@ -1,13 +1,22 @@
 #/bin/sh
 
+DEBIAN_FRONTEND=noninteractive
 echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes
 
-curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh && \
-sudo usermod -aG docker \${USER}
-sudo setfacl --modify user:\${USER}:rw /var/run/docker.sock
+sudo apt update
+sudo curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh && \
+sudo usermod -aG docker ubuntu
+sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock
 
+curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
+sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+sudo apt update && sudo apt-get install -y terraform
+
+curl -sL https://deb.nodesource.com/setup_12.x | sudo bash
+sudo apt update && sudo apt-get install -y nodejs
+
+sudo apt install -y ubuntu-drivers-common git
+sudo ubuntu-drivers autoinstall 
 curl -s -L https://nvidia.GitHub.io/nvidia-docker/gpgkey | sudo apt-key add - && \
-curl -s -L https://nvidia.GitHub.io/nvidia-docker/ubuntu18.04/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list && \
-sudo apt update && sudo apt install -y ubuntu-drivers-common  && \
-sudo ubuntu-drivers autoinstall  && \
-sudo apt install -y nvidia-container-toolkit
+curl -s -L https://nvidia.GitHub.io/nvidia-docker/ubuntu18.04/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+sudo apt update && sudo apt install -y nvidia-container-toolkit

go.mod

@@ -3,7 +3,15 @@ module terraform-provider-iterative
 go 1.14
 
 require (
+	github.com/Azure-Samples/azure-sdk-for-go-samples v0.0.0-20201110054206-ffcdafe9818d // indirect
+	github.com/Azure/azure-sdk-for-go v49.0.0+incompatible
+	github.com/Azure/go-autorest/autorest v0.11.13 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.3
+	github.com/Azure/go-autorest/autorest/to v0.4.0
 	github.com/aws/aws-sdk-go v1.34.13
-	github.com/hashicorp/terraform-plugin-sdk/v2 v2.0.1
+	github.com/hashicorp/terraform v0.14.2
+	github.com/hashicorp/terraform-plugin-sdk/v2 v2.4.0
 	github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf
+	github.com/zclconf/go-cty v1.7.0 // indirect
+	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
 )

go.sum

@@ -0,0 +1,285 @@
+package aws
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+//ResourceMachineCreate creates AWS instance
+func ResourceMachineCreate(ctx context.Context, d *schema.ResourceData, m interface{}) error {
+	userData := d.Get("startup_script").(string)
+	instanceName := d.Get("name").(string)
+	pairName := d.Id()
+	hddSize := d.Get("instance_hdd_size").(int)
+	region := getRegion(d.Get("region").(string))
+	instanceType := getInstanceType(d.Get("instance_type").(string), d.Get("instance_gpu").(string))
+	ami := d.Get("image").(string)
+	keyPublic := d.Get("ssh_public").(string)
+	securityGroup := d.Get("aws_security_group").(string)
+	if ami == "" {
+		ami = "iterative-cml"
+	}
+
+	svc, err := awsClient(region)
+	if err != nil {
+		return err
+	}
+
+	// Image
+	imagesRes, err := svc.DescribeImages(&ec2.DescribeImagesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("name"),
+				Values: []*string{aws.String(ami)},
+			},
+			{
+				Name:   aws.String("architecture"),
+				Values: []*string{aws.String("x86_64")},
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+	if len(imagesRes.Images) == 0 {
+		return errors.New(ami + " ami not found in region")
+	}
+
+	sort.Slice(imagesRes.Images, func(i, j int) bool {
+		itime, _ := time.Parse(time.RFC3339, aws.StringValue(imagesRes.Images[i].CreationDate))
+		jtime, _ := time.Parse(time.RFC3339, aws.StringValue(imagesRes.Images[j].CreationDate))
+		return itime.Unix() > jtime.Unix()
+	})
+
+	instanceAmi := *imagesRes.Images[0].ImageId
+
+	// key-pair
+	svc.ImportKeyPair(&ec2.ImportKeyPairInput{
+		KeyName:           aws.String(pairName),
+		PublicKeyMaterial: []byte(keyPublic),
+	})
+	//if err != nil {
+	//return err
+	//}
+
+	// securityGroup
+	var vpcID, sgID string
+	if len(securityGroup) == 0 {
+		securityGroup = "cml"
+
+		vpcsDesc, _ := svc.DescribeVpcs(&ec2.DescribeVpcsInput{})
+		vpcID = *vpcsDesc.Vpcs[0].VpcId
+
+		gpResult, err := svc.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
+			GroupName:   aws.String(securityGroup),
+			Description: aws.String("CML security group"),
+			VpcId:       aws.String(vpcID),
+		})
+
+		if err == nil {
+			ipPermissions := []*ec2.IpPermission{
+				(&ec2.IpPermission{}).
+					SetIpProtocol("-1").
+					SetFromPort(-1).
+					SetToPort(-1).
+					SetIpRanges([]*ec2.IpRange{
+						{CidrIp: aws.String("0.0.0.0/0")},
+					}),
+			}
+
+			svc.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
+				GroupId:       aws.String(*gpResult.GroupId),
+				IpPermissions: ipPermissions,
+			})
+
+			svc.AuthorizeSecurityGroupEgress(&ec2.AuthorizeSecurityGroupEgressInput{
+				GroupId:       aws.String(*gpResult.GroupId),
+				IpPermissions: ipPermissions,
+			})
+		}
+	}
+
+	sgDesc, err := svc.DescribeSecurityGroupsWithContext(ctx, &ec2.DescribeSecurityGroupsInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("group-name"),
+				Values: []*string{aws.String(securityGroup)},
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	sgID = *sgDesc.SecurityGroups[0].GroupId
+	vpcID = *sgDesc.SecurityGroups[0].VpcId
+
+	subDesc, _ := svc.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("vpc-id"),
+				Values: []*string{aws.String(vpcID)},
+			},
+		},
+	})
+
+	//launch instance
+	runResult, err := svc.RunInstancesWithContext(ctx, &ec2.RunInstancesInput{
+		UserData:         aws.String(userData),
+		ImageId:          aws.String(instanceAmi),
+		KeyName:          aws.String(pairName),
+		InstanceType:     aws.String(instanceType),
+		MinCount:         aws.Int64(1),
+		MaxCount:         aws.Int64(1),
+		SecurityGroupIds: []*string{aws.String(sgID)},
+		SubnetId:         aws.String(*subDesc.Subnets[0].SubnetId),
+		BlockDeviceMappings: []*ec2.BlockDeviceMapping{
+			{
+				//VirtualName: aws.String("Root"),
+				DeviceName: aws.String("/dev/sda1"),
+				Ebs: &ec2.EbsBlockDevice{
+					DeleteOnTermination: aws.Bool(true),
+					Encrypted:           aws.Bool(false),
+					//Iops:                aws.Int64(0),
+					VolumeSize: aws.Int64(int64(hddSize)),
+					VolumeType: aws.String("gp2"),
+				},
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	instanceID := *runResult.Instances[0].InstanceId
+
+	// Add name to the created instance
+	_, err = svc.CreateTags(&ec2.CreateTagsInput{
+		Resources: []*string{aws.String(instanceID)},
+		Tags: []*ec2.Tag{
+			{
+				Key:   aws.String("Name"),
+				Value: aws.String(instanceName),
+			},
+			{
+				Key:   aws.String("Id"),
+				Value: aws.String(d.Id()),
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	statusInput := ec2.DescribeInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("instance-state-name"),
+				Values: []*string{aws.String("running")},
+			},
+		},
+	}
+
+	svc.WaitUntilInstanceExistsWithContext(ctx, &statusInput)
+
+	descResult, err := svc.DescribeInstancesWithContext(ctx, &statusInput)
+	if err != nil {
+		return err
+	}
+
+	instanceDesc := descResult.Reservations[0].Instances[0]
+	d.Set("instance_ip", instanceDesc.PublicIpAddress)
+	d.Set("instance_launch_time", instanceDesc.LaunchTime.Format(time.RFC3339))
+
+	return nil
+}
+
+//ResourceMachineDelete deletes AWS instance
+func ResourceMachineDelete(ctx context.Context, d *schema.ResourceData, m interface{}) error {
+	id := aws.String(d.Id())
+	region := getRegion(d.Get("region").(string))
+
+	svc, err := awsClient(region)
+	if err != nil {
+		return err
+	}
+
+	descResult, err := svc.DescribeInstancesWithContext(ctx, &ec2.DescribeInstancesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("tag:Id"),
+				Values: []*string{id},
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	svc.DeleteKeyPair(&ec2.DeleteKeyPairInput{
+		KeyName: id,
+	})
+
+	instanceID := *descResult.Reservations[0].Instances[0].InstanceId
+	_, err = svc.TerminateInstances(&ec2.TerminateInstancesInput{
+		InstanceIds: []*string{
+			aws.String(instanceID),
+		},
+		DryRun: aws.Bool(false),
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func awsClient(region string) (*ec2.EC2, error) {
+	sess, err := session.NewSession(&aws.Config{
+		Region: aws.String(region),
+		//Credentials: credentials.NewStaticCredentials(conf.AWS_ACCESS_KEY_ID, conf.AWS_SECRET_ACCESS_KEY, ""),
+	})
+
+	svc := ec2.New(sess)
+	return svc, err
+}
+
+func getRegion(region string) string {
+	instanceRegions := make(map[string]string)
+	instanceRegions["us-east"] = "us-east-1"
+	instanceRegions["us-west"] = "us-west-1"
+	instanceRegions["eu-north"] = "eu-north-1"
+	instanceRegions["eu-west"] = "eu-west-1"
+	if val, ok := instanceRegions[region]; ok {
+		return val
+	}
+
+	return region
+}
+
+func getInstanceType(instanceType string, instanceGPU string) string {
+	instanceTypes := make(map[string]string)
+	instanceTypes["m"] = "m5.2xlarge"
+	instanceTypes["l"] = "m5.8xlarge"
+	instanceTypes["xl"] = "m5.16xlarge"
+	instanceTypes["mk80"] = "p2.xlarge"
+	instanceTypes["lk80"] = "p2.8xlarge"
+	instanceTypes["xlk80"] = "p2.16xlarge"
+	instanceTypes["mtesla"] = "p3.xlarge"
+	instanceTypes["ltesla"] = "p3.8xlarge"
+	instanceTypes["xltesla"] = "p3.16xlarge"
+
+	if val, ok := instanceTypes[instanceType+instanceGPU]; ok {
+		return val
+	}
+
+	return instanceType
+}