Simplify AMI build system (#409)

* Refactor image build system

Missing parts:
* Switch from Bash to Ansible
* Finish and enable Azure and Google Cloud

* Add remaining local changes

* Remove dead code

* Remove dead code

* Restyled by jq

* Merge master into improved-packer

* Document `packer` image build process

* Fix package name

* Update golden tests

Restyled by prettier-markdown

Restyled by whitespace

* Update and rename build-cml-ami.yml to build-ami.yml

Co-authored-by: Restyled.io <commits@restyled.io>

Author: 0x2b3bfa0

.github/workflows/build-ami.yml

@@ -0,0 +1,13 @@
+name: build-ami
+on: 
+  push:
+    tags: ['ami*']
+jobs:
+  build-ami:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: packer build environment
+        env:
+            AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+            AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/build-cml-ami.yml

@@ -0,0 +1,8 @@
+# Amazon Machine Images
+
+## Updating Images
+
+1. Edit `setup.sh` and open a pull request with the changes.
+2. Run `packer build .` in this directory and wait for the operation to complete.
+
+This operation needs to be run from the [`dvc-cml-terraform-provider`](https://github.com/iterative/itops/blob/e2423bf5b253896c68432a7e20d186918ed00703/cml/terraform/cml-terraform-provider.tf#L1-L3) IAM user, so `packer` can assume the [`cml-packer`](https://github.com/iterative/itops/blob/e2423bf5b253896c68432a7e20d186918ed00703/cml/terraform/packer-role.tf) role.

cml/ami-test.json

@@ -0,0 +1,111 @@
+packer {
+  required_plugins {
+    amazon = {
+      version = ">= 1.0.0"
+      source  = "github.com/hashicorp/amazon"
+    }
+  }
+}
+
+variables {
+  image_name        = "iterative-cml"
+  image_description = "CML (Continuous Machine Learning) Ubuntu 18.04"
+}
+
+variables {
+  aws_build_region       = "us-west-1"
+  aws_build_instance     = "g2.2xlarge"
+  aws_build_ubuntu_image = "*ubuntu-*-18.04-amd64-server-*"
+}
+
+variables {
+  aws_role_session_name = "cml-packer-session"
+  aws_role_arn          = "arn:aws:iam::260760892802:role/cml-packer"
+  aws_subnet_id         = "subnet-09fca08419c2f0575"
+  aws_security_group_id = "sg-0b7df7d9f902ca7ec"
+}
+
+locals {
+  aws_tags = {
+    ManagedBy   = "packer"
+    Name        = var.image_name
+    Environment = "prod"
+  }
+
+  aws_release_regions = [
+    "af-south-1",
+    "ap-east-1",
+    "ap-northeast-1",
+    "ap-northeast-2",
+    "ap-northeast-3",
+    "ap-south-1",
+    "ap-southeast-1",
+    "ap-southeast-2",
+    "ca-central-1",
+    "eu-central-1",
+    "eu-north-1",
+    "eu-south-1",
+    "eu-west-1",
+    "eu-west-2",
+    "eu-west-3",
+    "me-south-1",
+    "sa-east-1",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2"
+  ]
+}
+
+data "amazon-ami" "ubuntu" {
+  region      = var.aws_build_region
+  owners      = ["099720109477"]
+  most_recent = true
+
+  filters = {
+    name                = "ubuntu/images/${var.aws_build_ubuntu_image}"
+    root-device-type    = "ebs"
+    virtualization-type = "hvm"
+  }
+
+  assume_role {
+    role_arn     = var.aws_role_arn
+    session_name = var.aws_role_session_name
+  }
+}
+
+source "amazon-ebs" "source" {
+  ami_groups      = ["all"]
+  ami_name        = var.image_name
+  ami_description = var.image_description
+  ami_regions     = local.aws_release_regions
+
+  region        = var.aws_build_region
+  instance_type = var.aws_build_instance
+
+  source_ami    = data.amazon-ami.ubuntu.id
+  ssh_username  = "ubuntu"
+
+  security_group_id = var.aws_security_group_id
+  subnet_id         = var.aws_subnet_id
+
+  force_delete_snapshot = true
+  force_deregister      = true
+
+  tags            = local.aws_tags
+  run_tags        = local.aws_tags
+  run_volume_tags = local.aws_tags
+
+  assume_role {
+    role_arn     = var.aws_role_arn
+    session_name = var.aws_role_session_name
+  }
+}
+
+build {
+  sources = ["source.amazon-ebs.source"]
+
+  provisioner "shell" {
+    script = "${path.root}/setup.sh"
+  }
+}

cml/ami.json

@@ -25,6 +25,9 @@ if [ ! -f "$FILE" ]; then
   sudo apt install -y ubuntu-drivers-common
   sudo ubuntu-drivers autoinstall
 
+  sudo curl https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.5.0/linux-amd64/docker-credential-ecr-login --output /usr/bin/docker-credential-ecr-login
+  sudo chmod 755 /usr/bin/docker-credential-ecr-login
+
   curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
   curl -s -L https://nvidia.github.io/nvidia-docker/ubuntu18.04/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
   sudo apt update && sudo apt install -y nvidia-docker2