Create artifact registry driver interface. (#44)

- Create artifact registry driver interface, such that a new driver
needs to implement methods like `generate`, `publish`, and `download`
- Add artifact_registries & artifact_registry configuration to
deploy.yaml
- Add GCS driver

This is a breaking change to deploy.yaml, so we upgrade KDT and
deploy.yaml to v3.

This change includes some cleanups we have for v3 in general including:
- Remove unused deploy.yaml from KDT
- Remove old KDT v1 aliases

Author: julianvmodesto

bin/deploy

@@ -3,12 +3,13 @@
 
 require 'tmpdir'
 
-require 'kube_deploy_tools/deploy_artifact'
 require 'kube_deploy_tools/formatted_logger'
 require 'kube_deploy_tools/kubectl'
 require 'kube_deploy_tools/shellrunner'
 require 'kube_deploy_tools/deploy'
 require 'kube_deploy_tools/deploy/options'
+require 'kube_deploy_tools/deploy_config_file'
+require 'kube_deploy_tools/artifact_registry'
 
 def options
   @options ||= KubeDeployTools::Deploy::Optparser.new.parse(ARGV)
@@ -21,25 +22,28 @@ namespace = KubeDeployTools::Deploy::kube_namespace(context: context, kubeconfig
 KubeDeployTools::Logger.logger = KubeDeployTools::FormattedLogger.build(context: context, namespace: namespace)
 kubectl = KubeDeployTools::Kubectl.new(
   context: options.context,
-  kubeconfig: options.kubeconfig,
+  kubeconfig: options.kubeconfig
 )
 
-Dir.mktmpdir { |tmpdir|
+config = KubeDeployTools::DeployConfigFile.new('deploy.yaml')
+artifact_registry = config.artifact_registries[config.artifact_registry].driver
+
+Dir.mktmpdir do |tmpdir|
   if options.from_files
-    deploy_artifact_path = KubeDeployTools::DeployArtifact.new(
-      input_path: options.from_files,
-    ).path
+    deploy_artifact_path = options.from_files
   else
-    deploy_artifact_path = KubeDeployTools::DeployArtifact.new(
-      input_path: KubeDeployTools.get_remote_deploy_artifact_url(
-        project: options.project,
-        build_number: options.build_number,
-        flavor: options.flavor,
-        name: options.artifact,
-      ),
-      output_dir_path: tmpdir,
-      pre_apply_hook: options.pre_apply_hook
-    ).path
+    if options.build_number == 'latest'
+      build_number = artifact_registry.get_latest_build_number(options.project)
+    end
+
+    deploy_artifact_path = artifact_registry.download(
+      project: options.project,
+      build_number: build_number,
+      flavor: options.flavor,
+      name: options.artifact,
+      pre_apply_hook: options.pre_apply_hook,
+      output_dir: tmpdir
+    )
   end
 
   deploy = KubeDeployTools::Deploy.new(
@@ -54,4 +58,4 @@ Dir.mktmpdir { |tmpdir|
     dry_run: options.dry_run,
     send_report: options.send_report
   )
-}
+end

bin/publish

@@ -18,7 +18,11 @@ end
 KubeDeployTools::Logger.logger = KubeDeployTools::FormattedLogger.build
 KubeDeployTools::Shellrunner.shellrunner = KubeDeployTools::Shellrunner.new
 
+config = KubeDeployTools::DeployConfigFile.new(options.manifest_file)
+artifact_registry = config.artifact_registries[config.artifact_registry]
+
 KubeDeployTools::Publish.new(
   manifest: options.manifest_file,
+  artifact_registry: artifact_registry,
   output_dir: options.output_path,
 ).publish

deploy.yaml

@@ -1,5 +1,4 @@
 
-require 'kube_deploy_tools/deploy_artifact'
 require 'kube_deploy_tools/formatted_logger'
 require 'kube_deploy_tools/kubectl'
 require 'kube_deploy_tools/shellrunner'

lib/kube_deploy_tools.rb

@@ -0,0 +1,31 @@
+require 'kube_deploy_tools/artifact_registry/driver'
+require 'kube_deploy_tools/formatted_logger'
+
+module KubeDeployTools
+  # Read-only model for artifact_registries[] array element in KDT deploy.yaml
+  # configuration file.
+  class ArtifactRegistry
+    attr_accessor :name, :driver_name, :config, :driver
+
+    def initialize(h)
+      @name = h['name']
+      @driver_name = h['driver']
+      @config = h['config']
+
+      if !ArtifactRegistry::Driver::MAPPINGS.key?(@driver_name)
+        Logger.warn("Unsupported .artifact_registries.driver: #{@driver_name}")
+      else
+        @driver = ArtifactRegistry::Driver::MAPPINGS
+          .fetch(@driver_name)
+          .new(config: @config)
+      end
+    end
+
+    def ==(o)
+      @name == o.name
+      @driver == o.driver
+      @prefix == o.prefix
+      @config == o.config
+    end
+  end
+end

lib/kube_deploy_tools/artifact_registry.rb

@@ -0,0 +1,13 @@
+require_relative 'driver_artifactory'
+require_relative 'driver_gcs'
+
+module KubeDeployTools
+  class ArtifactRegistry
+    module Driver
+      MAPPINGS = {
+        'artifactory' => Artifactory,
+        'gcs' => GCS,
+      }
+    end
+  end
+end

lib/kube_deploy_tools/artifact_registry/driver.rb

@@ -0,0 +1,155 @@
+require_relative 'driver_base'
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/shellrunner'
+require 'kube_deploy_tools/object'
+
+require 'artifactory'
+require 'fileutils'
+require 'uri'
+
+
+EXT_TAR_GZ = ".tar.gz"
+
+module KubeDeployTools
+  class ArtifactRegistry::Driver::Artifactory < ArtifactRegistry::Driver::Base
+    # Artifactory configuration is configurable by environment variables
+    # by default:
+    # export ARTIFACTORY_ENDPOINT=http://my.storage.server/artifactory
+    # export ARTIFACTORY_USERNAME=admin
+    # export ARTIFACTORY_PASSWORD=password
+    # See https://github.com/chef/artifactory-client#create-a-connection.
+
+    def initialize(config:)
+      @config = config
+
+      Artifactory.endpoint = @config.fetch('endpoint', '')
+      @repo = @config.fetch('repo', '')
+    end
+
+
+    def get_local_artifact_path(name:, flavor:, local_dir:)
+      artifact_name = get_artifact_name(name: name, flavor: flavor)
+
+      local_artifact_path = File.join(local_dir, artifact_name)
+
+      local_artifact_path
+    end
+
+    def get_registry_artifact_path(name:, flavor:, project:, build_number:)
+      # NOTE(joshk): If the naming format changes, it represents a breaking
+      # change where all past clients will not be able to download new builds and
+      # new clients will not be able to download old builds. Change with caution.
+      "#{project}/#{build_number}/#{get_artifact_name(name: name, flavor: flavor)}"
+    end
+
+    def upload(local_dir:, name:, flavor:, project:, build_number:)
+      # Pack up contents of each flavor_dir to a correctly named artifact.
+      flavor_dir = File.join(local_dir, "#{name}_#{flavor}")
+
+      package(
+        name: name,
+        flavor: flavor,
+        input_dir: flavor_dir,
+        output_dir: local_dir,
+      )
+
+      local_artifact_path = get_local_artifact_path(
+        local_dir: local_dir,
+        name: name,
+        flavor: flavor,
+      )
+
+      registry_artifact_path = get_registry_artifact_path(
+        project: project,
+        name: name,
+        flavor: flavor,
+        build_number: build_number,
+      )
+      artifactory_url = "#{Artifactory.endpoint}/#{@repo}/#{registry_artifact_path}"
+      Logger.info("Uploading #{local_artifact_path} to #{artifactory_url}")
+      artifact = Artifactory::Resource::Artifact.new(local_path: local_artifact_path)
+      artifact.upload(@repo, registry_artifact_path)
+    end
+
+    def get_artifact_name(name:, flavor:)
+      "manifests_#{name}_#{flavor}#{EXT_TAR_GZ}"
+    end
+
+    def package(name:, flavor:, input_dir:, output_dir:)
+      local_artifact_path = get_local_artifact_path(name: name, flavor: flavor, local_dir: output_dir)
+
+      Shellrunner.check_call('tar', '-C', input_dir, '-czf', local_artifact_path, '.')
+
+      local_artifact_path
+    end
+
+    def download(project:, build_number:, flavor:, name:, pre_apply_hook:, output_dir:)
+      registry_artifact_path = get_registry_artifact_path(
+        name: name, flavor: flavor, project: project, build_number: build_number)
+
+      registry_artifact_full_path = [
+        Artifactory.endpoint,
+        @repo,
+        registry_artifact_path,
+      ].join('/')
+
+      local_artifact_path = download_artifact(registry_artifact_full_path, output_dir)
+      local_artifact_path = uncompress_artifact(local_artifact_path, output_dir)
+
+      if pre_apply_hook
+        out, err, status = Shellrunner.run_call(pre_apply_hook, local_artifact_path)
+        if !status.success?
+          raise "Failed to run post download hook #{pre_apply_hook}"
+        end
+      end
+
+      local_artifact_path
+    end
+
+    def get_latest_build_number(project)
+      project_url = [
+        Artifactory.endpoint,
+        @repo,
+        "#{project}/"
+      ].join('/')
+      project_builds_html = Shellrunner.run_call('curl', project_url).first
+      # store build entries string from html into an array
+      build_links_pattern = /(?<=">).+(?=\s{4})/
+      build_entries = project_builds_html.scan(build_links_pattern) # example of element: 10/</a>    13-Nov-2017 13:51
+      build_number_pattern = /^\d+/
+      build_number = build_entries.
+        map { |x| x.match(build_number_pattern).to_s.to_i }.
+        max.
+        to_s
+      if build_number.empty?
+        raise "Failed to find a valid build number. Project URL: #{project_url}"
+      end
+      build_number
+    end
+
+    def download_artifact(input_path, output_dir_path)
+      uri = URI.parse(input_path)
+      filename = File.basename(uri.path)
+      output_path = File.join(output_dir_path, filename)
+      out, err, status = Shellrunner.run_call('curl', '-o', output_path, input_path, '--silent', '--fail')
+      if !status.success?
+        raise "Failed to download remote deploy artifact #{uri}"
+      end
+
+      output_path
+    end
+
+    def uncompress_artifact(input_path, output_dir_path)
+      dirname = File.basename(input_path).chomp(EXT_TAR_GZ)
+      output_path = File.join(output_dir_path, dirname)
+      FileUtils.mkdir_p(output_path)
+      out, err, status = Shellrunner.run_call('tar', '-xzf', input_path, '-C', output_path)
+      if !status.success?
+        raise "Failed to uncompress deploy artifact #{input_path}"
+      end
+
+      output_path
+    end
+  end
+end

lib/kube_deploy_tools/artifact_registry/driver_artifactory.rb

@@ -0,0 +1,37 @@
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/shellrunner'
+
+# Abstract Driver class that specific implementations inherit
+module KubeDeployTools
+  class ArtifactRegistry
+    module Driver
+      class Base
+        def initialize(config:)
+          @config = config
+        end
+
+        # upload should publish the artifact identified by the given name and flavor
+        # in the input directory to the corresponding location in the artifact
+        # registry. The project and build number should be included in the
+        # namespace of the artifact registry path for this artifact.
+        def upload(local_dir:, name:, flavor:, project:, build_number:)
+          raise "#{self.class}#publish not implemented"
+        end
+
+        # download should retrieve the artifact namespaced with the given
+        # project and build number and identified by the name and flavor.
+        # The artifact should be put into the output directory.
+        # An optional pre-apply hook will process each artifact at the end.
+        def download(project:, build_number:, flavor:, name:, pre_apply_hook:, output_dir:)
+          raise "#{self.class}#download not implemented"
+        end
+
+        # get_latest_build_number should find the artifact from the most recent
+        # build
+        def get_latest_build_number(project)
+          raise "#{self.class}#get_latest_build_number not implemented"
+        end
+      end
+    end
+  end
+end