ci: download the latest github runner on a VM start (#7987)

Author: nikitka

ydb/ci/gh-runner-controller/src/scaler/cloud_config.py

@@ -3,7 +3,7 @@
 import yaml
 
 
-def create_userdata(repo_url, gh_token, runner_name, runner_labels, ssh_keys):
+def create_userdata(repo_url, gh_token, runner_name, runner_labels, ssh_keys, agent_mirror_url_prefix):
     runner_username = "runner"
 
     install_script = pkgutil.get_data(__name__, "scripts/install_runner.sh")
@@ -16,6 +16,7 @@ def create_userdata(repo_url, gh_token, runner_name, runner_labels, ssh_keys):
 GITHUB_TOKEN="{gh_token}"
 RUNNER_NAME="{runner_name}"
 RUNNER_LABELS="{runner_labels}"
+AGENT_MIRROR_URL_PREFIX="{agent_mirror_url_prefix}"
 """.strip().encode(
         "utf8"
     )

ydb/ci/gh-runner-controller/src/scaler/controller.py

@@ -189,7 +189,8 @@ def start_runner(self, preset_name: str):
             labels.extend(preset['additional_labels'])
 
         vm_labels = {self.prefix: runner_name}
-        user_data = create_userdata(self.gh.html_url, new_runner_token, runner_name, labels, self.cfg.ssh_keys)
+        user_data = create_userdata(self.gh.html_url, new_runner_token, runner_name, labels, self.cfg.ssh_keys,
+                                    self.cfg.agent_mirror_url_prefix)
 
         placement = random.choice(self.cfg.yc_zones)
         zone_id = placement['zone_id']
@@ -198,15 +199,15 @@ def start_runner(self, preset_name: str):
         self.logger.info("start runner %s in %s (%s)", runner_name, zone_id, labels)
 
         try:
-            instance = self.yc.start_vm(zone_id, subnet_id, runner_name, preset_name, user_data, vm_labels)
+            response = self.yc.start_vm(zone_id, subnet_id, runner_name, preset_name, user_data, vm_labels)
         except grpc.RpcError as rpc_error:
             # noinspection PyUnresolvedReferences
             if rpc_error.code() == grpc.StatusCode.RESOURCE_EXHAUSTED:
                 raise VMQuotaError()
             else:
                 raise
 
-        self.logger.info("instance %s started", instance.id)
+        self.logger.info("starting, operation_id=%s", response.id)
         return runner_name
 
     def delete_runner(self, runner):

ydb/ci/gh-runner-controller/src/scaler/scripts/install_runner.sh

@@ -11,7 +11,7 @@ function fail() {
 }
 
 function get_instance_id {
-  curl -H Metadata-Flavor:Google 169.254.169.254/computeMetadata/v1/instance/vendor/identity/document | jq -r '.instanceId'
+  curl -sS --retry 8 -H Metadata-Flavor:Google 169.254.169.254/computeMetadata/v1/instance/vendor/identity/document | jq -r '.instanceId'
 }
 
 vars_to_check=("REPO_URL" "GITHUB_TOKEN" "RUNNER_NAME" "RUNNER_LABELS" "RUNNER_USERNAME")
@@ -24,15 +24,23 @@ done
 
 
 H=/home/$RUNNER_USERNAME
+mkdir "$H"/actions_runner
+cd "$_"
+
+{
+  agent_latest_version=$(curl -sS --retry 8 "$AGENT_MIRROR_URL_PREFIX/latest") && \
+  agent_download_url="$AGENT_MIRROR_URL_PREFIX/$agent_latest_version" && \
+  curl -sS --retry 8 "$agent_download_url" | tar -xz
+} || {
+  # use bundled agent
+  cp -rT /opt/cache/actions-runner/latest/ "$H"/actions_runner
+}
 
-cp -r /opt/cache/actions-runner/latest "$H"/actions_runner
-chown "$RUNNER_USERNAME":"$RUNNER_USERNAME" "$H"/actions_runner
-
-cd "$H"/actions_runner
+chown -R "$RUNNER_USERNAME":"$RUNNER_USERNAME" "$H"/actions_runner
 
 instance_id=$(get_instance_id)
 
-sudo -u "$RUNNER_USERNAME" ./config.sh --unattended --url "${REPO_URL}" --token "${GITHUB_TOKEN}" --name "${RUNNER_NAME}" --labels "${RUNNER_LABELS},instance:${instance_id}"
+sudo -u "$RUNNER_USERNAME" ./config.sh --unattended --disableupdate --url "${REPO_URL}" --token "${GITHUB_TOKEN}" --name "${RUNNER_NAME}" --labels "${RUNNER_LABELS},instance:${instance_id}"
 
 ./svc.sh install "${RUNNER_USERNAME}" || fail "failed to install service"