ci: add unified-agent into github runner VM (#8474)

Author: nikitka

ydb/ci/gh-runner-controller/src/scaler/scripts/install_runner.sh

@@ -49,3 +49,6 @@ SVC_NAME=$(cat .service)
 systemctl daemon-reload || fail "failed to reload systemd"
 systemctl enable $SVC_NAME
 systemctl start $SVC_NAME
+
+echo "GH_RUNNER_NAME=${RUNNER_NAME}"> /etc/default/unified_agent
+systemctl restart unified-agent.service

ydb/ci/gh-runner-controller/src/scaler/yc.py

@@ -107,9 +107,11 @@ def start_vm(self, zone_id: str, subnet_id: str, instance_name: str, preset_name
 
         request = CreateInstanceRequest(
             name=instance_name,
+            hostname=f'{instance_name}.auto.internal',
             folder_id=self.cfg.yc_folder_id,
             zone_id=zone_id,
             platform_id=preset['platform_id'],
+            service_account_id=self.cfg.service_account_id,
             resources_spec=ResourcesSpec(
                 cores=int(preset["cpu_cores"]),
                 memory=int(preset["memory_gb"] * 1024 * 1024 * 1024),

ydb/ci/gh-runner-image/conf/unified-agent-linux.yml

@@ -0,0 +1,47 @@
+storages:
+  - name: main
+    plugin: fs
+    config:
+      directory: /var/lib/yandex/unified_agent/main
+      max_partition_size: 100mb
+      max_segment_size: 10mb
+
+channels:
+  - name: cloud_monitoring
+    channel:
+      pipe:
+        - storage_ref:
+            name: main
+        - filter:
+            plugin: transform_metric_labels
+            config:
+              labels:
+                - gh_runner_name: "{$env('GH_RUNNER_NAME')}"
+      output:
+        plugin: yc_metrics
+        config:
+          folder_id: "${FOLDER_ID}"
+          iam:
+            cloud_meta: { }
+
+routes:
+  - input:
+      plugin: linux_metrics
+      config:
+        namespace: sys
+    channel:
+      channel_ref:
+        name: cloud_monitoring
+  
+  - input:
+      plugin: agent_metrics
+      config:
+        namespace: ua
+    channel:
+      pipe:
+        - filter:
+            plugin: filter_metrics
+            config:
+              match: "{scope=health}"
+      channel_ref:
+        name: cloud_monitoring

ydb/ci/gh-runner-image/image.pkr.hcl

@@ -20,6 +20,10 @@ source "yandex" "this" {
   state_timeout = "10m"
 
   use_ipv4_nat = true
+  metadata = {
+    user-data : "#cloud-config\npackage_update: false\npackage_upgrade: false"
+    serial-port-enable : "1"
+  }
 
 }
 
@@ -37,7 +41,7 @@ apt-get -o DPkg::Lock::Timeout=600 -y --no-install-recommends dist-upgrade
 apt-get -y install --no-install-recommends \
   antlr3 clang-12 clang-14 cmake docker.io git jq libaio-dev libaio1 libicu70 libidn11-dev libkrb5-3 \
   liblttng-ust1 lld-14 llvm-14 m4 make ninja-build parallel postgresql-client postgresql-client \
-  python-is-python3 python3-pip s3cmd s3cmd zlib1g
+  python-is-python3 python3-pip s3cmd s3cmd zlib1g linux-tools-common linux-tools-generic
 
 apt-get -y purge lxd-agent-loader snapd modemmanager
 apt-get -y autoremove
@@ -74,12 +78,34 @@ EOF
     destination = "/tmp/install-agent.sh"
   }
 
+  provisioner "file" {
+    content = templatefile("conf/unified-agent-linux.yml", {
+      FOLDER_ID = var.folder_id
+    })
+    destination = "/tmp/yc-vm.yml"
+  }
+
+  provisioner "file" {
+    content     = <<EOF
+#!/bin/env/sh
+set -xe
+curl -s -O "https://storage.yandexcloud.net/yc-unified-agent/releases/${var.unified_agent_version}/deb/${var.unified_agent_ubuntu_name}/yandex-unified-agent_${var.unified_agent_version}_amd64.deb"
+dpkg -i yandex-unified-agent_${var.unified_agent_version}_amd64.deb
+
+mv /tmp/yc-vm.yml /etc/yandex/unified_agent/conf.d/
+chown unified_agent:unified_agent /etc/yandex/unified_agent/conf.d/yc-vm.yml
+EOF
+    destination = "/tmp/install-unified-agent.sh"
+  }
+
   provisioner "shell" {
     inline = [
       "sudo bash /tmp/install-packages.sh",
       "sudo bash /tmp/install-agent.sh",
-      "sudo rm /tmp/install-packages.sh /tmp/install-agent.sh",
+      "sudo bash /tmp/install-unified-agent.sh",
+      "sudo rm /tmp/install-packages.sh /tmp/install-agent.sh /tmp/install-unified-agent.sh",
       "sudo time sync",
     ]
   }
+
 }

ydb/ci/gh-runner-image/ydbtech.pkr.hcl

@@ -1,19 +1,29 @@
 variable "folder_id" {
-  type = string
+  type    = string
   default = "b1grf3mpoatgflnlavjd"
 }
 
 variable "zone_id" {
-  type = string
+  type    = string
   default = "ru-central1-d"
 }
 
 variable "subnet_id" {
-  type = string
+  type    = string
   default = "fl8gu1e1h8kqnlfj4uqp"
 }
 
 variable "github_runner_version" {
-  type = string
+  type    = string
   default = "2.319.1"
 }
+
+variable "unified_agent_ubuntu_name" {
+  type    = string
+  default = "ubuntu-22.04-jammy"
+}
+
+variable "unified_agent_version" {
+  type    = string
+  default = "24.07.02"
+}

ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/vm-sa.tf

@@ -0,0 +1,12 @@
+resource "yandex_iam_service_account" "vm-sa" {
+  name = "gh-runner-vm-sa"
+}
+
+
+resource "yandex_resourcemanager_folder_iam_binding" "vm-sa-monitoring-binding" {
+  folder_id = var.folder_id
+  members = [
+    "serviceAccount:${yandex_iam_service_account.vm-sa.id}"
+  ]
+  role = "monitoring.editor"
+}