Skip to content

Commit a674984

Browse files
brycekahlegeyslan
brycekahle
and
geyslan
committed
Add job to update amazon 2 in cron workflow
Co-authored-by: Geyslan Gregório <geyslan@gmail.com>
1 parent 253a198 commit a674984

File tree

2 files changed

+187
-10
lines changed

2 files changed

+187
-10
lines changed

‎.github/workflows/cron.yml‎

Lines changed: 125 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,116 @@ on:
44
- cron: "0 1 * * *"
55
workflow_dispatch: {}
66
jobs:
7+
amazon-update:
8+
name: Update Amazon 2 BTF Archive
9+
runs-on: ubuntu-latest
10+
container:
11+
image: amazonlinux:2023
12+
steps:
13+
- name: Install needed amazon packages
14+
run: |
15+
yum install -y yum-utils tar gzip xz golang clang make cmake git libdwarf-devel elfutils-libelf-devel elfutils-devel rsync
16+
shell: bash
17+
#
18+
- name: Setup Amazon Debuginfo Repositories
19+
run: |
20+
# disable default debuginfo repositories
21+
yum-config-manager -y --disable amazonlinux-debuginfo
22+
23+
# add Amazon Linux 2 debuginfo repositories
24+
append_repo() {
25+
local arch=$1
26+
local repo_name="amzn2-core-debuginfo-$arch"
27+
28+
echo "[$repo_name]
29+
name=Amazon Linux 2 core repository - debuginfo packages $arch
30+
mirrorlist=http://amazonlinux.default.amazonaws.com/2/core/latest/debuginfo/$arch/mirror.list
31+
enabled=1
32+
"
33+
}
34+
35+
cat << EOF >> /etc/yum.repos.d/amazonlinux.repo
36+
$(append_repo "x86_64")
37+
$(append_repo "aarch64")
38+
EOF
39+
shell: bash
40+
#
41+
- name: Checkout BTFHub
42+
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
43+
with:
44+
repository: aquasecurity/btfhub
45+
submodules: recursive
46+
path: ./btfhub
47+
#
48+
- name: Build and install pahole
49+
run: |
50+
cd btfhub/3rdparty/dwarves
51+
mkdir build
52+
cd build
53+
cmake -D__LIB=lib -DDWARF_INCLUDE_DIR=/usr/include ..
54+
make install
55+
echo "/usr/local/lib" >> /etc/ld.so.conf.d/pahole.conf
56+
ldconfig
57+
shell: bash
58+
#
59+
- name: Build and install bpftool
60+
run: |
61+
cd btfhub/3rdparty/bpftool
62+
make -C src clean
63+
CC=clang make -C src all
64+
cp ./src/bpftool /usr/sbin/bpftool
65+
make -C src clean
66+
shell: bash
67+
#
68+
- name: Compile BTFHub Tool
69+
run: |
70+
cd btfhub
71+
make
72+
shell: bash
73+
#
74+
- name: Checkout BTFHub Archive
75+
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
76+
with:
77+
repository: aquasecurity/btfhub-archive
78+
token: ${{ secrets.GEYSLAN_BTFHUB_PAT }}
79+
persist-credentials: false
80+
fetch-depth: 1
81+
path: ./btfhub-archive
82+
sparse-checkout: |
83+
amzn
84+
#
85+
- name: Bring current BTFHub Archive
86+
run: |
87+
cd btfhub
88+
make bring
89+
shell: bash
90+
#
91+
- name: Fetch and Generate new BTFs (AMAZON 2)
92+
run: |
93+
cd btfhub
94+
./btfhub -workers 6 -d amzn -r 2
95+
#
96+
- name: Take new BTFs to BTFHub Archive
97+
run: |
98+
cd btfhub
99+
make take
100+
#
101+
- name: Check Status
102+
run: |
103+
cd btfhub-archive
104+
git status
105+
#
106+
- name: Commit and Push to BTFHub Archive
107+
uses: actions-js/push@5a7cbd780d82c0c937b5977586e641b2fd94acc5 # v1.5
108+
with:
109+
directory: ./btfhub-archive
110+
author_email: 'geyslan@gmail.com'
111+
author_name: 'Geyslan Gregório'
112+
github_token: ${{ secrets.GEYSLAN_BTFHUB_PAT }}
113+
message: 'Update BTFHUB Archive from BTFHUB'
114+
repository: aquasecurity/btfhub-archive
115+
branch: main
116+
7117
build:
8118
name: Update BTF Archive
9119
env:
@@ -24,7 +134,7 @@ jobs:
24134
swapon /swapfile
25135
shell: bash
26136
#
27-
- name: Check out BTFHub
137+
- name: Checkout BTFHub
28138
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
29139
with:
30140
repository: aquasecurity/btfhub
@@ -44,6 +154,13 @@ jobs:
44154
persist-credentials: false
45155
fetch-depth: 1
46156
path: ./btfhub-archive
157+
sparse-checkout: |
158+
centos
159+
debian
160+
fedora
161+
ol
162+
ubuntu
163+
#
47164
- name: Bring current BTFHub Archive
48165
run: |
49166
cd btfhub
@@ -56,22 +173,17 @@ jobs:
56173
make
57174
shell: bash
58175
#
59-
- name: Fetch and Generate new BTFs (UBUNTU)
176+
- name: Fetch and Generate new BTFs (CENTOS)
60177
run: |
61178
cd btfhub
62-
./btfhub -workers 6 -d ubuntu
179+
./btfhub -workers 6 -d centos
63180
# debian stretch seems to be gone, updates for buster and bullseye only
64181
- name: Fetch and Generate new BTFs (DEBIAN)
65182
run: |
66183
cd btfhub
67184
./btfhub -workers 6 -d debian -r buster
68185
./btfhub -workers 6 -d debian -r bullseye
69186
#
70-
- name: Fetch and Generate new BTFs (CENTOS)
71-
run: |
72-
cd btfhub
73-
./btfhub -workers 6 -d centos
74-
#
75187
- name: Fetch and Generate new BTFs (FEDORA)
76188
run: |
77189
cd btfhub
@@ -82,6 +194,11 @@ jobs:
82194
cd btfhub
83195
./btfhub -workers 6 -d ol
84196
#
197+
- name: Fetch and Generate new BTFs (UBUNTU)
198+
run: |
199+
cd btfhub
200+
./btfhub -workers 6 -d ubuntu
201+
#
85202
- name: Take new BTFs to BTFHub Archive
86203
run: |
87204
cd btfhub

‎pkg/repo/amazon.go‎

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,18 @@
11
package repo
22

33
import (
4+
"bufio"
5+
"bytes"
46
"context"
57
"errors"
68
"fmt"
9+
"io"
710
"log"
11+
"os/exec"
812
"sort"
13+
"strings"
14+
15+
"golang.org/x/exp/maps"
916

1017
"github.com/aquasecurity/btfhub/pkg/job"
1118
"github.com/aquasecurity/btfhub/pkg/kernel"
@@ -34,11 +41,12 @@ func (d *AmazonRepo) GetKernelPackages(
3441
force bool,
3542
jobChan chan<- job.Job,
3643
) error {
37-
searchOut, err := yumSearch(ctx, "kernel-debuginfo")
44+
altArch := d.archs[arch]
45+
searchOut, err := repoquery(ctx, "kernel-debuginfo", altArch)
3846
if err != nil {
3947
return err
4048
}
41-
pkgs, err := parseYumPackages(searchOut, kernel.NewKernelVersion(""))
49+
pkgs, err := parseRepoqueryPackages(searchOut, kernel.NewKernelVersion(""))
4250
if err != nil {
4351
return fmt.Errorf("parse package listing: %s", err)
4452
}
@@ -57,3 +65,55 @@ func (d *AmazonRepo) GetKernelPackages(
5765

5866
return nil
5967
}
68+
69+
func repoquery(ctx context.Context, pkg string, arch string) (*bytes.Buffer, error) {
70+
stdout := &bytes.Buffer{}
71+
stderr := &bytes.Buffer{}
72+
binary, args := utils.SudoCMD("repoquery", "--archlist="+arch, "--show-duplicates", pkg)
73+
cmd := exec.CommandContext(ctx, binary, args...)
74+
cmd.Stdout = stdout
75+
cmd.Stderr = stderr
76+
if err := cmd.Run(); err != nil {
77+
return nil, fmt.Errorf("repoquery search %s: %s\n%s", pkg, err, stderr.String())
78+
}
79+
return stdout, nil
80+
}
81+
82+
func parseRepoqueryPackages(rdr io.Reader, minVersion kernel.Version) ([]pkg.Package, error) {
83+
pkgs := map[string]pkg.Package{}
84+
bio := bufio.NewScanner(rdr)
85+
for bio.Scan() {
86+
line := bio.Text()
87+
if !strings.HasPrefix(line, "kernel-debuginfo-") {
88+
continue
89+
}
90+
if strings.HasPrefix(line, "kernel-debuginfo-common-") {
91+
continue
92+
}
93+
_, version, found := strings.Cut(line, ":")
94+
if !found {
95+
continue
96+
}
97+
filename := version
98+
lastdot := strings.LastIndex(filename, ".")
99+
if lastdot == -1 {
100+
continue
101+
}
102+
p := &pkg.RHELPackage{
103+
Name: fmt.Sprintf("kernel-debuginfo-%s", version),
104+
NameOfFile: filename,
105+
KernelVersion: kernel.NewKernelVersion(filename[:lastdot]),
106+
Architecture: filename[lastdot+1:],
107+
}
108+
if !minVersion.IsZero() && p.Version().Less(minVersion) {
109+
continue
110+
}
111+
if _, ok := pkgs[p.Name]; !ok {
112+
pkgs[p.Name] = p
113+
}
114+
}
115+
if err := bio.Err(); err != nil {
116+
return nil, err
117+
}
118+
return maps.Values(pkgs), nil
119+
}

0 commit comments

Comments
 (0)