Skip to content

Commit d524ce8

Browse files
Add SONiC flavor to support migration (#114)
* Add make target to download SONiC QEMU image * Add python script to provision the virtual machine via telnet * Add additional topology file for SONiC
1 parent bcbb8ee commit d524ce8

File tree

7 files changed

+267
-3
lines changed

7 files changed

+267
-3
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ ansible-common
88
metal-hammer*
99
requirements.yaml
1010
.extra_vars.yaml
11+
sonic-vs.img

Makefile

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,18 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:
1818

1919
MACHINE_OS=ubuntu-20.04
2020

21+
SONIC_REMOTE_IMG := https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=master&platform=vs&buildId=125016&target=target%2Fsonic-vs.img.gz
22+
2123
# Machine flavors
2224
ifeq ($(MINI_LAB_FLAVOR),default)
2325
LAB_MACHINES=machine01,machine02
26+
LAB_TOPOLOGY=mini-lab.cumulus.yaml
2427
else ifeq ($(MINI_LAB_FLAVOR),cluster-api)
2528
LAB_MACHINES=machine01,machine02,machine03
29+
LAB_TOPOLOGY=mini-lab.cumulus.yaml
30+
else ifeq ($(MINI_LAB_FLAVOR),sonic)
31+
LAB_MACHINES=machine01,machine02
32+
LAB_TOPOLOGY=mini-lab.sonic.yaml
2633
else
2734
$(error Unknown flavor $(MINI_LAB_FLAVOR))
2835
endif
@@ -70,8 +77,8 @@ partition: partition-bake
7077
.PHONY: partition-bake
7178
partition-bake:
7279
# docker pull $(MINI_LAB_VM_IMAGE)
73-
@if ! sudo containerlab --topo mini-lab.clab.yaml inspect | grep -i running > /dev/null; then \
74-
sudo --preserve-env containerlab deploy --topo mini-lab.clab.yaml --reconfigure && \
80+
@if ! sudo containerlab --topo $(LAB_TOPOLOGY) inspect | grep -i running > /dev/null; then \
81+
sudo --preserve-env containerlab deploy --topo $(LAB_TOPOLOGY) --reconfigure && \
7582
./scripts/deactivate_offloading.sh; fi
7683

7784
.PHONY: env
@@ -110,7 +117,7 @@ cleanup-control-plane:
110117

111118
.PHONY: cleanup-partition
112119
cleanup-partition:
113-
sudo containerlab destroy --topo mini-lab.clab.yaml
120+
sudo containerlab destroy --topo $(LAB_TOPOLOGY)
114121

115122
.PHONY: _privatenet
116123
_privatenet: env
@@ -222,3 +229,6 @@ dev-env:
222229
.PHONY: build-vms-image
223230
build-vms-image:
224231
cd images && docker build -f Dockerfile.vms -t $(MINI_LAB_VM_IMAGE) . && cd -
232+
233+
sonic-vs.img:
234+
curl --location --output - "${SONIC_REMOTE_IMG}" | gunzip > sonic-vs.img

README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@ There's few versions of mini-lab environment that you can run. We call them flav
208208

209209
- `default` -- runs 2 machines.
210210
- `cluster-api` -- runs 3 machines. Usefull for testing Control plane and worker node deployment with [Cluster API provider](https://github.com/metal-stack/cluster-api-provider-metalstack).
211+
- `sonic` -- use SONiC as network operating system for the leaves
211212

212213
In order to start specific flavor, you can define the flavor as follows:
213214

File renamed without changes.

mini-lab.sonic.yaml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: mini-lab
2+
prefix: ""
3+
4+
mgmt:
5+
network: bridge
6+
7+
topology:
8+
kinds:
9+
linux:
10+
image: ${MINI_LAB_VM_IMAGE}
11+
binds:
12+
- /dev:/dev
13+
- scripts:/mini-lab
14+
15+
nodes:
16+
leaf01:
17+
kind: linux
18+
binds:
19+
- files/ssh/id_rsa.pub:/id_rsa.pub
20+
- sonic-vs.img:/sonic-vs.img
21+
cmd: /mini-lab/sonic_entrypoint.py
22+
leaf02:
23+
kind: linux
24+
binds:
25+
- files/ssh/id_rsa.pub:/id_rsa.pub
26+
- sonic-vs.img:/sonic-vs.img
27+
cmd: /mini-lab/sonic_entrypoint.py
28+
vms:
29+
kind: linux
30+
31+
links:
32+
- endpoints: ["leaf01:eth1", "vms:lan0"]
33+
- endpoints: ["leaf02:eth1", "vms:lan1"]
34+
- endpoints: ["leaf01:eth2", "vms:lan2"]
35+
- endpoints: ["leaf02:eth2", "vms:lan3"]
36+
- endpoints: ["leaf01:eth3", "vms:lan4"]
37+
- endpoints: ["leaf02:eth3", "vms:lan5"]

scripts/mirror_tap_to_eth.sh

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
# Script is taken from https://netdevops.me/2021/transparently-redirecting-packets/frames-between-interfaces/
4+
# Read it for better understanding
5+
6+
set -o errexit
7+
8+
TAP_IF=$1
9+
# get interface index number up to 3 digits (everything after first three chars)
10+
# tap0 -> 0
11+
# tap123 -> 123
12+
INDEX=${TAP_IF:3:3}
13+
14+
ip link set $TAP_IF up
15+
16+
# create tc eth<->tap redirect rules
17+
tc qdisc add dev eth$INDEX ingress
18+
tc filter add dev eth$INDEX parent ffff: protocol all matchall action mirred egress redirect dev $TAP_IF
19+
20+
tc qdisc add dev $TAP_IF ingress
21+
tc filter add dev $TAP_IF parent ffff: protocol all matchall action mirred egress redirect dev eth$INDEX

scripts/sonic_entrypoint.py

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
#!/usr/bin/python3
2+
import fcntl
3+
import logging
4+
import os
5+
import signal
6+
import socket
7+
import struct
8+
import subprocess
9+
import sys
10+
import telnetlib
11+
import time
12+
13+
BASE_IMG = '/sonic-vs.img'
14+
USER = 'admin'
15+
PASSWORD = 'YourPaSsWoRd'
16+
17+
18+
class Qemu:
19+
def __init__(self, name: str, memory: str, interfaces: int):
20+
self._name = name
21+
self._memory = memory
22+
self._interfaces = interfaces
23+
self._p = None
24+
self._disk = '/overlay.img'
25+
26+
def prepare_overlay(self, base: str) -> None:
27+
cmd = [
28+
'qemu-img',
29+
'create',
30+
'-f', 'qcow2',
31+
'-b', base,
32+
self._disk,
33+
]
34+
subprocess.run(cmd, check=True)
35+
36+
def start(self) -> None:
37+
cmd = [
38+
'qemu-system-x86_64',
39+
'-cpu', 'host',
40+
'-display', 'none',
41+
'-enable-kvm',
42+
'-machine', 'q35',
43+
'-name', self._name,
44+
'-m', self._memory,
45+
'-drive', f'if=virtio,format=qcow2,file={self._disk}',
46+
'-serial', 'telnet:127.0.0.1:5000,server,nowait',
47+
]
48+
49+
for i in range(self._interfaces):
50+
with open(f'/sys/class/net/eth{i}/address', 'r') as f:
51+
mac = f.read().strip()
52+
cmd.append('-device')
53+
cmd.append(f'virtio-net,netdev=hn{i},mac={mac}')
54+
cmd.append(f'-netdev')
55+
cmd.append(f'tap,id=hn{i},ifname=tap{i},script=/mini-lab/mirror_tap_to_eth.sh,downscript=no')
56+
57+
self._p = subprocess.Popen(cmd)
58+
59+
def wait(self) -> None:
60+
self._p.wait()
61+
62+
63+
class Telnet:
64+
def __init__(self):
65+
self._tn: telnetlib.Telnet | None = None
66+
67+
def connect(self, host: str, port: int, max_retries=60) -> bool:
68+
for i in range(1, max_retries + 1):
69+
try:
70+
self._tn = telnetlib.Telnet(host, port)
71+
return True
72+
except:
73+
time.sleep(1)
74+
if i == max_retries:
75+
return False
76+
77+
def close(self):
78+
self._tn.close()
79+
80+
def wait_until(self, match: str):
81+
self._tn.read_until(match.encode('ascii'))
82+
83+
def write_and_wait(self, data: str, match: str = '$ ') -> str:
84+
self._tn.write(data.encode('ascii') + b'\n')
85+
return self._tn.read_until(match.encode('ascii')).decode('utf-8')
86+
87+
def write_test(self, data: str) -> str:
88+
self._tn.write(data.encode('ascii') + b'\n')
89+
time.sleep(5)
90+
return self._tn.read_some().decode('utf-8')
91+
92+
93+
def main():
94+
signal.signal(signal.SIGINT, handle_exit)
95+
signal.signal(signal.SIGTERM, handle_exit)
96+
97+
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
98+
logger = logging.getLogger()
99+
100+
name = os.getenv('CLAB_LABEL_CLAB_NODE_NAME', default='switch')
101+
memory = os.getenv('VM_MEMORY', default='2048')
102+
interfaces = int(os.getenv('CLAB_INTFS', 0)) + 1
103+
104+
logger.info(f'Waiting for {interfaces} interfaces to be connected')
105+
wait_until_all_interfaces_are_connected(interfaces)
106+
107+
vm = Qemu(name, memory, interfaces)
108+
109+
logger.info('Prepare disk')
110+
vm.prepare_overlay(BASE_IMG)
111+
112+
logger.info('Start QEMU')
113+
vm.start()
114+
115+
logger.info('Try to connect via telnet...')
116+
tn = Telnet()
117+
if not tn.connect('127.0.0.1', 5000):
118+
logger.error('Cannot connect to telnet server')
119+
sys.exit(1)
120+
121+
logger.info('Connected via telnet and waiting for login prompt')
122+
tn.wait_until('login: ')
123+
124+
logger.info('Try to login')
125+
tn.write_and_wait(USER, 'Password: ')
126+
tn.write_and_wait(PASSWORD)
127+
128+
logger.info('Authorize ssh key')
129+
authorize_ssh_key(tn)
130+
131+
logger.info('Wait until config-setup is done')
132+
if not wait_until_config_setup_is_done(tn):
133+
logger.error('config-setup still not done')
134+
sys.exit(1)
135+
136+
net = get_ip_address('eth0') + '/16'
137+
logger.info(f'Configure {net} on eth0')
138+
tn.write_and_wait(f'sudo config interface ip add eth0 {net}')
139+
tn.write_and_wait('sudo config save --yes')
140+
141+
tn.close()
142+
143+
logger.info('Wait until QEMU terminated')
144+
vm.wait()
145+
146+
147+
def handle_exit(signal, frame):
148+
sys.exit(0)
149+
150+
151+
def wait_until_all_interfaces_are_connected(interfaces: int) -> None:
152+
while True:
153+
i = 0
154+
for iface in os.listdir('/sys/class/net/'):
155+
if iface.startswith('eth'):
156+
i += 1
157+
if i == interfaces:
158+
break
159+
time.sleep(1)
160+
161+
162+
def get_ip_address(iface: str) -> str:
163+
# Source: https://bit.ly/3dROGBN
164+
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
165+
return socket.inet_ntoa(fcntl.ioctl(
166+
s.fileno(),
167+
0x8915, # SIOCGIFADDR
168+
struct.pack('256s', iface.encode('utf_8'))
169+
)[20:24])
170+
171+
172+
def wait_until_config_setup_is_done(tn: Telnet, max_retries: int = 60) -> bool:
173+
for i in range(1, max_retries + 1):
174+
# updategraph is started after the config-setup
175+
result = tn.write_and_wait('systemctl is-active updategraph')
176+
if not 'inactive' in result:
177+
return True
178+
time.sleep(1)
179+
if i == max_retries:
180+
return False
181+
182+
183+
def authorize_ssh_key(tn: Telnet) -> None:
184+
with open('/id_rsa.pub') as f:
185+
key = f.read().strip()
186+
187+
tn.write_and_wait(f'echo "{key}" > authorized_keys')
188+
tn.write_and_wait('sudo mkdir /root/.ssh')
189+
tn.write_and_wait('sudo chmod 0600 /root/.ssh')
190+
tn.write_and_wait('sudo cp authorized_keys /root/.ssh/')
191+
192+
193+
if __name__ == '__main__':
194+
main()

0 commit comments

Comments
 (0)