Fix: recreate VXLAN device (flannel.*) when external interface is deleted and re-added

main.go

@@ -301,6 +301,7 @@ func main() {
 		PublicIPv6: opts.publicIPv6,
 	}
 	// Check the default interface only if no interfaces are specified
+
 	if len(opts.iface) == 0 && len(opts.ifaceRegex) == 0 && len(opts.ifaceCanReach) == 0 {
 		if len(opts.publicIP) > 0 {
 			extIface, err = ipmatch.LookupExtIface(opts.publicIP, "", "", ipStack, optsPublicIP)
@@ -431,8 +432,8 @@ func main() {
 				for {
 					select {
 					case <-ctx.Done():
-					     break
-				     	case <-time.After(time.Duration(opts.iptablesResyncSeconds) * time.Second):
+						break
+					case <-time.After(time.Duration(opts.iptablesResyncSeconds) * time.Second):
 						if err := ip.AddBlackholeV4Route(bn.Lease().Subnet.ToIPNet()); err != nil {
 							log.Errorf("Failed to setup blackhole route, %v", err)
 						}
@@ -445,8 +446,8 @@ func main() {
 				for {
 					select {
 					case <-ctx.Done():
-					     break
-				     	case <-time.After(time.Duration(opts.iptablesResyncSeconds) * time.Second):
+						break
+					case <-time.After(time.Duration(opts.iptablesResyncSeconds) * time.Second):
 						if err := ip.AddBlackholeV6Route(bn.Lease().IPv6Subnet.ToIPNet()); err != nil {
 							log.Errorf("Failed to setup blackhole route, %v", err)
 						}

pkg/backend/common.go

@@ -25,6 +25,7 @@ import (
 
 type ExternalInterface struct {
 	Iface       *net.Interface
+	IfaceName   string
 	IfaceAddr   net.IP
 	IfaceV6Addr net.IP
 	ExtAddr     net.IP

pkg/backend/vxlan/vxlan.go

@@ -121,32 +121,76 @@ func newSubnetAttrs(publicIP net.IP, publicIPv6 net.IP, vnid uint32, dev, v6Dev
 
 func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
 	// Parse our configuration
-	cfg := struct {
-		VNI           int
-		Port          int
-		MTU           int
-		GBP           bool
-		Learning      bool
-		DirectRouting bool
-	}{
+	cfg, err := parseVXLANConfig(config.Backend, be.extIface.Iface.MTU)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding VXLAN backend config: %w", err)
+	}
+	log.Infof("VXLAN config: VNI=%d Port=%d GBP=%v Learning=%v DirectRouting=%v", cfg.VNI, cfg.Port, cfg.GBP, cfg.Learning, cfg.DirectRouting)
+
+	dev, v6Dev, err := createVXLANDevice(ctx, config, cfg, be.subnetMgr, be.extIface.Iface.Index, be.extIface.ExtAddr, be.extIface.ExtV6Addr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create vxlan device: %w", err)
+	}
+
+	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, be.extIface.ExtV6Addr, uint32(cfg.VNI), dev, v6Dev)
+	if err != nil {
+		return nil, err
+	}
+
+	lease, err := be.subnetMgr.AcquireLease(ctx, subnetAttrs)
+	switch err {
+	case nil:
+	case context.Canceled, context.DeadlineExceeded:
+		return nil, err
+	default:
+		return nil, fmt.Errorf("failed to acquire lease: %v", err)
+	}
+
+	// Ensure that the device has a /32 address so that no broadcast routes are created.
+	// This IP is just used as a source address for host to workload traffic (so
+	// the return path for the traffic has an address on the flannel network to use as the destination)
+	if err := configureDeviceIPv4IPv6(dev, v6Dev, lease, config); err != nil {
+		return nil, err
+	}
+
+	return newNetwork(be.subnetMgr, be.extIface, dev, v6Dev, ip.IP4Net{}, lease, cfg.MTU)
+}
+
+type VXLANConfig struct {
+	VNI           int  `json:"vni"`
+	Port          int  `json:"port"`
+	MTU           int  `json:"mtu"`
+	GBP           bool `json:"gbp"`
+	Learning      bool `json:"learning"`
+	DirectRouting bool `json:"directRouting"`
+}
+
+func parseVXLANConfig(config json.RawMessage, defaultMTU int) (VXLANConfig, error) {
+	cfg := VXLANConfig{
 		VNI: defaultVNI,
-		MTU: be.extIface.Iface.MTU,
+		MTU: defaultMTU,
 	}
 
-	if len(config.Backend) > 0 {
-		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
-			return nil, fmt.Errorf("error decoding VXLAN backend config: %v", err)
+	if len(config) > 0 {
+		if err := json.Unmarshal(config, &cfg); err != nil {
+			return VXLANConfig{}, err
 		}
 	}
-	log.Infof("VXLAN config: VNI=%d Port=%d GBP=%v Learning=%v DirectRouting=%v", cfg.VNI, cfg.Port, cfg.GBP, cfg.Learning, cfg.DirectRouting)
-
-	var dev, v6Dev *vxlanDevice
-	var err error
+	return cfg, nil
+}
 
+func createVXLANDevice(ctx context.Context,
+	config *subnet.Config,
+	cfg VXLANConfig,
+	subnetMgr subnet.Manager,
+	extIfaceID int,
+	extIfaceIP net.IP,
+	extIfaceV6IP net.IP,
+) (dev, v6Dev *vxlanDevice, err error) {
 	// When flannel is restarted, it will get the MAC address from the node annotations to set flannel.1 MAC address
 	var hwAddr, hwAddrv6 net.HardwareAddr
 
-	macStr, macStrv6 := be.subnetMgr.GetStoredMacAddresses(ctx)
+	macStr, macStrv6 := subnetMgr.GetStoredMacAddresses(ctx)
 	if macStr != "" {
 		hwAddr, err = net.ParseMAC(macStr)
 		if err != nil {
@@ -160,8 +204,8 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
 			vni:       uint32(cfg.VNI),
 			name:      fmt.Sprintf("flannel.%d", cfg.VNI),
 			MTU:       cfg.MTU,
-			vtepIndex: be.extIface.Iface.Index,
-			vtepAddr:  be.extIface.IfaceAddr,
+			vtepIndex: extIfaceID,
+			vtepAddr:  extIfaceIP,
 			vtepPort:  cfg.Port,
 			gbp:       cfg.GBP,
 			learning:  cfg.Learning,
@@ -170,7 +214,7 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
 
 		dev, err = newVXLANDevice(&devAttrs)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		dev.directRouting = cfg.DirectRouting
 	}
@@ -188,54 +232,45 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
 			vni:       uint32(cfg.VNI),
 			name:      fmt.Sprintf("flannel-v6.%d", cfg.VNI),
 			MTU:       cfg.MTU,
-			vtepIndex: be.extIface.Iface.Index,
-			vtepAddr:  be.extIface.IfaceV6Addr,
+			vtepIndex: extIfaceID,
+			vtepAddr:  extIfaceV6IP,
 			vtepPort:  cfg.Port,
 			gbp:       cfg.GBP,
 			learning:  cfg.Learning,
 			hwAddr:    hwAddrv6,
 		}
 		v6Dev, err = newVXLANDevice(&v6DevAttrs)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		v6Dev.directRouting = cfg.DirectRouting
 	}
 
-	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, be.extIface.ExtV6Addr, uint32(cfg.VNI), dev, v6Dev)
-	if err != nil {
-		return nil, err
-	}
-
-	lease, err := be.subnetMgr.AcquireLease(ctx, subnetAttrs)
-	switch err {
-	case nil:
-	case context.Canceled, context.DeadlineExceeded:
-		return nil, err
-	default:
-		return nil, fmt.Errorf("failed to acquire lease: %v", err)
-	}
+	return dev, v6Dev, nil
+}
 
-	// Ensure that the device has a /32 address so that no broadcast routes are created.
-	// This IP is just used as a source address for host to workload traffic (so
-	// the return path for the traffic has an address on the flannel network to use as the destination)
+func configureDeviceIPv4IPv6(dev *vxlanDevice, v6Dev *vxlanDevice, lease *lease.Lease, config *subnet.Config) error {
+	// Configure IPv4 if enabled
 	if config.EnableIPv4 {
 		if lease.Subnet.Empty() {
-			return nil, fmt.Errorf("failed to configure interface %s: IPv4 is enabled but the lease has no IPv4", dev.link.Attrs().Name)
+			return fmt.Errorf("failed to configure interface %s: IPv4 is enabled but the lease has no IPv4", dev.link.Attrs().Name)
 		}
 		if err := dev.Configure(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, config.Network); err != nil {
-			return nil, fmt.Errorf("failed to configure interface %s: %w", dev.link.Attrs().Name, err)
+			return fmt.Errorf("failed to configure interface %s: %w", dev.link.Attrs().Name, err)
 		}
 	}
+
+	// Configure IPv6 if enabled
 	if config.EnableIPv6 {
 		if lease.IPv6Subnet.Empty() {
-			return nil, fmt.Errorf("failed to configure interface %s: IPv6 is enabled but the lease has no IPv6", v6Dev.link.Attrs().Name)
+			return fmt.Errorf("failed to configure interface %s: IPv6 is enabled but the lease has no IPv6", v6Dev.link.Attrs().Name)
 		}
 		if err := v6Dev.ConfigureIPv6(ip.IP6Net{IP: lease.IPv6Subnet.IP, PrefixLen: 128}, config.IPv6Network); err != nil {
-			return nil, fmt.Errorf("failed to configure interface %s: %w", v6Dev.link.Attrs().Name, err)
+			return fmt.Errorf("failed to configure interface %s: %w", v6Dev.link.Attrs().Name, err)
 		}
 	}
-	return newNetwork(be.subnetMgr, be.extIface, dev, v6Dev, ip.IP4Net{}, lease, cfg.MTU)
+
+	return nil
 }
 
 // So we can make it JSON (un)marshalable