Skip to content

Commit 6eee45c

Browse files
EricMountainEricMountain
authored
Merge pull request #21 from DataDog/eric.mountain/kube-proxy-ipvs-for-big-kernel
use 'nf_conntrack' instead of 'nf_conntrack_ipv4' for linux kernel >= 4.19
2 parents efd1bc5 + 24262f5 commit 6eee45c

File tree

9 files changed

+785
-62
lines changed

9 files changed

+785
-62
lines changed

‎pkg/proxy/ipvs/proxier.go‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -151,14 +151,6 @@ var ipsetWithIptablesChain = []struct {
151151
{kubeNodePortLocalSetTCP, string(KubeNodePortChain), "RETURN", "dst"},
152152
}
153153

154-
var ipvsModules = []string{
155-
"ip_vs",
156-
"ip_vs_rr",
157-
"ip_vs_wrr",
158-
"ip_vs_sh",
159-
"nf_conntrack_ipv4",
160-
}
161-
162154
// In IPVS proxy mode, the following flags need to be set
163155
const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet"
164156
const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables"
@@ -449,14 +441,12 @@ func NewLinuxKernelHandler() *LinuxKernelHandler {
449441
// GetModules returns all installed kernel modules.
450442
func (handle *LinuxKernelHandler) GetModules() ([]string, error) {
451443
// Check whether IPVS required kernel modules are built-in
452-
kernelVersionFile := "/proc/sys/kernel/osrelease"
453-
b, err := ioutil.ReadFile(kernelVersionFile)
444+
kernelVersion, ipvsModules, err := utilipvs.GetKernelVersionAndIPVSMods(handle.executor)
454445
if err != nil {
455-
glog.Errorf("Failed to read file %s with error %v", kernelVersionFile, err)
446+
return nil, err
456447
}
457-
kernelVersion := strings.TrimSpace(string(b))
458448
builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion)
459-
b, err = ioutil.ReadFile(builtinModsFilePath)
449+
b, err := ioutil.ReadFile(builtinModsFilePath)
460450
if err != nil {
461451
glog.Errorf("Failed to read file %s with error %v", builtinModsFilePath, err)
462452
}
@@ -497,6 +487,8 @@ func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, err
497487
}
498488
wantModules := sets.NewString()
499489
loadModules := sets.NewString()
490+
linuxKernelHandler := NewLinuxKernelHandler()
491+
_, ipvsModules, _ := utilipvs.GetKernelVersionAndIPVSMods(linuxKernelHandler.executor)
500492
wantModules.Insert(ipvsModules...)
501493
loadModules.Insert(mods...)
502494
modules := wantModules.Difference(loadModules).UnsortedList()

‎pkg/util/ipvs/BUILD‎

Lines changed: 4 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -80,42 +80,14 @@ go_library(
8080
"//conditions:default": [],
8181
}),
8282
importpath = "k8s.io/kubernetes/pkg/util/ipvs",
83-
deps = select({
84-
"@io_bazel_rules_go//go/platform:android": [
85-
"//vendor/k8s.io/utils/exec:go_default_library",
86-
],
87-
"@io_bazel_rules_go//go/platform:darwin": [
88-
"//vendor/k8s.io/utils/exec:go_default_library",
89-
],
90-
"@io_bazel_rules_go//go/platform:dragonfly": [
91-
"//vendor/k8s.io/utils/exec:go_default_library",
92-
],
93-
"@io_bazel_rules_go//go/platform:freebsd": [
94-
"//vendor/k8s.io/utils/exec:go_default_library",
95-
],
83+
deps = [
84+
"//vendor/k8s.io/apimachinery/pkg/util/version:go_default_library",
85+
"//vendor/k8s.io/utils/exec:go_default_library",
86+
] + select({
9687
"@io_bazel_rules_go//go/platform:linux": [
9788
"//vendor/github.com/docker/libnetwork/ipvs:go_default_library",
9889
"//vendor/github.com/golang/glog:go_default_library",
9990
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
100-
"//vendor/k8s.io/utils/exec:go_default_library",
101-
],
102-
"@io_bazel_rules_go//go/platform:nacl": [
103-
"//vendor/k8s.io/utils/exec:go_default_library",
104-
],
105-
"@io_bazel_rules_go//go/platform:netbsd": [
106-
"//vendor/k8s.io/utils/exec:go_default_library",
107-
],
108-
"@io_bazel_rules_go//go/platform:openbsd": [
109-
"//vendor/k8s.io/utils/exec:go_default_library",
110-
],
111-
"@io_bazel_rules_go//go/platform:plan9": [
112-
"//vendor/k8s.io/utils/exec:go_default_library",
113-
],
114-
"@io_bazel_rules_go//go/platform:solaris": [
115-
"//vendor/k8s.io/utils/exec:go_default_library",
116-
],
117-
"@io_bazel_rules_go//go/platform:windows": [
118-
"//vendor/k8s.io/utils/exec:go_default_library",
11991
],
12092
"//conditions:default": [],
12193
}),

‎pkg/util/ipvs/ipvs.go‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,12 @@ package ipvs
1919
import (
2020
"net"
2121
"strconv"
22+
"strings"
2223
"time"
24+
25+
"fmt"
26+
"k8s.io/apimachinery/pkg/util/version"
27+
"k8s.io/utils/exec"
2328
)
2429

2530
// Interface is an injectable interface for running ipvs commands. Implementations must be goroutine-safe.
@@ -68,14 +73,21 @@ const (
6873
IPVSProxyMode = "ipvs"
6974
)
7075

71-
// Sets of IPVS required kernel modules.
72-
var ipvsModules = []string{
73-
"ip_vs",
74-
"ip_vs_rr",
75-
"ip_vs_wrr",
76-
"ip_vs_sh",
77-
"nf_conntrack_ipv4",
78-
}
76+
// IPVS required kernel modules.
77+
const (
78+
// ModIPVS is the kernel module "ip_vs"
79+
ModIPVS string = "ip_vs"
80+
// ModIPVSRR is the kernel module "ip_vs_rr"
81+
ModIPVSRR string = "ip_vs_rr"
82+
// ModIPVSWRR is the kernel module "ip_vs_wrr"
83+
ModIPVSWRR string = "ip_vs_wrr"
84+
// ModIPVSSH is the kernel module "ip_vs_sh"
85+
ModIPVSSH string = "ip_vs_sh"
86+
// ModNfConntrackIPV4 is the module "nf_conntrack_ipv4"
87+
ModNfConntrackIPV4 string = "nf_conntrack_ipv4"
88+
// ModNfConntrack is the kernel module "nf_conntrack"
89+
ModNfConntrack string = "nf_conntrack"
90+
)
7991

8092
// Equal check the equality of virtual server.
8193
// We don't use struct == since it doesn't work because of slice.
@@ -110,3 +122,29 @@ func (rs *RealServer) Equal(other *RealServer) bool {
110122
rs.Port == other.Port &&
111123
rs.Weight == other.Weight
112124
}
125+
126+
// GetKernelVersionAndIPVSMods returns the linux kernel version and the required ipvs modules
127+
func GetKernelVersionAndIPVSMods(Executor exec.Interface) (kernelVersion string, ipvsModules []string, err error) {
128+
kernelVersionFile := "/proc/sys/kernel/osrelease"
129+
out, err := Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput()
130+
if err != nil {
131+
return "", nil, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out)
132+
}
133+
kernelVersion = strings.TrimSpace(string(out))
134+
// parse kernel version
135+
ver1, err := version.ParseGeneric(kernelVersion)
136+
if err != nil {
137+
return kernelVersion, nil, fmt.Errorf("error parsing kernel version: %v(%s)", err, kernelVersion)
138+
}
139+
// "nf_conntrack_ipv4" has been removed since v4.19
140+
// see https://github.com/torvalds/linux/commit/a0ae2562c6c4b2721d9fddba63b7286c13517d9f
141+
ver2, _ := version.ParseGeneric("4.19")
142+
// get required ipvs modules
143+
if ver1.LessThan(ver2) {
144+
ipvsModules = append(ipvsModules, ModIPVS, ModIPVSRR, ModIPVSWRR, ModIPVSSH, ModNfConntrackIPV4)
145+
} else {
146+
ipvsModules = append(ipvsModules, ModIPVS, ModIPVSRR, ModIPVSWRR, ModIPVSSH, ModNfConntrack)
147+
}
148+
149+
return kernelVersion, ipvsModules, nil
150+
}

‎pkg/util/ipvs/kernelcheck_linux.go‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ func (r RequiredIPVSKernelModulesAvailableCheck) Name() string {
4444
func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) {
4545
glog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not")
4646

47+
kernelVersion, ipvsModules, err := GetKernelVersionAndIPVSMods(r.Executor)
48+
if err != nil {
49+
errors = append(errors, err)
50+
}
51+
4752
// Find out loaded kernel modules
4853
out, err := r.Executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
4954
if err != nil {
@@ -60,14 +65,6 @@ func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []err
6065

6166
// Check builtin modules exist or not
6267
if len(modules) != 0 {
63-
kernelVersionFile := "/proc/sys/kernel/osrelease"
64-
b, err := r.Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput()
65-
if err != nil {
66-
errors = append(errors, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out))
67-
return nil, errors
68-
}
69-
70-
kernelVersion := strings.TrimSpace(string(b))
7168
builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion)
7269
out, err := r.Executor.Command("cut", "-f1", "-d", " ", builtinModsFilePath).CombinedOutput()
7370
if err != nil {

‎pkg/util/ipvs/kernelcheck_linux_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,8 @@ func TestRequiredIPVSKernelModulesAvailableCheck(t *testing.T) {
9797
for i, tc := range cases {
9898
fcmd := fakeexec.FakeCmd{
9999
CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
100-
func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil },
101100
func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil },
101+
func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil },
102102
func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil },
103103
},
104104
}

‎staging/src/k8s.io/apimachinery/pkg/util/version/BUILD‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package(default_visibility = ["//visibility:public"])
2+
3+
load(
4+
"@io_bazel_rules_go//go:def.bzl",
5+
"go_library",
6+
"go_test",
7+
)
8+
9+
go_library(
10+
name = "go_default_library",
11+
srcs = [
12+
"doc.go",
13+
"version.go",
14+
],
15+
importmap = "k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/version",
16+
importpath = "k8s.io/apimachinery/pkg/util/version",
17+
)
18+
19+
go_test(
20+
name = "go_default_test",
21+
srcs = ["version_test.go"],
22+
embed = [":go_default_library"],
23+
)
24+
25+
filegroup(
26+
name = "package-srcs",
27+
srcs = glob(["**"]),
28+
tags = ["automanaged"],
29+
visibility = ["//visibility:private"],
30+
)
31+
32+
filegroup(
33+
name = "all-srcs",
34+
srcs = [":package-srcs"],
35+
tags = ["automanaged"],
36+
)

‎staging/src/k8s.io/apimachinery/pkg/util/version/doc.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
Copyright 2016 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package version provides utilities for version number comparisons
18+
package version // import "k8s.io/apimachinery/pkg/util/version"

0 commit comments

Comments
 (0)