|
| 1 | +package e2e |
| 2 | + |
| 3 | +import ( |
| 4 | + "fmt" |
| 5 | + "strconv" |
| 6 | + "strings" |
| 7 | + "sync" |
| 8 | + |
| 9 | + corev1 "k8s.io/api/core/v1" |
| 10 | + "k8s.io/apimachinery/pkg/api/resource" |
| 11 | + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 12 | + api "sigs.k8s.io/hierarchical-namespaces/api/v1alpha2" |
| 13 | + "sigs.k8s.io/yaml" |
| 14 | + |
| 15 | + "github.com/google/uuid" |
| 16 | + . "github.com/onsi/ginkgo/v2" |
| 17 | + . "github.com/onsi/gomega" |
| 18 | + . "sigs.k8s.io/hierarchical-namespaces/pkg/testutils" |
| 19 | +) |
| 20 | + |
| 21 | +var _ = Describe("Scoped Hierarchical Resource Quota", Label("pfnet"), func() { |
| 22 | + var ( |
| 23 | + parentNs, childNs, priorityName string |
| 24 | + cleanupNs, cleanupPriority func() |
| 25 | + scopeSelector corev1.ScopeSelector |
| 26 | + normalHRQ api.HierarchicalResourceQuota |
| 27 | + ) |
| 28 | + |
| 29 | + BeforeEach(func() { |
| 30 | + parentNs, childNs, cleanupNs = setUpParentAndChild() |
| 31 | + scopeSelector, priorityName, cleanupPriority = genPriorityScopeSelector() |
| 32 | + |
| 33 | + // For checking not-scoped HRQ with scoped HRQ. |
| 34 | + normalHRQ = setScopedHRQ("normal-hrq", parentNs, corev1.ResourceList{corev1.ResourcePods: resource.MustParse("100")}, nil) |
| 35 | + }) |
| 36 | + |
| 37 | + AfterEach(func() { |
| 38 | + var podCount int |
| 39 | + for _, ns := range []string{parentNs, childNs} { |
| 40 | + out, err := RunCommand("kubectl get --no-headers pods -n " + ns) |
| 41 | + Expect(err).NotTo(HaveOccurred()) |
| 42 | + if strings.Contains(out, "No resources found") { |
| 43 | + continue |
| 44 | + } |
| 45 | + |
| 46 | + pods := strings.Count(out, "\n") |
| 47 | + Expect(err).NotTo(HaveOccurred()) |
| 48 | + podCount += pods |
| 49 | + } |
| 50 | + |
| 51 | + FieldShouldContainWithTimeout("hrq", parentNs, normalHRQ.Name, ".status.used", "pods:"+strconv.Itoa(podCount), 300) |
| 52 | + |
| 53 | + // For debugging after failed tests |
| 54 | + if !CurrentSpecReport().Failed() { |
| 55 | + cleanupPriority() |
| 56 | + cleanupNs() |
| 57 | + } |
| 58 | + }) |
| 59 | + |
| 60 | + It("should create RQs with correct limits in the descendants (including itself) for Scoped HRQs", func() { |
| 61 | + hrqA := setScopedHRQ("a-hrq", parentNs, corev1.ResourceList{corev1.ResourcePods: resource.MustParse("3")}, &scopeSelector) |
| 62 | + hrqB := setScopedHRQ("b-hrq", childNs, corev1.ResourceList{corev1.ResourcePods: resource.MustParse("2")}, &scopeSelector) |
| 63 | + |
| 64 | + rqAName := api.ResourceQuotaSingletonName + "-" + hrqA.Name |
| 65 | + rqBName := api.ResourceQuotaSingletonName + "-" + hrqB.Name |
| 66 | + |
| 67 | + FieldShouldContain("resourcequota", parentNs, rqAName, ".spec.hard", "pods:3") |
| 68 | + FieldShouldContain("resourcequota", childNs, rqBName, ".spec.hard", "pods:2") |
| 69 | + |
| 70 | + expect := selectorStr(priorityName) |
| 71 | + FieldShouldContain("resourcequota", parentNs, rqAName, ".spec.scopeSelector", expect) |
| 72 | + FieldShouldContain("resourcequota", childNs, rqBName, ".spec.scopeSelector", expect) |
| 73 | + }) |
| 74 | + |
| 75 | + It("should remove obsolete (empty) RQ if there's no longer a Scoped HRQ in the ancestor", func() { |
| 76 | + hrq := setScopedHRQ("a-hrq", parentNs, corev1.ResourceList{corev1.ResourcePods: resource.MustParse("3")}, &scopeSelector) |
| 77 | + rqName := api.ResourceQuotaSingletonName + "-" + hrq.Name |
| 78 | + |
| 79 | + MustRun("kubectl delete hrq -n", parentNs, hrq.Name) |
| 80 | + |
| 81 | + RunShouldNotContain(rqName, propagationTime, "kubectl get resourcequota -n" , parentNs) |
| 82 | + RunShouldNotContain(rqName, propagationTime, "kubectl get resourcequota -n" , childNs) |
| 83 | + }) |
| 84 | + |
| 85 | + It("should update the .status.used field of the HRQ when pods are created", func() { |
| 86 | + hrq := setScopedHRQ("status-updated", parentNs, corev1.ResourceList{corev1.ResourcePods: resource.MustParse("2")}, &scopeSelector) |
| 87 | + |
| 88 | + for i := 0; i < 2; i++ { |
| 89 | + _, err := mustCreatePodWithPrioirty(fmt.Sprint(i), childNs, priorityName) |
| 90 | + Expect(err).NotTo(HaveOccurred()) |
| 91 | + FieldShouldContain("hrq", parentNs, hrq.Name, ".status.used", "pods:"+fmt.Sprint(i+1)) |
| 92 | + } |
| 93 | + |
| 94 | + _, err := mustCreatePod("normal", childNs) |
| 95 | + Expect(err).NotTo(HaveOccurred()) |
| 96 | + }) |
| 97 | + |
| 98 | + It("should reject creating a pod if the HRQ is exceeded", func() { |
| 99 | + hrq := setScopedHRQ("ok-ng", parentNs, corev1.ResourceList{corev1.ResourcePods: resource.MustParse("1")}, &scopeSelector) |
| 100 | + |
| 101 | + _, err := mustCreatePodWithPrioirty("ok", childNs, priorityName) |
| 102 | + Expect(err).NotTo(HaveOccurred()) |
| 103 | + FieldShouldContain("hrq", parentNs, hrq.Name, ".status.used", "pods:1") |
| 104 | + |
| 105 | + _, err = mustCreatePodWithPrioirty("ng", childNs, priorityName) |
| 106 | + Expect(err).ShouldNot(Succeed()) |
| 107 | + |
| 108 | + _, err = mustCreatePod("normal-pod", childNs) |
| 109 | + Expect(err).NotTo(HaveOccurred()) |
| 110 | + |
| 111 | + FieldShouldContain("hrq", parentNs, hrq.Name, ".status.used", "pods:1") |
| 112 | + }) |
| 113 | +}) |
| 114 | + |
| 115 | +func mustCreatePod(prefix, nsnm string) (corev1.Pod, error) { |
| 116 | + return mustCreatePodWithPrioirty(prefix, nsnm, "") |
| 117 | +} |
| 118 | + |
| 119 | +func mustCreatePodWithPrioirty(prefix, nsnm, priority string) (corev1.Pod, error) { |
| 120 | + name := prefix + "-" + uuid.New().String() |
| 121 | + spec := corev1.PodSpec{ |
| 122 | + PriorityClassName: priority, |
| 123 | + Containers: []corev1.Container{ |
| 124 | + { |
| 125 | + Name: "test", |
| 126 | + Image: "nginx", |
| 127 | + }, |
| 128 | + }, |
| 129 | + } |
| 130 | + if priority != "" { |
| 131 | + spec.PriorityClassName = priority |
| 132 | + } |
| 133 | + |
| 134 | + pod := corev1.Pod { |
| 135 | + TypeMeta: metav1.TypeMeta{ |
| 136 | + Kind: "Pod", |
| 137 | + APIVersion: "v1", |
| 138 | + }, |
| 139 | + ObjectMeta: metav1.ObjectMeta{ |
| 140 | + Name: name, |
| 141 | + Namespace: nsnm, |
| 142 | + }, |
| 143 | + Spec: spec, |
| 144 | + } |
| 145 | + |
| 146 | + manifest, err := yaml.Marshal(pod) |
| 147 | + Expect(err).NotTo(HaveOccurred()) |
| 148 | + |
| 149 | + fn := writeTempFile(string(manifest)) |
| 150 | + GinkgoT().Log("Wrote " + fn + ":\n" + string(manifest)) |
| 151 | + defer removeFile(fn) |
| 152 | + |
| 153 | + return pod, TryRun("kubectl apply -f " + fn) |
| 154 | +} |
| 155 | + |
| 156 | +func setScopedHRQ(nm, nsnm string, resourceList corev1.ResourceList, scopeSelector *corev1.ScopeSelector) api.HierarchicalResourceQuota { |
| 157 | + hrq := api.HierarchicalResourceQuota{ |
| 158 | + TypeMeta: metav1.TypeMeta{ |
| 159 | + Kind: "HierarchicalResourceQuota", |
| 160 | + APIVersion: "hnc.x-k8s.io/v1alpha2", |
| 161 | + }, |
| 162 | + ObjectMeta: metav1.ObjectMeta{ |
| 163 | + Name: nm, |
| 164 | + Namespace: nsnm, |
| 165 | + }, |
| 166 | + Spec: api.HierarchicalResourceQuotaSpec{ |
| 167 | + Hard: resourceList, |
| 168 | + ScopeSelector: scopeSelector, |
| 169 | + }, |
| 170 | + } |
| 171 | + manifest, err := yaml.Marshal(hrq) |
| 172 | + Expect(err).NotTo(HaveOccurred()) |
| 173 | + |
| 174 | + MustApplyYAML(string(manifest)) |
| 175 | + RunShouldContain(nm, propagationTime, "kubectl get hrq -n", nsnm, nm) |
| 176 | + |
| 177 | + return hrq |
| 178 | +} |
| 179 | + |
| 180 | +func setUpParentAndChild() (string, string, func()) { |
| 181 | + parentNs := createNS("hrq-test-parent-") |
| 182 | + childNs := createSubNS(parentNs) |
| 183 | + cleanup := func() { |
| 184 | + MustRunWithTimeout(cleanupTimeout, "kubectl annotate ns", parentNs, "hnc.x-k8s.io/subnamespace-of-") |
| 185 | + MustRunWithTimeout(cleanupTimeout, "kubectl annotate ns", childNs, "hnc.x-k8s.io/subnamespace-of-") |
| 186 | + var wg sync.WaitGroup |
| 187 | + for _, ns := range []string{parentNs, childNs} { |
| 188 | + wg.Add(1) |
| 189 | + go func(ns string) { |
| 190 | + MustRun("kubectl delete ns", ns) |
| 191 | + wg.Done() |
| 192 | + }(ns) |
| 193 | + } |
| 194 | + wg.Wait() |
| 195 | + } |
| 196 | + |
| 197 | + RunShouldContain(childNs, propagationTime, "kubectl get ns") |
| 198 | + return parentNs, childNs, cleanup |
| 199 | +} |
| 200 | + |
| 201 | +func createNS(prefix string) string { |
| 202 | + nsName := prefix + uuid.New().String() |
| 203 | + MustRun("kubectl create ns", nsName) |
| 204 | + return nsName |
| 205 | +} |
| 206 | + |
| 207 | +func createSubNS(parent string) string { |
| 208 | + nsName := "hrq-test-child-" + uuid.New().String() |
| 209 | + MustRun("kubectl hns create", nsName, "-n", parent) |
| 210 | + return nsName |
| 211 | +} |
| 212 | + |
| 213 | +func genPriorityScopeSelector() (corev1.ScopeSelector, string, func()) { |
| 214 | + priority := uuid.New().String() |
| 215 | + err := TryRun("kubectl create priorityclass", priority, "--value=100") |
| 216 | + Expect(err).NotTo(HaveOccurred()) |
| 217 | + cleanup := func() { |
| 218 | + err := TryRun("kubectl delete priorityclass", priority) |
| 219 | + Expect(err).NotTo(HaveOccurred()) |
| 220 | + } |
| 221 | + |
| 222 | + return corev1.ScopeSelector{ |
| 223 | + MatchExpressions: []corev1.ScopedResourceSelectorRequirement{ |
| 224 | + { |
| 225 | + Operator: corev1.ScopeSelectorOpIn, |
| 226 | + ScopeName: "PriorityClass", |
| 227 | + Values: []string{priority}, |
| 228 | + }, |
| 229 | + }, |
| 230 | + }, priority, cleanup |
| 231 | +} |
| 232 | + |
| 233 | +func selectorStr(priorityName string) string { |
| 234 | + return "map[matchExpressions:[map[operator:In scopeName:PriorityClass values:[" + priorityName + "]]]]" |
| 235 | +} |
0 commit comments