Skip to content

Commit 4a88576

Browse files
yutpengl-technicore
yutpeng
authored and
l-technicore
committed
Add OpenShift Tag to identify Openshift usage and support secondary vnic function
1 parent 5adce2f commit 4a88576

File tree

6 files changed

+353
-7
lines changed

6 files changed

+353
-7
lines changed

‎pkg/cloudprovider/providers/oci/instances.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ const (
3333
VirtualNodePoolIdAnnotation = "oci.oraclecloud.com/virtual-node-pool-id"
3434
IPv4NodeIPFamilyLabel = "oci.oraclecloud.com/ip-family-ipv4"
3535
IPv6NodeIPFamilyLabel = "oci.oraclecloud.com/ip-family-ipv6"
36+
OpenShiftTagNamesapcePrefix = "openshift-"
37+
OpenShiftBootVolumeType = "boot-volume-type"
38+
OpenShiftBootVolumeISCSI = "ISCSI"
3639
)
3740

3841
var _ cloudprovider.Instances = &CloudProvider{}
@@ -124,6 +127,59 @@ func (cp *CloudProvider) extractNodeAddresses(ctx context.Context, instanceID st
124127
}
125128
}
126129

130+
OpenShiftTagNamesapce := cp.getOpenShiftTagNamespaceByInstance(ctx, instanceID)
131+
132+
if OpenShiftTagNamesapce != "" {
133+
secondaryVnics, err := cp.client.Compute().GetSecondaryVNICsForInstance(ctx, compartmentID, instanceID)
134+
if err != nil {
135+
return nil, err
136+
}
137+
138+
if secondaryVnics == nil || len(secondaryVnics) == 0 {
139+
return addresses, nil
140+
}
141+
for _, secondaryVnic := range secondaryVnics {
142+
if cp.checkOpenShiftISCSIBootVolumeTagByVnic(ctx, secondaryVnic, OpenShiftTagNamesapce) {
143+
if (secondaryVnic.IsPrimary == nil || !*secondaryVnic.IsPrimary) && secondaryVnic.PrivateIp != nil && *secondaryVnic.PrivateIp != "" {
144+
ip := net.ParseIP(*secondaryVnic.PrivateIp)
145+
if ip == nil {
146+
return nil, fmt.Errorf("instance has invalid private address: %q", *secondaryVnic.PrivateIp)
147+
}
148+
addresses = append(addresses, api.NodeAddress{Type: api.NodeInternalIP, Address: ip.String()})
149+
}
150+
151+
if (secondaryVnic.IsPrimary == nil || !*secondaryVnic.IsPrimary) && secondaryVnic.PublicIp != nil && *secondaryVnic.PublicIp != "" {
152+
ip := net.ParseIP(*secondaryVnic.PublicIp)
153+
if ip == nil {
154+
return nil, errors.Errorf("instance has invalid public address: %q", *secondaryVnic.PublicIp)
155+
}
156+
addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()})
157+
}
158+
}
159+
}
160+
nodeIpFamily, err := cp.getNodeIpFamily(instanceID)
161+
if err != nil {
162+
return nil, err
163+
}
164+
if contains(nodeIpFamily, IPv6) {
165+
if vnic.Ipv6Addresses != nil {
166+
for _, ipv6Addresses := range vnic.Ipv6Addresses {
167+
if ipv6Addresses != "" {
168+
ip := net.ParseIP(ipv6Addresses)
169+
if ip == nil {
170+
return nil, errors.Errorf("instance has invalid ipv6 address: %q", vnic.Ipv6Addresses[0])
171+
}
172+
if ip.IsPrivate() {
173+
addresses = append(addresses, api.NodeAddress{Type: api.NodeInternalIP, Address: ip.String()})
174+
} else {
175+
addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()})
176+
}
177+
}
178+
}
179+
}
180+
}
181+
}
182+
127183
// OKE does not support setting DNS since this changes the override hostname we setup to be the ip address.
128184
// Changing this can have wide reaching impact.
129185
//
@@ -372,3 +428,35 @@ func (cp *CloudProvider) getCompartmentIDByNodeName(nodeName string) (string, er
372428
cp.logger.Debug("CompartmentID annotation is not present")
373429
return "", errors.New("compartmentID annotation missing in the node. Would retry")
374430
}
431+
432+
func (cp *CloudProvider) getOpenShiftTagNamespaceByInstance(ctx context.Context, instanceID string) string {
433+
instance, err := cp.client.Compute().GetInstance(ctx, instanceID)
434+
if err != nil {
435+
return ""
436+
}
437+
438+
if instance.DefinedTags == nil {
439+
return ""
440+
}
441+
442+
for namespace := range instance.DefinedTags {
443+
if strings.HasPrefix(namespace, OpenShiftTagNamesapcePrefix) {
444+
return namespace
445+
}
446+
}
447+
return ""
448+
}
449+
450+
func (cp *CloudProvider) checkOpenShiftISCSIBootVolumeTagByVnic(ctx context.Context, vnic *core.Vnic, namespace string) bool {
451+
if vnic.DefinedTags == nil {
452+
return false
453+
}
454+
455+
if tags, namespaceExists := vnic.DefinedTags[namespace]; namespaceExists {
456+
// Check if the boot volume type key exists and its value is ISCSI
457+
if bootVolume, keyExists := tags[OpenShiftBootVolumeType]; keyExists && bootVolume == OpenShiftBootVolumeISCSI {
458+
return true
459+
}
460+
}
461+
return false
462+
}

‎pkg/cloudprovider/providers/oci/instances_test.go

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,24 @@ var (
136136
SubnetId: common.String("ipv6-gua-ipv4-instance"),
137137
Ipv6Addresses: []string{"2001:0db8:85a3:0000:0000:8a2e:0370:7334"},
138138
},
139+
"ocid1.openshift-instance-ipv4": {
140+
IsPrimary: common.Bool(false),
141+
PrivateIp: common.String("10.0.0.1"),
142+
PublicIp: common.String("0.0.0.1"),
143+
HostnameLabel: common.String("openshift-instance"),
144+
SubnetId: common.String("subnetwithdnslabel"),
145+
},
146+
"ocid1.openshift-instance-invalid": {
147+
PrivateIp: common.String("10.0.0."),
148+
HostnameLabel: common.String("openshift-instance-invalid"),
149+
SubnetId: common.String("subnetwithdnslabel"),
150+
},
151+
"ocid1.openshift-instance-ipv6": {
152+
IsPrimary: common.Bool(false),
153+
HostnameLabel: common.String("openshift-instance"),
154+
SubnetId: common.String("ipv6-instance"),
155+
Ipv6Addresses: []string{"2001:0db8:85a3:0000:0000:8a2e:0370:7334"},
156+
},
139157
}
140158

141159
instances = map[string]*core.Instance{
@@ -205,6 +223,33 @@ var (
205223
Id: common.String("ocid1.ipv6-gua-ipv4-instance"),
206224
CompartmentId: common.String("ipv6-gua-ipv4-instance"),
207225
},
226+
"openshift-instance-ipv4": {
227+
Id: common.String("ocid1.openshift-instance-ipv4"),
228+
CompartmentId: common.String("default"),
229+
DefinedTags: map[string]map[string]interface{}{
230+
"openshift-namespace": {
231+
"role": "compute",
232+
},
233+
},
234+
},
235+
"openshift-instance-invalid": {
236+
Id: common.String("ocid1.openshift-instance-invalid"),
237+
CompartmentId: common.String("default"),
238+
DefinedTags: map[string]map[string]interface{}{
239+
"openshift-namespace": {
240+
"role": "compute",
241+
},
242+
},
243+
},
244+
"openshift-instance-ipv6": {
245+
Id: common.String("ocid1.openshift-instance-ipv6"),
246+
CompartmentId: common.String("default"),
247+
DefinedTags: map[string]map[string]interface{}{
248+
"openshift-namespace": {
249+
"role": "compute",
250+
},
251+
},
252+
},
208253
}
209254
subnets = map[string]*core.Subnet{
210255
"subnetwithdnslabel": {
@@ -451,6 +496,20 @@ var (
451496
ProviderID: "ocid1.instance-id-ipv6",
452497
},
453498
},
499+
"openshift-instance-id-ipv6": {
500+
ObjectMeta: metav1.ObjectMeta{
501+
Annotations: map[string]string{
502+
CompartmentIDAnnotation: "default",
503+
},
504+
Labels: map[string]string{
505+
IPv6NodeIPFamilyLabel: "true",
506+
},
507+
Name: "Node-Ipv6",
508+
},
509+
Spec: v1.NodeSpec{
510+
ProviderID: "ocid1.openshift-instance-ipv6",
511+
},
512+
},
454513
}
455514

456515
podList = map[string]*v1.Pod{
@@ -873,6 +932,10 @@ func (MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compartm
873932
return instanceVnics[instanceID], nil
874933
}
875934

935+
func (MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
936+
return []*core.Vnic{instanceVnics[instanceID]}, nil
937+
}
938+
876939
func (MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) {
877940
return nil, nil
878941
}
@@ -1436,6 +1499,29 @@ func TestExtractNodeAddresses(t *testing.T) {
14361499
},
14371500
err: nil,
14381501
},
1502+
{
1503+
name: "openshift-instance-ipv4",
1504+
in: "ocid1.openshift-instance-ipv4",
1505+
out: []v1.NodeAddress{
1506+
{Type: v1.NodeInternalIP, Address: "10.0.0.1"},
1507+
{Type: v1.NodeExternalIP, Address: "0.0.0.1"},
1508+
},
1509+
err: nil,
1510+
},
1511+
{
1512+
name: "openshift-instance-invalid",
1513+
in: "ocid1.openshift-instance-invalid",
1514+
out: nil,
1515+
err: errors.New(`instance has invalid private address: "10.0.0."`),
1516+
},
1517+
{
1518+
name: "openshift-instance-ipv6",
1519+
in: "ocid1.openshift-instance-ipv6",
1520+
out: []v1.NodeAddress{
1521+
{Type: v1.NodeExternalIP, Address: "2001:db8:85a3::8a2e:370:7334"},
1522+
},
1523+
err: nil,
1524+
},
14391525
}
14401526

14411527
cp := &CloudProvider{
@@ -1931,3 +2017,113 @@ func (m mockServiceError) GetOpcRequestID() string {
19312017
func (m mockServiceError) Error() string {
19322018
return m.Message
19332019
}
2020+
2021+
func TestGetOpenShiftTagNamespaceByInstance(t *testing.T) {
2022+
testCases := []struct {
2023+
name string
2024+
instanceID string
2025+
expected string
2026+
}{
2027+
{
2028+
name: "Instance with OpenShift namespace",
2029+
instanceID: "openshift-instance-ipv4",
2030+
expected: "openshift-namespace",
2031+
},
2032+
{
2033+
name: "Instance without OpenShift namespace",
2034+
instanceID: "basic-complete",
2035+
expected: "",
2036+
},
2037+
{
2038+
name: "Non-existent instance",
2039+
instanceID: "non-existent-instance-id",
2040+
expected: "",
2041+
},
2042+
}
2043+
2044+
cp := &CloudProvider{
2045+
client: MockOCIClient{},
2046+
config: &providercfg.Config{CompartmentID: "testCompartment"},
2047+
NodeLister: &mockNodeLister{},
2048+
instanceCache: &mockInstanceCache{},
2049+
}
2050+
2051+
for _, tt := range testCases {
2052+
t.Run(tt.name, func(t *testing.T) {
2053+
result := cp.getOpenShiftTagNamespaceByInstance(context.Background(), tt.instanceID)
2054+
if !reflect.DeepEqual(result, tt.expected) {
2055+
t.Errorf("expected %s, got %s", tt.expected, result)
2056+
}
2057+
})
2058+
}
2059+
}
2060+
2061+
func TestCheckOpenShiftISCSIBootVolumeTagByVnic(t *testing.T) {
2062+
testCases := []struct {
2063+
name string
2064+
vnic *core.Vnic
2065+
namespace string
2066+
expected bool
2067+
}{
2068+
{
2069+
name: "VNIC with ISCSI boot volume tag",
2070+
vnic: &core.Vnic{
2071+
DefinedTags: map[string]map[string]interface{}{
2072+
"openshift-namespace": {
2073+
"boot-volume-type": "ISCSI",
2074+
},
2075+
},
2076+
},
2077+
namespace: "openshift-namespace",
2078+
expected: true,
2079+
},
2080+
{
2081+
name: "VNIC without ISCSI boot volume tag",
2082+
vnic: &core.Vnic{
2083+
DefinedTags: map[string]map[string]interface{}{
2084+
"openshift-namespace": {
2085+
"boot-volume-type": "NVMe",
2086+
},
2087+
},
2088+
},
2089+
namespace: "openshift-namespace",
2090+
expected: false,
2091+
},
2092+
{
2093+
name: "VNIC with no defined tags",
2094+
vnic: &core.Vnic{
2095+
DefinedTags: nil,
2096+
},
2097+
namespace: "openshift-namespace",
2098+
expected: false,
2099+
},
2100+
{
2101+
name: "Namespace not found in VNIC tags",
2102+
vnic: &core.Vnic{
2103+
DefinedTags: map[string]map[string]interface{}{
2104+
"another-namespace": {
2105+
"bootVolumeType": "ISCSI",
2106+
},
2107+
},
2108+
},
2109+
namespace: "openshift-namespace",
2110+
expected: false,
2111+
},
2112+
}
2113+
2114+
cp := &CloudProvider{
2115+
client: MockOCIClient{},
2116+
config: &providercfg.Config{CompartmentID: "testCompartment"},
2117+
NodeLister: &mockNodeLister{},
2118+
instanceCache: &mockInstanceCache{},
2119+
}
2120+
2121+
for _, tt := range testCases {
2122+
t.Run(tt.name, func(t *testing.T) {
2123+
result := cp.checkOpenShiftISCSIBootVolumeTagByVnic(context.Background(), tt.vnic, tt.namespace)
2124+
if !reflect.DeepEqual(result, tt.expected) {
2125+
t.Errorf("expected %v, got %v", tt.expected, result)
2126+
}
2127+
})
2128+
}
2129+
}

‎pkg/csi/driver/bv_controller_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -616,6 +616,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa
616616
return nil, nil
617617
}
618618

619+
func (MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
620+
return nil, nil
621+
}
622+
619623
func (c *MockComputeClient) ListVnicAttachments(ctx context.Context, compartmentID, instanceID string) ([]core.VnicAttachment, error) {
620624
return nil, nil
621625
}

0 commit comments

Comments
 (0)