Skip to content

Commit 8c40cf3

Browse files
mrunalpagnisl-technicore
authored andcommitted
Use vendor dependency for mount utils
NLB fix cherry pick from open source
1 parent 2c59400 commit 8c40cf3

File tree

12 files changed

+211
-1140
lines changed

12 files changed

+211
-1140
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ require (
6969
k8s.io/klog/v2 v2.60.1
7070
k8s.io/kubelet v0.24.1
7171
k8s.io/kubernetes v1.24.1
72+
k8s.io/mount-utils v0.24.1
7273
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
7374
sigs.k8s.io/controller-runtime v0.11.2
7475
sigs.k8s.io/sig-storage-lib-external-provisioner/v6 v6.3.0
@@ -172,7 +173,6 @@ require (
172173
k8s.io/kube-openapi v0.0.0-20220401212409-b28bf2818661 // indirect
173174
k8s.io/kube-scheduler v0.0.0 // indirect
174175
k8s.io/kubectl v0.0.0 // indirect
175-
k8s.io/mount-utils v0.24.1 // indirect
176176
oracle.com/oci/httpsigner v0.0.0-20190320175442-e8cb27ebf440 // indirect
177177
oracle.com/oci/ociauthz v0.0.0-20200515161105-5b1e37d2dc95 // indirect
178178
oracle.com/oci/tagging v0.0.0-20190321202046-20a2e48911da // indirect

pkg/csi-util/utils.go

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"os"
7+
"os/exec"
78
"strconv"
89
"strings"
910
"sync"
@@ -47,6 +48,12 @@ const (
4748
LowCostPerformanceOption = 0
4849
BalancedPerformanceOption = 10
4950
HigherPerformanceOption = 20
51+
52+
InTransitEncryptionPackageName = "oci-fss-utils"
53+
FIPS_ENABLED_FILE_PATH = "/host/proc/sys/crypto/fips_enabled"
54+
FINDMNT_COMMAND = "findmnt"
55+
CAT_COMMAND = "cat"
56+
RPM_COMMAND = "rpm"
5057
)
5158

5259
//Util interface
@@ -348,3 +355,96 @@ func RoundUpSize(volumeSizeBytes int64, allocationUnitBytes int64) int64 {
348355
func RoundUpMinSize() int64 {
349356
return RoundUpSize(MinimumVolumeSizeInBytes, 1*client.GiB)
350357
}
358+
359+
func IsFipsEnabled() (string, error) {
360+
command := exec.Command(CAT_COMMAND, FIPS_ENABLED_FILE_PATH)
361+
output, err := command.CombinedOutput()
362+
if err != nil {
363+
return "", fmt.Errorf("command failed: %v\narguments: %s\nOutput: %v\n", err, CAT_COMMAND, string(output))
364+
}
365+
366+
return string(output), nil
367+
}
368+
369+
func IsInTransitEncryptionPackageInstalled() (bool, error) {
370+
args := []string{"-q", "-a", "--root=/host"}
371+
command := exec.Command(RPM_COMMAND, args...)
372+
output, err := command.CombinedOutput()
373+
if err != nil {
374+
return false, fmt.Errorf("command failed: %v\narguments: %s\nOutput: %v\n", err, RPM_COMMAND, string(output))
375+
}
376+
377+
if len(output) > 0 {
378+
list := string(output)
379+
if strings.Contains(list, InTransitEncryptionPackageName) {
380+
return true, nil
381+
}
382+
return false, nil
383+
}
384+
return false, nil
385+
}
386+
387+
func FindMount(target string) ([]string, error) {
388+
mountArgs := []string{"-n", "-o", "SOURCE", "-T", target}
389+
command := exec.Command(FINDMNT_COMMAND, mountArgs...)
390+
output, err := command.CombinedOutput()
391+
if err != nil {
392+
return nil, fmt.Errorf("findmnt failed: %v\narguments: %s\nOutput: %v\n", err, mountArgs, string(output))
393+
}
394+
395+
sources := strings.Fields(string(output))
396+
return sources, nil
397+
}
398+
399+
func Rescan(logger *zap.SugaredLogger, devicePath string) error {
400+
401+
lsblkargs := []string{"-n", "-o", "NAME", devicePath}
402+
lsblkcmd := exec.Command("lsblk", lsblkargs...)
403+
lsblkoutput, err := lsblkcmd.CombinedOutput()
404+
if err != nil {
405+
return fmt.Errorf("Failed to find device name associated with devicePath %s", devicePath)
406+
}
407+
deviceName := strings.TrimSpace(string(lsblkoutput))
408+
if strings.HasPrefix(deviceName, "/dev/") {
409+
deviceName = strings.TrimPrefix(deviceName, "/dev/")
410+
}
411+
logger.With("deviceName", deviceName).Info("Rescanning")
412+
413+
// run command dd iflag=direct if=/dev/<device_name> of=/dev/null count=1
414+
// https://docs.oracle.com/en-us/iaas/Content/Block/Tasks/rescanningdisk.htm#Rescanni
415+
devicePathFileArg := fmt.Sprintf("if=%s", devicePath)
416+
args := []string{"iflag=direct", devicePathFileArg, "of=/dev/null", "count=1"}
417+
cmd := exec.Command("dd", args...)
418+
output, err := cmd.CombinedOutput()
419+
if err != nil {
420+
return fmt.Errorf("command failed: %v\narguments: %s\nOutput: %v\n", err, "dd", string(output))
421+
}
422+
logger.With("command", "dd", "output", string(output)).Debug("dd output")
423+
// run command echo 1 | tee /sys/class/block/%s/device/rescan
424+
// https://docs.oracle.com/en-us/iaas/Content/Block/Tasks/rescanningdisk.htm#Rescanni
425+
cmdStr := fmt.Sprintf("echo 1 | tee /sys/class/block/%s/device/rescan", deviceName)
426+
cmd = exec.Command("bash", "-c", cmdStr)
427+
output, err = cmd.CombinedOutput()
428+
if err != nil {
429+
return fmt.Errorf("command failed: %v\narguments: %s\nOutput: %v\n", err, cmdStr, string(output))
430+
}
431+
logger.With("command", cmdStr, "output", string(output)).Debug("rescan output")
432+
433+
return nil
434+
}
435+
436+
func GetBlockSizeBytes(logger *zap.SugaredLogger, devicePath string) (int64, error) {
437+
args := []string{"--getsize64", devicePath}
438+
cmd := exec.Command("blockdev", args...)
439+
output, err := cmd.CombinedOutput()
440+
if err != nil {
441+
return -1, fmt.Errorf("command failed: %v\narguments: %s\nOutput: %v\n", err, "blockdev", string(output))
442+
}
443+
strOut := strings.TrimSpace(string(output))
444+
logger.With("devicePath", devicePath, "command", "blockdev", "output", strOut).Debugf("Get block device size in bytes successful")
445+
gotSizeBytes, err := strconv.ParseInt(strOut, 10, 64)
446+
if err != nil {
447+
return -1, fmt.Errorf("failed to parse size %s into an int64 size", strOut)
448+
}
449+
return gotSizeBytes, nil
450+
}

pkg/csi/driver/bv_node.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -593,7 +593,7 @@ func (d BlockVolumeNodeDriver) NodeExpandVolume(ctx context.Context, req *csi.No
593593
return nil, status.Error(codes.InvalidArgument, "unknown attachment type. supported attachment types are iscsi and paravirtualized")
594594
}
595595

596-
if err := mountHandler.Rescan(devicePath); err != nil {
596+
if err := csi_util.Rescan(logger, devicePath); err != nil {
597597
return nil, status.Errorf(codes.Internal, "Failed to rescan volume %q (%q): %v", volumeID, devicePath, err)
598598
}
599599
logger.With("devicePath", devicePath).Debug("Rescan completed")
@@ -602,7 +602,7 @@ func (d BlockVolumeNodeDriver) NodeExpandVolume(ctx context.Context, req *csi.No
602602
return nil, status.Errorf(codes.Internal, "Failed to resize volume %q (%q): %v", volumeID, devicePath, err)
603603
}
604604

605-
allocatedSizeBytes, err := mountHandler.GetBlockSizeBytes(devicePath)
605+
allocatedSizeBytes, err := csi_util.GetBlockSizeBytes(logger, devicePath)
606606
if err != nil {
607607
return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to get size of block volume at path %s: %v", devicePath, err))
608608
}

pkg/csi/driver/controller.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,7 @@ func (d *ControllerDriver) CreateVolume(ctx context.Context, req *csi.CreateVolu
268268
}
269269
ctx, cancel := context.WithTimeout(ctx, timeout)
270270
defer cancel()
271+
log.Info("Waiting for volume to become available.")
271272
_, err = d.client.BlockStorage().AwaitVolumeAvailableORTimeout(ctx, *provisionedVolume.Id)
272273
if err != nil {
273274
log.With("volumeName", volumeName).Error("Create volume failed with time out")

pkg/csi/driver/fss_node.go

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@ import (
1212
"google.golang.org/grpc/codes"
1313
"google.golang.org/grpc/status"
1414
kubeAPI "k8s.io/api/core/v1"
15+
"k8s.io/mount-utils"
1516

1617
"github.com/container-storage-interface/spec/lib/go/csi"
17-
"github.com/oracle/oci-cloud-controller-manager/pkg/util/mount"
18+
csi_util "github.com/oracle/oci-cloud-controller-manager/pkg/csi-util"
19+
"github.com/oracle/oci-cloud-controller-manager/pkg/util/disk"
1820
)
1921

2022
const (
@@ -53,22 +55,22 @@ func (d FSSNodeDriver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVo
5355
return nil, status.Errorf(codes.InvalidArgument, "EncryptInTransit must be a boolean value")
5456
}
5557

56-
mounter := mount.New(logger, mountPath)
58+
mounter := mount.New(mountPath)
5759

5860
var options []string
5961
if encryptInTransit {
60-
isPackageInstalled, err := mount.IsInTransitEncryptionPackageInstalled(mounter)
62+
isPackageInstalled, err := csi_util.IsInTransitEncryptionPackageInstalled()
6163
if err != nil {
6264
logger.With(zap.Error(err)).Error("FSS in-transit encryption Package installation check failed")
6365
return nil, status.Error(codes.Internal, "FSS in-transit encryption Package installation check failed")
6466
}
6567
if !isPackageInstalled {
66-
logger.Error("Package %s not installed for in-transit encryption", mount.InTransitEncryptionPackageName)
67-
return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("Package %s not installed for in-transit encryption", mount.InTransitEncryptionPackageName))
68+
logger.Error("Package %s not installed for in-transit encryption", csi_util.InTransitEncryptionPackageName)
69+
return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("Package %s not installed for in-transit encryption", csi_util.InTransitEncryptionPackageName))
6870
}
6971
logger.Debug("In-transit encryption enabled")
7072
fsType = "oci-fss"
71-
content, err := mount.IsFipsEnabled(mounter)
73+
content, err := csi_util.IsFipsEnabled()
7274
if err != nil {
7375
logger.With(zap.Error(err)).Error("Could not verify if FIPS enabled")
7476
return nil, status.Error(codes.Internal, "Could not verify if FIPS enabled")
@@ -174,7 +176,7 @@ func (d FSSNodeDriver) NodePublishVolume(ctx context.Context, req *csi.NodePubli
174176

175177
var fsType = ""
176178

177-
mounter := mount.New(logger, mountPath)
179+
mounter := mount.New(mountPath)
178180

179181
targetPath := req.GetTargetPath()
180182
readOnly := req.GetReadonly()
@@ -231,7 +233,7 @@ func (d FSSNodeDriver) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnp
231233

232234
logger := d.logger.With("volumeID", req.VolumeId, "targetPath", req.TargetPath)
233235

234-
mounter := mount.New(logger, mountPath)
236+
mounter := mount.New(mountPath)
235237
targetPath := req.GetTargetPath()
236238

237239
// Use mount.IsNotMountPoint because mounter.IsLikelyNotMountPoint can't detect bind mounts
@@ -302,7 +304,7 @@ func (d FSSNodeDriver) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnsta
302304
}
303305

304306
func (d FSSNodeDriver) unmountAndCleanup(logger *zap.SugaredLogger, targetPath string, exportPath string, mountTargetIP string) error {
305-
mounter := mount.New(logger, mountPath)
307+
mounter := mount.New(mountPath)
306308
// Use mount.IsNotMountPoint because mounter.IsLikelyNotMountPoint can't detect bind mounts
307309
isNotMountPoint, err := mount.IsNotMountPoint(mounter, targetPath)
308310
if err != nil {
@@ -323,7 +325,7 @@ func (d FSSNodeDriver) unmountAndCleanup(logger *zap.SugaredLogger, targetPath s
323325
return nil
324326
}
325327

326-
sources, err := mount.FindMount(mounter, targetPath)
328+
sources, err := csi_util.FindMount(targetPath)
327329
if err != nil {
328330
logger.With(zap.Error(err)).Error("Find Mount failed for target path")
329331
return status.Error(codes.Internal, "Find Mount failed for target path")
@@ -341,7 +343,7 @@ func (d FSSNodeDriver) unmountAndCleanup(logger *zap.SugaredLogger, targetPath s
341343
logger.Debugf("Sources mounted at staging target path %v", sources)
342344

343345
if inTransitEncryption {
344-
err = mounter.UnmountWithEncrypt(targetPath)
346+
err = disk.UnmountWithEncrypt(logger, targetPath)
345347
} else {
346348
err = mounter.Unmount(targetPath)
347349
}

0 commit comments

Comments
 (0)