Skip to content

Commit c3734b3

Browse files
janekbaraniewskijanekbaraniewski
authored
Add cleanup modes to namespace toHost syncing (#2856)
* Add imported marked annotation to prevent deletion of imported namespaces * Add hostNamespaces config struct to vcluster config * Add GetValues to helm client implementation * Add cleanup handlers for namespaces and use them when removing vcluster helm release * Move cleanup code to pkg/cli, simplify helm.GetValues * Fix errors logging * Disable UID deletion for namespace importer This caused recreation of namespaces imported into vclsuter on host * Rename vcluster config in delete helm cmd
1 parent ea8d7b3 commit c3734b3

File tree

9 files changed

+296
-33
lines changed

9 files changed

+296
-33
lines changed

‎chart/values.schema.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3645,6 +3645,16 @@
36453645
"scope"
36463646
]
36473647
},
3648+
"SyncHostSettings": {
3649+
"properties": {
3650+
"cleanup": {
3651+
"type": "string",
3652+
"description": "Cleanup defines the cleanup policy for host resources when vCluster is deleted.\nAllowed values: \"all\", \"synced\", \"none\"."
3653+
}
3654+
},
3655+
"additionalProperties": false,
3656+
"type": "object"
3657+
},
36483658
"SyncNodeSelector": {
36493659
"properties": {
36503660
"all": {
@@ -3898,6 +3908,10 @@
38983908
},
38993909
"type": "object",
39003910
"description": "ExtraLabels are additional labels to add to the namespace in the host cluster."
3911+
},
3912+
"hostNamespaces": {
3913+
"$ref": "#/$defs/SyncHostSettings",
3914+
"description": "HostNamespaces defines configuration for handling host namespaces."
39013915
}
39023916
},
39033917
"additionalProperties": false,

‎chart/values.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ sync:
108108
enabled: false
109109
# MappingsOnly defines if creation of namespaces not matched by mappings should be allowed.
110110
mappingsOnly: false
111+
# HostNamespaces defines configuration for handling host namespaces.
112+
hostNamespaces:
113+
# Cleanup defines the cleanup policy for host resources when vCluster is deleted.
114+
# Allowed values: "all", "synced", "none".
115+
cleanup: "synced"
111116

112117
# Configure what resources vCluster should sync from the host cluster to the virtual cluster.
113118
fromHost:

‎config/config.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -896,6 +896,27 @@ type SyncToHostNamespaces struct {
896896

897897
// ExtraLabels are additional labels to add to the namespace in the host cluster.
898898
ExtraLabels map[string]string `json:"extraLabels,omitempty"`
899+
900+
// HostNamespaces defines configuration for handling host namespaces.
901+
HostNamespaces SyncHostSettings `json:"hostNamespaces,omitempty"`
902+
}
903+
904+
// HostDeletionPolicy defines the policy for deletion of synced host resources.
905+
type HostDeletionPolicy string
906+
907+
const (
908+
// HostDeletionPolicyAll signifies a policy affecting all resources in host cluster matching any of syncing rules.
909+
HostDeletionPolicyAll HostDeletionPolicy = "all"
910+
// HostDeletionPolicySynced signifies a policy affecting only resources in host cluster created through syncing process from vCluster.
911+
HostDeletionPolicySynced HostDeletionPolicy = "synced"
912+
// HostDeletionPolicyNone signifies that no host resources are affected by this policy.
913+
HostDeletionPolicyNone HostDeletionPolicy = "none"
914+
)
915+
916+
type SyncHostSettings struct {
917+
// Cleanup defines the cleanup policy for host resources when vCluster is deleted.
918+
// Allowed values: "all", "synced", "none".
919+
Cleanup HostDeletionPolicy `json:"cleanup,omitempty"`
899920
}
900921

901922
type SyncToHostCustomResource struct {

‎config/values.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ sync:
5757
namespaces:
5858
enabled: false
5959
mappingsOnly: false
60+
hostNamespaces:
61+
cleanup: "synced"
6062

6163
fromHost:
6264
events:

‎pkg/cli/cleanup_namespaces.go

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
package cli
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
8+
"github.com/loft-sh/log"
9+
"github.com/loft-sh/vcluster/config"
10+
"github.com/loft-sh/vcluster/pkg/util/namespaces"
11+
"github.com/loft-sh/vcluster/pkg/util/translate"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/client-go/kubernetes"
14+
)
15+
16+
// NamespaceCleanupHandler defines the function signature for namespace cleanup operations.
17+
type NamespaceCleanupHandler func(
18+
ctx context.Context,
19+
mainPhysicalNamespace string,
20+
vClusterName string,
21+
nsSyncConfig config.SyncToHostNamespaces,
22+
k8sClient *kubernetes.Clientset,
23+
logger log.Logger,
24+
) error
25+
26+
// GetNamespaceCleanupHandler returns a NamespaceCleanupHandler function based on the provided policy.
27+
func GetNamespaceCleanupHandler(policy config.HostDeletionPolicy) (NamespaceCleanupHandler, error) {
28+
switch policy {
29+
case config.HostDeletionPolicyAll:
30+
return cleanupAllNamespaces, nil
31+
case config.HostDeletionPolicySynced:
32+
return cleanupSyncedNamespaces, nil
33+
case config.HostDeletionPolicyNone:
34+
return cleanupNoneNamespaces, nil
35+
default:
36+
return nil, fmt.Errorf("unsupported host namespace cleanup policy: %s", policy)
37+
}
38+
}
39+
40+
// cleanupNoneNamespaces is a no-op handler for the 'none' policy.
41+
func cleanupNoneNamespaces(
42+
_ context.Context,
43+
_ string, _ string,
44+
_ config.SyncToHostNamespaces,
45+
_ *kubernetes.Clientset,
46+
_ log.Logger,
47+
) error {
48+
return nil
49+
}
50+
51+
// cleanupSyncedNamespaces handles deletion of namespaces for the 'synced' policy.
52+
// It deletes namespaces from the host cluster that were created as a result of syncing process from vCluster,
53+
func cleanupSyncedNamespaces(
54+
ctx context.Context,
55+
mainPhysicalNamespace string,
56+
vClusterName string,
57+
_ config.SyncToHostNamespaces,
58+
k8sClient *kubernetes.Clientset,
59+
logger log.Logger,
60+
) error {
61+
logger.Infof("Starting cleanup of vCluster '%s' namespaces.", vClusterName)
62+
63+
if mainPhysicalNamespace == "" || vClusterName == "" {
64+
return fmt.Errorf("main physical namespace or vCluster name is empty")
65+
}
66+
67+
labelSelector := translate.MarkerLabel + "=" + translate.SafeConcatName(mainPhysicalNamespace, "x", vClusterName)
68+
nsList, err := k8sClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
69+
if err != nil {
70+
return fmt.Errorf("list namespaces: %w", err)
71+
}
72+
73+
if nsList == nil || len(nsList.Items) == 0 {
74+
logger.Infof("No additional managed namespaces found with label selector '%s'.", labelSelector)
75+
return nil
76+
}
77+
78+
var errs []error
79+
for _, ns := range nsList.Items {
80+
// Check if namespace was imported, if yes we skip deletion for 'synced' policy.
81+
if ns.Annotations != nil && ns.Annotations[translate.ImportedMarkerAnnotation] == "true" {
82+
logger.Infof("Namespace %s was imported, skip cleanup.", ns.Name)
83+
continue
84+
}
85+
86+
logger.Infof("Attempting to delete virtual cluster namespace %s.", ns.Name)
87+
err := k8sClient.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
88+
if err != nil {
89+
errs = append(errs, fmt.Errorf("namespace %s: %w", ns.Name, err))
90+
} else {
91+
logger.Donef("Successfully deleted virtual cluster namespace %s.", ns.Name)
92+
}
93+
}
94+
95+
if len(errs) > 0 {
96+
return fmt.Errorf("cleanup of vCluster '%s' namespaces finished with errors: %w", vClusterName, errors.Join(errs...))
97+
}
98+
99+
logger.Infof("Cleanup of vCluster '%s' namespaces finished.", vClusterName)
100+
return nil
101+
}
102+
103+
// cleanupAllNamespaces handles deletion of namespaces for the 'all' policy.
104+
// It deletes all namespaces matching target patterns in mappings, regardless of whether they were imported, created through syncing or not.
105+
func cleanupAllNamespaces(
106+
ctx context.Context,
107+
_ string,
108+
vClusterName string,
109+
nsSyncConfig config.SyncToHostNamespaces,
110+
k8sClient *kubernetes.Clientset,
111+
logger log.Logger,
112+
) error {
113+
logger.Infof("Starting cleanup of vCluster '%s' namespaces.", vClusterName)
114+
mappingsConfig := nsSyncConfig.Mappings
115+
116+
if len(mappingsConfig.ByName) == 0 {
117+
logger.Infof("No namespace mappings defined.")
118+
logger.Infof("Cleanup of vCluster '%s' namespaces finished.", vClusterName)
119+
return nil
120+
}
121+
logger.Debugf("Processing %d namespace mappings for potential deletion.", len(mappingsConfig.ByName))
122+
123+
hostNamespaces, err := k8sClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
124+
if err != nil {
125+
return fmt.Errorf("list all host namespaces for mapping check: %w", err)
126+
}
127+
128+
if hostNamespaces == nil || len(hostNamespaces.Items) == 0 {
129+
logger.Infof("No host namespaces found to check against mappings.")
130+
logger.Infof("Cleanup of vCluster '%s' namespaces finished - no namespaces found.", vClusterName)
131+
return nil
132+
}
133+
134+
var errs []error
135+
for _, hostNs := range hostNamespaces.Items {
136+
// Check if this hostNs matches any mapping rule target
137+
for _, hostTargetPatternRaw := range mappingsConfig.ByName {
138+
processedHostTargetPattern := namespaces.ProcessNamespaceName(hostTargetPatternRaw, vClusterName)
139+
140+
var currentRuleMatches bool
141+
if namespaces.IsPattern(processedHostTargetPattern) {
142+
_, currentRuleMatches = namespaces.MatchAndExtractWildcard(hostNs.Name, processedHostTargetPattern)
143+
} else {
144+
currentRuleMatches = (hostNs.Name == processedHostTargetPattern)
145+
}
146+
147+
if currentRuleMatches {
148+
logger.Infof("Attempting to delete virtual cluster namespace %s.", hostNs.Name)
149+
err := k8sClient.CoreV1().Namespaces().Delete(ctx, hostNs.Name, metav1.DeleteOptions{})
150+
if err != nil {
151+
errs = append(errs, fmt.Errorf("delete virtual cluster namespace %s: %w", hostNs.Name, err))
152+
} else {
153+
logger.Donef("Successfully deleted virtual cluster namespace %s.", hostNs.Name)
154+
}
155+
// This namespace has been handled. Skip other mappings and move to the next one.
156+
break
157+
}
158+
}
159+
}
160+
161+
if len(errs) > 0 {
162+
return fmt.Errorf("cleanup of vCluster '%s' namespaces finished with errors: %w", vClusterName, errors.Join(errs...))
163+
}
164+
165+
logger.Infof("Cleanup of vCluster '%s' namespaces finished.", vClusterName)
166+
return nil
167+
}

‎pkg/cli/delete_helm.go

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ import (
77
"os/exec"
88
"time"
99

10+
"github.com/ghodss/yaml"
1011
managementv1 "github.com/loft-sh/api/v4/pkg/apis/management/v1"
1112
"github.com/loft-sh/log"
13+
"github.com/loft-sh/vcluster/config"
1214
"github.com/loft-sh/vcluster/pkg/cli/find"
1315
"github.com/loft-sh/vcluster/pkg/cli/flags"
1416
"github.com/loft-sh/vcluster/pkg/cli/localkubernetes"
@@ -117,9 +119,22 @@ func DeleteHelm(ctx context.Context, platformClient platform.Client, options *De
117119
}
118120
}
119121

122+
helmClient := helm.NewClient(cmd.rawConfig, cmd.log, helmBinaryPath)
123+
// before removing vCluster release, we need to get the config from values for later use
124+
values, err := helmClient.GetValues(ctx, vClusterName, cmd.Namespace, true)
125+
if err != nil {
126+
return err
127+
}
128+
129+
vclusterConfig := &config.Config{}
130+
err = yaml.Unmarshal(values, vclusterConfig)
131+
if err != nil {
132+
return err
133+
}
134+
120135
// we have to delete the chart
121136
cmd.log.Infof("Delete vcluster %s...", vClusterName)
122-
err = helm.NewClient(cmd.rawConfig, cmd.log, helmBinaryPath).Delete(vClusterName, cmd.Namespace)
137+
err = helmClient.Delete(vClusterName, cmd.Namespace)
123138
if err != nil {
124139
return err
125140
}
@@ -179,7 +194,18 @@ func DeleteHelm(ctx context.Context, platformClient platform.Client, options *De
179194
cmd.DeleteNamespace = false
180195
}
181196

182-
// try to delete the namespace
197+
// if namespace sync is enabled, use cleanup handlers to handle namespace cleanup
198+
if vclusterConfig.Sync.ToHost.Namespaces.Enabled {
199+
runNamespaceCleanup, err := GetNamespaceCleanupHandler(vclusterConfig.Sync.ToHost.Namespaces.HostNamespaces.Cleanup)
200+
if err != nil {
201+
return fmt.Errorf("get cleanup handler: %w", err)
202+
}
203+
if err := runNamespaceCleanup(ctx, cmd.Namespace, vClusterName, vclusterConfig.Sync.ToHost.Namespaces, cmd.kubeClient, cmd.log); err != nil {
204+
return fmt.Errorf("run namespace cleanup: %w", err)
205+
}
206+
}
207+
208+
// check if we should cleanup vCluster host namespace
183209
if cmd.DeleteNamespace {
184210
// delete namespace
185211
err = cmd.kubeClient.CoreV1().Namespaces().Delete(ctx, cmd.Namespace, metav1.DeleteOptions{})
@@ -191,29 +217,7 @@ func DeleteHelm(ctx context.Context, platformClient platform.Client, options *De
191217
cmd.log.Donef("Successfully deleted virtual cluster namespace %s", cmd.Namespace)
192218
}
193219

194-
// delete multi namespace mode namespaces
195-
namespaces, err := cmd.kubeClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{
196-
LabelSelector: translate.MarkerLabel + "=" + translate.SafeConcatName(cmd.Namespace, "x", vClusterName),
197-
})
198-
if err != nil && !kerrors.IsForbidden(err) {
199-
return fmt.Errorf("list namespaces: %w", err)
200-
}
201-
202-
// delete all namespaces
203-
if namespaces != nil && len(namespaces.Items) > 0 {
204-
for _, namespace := range namespaces.Items {
205-
err = cmd.kubeClient.CoreV1().Namespaces().Delete(ctx, namespace.Name, metav1.DeleteOptions{})
206-
if err != nil {
207-
if !kerrors.IsNotFound(err) {
208-
return fmt.Errorf("delete namespace: %w", err)
209-
}
210-
} else {
211-
cmd.log.Donef("Successfully deleted virtual cluster namespace %s", namespace.Name)
212-
}
213-
}
214-
}
215-
216-
// wait for vcluster deletion
220+
// wait for namespace deletion
217221
if cmd.Wait {
218222
cmd.log.Info("Waiting for virtual cluster to be deleted...")
219223
for {

‎pkg/controllers/resources/namespaces/syncer.go

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/loft-sh/vcluster/pkg/syncer/synccontext"
1111
"github.com/loft-sh/vcluster/pkg/syncer/translator"
1212
syncertypes "github.com/loft-sh/vcluster/pkg/syncer/types"
13+
"github.com/loft-sh/vcluster/pkg/util/translate"
1314
corev1 "k8s.io/api/core/v1"
1415
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1516
utilerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -67,7 +68,8 @@ var _ syncertypes.OptionsProvider = &namespaceSyncer{}
6768

6869
func (s *namespaceSyncer) Options() *syncertypes.Options {
6970
return &syncertypes.Options{
70-
ObjectCaching: true,
71+
ObjectCaching: true,
72+
DisableUIDDeletion: true,
7173
}
7274
}
7375

@@ -110,13 +112,31 @@ func (s *namespaceSyncer) Sync(ctx *synccontext.SyncContext, event *synccontext.
110112
func (s *namespaceSyncer) SyncToVirtual(ctx *synccontext.SyncContext, event *synccontext.SyncToVirtualEvent[*corev1.Namespace]) (_ ctrl.Result, retErr error) {
111113
// virtual object is not here anymore, so we delete
112114
if event.VirtualOld != nil || event.Host.DeletionTimestamp != nil {
115+
// first, lets check if host object was imported - if so, we don't delete it
116+
if event.Host.Annotations != nil && event.Host.Annotations[translate.ImportedMarkerAnnotation] == "true" {
117+
ctx.Log.Infof("host object %s/%s was imported, not deleting it", event.Host.Namespace, event.Host.Name)
118+
return ctrl.Result{}, nil
119+
}
120+
113121
return patcher.DeleteHostObject(ctx, event.Host, event.VirtualOld, "virtual object was deleted")
114122
}
115123

124+
// add marker annotation to host object and update it
125+
_, err := controllerutil.CreateOrPatch(ctx, ctx.PhysicalClient, event.Host, func() error {
126+
if event.Host.Annotations == nil {
127+
event.Host.Annotations = map[string]string{}
128+
}
129+
event.Host.Annotations[translate.ImportedMarkerAnnotation] = "true"
130+
return nil
131+
})
132+
if err != nil {
133+
return ctrl.Result{}, fmt.Errorf("create or patch host object: %w", err)
134+
}
135+
116136
newNamespace := s.translateToVirtual(ctx, event.Host)
117137
ctx.Log.Infof("create virtual namespace %s", newNamespace.Name)
118138

119-
err := pro.ApplyPatchesVirtualObject(ctx, nil, newNamespace, event.Host, ctx.Config.Sync.ToHost.Namespaces.Patches, false)
139+
err = pro.ApplyPatchesVirtualObject(ctx, nil, newNamespace, event.Host, ctx.Config.Sync.ToHost.Namespaces.Patches, false)
120140
if err != nil {
121141
return ctrl.Result{}, err
122142
}

0 commit comments

Comments
 (0)