step Initializing before Provisioning (#196)

kobzonega · web-flow · commit d9edaebc1fd7 · 2024-05-01T22:32:32.000+02:00
diff --git a/docs/storage-state-machine-schema.drawio.png b/docs/storage-state-machine-schema.drawio.png
diff --git a/e2e/tests/smoke_test.go b/e2e/tests/smoke_test.go
@@ -102,7 +102,7 @@ func waitUntilDatabaseReady(ctx context.Context, databaseName, databaseNamespace
 
 		return meta.IsStatusConditionPresentAndEqual(
 			database.Status.Conditions,
-			DatabaseTenantInitializedCondition,
+			DatabaseInitializedCondition,
 			metav1.ConditionTrue,
 		) && database.Status.State == testobjects.ReadyStatus
 	}, Timeout, Interval).Should(BeTrue())
@@ -521,7 +521,7 @@ var _ = Describe("Operator smoke test", func() {
 		checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-storage", storageSample.Spec.Nodes)
 	})
 
-	It("storage.State goes Pending -> Preparing -> Provisioning -> Initializing -> Ready", func() {
+	It("storage.State goes Pending -> Preparing -> Initializing -> Provisioning -> Ready", func() {
 		Expect(k8sClient.Create(ctx, storageSample)).Should(Succeed())
 		defer func() {
 			Expect(k8sClient.Delete(ctx, storageSample)).Should(Succeed())
@@ -537,9 +537,9 @@ var _ = Describe("Operator smoke test", func() {
 
 		allowedChanges := map[ClusterState]ClusterState{
 			StoragePending:      StoragePreparing,
-			StoragePreparing:    StorageProvisioning,
-			StorageProvisioning: StorageInitializing,
-			StorageInitializing: StorageReady,
+			StoragePreparing:    StorageInitializing,
+			StorageInitializing: StorageProvisioning,
+			StorageProvisioning: StorageReady,
 		}
 
 		re := regexp.MustCompile(`Storage moved from ([a-zA-Z]+) to ([a-zA-Z]+)`)
diff --git a/internal/controllers/constants/constants.go b/internal/controllers/constants/constants.go
@@ -15,13 +15,17 @@ const (
 	DatabaseNodeSetKind       = "DatabaseNodeSet"
 	RemoteDatabaseNodeSetKind = "RemoteDatabaseNodeSet"
 
-	StoragePausedCondition             = "StoragePaused"
-	StorageInitializedCondition        = "StorageReady"
-	StorageNodeSetReadyCondition       = "StorageNodeSetReady"
-	DatabasePausedCondition            = "DatabasePaused"
-	DatabaseTenantInitializedCondition = "TenantInitialized"
-	DatabaseNodeSetReadyCondition      = "DatabaseNodeSetReady"
-	RemoteResourceSyncedCondition      = "ResourceSynced"
+	// For backward compatibility
+	OldStorageInitializedCondition  = "StorageReady"
+	OldDatabaseInitializedCondition = "TenantInitialized"
+
+	StoragePausedCondition        = "StoragePaused"
+	StorageInitializedCondition   = "StorageInitialized"
+	StorageNodeSetReadyCondition  = "StorageNodeSetReady"
+	DatabasePausedCondition       = "DatabasePaused"
+	DatabaseInitializedCondition  = "DatabaseInitialized"
+	DatabaseNodeSetReadyCondition = "DatabaseNodeSetReady"
+	RemoteResourceSyncedCondition = "ResourceSynced"
 
 	Stop     = true
 	Continue = false
@@ -30,10 +34,11 @@ const (
 	ReasonNotRequired = "NotRequired"
 	ReasonCompleted   = "Completed"
 
-	DefaultRequeueDelay               = 10 * time.Second
-	StatusUpdateRequeueDelay          = 1 * time.Second
-	SelfCheckRequeueDelay             = 30 * time.Second
-	StorageInitializationRequeueDelay = 5 * time.Second
+	DefaultRequeueDelay                = 10 * time.Second
+	StatusUpdateRequeueDelay           = 1 * time.Second
+	SelfCheckRequeueDelay              = 30 * time.Second
+	StorageInitializationRequeueDelay  = 30 * time.Second
+	DatabaseInitializationRequeueDelay = 30 * time.Second
 
 	DatabasePending      ClusterState = "Pending"
 	DatabasePreparing    ClusterState = "Preparing"
@@ -62,7 +67,6 @@ const (
 	ResourceSyncPending RemoteResourceState = "Pending"
 	ResourceSyncSuccess RemoteResourceState = "Synced"
 
-	TenantCreationRequeueDelay      = 30 * time.Second
 	StorageAwaitRequeueDelay        = 30 * time.Second
 	SharedDatabaseAwaitRequeueDelay = 30 * time.Second
 
diff --git a/internal/controllers/database/init.go b/internal/controllers/database/init.go
@@ -44,21 +44,32 @@ func (r *Reconciler) setInitialStatus(
 ) (bool, ctrl.Result, error) {
 	r.Log.Info("running step setInitialStatus")
 
+	if meta.IsStatusConditionTrue(database.Status.Conditions, OldDatabaseInitializedCondition) {
+		meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
+			Type:    DatabaseInitializedCondition,
+			Status:  "True",
+			Reason:  ReasonCompleted,
+			Message: "Database initialized successfully",
+		})
+		database.Status.State = DatabaseReady
+		return r.updateStatus(ctx, database)
+	}
+
 	if value, ok := database.Annotations[v1alpha1.AnnotationSkipInitialization]; ok && value == v1alpha1.AnnotationValueTrue {
-		if meta.FindStatusCondition(database.Status.Conditions, DatabaseTenantInitializedCondition) == nil ||
-			meta.IsStatusConditionFalse(database.Status.Conditions, DatabaseTenantInitializedCondition) {
+		if meta.FindStatusCondition(database.Status.Conditions, DatabaseInitializedCondition) == nil ||
+			meta.IsStatusConditionFalse(database.Status.Conditions, DatabaseInitializedCondition) {
 			return r.processSkipInitPipeline(ctx, database)
 		}
 		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil
 	}
 
 	if database.Status.State == DatabasePending ||
-		meta.FindStatusCondition(database.Status.Conditions, DatabaseTenantInitializedCondition) == nil {
+		meta.FindStatusCondition(database.Status.Conditions, DatabaseInitializedCondition) == nil {
 		meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
-			Type:    DatabaseTenantInitializedCondition,
+			Type:    DatabaseInitializedCondition,
 			Status:  "False",
 			Reason:  ReasonInProgress,
-			Message: "Tenant creation in progress",
+			Message: "Database has not been initialized yet",
 		})
 		database.Status.State = DatabasePreparing
 		return r.updateStatus(ctx, database)
@@ -73,21 +84,26 @@ func (r *Reconciler) setInitDatabaseCompleted(
 	message string,
 ) (bool, ctrl.Result, error) {
 	meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
-		Type:    DatabaseTenantInitializedCondition,
+		Type:    DatabaseInitializedCondition,
 		Status:  "True",
 		Reason:  ReasonCompleted,
 		Message: message,
 	})
+	database.Status.State = DatabaseProvisioning
 
-	database.Status.State = DatabaseReady
 	return r.updateStatus(ctx, database)
 }
 
-func (r *Reconciler) handleTenantCreation(
+func (r *Reconciler) initializeDatabase(
 	ctx context.Context,
 	database *resources.DatabaseBuilder,
 ) (bool, ctrl.Result, error) {
-	r.Log.Info("running step handleTenantCreation")
+	r.Log.Info("running step initializeDatabase")
+
+	if database.Status.State == DatabasePreparing {
+		database.Status.State = DatabaseInitializing
+		return r.updateStatus(ctx, database)
+	}
 
 	path := database.GetDatabasePath()
 	var storageUnits []v1alpha1.StorageUnit
@@ -197,7 +213,7 @@ func (r *Reconciler) handleTenantCreation(
 			"InitializingFailed",
 			fmt.Sprintf("Error creating tenant %s: %s", tenant.Path, err),
 		)
-		return Stop, ctrl.Result{RequeueAfter: TenantCreationRequeueDelay}, err
+		return Stop, ctrl.Result{RequeueAfter: DatabaseInitializationRequeueDelay}, err
 	}
 	r.Recorder.Event(
 		database,
@@ -213,9 +229,5 @@ func (r *Reconciler) handleTenantCreation(
 		"Database is initialized",
 	)
 
-	return r.setInitDatabaseCompleted(
-		ctx,
-		database,
-		"Database initialization is completed",
-	)
+	return r.setInitDatabaseCompleted(ctx, database, "Database initialized successfully")
 }
diff --git a/internal/controllers/database/sync.go b/internal/controllers/database/sync.go
@@ -62,8 +62,28 @@ func (r *Reconciler) Sync(ctx context.Context, ydbCr *v1alpha1.Database) (ctrl.R
 		return result, err
 	}
 
-	if !meta.IsStatusConditionTrue(database.Status.Conditions, DatabaseTenantInitializedCondition) {
-		return r.handleFirstStart(ctx, &database)
+	if !meta.IsStatusConditionTrue(database.Status.Conditions, DatabaseInitializedCondition) {
+		return r.handleTenantCreation(ctx, &database)
+	}
+
+	if database.Spec.NodeSets != nil {
+		stop, result, err = r.waitForDatabaseNodeSetsToReady(ctx, &database)
+		if stop {
+			return result, err
+		}
+	} else {
+		stop, result, err = r.waitForStatefulSetToScale(ctx, &database)
+		if stop {
+			return result, err
+		}
+	}
+
+	if database.Status.State != DatabaseReady {
+		database.Status.State = DatabaseReady
+		stop, result, err = r.updateStatus(ctx, &database)
+		if stop {
+			return result, err
+		}
 	}
 
 	return ctrl.Result{}, nil
@@ -130,7 +150,7 @@ func (r *Reconciler) waitForDatabaseNodeSetsToReady(
 ) (bool, ctrl.Result, error) {
 	r.Log.Info("running step waitForDatabaseNodeSetToReady")
 
-	if database.Status.State == DatabasePreparing {
+	if database.Status.State == DatabaseInitializing {
 		r.Recorder.Event(
 			database,
 			corev1.EventTypeNormal,
@@ -552,7 +572,7 @@ func (r *Reconciler) handlePauseResume(
 	return Continue, ctrl.Result{}, nil
 }
 
-func (r *Reconciler) handleFirstStart(
+func (r *Reconciler) handleTenantCreation(
 	ctx context.Context,
 	database *resources.DatabaseBuilder,
 ) (ctrl.Result, error) {
@@ -561,19 +581,7 @@ func (r *Reconciler) handleFirstStart(
 		return result, err
 	}
 
-	if database.Spec.NodeSets != nil {
-		stop, result, err = r.waitForDatabaseNodeSetsToReady(ctx, database)
-		if stop {
-			return result, err
-		}
-	} else {
-		stop, result, err = r.waitForStatefulSetToScale(ctx, database)
-		if stop {
-			return result, err
-		}
-	}
-
-	stop, result, err = r.handleTenantCreation(ctx, database)
+	stop, result, err = r.initializeDatabase(ctx, database)
 	if stop {
 		return result, err
 	}
diff --git a/internal/controllers/databasenodeset/controller_test.go b/internal/controllers/databasenodeset/controller_test.go
@@ -80,7 +80,7 @@ var _ = Describe("DatabaseNodeSet controller medium tests", func() {
 				Name:      storageSample.Name,
 				Namespace: testobjects.YdbNamespace,
 			}, &foundStorage))
-			return foundStorage.Status.State == StorageProvisioning
+			return foundStorage.Status.State == StorageInitializing
 		}, test.Timeout, test.Interval).Should(BeTrue())
 
 		By("set status Ready to Storage...")
@@ -111,7 +111,7 @@ var _ = Describe("DatabaseNodeSet controller medium tests", func() {
 				Name:      databaseSample.Name,
 				Namespace: testobjects.YdbNamespace,
 			}, &foundDatabase))
-			return foundDatabase.Status.State == DatabaseProvisioning
+			return foundDatabase.Status.State == DatabaseInitializing
 		}, test.Timeout, test.Interval).Should(BeTrue())
 
 		By("checking that DatabaseNodeSet created on local cluster...")
diff --git a/internal/controllers/remotedatabasenodeset/controller_test.go b/internal/controllers/remotedatabasenodeset/controller_test.go
@@ -257,7 +257,7 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() {
 				Name:      storageSample.Name,
 				Namespace: testobjects.YdbNamespace,
 			}, &foundStorage))
-			return foundStorage.Status.State == StorageProvisioning
+			return foundStorage.Status.State == StorageInitializing
 		}, test.Timeout, test.Interval).Should(BeTrue())
 
 		By("set status Ready to Storage...")
@@ -280,7 +280,7 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() {
 				Name:      databaseSample.Name,
 				Namespace: testobjects.YdbNamespace,
 			}, &foundDatabase))
-			return foundDatabase.Status.State == DatabaseProvisioning
+			return foundDatabase.Status.State == DatabaseInitializing
 		}, test.Timeout, test.Interval).Should(BeTrue())
 
 		By("checking that DatabaseNodeSet created on local cluster...")
diff --git a/internal/controllers/remotestoragenodeset/controller_test.go b/internal/controllers/remotestoragenodeset/controller_test.go
@@ -229,7 +229,7 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() {
 				Name:      storageSample.Name,
 				Namespace: testobjects.YdbNamespace,
 			}, &foundStorage))
-			return foundStorage.Status.State == StorageProvisioning
+			return foundStorage.Status.State == StorageInitializing
 		}, test.Timeout, test.Interval).Should(BeTrue())
 
 		By("checking that StorageNodeSet created on local cluster...")
diff --git a/internal/controllers/storage/init.go b/internal/controllers/storage/init.go
@@ -42,11 +42,14 @@ func (r *Reconciler) processSkipInitPipeline(
 		"Skipping initialization due to skip annotation present, be careful!",
 	)
 
-	return r.setInitStorageCompleted(
-		ctx,
-		storage,
-		"Storage initialization not performed because initialization is skipped",
-	)
+	meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
+		Type:    StorageInitializedCondition,
+		Status:  "True",
+		Reason:  ReasonCompleted,
+		Message: "Storage initialization not performed because initialization is skipped",
+	})
+	storage.Status.State = StorageReady
+	return r.updateStatus(ctx, storage)
 }
 
 func (r *Reconciler) setInitialStatus(
@@ -55,6 +58,17 @@ func (r *Reconciler) setInitialStatus(
 ) (bool, ctrl.Result, error) {
 	r.Log.Info("running step setInitialStatus")
 
+	if meta.IsStatusConditionTrue(storage.Status.Conditions, OldStorageInitializedCondition) {
+		meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
+			Type:    StorageInitializedCondition,
+			Status:  "True",
+			Reason:  ReasonCompleted,
+			Message: "Storage initialized successfully",
+		})
+		storage.Status.State = StorageReady
+		return r.updateStatus(ctx, storage)
+	}
+
 	// This block is special internal logic that skips all Storage initialization.
 	// It is needed when large clusters are migrated where `waitForStatefulSetToScale`
 	// does not make sense, since some nodes can be down for a long time (and it is okay, since
@@ -73,7 +87,7 @@ func (r *Reconciler) setInitialStatus(
 			Type:    StorageInitializedCondition,
 			Status:  "False",
 			Reason:  ReasonInProgress,
-			Message: "Storage is not ready yet",
+			Message: "Storage has not been initialized yet",
 		})
 		storage.Status.State = StoragePreparing
 		return r.updateStatus(ctx, storage)
@@ -92,8 +106,7 @@ func (r *Reconciler) setInitStorageCompleted(
 		Reason:  ReasonCompleted,
 		Message: message,
 	})
-
-	storage.Status.State = StorageReady
+	storage.Status.State = StorageProvisioning
 	return r.updateStatus(ctx, storage)
 }
 
@@ -103,7 +116,7 @@ func (r *Reconciler) initializeStorage(
 ) (bool, ctrl.Result, error) {
 	r.Log.Info("running step initializeStorage")
 
-	if storage.Status.State == StorageProvisioning {
+	if storage.Status.State == StoragePreparing {
 		storage.Status.State = StorageInitializing
 		return r.updateStatus(ctx, storage)
 	}
diff --git a/internal/controllers/storage/sync.go b/internal/controllers/storage/sync.go
