From b1ec1c238f72665909f724aba282cfcd8fd46e3b Mon Sep 17 00:00:00 2001 From: petruki <31597636+petruki@users.noreply.github.com> Date: Tue, 10 Sep 2024 19:49:33 -0700 Subject: [PATCH 1/3] Fixed snapshot version/content check before push to repo --- .../api/default_snapshot_version.json | 7 ++ src/core/api.go | 38 ++++++- src/core/api_test.go | 42 +++++++ src/core/handler.go | 41 +++++-- src/core/handler_test.go | 105 ++++++++++++++++-- src/utils/util.go | 3 +- 6 files changed, 213 insertions(+), 23 deletions(-) create mode 100644 resources/fixtures/api/default_snapshot_version.json diff --git a/resources/fixtures/api/default_snapshot_version.json b/resources/fixtures/api/default_snapshot_version.json new file mode 100644 index 0000000..708ee87 --- /dev/null +++ b/resources/fixtures/api/default_snapshot_version.json @@ -0,0 +1,7 @@ +{ + "data": { + "domain": { + "version": 1 + } + } +} \ No newline at end of file diff --git a/src/core/api.go b/src/core/api.go index 61f79f8..f389f57 100644 --- a/src/core/api.go +++ b/src/core/api.go @@ -22,6 +22,7 @@ type ApplyChangeResponse struct { } type IAPIService interface { + FetchSnapshotVersion(domainId string, environment string) (string, error) FetchSnapshot(domainId string, environment string) (string, error) ApplyChangesToAPI(domainId string, environment string, diff model.DiffResult) (ApplyChangeResponse, error) NewDataFromJson(jsonData []byte) model.Data @@ -45,6 +46,32 @@ func (c *ApiService) NewDataFromJson(jsonData []byte) model.Data { return data } +func (a *ApiService) FetchSnapshotVersion(domainId string, environment string) (string, error) { + // Generate a bearer token + token := generateBearerToken(a.ApiKey, domainId) + + // Define the GraphQL query + query := createQuerySnapshotVersion(domainId) + + // Create a new request + reqBody, _ := json.Marshal(GraphQLRequest{Query: query}) + req, _ := http.NewRequest("POST", a.ApiUrl+"/gitops-graphql", bytes.NewBuffer(reqBody)) + + // Set the request headers + setHeaders(req, token) + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + return string(body), nil +} + func (a *ApiService) FetchSnapshot(domainId string, environment string) (string, error) { // Generate a bearer token token := generateBearerToken(a.ApiKey, domainId) @@ -67,7 +94,6 @@ func (a *ApiService) FetchSnapshot(domainId string, environment string) (string, } defer resp.Body.Close() - // Read and print the response body, _ := io.ReadAll(resp.Body) return string(body), nil } @@ -91,7 +117,6 @@ func (a *ApiService) ApplyChangesToAPI(domainId string, environment string, diff } defer resp.Body.Close() - // Read and print the response body, _ := io.ReadAll(resp.Body) var response ApplyChangeResponse json.Unmarshal(body, &response) @@ -121,6 +146,15 @@ func setHeaders(req *http.Request, token string) { req.Header.Set("Authorization", "Bearer "+token) } +func createQuerySnapshotVersion(domainId string) string { + return fmt.Sprintf(` + { + domain(_id: "%s") { + version + } + }`, domainId) +} + func createQuery(domainId string, environment string) string { return fmt.Sprintf(` { diff --git a/src/core/api_test.go b/src/core/api_test.go index f073a55..2fe53c8 100644 --- a/src/core/api_test.go +++ b/src/core/api_test.go @@ -13,6 +13,48 @@ import ( const SWITCHER_API_JWT_SECRET = "SWITCHER_API_JWT_SECRET" +func TestFetchSnapshotVersion(t *testing.T) { + t.Run("Should return snapshot version", func(t *testing.T) { + responsePayload := utils.ReadJsonFromFile("../../resources/fixtures/api/default_snapshot_version.json") + fakeApiServer := givenApiResponse(http.StatusOK, responsePayload) + defer fakeApiServer.Close() + + apiService := NewApiService(SWITCHER_API_JWT_SECRET, fakeApiServer.URL) + version, _ := apiService.FetchSnapshotVersion("domainId", "default") + + assert.Contains(t, version, "version", "Missing version in response") + assert.Contains(t, version, "domain", "Missing domain in response") + }) + + t.Run("Should return error - invalid API key", func(t *testing.T) { + fakeApiServer := givenApiResponse(http.StatusUnauthorized, `{ "error": "Invalid API token" }`) + defer fakeApiServer.Close() + + apiService := NewApiService("INVALID_KEY", fakeApiServer.URL) + version, _ := apiService.FetchSnapshotVersion("domainId", "default") + + assert.Contains(t, version, "Invalid API token") + }) + + t.Run("Should return error - invalid domain", func(t *testing.T) { + responsePayload := utils.ReadJsonFromFile("../../resources/fixtures/api/error_invalid_domain.json") + fakeApiServer := givenApiResponse(http.StatusUnauthorized, responsePayload) + defer fakeApiServer.Close() + + apiService := NewApiService(SWITCHER_API_JWT_SECRET, fakeApiServer.URL) + version, _ := apiService.FetchSnapshotVersion("INVALID_DOMAIN", "default") + + assert.Contains(t, version, "errors") + }) + + t.Run("Should return error - invalid API URL", func(t *testing.T) { + apiService := NewApiService(config.GetEnv(SWITCHER_API_JWT_SECRET), "http://localhost:8080") + _, err := apiService.FetchSnapshotVersion("domainId", "default") + + assert.NotNil(t, err) + }) +} + func TestFetchSnapshot(t *testing.T) { t.Run("Should return snapshot", func(t *testing.T) { responsePayload := utils.ReadJsonFromFile("../../resources/fixtures/api/default_snapshot.json") diff --git a/src/core/handler.go b/src/core/handler.go index 76f0f87..cd9c977 100644 --- a/src/core/handler.go +++ b/src/core/handler.go @@ -89,11 +89,18 @@ func (c *CoreHandler) StartAccountHandler(accountId string, gitService IGitServi continue } - // Print account domain version - utils.Log(utils.LogLevelDebug, "[%s] Repository data: %s", accountId, utils.FormatJSON(repositoryData.Content)) + // Fetch snapshot version from API + snapshotVersionPayload, err := c.ApiService.FetchSnapshotVersion(account.Domain.ID, account.Environment) + + if err != nil { + utils.Log(utils.LogLevelError, "[%s] Failed to fetch snapshot version - %s", accountId, err.Error()) + c.updateDomainStatus(*account, model.StatusError, "Failed to fetch snapshot version - "+err.Error()) + time.Sleep(1 * time.Minute) + continue + } // Check if repository is out of sync - if isRepositoryOutSync(*account, repositoryData.CommitHash) { + if c.isRepositoryOutSync(*account, repositoryData.CommitHash, snapshotVersionPayload) { c.syncUp(*account, repositoryData, gitService) } @@ -126,7 +133,13 @@ func (c *CoreHandler) syncUp(account model.Account, repositoryData *model.Reposi changeSource := "" if snapshotApi.Domain.Version > account.Domain.Version { changeSource = "Repository" - account, err = c.applyChangesToRepository(account, snapshotApi, gitService) + if len(diff.Changes) > 0 { + account, err = c.applyChangesToRepository(account, snapshotApi, gitService) + } else { + utils.Log(utils.LogLevelInfo, "[%s] Repository is up to date", account.ID.Hex()) + account.Domain.Version = snapshotApi.Domain.Version + account.Domain.LastCommit = repositoryData.CommitHash + } } else if len(diff.Changes) > 0 { changeSource = "API" account = c.applyChangesToAPI(account, repositoryData) @@ -198,16 +211,15 @@ func (c *CoreHandler) applyChangesToRepository(account model.Account, snapshot m return account, err } -func isRepositoryOutSync(account model.Account, lastCommit string) bool { - utils.Log(utils.LogLevelDebug, "[%s] Checking account - Last commit: %s - Domain Version: %d", - account.ID.Hex(), account.Domain.LastCommit, account.Domain.Version) +func (c *CoreHandler) isRepositoryOutSync(account model.Account, lastCommit string, snapshotVersionPayload string) bool { + snapshotVersion := c.ApiService.NewDataFromJson([]byte(snapshotVersionPayload)).Snapshot.Domain.Version - return account.Domain.LastCommit == "" || account.Domain.LastCommit != lastCommit -} + utils.Log(utils.LogLevelDebug, "[%s] Checking account - Last commit: %s - Domain Version: %d - Snapshot Version: %d", + account.ID.Hex(), account.Domain.LastCommit, account.Domain.Version, snapshotVersion) -func getTimeWindow(window string) (int, time.Duration) { - duration, _ := time.ParseDuration(window) - return 1, duration + return account.Domain.LastCommit == "" || // First sync + account.Domain.LastCommit != lastCommit || // Repository out of sync + account.Domain.Version != snapshotVersion // API out of sync } func (c *CoreHandler) updateDomainStatus(account model.Account, status string, message string) { @@ -217,3 +229,8 @@ func (c *CoreHandler) updateDomainStatus(account model.Account, status string, m account.Domain.LastDate = time.Now().Format(time.ANSIC) c.AccountRepository.Update(&account) } + +func getTimeWindow(window string) (int, time.Duration) { + duration, _ := time.ParseDuration(window) + return 1, duration +} diff --git a/src/core/handler_test.go b/src/core/handler_test.go index 25fe4fe..5e61d94 100644 --- a/src/core/handler_test.go +++ b/src/core/handler_test.go @@ -107,6 +107,32 @@ func TestStartAccountHandler(t *testing.T) { tearDown() }) + t.Run("Should not sync when fetch snapshot version returns an error", func(t *testing.T) { + // Given + fakeGitService := NewFakeGitService() + fakeApiService := NewFakeApiService() + fakeApiService.throwErrorVersion = true + + coreHandler = NewCoreHandler(coreHandler.AccountRepository, fakeApiService, NewComparatorService()) + + account := givenAccount() + account.Domain.ID = "123-error-fetch-snapshot-version" + accountCreated, _ := coreHandler.AccountRepository.Create(&account) + + // Test + go coreHandler.StartAccountHandler(accountCreated.ID.Hex(), fakeGitService) + + time.Sleep(1 * time.Second) + + // Assert + accountFromDb, _ := coreHandler.AccountRepository.FetchByDomainId(accountCreated.Domain.ID) + assert.Equal(t, model.StatusError, accountFromDb.Domain.Status) + assert.Contains(t, accountFromDb.Domain.Message, "Failed to fetch snapshot version") + assert.Equal(t, "", accountFromDb.Domain.LastCommit) + + tearDown() + }) + t.Run("Should not sync after account is deleted", func(t *testing.T) { // Given fakeGitService := NewFakeGitService() @@ -159,6 +185,51 @@ func TestStartAccountHandler(t *testing.T) { tearDown() }) + t.Run("Should sync successfully when repository is up to date but not synced", func(t *testing.T) { + // Given + fakeGitService := NewFakeGitService() + fakeGitService.content = `{ + "domain": { + "group": [{ + "name": "Release 1", + "description": "Showcase configuration", + "activated": true, + "config": [{ + "key": "MY_SWITCHER", + "description": "", + "activated": true, + "strategies": [], + "components": [ + "switcher-playground" + ] + }] + }] + } + }` + fakeApiService := NewFakeApiService() + coreHandler = NewCoreHandler(coreHandler.AccountRepository, fakeApiService, NewComparatorService()) + + account := givenAccount() + account.Domain.ID = "123-up-to-date-not-synced" + accountCreated, _ := coreHandler.AccountRepository.Create(&account) + + // Test + go coreHandler.StartAccountHandler(accountCreated.ID.Hex(), fakeGitService) + + // Wait for goroutine to process + time.Sleep(1 * time.Second) + + // Assert + accountFromDb, _ := coreHandler.AccountRepository.FetchByDomainId(accountCreated.Domain.ID) + assert.Equal(t, model.StatusSynced, accountFromDb.Domain.Status) + assert.Contains(t, accountFromDb.Domain.Message, model.MessageSynced) + assert.Equal(t, "123", accountFromDb.Domain.LastCommit) + assert.Equal(t, 1, accountFromDb.Domain.Version) + assert.NotEqual(t, "", accountFromDb.Domain.LastDate) + + tearDown() + }) + t.Run("Should sync and prune successfully when repository is out of sync", func(t *testing.T) { // Given fakeGitService := NewFakeGitService() @@ -226,7 +297,7 @@ func TestStartAccountHandler(t *testing.T) { // Given fakeGitService := NewFakeGitService() fakeApiService := NewFakeApiService() - fakeApiService.throwError = true + fakeApiService.throwErrorSnapshot = true coreHandler = NewCoreHandler(coreHandler.AccountRepository, fakeApiService, NewComparatorService()) @@ -349,14 +420,24 @@ func (f *FakeGitService) UpdateRepositorySettings(repository string, token strin } type FakeApiService struct { - throwError bool - response string + throwErrorVersion bool + throwErrorSnapshot bool + responseVersion string + responseSnapshot string } func NewFakeApiService() *FakeApiService { return &FakeApiService{ - throwError: false, - response: `{ + throwErrorVersion: false, + throwErrorSnapshot: false, + responseVersion: `{ + "data": { + "domain": { + "version": 1 + } + } + }`, + responseSnapshot: `{ "data": { "domain": { "name": "Switcher GitOps", @@ -381,12 +462,20 @@ func NewFakeApiService() *FakeApiService { } } +func (f *FakeApiService) FetchSnapshotVersion(domainId string, environment string) (string, error) { + if f.throwErrorVersion { + return "", errors.New("Something went wrong") + } + + return f.responseVersion, nil +} + func (f *FakeApiService) FetchSnapshot(domainId string, environment string) (string, error) { - if f.throwError { - return "", assert.AnError + if f.throwErrorSnapshot { + return "", errors.New("Something went wrong") } - return f.response, nil + return f.responseSnapshot, nil } func (f *FakeApiService) ApplyChangesToAPI(domainId string, environment string, diff model.DiffResult) (ApplyChangeResponse, error) { diff --git a/src/utils/util.go b/src/utils/util.go index abcdc9e..dd3ed35 100644 --- a/src/utils/util.go +++ b/src/utils/util.go @@ -23,7 +23,8 @@ const ( func Log(logLevel string, message string, args ...interface{}) { currentLogLevel := config.GetEnv("LOG_LEVEL") - if logLevel == LogLevelDebug || logLevel == LogLevelError || currentLogLevel == logLevel { + if currentLogLevel == LogLevelDebug || currentLogLevel == LogLevelError || + currentLogLevel == logLevel || LogLevelError == logLevel { log.Printf("[%s] %s\n", logLevel, fmt.Sprintf(message, args...)) } } From c7778299231f5345e6600f5a81158ef986ef2c53 Mon Sep 17 00:00:00 2001 From: petruki <31597636+petruki@users.noreply.github.com> Date: Tue, 10 Sep 2024 21:08:38 -0700 Subject: [PATCH 2/3] Improved api module and handler logs --- src/core/api.go | 61 ++++++++++++++++++++++++--------------------- src/core/handler.go | 20 +++++++-------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/core/api.go b/src/core/api.go index f389f57..2d844ad 100644 --- a/src/core/api.go +++ b/src/core/api.go @@ -47,38 +47,44 @@ func (c *ApiService) NewDataFromJson(jsonData []byte) model.Data { } func (a *ApiService) FetchSnapshotVersion(domainId string, environment string) (string, error) { - // Generate a bearer token - token := generateBearerToken(a.ApiKey, domainId) - - // Define the GraphQL query query := createQuerySnapshotVersion(domainId) + responseBody, err := a.doGraphQLRequest(domainId, query) - // Create a new request - reqBody, _ := json.Marshal(GraphQLRequest{Query: query}) - req, _ := http.NewRequest("POST", a.ApiUrl+"/gitops-graphql", bytes.NewBuffer(reqBody)) + if err != nil { + return "", err + } - // Set the request headers - setHeaders(req, token) + return responseBody, nil +} + +func (a *ApiService) FetchSnapshot(domainId string, environment string) (string, error) { + query := createQuery(domainId, environment) + responseBody, err := a.doGraphQLRequest(domainId, query) - // Send the request - client := &http.Client{} - resp, err := client.Do(req) if err != nil { return "", err } - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - return string(body), nil + return responseBody, nil } -func (a *ApiService) FetchSnapshot(domainId string, environment string) (string, error) { +func (a *ApiService) ApplyChangesToAPI(domainId string, environment string, diff model.DiffResult) (ApplyChangeResponse, error) { + reqBody, _ := json.Marshal(diff) + responseBody, err := a.doPostRequest(a.ApiUrl+"/gitops/apply", domainId, reqBody) + + if err != nil { + return ApplyChangeResponse{}, err + } + + var response ApplyChangeResponse + json.Unmarshal([]byte(responseBody), &response) + return response, nil +} + +func (a *ApiService) doGraphQLRequest(domainId string, query string) (string, error) { // Generate a bearer token token := generateBearerToken(a.ApiKey, domainId) - // Define the GraphQL query - query := createQuery(domainId, environment) - // Create a new request reqBody, _ := json.Marshal(GraphQLRequest{Query: query}) req, _ := http.NewRequest("POST", a.ApiUrl+"/gitops-graphql", bytes.NewBuffer(reqBody)) @@ -94,17 +100,16 @@ func (a *ApiService) FetchSnapshot(domainId string, environment string) (string, } defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - return string(body), nil + responseBody, _ := io.ReadAll(resp.Body) + return string(responseBody), nil } -func (a *ApiService) ApplyChangesToAPI(domainId string, environment string, diff model.DiffResult) (ApplyChangeResponse, error) { +func (a *ApiService) doPostRequest(url string, domainId string, body []byte) (string, error) { // Generate a bearer token token := generateBearerToken(a.ApiKey, domainId) // Create a new request - reqBody, _ := json.Marshal(diff) - req, _ := http.NewRequest("POST", a.ApiUrl+"/gitops/apply", bytes.NewBuffer(reqBody)) + req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body)) // Set the request headers setHeaders(req, token) @@ -113,14 +118,12 @@ func (a *ApiService) ApplyChangesToAPI(domainId string, environment string, diff client := &http.Client{} resp, err := client.Do(req) if err != nil { - return ApplyChangeResponse{}, err + return "", err } defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - var response ApplyChangeResponse - json.Unmarshal(body, &response) - return response, nil + responseBody, _ := io.ReadAll(resp.Body) + return string(responseBody), nil } func generateBearerToken(apiKey string, subject string) string { diff --git a/src/core/handler.go b/src/core/handler.go index cd9c977..c7f354a 100644 --- a/src/core/handler.go +++ b/src/core/handler.go @@ -111,7 +111,7 @@ func (c *CoreHandler) StartAccountHandler(accountId string, gitService IGitServi } func (c *CoreHandler) syncUp(account model.Account, repositoryData *model.RepositoryData, gitService IGitService) { - utils.Log(utils.LogLevelInfo, "[%s] Syncing up", account.ID.Hex()) + utils.Log(utils.LogLevelInfo, "[%s - %s] Syncing up", account.ID.Hex(), account.Domain.Name) // Update account status: Out of sync account.Domain.LastCommit = repositoryData.CommitHash @@ -126,17 +126,17 @@ func (c *CoreHandler) syncUp(account model.Account, repositoryData *model.Reposi return } - utils.Log(utils.LogLevelDebug, "[%s] SnapshotAPI version: %s - SnapshotRepo version: %s", - account.ID.Hex(), fmt.Sprint(snapshotApi.Domain.Version), fmt.Sprint(account.Domain.Version)) + utils.Log(utils.LogLevelDebug, "[%s - %s] SnapshotAPI version: %s - SnapshotRepo version: %s", + account.ID.Hex(), account.Domain.Name, fmt.Sprint(snapshotApi.Domain.Version), fmt.Sprint(account.Domain.Version)) // Apply changes changeSource := "" if snapshotApi.Domain.Version > account.Domain.Version { changeSource = "Repository" - if len(diff.Changes) > 0 { + if account.Domain.Version == 0 || len(diff.Changes) > 0 { account, err = c.applyChangesToRepository(account, snapshotApi, gitService) } else { - utils.Log(utils.LogLevelInfo, "[%s] Repository is up to date", account.ID.Hex()) + utils.Log(utils.LogLevelInfo, "[%s - %s] Repository is up to date", account.ID.Hex(), account.Domain.Name) account.Domain.Version = snapshotApi.Domain.Version account.Domain.LastCommit = repositoryData.CommitHash } @@ -146,7 +146,7 @@ func (c *CoreHandler) syncUp(account model.Account, repositoryData *model.Reposi } if err != nil { - utils.Log(utils.LogLevelError, "[%s] Failed to apply changes [%s] - %s", account.ID.Hex(), changeSource, err.Error()) + utils.Log(utils.LogLevelError, "[%s - %s] Failed to apply changes [%s] - %s", account.ID.Hex(), account.Domain.Name, changeSource, err.Error()) c.updateDomainStatus(account, model.StatusError, "Failed to apply changes ["+changeSource+"] - "+err.Error()) return } @@ -184,7 +184,7 @@ func (c *CoreHandler) checkForChanges(account model.Account, content string) (mo } func (c *CoreHandler) applyChangesToAPI(account model.Account, repositoryData *model.RepositoryData) model.Account { - utils.Log(utils.LogLevelInfo, "[%s] Pushing changes to API", account.ID.Hex()) + utils.Log(utils.LogLevelInfo, "[%s - %s] Pushing changes to API", account.ID.Hex(), account.Domain.Name) // Push changes to API @@ -196,7 +196,7 @@ func (c *CoreHandler) applyChangesToAPI(account model.Account, repositoryData *m } func (c *CoreHandler) applyChangesToRepository(account model.Account, snapshot model.Snapshot, gitService IGitService) (model.Account, error) { - utils.Log(utils.LogLevelInfo, "[%s] Pushing changes to repository", account.ID.Hex()) + utils.Log(utils.LogLevelInfo, "[%s - %s] Pushing changes to repository", account.ID.Hex(), account.Domain.Name) // Remove version from domain snapshotContent := snapshot @@ -214,8 +214,8 @@ func (c *CoreHandler) applyChangesToRepository(account model.Account, snapshot m func (c *CoreHandler) isRepositoryOutSync(account model.Account, lastCommit string, snapshotVersionPayload string) bool { snapshotVersion := c.ApiService.NewDataFromJson([]byte(snapshotVersionPayload)).Snapshot.Domain.Version - utils.Log(utils.LogLevelDebug, "[%s] Checking account - Last commit: %s - Domain Version: %d - Snapshot Version: %d", - account.ID.Hex(), account.Domain.LastCommit, account.Domain.Version, snapshotVersion) + utils.Log(utils.LogLevelDebug, "[%s - %s] Checking account - Last commit: %s - Domain Version: %d - Snapshot Version: %d", + account.ID.Hex(), account.Domain.Name, account.Domain.LastCommit, account.Domain.Version, snapshotVersion) return account.Domain.LastCommit == "" || // First sync account.Domain.LastCommit != lastCommit || // Repository out of sync From a1cfb224116eebe019be72fc14d0704aa5250f29 Mon Sep 17 00:00:00 2001 From: petruki <31597636+petruki@users.noreply.github.com> Date: Tue, 10 Sep 2024 21:17:14 -0700 Subject: [PATCH 3/3] Reverted syncUp check for newly created accounts --- src/core/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/handler.go b/src/core/handler.go index c7f354a..f173dac 100644 --- a/src/core/handler.go +++ b/src/core/handler.go @@ -133,7 +133,7 @@ func (c *CoreHandler) syncUp(account model.Account, repositoryData *model.Reposi changeSource := "" if snapshotApi.Domain.Version > account.Domain.Version { changeSource = "Repository" - if account.Domain.Version == 0 || len(diff.Changes) > 0 { + if len(diff.Changes) > 0 { account, err = c.applyChangesToRepository(account, snapshotApi, gitService) } else { utils.Log(utils.LogLevelInfo, "[%s - %s] Repository is up to date", account.ID.Hex(), account.Domain.Name)