diff --git a/.github/gh-config-template/README.md b/.github/gh-config-template/README.md new file mode 100644 index 000000000..42efd0e27 --- /dev/null +++ b/.github/gh-config-template/README.md @@ -0,0 +1,8 @@ +# Generate github actions from template + +ytt -f ./gh_template.yml -f [ytt-helpers.star](https://github.com/cloudfoundry/wg-app-platform-runtime-ci/blob/main/shared/helpers/ytt-helpers.star) -f [index.yml](https://github.com/cloudfoundry/wg-app-platform-runtime-ci/blob/main/routing-release/index.yml) > ./workflows/tests-workflow.yml + +## Supported jobs +- Template tests +- Basic Verifications +- Unit and Integration tests \ No newline at end of file diff --git a/.github/gh-config-template/gh_template.yml b/.github/gh-config-template/gh_template.yml new file mode 100644 index 000000000..11c537f56 --- /dev/null +++ b/.github/gh-config-template/gh_template.yml @@ -0,0 +1,179 @@ +#@ load("@ytt:data", "data") +#@ load("ytt-helpers.star", "helpers") +# test commit +name: unit-integration-tests + +on: + pull_request: + branches: + - test-gh + types: + - opened + - reopened + - synchronize + +env: + MAPPING: | + build_nats_server=src/code.cloudfoundry.org/vendor/github.com/nats-io/nats-server/v2 + build_routing_api_cli=src/code.cloudfoundry.org/routing-api-cli + FLAGS: | + --keep-going + --trace + -r + --fail-on-pending + --randomize-all + --nodes=7 + --race + --timeout 30m + --flake-attempts 2 + RUN_AS: root + VERIFICATIONS: | + verify_go repo/$DIR + verify_go_version_match_bosh_release repo + verify_gofmt repo/$DIR + verify_govet repo/$DIR + verify_staticcheck repo/$DIR + FUNCTIONS: ci/routing-release/helpers/configure-binaries.bash + +jobs: + repo-clone: + runs-on: ubuntu-latest + steps: + - name: routing-release-repo + uses: actions/checkout@v4 + with: + repository: cloudfoundry/routing-release.git + ref: github-action + submodules: recursive + path: repo + - name: Check out wg-appruntime code + uses: actions/checkout@v4 + with: + repository: cloudfoundry/wg-app-platform-runtime-ci + path: ci + - name: zip repo artifacts + run: | + tar -czf repo-artifact.tar.gz repo + tar -czf ci-artifact.tar.gz ci + - name: upload artifact + uses: actions/upload-artifact@v4 + with: + name: repo + path: | + repo-artifact.tar.gz + ci-artifact.tar.gz + template-tests: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: cloudfoundry/tas-runtime-mysql-5.7 + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: "tar -xzvf repo-artifact.tar.gz\ntar -xzvf ci-artifact.tar.gz\n" + - name: template-tests + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-tests-templates/task.bash + test-on-mysql-5-7: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: cloudfoundry/tas-runtime-mysql-5.7 + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: build binaries + run: | + export DEFAULT_PARAMS="${GITHUB_WORKSPACE}/ci/routing-release/default-params/build-binaries/linux.yml" + "${GITHUB_WORKSPACE}"/ci/shared/tasks/build-binaries/task.bash +#@ for package in helpers.packages_with_configure_db(data.values.internal_repos): + - name: #@ "{}-mysql".format(package.name) + env: + DIR: #@ "src/code.cloudfoundry.org/{}".format(package.name) + DB: mysql + run: | + "${GITHUB_WORKSPACE}"/ci/routing-release/helpers/configure-binaries.bash + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 +#@ end + test-repos-withoutdb: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: cloudfoundry/tas-runtime-build + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: build binaries + run: | + export DEFAULT_PARAMS="${GITHUB_WORKSPACE}/ci/routing-release/default-params/build-binaries/linux.yml" + "${GITHUB_WORKSPACE}"/ci/shared/tasks/build-binaries/task.bash +#@ for package in helpers.packages_without_configure_db(data.values.internal_repos): + - name: #@ package.name + env: + DIR: #@ "src/code.cloudfoundry.org/{}".format(package.name) + run: | + export DIR=$DIR + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 +#@ end + test-on-postgres: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: cloudfoundry/tas-runtime-postgres + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: build binaries + run: | + export DEFAULT_PARAMS="${GITHUB_WORKSPACE}/ci/routing-release/default-params/build-binaries/linux.yml" + "${GITHUB_WORKSPACE}"/ci/shared/tasks/build-binaries/task.bash +#@ for package in helpers.packages_with_configure_db(data.values.internal_repos): + - name: #@ "{}-mysql".format(package.name) + env: + DIR: #@ "src/code.cloudfoundry.org/{}".format(package.name) + DB: postgres + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 +#@ end + test-on-mysql-8-0: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: cloudfoundry/tas-runtime-mysql-8.0 + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: build binaries + run: | + export DEFAULT_PARAMS="${GITHUB_WORKSPACE}/ci/routing-release/default-params/build-binaries/linux.yml" + "${GITHUB_WORKSPACE}"/ci/shared/tasks/build-binaries/task.bash +#@ for package in helpers.packages_with_configure_db(data.values.internal_repos): + - name: #@ "{}-mysql".format(package.name) + env: + DIR: #@ "src/code.cloudfoundry.org/{}".format(package.name) + DB: mysql + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 +#@ end \ No newline at end of file diff --git a/.github/workflows/tests-workflow.yml b/.github/workflows/tests-workflow.yml new file mode 100644 index 000000000..2d177a504 --- /dev/null +++ b/.github/workflows/tests-workflow.yml @@ -0,0 +1,217 @@ +name: unit-integration-tests +on: + push: + branches: + - develop + pull_request: + +env: + MAPPING: | + build_nats_server=src/code.cloudfoundry.org/vendor/github.com/nats-io/nats-server/v2 + build_routing_api_cli=src/code.cloudfoundry.org/routing-api-cli + FLAGS: | + --keep-going + --trace + -r + --fail-on-pending + --randomize-all + --nodes=7 + --race + --timeout 30m + --flake-attempts 2 + RUN_AS: root + VERIFICATIONS: | + verify_go repo/$DIR + verify_go_version_match_bosh_release repo + verify_gofmt repo/$DIR + verify_govet repo/$DIR + verify_staticcheck repo/$DIR + FUNCTIONS: ci/routing-release/helpers/configure-binaries.bash +jobs: + repo-clone: + runs-on: ubuntu-latest + steps: + - name: routing-release-repo + uses: actions/checkout@v4 + with: + repository: cloudfoundry/routing-release.git + ref: develop + submodules: recursive + path: repo + - name: Check out wg-appruntime code + uses: actions/checkout@v4 + with: + repository: cloudfoundry/wg-app-platform-runtime-ci + path: ci + - name: zip repo artifacts + run: | + tar -czf repo-artifact.tar.gz repo + tar -czf ci-artifact.tar.gz ci + - name: upload artifact + uses: actions/upload-artifact@v4 + with: + name: repo + path: | + repo-artifact.tar.gz + ci-artifact.tar.gz + template-tests: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: us-central1-docker.pkg.dev/cf-diego-pivotal/tas-runtime-dockerhub-mirror/cloudfoundry/tas-runtime-build + credentials: + username: _json_key + password: ${{ secrets.GCP_SERVICE_ACCOUNT_TAS_RUNTIME_BUILD_IMAGE_READER }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: "tar -xzvf repo-artifact.tar.gz\ntar -xzvf ci-artifact.tar.gz\n" + - name: template-tests + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-tests-templates/task.bash + test-on-mysql-5-7: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: us-central1-docker.pkg.dev/cf-diego-pivotal/tas-runtime-dockerhub-mirror/cloudfoundry/tas-runtime-mysql-5.7 + credentials: + username: _json_key + password: ${{ secrets.GCP_SERVICE_ACCOUNT_TAS_RUNTIME_BUILD_IMAGE_READER }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: "tar -xzvf repo-artifact.tar.gz\ntar -xzvf ci-artifact.tar.gz\n" + - name: build binaries + run: | + export DEFAULT_PARAMS="${GITHUB_WORKSPACE}/ci/routing-release/default-params/build-binaries/linux.yml" + "${GITHUB_WORKSPACE}"/ci/shared/tasks/build-binaries/task.bash + - name: gorouter-mysql + env: + DIR: src/code.cloudfoundry.org/gorouter + DB: mysql + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + - name: cf-tcp-router-mysql + env: + DIR: src/code.cloudfoundry.org/cf-tcp-router + DB: mysql + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + - name: routing-api-mysql + env: + DIR: src/code.cloudfoundry.org/routing-api + DB: mysql + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + test-repos-withoutdb: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: us-central1-docker.pkg.dev/cf-diego-pivotal/tas-runtime-dockerhub-mirror/cloudfoundry/tas-runtime-build + credentials: + username: _json_key + password: ${{ secrets.GCP_SERVICE_ACCOUNT_TAS_RUNTIME_BUILD_IMAGE_READER }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: "tar -xzvf repo-artifact.tar.gz\ntar -xzvf ci-artifact.tar.gz\n" + - name: build binaries + run: | + export DEFAULT_PARAMS="${GITHUB_WORKSPACE}/ci/routing-release/default-params/build-binaries/linux.yml" + "${GITHUB_WORKSPACE}"/ci/shared/tasks/build-binaries/task.bash + - name: multierror + env: + DIR: src/code.cloudfoundry.org/multierror + run: | + export DIR=$DIR + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + - name: route-registrar + env: + DIR: src/code.cloudfoundry.org/route-registrar + run: | + export DIR=$DIR + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + - name: routing-api-cli + env: + DIR: src/code.cloudfoundry.org/routing-api-cli + run: | + export DIR=$DIR + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + test-on-postgres: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: us-central1-docker.pkg.dev/cf-diego-pivotal/tas-runtime-dockerhub-mirror/cloudfoundry/tas-runtime-postgres + credentials: + username: _json_key + password: ${{ secrets.GCP_SERVICE_ACCOUNT_TAS_RUNTIME_BUILD_IMAGE_READER }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: "tar -xzvf repo-artifact.tar.gz\ntar -xzvf ci-artifact.tar.gz\n" + - name: build binaries + run: | + export DEFAULT_PARAMS="${GITHUB_WORKSPACE}/ci/routing-release/default-params/build-binaries/linux.yml" + "${GITHUB_WORKSPACE}"/ci/shared/tasks/build-binaries/task.bash + - name: gorouter-mysql + env: + DIR: src/code.cloudfoundry.org/gorouter + DB: postgres + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + - name: cf-tcp-router-mysql + env: + DIR: src/code.cloudfoundry.org/cf-tcp-router + DB: postgres + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + - name: routing-api-mysql + env: + DIR: src/code.cloudfoundry.org/routing-api + DB: postgres + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + test-on-mysql-8-0: + runs-on: ubuntu-latest + needs: repo-clone + container: + image: us-central1-docker.pkg.dev/cf-diego-pivotal/tas-runtime-dockerhub-mirror/cloudfoundry/tas-runtime-mysql-8.0 + credentials: + username: _json_key + password: ${{ secrets.GCP_SERVICE_ACCOUNT_TAS_RUNTIME_BUILD_IMAGE_READER }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: "tar -xzvf repo-artifact.tar.gz\ntar -xzvf ci-artifact.tar.gz\n" + - name: build binaries + run: | + export DEFAULT_PARAMS="${GITHUB_WORKSPACE}/ci/routing-release/default-params/build-binaries/linux.yml" + "${GITHUB_WORKSPACE}"/ci/shared/tasks/build-binaries/task.bash + - name: gorouter-mysql + env: + DIR: src/code.cloudfoundry.org/gorouter + DB: mysql + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + - name: cf-tcp-router-mysql + env: + DIR: src/code.cloudfoundry.org/cf-tcp-router + DB: mysql + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 + - name: routing-api-mysql + env: + DIR: src/code.cloudfoundry.org/routing-api + DB: mysql + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 diff --git a/src/code.cloudfoundry.org/gorouter/handlers/routeservice_test.go b/src/code.cloudfoundry.org/gorouter/handlers/routeservice_test.go index 7be3af618..7caa4cc57 100644 --- a/src/code.cloudfoundry.org/gorouter/handlers/routeservice_test.go +++ b/src/code.cloudfoundry.org/gorouter/handlers/routeservice_test.go @@ -129,7 +129,7 @@ var _ = Describe("Route Service Handler", func() { endpoint := route.NewEndpoint(&route.EndpointOpts{}) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) }) It("should not add route service metadata to the request for normal routes", func() { handler.ServeHTTP(resp, req) @@ -152,7 +152,7 @@ var _ = Describe("Route Service Handler", func() { BeforeEach(func() { endpoint := route.NewEndpoint(&route.EndpointOpts{RouteServiceUrl: "route-service.com"}) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) }) It("returns 502 Bad Gateway", func() { @@ -172,7 +172,7 @@ var _ = Describe("Route Service Handler", func() { BeforeEach(func() { endpoint := route.NewEndpoint(&route.EndpointOpts{}) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) }) It("should not add route service metadata to the request for normal routes", func() { handler.ServeHTTP(resp, req) @@ -211,7 +211,7 @@ var _ = Describe("Route Service Handler", func() { BeforeEach(func() { endpoint := route.NewEndpoint(&route.EndpointOpts{RouteServiceUrl: "https://route-service.com"}) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) }) It("sends the request to the route service with X-CF-Forwarded-Url using https scheme", func() { @@ -699,7 +699,7 @@ var _ = Describe("Route Service Handler", func() { endpoint := route.NewEndpoint(&route.EndpointOpts{RouteServiceUrl: "https://goodrouteservice.com"}) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) req.Header.Set("connection", "upgrade") req.Header.Set("upgrade", "websocket") @@ -718,7 +718,7 @@ var _ = Describe("Route Service Handler", func() { BeforeEach(func() { endpoint := route.NewEndpoint(&route.EndpointOpts{RouteServiceUrl: "https://bad%20service.com"}) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) }) It("returns a 500 internal server error response", func() { diff --git a/src/code.cloudfoundry.org/gorouter/metrics/compositereporter.go b/src/code.cloudfoundry.org/gorouter/metrics/compositereporter.go index 6b2344272..290d08219 100644 --- a/src/code.cloudfoundry.org/gorouter/metrics/compositereporter.go +++ b/src/code.cloudfoundry.org/gorouter/metrics/compositereporter.go @@ -39,6 +39,8 @@ type MetricReporter interface { CaptureRoutesPruned(prunedRoutes uint64) CaptureLookupTime(t time.Duration) CaptureRegistryMessage(msg ComponentTagged, action string) + CaptureRoutesRegistered() + CaptureRoutesUnregistered() CaptureRouteRegistrationLatency(t time.Duration) CaptureUnregistryMessage(msg ComponentTagged) CaptureFoundFileDescriptors(files int) @@ -198,6 +200,18 @@ func (m MultiMetricReporter) CaptureUnregistryMessage(msg ComponentTagged) { } } +func (m MultiMetricReporter) CaptureRoutesRegistered() { + for _, r := range m { + r.CaptureRoutesRegistered() + } +} + +func (m MultiMetricReporter) CaptureRoutesUnregistered() { + for _, r := range m { + r.CaptureRoutesUnregistered() + } +} + func (m MultiMetricReporter) CaptureFoundFileDescriptors(files int) { for _, r := range m { r.CaptureFoundFileDescriptors(files) diff --git a/src/code.cloudfoundry.org/gorouter/metrics/fakes/fake_metricreporter.go b/src/code.cloudfoundry.org/gorouter/metrics/fakes/fake_metricreporter.go index 54e624998..ceb391e80 100644 --- a/src/code.cloudfoundry.org/gorouter/metrics/fakes/fake_metricreporter.go +++ b/src/code.cloudfoundry.org/gorouter/metrics/fakes/fake_metricreporter.go @@ -97,6 +97,14 @@ type FakeMetricReporter struct { captureRoutesPrunedArgsForCall []struct { arg1 uint64 } + CaptureRoutesRegisteredStub func() + captureRoutesRegisteredMutex sync.RWMutex + captureRoutesRegisteredArgsForCall []struct { + } + CaptureRoutesUnregisteredStub func() + captureRoutesUnregisteredMutex sync.RWMutex + captureRoutesUnregisteredArgsForCall []struct { + } CaptureRoutingRequestStub func(*route.Endpoint) captureRoutingRequestMutex sync.RWMutex captureRoutingRequestArgsForCall []struct { @@ -659,6 +667,54 @@ func (fake *FakeMetricReporter) CaptureRoutesPrunedArgsForCall(i int) uint64 { return argsForCall.arg1 } +func (fake *FakeMetricReporter) CaptureRoutesRegistered() { + fake.captureRoutesRegisteredMutex.Lock() + fake.captureRoutesRegisteredArgsForCall = append(fake.captureRoutesRegisteredArgsForCall, struct { + }{}) + stub := fake.CaptureRoutesRegisteredStub + fake.recordInvocation("CaptureRoutesRegistered", []interface{}{}) + fake.captureRoutesRegisteredMutex.Unlock() + if stub != nil { + fake.CaptureRoutesRegisteredStub() + } +} + +func (fake *FakeMetricReporter) CaptureRoutesRegisteredCallCount() int { + fake.captureRoutesRegisteredMutex.RLock() + defer fake.captureRoutesRegisteredMutex.RUnlock() + return len(fake.captureRoutesRegisteredArgsForCall) +} + +func (fake *FakeMetricReporter) CaptureRoutesRegisteredCalls(stub func()) { + fake.captureRoutesRegisteredMutex.Lock() + defer fake.captureRoutesRegisteredMutex.Unlock() + fake.CaptureRoutesRegisteredStub = stub +} + +func (fake *FakeMetricReporter) CaptureRoutesUnregistered() { + fake.captureRoutesUnregisteredMutex.Lock() + fake.captureRoutesUnregisteredArgsForCall = append(fake.captureRoutesUnregisteredArgsForCall, struct { + }{}) + stub := fake.CaptureRoutesUnregisteredStub + fake.recordInvocation("CaptureRoutesUnregistered", []interface{}{}) + fake.captureRoutesUnregisteredMutex.Unlock() + if stub != nil { + fake.CaptureRoutesUnregisteredStub() + } +} + +func (fake *FakeMetricReporter) CaptureRoutesUnregisteredCallCount() int { + fake.captureRoutesUnregisteredMutex.RLock() + defer fake.captureRoutesUnregisteredMutex.RUnlock() + return len(fake.captureRoutesUnregisteredArgsForCall) +} + +func (fake *FakeMetricReporter) CaptureRoutesUnregisteredCalls(stub func()) { + fake.captureRoutesUnregisteredMutex.Lock() + defer fake.captureRoutesUnregisteredMutex.Unlock() + fake.CaptureRoutesUnregisteredStub = stub +} + func (fake *FakeMetricReporter) CaptureRoutingRequest(arg1 *route.Endpoint) { fake.captureRoutingRequestMutex.Lock() fake.captureRoutingRequestArgsForCall = append(fake.captureRoutingRequestArgsForCall, struct { @@ -901,6 +957,10 @@ func (fake *FakeMetricReporter) Invocations() map[string][][]interface{} { defer fake.captureRouteStatsMutex.RUnlock() fake.captureRoutesPrunedMutex.RLock() defer fake.captureRoutesPrunedMutex.RUnlock() + fake.captureRoutesRegisteredMutex.RLock() + defer fake.captureRoutesRegisteredMutex.RUnlock() + fake.captureRoutesUnregisteredMutex.RLock() + defer fake.captureRoutesUnregisteredMutex.RUnlock() fake.captureRoutingRequestMutex.RLock() defer fake.captureRoutingRequestMutex.RUnlock() fake.captureRoutingResponseMutex.RLock() diff --git a/src/code.cloudfoundry.org/gorouter/metrics/metricsreporter.go b/src/code.cloudfoundry.org/gorouter/metrics/metricsreporter.go index db46b85fc..c406b2065 100644 --- a/src/code.cloudfoundry.org/gorouter/metrics/metricsreporter.go +++ b/src/code.cloudfoundry.org/gorouter/metrics/metricsreporter.go @@ -168,6 +168,14 @@ func (m *Metrics) CaptureUnregistryMessage(msg ComponentTagged) { } } +func (m *Metrics) CaptureRoutesRegistered() { + m.Batcher.BatchIncrementCounter("routes_registered") +} + +func (m *Metrics) CaptureRoutesUnregistered() { + m.Batcher.BatchIncrementCounter("routes_unregistered") +} + func (m *Metrics) CaptureWebSocketUpdate() { m.Batcher.BatchIncrementCounter("websocket_upgrades") } diff --git a/src/code.cloudfoundry.org/gorouter/metrics/metricsreporter_test.go b/src/code.cloudfoundry.org/gorouter/metrics/metricsreporter_test.go index 7d07adf15..b88432fd6 100644 --- a/src/code.cloudfoundry.org/gorouter/metrics/metricsreporter_test.go +++ b/src/code.cloudfoundry.org/gorouter/metrics/metricsreporter_test.go @@ -467,7 +467,7 @@ var _ = Describe("MetricsReporter", func() { It("sends number of nats messages received from each component", func() { endpoint.Tags = map[string]string{} - metricReporter.CaptureRegistryMessage(endpoint, route.ADDED.String()) + metricReporter.CaptureRegistryMessage(endpoint, route.EndpointAdded.String()) Expect(batcher.BatchIncrementCounterCallCount()).To(Equal(1)) Expect(batcher.BatchIncrementCounterArgsForCall(0)).To(Equal("registry_message")) @@ -475,10 +475,10 @@ var _ = Describe("MetricsReporter", func() { It("sends number of nats messages received from each component", func() { endpoint.Tags = map[string]string{"component": "uaa"} - metricReporter.CaptureRegistryMessage(endpoint, route.ADDED.String()) + metricReporter.CaptureRegistryMessage(endpoint, route.EndpointAdded.String()) endpoint.Tags = map[string]string{"component": "route-emitter"} - metricReporter.CaptureRegistryMessage(endpoint, route.ADDED.String()) + metricReporter.CaptureRegistryMessage(endpoint, route.EndpointAdded.String()) Expect(batcher.BatchIncrementCounterCallCount()).To(Equal(2)) Expect(batcher.BatchIncrementCounterArgsForCall(0)).To(Equal("registry_message.uaa")) @@ -524,6 +524,18 @@ var _ = Describe("MetricsReporter", func() { Expect(count).To(Equal(uint64(5))) }) + It("increments the routes_registered metric", func() { + metricReporter.CaptureRoutesRegistered() + Expect(batcher.BatchIncrementCounterCallCount()).To(Equal(1)) + Expect(batcher.BatchIncrementCounterArgsForCall(0)).To(Equal("routes_registered")) + }) + + It("increments the routes_unregistered metric", func() { + metricReporter.CaptureRoutesUnregistered() + Expect(batcher.BatchIncrementCounterCallCount()).To(Equal(1)) + Expect(batcher.BatchIncrementCounterArgsForCall(0)).To(Equal("routes_unregistered")) + }) + It("increments the backend_tls_handshake_failed metric", func() { metricReporter.CaptureBackendTLSHandshakeFailed() Expect(batcher.BatchIncrementCounterCallCount()).To(Equal(1)) diff --git a/src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics.go b/src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics.go index 0ed12a78d..cab0c8705 100644 --- a/src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics.go +++ b/src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics.go @@ -18,6 +18,8 @@ type Metrics struct { RouteRegistration mr.CounterVec RouteUnregistration mr.CounterVec RoutesPruned mr.Counter + RoutesRegistered mr.Counter + RoutesUnregistered mr.Counter TotalRoutes mr.Gauge TimeSinceLastRegistryUpdate mr.Gauge RouteLookupTime mr.Histogram @@ -64,6 +66,8 @@ func NewMetrics(registry *mr.Registry, perRequestMetricsReporting bool, meterCon RouteRegistration: registry.NewCounterVec("registry_message", "number of route registration messages", []string{"component", "action"}), RouteUnregistration: registry.NewCounterVec("unregistry_message", "number of unregister messages", []string{"component"}), RoutesPruned: registry.NewCounter("routes_pruned", "number of pruned routes"), + RoutesRegistered: registry.NewCounter("routes_registered", "number of registered routes"), + RoutesUnregistered: registry.NewCounter("routes_unregistered", "number of unregistered routes"), TotalRoutes: registry.NewGauge("total_routes", "number of total routes"), TimeSinceLastRegistryUpdate: registry.NewGauge("ms_since_last_registry_update", "time since last registry update in ms"), RouteLookupTime: registry.NewHistogram("route_lookup_time", "route lookup time per request in ns", meterConfig.RouteLookupTimeHistogramBuckets), @@ -107,6 +111,13 @@ func (metrics *Metrics) CaptureRoutesPruned(routesPruned uint64) { metrics.RoutesPruned.Add(float64(routesPruned)) } +func (metrics *Metrics) CaptureRoutesRegistered() { + metrics.RoutesRegistered.Add(1) +} + +func (metrics *Metrics) CaptureRoutesUnregistered() { + metrics.RoutesUnregistered.Add(1) +} func (metrics *Metrics) CaptureTotalRoutes(totalRoutes int) { metrics.TotalRoutes.Set(float64(totalRoutes)) } diff --git a/src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics_test.go b/src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics_test.go index 6e25a9245..d0831c89f 100644 --- a/src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics_test.go +++ b/src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics_test.go @@ -35,12 +35,12 @@ var _ = Describe("Metrics", func() { It("sends number of nats messages received from each component", func() { endpoint.Tags = map[string]string{} - m.CaptureRegistryMessage(endpoint, route.UPDATED.String()) - expected := fmt.Sprintf("registry_message{action=\"%s\",component=\"\"} 1", route.UPDATED.String()) + m.CaptureRegistryMessage(endpoint, route.EndpointUpdated.String()) + expected := fmt.Sprintf("registry_message{action=\"%s\",component=\"\"} 1", route.EndpointUpdated.String()) Expect(getMetrics(r.Port())).To(ContainSubstring(expected)) - m.CaptureRegistryMessage(endpoint, route.UPDATED.String()) - expected = fmt.Sprintf("registry_message{action=\"%s\",component=\"\"} 2", route.UPDATED.String()) + m.CaptureRegistryMessage(endpoint, route.EndpointUpdated.String()) + expected = fmt.Sprintf("registry_message{action=\"%s\",component=\"\"} 2", route.EndpointUpdated.String()) Expect(getMetrics(r.Port())).To(ContainSubstring(expected)) }) @@ -99,6 +99,15 @@ var _ = Describe("Metrics", func() { Expect(getMetrics(r.Port())).To(ContainSubstring(`routes_pruned 50`)) }) + It("increments the routes registered metric", func() { + m.CaptureRoutesRegistered() + Expect(getMetrics(r.Port())).To(ContainSubstring(`routes_registered 1`)) + }) + It("increments the routes unregistered metric", func() { + m.CaptureRoutesUnregistered() + Expect(getMetrics(r.Port())).To(ContainSubstring(`routes_unregistered 1`)) + }) + Describe("captures route registration latency", func() { It("properly splits the latencies apart", func() { m.CaptureRouteRegistrationLatency(1234 * time.Microsecond) diff --git a/src/code.cloudfoundry.org/gorouter/proxy/round_tripper/proxy_round_tripper_test.go b/src/code.cloudfoundry.org/gorouter/proxy/round_tripper/proxy_round_tripper_test.go index 6be15d018..9d270867c 100644 --- a/src/code.cloudfoundry.org/gorouter/proxy/round_tripper/proxy_round_tripper_test.go +++ b/src/code.cloudfoundry.org/gorouter/proxy/round_tripper/proxy_round_tripper_test.go @@ -160,7 +160,7 @@ var _ = Describe("ProxyRoundTripper", func() { }) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) } proxyRoundTripper = round_tripper.NewProxyRoundTripper( @@ -418,11 +418,11 @@ var _ = Describe("ProxyRoundTripper", func() { if transport.RoundTripCallCount() == 1 { endpoint := endpointFor(4) updated := routePool.Put(endpoint) - Expect(updated).To(Equal(route.UPDATED)) + Expect(updated).To(Equal(route.EndpointRefreshed)) endpoint = endpointFor(5) updated = routePool.Put(endpoint) - Expect(updated).To(Equal(route.UPDATED)) + Expect(updated).To(Equal(route.EndpointRefreshed)) } return nil, &net.OpError{Op: "dial", Err: errors.New("connection refused")} @@ -445,7 +445,7 @@ var _ = Describe("ProxyRoundTripper", func() { for i := 6; i <= 7; i++ { endpoint := endpointFor(i) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) } } @@ -467,7 +467,7 @@ var _ = Describe("ProxyRoundTripper", func() { transport.RoundTripStub = func(*http.Request) (*http.Response, error) { if transport.RoundTripCallCount() == 2 { added := routePool.Put(endpointFor(6)) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) removed := routePool.Remove(endpointFor(2)) Expect(removed).To(BeTrue()) @@ -495,7 +495,7 @@ var _ = Describe("ProxyRoundTripper", func() { transport.RoundTripStub = func(*http.Request) (*http.Response, error) { if transport.RoundTripCallCount() == 5 { added := routePool.Put(endpointFor(6)) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) removed := routePool.Remove(endpointFor(2)) Expect(removed).To(BeTrue()) @@ -603,7 +603,7 @@ var _ = Describe("ProxyRoundTripper", func() { }) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) _, err := proxyRoundTripper.RoundTrip(req) Expect(err).To(MatchError(ContainSubstring("tls: handshake failure"))) @@ -797,14 +797,14 @@ var _ = Describe("ProxyRoundTripper", func() { Port: 20222, UseTLS: true, }) - Expect(routePool.Put(tlsEndpoint)).To(Equal(route.ADDED)) + Expect(routePool.Put(tlsEndpoint)).To(Equal(route.EndpointAdded)) nonTLSEndpoint := route.NewEndpoint(&route.EndpointOpts{ Host: "3.3.3.3", Port: 30333, UseTLS: false, }) - Expect(routePool.Put(nonTLSEndpoint)).To(Equal(route.ADDED)) + Expect(routePool.Put(nonTLSEndpoint)).To(Equal(route.EndpointAdded)) }) Context("when retrying different backends", func() { @@ -858,7 +858,7 @@ var _ = Describe("ProxyRoundTripper", func() { }) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) transport.RoundTripReturns( &http.Response{StatusCode: http.StatusTeapot}, nil, ) @@ -881,7 +881,7 @@ var _ = Describe("ProxyRoundTripper", func() { }) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.UPDATED)) + Expect(added).To(Equal(route.EndpointUpdated)) transport.RoundTripReturns( &http.Response{StatusCode: http.StatusTeapot}, nil, ) @@ -918,7 +918,7 @@ var _ = Describe("ProxyRoundTripper", func() { Host: "1.1.1.1", Port: 9091, UseTLS: true, PrivateInstanceId: "instanceId-2", }) added := routePool.Put(endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) _, err := proxyRoundTripper.RoundTrip(req) Expect(err).ToNot(HaveOccurred()) @@ -1202,9 +1202,9 @@ var _ = Describe("ProxyRoundTripper", func() { }) added := routePool.Put(endpoint1) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) added = routePool.Put(endpoint2) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) removed := routePool.Remove(endpoint) Expect(removed).To(BeTrue()) }) @@ -1505,7 +1505,7 @@ var _ = Describe("ProxyRoundTripper", func() { new_endpoint := route.NewEndpoint(&route.EndpointOpts{PrivateInstanceId: "id-5"}) added := routePool.Put(new_endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) }) Context("when route service headers are not on the request", func() { @@ -1568,7 +1568,7 @@ var _ = Describe("ProxyRoundTripper", func() { new_endpoint := route.NewEndpoint(&route.EndpointOpts{PrivateInstanceId: "id-5"}) added := routePool.Put(new_endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) }) Context("when route service headers are not on the request", func() { @@ -1634,7 +1634,7 @@ var _ = Describe("ProxyRoundTripper", func() { new_endpoint := route.NewEndpoint(&route.EndpointOpts{PrivateInstanceId: "id-5"}) added := routePool.Put(new_endpoint) - Expect(added).To(Equal(route.ADDED)) + Expect(added).To(Equal(route.EndpointAdded)) }) Context("when route service headers are not on the request", func() { diff --git a/src/code.cloudfoundry.org/gorouter/registry/registry.go b/src/code.cloudfoundry.org/gorouter/registry/registry.go index 21e146fbe..6f0ae83d3 100644 --- a/src/code.cloudfoundry.org/gorouter/registry/registry.go +++ b/src/code.cloudfoundry.org/gorouter/registry/registry.go @@ -88,32 +88,42 @@ func (r *RouteRegistry) Register(uri route.Uri, endpoint *route.Endpoint) { return } - endpointAdded := r.register(uri, endpoint) + poolPutResult, routePoolAdded := r.register(uri, endpoint) - r.reporter.CaptureRegistryMessage(endpoint, endpointAdded.String()) + r.reporter.CaptureRegistryMessage(endpoint, poolPutResult.String()) - if endpointAdded == route.ADDED && !endpoint.UpdatedAt.IsZero() { + if poolPutResult == route.EndpointAdded && !endpoint.UpdatedAt.IsZero() { r.reporter.CaptureRouteRegistrationLatency(time.Since(endpoint.UpdatedAt)) } - switch endpointAdded { - case route.ADDED: + if routePoolAdded { + r.logger.Info("route-registered", slog.Any("uri", uri)) + // for backward compatibility: + r.logger.Debug("uri-added", slog.Any("uri", uri)) + r.reporter.CaptureRoutesRegistered() + } + + switch poolPutResult { + case route.EndpointAdded: if r.logger.Enabled(context.Background(), slog.LevelInfo) { r.logger.Info("endpoint-registered", buildSlogAttrs(uri, endpoint)...) } - case route.UPDATED: - if r.logger.Enabled(context.Background(), slog.LevelDebug) { - r.logger.Debug("endpoint-registered", buildSlogAttrs(uri, endpoint)...) + case route.EndpointUpdated: + if r.logger.Enabled(context.Background(), slog.LevelInfo) { + r.logger.Info("endpoint-registered", buildSlogAttrs(uri, endpoint)...) } - default: + case route.EndpointUnmodified: if r.logger.Enabled(context.Background(), slog.LevelDebug) { r.logger.Debug("endpoint-not-registered", buildSlogAttrs(uri, endpoint)...) } + case route.EndpointRefreshed: + if r.logger.Enabled(context.Background(), slog.LevelDebug) { + r.logger.Debug("endpoint-refreshed", buildSlogAttrs(uri, endpoint)...) + } } - } -func (r *RouteRegistry) register(uri route.Uri, endpoint *route.Endpoint) route.PoolPutResult { +func (r *RouteRegistry) register(uri route.Uri, endpoint *route.Endpoint) (putResult route.PoolPutResult, routePoolAdded bool) { r.RLock() defer r.RUnlock() @@ -124,7 +134,7 @@ func (r *RouteRegistry) register(uri route.Uri, endpoint *route.Endpoint) route. if pool == nil { // release read lock, insertRouteKey() will acquire a write lock. r.RUnlock() - pool = r.insertRouteKey(routekey, uri) + pool, routePoolAdded = r.insertRouteKey(routekey, uri) r.RLock() } @@ -132,36 +142,37 @@ func (r *RouteRegistry) register(uri route.Uri, endpoint *route.Endpoint) route. endpoint.StaleThreshold = r.dropletStaleThreshold } - endpointAdded := pool.Put(endpoint) + putResult = pool.Put(endpoint) // Overwrites the load balancing algorithm of a pool by that of a specified endpoint, if that is valid. r.SetTimeOfLastUpdate(t) - return endpointAdded + return putResult, routePoolAdded } -// insertRouteKey acquires a write lock, inserts the route key into the registry and releases the write lock. -func (r *RouteRegistry) insertRouteKey(routekey route.Uri, uri route.Uri) *route.EndpointPool { +// insertRouteKey acquires a write lock, inserts the route key into the registry and releases the +// write lock. If a pool already exists it returns that instead. +func (r *RouteRegistry) insertRouteKey(routekey route.Uri, uri route.Uri) (pool *route.EndpointPool, poolAdded bool) { r.Lock() defer r.Unlock() // double check that the route key is still not found, now with the write lock. - pool := r.byURI.Find(routekey) - if pool == nil { - host, contextPath := splitHostAndContextPath(uri) - pool = route.NewPool(&route.PoolOpts{ - Logger: r.logger, - RetryAfterFailure: r.dropletStaleThreshold / 4, - Host: host, - ContextPath: contextPath, - MaxConnsPerBackend: r.maxConnsPerBackend, - LoadBalancingAlgorithm: r.DefaultLoadBalancingAlgorithm, - }) - r.byURI.Insert(routekey, pool) - r.logger.Info("route-registered", slog.Any("uri", routekey)) - // for backward compatibility: - r.logger.Debug("uri-added", slog.Any("uri", routekey)) + pool = r.byURI.Find(routekey) + if pool != nil { + return pool, false } - return pool + + host, contextPath := splitHostAndContextPath(uri) + pool = route.NewPool(&route.PoolOpts{ + Logger: r.logger, + RetryAfterFailure: r.dropletStaleThreshold / 4, + Host: host, + ContextPath: contextPath, + MaxConnsPerBackend: r.maxConnsPerBackend, + LoadBalancingAlgorithm: r.DefaultLoadBalancingAlgorithm, + }) + r.byURI.Insert(routekey, pool) + + return pool, true } func (r *RouteRegistry) Unregister(uri route.Uri, endpoint *route.Endpoint) { @@ -169,43 +180,57 @@ func (r *RouteRegistry) Unregister(uri route.Uri, endpoint *route.Endpoint) { return } - r.unregister(uri, endpoint) + endpointRemoved, routePoolRemoved := r.unregister(uri, endpoint) r.reporter.CaptureUnregistryMessage(endpoint) + var logMsg string + if endpointRemoved { + logMsg = "endpoint-unregistered" + } else { + logMsg = "endpoint-not-unregistered" + } + + if r.logger.Enabled(context.Background(), slog.LevelInfo) { + r.logger.Info(logMsg, buildSlogAttrs(uri, endpoint)...) + } + + if routePoolRemoved { + r.logger.Info("route-unregistered", slog.Any("uri", uri)) + r.reporter.CaptureRoutesUnregistered() + } else { + r.logger.Info("route-not-unregistered", slog.Any("uri", uri)) + } } -func (r *RouteRegistry) unregister(uri route.Uri, endpoint *route.Endpoint) { +func (r *RouteRegistry) unregister(uri route.Uri, endpoint *route.Endpoint) (endpointRemoved, routePoolRemoved bool) { r.Lock() defer r.Unlock() uri = uri.RouteKey() pool := r.byURI.Find(uri) - if pool != nil { - endpointRemoved := pool.Remove(endpoint) - if endpointRemoved { - if r.logger.Enabled(context.Background(), slog.LevelInfo) { - r.logger.Info("endpoint-unregistered", buildSlogAttrs(uri, endpoint)...) - } - } else { - if r.logger.Enabled(context.Background(), slog.LevelInfo) { - r.logger.Info("endpoint-not-unregistered", buildSlogAttrs(uri, endpoint)...) - } - } + if pool == nil { + return false, false + } - if pool.IsEmpty() { - if r.EmptyPoolResponseCode503 && r.EmptyPoolTimeout > 0 { - if time.Since(pool.LastUpdated()) > r.EmptyPoolTimeout { - r.byURI.Delete(uri) - r.logger.Info("route-unregistered", slog.Any("uri", uri)) - } - } else { - r.byURI.Delete(uri) - r.logger.Info("route-unregistered", slog.Any("uri", uri)) - } - } + endpointRemoved = pool.Remove(endpoint) + if !endpointRemoved { + return false, false } + + if !pool.IsEmpty() { + return true, false + } + + // If we have empty pool responses, the timeout for empty pools is greater than zero and the + // timeout of this pool has not yet expired, don't remove it yet. + if r.EmptyPoolResponseCode503 && r.EmptyPoolTimeout > 0 && time.Since(pool.LastUpdated()) <= r.EmptyPoolTimeout { + return true, false + } + + r.byURI.Delete(uri) + return true, true } func (r *RouteRegistry) Lookup(uri route.Uri) *route.EndpointPool { diff --git a/src/code.cloudfoundry.org/gorouter/registry/registry_test.go b/src/code.cloudfoundry.org/gorouter/registry/registry_test.go index 73e5bd3fe..83caa6b02 100644 --- a/src/code.cloudfoundry.org/gorouter/registry/registry_test.go +++ b/src/code.cloudfoundry.org/gorouter/registry/registry_test.go @@ -650,7 +650,12 @@ var _ = Describe("RouteRegistry", func() { Context("EmptyPoolResponseCode503 is true and EmptyPoolTimeout greater than 0", func() { JustBeforeEach(func() { r.EmptyPoolResponseCode503 = true - r.EmptyPoolTimeout = 5 * time.Second + r.EmptyPoolTimeout = 1 * time.Second + r.StartPruningCycle() + }) + + JustAfterEach(func() { + r.StopPruningCycle() }) It("Removes the route after EmptyPoolTimeout period of time is passed", func() { @@ -659,10 +664,7 @@ var _ = Describe("RouteRegistry", func() { r.Unregister("bar", barEndpoint) Expect(r.NumUris()).To(Equal(1)) - time.Sleep(r.EmptyPoolTimeout) - r.Unregister("bar", barEndpoint) - Expect(r.NumUris()).To(Equal(0)) - + Eventually(r.NumUris).WithTimeout(r.EmptyPoolTimeout + time.Second).Should(Equal(0)) }) }) diff --git a/src/code.cloudfoundry.org/gorouter/route/pool.go b/src/code.cloudfoundry.org/gorouter/route/pool.go index 14ffdb900..0d9fd8f2d 100644 --- a/src/code.cloudfoundry.org/gorouter/route/pool.go +++ b/src/code.cloudfoundry.org/gorouter/route/pool.go @@ -25,21 +25,24 @@ type PoolPutResult int func (p PoolPutResult) String() string { switch p { - case UNMODIFIED: + case EndpointUnmodified: return "unmodified" - case UPDATED: + case EndpointUpdated: return "updated" - case ADDED: + case EndpointAdded: return "added" + case EndpointRefreshed: + return "refreshed" default: panic("invalid PoolPutResult") } } const ( - UNMODIFIED = PoolPutResult(iota) - UPDATED - ADDED + EndpointUnmodified = PoolPutResult(iota) + EndpointUpdated + EndpointAdded + EndpointRefreshed ) func NewCounter(initial int64) *Counter { @@ -277,35 +280,58 @@ func (p *EndpointPool) Put(endpoint *Endpoint) PoolPutResult { p.Lock() defer p.Unlock() - var result PoolPutResult + var equal bool e, found := p.index[endpoint.CanonicalAddr()] if found { - result = UPDATED - if !e.endpoint.Equal(endpoint) { - e.Lock() - defer e.Unlock() + // Only calculate equal once, it's expensive. + equal = e.endpoint.Equal(endpoint) + } - if !e.endpoint.ModificationTag.SucceededBy(&endpoint.ModificationTag) { - return UNMODIFIED - } + switch { + case found && equal: + // This is the most common case. The endpoint has not changed but was simply re-announced + // to ensure gorouter is still aware of it. + e.updated = time.Now() + p.Update() - oldEndpoint := e.endpoint - e.endpoint = endpoint + return EndpointRefreshed - if oldEndpoint.PrivateInstanceId != endpoint.PrivateInstanceId { - delete(p.index, oldEndpoint.PrivateInstanceId) - p.index[endpoint.PrivateInstanceId] = e - } + case found && !e.endpoint.ModificationTag.SucceededBy(&endpoint.ModificationTag): + // This exists to protect against flapping when a route receives a change (e.g. a new + // route-service URL) and messages for the old and new config are still floating around. + return EndpointUnmodified - if oldEndpoint.ServerCertDomainSAN == endpoint.ServerCertDomainSAN { - endpoint.SetRoundTripper(oldEndpoint.RoundTripper()) - } + case found && !equal: + // The same endpoint was announced with different data, replace the old endpoint with the + // new one. + e.Lock() + defer e.Unlock() + + oldEndpoint := e.endpoint + e.endpoint = endpoint + + if oldEndpoint.PrivateInstanceId != endpoint.PrivateInstanceId { + delete(p.index, oldEndpoint.PrivateInstanceId) + p.index[endpoint.PrivateInstanceId] = e + } + + if oldEndpoint.ServerCertDomainSAN == endpoint.ServerCertDomainSAN { + endpoint.SetRoundTripper(oldEndpoint.RoundTripper()) } - } else { - result = ADDED + + p.RouteSvcUrl = e.endpoint.RouteServiceUrl + p.setPoolLoadBalancingAlgorithm(e.endpoint) + e.updated = time.Now() + p.Update() + + return EndpointUpdated + + case !found: + // New endpoint. e = &endpointElem{ endpoint: endpoint, index: len(p.endpoints), + updated: time.Now(), maxConnsPerBackend: p.maxConnsPerBackend, } @@ -314,14 +340,15 @@ func (p *EndpointPool) Put(endpoint *Endpoint) PoolPutResult { p.index[endpoint.CanonicalAddr()] = e p.index[endpoint.PrivateInstanceId] = e - } - p.RouteSvcUrl = e.endpoint.RouteServiceUrl - p.setPoolLoadBalancingAlgorithm(e.endpoint) - e.updated = time.Now() - // set the update time of the pool - p.Update() + p.RouteSvcUrl = e.endpoint.RouteServiceUrl + p.setPoolLoadBalancingAlgorithm(e.endpoint) + p.Update() + + return EndpointAdded - return result + default: + panic("quantum state discovered") + } } func (p *EndpointPool) RouteServiceUrl() string { diff --git a/src/code.cloudfoundry.org/gorouter/route/pool_test.go b/src/code.cloudfoundry.org/gorouter/route/pool_test.go index d43f7bf19..7d0655141 100644 --- a/src/code.cloudfoundry.org/gorouter/route/pool_test.go +++ b/src/code.cloudfoundry.org/gorouter/route/pool_test.go @@ -142,7 +142,7 @@ var _ = Describe("EndpointPool", func() { endpoint := &route.Endpoint{} b := pool.Put(endpoint) - Expect(b).To(Equal(route.ADDED)) + Expect(b).To(Equal(route.EndpointAdded)) }) It("handles duplicate endpoints", func() { @@ -152,7 +152,7 @@ var _ = Describe("EndpointPool", func() { pool.MarkUpdated(time.Now().Add(-(10 * time.Minute))) b := pool.Put(endpoint) - Expect(b).To(Equal(route.UPDATED)) + Expect(b).To(Equal(route.EndpointRefreshed)) prunedEndpoints := pool.PruneEndpoints() Expect(prunedEndpoints).To(BeEmpty()) @@ -163,7 +163,7 @@ var _ = Describe("EndpointPool", func() { endpoint2 := route.NewEndpoint(&route.EndpointOpts{Host: "1.2.3.4", Port: 5678}) pool.Put(endpoint1) - Expect(pool.Put(endpoint2)).To(Equal(route.UPDATED)) + Expect(pool.Put(endpoint2)).To(Equal(route.EndpointRefreshed)) }) Context("with modification tags", func() { @@ -175,13 +175,13 @@ var _ = Describe("EndpointPool", func() { modTag2 = models.ModificationTag{Guid: "abc"} endpoint1 := route.NewEndpoint(&route.EndpointOpts{Host: "1.2.3.4", Port: 5678, ModificationTag: modTag}) - Expect(pool.Put(endpoint1)).To(Equal(route.ADDED)) + Expect(pool.Put(endpoint1)).To(Equal(route.EndpointAdded)) }) It("updates an endpoint with modification tag", func() { endpoint := route.NewEndpoint(&route.EndpointOpts{Host: "1.2.3.4", Port: 5678, ModificationTag: modTag2}) - Expect(pool.Put(endpoint)).To(Equal(route.UPDATED)) + Expect(pool.Put(endpoint)).To(Equal(route.EndpointUpdated)) Expect(pool.Endpoints(logger.Logger, "", false, azPreference, az).Next(0).ModificationTag).To(Equal(modTag2)) }) @@ -189,14 +189,14 @@ var _ = Describe("EndpointPool", func() { BeforeEach(func() { modTag2.Increment() endpoint := route.NewEndpoint(&route.EndpointOpts{Host: "1.2.3.4", Port: 5678, ModificationTag: modTag2}) - pool.Put(endpoint) + Expect(pool.Put(endpoint)).To(Equal(route.EndpointUpdated)) }) It("doesnt update an endpoint", func() { olderModTag := models.ModificationTag{Guid: "abc"} endpoint := route.NewEndpoint(&route.EndpointOpts{Host: "1.2.3.4", Port: 5678, ModificationTag: olderModTag}) - Expect(pool.Put(endpoint)).To(Equal(route.UNMODIFIED)) + Expect(pool.Put(endpoint)).To(Equal(route.EndpointUnmodified)) Expect(pool.Endpoints(logger.Logger, "", false, azPreference, az).Next(0).ModificationTag).To(Equal(modTag2)) }) }) @@ -367,13 +367,13 @@ var _ = Describe("EndpointPool", func() { endpoint := &route.Endpoint{} endpointRS := &route.Endpoint{RouteServiceUrl: "my-url"} b := pool.Put(endpoint) - Expect(b).To(Equal(route.ADDED)) + Expect(b).To(Equal(route.EndpointAdded)) url := pool.RouteServiceUrl() Expect(url).To(BeEmpty()) b = pool.Put(endpointRS) - Expect(b).To(Equal(route.UPDATED)) + Expect(b).To(Equal(route.EndpointUpdated)) url = pool.RouteServiceUrl() Expect(url).To(Equal("my-url")) }) @@ -387,27 +387,27 @@ var _ = Describe("EndpointPool", func() { Context("when any endpoint updates its route_service_url", func() { It("returns the route_service_url most recently updated in the pool", func() { endpointRS1 := route.NewEndpoint(&route.EndpointOpts{Host: "host-1", Port: 1234, RouteServiceUrl: "first-url"}) + endpointRS1Updated := route.NewEndpoint(&route.EndpointOpts{Host: "host-1", Port: 1234, RouteServiceUrl: "third-url"}) endpointRS2 := route.NewEndpoint(&route.EndpointOpts{Host: "host-2", Port: 2234, RouteServiceUrl: "second-url"}) + endpointRS2Updated := route.NewEndpoint(&route.EndpointOpts{Host: "host-2", Port: 2234, RouteServiceUrl: "fourth-url"}) b := pool.Put(endpointRS1) - Expect(b).To(Equal(route.ADDED)) + Expect(b).To(Equal(route.EndpointAdded)) url := pool.RouteServiceUrl() Expect(url).To(Equal("first-url")) b = pool.Put(endpointRS2) - Expect(b).To(Equal(route.ADDED)) + Expect(b).To(Equal(route.EndpointAdded)) url = pool.RouteServiceUrl() Expect(url).To(Equal("second-url")) - endpointRS1.RouteServiceUrl = "third-url" - b = pool.Put(endpointRS1) - Expect(b).To(Equal(route.UPDATED)) + b = pool.Put(endpointRS1Updated) + Expect(b).To(Equal(route.EndpointUpdated)) url = pool.RouteServiceUrl() Expect(url).To(Equal("third-url")) - endpointRS2.RouteServiceUrl = "fourth-url" - b = pool.Put(endpointRS2) - Expect(b).To(Equal(route.UPDATED)) + b = pool.Put(endpointRS2Updated) + Expect(b).To(Equal(route.EndpointUpdated)) url = pool.RouteServiceUrl() Expect(url).To(Equal("fourth-url")) }) @@ -514,7 +514,7 @@ var _ = Describe("EndpointPool", func() { modTag = models.ModificationTag{Guid: "abc"} endpoint1 := route.NewEndpoint(&route.EndpointOpts{Host: "1.2.3.4", Port: 5678, ModificationTag: modTag}) - Expect(pool.Put(endpoint1)).To(Equal(route.ADDED)) + Expect(pool.Put(endpoint1)).To(Equal(route.EndpointAdded)) }) It("removes an endpoint with modification tag", func() {