From 6c16467dfd52e9eb8e0c732f48f5ab2f09019bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aitor=20P=C3=A9rez=20Cedres?= Date: Wed, 9 Aug 2023 11:59:20 +0100 Subject: [PATCH 01/24] Bump to Go 1.21 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aitor PĂ©rez Cedres --- Makefile | 4 +-- go.mod | 17 ++++++----- go.sum | 54 ++++++++++++++++++---------------- main.go | 8 ++--- pkg/e2e/end_to_end_test.go | 12 ++++---- pkg/raw/client.go | 2 +- pkg/raw/log.go | 2 +- pkg/raw/stream_suite_test.go | 8 ++--- pkg/stream/environment.go | 2 +- pkg/stream/environment_test.go | 8 ++--- pkg/stream/locator.go | 2 +- pkg/stream/locator_test.go | 10 ++----- pkg/stream/test_helpers.go | 2 +- 13 files changed, 65 insertions(+), 66 deletions(-) diff --git a/Makefile b/Makefile index 76a9a868..9490699a 100644 --- a/Makefile +++ b/Makefile @@ -50,8 +50,8 @@ install-tools: ## Install tool dependencies for development ### Golang targets .PHONY: go-mod-tidy -go-mod-tidy: ## Run 'go mod tidy' with compatibility to Go 1.19 - $(GO) mod tidy -go=1.19 +go-mod-tidy: ## Run 'go mod tidy' with compatibility to Go 1.21 + $(GO) mod tidy -go=1.21 .PHONY: go-generate-mocks go-generate-mocks: | $(MOCKGEN) ## Generate Mocks for testing diff --git a/go.mod b/go.mod index eb2af435..e90513fb 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,14 @@ module github.com/rabbitmq/rabbitmq-stream-go-client/v2 -go 1.19 +go 1.21 + +toolchain go1.21.0 require ( - github.com/michaelklishin/rabbit-hole/v2 v2.13.0 + github.com/michaelklishin/rabbit-hole/v2 v2.15.0 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 go.uber.org/mock v0.2.0 - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/mod v0.12.0 ) @@ -15,10 +16,10 @@ require ( github.com/go-logr/logr v1.2.4 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect - golang.org/x/net v0.12.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect - golang.org/x/tools v0.11.0 // indirect + github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + golang.org/x/tools v0.12.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 06f184d2..4459025f 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,7 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -32,8 +33,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c= +github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -42,8 +43,8 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/michaelklishin/rabbit-hole/v2 v2.13.0 h1:4idonhoyXqm47o+A8Sf8LwN6GNOaDYqHWpdU/jqV1gE= -github.com/michaelklishin/rabbit-hole/v2 v2.13.0/go.mod h1:JGRQOHJaoefurvQP6oX4kLVJvv8hggrjGZ/4lTMpzpg= +github.com/michaelklishin/rabbit-hole/v2 v2.15.0 h1:asuENwbu5UsgPBHKgOzHY6VVrjNePurjJoE+8+EWeLA= +github.com/michaelklishin/rabbit-hole/v2 v2.15.0/go.mod h1:o0k0caEjRjboLEylRXVR7aOkuI2vZ6gLXZ78JyonVkA= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -60,6 +61,9 @@ github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkA github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= +github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= +github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -73,12 +77,15 @@ github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2 github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= +github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rabbitmq/amqp091-go v1.7.0 h1:V5CF5qPem5OGSnEo8BoSbsDGwejg6VUJsKEdneaoTUo= -github.com/rabbitmq/amqp091-go v1.7.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI= +github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA= +github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -87,10 +94,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU= go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -98,26 +104,21 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -129,8 +130,9 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -148,9 +150,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -164,8 +164,9 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -173,6 +174,7 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -181,20 +183,20 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= -golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index 63318750..565e7f3d 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,7 @@ import ( "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/constants" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/stream" - "golang.org/x/exp/slog" + "log/slog" "os" "time" ) @@ -20,10 +20,10 @@ func main() { } func runSmartClient() { - h := slog.HandlerOptions{ + slogOpts := &slog.HandlerOptions{ Level: slog.LevelDebug, } - log := slog.New(h.NewTextHandler(os.Stdout)) + log := slog.New(slog.NewTextHandler(os.Stdout, slogOpts)) ctx := raw.NewContextWithLogger(context.Background(), *log) @@ -66,7 +66,7 @@ func runSmartClient() { } func runRawClient() { - log := slog.New(slog.NewTextHandler(os.Stdout)) + log := slog.New(slog.NewTextHandler(os.Stdout, nil)) streamName := "test-streamName" config, err := raw.NewClientConfiguration("rabbitmq-stream://guest:guest@localhost:5552") if err != nil { diff --git a/pkg/e2e/end_to_end_test.go b/pkg/e2e/end_to_end_test.go index cf2b5de6..aef96bde 100644 --- a/pkg/e2e/end_to_end_test.go +++ b/pkg/e2e/end_to_end_test.go @@ -13,14 +13,14 @@ import ( "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/common" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/constants" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" - "golang.org/x/exp/slog" "io" + "log/slog" "os" "sync" "time" ) -var e2eLogger = slog.New(slog.NewTextHandler(GinkgoWriter)) +var e2eLogger = slog.New(slog.NewTextHandler(GinkgoWriter, nil)) var _ = Describe("E2E", Serial, Label("e2e"), func() { const ( @@ -134,8 +134,8 @@ var _ = Describe("E2E", Serial, Label("e2e"), func() { // Send and Recveive Messages, assert messages received are valid. It("sends, and receives messages", Label("behaviour"), func(ctx SpecContext) { - h := slog.HandlerOptions{Level: slog.LevelDebug}.NewTextHandler(GinkgoWriter) - debugLogger := slog.New(h) + opts := &slog.HandlerOptions{Level: slog.LevelDebug} + debugLogger := slog.New(slog.NewTextHandler(GinkgoWriter, opts)) itCtx := raw.NewContextWithLogger(ctx, *debugLogger) streamClientConfiguration, err := raw.NewClientConfiguration(rabbitmqUri) Expect(err).ToNot(HaveOccurred()) @@ -243,8 +243,8 @@ var _ = Describe("E2E", Serial, Label("e2e"), func() { // With the HTTP API, we can check the connection name and kill it. // The client has to notify the disconnection. It("connection name and notify disconnection", Label("behaviour"), func(ctx SpecContext) { - h := slog.HandlerOptions{Level: slog.LevelDebug}.NewTextHandler(GinkgoWriter) - debugLogger := slog.New(h) + options := &slog.HandlerOptions{Level: slog.LevelDebug} + debugLogger := slog.New(slog.NewTextHandler(GinkgoWriter, options)) itCtx := raw.NewContextWithLogger(ctx, *debugLogger) streamClientConfiguration, err := raw.NewClientConfiguration(rabbitmqUri) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/raw/client.go b/pkg/raw/client.go index bc404a4a..a38a0ae7 100644 --- a/pkg/raw/client.go +++ b/pkg/raw/client.go @@ -7,8 +7,8 @@ import ( "fmt" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/internal" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/common" - "golang.org/x/exp/slog" "io" + "log/slog" "math" "net" "reflect" diff --git a/pkg/raw/log.go b/pkg/raw/log.go index 1f6e586f..5f867e79 100644 --- a/pkg/raw/log.go +++ b/pkg/raw/log.go @@ -2,7 +2,7 @@ package raw import ( "context" - "golang.org/x/exp/slog" + "log/slog" ) // The provided key must be comparable and should not be of type string or any diff --git a/pkg/raw/stream_suite_test.go b/pkg/raw/stream_suite_test.go index 7b676853..040abb92 100644 --- a/pkg/raw/stream_suite_test.go +++ b/pkg/raw/stream_suite_test.go @@ -6,8 +6,8 @@ import ( "encoding/binary" "errors" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" - "golang.org/x/exp/slog" "io" + "log/slog" "net" "os" "testing" @@ -26,10 +26,10 @@ func TestStream(t *testing.T) { } var _ = BeforeSuite(func() { - h := slog.HandlerOptions{ + opts := &slog.HandlerOptions{ Level: slog.LevelDebug, - }.NewTextHandler(GinkgoWriter) - logger = slog.New(h) + } + logger = slog.New(slog.NewTextHandler(GinkgoWriter, opts)) }) type autoIncrementingSequence struct { diff --git a/pkg/stream/environment.go b/pkg/stream/environment.go index e957d32c..4ebd030c 100644 --- a/pkg/stream/environment.go +++ b/pkg/stream/environment.go @@ -4,7 +4,7 @@ import ( "context" "fmt" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" - "golang.org/x/exp/slog" + "log/slog" "math/rand" "time" ) diff --git a/pkg/stream/environment_test.go b/pkg/stream/environment_test.go index caae5ecb..8d4103dd 100644 --- a/pkg/stream/environment_test.go +++ b/pkg/stream/environment_test.go @@ -9,7 +9,7 @@ import ( "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/stream" "go.uber.org/mock/gomock" - "golang.org/x/exp/slog" + "log/slog" "reflect" "sync" "time" @@ -199,7 +199,7 @@ var _ = Describe("Environment", func() { It("logs locator operation errors", func() { // setup logBuffer := gbytes.NewBuffer() - logger := slog.New(slog.NewTextHandler(logBuffer)) + logger := slog.New(slog.NewTextHandler(logBuffer, nil)) ctx := raw.NewContextWithLogger(context.Background(), *logger) // act @@ -281,7 +281,7 @@ var _ = Describe("Environment", func() { It("logs the error and moves on", func() { logBuffer := gbytes.NewBuffer() - logger := slog.New(slog.NewTextHandler(logBuffer)) + logger := slog.New(slog.NewTextHandler(logBuffer, nil)) ctx := raw.NewContextWithLogger(context.Background(), *logger) environment.Close(ctx) @@ -417,7 +417,7 @@ var _ = Describe("Environment", func() { It("logs intermediate error messages", func() { // setup logBuffer := gbytes.NewBuffer() - logger := slog.New(slog.NewTextHandler(logBuffer)) + logger := slog.New(slog.NewTextHandler(logBuffer, nil)) ctx := raw.NewContextWithLogger(context.Background(), *logger) mockRawClient.EXPECT(). diff --git a/pkg/stream/locator.go b/pkg/stream/locator.go index 8ed838ac..f1ae70dd 100644 --- a/pkg/stream/locator.go +++ b/pkg/stream/locator.go @@ -3,8 +3,8 @@ package stream import ( "context" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" - "golang.org/x/exp/slog" "golang.org/x/mod/semver" + "log/slog" "net" "sync" "time" diff --git a/pkg/stream/locator_test.go b/pkg/stream/locator_test.go index c269b021..b6edb023 100644 --- a/pkg/stream/locator_test.go +++ b/pkg/stream/locator_test.go @@ -3,13 +3,12 @@ package stream import ( "context" "errors" - "time" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" - "golang.org/x/exp/slog" + "log/slog" + "time" ) var _ = Describe("Locator", func() { @@ -23,10 +22,7 @@ var _ = Describe("Locator", func() { ) BeforeEach(func() { - h := slog.HandlerOptions{ - Level: slog.LevelDebug, - }.NewTextHandler(GinkgoWriter) - logger = slog.New(h) + logger = slog.New(slog.NewTextHandler(GinkgoWriter, &slog.HandlerOptions{Level: slog.LevelDebug})) loc = &locator{ log: logger, diff --git a/pkg/stream/test_helpers.go b/pkg/stream/test_helpers.go index b6cbc3f1..ced60239 100644 --- a/pkg/stream/test_helpers.go +++ b/pkg/stream/test_helpers.go @@ -5,7 +5,7 @@ package stream import ( "context" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" - "golang.org/x/exp/slog" + "log/slog" "time" ) From 859f58d6538be4bc915ba78572438d0eb9ad5e2f Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Mon, 25 Sep 2023 17:18:52 +0100 Subject: [PATCH 02/24] Standard producer basic functionality First pass for smart producer. Signed-off-by: Aitor Perez Cedres --- pkg/stream/environment.go | 11 +-- pkg/stream/locator.go | 8 +- pkg/stream/locator_test.go | 2 +- pkg/stream/producer.go | 191 ++++++++++++++++++++++++++++++++++++ pkg/stream/producer_test.go | 151 ++++++++++++++++++++++++++++ pkg/stream/test_helpers.go | 4 +- pkg/stream/types.go | 19 +++- pkg/stream/util.go | 33 +++++++ 8 files changed, 402 insertions(+), 17 deletions(-) create mode 100644 pkg/stream/producer.go create mode 100644 pkg/stream/producer_test.go diff --git a/pkg/stream/environment.go b/pkg/stream/environment.go index 4ebd030c..02484ca8 100644 --- a/pkg/stream/environment.go +++ b/pkg/stream/environment.go @@ -15,9 +15,9 @@ const ( ) type Environment struct { - configuration EnvironmentConfiguration - locators []*locator - backOffPolicy func(int) time.Duration + configuration EnvironmentConfiguration + locators []*locator + retryPolicy backoffDurationFunc locatorSelectSequential bool } @@ -25,10 +25,7 @@ func NewEnvironment(ctx context.Context, configuration EnvironmentConfiguration) e := &Environment{ configuration: configuration, locators: make([]*locator, 0, len(configuration.Uris)), - } - - e.backOffPolicy = func(attempt int) time.Duration { - return time.Second * time.Duration(attempt<<1) + retryPolicy: defaultBackOffPolicy, } if !configuration.LazyInitialization { diff --git a/pkg/stream/locator.go b/pkg/stream/locator.go index f1ae70dd..6e5b9963 100644 --- a/pkg/stream/locator.go +++ b/pkg/stream/locator.go @@ -22,7 +22,7 @@ type locator struct { client raw.Clienter isSet bool clientClose <-chan error - backOffPolicy func(int) time.Duration + retryPolicy backoffDurationFunc addressResolver net.Addr // TODO: placeholder for address resolver } @@ -35,10 +35,8 @@ func newLocator(c raw.ClientConfiguration, logger *slog.Logger) *locator { slog.String("host", c.RabbitmqBrokers().Host), slog.Int("port", c.RabbitmqBrokers().Port), ), - rawClientConf: c, - backOffPolicy: func(attempt int) time.Duration { - return time.Second * time.Duration(attempt<<1) - }, + rawClientConf: c, + retryPolicy: defaultBackOffPolicy, client: nil, isSet: false, addressResolver: nil, diff --git a/pkg/stream/locator_test.go b/pkg/stream/locator_test.go index b6edb023..7b13498e 100644 --- a/pkg/stream/locator_test.go +++ b/pkg/stream/locator_test.go @@ -31,7 +31,7 @@ var _ = Describe("Locator", func() { client: nil, isSet: true, clientClose: nil, - backOffPolicy: backOffPolicy, + retryPolicy: backOffPolicy, } }) diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go new file mode 100644 index 00000000..e69d7f3f --- /dev/null +++ b/pkg/stream/producer.go @@ -0,0 +1,191 @@ +package stream + +import ( + "context" + "errors" + "fmt" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/common" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" + "sync" + "time" +) + +var ( + errMessageBufferFull = errors.New("message buffer is full") +) + +type ProducerOptions struct { + MaxInFlight int + MaxBufferedMessages int +} + +type standardProducer struct { + publisherId uint8 + rawClient raw.Clienter + rawClientMu *sync.Mutex + publishingIdSeq autoIncrementingSequence[uint64] + opts ProducerOptions + // buffer mutex + bufferMu *sync.Mutex + messageBuffer []common.PublishingMessager + retryDuration backoffDurationFunc + done chan struct{} + cancel context.CancelFunc + destructor sync.Once +} + +func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts ProducerOptions) *standardProducer { + if opts.MaxInFlight <= 0 { + opts.MaxInFlight = 10_000 + } + if opts.MaxBufferedMessages <= 0 { + opts.MaxBufferedMessages = 100 + } + p := &standardProducer{ + publisherId: publisherId, + rawClient: rawClient, + rawClientMu: &sync.Mutex{}, + publishingIdSeq: autoIncrementingSequence[uint64]{}, + opts: opts, + bufferMu: &sync.Mutex{}, + messageBuffer: make([]common.PublishingMessager, 0, opts.MaxBufferedMessages), + retryDuration: func(i int) time.Duration { + return time.Second * (1 << i) + }, + done: make(chan struct{}), + } + + ctx, cancel := context.WithCancel(context.Background()) + p.cancel = cancel + + go p.sendLoopAsync(ctx) + return p +} + +func (s *standardProducer) close() { + s.cancel() + s.destructor.Do(func() { + close(s.done) + }) +} + +// push is not thread-safe. A bufferMu lock must be acquired before calling this function. +func (s *standardProducer) push(publishingId uint64, message Message) error { + if len(s.messageBuffer) >= s.opts.MaxBufferedMessages { + return errMessageBufferFull + } + s.messageBuffer = append(s.messageBuffer, raw.NewPublishingMessage(publishingId, message)) + return nil +} + +// synchronously sends the messages accumulated in the message buffer. If sending +// is successful, it clears the message buffer. +func (s *standardProducer) doSend(ctx context.Context) error { + s.bufferMu.Lock() + s.rawClientMu.Lock() + defer s.rawClientMu.Unlock() + defer s.bufferMu.Unlock() + + if len(s.messageBuffer) == 0 { + return nil + } + + var err error + for i := 0; i < maxAttempt; i++ { + err = s.rawClient.Send(ctx, s.publisherId, s.messageBuffer) + if isNonRetryableError(err) { + return err + } + if err == nil { + break + } + + <-time.After(s.retryDuration(i)) + } + + if err != nil { + return err + } + + for i := 0; i < len(s.messageBuffer); i++ { + // FIXME: keep track of pending messages in a different list + s.messageBuffer[i] = nil + } + s.messageBuffer = s.messageBuffer[:0] + + return nil +} + +func (s *standardProducer) sendLoopAsync(ctx context.Context) { + // FIXME: configurable ticker + t := time.NewTicker(time.Millisecond * 300) + + for { + select { + case <-t.C: + // send + err := s.doSend(ctx) + if err != nil { + // log error + panic(err) + } + case <-s.done: + case <-ctx.Done(): + // exit + t.Stop() + return + } + } +} + +// Public API + +func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { + //TODO implement me + s.bufferMu.Lock() + if len(s.messageBuffer) == s.opts.MaxBufferedMessages { + s.bufferMu.Unlock() + if err := s.doSend(ctx); err != nil { + return err + } + } + + _ = s.push(s.publishingIdSeq.next(), &msg) + s.bufferMu.Unlock() + + return nil +} + +// SendBatch synchronously sends messages to the broker. Each messages gets +// a publishing ID assigned automatically. +func (s *standardProducer) SendBatch(ctx context.Context, messages []amqp.Message) error { + if len(messages) == 0 { + return ErrEmptyBatch + } + + if n := len(messages); n > s.opts.MaxInFlight { + return fmt.Errorf("%w: max %d, batch len %d", ErrBatchTooLarge, s.opts.MaxInFlight, n) + } + + var pMsgs = make([]common.PublishingMessager, 0, len(messages)) + for i := 0; i < len(messages); i++ { + pMsgs = append(pMsgs, raw.NewPublishingMessage(s.publishingIdSeq.next(), &messages[i])) + } + + s.rawClientMu.Lock() + defer s.rawClientMu.Unlock() + + return s.rawClient.Send(ctx, s.publisherId, pMsgs) +} + +// SendWithId always returns an error in the standard producer because the +// publishing ID is tracked internally. +func (s *standardProducer) SendWithId(_ context.Context, _ uint64, _ amqp.Message) error { + return fmt.Errorf("%w: standard producer does not support sending with ID", ErrUnsupportedOperation) +} + +func (s *standardProducer) GetLastPublishedId() uint64 { + //TODO implement me + panic("implement me") +} diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go new file mode 100644 index 00000000..48a65947 --- /dev/null +++ b/pkg/stream/producer_test.go @@ -0,0 +1,151 @@ +package stream + +import ( + "context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/common" + "go.uber.org/mock/gomock" + "reflect" + "sync" + "time" +) + +var _ = Describe("Smart Producer", func() { + + var ( + mockController *gomock.Controller + fakeRawClient *MockRawClient + ctxType = reflect.TypeOf((*context.Context)(nil)).Elem() + ) + + BeforeEach(func() { + mockController = gomock.NewController(GinkgoT()) + fakeRawClient = NewMockRawClient(mockController) + }) + + Describe("send batch", func() { + When("the batch list is empty", func() { + It("returns an error", func() { + p := standardProducer{ + publisherId: 0, + rawClient: fakeRawClient, + rawClientMu: &sync.Mutex{}, + publishingIdSeq: autoIncrementingSequence[uint64]{}, + } + + Expect(p.SendBatch(context.Background(), []amqp.Message{})).To(MatchError("batch list is empty")) + }) + }) + + It("sends messages batched by the user", func() { + // setup + var capturedPublishingIds []uint64 + gomock.InOrder(fakeRawClient.EXPECT(). + Send( + gomock.AssignableToTypeOf(ctxType), + gomock.Eq(uint8(1)), + gomock.All( + gomock.Len(1), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ), + ). + Do(func(_ context.Context, _ uint8, pMessages []common.PublishingMessager) error { + capturedPublishingIds = []uint64{pMessages[0].PublishingId()} + return nil + }), + fakeRawClient.EXPECT(). + Send( + gomock.AssignableToTypeOf(ctxType), + gomock.Eq(uint8(1)), + gomock.All( + gomock.Len(3), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ), + ). + Do(func(_ context.Context, _ uint8, pMessages []common.PublishingMessager) error { + capturedPublishingIds = make([]uint64, 0, 3) + for i := 0; i < len(pMessages); i++ { + capturedPublishingIds = append(capturedPublishingIds, pMessages[i].PublishingId()) + } + return nil + }), + ) + + p := standardProducer{ + publisherId: 1, + rawClient: fakeRawClient, + rawClientMu: &sync.Mutex{}, + opts: ProducerOptions{100, 100}, + } + + // test + batch := []amqp.Message{{Data: []byte("message 1")}} + Expect(p.SendBatch(context.Background(), batch)).To(Succeed()) + Expect(capturedPublishingIds).To(ConsistOf(uint64(0))) + + batch = append(batch, amqp.Message{Data: []byte("message 2")}, amqp.Message{Data: []byte("message 3")}) + Expect(p.SendBatch(context.Background(), batch)).To(Succeed()) + Expect(capturedPublishingIds).To(ConsistOf(uint64(1), uint64(2), uint64(3))) + }) + + When("the batch list is larger than max in flight", func() { + It("returns an error", func() { + p := standardProducer{ + publisherId: 0, + rawClient: fakeRawClient, + rawClientMu: &sync.Mutex{}, + publishingIdSeq: autoIncrementingSequence[uint64]{}, + opts: ProducerOptions{1, 1}, + } + msgs := make([]amqp.Message, 10) + Expect(p.SendBatch(context.Background(), msgs)).To(MatchError(ErrBatchTooLarge)) + }) + }) + }) + + Describe("send with ID", func() { + It("always returns an error", func() { + p := &standardProducer{} + Expect( + p.SendWithId(context.Background(), 123, amqp.Message{Data: []byte("this will return an error")}), + ).To(MatchError(ErrUnsupportedOperation)) + }) + }) + + Describe("send", func() { + It("accumulates and sends messages", func() { + m := &sync.Mutex{} + var capturedIds = make([]uint64, 0) + fakeRawClient.EXPECT(). + Send(gomock.AssignableToTypeOf(ctxType), gomock.Eq(uint8(42)), + gomock.All( + gomock.Len(3), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ), + ). + Do(func(_ context.Context, _ uint8, pMessages []common.PublishingMessager) error { + m.Lock() + for i := 0; i < len(pMessages); i++ { + capturedIds = append(capturedIds, pMessages[i].PublishingId()) + } + m.Unlock() + return nil + }) + + p := newStandardProducer(42, fakeRawClient, ProducerOptions{5, 5}) + + Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 1")})).To(Succeed()) + Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 2")})).To(Succeed()) + Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 3")})).To(Succeed()) + + Eventually(func() []uint64 { + m.Lock() + defer m.Unlock() + return capturedIds + }).Within(time.Second * 1).WithPolling(time.Millisecond * 200).Should(ConsistOf(uint64(0), uint64(1), uint64(2))) + }) + + }) +}) diff --git a/pkg/stream/test_helpers.go b/pkg/stream/test_helpers.go index ced60239..2059485e 100644 --- a/pkg/stream/test_helpers.go +++ b/pkg/stream/test_helpers.go @@ -33,7 +33,7 @@ func (e *Environment) AppendLocatorRawClient(c raw.Clienter) { client: c, isSet: true, log: slog.New(&discardHandler{}), - backOffPolicy: func(int) time.Duration { + retryPolicy: func(int) time.Duration { return time.Millisecond * 10 }, }) @@ -49,7 +49,7 @@ func (e *Environment) SetServerVersion(v string) { } func (e *Environment) SetBackoffPolicy(f func(int) time.Duration) { - e.backOffPolicy = f + e.retryPolicy = f } func (e *Environment) SetLocatorSelectSequential(v bool) { diff --git a/pkg/stream/types.go b/pkg/stream/types.go index 6ca8223d..caac1e77 100644 --- a/pkg/stream/types.go +++ b/pkg/stream/types.go @@ -1,7 +1,11 @@ package stream import ( + "context" + "errors" "fmt" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/common" "time" ) @@ -13,8 +17,10 @@ const ( ) var ( - ErrNoLocators = fmt.Errorf("no locators configured") - ErrUnsupportedOperation = fmt.Errorf("unsupported operation") + ErrNoLocators = errors.New("no locators configured") + ErrUnsupportedOperation = errors.New("unsupported operation") + ErrBatchTooLarge = errors.New("too many messages in batch") + ErrEmptyBatch = errors.New("batch list is empty") ) type ByteCapacity uint64 @@ -33,3 +39,12 @@ type CreateStreamOptions struct { MaxLength ByteCapacity MaxSegmentSize ByteCapacity } + +type Producer interface { + Send(ctx context.Context, msg amqp.Message) error + SendBatch(ctx context.Context, messages []amqp.Message) error + SendWithId(ctx context.Context, publishingId uint64, msg amqp.Message) error + GetLastPublishedId() uint64 +} + +type Message = common.Message diff --git a/pkg/stream/util.go b/pkg/stream/util.go index 4ac39886..d6d6e9ba 100644 --- a/pkg/stream/util.go +++ b/pkg/stream/util.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" + "sync" + "time" ) const ( @@ -61,6 +63,37 @@ func maybeApplyDefaultTimeout(ctx context.Context) (context.Context, context.Can return ctx, nil } +// Borrowed from experimental Go library +// https://pkg.go.dev/golang.org/x/exp/constraints#Integer +// Making a copy & paste to avoid depending on golang.org/x/exp +// FIXME probably need to include copyright/license from golang.org/x/exp +type integer interface { + ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr +} + +type autoIncrementingSequence[T integer] struct { + sync.Mutex + value T +} + +func newAutoIncrementingSequence[T integer](startingValue T) *autoIncrementingSequence[T] { + return &autoIncrementingSequence[T]{value: startingValue} +} + +func (a *autoIncrementingSequence[T]) next() (next T) { + a.Lock() + defer a.Unlock() + next = a.value + a.value += 1 + return +} + +type backoffDurationFunc func(int) time.Duration + +var defaultBackOffPolicy backoffDurationFunc = func(i int) time.Duration { + return time.Second * time.Duration(i<<1) +} + func validateStringParameter(p string) bool { if len(p) == 0 || p == " " { return false From 1a73f89ba1d35be87f5b3637fa542cf128312230 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 28 Sep 2023 12:23:50 +0100 Subject: [PATCH 03/24] Make batch publishing delay configurable GA version of `slog` has a different signature for `slog.New`. This commit updates some code that was using the old signature. Signed-off-by: Aitor Perez Cedres --- pkg/stream/environment_test.go | 6 +++--- pkg/stream/producer.go | 22 +++++++++++++++++++--- pkg/stream/producer_test.go | 6 +++--- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/pkg/stream/environment_test.go b/pkg/stream/environment_test.go index 8d4103dd..d3ff0644 100644 --- a/pkg/stream/environment_test.go +++ b/pkg/stream/environment_test.go @@ -512,7 +512,7 @@ var _ = Describe("Environment", func() { It("logs intermediate error messages", func() { // setup logBuffer := gbytes.NewBuffer() - logger := slog.New(slog.NewTextHandler(logBuffer)) + logger := slog.New(slog.NewTextHandler(logBuffer, nil)) ctx := raw.NewContextWithLogger(context.Background(), *logger) mockRawClient.EXPECT(). @@ -607,7 +607,7 @@ var _ = Describe("Environment", func() { It("logs intermediate error messages", func() { // setup logBuffer := gbytes.NewBuffer() - logger := slog.New(slog.NewTextHandler(logBuffer)) + logger := slog.New(slog.NewTextHandler(logBuffer, nil)) ctx := raw.NewContextWithLogger(context.Background(), *logger) mockRawClient.EXPECT(). @@ -709,7 +709,7 @@ var _ = Describe("Environment", func() { It("logs intermediate error messages", func() { // setup logBuffer := gbytes.NewBuffer() - logger := slog.New(slog.NewTextHandler(logBuffer)) + logger := slog.New(slog.NewTextHandler(logBuffer, nil)) ctx := raw.NewContextWithLogger(context.Background(), *logger) mockRawClient.EXPECT(). diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index e69d7f3f..490a4a4e 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -15,9 +15,19 @@ var ( errMessageBufferFull = errors.New("message buffer is full") ) +const ( + defaultBatchPublishingDelay = time.Millisecond * 100 +) + type ProducerOptions struct { - MaxInFlight int + // The maximum number of unconfirmed outbound messages. Producer.Send will start + // blocking when the limit is reached. + MaxInFlight int + // The maximum number of messages to accumulate before sending them to the + // broker. MaxBufferedMessages int + // Period to send a batch of messages. + BatchPublishingDelay time.Duration } type standardProducer struct { @@ -118,9 +128,15 @@ func (s *standardProducer) doSend(ctx context.Context) error { } func (s *standardProducer) sendLoopAsync(ctx context.Context) { - // FIXME: configurable ticker - t := time.NewTicker(time.Millisecond * 300) + var publishingDelay time.Duration + if s.opts.BatchPublishingDelay == 0 { + // important to make this check because NewTicker(0) panics + publishingDelay = defaultBatchPublishingDelay + } else { + publishingDelay = s.opts.BatchPublishingDelay + } + t := time.NewTicker(publishingDelay) for { select { case <-t.C: diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index 48a65947..d2fe91b8 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -77,7 +77,7 @@ var _ = Describe("Smart Producer", func() { publisherId: 1, rawClient: fakeRawClient, rawClientMu: &sync.Mutex{}, - opts: ProducerOptions{100, 100}, + opts: ProducerOptions{MaxInFlight: 100, MaxBufferedMessages: 100}, } // test @@ -97,7 +97,7 @@ var _ = Describe("Smart Producer", func() { rawClient: fakeRawClient, rawClientMu: &sync.Mutex{}, publishingIdSeq: autoIncrementingSequence[uint64]{}, - opts: ProducerOptions{1, 1}, + opts: ProducerOptions{MaxInFlight: 1, MaxBufferedMessages: 1}, } msgs := make([]amqp.Message, 10) Expect(p.SendBatch(context.Background(), msgs)).To(MatchError(ErrBatchTooLarge)) @@ -134,7 +134,7 @@ var _ = Describe("Smart Producer", func() { return nil }) - p := newStandardProducer(42, fakeRawClient, ProducerOptions{5, 5}) + p := newStandardProducer(42, fakeRawClient, ProducerOptions{MaxInFlight: 5, MaxBufferedMessages: 5}) Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 1")})).To(Succeed()) Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 2")})).To(Succeed()) From 5d58aaeb063734f048b720fb120dee3f292e7b76 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 28 Sep 2023 16:46:47 +0100 Subject: [PATCH 04/24] Smart Send: force send when buffer is full Signed-off-by: Aitor Perez Cedres --- pkg/stream/producer.go | 50 +++++++++++++++++++++------------ pkg/stream/producer_test.go | 56 +++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 490a4a4e..622415e8 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -17,6 +17,8 @@ var ( const ( defaultBatchPublishingDelay = time.Millisecond * 100 + defaultMaxInFlight = 10_000 + maxBufferedMessages = 100 ) type ProducerOptions struct { @@ -30,12 +32,24 @@ type ProducerOptions struct { BatchPublishingDelay time.Duration } +func (p *ProducerOptions) validate() { + if p.MaxInFlight <= 0 { + p.MaxInFlight = defaultMaxInFlight + } + if p.MaxBufferedMessages <= 0 { + p.MaxBufferedMessages = maxBufferedMessages + } + if p.BatchPublishingDelay == 0 { + p.BatchPublishingDelay = defaultBatchPublishingDelay + } +} + type standardProducer struct { publisherId uint8 rawClient raw.Clienter rawClientMu *sync.Mutex publishingIdSeq autoIncrementingSequence[uint64] - opts ProducerOptions + opts *ProducerOptions // buffer mutex bufferMu *sync.Mutex messageBuffer []common.PublishingMessager @@ -45,13 +59,8 @@ type standardProducer struct { destructor sync.Once } -func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts ProducerOptions) *standardProducer { - if opts.MaxInFlight <= 0 { - opts.MaxInFlight = 10_000 - } - if opts.MaxBufferedMessages <= 0 { - opts.MaxBufferedMessages = 100 - } +func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *ProducerOptions) *standardProducer { + opts.validate() p := &standardProducer{ publisherId: publisherId, rawClient: rawClient, @@ -90,12 +99,12 @@ func (s *standardProducer) push(publishingId uint64, message Message) error { } // synchronously sends the messages accumulated in the message buffer. If sending -// is successful, it clears the message buffer. +// is successful, it clears the message buffer. The caller MUST hold a lock +// on the buffer mutex. Calling this function without a lock on buffer mutex is +// undefined behaviour func (s *standardProducer) doSend(ctx context.Context) error { - s.bufferMu.Lock() s.rawClientMu.Lock() defer s.rawClientMu.Unlock() - defer s.bufferMu.Unlock() if len(s.messageBuffer) == 0 { return nil @@ -141,7 +150,9 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { select { case <-t.C: // send + s.bufferMu.Lock() err := s.doSend(ctx) + s.bufferMu.Unlock() if err != nil { // log error panic(err) @@ -160,15 +171,18 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { //TODO implement me s.bufferMu.Lock() - if len(s.messageBuffer) == s.opts.MaxBufferedMessages { - s.bufferMu.Unlock() - if err := s.doSend(ctx); err != nil { - return err - } + defer s.bufferMu.Unlock() + err := s.push(s.publishingIdSeq.next(), &msg) + if err != nil && errors.Is(err, errMessageBufferFull) { + return s.doSend(ctx) + } else if err != nil { + // this should never happen + panic(err) } - _ = s.push(s.publishingIdSeq.next(), &msg) - s.bufferMu.Unlock() + if len(s.messageBuffer) == s.opts.MaxBufferedMessages { + return s.doSend(ctx) + } return nil } diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index d2fe91b8..f291a1b7 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -77,7 +77,7 @@ var _ = Describe("Smart Producer", func() { publisherId: 1, rawClient: fakeRawClient, rawClientMu: &sync.Mutex{}, - opts: ProducerOptions{MaxInFlight: 100, MaxBufferedMessages: 100}, + opts: &ProducerOptions{MaxInFlight: 100, MaxBufferedMessages: 100}, } // test @@ -97,7 +97,7 @@ var _ = Describe("Smart Producer", func() { rawClient: fakeRawClient, rawClientMu: &sync.Mutex{}, publishingIdSeq: autoIncrementingSequence[uint64]{}, - opts: ProducerOptions{MaxInFlight: 1, MaxBufferedMessages: 1}, + opts: &ProducerOptions{MaxInFlight: 1, MaxBufferedMessages: 1}, } msgs := make([]amqp.Message, 10) Expect(p.SendBatch(context.Background(), msgs)).To(MatchError(ErrBatchTooLarge)) @@ -134,7 +134,11 @@ var _ = Describe("Smart Producer", func() { return nil }) - p := newStandardProducer(42, fakeRawClient, ProducerOptions{MaxInFlight: 5, MaxBufferedMessages: 5}) + p := newStandardProducer(42, fakeRawClient, &ProducerOptions{ + MaxInFlight: 5, + MaxBufferedMessages: 5, + BatchPublishingDelay: time.Millisecond * 200, // batching delay must be lower than Eventually's timeout + }) Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 1")})).To(Succeed()) Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 2")})).To(Succeed()) @@ -147,5 +151,51 @@ var _ = Describe("Smart Producer", func() { }).Within(time.Second * 1).WithPolling(time.Millisecond * 200).Should(ConsistOf(uint64(0), uint64(1), uint64(2))) }) + It("publishes messages when buffer is full", func() { + m := &sync.Mutex{} + var capturedIds = make([]uint64, 0) + fakeRawClient.EXPECT(). + Send(gomock.AssignableToTypeOf(ctxType), gomock.Eq(uint8(42)), + gomock.All( + gomock.Len(3), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ), + ). + Do(func(_ context.Context, _ uint8, pMessages []common.PublishingMessager) error { + m.Lock() + for i := 0; i < len(pMessages); i++ { + capturedIds = append(capturedIds, pMessages[i].PublishingId()) + } + m.Unlock() + return nil + }). + Times(2) + + p := newStandardProducer(42, fakeRawClient, &ProducerOptions{ + MaxInFlight: 3, + MaxBufferedMessages: 3, + BatchPublishingDelay: time.Minute, // long batch delay so that publishing happens because buffer is full + }) + + Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 1")})).To(Succeed()) + Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 2")})).To(Succeed()) + Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 3")})).To(Succeed()) + Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 4")})).To(Succeed()) + Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 5")})).To(Succeed()) + Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 6")})).To(Succeed()) + Eventually(func() []uint64 { + m.Lock() + defer m.Unlock() + return capturedIds + }).Within(time.Millisecond * 200).WithPolling(time.Millisecond * 20).Should(ConsistOf( + uint64(0), + uint64(1), + uint64(2), + uint64(3), + uint64(4), + uint64(5), + )) + + }) }) }) From 99c2d98881af0acd305abe16484706474aba49dd Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Tue, 3 Oct 2023 16:53:15 +0100 Subject: [PATCH 05/24] Publish confirmation tracker Internal component to track publisher confirmations. Signed-off-by: Aitor Perez Cedres --- pkg/stream/confirmation.go | 41 +++++++++++ pkg/stream/confirmation_test.go | 120 ++++++++++++++++++++++++++++++++ pkg/stream/producer.go | 24 +++++-- pkg/stream/types.go | 10 +-- 4 files changed, 184 insertions(+), 11 deletions(-) create mode 100644 pkg/stream/confirmation.go create mode 100644 pkg/stream/confirmation_test.go diff --git a/pkg/stream/confirmation.go b/pkg/stream/confirmation.go new file mode 100644 index 00000000..b1737425 --- /dev/null +++ b/pkg/stream/confirmation.go @@ -0,0 +1,41 @@ +package stream + +import ( + "fmt" + "sync" +) + +type confirmationTracker struct { + *sync.Mutex + messages map[uint64]PublishingMessage +} + +func (p *confirmationTracker) add(m PublishingMessage) { + p.Lock() + defer p.Unlock() + id := m.PublishingId() + p.messages[id] = m +} + +func (p *confirmationTracker) addMany(m ...PublishingMessage) { + if len(m) == 0 { + return + } + + p.Lock() + defer p.Unlock() + for _, message := range m { + p.messages[message.PublishingId()] = message + } +} + +func (p *confirmationTracker) confirm(id uint64) (PublishingMessage, error) { + p.Lock() + defer p.Unlock() + pm, found := p.messages[id] + if !found { + return nil, fmt.Errorf("%w: publishingID %d", ErrUntrackedConfirmation, id) + } + delete(p.messages, id) + return pm, nil +} diff --git a/pkg/stream/confirmation_test.go b/pkg/stream/confirmation_test.go new file mode 100644 index 00000000..465e7b51 --- /dev/null +++ b/pkg/stream/confirmation_test.go @@ -0,0 +1,120 @@ +package stream + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" + "sync" +) + +var _ = Describe("confirmation", func() { + + var ( + ctracker confirmationTracker + ) + + BeforeEach(func() { + ctracker = confirmationTracker{ + Mutex: &sync.Mutex{}, + messages: make(map[uint64]PublishingMessage, 10), + } + }) + + It("adds messages and maps them by publishing ID", func() { + m := &amqp.Message{Data: []byte("message")} + + ctracker.add(raw.NewPublishingMessage(0, m)) + ctracker.add(raw.NewPublishingMessage(1, m)) + ctracker.add(raw.NewPublishingMessage(2, m)) + + Expect(ctracker.messages).To(HaveLen(3)) + Expect(ctracker.messages).To(SatisfyAll( + HaveKey(BeNumerically("==", 0)), + HaveKey(BeNumerically("==", 1)), + HaveKey(BeNumerically("==", 2)), + )) + }) + + When("multiple routines add confirmations", func() { + It("does not race", func() { + // this test is effective when the race detector is active + // use ginkgo --race [...ginkgo args...] + By("adding them one by one") + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func(i int) { + m := &amqp.Message{Data: []byte("message")} + ctracker.add(raw.NewPublishingMessage(uint64(i), m)) + wg.Done() + }(i) + } + wg.Wait() + Expect(ctracker.messages).To(HaveLen(20)) + + By("adding them in batches") + ctracker.messages = make(map[uint64]PublishingMessage, 20) + for i := 0; i < 20; i += 3 { + wg.Add(1) + go func(i int) { + m := &amqp.Message{Data: []byte("message")} + ctracker.addMany( + raw.NewPublishingMessage(uint64(i), m), + raw.NewPublishingMessage(uint64(i+1), m), + raw.NewPublishingMessage(uint64(i+2), m), + ) + wg.Done() + }(i) + } + wg.Wait() + Expect(ctracker.messages).To(HaveLen(21)) + }) + }) + + It("adds many messages", func() { + m := &amqp.Message{Data: []byte("amazing data")} + ctracker.addMany() + + ctracker.addMany(raw.NewPublishingMessage(0, m)) + Expect(ctracker.messages).To(HaveLen(1)) + + ctracker.addMany( + raw.NewPublishingMessage(5, m), + raw.NewPublishingMessage(6, m), + raw.NewPublishingMessage(7, m), + ) + Expect(ctracker.messages).To(HaveLen(4)) + Expect(ctracker.messages).To(SatisfyAll( + HaveKey(BeNumerically("==", 5)), + HaveKey(BeNumerically("==", 6)), + HaveKey(BeNumerically("==", 7)), + )) + }) + + Context("confirm", func() { + var ( + m = &amqp.Message{Data: []byte("superb message")} + ) + + It("confirms one", func() { + ctracker.add(raw.NewPublishingMessage(6, m)) + + pubMsg, err := ctracker.confirm(6) + Expect(err).ToNot(HaveOccurred()) + Expect(pubMsg.PublishingId()).To(BeNumerically("==", 6)) + Expect(pubMsg.Message()).To(Equal(m)) + Expect(ctracker.messages).To(HaveLen(0)) + }) + + When("a message is not tracked", func() { + It("returns an error", func() { + ctracker.add(raw.NewPublishingMessage(1, m)) + + pubMsg, err := ctracker.confirm(123) + Expect(pubMsg).To(BeNil()) + Expect(err).To(MatchError(ContainSubstring("message confirmation not tracked"))) + }) + }) + }) +}) diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 622415e8..0bc2d738 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -51,12 +51,14 @@ type standardProducer struct { publishingIdSeq autoIncrementingSequence[uint64] opts *ProducerOptions // buffer mutex - bufferMu *sync.Mutex - messageBuffer []common.PublishingMessager - retryDuration backoffDurationFunc - done chan struct{} - cancel context.CancelFunc - destructor sync.Once + bufferMu *sync.Mutex + messageBuffer []PublishingMessage + retryDuration backoffDurationFunc + done chan struct{} + cancel context.CancelFunc + destructor sync.Once + unconfirmedMessage confirmationTracker + confirmedPublish chan uint64 } func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *ProducerOptions) *standardProducer { @@ -72,7 +74,12 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *Produc retryDuration: func(i int) time.Duration { return time.Second * (1 << i) }, - done: make(chan struct{}), + done: make(chan struct{}), + confirmedPublish: make(chan uint64), + unconfirmedMessage: confirmationTracker{ + Mutex: &sync.Mutex{}, + messages: make(map[uint64]PublishingMessage, opts.MaxInFlight), + }, } ctx, cancel := context.WithCancel(context.Background()) @@ -168,6 +175,9 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { // Public API +// Send an AMQP 1.0 message asynchronously. Messages are accumulated in a buffer, +// and sent after a delay, or when the buffer becomes full, whichever happens +// first. func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { //TODO implement me s.bufferMu.Lock() diff --git a/pkg/stream/types.go b/pkg/stream/types.go index caac1e77..ea7a983a 100644 --- a/pkg/stream/types.go +++ b/pkg/stream/types.go @@ -17,10 +17,11 @@ const ( ) var ( - ErrNoLocators = errors.New("no locators configured") - ErrUnsupportedOperation = errors.New("unsupported operation") - ErrBatchTooLarge = errors.New("too many messages in batch") - ErrEmptyBatch = errors.New("batch list is empty") + ErrNoLocators = errors.New("no locators configured") + ErrUnsupportedOperation = errors.New("unsupported operation") + ErrBatchTooLarge = errors.New("too many messages in batch") + ErrEmptyBatch = errors.New("batch list is empty") + ErrUntrackedConfirmation = errors.New("message confirmation not tracked") ) type ByteCapacity uint64 @@ -48,3 +49,4 @@ type Producer interface { } type Message = common.Message +type PublishingMessage = common.PublishingMessager From 19c8d3924127cc19f7e5055a0f4d9c0087311bd7 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Wed, 4 Oct 2023 17:49:08 +0100 Subject: [PATCH 06/24] Add a semaphore to confirmation tracker The semaphore provides an simple mechanism to synchronise the addition and release of pending confirmation messages. Given how channels work in Go, the functions to add pending confirmations will block if the semaphore does not have more slots or permits available. Signed-off-by: Aitor Perez Cedres --- pkg/stream/confirmation.go | 41 ++++++++++++++++++++++++++------- pkg/stream/confirmation_test.go | 15 +++++++----- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/pkg/stream/confirmation.go b/pkg/stream/confirmation.go index b1737425..049c93c1 100644 --- a/pkg/stream/confirmation.go +++ b/pkg/stream/confirmation.go @@ -6,36 +6,61 @@ import ( ) type confirmationTracker struct { - *sync.Mutex - messages map[uint64]PublishingMessage + mapMu *sync.Mutex + messages map[uint64]PublishingMessage + unconfirmedMessagesSemaphore chan struct{} + cap int +} + +func newConfirmationTracker(capacity int) *confirmationTracker { + return &confirmationTracker{ + mapMu: &sync.Mutex{}, + messages: make(map[uint64]PublishingMessage, capacity), + unconfirmedMessagesSemaphore: make(chan struct{}, capacity), + cap: capacity, + } } func (p *confirmationTracker) add(m PublishingMessage) { - p.Lock() - defer p.Unlock() + p.mapMu.Lock() + defer p.mapMu.Unlock() + p.unconfirmedMessagesSemaphore <- struct{}{} + id := m.PublishingId() p.messages[id] = m } +// adds a batch of messages to track confirmation. This function +// blocks if: +// +// len(m) + len(confirmationTracker.messages) > cap func (p *confirmationTracker) addMany(m ...PublishingMessage) { if len(m) == 0 { return } - p.Lock() - defer p.Unlock() + p.mapMu.Lock() + defer p.mapMu.Unlock() + + // we should not block here if there's sufficient capacity + // by acquiring the map mutex, we ensure that other functions + // don't "steal" or "race" semaphore's permits + for i := 0; i < len(m); i++ { + p.unconfirmedMessagesSemaphore <- struct{}{} + } for _, message := range m { p.messages[message.PublishingId()] = message } } func (p *confirmationTracker) confirm(id uint64) (PublishingMessage, error) { - p.Lock() - defer p.Unlock() + p.mapMu.Lock() + defer p.mapMu.Unlock() pm, found := p.messages[id] if !found { return nil, fmt.Errorf("%w: publishingID %d", ErrUntrackedConfirmation, id) } + <-p.unconfirmedMessagesSemaphore delete(p.messages, id) return pm, nil } diff --git a/pkg/stream/confirmation_test.go b/pkg/stream/confirmation_test.go index 465e7b51..0d9967ce 100644 --- a/pkg/stream/confirmation_test.go +++ b/pkg/stream/confirmation_test.go @@ -11,14 +11,11 @@ import ( var _ = Describe("confirmation", func() { var ( - ctracker confirmationTracker + ctracker *confirmationTracker ) BeforeEach(func() { - ctracker = confirmationTracker{ - Mutex: &sync.Mutex{}, - messages: make(map[uint64]PublishingMessage, 10), - } + ctracker = newConfirmationTracker(10) }) It("adds messages and maps them by publishing ID", func() { @@ -34,6 +31,7 @@ var _ = Describe("confirmation", func() { HaveKey(BeNumerically("==", 1)), HaveKey(BeNumerically("==", 2)), )) + Expect(ctracker.unconfirmedMessagesSemaphore).To(HaveLen(3)) }) When("multiple routines add confirmations", func() { @@ -41,6 +39,7 @@ var _ = Describe("confirmation", func() { // this test is effective when the race detector is active // use ginkgo --race [...ginkgo args...] By("adding them one by one") + ctracker = newConfirmationTracker(20) var wg sync.WaitGroup for i := 0; i < 20; i++ { wg.Add(1) @@ -54,7 +53,7 @@ var _ = Describe("confirmation", func() { Expect(ctracker.messages).To(HaveLen(20)) By("adding them in batches") - ctracker.messages = make(map[uint64]PublishingMessage, 20) + ctracker = newConfirmationTracker(21) for i := 0; i < 20; i += 3 { wg.Add(1) go func(i int) { @@ -78,6 +77,7 @@ var _ = Describe("confirmation", func() { ctracker.addMany(raw.NewPublishingMessage(0, m)) Expect(ctracker.messages).To(HaveLen(1)) + Expect(ctracker.unconfirmedMessagesSemaphore).To(HaveLen(1)) ctracker.addMany( raw.NewPublishingMessage(5, m), @@ -90,6 +90,7 @@ var _ = Describe("confirmation", func() { HaveKey(BeNumerically("==", 6)), HaveKey(BeNumerically("==", 7)), )) + Expect(ctracker.unconfirmedMessagesSemaphore).To(HaveLen(4)) }) Context("confirm", func() { @@ -105,6 +106,7 @@ var _ = Describe("confirmation", func() { Expect(pubMsg.PublishingId()).To(BeNumerically("==", 6)) Expect(pubMsg.Message()).To(Equal(m)) Expect(ctracker.messages).To(HaveLen(0)) + Expect(ctracker.unconfirmedMessagesSemaphore).To(HaveLen(0)) }) When("a message is not tracked", func() { @@ -114,6 +116,7 @@ var _ = Describe("confirmation", func() { pubMsg, err := ctracker.confirm(123) Expect(pubMsg).To(BeNil()) Expect(err).To(MatchError(ContainSubstring("message confirmation not tracked"))) + Expect(ctracker.unconfirmedMessagesSemaphore).To(HaveLen(1)) }) }) }) From 67a89978028f336957c7e747dd8f1b9371387801 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Wed, 4 Oct 2023 18:01:30 +0100 Subject: [PATCH 07/24] Ground work for publisher confirms Signed-off-by: Aitor Perez Cedres --- pkg/stream/producer.go | 34 ++++++++++++++++++++-- pkg/stream/producer_test.go | 57 ++++++++++++++++++++++--------------- 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 0bc2d738..1548ec8d 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -30,6 +30,9 @@ type ProducerOptions struct { MaxBufferedMessages int // Period to send a batch of messages. BatchPublishingDelay time.Duration + // Time before enqueueing of a message fail when the maximum number of + // unconfirmed is reached. + EnqueueTimeout time.Duration } func (p *ProducerOptions) validate() { @@ -42,6 +45,9 @@ func (p *ProducerOptions) validate() { if p.BatchPublishingDelay == 0 { p.BatchPublishingDelay = defaultBatchPublishingDelay } + if p.EnqueueTimeout < 0 { + p.EnqueueTimeout = time.Second * 10 + } } type standardProducer struct { @@ -66,7 +72,7 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *Produc p := &standardProducer{ publisherId: publisherId, rawClient: rawClient, - rawClientMu: &sync.Mutex{}, + rawClientMu: &sync.Mutex{}, // FIXME: this has to come as argument. this mutex must be shared among all users of the client publishingIdSeq: autoIncrementingSequence[uint64]{}, opts: opts, bufferMu: &sync.Mutex{}, @@ -77,7 +83,7 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *Produc done: make(chan struct{}), confirmedPublish: make(chan uint64), unconfirmedMessage: confirmationTracker{ - Mutex: &sync.Mutex{}, + mapMu: &sync.Mutex{}, messages: make(map[uint64]PublishingMessage, opts.MaxInFlight), }, } @@ -86,6 +92,7 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *Produc p.cancel = cancel go p.sendLoopAsync(ctx) + go p.confirmationListenerAsync() return p } @@ -165,6 +172,9 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { panic(err) } case <-s.done: + // exit + t.Stop() + return case <-ctx.Done(): // exit t.Stop() @@ -173,6 +183,18 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { } } +func (s *standardProducer) confirmationListenerAsync() { + for { + select { + case <-s.done: + return + case id := <-s.confirmedPublish: + _, _ = s.unconfirmedMessage.confirm(id) + // TODO: invoke a callback, or send a notification to a channel, or log the error, or all of the above (or change confirm() to no-op instead of error) or make this comment shorter :) + } + } +} + // Public API // Send an AMQP 1.0 message asynchronously. Messages are accumulated in a buffer, @@ -182,7 +204,15 @@ func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { //TODO implement me s.bufferMu.Lock() defer s.bufferMu.Unlock() + + // if not max in flight + // then push + // else return error fail publish + //var err error err := s.push(s.publishingIdSeq.next(), &msg) + if s.opts.EnqueueTimeout != 0 { + + } if err != nil && errors.Is(err, errMessageBufferFull) { return s.doSend(ctx) } else if err != nil { diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index f291a1b7..9424f800 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -26,15 +26,17 @@ var _ = Describe("Smart Producer", func() { }) Describe("send batch", func() { + var ( + p *standardProducer + ) + + AfterEach(func() { + p.close() + }) + When("the batch list is empty", func() { It("returns an error", func() { - p := standardProducer{ - publisherId: 0, - rawClient: fakeRawClient, - rawClientMu: &sync.Mutex{}, - publishingIdSeq: autoIncrementingSequence[uint64]{}, - } - + p = newStandardProducer(0, fakeRawClient, &ProducerOptions{}) Expect(p.SendBatch(context.Background(), []amqp.Message{})).To(MatchError("batch list is empty")) }) }) @@ -73,12 +75,10 @@ var _ = Describe("Smart Producer", func() { }), ) - p := standardProducer{ - publisherId: 1, - rawClient: fakeRawClient, - rawClientMu: &sync.Mutex{}, - opts: &ProducerOptions{MaxInFlight: 100, MaxBufferedMessages: 100}, - } + p = newStandardProducer(1, fakeRawClient, &ProducerOptions{ + MaxInFlight: 100, + MaxBufferedMessages: 100, + }) // test batch := []amqp.Message{{Data: []byte("message 1")}} @@ -92,13 +92,10 @@ var _ = Describe("Smart Producer", func() { When("the batch list is larger than max in flight", func() { It("returns an error", func() { - p := standardProducer{ - publisherId: 0, - rawClient: fakeRawClient, - rawClientMu: &sync.Mutex{}, - publishingIdSeq: autoIncrementingSequence[uint64]{}, - opts: &ProducerOptions{MaxInFlight: 1, MaxBufferedMessages: 1}, - } + p = newStandardProducer(0, fakeRawClient, &ProducerOptions{ + MaxInFlight: 1, + MaxBufferedMessages: 1, + }) msgs := make([]amqp.Message, 10) Expect(p.SendBatch(context.Background(), msgs)).To(MatchError(ErrBatchTooLarge)) }) @@ -115,6 +112,14 @@ var _ = Describe("Smart Producer", func() { }) Describe("send", func() { + var ( + p *standardProducer + ) + + AfterEach(func() { + p.close() + }) + It("accumulates and sends messages", func() { m := &sync.Mutex{} var capturedIds = make([]uint64, 0) @@ -134,7 +139,7 @@ var _ = Describe("Smart Producer", func() { return nil }) - p := newStandardProducer(42, fakeRawClient, &ProducerOptions{ + p = newStandardProducer(42, fakeRawClient, &ProducerOptions{ MaxInFlight: 5, MaxBufferedMessages: 5, BatchPublishingDelay: time.Millisecond * 200, // batching delay must be lower than Eventually's timeout @@ -152,6 +157,7 @@ var _ = Describe("Smart Producer", func() { }) It("publishes messages when buffer is full", func() { + // setup m := &sync.Mutex{} var capturedIds = make([]uint64, 0) fakeRawClient.EXPECT(). @@ -171,12 +177,17 @@ var _ = Describe("Smart Producer", func() { }). Times(2) - p := newStandardProducer(42, fakeRawClient, &ProducerOptions{ - MaxInFlight: 3, + p = newStandardProducer(42, fakeRawClient, &ProducerOptions{ + MaxInFlight: 10, MaxBufferedMessages: 3, BatchPublishingDelay: time.Minute, // long batch delay so that publishing happens because buffer is full }) + DeferCleanup(func() { + p.close() + }) + + // act Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 1")})).To(Succeed()) Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 2")})).To(Succeed()) Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 3")})).To(Succeed()) From f064e2556202b6b6fa4d8e4f8b995bdffa448dc4 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 5 Oct 2023 17:30:56 +0100 Subject: [PATCH 08/24] Refactor publisher message queue and confirm tracker Channels synchronise naturally, and also act as a synchronised queue. Those two properties make channels an excellent choice to implement a semaphore and a message queue buffer. Signed-off-by: Aitor Perez Cedres --- pkg/stream/confirmation.go | 15 +++++ pkg/stream/message_accumulator.go | 55 ++++++++++++++++ pkg/stream/producer.go | 104 +++++++++++++----------------- pkg/stream/producer_test.go | 4 -- pkg/stream/types.go | 1 + 5 files changed, 115 insertions(+), 64 deletions(-) create mode 100644 pkg/stream/message_accumulator.go diff --git a/pkg/stream/confirmation.go b/pkg/stream/confirmation.go index 049c93c1..7c63c423 100644 --- a/pkg/stream/confirmation.go +++ b/pkg/stream/confirmation.go @@ -3,6 +3,7 @@ package stream import ( "fmt" "sync" + "time" ) type confirmationTracker struct { @@ -30,6 +31,20 @@ func (p *confirmationTracker) add(m PublishingMessage) { p.messages[id] = m } +func (p *confirmationTracker) addWithTimeout(m PublishingMessage, timeout time.Duration) error { + p.mapMu.Lock() + defer p.mapMu.Unlock() + + select { + case p.unconfirmedMessagesSemaphore <- struct{}{}: + id := m.PublishingId() + p.messages[id] = m + return nil + case <-time.After(timeout): + return ErrEnqueueTimeout + } +} + // adds a batch of messages to track confirmation. This function // blocks if: // diff --git a/pkg/stream/message_accumulator.go b/pkg/stream/message_accumulator.go new file mode 100644 index 00000000..d88de1fd --- /dev/null +++ b/pkg/stream/message_accumulator.go @@ -0,0 +1,55 @@ +package stream + +import ( + "errors" + "time" +) + +type messageAccumulator struct { + messages chan PublishingMessage + cap int +} + +func newMessageAccumulator(capacity int) *messageAccumulator { + return &messageAccumulator{cap: capacity, messages: make(chan PublishingMessage, capacity)} +} + +// addWithTimeout a message to the queue. If the queue is full, it will wait at +// least, timeout duration before unblocking and returning an error. Returns true +// if this queue operation filled the message queue +func (m *messageAccumulator) addWithTimeout(message PublishingMessage, timeout time.Duration) (bool, error) { + if m == nil { + return false, errors.New("message can't be nil") + } + select { + case m.messages <- message: + return len(m.messages) == m.cap, nil + case <-time.After(timeout): + return false, ErrEnqueueTimeout + } +} + +// add queues a message into the message queue. It blocks if the queue is full. +// returns true if this queueing operation filled the queue to max capacity +func (m *messageAccumulator) add(message PublishingMessage) (bool, error) { + if message == nil { + return false, errors.New("message can't be nil") + } + + m.messages <- message + return len(m.messages) == m.cap, nil +} + +// get the head of the queue or nil if it's empty +func (m *messageAccumulator) get() PublishingMessage { + select { + case pm := <-m.messages: + return pm + default: + return nil + } +} + +func (m *messageAccumulator) isEmpty() bool { + return len(m.messages) == 0 +} diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 1548ec8d..32a630c2 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -51,19 +51,17 @@ func (p *ProducerOptions) validate() { } type standardProducer struct { - publisherId uint8 - rawClient raw.Clienter - rawClientMu *sync.Mutex - publishingIdSeq autoIncrementingSequence[uint64] - opts *ProducerOptions - // buffer mutex - bufferMu *sync.Mutex - messageBuffer []PublishingMessage + publisherId uint8 + rawClient raw.Clienter + rawClientMu *sync.Mutex + publishingIdSeq autoIncrementingSequence[uint64] + opts *ProducerOptions + accumulator *messageAccumulator retryDuration backoffDurationFunc done chan struct{} cancel context.CancelFunc destructor sync.Once - unconfirmedMessage confirmationTracker + unconfirmedMessage *confirmationTracker confirmedPublish chan uint64 } @@ -75,17 +73,13 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *Produc rawClientMu: &sync.Mutex{}, // FIXME: this has to come as argument. this mutex must be shared among all users of the client publishingIdSeq: autoIncrementingSequence[uint64]{}, opts: opts, - bufferMu: &sync.Mutex{}, - messageBuffer: make([]common.PublishingMessager, 0, opts.MaxBufferedMessages), + accumulator: newMessageAccumulator(opts.MaxBufferedMessages), retryDuration: func(i int) time.Duration { return time.Second * (1 << i) }, - done: make(chan struct{}), - confirmedPublish: make(chan uint64), - unconfirmedMessage: confirmationTracker{ - mapMu: &sync.Mutex{}, - messages: make(map[uint64]PublishingMessage, opts.MaxInFlight), - }, + done: make(chan struct{}), + confirmedPublish: make(chan uint64), + unconfirmedMessage: newConfirmationTracker(opts.MaxInFlight), } ctx, cancel := context.WithCancel(context.Background()) @@ -103,15 +97,6 @@ func (s *standardProducer) close() { }) } -// push is not thread-safe. A bufferMu lock must be acquired before calling this function. -func (s *standardProducer) push(publishingId uint64, message Message) error { - if len(s.messageBuffer) >= s.opts.MaxBufferedMessages { - return errMessageBufferFull - } - s.messageBuffer = append(s.messageBuffer, raw.NewPublishingMessage(publishingId, message)) - return nil -} - // synchronously sends the messages accumulated in the message buffer. If sending // is successful, it clears the message buffer. The caller MUST hold a lock // on the buffer mutex. Calling this function without a lock on buffer mutex is @@ -120,33 +105,25 @@ func (s *standardProducer) doSend(ctx context.Context) error { s.rawClientMu.Lock() defer s.rawClientMu.Unlock() - if len(s.messageBuffer) == 0 { + if s.accumulator.isEmpty() { return nil } - var err error - for i := 0; i < maxAttempt; i++ { - err = s.rawClient.Send(ctx, s.publisherId, s.messageBuffer) - if isNonRetryableError(err) { - return err - } - if err == nil { + // TODO: explore if we can have this buffer in a sync.Pool + messages := make([]PublishingMessage, 0, s.opts.MaxBufferedMessages) + for batchSize := 0; batchSize < s.opts.MaxBufferedMessages; batchSize++ { + pm := s.accumulator.get() + if pm == nil { break } - - <-time.After(s.retryDuration(i)) + messages = append(messages, pm) } + err := s.rawClient.Send(ctx, s.publisherId, messages) if err != nil { return err } - for i := 0; i < len(s.messageBuffer); i++ { - // FIXME: keep track of pending messages in a different list - s.messageBuffer[i] = nil - } - s.messageBuffer = s.messageBuffer[:0] - return nil } @@ -164,9 +141,7 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { select { case <-t.C: // send - s.bufferMu.Lock() err := s.doSend(ctx) - s.bufferMu.Unlock() if err != nil { // log error panic(err) @@ -202,26 +177,35 @@ func (s *standardProducer) confirmationListenerAsync() { // first. func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { //TODO implement me - s.bufferMu.Lock() - defer s.bufferMu.Unlock() - - // if not max in flight - // then push - // else return error fail publish - //var err error - err := s.push(s.publishingIdSeq.next(), &msg) + var send bool if s.opts.EnqueueTimeout != 0 { + var err error + pm := raw.NewPublishingMessage(s.publishingIdSeq.next(), &msg) + err = s.unconfirmedMessage.addWithTimeout(pm, s.opts.EnqueueTimeout) + if err != nil { + return err + } - } - if err != nil && errors.Is(err, errMessageBufferFull) { - return s.doSend(ctx) - } else if err != nil { - // this should never happen - panic(err) + send, err = s.accumulator.addWithTimeout(pm, s.opts.EnqueueTimeout) + if err != nil { + return fmt.Errorf("error sending message: %w", err) + } + } else { + pm := raw.NewPublishingMessage(s.publishingIdSeq.next(), &msg) + s.unconfirmedMessage.add(pm) + + var err error + send, err = s.accumulator.add(pm) + if err != nil { + return fmt.Errorf("error sending message: %w", err) + } } - if len(s.messageBuffer) == s.opts.MaxBufferedMessages { - return s.doSend(ctx) + if send { + err := s.doSend(ctx) + if err != nil { + return err + } } return nil diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index 9424f800..ab88e9a6 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -183,10 +183,6 @@ var _ = Describe("Smart Producer", func() { BatchPublishingDelay: time.Minute, // long batch delay so that publishing happens because buffer is full }) - DeferCleanup(func() { - p.close() - }) - // act Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 1")})).To(Succeed()) Expect(p.Send(context.Background(), amqp.Message{Data: []byte("message 2")})).To(Succeed()) diff --git a/pkg/stream/types.go b/pkg/stream/types.go index ea7a983a..75bd9a20 100644 --- a/pkg/stream/types.go +++ b/pkg/stream/types.go @@ -22,6 +22,7 @@ var ( ErrBatchTooLarge = errors.New("too many messages in batch") ErrEmptyBatch = errors.New("batch list is empty") ErrUntrackedConfirmation = errors.New("message confirmation not tracked") + ErrEnqueueTimeout = errors.New("timed out queueing message") ) type ByteCapacity uint64 From c3f84680a3870fbf0763df6948201054f6bcbc22 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 19 Oct 2023 16:09:01 +0100 Subject: [PATCH 09/24] Add dedicated type for message confirmation We want to provide a better abstraction for message confirmations in the smart layer. The publish confirm frame sends a potentially long list of IDs that are confirmed. In the stream protocol, a confirm for ID N does not confirm messages with IDs < N. Therefore, we have to keep track of all pending publishing IDs, and associate them with their AMQP message. Signed-off-by: Aitor Perez Cedres --- pkg/stream/confirmation.go | 64 ++++++++++++++++-- pkg/stream/confirmation_test.go | 115 ++++++++++++++++++++++++++------ pkg/stream/producer.go | 35 ++++++++-- 3 files changed, 180 insertions(+), 34 deletions(-) diff --git a/pkg/stream/confirmation.go b/pkg/stream/confirmation.go index 7c63c423..227da802 100644 --- a/pkg/stream/confirmation.go +++ b/pkg/stream/confirmation.go @@ -2,13 +2,65 @@ package stream import ( "fmt" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" "sync" "time" ) +type ConfirmationStatus int + +const ( + // WaitingConfirmation for a publishing message + WaitingConfirmation ConfirmationStatus = 0 + // Confirmed and received message by the server + Confirmed ConfirmationStatus = 1 + // ClientTimeout and gave up waiting for a confirmation + ClientTimeout ConfirmationStatus = 2 + // NotAvailable Stream, often because stream was deleted + NotAvailable ConfirmationStatus = 6 + // InternalError from the server + InternalError ConfirmationStatus = 15 + // TODO: do we need this? + AccessRefused ConfirmationStatus = 16 + PreconditionFailed ConfirmationStatus = 17 + PublisherDoesNotExist ConfirmationStatus = 18 + UndefinedError ConfirmationStatus = 200 +) + +// TODO: docs +type MessageConfirmation struct { + // publishing ID of the message/s in this confirmation + publishingId uint64 + // list of AMQP messages in this confirmation. Its len will be 1, except with + // sub-batch entries + messages []amqp.Message + // status of the confirmation + status ConfirmationStatus + // time when this message confirmation was created + insert time.Time + // stream name where the message/s were published to + stream string +} + +func (m *MessageConfirmation) PublishingId() uint64 { + return m.publishingId +} + +func (m *MessageConfirmation) Messages() []amqp.Message { + return m.messages +} + +func (m *MessageConfirmation) Status() ConfirmationStatus { + return m.status +} + +func (m *MessageConfirmation) Stream() string { + return m.stream +} + type confirmationTracker struct { mapMu *sync.Mutex - messages map[uint64]PublishingMessage + messages map[uint64]*MessageConfirmation unconfirmedMessagesSemaphore chan struct{} cap int } @@ -16,13 +68,13 @@ type confirmationTracker struct { func newConfirmationTracker(capacity int) *confirmationTracker { return &confirmationTracker{ mapMu: &sync.Mutex{}, - messages: make(map[uint64]PublishingMessage, capacity), + messages: make(map[uint64]*MessageConfirmation, capacity), unconfirmedMessagesSemaphore: make(chan struct{}, capacity), cap: capacity, } } -func (p *confirmationTracker) add(m PublishingMessage) { +func (p *confirmationTracker) add(m *MessageConfirmation) { p.mapMu.Lock() defer p.mapMu.Unlock() p.unconfirmedMessagesSemaphore <- struct{}{} @@ -31,7 +83,7 @@ func (p *confirmationTracker) add(m PublishingMessage) { p.messages[id] = m } -func (p *confirmationTracker) addWithTimeout(m PublishingMessage, timeout time.Duration) error { +func (p *confirmationTracker) addWithTimeout(m *MessageConfirmation, timeout time.Duration) error { p.mapMu.Lock() defer p.mapMu.Unlock() @@ -49,7 +101,7 @@ func (p *confirmationTracker) addWithTimeout(m PublishingMessage, timeout time.D // blocks if: // // len(m) + len(confirmationTracker.messages) > cap -func (p *confirmationTracker) addMany(m ...PublishingMessage) { +func (p *confirmationTracker) addMany(m ...*MessageConfirmation) { if len(m) == 0 { return } @@ -68,7 +120,7 @@ func (p *confirmationTracker) addMany(m ...PublishingMessage) { } } -func (p *confirmationTracker) confirm(id uint64) (PublishingMessage, error) { +func (p *confirmationTracker) confirm(id uint64) (*MessageConfirmation, error) { p.mapMu.Lock() defer p.mapMu.Unlock() pm, found := p.messages[id] diff --git a/pkg/stream/confirmation_test.go b/pkg/stream/confirmation_test.go index 0d9967ce..e22d6adc 100644 --- a/pkg/stream/confirmation_test.go +++ b/pkg/stream/confirmation_test.go @@ -4,8 +4,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" - "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "sync" + "time" ) var _ = Describe("confirmation", func() { @@ -19,11 +19,29 @@ var _ = Describe("confirmation", func() { }) It("adds messages and maps them by publishing ID", func() { - m := &amqp.Message{Data: []byte("message")} - - ctracker.add(raw.NewPublishingMessage(0, m)) - ctracker.add(raw.NewPublishingMessage(1, m)) - ctracker.add(raw.NewPublishingMessage(2, m)) + m := amqp.Message{Data: []byte("message")} + + ctracker.add( + &MessageConfirmation{ + publishingId: 0, + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "foo", + }) + ctracker.add( + &MessageConfirmation{ + publishingId: 1, + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "foo", + }) + ctracker.add( + &MessageConfirmation{ + publishingId: 2, + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "foo", + }) Expect(ctracker.messages).To(HaveLen(3)) Expect(ctracker.messages).To(SatisfyAll( @@ -44,8 +62,13 @@ var _ = Describe("confirmation", func() { for i := 0; i < 20; i++ { wg.Add(1) go func(i int) { - m := &amqp.Message{Data: []byte("message")} - ctracker.add(raw.NewPublishingMessage(uint64(i), m)) + m := amqp.Message{Data: []byte("message")} + ctracker.add(&MessageConfirmation{ + publishingId: uint64(i), + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }) wg.Done() }(i) } @@ -57,11 +80,26 @@ var _ = Describe("confirmation", func() { for i := 0; i < 20; i += 3 { wg.Add(1) go func(i int) { - m := &amqp.Message{Data: []byte("message")} + m := amqp.Message{Data: []byte("message")} ctracker.addMany( - raw.NewPublishingMessage(uint64(i), m), - raw.NewPublishingMessage(uint64(i+1), m), - raw.NewPublishingMessage(uint64(i+2), m), + &MessageConfirmation{ + publishingId: uint64(i), + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }, + &MessageConfirmation{ + publishingId: uint64(i + 1), + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }, + &MessageConfirmation{ + publishingId: uint64(i + 2), + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }, ) wg.Done() }(i) @@ -72,17 +110,37 @@ var _ = Describe("confirmation", func() { }) It("adds many messages", func() { - m := &amqp.Message{Data: []byte("amazing data")} + m := amqp.Message{Data: []byte("amazing data")} ctracker.addMany() - ctracker.addMany(raw.NewPublishingMessage(0, m)) + ctracker.addMany(&MessageConfirmation{ + publishingId: 0, + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }) Expect(ctracker.messages).To(HaveLen(1)) Expect(ctracker.unconfirmedMessagesSemaphore).To(HaveLen(1)) ctracker.addMany( - raw.NewPublishingMessage(5, m), - raw.NewPublishingMessage(6, m), - raw.NewPublishingMessage(7, m), + &MessageConfirmation{ + publishingId: 5, + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }, + &MessageConfirmation{ + publishingId: 6, + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }, + &MessageConfirmation{ + publishingId: 7, + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }, ) Expect(ctracker.messages).To(HaveLen(4)) Expect(ctracker.messages).To(SatisfyAll( @@ -95,23 +153,38 @@ var _ = Describe("confirmation", func() { Context("confirm", func() { var ( - m = &amqp.Message{Data: []byte("superb message")} + m = amqp.Message{Data: []byte("superb message")} ) It("confirms one", func() { - ctracker.add(raw.NewPublishingMessage(6, m)) + ctracker.add(&MessageConfirmation{ + publishingId: 6, + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }) pubMsg, err := ctracker.confirm(6) Expect(err).ToNot(HaveOccurred()) Expect(pubMsg.PublishingId()).To(BeNumerically("==", 6)) - Expect(pubMsg.Message()).To(Equal(m)) + Expect(pubMsg.Messages()).To( + SatisfyAll( + HaveLen(1), + ContainElement(m), + ), + ) Expect(ctracker.messages).To(HaveLen(0)) Expect(ctracker.unconfirmedMessagesSemaphore).To(HaveLen(0)) }) When("a message is not tracked", func() { It("returns an error", func() { - ctracker.add(raw.NewPublishingMessage(1, m)) + ctracker.add(&MessageConfirmation{ + publishingId: 1, + messages: []amqp.Message{m}, + insert: time.Time{}, + stream: "my-stream", + }) pubMsg, err := ctracker.confirm(123) Expect(pubMsg).To(BeNil()) diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 32a630c2..17df7170 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -33,6 +33,10 @@ type ProducerOptions struct { // Time before enqueueing of a message fail when the maximum number of // unconfirmed is reached. EnqueueTimeout time.Duration + // TODO: docs + ConfirmationHandler func(confirmation *MessageConfirmation) + // Used internally. Must be set by the producer manager + stream string } func (p *ProducerOptions) validate() { @@ -51,8 +55,12 @@ func (p *ProducerOptions) validate() { } type standardProducer struct { - publisherId uint8 - rawClient raw.Clienter + publisherId uint8 + // shared Raw Client connection among all clients in the same manager + rawClient raw.Clienter + // this mutex must be shared among all components that have access to this + // rawClient. The Raw Client is not thread-safe and its access must be + // synchronised rawClientMu *sync.Mutex publishingIdSeq autoIncrementingSequence[uint64] opts *ProducerOptions @@ -62,7 +70,9 @@ type standardProducer struct { cancel context.CancelFunc destructor sync.Once unconfirmedMessage *confirmationTracker - confirmedPublish chan uint64 + // this channel is used by the producer manager to send confirmation notifications + // the end-user does not have access to this channel + confirmedPublish chan uint64 } func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *ProducerOptions) *standardProducer { @@ -86,7 +96,7 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *Produc p.cancel = cancel go p.sendLoopAsync(ctx) - go p.confirmationListenerAsync() + go p.confirmationListenerLoop() return p } @@ -158,13 +168,14 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { } } -func (s *standardProducer) confirmationListenerAsync() { +func (s *standardProducer) confirmationListenerLoop() { for { select { case <-s.done: return case id := <-s.confirmedPublish: _, _ = s.unconfirmedMessage.confirm(id) + // Probably a callback // TODO: invoke a callback, or send a notification to a channel, or log the error, or all of the above (or change confirm() to no-op instead of error) or make this comment shorter :) } } @@ -181,7 +192,12 @@ func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { if s.opts.EnqueueTimeout != 0 { var err error pm := raw.NewPublishingMessage(s.publishingIdSeq.next(), &msg) - err = s.unconfirmedMessage.addWithTimeout(pm, s.opts.EnqueueTimeout) + err = s.unconfirmedMessage.addWithTimeout(&MessageConfirmation{ + publishingId: pm.PublishingId(), + messages: []amqp.Message{msg}, + insert: time.Now(), + stream: s.opts.stream, + }, s.opts.EnqueueTimeout) if err != nil { return err } @@ -192,7 +208,12 @@ func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { } } else { pm := raw.NewPublishingMessage(s.publishingIdSeq.next(), &msg) - s.unconfirmedMessage.add(pm) + s.unconfirmedMessage.add(&MessageConfirmation{ + publishingId: pm.PublishingId(), + messages: []amqp.Message{msg}, + insert: time.Now(), + stream: s.opts.stream, + }) var err error send, err = s.accumulator.add(pm) From e55ec5df71ee7e824059670d0897ed0fc2e86882 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 19 Oct 2023 16:15:58 +0100 Subject: [PATCH 10/24] Refactor Makefile for local binaries To use a local binary version, and avoid changing the system binary for Ginkgo CLI that may be present. The same for MockGen. Signed-off-by: Aitor Perez Cedres --- .gitignore | 1 + Makefile | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 5cc44465..d5422624 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ coverage.txt # Dependency directories (remove the comment below to include it) # vendor/ +/bin/ diff --git a/Makefile b/Makefile index 9490699a..c4787f4f 100644 --- a/Makefile +++ b/Makefile @@ -21,23 +21,27 @@ help: GO ?= $(shell which go) GOPATH ?= $(shell $(GO) env GOPATH) +GOBIN ?= $(CURDIR)/bin +$(GOBIN): + mkdir -pv $(GOBIN) + define GO_TOOLS -"github.com/golang/mock/mockgen" \ +"go.uber.org/mock/mockgen" \ "github.com/onsi/ginkgo/v2/ginkgo" endef -GINKGO ?= $(GOPATH)/bin/ginkgo -$(GINKGO): +GINKGO ?= $(GOBIN)/ginkgo +$(GINKGO): | $(GOBIN) @printf "$(GREEN)Installing ginkgo CLI$(NORMAL)\n" - $(GO) install -mod=mod github.com/onsi/ginkgo/v2/ginkgo + GOBIN="$(GOBIN)" $(GO) install -mod=mod github.com/onsi/ginkgo/v2/ginkgo .PHONY: ginkgo ginkgo: | $(GINKGO) -MOCKGEN ?= $(GOPATH)/bin/mockgen +MOCKGEN ?= $(GOBIN)/mockgen $(MOCKGEN): @printf "$(GREEN)Installing mockgen CLI$(NORMAL)\n" - $(GO) install -mod=mod go.uber.org/mock/mockgen + GOBIN="$(GOBIN)" $(GO) install -mod=mod go.uber.org/mock/mockgen .PHONY: mockgen mockgen: | $(MOCKGEN) From 13d8f24eadcbf41a32f7287c85f7e6e45c6c91cc Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 19 Oct 2023 18:02:29 +0100 Subject: [PATCH 11/24] Message confirmations in Send Signed-off-by: Aitor Perez Cedres --- pkg/stream/producer.go | 19 ++++++-- pkg/stream/producer_test.go | 94 +++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 4 deletions(-) diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 17df7170..831eb323 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -88,8 +88,8 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *Produc return time.Second * (1 << i) }, done: make(chan struct{}), - confirmedPublish: make(chan uint64), unconfirmedMessage: newConfirmationTracker(opts.MaxInFlight), + confirmedPublish: make(chan uint64), } ctx, cancel := context.WithCancel(context.Background()) @@ -174,9 +174,16 @@ func (s *standardProducer) confirmationListenerLoop() { case <-s.done: return case id := <-s.confirmedPublish: - _, _ = s.unconfirmedMessage.confirm(id) - // Probably a callback - // TODO: invoke a callback, or send a notification to a channel, or log the error, or all of the above (or change confirm() to no-op instead of error) or make this comment shorter :) + msgConfirm, err := s.unconfirmedMessage.confirm(id) + if err != nil { + // TODO: log the error instead + panic(err) + } + if s.opts.ConfirmationHandler != nil { + msgConfirm.status = Confirmed + s.opts.ConfirmationHandler(msgConfirm) + } + // TODO: do we need an else { msgConfirm = nil } to ease the job of the GC? } } } @@ -225,6 +232,10 @@ func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { if send { err := s.doSend(ctx) if err != nil { + // at this point, we are tracking messages as unconfirmed, + // however, it is very likely they have never reached the broker + // a background worker will have to time out the confirmation + // This situation makes the infinite enqueue timeout dangerous return err } } diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index ab88e9a6..f20e4ed4 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -100,6 +100,12 @@ var _ = Describe("Smart Producer", func() { Expect(p.SendBatch(context.Background(), msgs)).To(MatchError(ErrBatchTooLarge)) }) }) + + When("the pending confirmations + the batch is larger than max in flight", func() { + It("returns an error", func() { + Skip("TODO") + }) + }) }) Describe("send with ID", func() { @@ -204,5 +210,93 @@ var _ = Describe("Smart Producer", func() { )) }) + + Context("message confirmations", Focus, func() { + It("calls the confirmation handler", func() { + // setup + pingBack := make(chan MessageConfirmation, 1) + wait := make(chan struct{}) + fakeRawClient.EXPECT(). + Send(gomock.AssignableToTypeOf(ctxType), + gomock.AssignableToTypeOf(uint8(42)), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ).DoAndReturn(func(_ context.Context, _ uint8, _ []common.PublishingMessager) error { + close(wait) + return nil + }) + + p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ + MaxInFlight: 10, + MaxBufferedMessages: 10, + BatchPublishingDelay: time.Millisecond * 50, // we want fast batching + EnqueueTimeout: 0, + ConfirmationHandler: func(confirm *MessageConfirmation) { + pingBack <- *confirm + }, + stream: "test-stream", + }) + + // routines started, and should not be sending (there's nothing to send) + Consistently(pingBack).ShouldNot(Receive()) + + Expect( + p.Send(context.Background(), amqp.Message{Data: []byte("rabbitmq is awesome")}), + ).To(Succeed()) + select { + case <-wait: + case <-time.After(time.Second): + Fail("expected to be unblocked by the mock, but we are still waiting") + } + + // faking the producer manager receiving and forwarding a publish confirm + p.confirmedPublish <- 0 + + var mc MessageConfirmation + Eventually(pingBack).Should(Receive(&mc)) + Expect(mc.stream).To(Equal("test-stream")) + Expect(mc.publishingId).To(BeNumerically("==", 0)) + Expect(mc.status).To(Equal(Confirmed)) + Expect(mc.messages).To(HaveLen(1)) + Expect(mc.messages[0].Data).To(BeEquivalentTo("rabbitmq is awesome")) + }) + + When("the pending confirmations + the batch is larger than max in flight", func() { + It("returns an error", func() { + // setup + wait := make(chan struct{}) + fakeRawClient.EXPECT(). + Send(gomock.AssignableToTypeOf(ctxType), + gomock.AssignableToTypeOf(uint8(42)), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ).DoAndReturn(func(_ context.Context, _ uint8, _ []common.PublishingMessager) error { + close(wait) + return nil + }) + + p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ + MaxInFlight: 3, + MaxBufferedMessages: 3, + BatchPublishingDelay: time.Millisecond * 5000, // we want to publish on max buffered + EnqueueTimeout: time.Millisecond * 10, // we want to time out quickly + ConfirmationHandler: nil, + stream: "test-stream", + }) + + // act + message := amqp.Message{Data: []byte("rabbitmq is the best messaging broker")} + Expect(p.Send(context.Background(), message)).To(Succeed()) + Expect(p.Send(context.Background(), message)).To(Succeed()) + Expect(p.Send(context.Background(), message)).To(Succeed()) + + select { + case <-wait: + case <-time.After(time.Second): + Fail("time out waiting for the mock to unblock us") + } + + Expect(p.Send(context.Background(), message)).To(MatchError(ErrEnqueueTimeout)) + }) + }) + }) }) }) From e239a2373c1d944b94683f1933d719eeaa6aae45 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Mon, 23 Oct 2023 13:56:33 +0100 Subject: [PATCH 12/24] Refactor publish confirmations Signed-off-by: Aitor Perez Cedres --- pkg/stream/confirmation.go | 22 +++++++++++++++ pkg/stream/message_accumulator.go | 4 ++- pkg/stream/producer.go | 46 ++++++++++++++----------------- pkg/stream/producer_test.go | 32 ++++++++++++++++++--- pkg/stream/types.go | 1 + 5 files changed, 74 insertions(+), 31 deletions(-) diff --git a/pkg/stream/confirmation.go b/pkg/stream/confirmation.go index 227da802..5feb292c 100644 --- a/pkg/stream/confirmation.go +++ b/pkg/stream/confirmation.go @@ -74,6 +74,8 @@ func newConfirmationTracker(capacity int) *confirmationTracker { } } +// Adds a message confirmation to the tracker. It blocks if the maximum +// number of pending confirmations is reached. func (p *confirmationTracker) add(m *MessageConfirmation) { p.mapMu.Lock() defer p.mapMu.Unlock() @@ -83,16 +85,34 @@ func (p *confirmationTracker) add(m *MessageConfirmation) { p.messages[id] = m } +// Tracks a pending message confirmation. It uses timeout to wait when the +// maximum number of pending message confirmations is reached. If timeout is 0, +// then it tries to add the message confirmation to the tracker, and returns an +// error if the tracker is full i.e. max number of pending confirmations is +// reached. It also returns an error if the timeout elapses and the message is +// not added to the tracker. func (p *confirmationTracker) addWithTimeout(m *MessageConfirmation, timeout time.Duration) error { p.mapMu.Lock() defer p.mapMu.Unlock() + if timeout == 0 { + select { + case p.unconfirmedMessagesSemaphore <- struct{}{}: + id := m.PublishingId() + p.messages[id] = m + return nil + default: + return ErrMaxMessagesInFlight + } + } + select { case p.unconfirmedMessagesSemaphore <- struct{}{}: id := m.PublishingId() p.messages[id] = m return nil case <-time.After(timeout): + // maybe it's better to return ErrMaxMessagesInFlight return ErrEnqueueTimeout } } @@ -120,6 +140,8 @@ func (p *confirmationTracker) addMany(m ...*MessageConfirmation) { } } +// Removes the message confirmation for a publishing id. It returns +// an error if the message was not tracked. func (p *confirmationTracker) confirm(id uint64) (*MessageConfirmation, error) { p.mapMu.Lock() defer p.mapMu.Unlock() diff --git a/pkg/stream/message_accumulator.go b/pkg/stream/message_accumulator.go index d88de1fd..9f86bf12 100644 --- a/pkg/stream/message_accumulator.go +++ b/pkg/stream/message_accumulator.go @@ -8,6 +8,8 @@ import ( type messageAccumulator struct { messages chan PublishingMessage cap int + // Accumulator is thread-safe because it uses a channel. A channel synchronises + // itself } func newMessageAccumulator(capacity int) *messageAccumulator { @@ -18,7 +20,7 @@ func newMessageAccumulator(capacity int) *messageAccumulator { // least, timeout duration before unblocking and returning an error. Returns true // if this queue operation filled the message queue func (m *messageAccumulator) addWithTimeout(message PublishingMessage, timeout time.Duration) (bool, error) { - if m == nil { + if message == nil { return false, errors.New("message can't be nil") } select { diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 831eb323..22ec8e60 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -7,6 +7,7 @@ import ( "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/common" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" + "log" "sync" "time" ) @@ -108,9 +109,7 @@ func (s *standardProducer) close() { } // synchronously sends the messages accumulated in the message buffer. If sending -// is successful, it clears the message buffer. The caller MUST hold a lock -// on the buffer mutex. Calling this function without a lock on buffer mutex is -// undefined behaviour +// is successful, it clears the message buffer. func (s *standardProducer) doSend(ctx context.Context) error { s.rawClientMu.Lock() defer s.rawClientMu.Unlock() @@ -127,6 +126,19 @@ func (s *standardProducer) doSend(ctx context.Context) error { break } messages = append(messages, pm) + + m := pm.Message().(*amqp.Message) + err := s.unconfirmedMessage.addWithTimeout(&MessageConfirmation{ + publishingId: pm.PublishingId(), + messages: []amqp.Message{*m}, + status: WaitingConfirmation, + insert: time.Now(), + stream: s.opts.stream, + }, s.opts.EnqueueTimeout) + if err != nil { + // TODO: log error + break + } } err := s.rawClient.Send(ctx, s.publisherId, messages) @@ -153,8 +165,9 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { // send err := s.doSend(ctx) if err != nil { - // log error - panic(err) + // FIXME: log error using logger + //panic(err) + log.Printf("error sending: %v", err) } case <-s.done: // exit @@ -198,32 +211,13 @@ func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { var send bool if s.opts.EnqueueTimeout != 0 { var err error - pm := raw.NewPublishingMessage(s.publishingIdSeq.next(), &msg) - err = s.unconfirmedMessage.addWithTimeout(&MessageConfirmation{ - publishingId: pm.PublishingId(), - messages: []amqp.Message{msg}, - insert: time.Now(), - stream: s.opts.stream, - }, s.opts.EnqueueTimeout) - if err != nil { - return err - } - - send, err = s.accumulator.addWithTimeout(pm, s.opts.EnqueueTimeout) + send, err = s.accumulator.addWithTimeout(raw.NewPublishingMessage(s.publishingIdSeq.next(), &msg), s.opts.EnqueueTimeout) if err != nil { return fmt.Errorf("error sending message: %w", err) } } else { - pm := raw.NewPublishingMessage(s.publishingIdSeq.next(), &msg) - s.unconfirmedMessage.add(&MessageConfirmation{ - publishingId: pm.PublishingId(), - messages: []amqp.Message{msg}, - insert: time.Now(), - stream: s.opts.stream, - }) - var err error - send, err = s.accumulator.add(pm) + send, err = s.accumulator.add(raw.NewPublishingMessage(s.publishingIdSeq.next(), &msg)) if err != nil { return fmt.Errorf("error sending message: %w", err) } diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index f20e4ed4..5a284732 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -211,7 +211,7 @@ var _ = Describe("Smart Producer", func() { }) - Context("message confirmations", Focus, func() { + Context("message confirmations", func() { It("calls the confirmation handler", func() { // setup pingBack := make(chan MessageConfirmation, 1) @@ -260,8 +260,8 @@ var _ = Describe("Smart Producer", func() { Expect(mc.messages[0].Data).To(BeEquivalentTo("rabbitmq is awesome")) }) - When("the pending confirmations + the batch is larger than max in flight", func() { - It("returns an error", func() { + When("the pending confirmations is greater or equal than max in flight", func() { + It("returns an error after a timeout", func() { // setup wait := make(chan struct{}) fakeRawClient.EXPECT(). @@ -294,7 +294,31 @@ var _ = Describe("Smart Producer", func() { Fail("time out waiting for the mock to unblock us") } - Expect(p.Send(context.Background(), message)).To(MatchError(ErrEnqueueTimeout)) + //Expect(p.Send(context.Background(), message)).To(MatchError(ErrEnqueueTimeout)) + Expect(p.Send(context.Background(), message)).To(Succeed()) + }) + + It("waits until some messages are confirmed", Pending, func() { + // setup + wait := make(chan struct{}) + fakeRawClient.EXPECT(). + Send(gomock.AssignableToTypeOf(ctxType), + gomock.AssignableToTypeOf(uint8(42)), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ).DoAndReturn(func(_ context.Context, _ uint8, _ []common.PublishingMessager) error { + close(wait) + return nil + }) + + p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ + MaxInFlight: 2, + MaxBufferedMessages: 3, + BatchPublishingDelay: time.Millisecond * 5000, // we want to publish on max buffered + EnqueueTimeout: time.Millisecond * 10, // we want to time out quickly + ConfirmationHandler: nil, + stream: "test-stream", + }) + }) }) }) diff --git a/pkg/stream/types.go b/pkg/stream/types.go index 75bd9a20..35218915 100644 --- a/pkg/stream/types.go +++ b/pkg/stream/types.go @@ -23,6 +23,7 @@ var ( ErrEmptyBatch = errors.New("batch list is empty") ErrUntrackedConfirmation = errors.New("message confirmation not tracked") ErrEnqueueTimeout = errors.New("timed out queueing message") + ErrMaxMessagesInFlight = errors.New("maximum number of messages in flight") ) type ByteCapacity uint64 From 717ee7a3e81c1b50d20f5598afc578dd9ad6bcca Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Tue, 24 Oct 2023 17:45:57 +0100 Subject: [PATCH 13/24] Refactor message confirmation At the raw layer, the client uses two different commands for publish confirmation and publish error. At the smart layer, we abstract that away and consider a publish error a "nack" or negative publish confirmation. We have to adapt the channel in the producer, so that it can receive publish confirmations or publish errors in the same channel. Signed-off-by: Aitor Perez Cedres --- internal/constants.go | 1 - pkg/raw/client.go | 2 ++ pkg/stream/confirmation.go | 51 +++++++++++++++++++++++++++++-------- pkg/stream/producer.go | 28 +++++++++++++++----- pkg/stream/producer_test.go | 40 +++++++++-------------------- 5 files changed, 76 insertions(+), 46 deletions(-) delete mode 100644 internal/constants.go diff --git a/internal/constants.go b/internal/constants.go deleted file mode 100644 index 5bf0569c..00000000 --- a/internal/constants.go +++ /dev/null @@ -1 +0,0 @@ -package internal diff --git a/pkg/raw/client.go b/pkg/raw/client.go index a38a0ae7..56ac861a 100644 --- a/pkg/raw/client.go +++ b/pkg/raw/client.go @@ -207,6 +207,8 @@ func (tc *Client) handleIncoming(ctx context.Context) error { for { select { case <-ctx.Done(): + // FIXME: most of the time, cancelling the context will be OK and the intended shutdown + // probably should not return an error, just log that frame handler is stopping log.Info("context cancelled", "reason", ctx.Err()) return ctx.Err() default: diff --git a/pkg/stream/confirmation.go b/pkg/stream/confirmation.go index 5feb292c..af287b4a 100644 --- a/pkg/stream/confirmation.go +++ b/pkg/stream/confirmation.go @@ -3,6 +3,7 @@ package stream import ( "fmt" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "sync" "time" ) @@ -11,20 +12,25 @@ type ConfirmationStatus int const ( // WaitingConfirmation for a publishing message - WaitingConfirmation ConfirmationStatus = 0 + WaitingConfirmation ConfirmationStatus = iota // Confirmed and received message by the server - Confirmed ConfirmationStatus = 1 + Confirmed // ClientTimeout and gave up waiting for a confirmation - ClientTimeout ConfirmationStatus = 2 + ClientTimeout // NotAvailable Stream, often because stream was deleted - NotAvailable ConfirmationStatus = 6 + NotAvailable // InternalError from the server - InternalError ConfirmationStatus = 15 - // TODO: do we need this? - AccessRefused ConfirmationStatus = 16 - PreconditionFailed ConfirmationStatus = 17 - PublisherDoesNotExist ConfirmationStatus = 18 - UndefinedError ConfirmationStatus = 200 + InternalError + // AccessRefused user did not have permissions to publish to the stream + AccessRefused + // PreconditionFailed means that certain conditions required to publish + // were not met e.g. stream does not exist + PreconditionFailed + // PublisherDoesNotExist happens when the client tries to publish before the + // publisher was registered and assigned an ID + PublisherDoesNotExist + // UndefinedError is for any other error + UndefinedError ) // TODO: docs @@ -58,6 +64,31 @@ func (m *MessageConfirmation) Stream() string { return m.stream } +type publishConfirmOrError struct { + publishingId uint64 + statusCode uint16 +} + +// status translates a raw response code into a ConfirmationStatus +func (p *publishConfirmOrError) status() ConfirmationStatus { + switch p.statusCode { + case raw.ResponseCodeOK: + return Confirmed + case raw.ResponseCodeStreamDoesNotExist, raw.ResponseCodeStreamNotAvailable: + return NotAvailable + case raw.ResponseCodeInternalError: + return InternalError + case raw.ResponseCodeAccessRefused: + return AccessRefused + case raw.ResponseCodePublisherDoesNotExist: + return PublisherDoesNotExist + case raw.ResponseCodePreconditionFailed: + return PreconditionFailed + default: + return UndefinedError + } +} + type confirmationTracker struct { mapMu *sync.Mutex messages map[uint64]*MessageConfirmation diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 22ec8e60..503c1e03 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -73,7 +73,7 @@ type standardProducer struct { unconfirmedMessage *confirmationTracker // this channel is used by the producer manager to send confirmation notifications // the end-user does not have access to this channel - confirmedPublish chan uint64 + confirmedPublish chan *publishConfirmOrError } func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *ProducerOptions) *standardProducer { @@ -90,7 +90,7 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *Produc }, done: make(chan struct{}), unconfirmedMessage: newConfirmationTracker(opts.MaxInFlight), - confirmedPublish: make(chan uint64), + confirmedPublish: make(chan *publishConfirmOrError), } ctx, cancel := context.WithCancel(context.Background()) @@ -114,19 +114,22 @@ func (s *standardProducer) doSend(ctx context.Context) error { s.rawClientMu.Lock() defer s.rawClientMu.Unlock() - if s.accumulator.isEmpty() { + if !s.canSend() { + // TODO: maybe log a message return nil } // TODO: explore if we can have this buffer in a sync.Pool messages := make([]PublishingMessage, 0, s.opts.MaxBufferedMessages) - for batchSize := 0; batchSize < s.opts.MaxBufferedMessages; batchSize++ { + for batchSize := 0; batchSize < s.opts.MaxBufferedMessages && s.canSend(); batchSize++ { pm := s.accumulator.get() if pm == nil { break } messages = append(messages, pm) + // the smart layer only supports AMQP 1.0 message formats + // we should never panic here m := pm.Message().(*amqp.Message) err := s.unconfirmedMessage.addWithTimeout(&MessageConfirmation{ publishingId: pm.PublishingId(), @@ -141,6 +144,12 @@ func (s *standardProducer) doSend(ctx context.Context) error { } } + if len(messages) == 0 { + // this case happens when pending confirms == max in-flight messages + // we return so that we don't send an empty Publish frame + return nil + } + err := s.rawClient.Send(ctx, s.publisherId, messages) if err != nil { return err @@ -149,6 +158,11 @@ func (s *standardProducer) doSend(ctx context.Context) error { return nil } +func (s *standardProducer) canSend() bool { + return len(s.unconfirmedMessage.unconfirmedMessagesSemaphore) != s.opts.MaxInFlight && + !s.accumulator.isEmpty() +} + func (s *standardProducer) sendLoopAsync(ctx context.Context) { var publishingDelay time.Duration if s.opts.BatchPublishingDelay == 0 { @@ -186,14 +200,14 @@ func (s *standardProducer) confirmationListenerLoop() { select { case <-s.done: return - case id := <-s.confirmedPublish: - msgConfirm, err := s.unconfirmedMessage.confirm(id) + case confirmOrError := <-s.confirmedPublish: + msgConfirm, err := s.unconfirmedMessage.confirm(confirmOrError.publishingId) if err != nil { // TODO: log the error instead panic(err) } + msgConfirm.status = confirmOrError.status() if s.opts.ConfirmationHandler != nil { - msgConfirm.status = Confirmed s.opts.ConfirmationHandler(msgConfirm) } // TODO: do we need an else { msgConfirm = nil } to ease the job of the GC? diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index 5a284732..3ae2ed8e 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -248,8 +248,11 @@ var _ = Describe("Smart Producer", func() { Fail("expected to be unblocked by the mock, but we are still waiting") } - // faking the producer manager receiving and forwarding a publish confirm - p.confirmedPublish <- 0 + // faking the producer manager receiving and forwarding a 'publish confirm' + p.confirmedPublish <- &publishConfirmOrError{ + publishingId: 0, + statusCode: 1, + } var mc MessageConfirmation Eventually(pingBack).Should(Receive(&mc)) @@ -261,7 +264,7 @@ var _ = Describe("Smart Producer", func() { }) When("the pending confirmations is greater or equal than max in flight", func() { - It("returns an error after a timeout", func() { + It("does not send the message and keeps the message in the message buffer", func() { // setup wait := make(chan struct{}) fakeRawClient.EXPECT(). @@ -276,7 +279,7 @@ var _ = Describe("Smart Producer", func() { p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ MaxInFlight: 3, MaxBufferedMessages: 3, - BatchPublishingDelay: time.Millisecond * 5000, // we want to publish on max buffered + BatchPublishingDelay: time.Millisecond * 1500, // we want to publish on max buffered EnqueueTimeout: time.Millisecond * 10, // we want to time out quickly ConfirmationHandler: nil, stream: "test-stream", @@ -294,31 +297,12 @@ var _ = Describe("Smart Producer", func() { Fail("time out waiting for the mock to unblock us") } - //Expect(p.Send(context.Background(), message)).To(MatchError(ErrEnqueueTimeout)) Expect(p.Send(context.Background(), message)).To(Succeed()) - }) - - It("waits until some messages are confirmed", Pending, func() { - // setup - wait := make(chan struct{}) - fakeRawClient.EXPECT(). - Send(gomock.AssignableToTypeOf(ctxType), - gomock.AssignableToTypeOf(uint8(42)), - gomock.AssignableToTypeOf([]common.PublishingMessager{}), - ).DoAndReturn(func(_ context.Context, _ uint8, _ []common.PublishingMessager) error { - close(wait) - return nil - }) - - p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ - MaxInFlight: 2, - MaxBufferedMessages: 3, - BatchPublishingDelay: time.Millisecond * 5000, // we want to publish on max buffered - EnqueueTimeout: time.Millisecond * 10, // we want to time out quickly - ConfirmationHandler: nil, - stream: "test-stream", - }) - + Consistently(p.accumulator.messages). + WithPolling(time.Millisecond * 200). + Within(time.Millisecond * 1600). + Should(HaveLen(1)) // 3 messages were sent, 1 message is buffered + Expect(p.unconfirmedMessage.messages).To(HaveLen(3)) // 3 messages were sent, 3 confirmations are pending }) }) }) From d2e9362d1a50fd42101eb23831b727f2063d1bd2 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 26 Oct 2023 13:42:15 +0100 Subject: [PATCH 14/24] Add unconfirmed messages in Send batch Signed-off-by: Aitor Perez Cedres --- pkg/stream/confirmation.go | 3 ++ pkg/stream/producer.go | 26 +++++++++++--- pkg/stream/producer_test.go | 69 +++++++++++++++++++++++++++++++++++-- pkg/stream/util.go | 9 +++++ 4 files changed, 100 insertions(+), 7 deletions(-) diff --git a/pkg/stream/confirmation.go b/pkg/stream/confirmation.go index af287b4a..e8a72d18 100644 --- a/pkg/stream/confirmation.go +++ b/pkg/stream/confirmation.go @@ -64,6 +64,9 @@ func (m *MessageConfirmation) Stream() string { return m.stream } +// this type represents a publishing confirmation. In the smart layer, a +// confirmation can be a negative/error confirmation. It is a similar concept of +// a "nack" or negative ack in AMQP. type publishConfirmOrError struct { publishingId uint64 statusCode uint16 diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 503c1e03..1d416731 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -35,7 +35,7 @@ type ProducerOptions struct { // unconfirmed is reached. EnqueueTimeout time.Duration // TODO: docs - ConfirmationHandler func(confirmation *MessageConfirmation) + ConfirmationHandler func(*MessageConfirmation) // Used internally. Must be set by the producer manager stream string } @@ -159,8 +159,7 @@ func (s *standardProducer) doSend(ctx context.Context) error { } func (s *standardProducer) canSend() bool { - return len(s.unconfirmedMessage.unconfirmedMessagesSemaphore) != s.opts.MaxInFlight && - !s.accumulator.isEmpty() + return len(s.unconfirmedMessage.unconfirmedMessagesSemaphore) != s.opts.MaxInFlight } func (s *standardProducer) sendLoopAsync(ctx context.Context) { @@ -253,6 +252,10 @@ func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { // SendBatch synchronously sends messages to the broker. Each messages gets // a publishing ID assigned automatically. +// +// This function blocks if the number of messages in flight (i.e. not confirmed +// by the broker) is greater than len(messages). This function will observe +// context cancellation func (s *standardProducer) SendBatch(ctx context.Context, messages []amqp.Message) error { if len(messages) == 0 { return ErrEmptyBatch @@ -264,7 +267,22 @@ func (s *standardProducer) SendBatch(ctx context.Context, messages []amqp.Messag var pMsgs = make([]common.PublishingMessager, 0, len(messages)) for i := 0; i < len(messages); i++ { - pMsgs = append(pMsgs, raw.NewPublishingMessage(s.publishingIdSeq.next(), &messages[i])) + // checking context cancellation here because adding a message + // confirmation is blocking + if isContextCancelled(ctx) { + return ctx.Err() + } + + next := s.publishingIdSeq.next() + pMsgs = append(pMsgs, raw.NewPublishingMessage(next, &messages[i])) + + s.unconfirmedMessage.add(&MessageConfirmation{ + publishingId: next, + messages: []amqp.Message{messages[i]}, + status: WaitingConfirmation, + insert: time.Now(), + stream: s.opts.stream, + }) } s.rawClientMu.Lock() diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index 3ae2ed8e..72872bbd 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -2,6 +2,7 @@ package stream import ( "context" + "errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" @@ -102,9 +103,65 @@ var _ = Describe("Smart Producer", func() { }) When("the pending confirmations + the batch is larger than max in flight", func() { - It("returns an error", func() { - Skip("TODO") - }) + It("waits until some messages are confirmed", func(ctx context.Context) { + Skip("coordination is hard") + // setup + gomock.InOrder(fakeRawClient.EXPECT(). + Send( + gomock.AssignableToTypeOf(ctxType), + gomock.Eq(uint8(10)), + gomock.All( + gomock.Len(2), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ), + ), + fakeRawClient.EXPECT(). + Send( + gomock.AssignableToTypeOf(ctxType), + gomock.Eq(uint8(10)), + gomock.All( + gomock.Len(1), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ), + ). + DoAndReturn(func(_ context.Context, _ uint8, pMessages []common.PublishingMessager) error { + if p.opts.MaxInFlight >= 2 { + return errors.New("not permitted to send") + } + return nil + }), + ) + + const testMaxInFlight = 2 + p = newStandardProducer(10, fakeRawClient, &ProducerOptions{ + MaxInFlight: testMaxInFlight, + MaxBufferedMessages: 2, + BatchPublishingDelay: 0, + EnqueueTimeout: 0, + ConfirmationHandler: nil, + stream: "batch-send-test", + }) + + // act + m := amqp.Message{Data: []byte("i am a message!")} + Expect(p.SendBatch(context.Background(), []amqp.Message{m, m})).To(Succeed()) + + go func() { + p.confirmedPublish <- &publishConfirmOrError{ + publishingId: 0, + statusCode: 1, // confirmed đź‘Ť + } + }() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer GinkgoRecover() + defer wg.Done() + Expect(p.SendBatch(ctx, []amqp.Message{m})).To(Succeed()) + }() + wg.Wait() + }, SpecTimeout(time.Second)) }) }) @@ -305,6 +362,12 @@ var _ = Describe("Smart Producer", func() { Expect(p.unconfirmedMessage.messages).To(HaveLen(3)) // 3 messages were sent, 3 confirmations are pending }) }) + + When("number of messages accumulated are greater than max in flight", func() { + It("does-something", func() { + Skip("TODO") + }) + }) }) }) }) diff --git a/pkg/stream/util.go b/pkg/stream/util.go index d6d6e9ba..51ea592a 100644 --- a/pkg/stream/util.go +++ b/pkg/stream/util.go @@ -101,3 +101,12 @@ func validateStringParameter(p string) bool { return true } + +func isContextCancelled(ctx context.Context) bool { + select { + case <-ctx.Done(): + return true + default: + return false + } +} From 8a84d9fb18b13fce42df5ef5d6fd394488d4286b Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Fri, 27 Oct 2023 14:12:45 +0100 Subject: [PATCH 15/24] Confirm timeouts in standard producer Signed-off-by: Aitor Perez Cedres --- pkg/stream/confirmation.go | 38 +++++- pkg/stream/producer.go | 100 ++++++++++---- pkg/stream/producer_test.go | 255 +++++++++++++++++------------------- 3 files changed, 233 insertions(+), 160 deletions(-) diff --git a/pkg/stream/confirmation.go b/pkg/stream/confirmation.go index e8a72d18..9ccf19f3 100644 --- a/pkg/stream/confirmation.go +++ b/pkg/stream/confirmation.go @@ -33,7 +33,13 @@ const ( UndefinedError ) -// TODO: docs +// MessageConfirmation represent a confirmation, or negative confirmation, +// from the broker. The confirmation has the messages that have been confirmed, +// or errored, the publishing ID assigned during Send operation, and the status +// of the confirmation. It is important to inspect the status to determine +// if the message has been confirmed, timed out, or errored. +// +// See also ConfirmationHandler in ProducerOptions. type MessageConfirmation struct { // publishing ID of the message/s in this confirmation publishingId uint64 @@ -187,3 +193,33 @@ func (p *confirmationTracker) confirm(id uint64) (*MessageConfirmation, error) { delete(p.messages, id) return pm, nil } + +// Removes a message confirmation with id from the tracker and returns a message +// confirmation with status ClientTimeout. Returns an error if the id is not +// tracked. +func (p *confirmationTracker) timeoutMessage(id uint64) (*MessageConfirmation, error) { + p.mapMu.Lock() + defer p.mapMu.Unlock() + pm, found := p.messages[id] + if !found { + return nil, fmt.Errorf("%w: publishingID %d", ErrUntrackedConfirmation, id) + } + <-p.unconfirmedMessagesSemaphore + delete(p.messages, id) + pm.status = ClientTimeout + return pm, nil +} + +func (p *confirmationTracker) idsBefore(t time.Time) []uint64 { + var s []uint64 + + p.mapMu.Lock() + for id, c := range p.messages { + if c.insert.Before(t) { + s = append(s, id) + } + } + p.mapMu.Unlock() + + return s +} diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 1d416731..111c8443 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -2,7 +2,6 @@ package stream import ( "context" - "errors" "fmt" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/common" @@ -12,46 +11,74 @@ import ( "time" ) -var ( - errMessageBufferFull = errors.New("message buffer is full") -) - +// Default values for ProducerOptions const ( - defaultBatchPublishingDelay = time.Millisecond * 100 - defaultMaxInFlight = 10_000 - maxBufferedMessages = 100 + DefaultBatchPublishingDelay = time.Millisecond * 100 + DefaultMaxInFlight = 10_000 + DefaultMaxBufferedMessages = 100 + DefaultConfirmTimeout = time.Second * 10 + DefaultEnqueueTimeout = time.Second * 10 ) type ProducerOptions struct { // The maximum number of unconfirmed outbound messages. Producer.Send will start - // blocking when the limit is reached. + // blocking when the limit is reached. Only accepts positive integers. Any negative + // value will be set to the default. MaxInFlight int // The maximum number of messages to accumulate before sending them to the - // broker. + // broker. Only accepts positive integers. Any negative value will be set to the + // default. MaxBufferedMessages int - // Period to send a batch of messages. + // Accumulating period to send a batch of messages. BatchPublishingDelay time.Duration - // Time before enqueueing of a message fail when the maximum number of - // unconfirmed is reached. + // Time before failing a message enqueue. This acts as a timeout to Send a + // message. If the message is not accumulated before this timeout, the publish + // operation will be considered failed. EnqueueTimeout time.Duration - // TODO: docs + // Handler function for publish confirmations. The function receives a pointer + // to a message confirmation. The handler should inspect to confirmation to + // determine whether the message was received, timed out, or else. See also + // ConfirmationStatus. + // + // The following code logs the confirmations received: + // + // opt := &ProducerOptions{ + // ConfirmationHandler: func(c *MessageConfirmation) { + // log.Printf("Received message confirmation: ID: %d, Status: %s", c.PublishingId(), c.Status()) + // // some code + // }, + // } + // + // The handler is invoked synchronously by the background routine handling + // confirms. It is critical to not perform long or expensive operations inside + // the handler, as it will stall the progress of the background routine, and + // subsequent invocations of confirmation handlers. ConfirmationHandler func(*MessageConfirmation) + // Time before the client calls the confirm callback to signal outstanding + // unconfirmed messages timed out. + // + // Set to -1 to disable confirmation timeouts. Any other negative value + // is invalid, and it will be set to the default. + ConfirmTimeout time.Duration // Used internally. Must be set by the producer manager stream string } func (p *ProducerOptions) validate() { if p.MaxInFlight <= 0 { - p.MaxInFlight = defaultMaxInFlight + p.MaxInFlight = DefaultMaxInFlight } if p.MaxBufferedMessages <= 0 { - p.MaxBufferedMessages = maxBufferedMessages + p.MaxBufferedMessages = DefaultMaxBufferedMessages } if p.BatchPublishingDelay == 0 { - p.BatchPublishingDelay = defaultBatchPublishingDelay + p.BatchPublishingDelay = DefaultBatchPublishingDelay } if p.EnqueueTimeout < 0 { - p.EnqueueTimeout = time.Second * 10 + p.EnqueueTimeout = DefaultEnqueueTimeout + } + if p.ConfirmTimeout == 0 || p.ConfirmTimeout < -1 { + p.ConfirmTimeout = DefaultConfirmTimeout } } @@ -98,6 +125,11 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *Produc go p.sendLoopAsync(ctx) go p.confirmationListenerLoop() + + if opts.ConfirmTimeout != -1 { + go p.confirmationTimeoutLoop() + } + return p } @@ -166,7 +198,7 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { var publishingDelay time.Duration if s.opts.BatchPublishingDelay == 0 { // important to make this check because NewTicker(0) panics - publishingDelay = defaultBatchPublishingDelay + publishingDelay = DefaultBatchPublishingDelay } else { publishingDelay = s.opts.BatchPublishingDelay } @@ -176,12 +208,13 @@ func (s *standardProducer) sendLoopAsync(ctx context.Context) { select { case <-t.C: // send - err := s.doSend(ctx) + ctx2, cancel := maybeApplyDefaultTimeout(ctx) + err := s.doSend(ctx2) if err != nil { - // FIXME: log error using logger - //panic(err) + // FIXME: log error using slog.logger log.Printf("error sending: %v", err) } + cancel() case <-s.done: // exit t.Stop() @@ -214,6 +247,29 @@ func (s *standardProducer) confirmationListenerLoop() { } } +func (s *standardProducer) confirmationTimeoutLoop() { + t := time.NewTicker(s.opts.ConfirmTimeout) + + for { + select { + case now := <-t.C: + expiredIds := s.unconfirmedMessage.idsBefore(now.Add(-s.opts.ConfirmTimeout)) + for _, id := range expiredIds { + confirmation, err := s.unconfirmedMessage.timeoutMessage(id) + if err != nil { + // TODO: log error + continue + } + if s.opts.ConfirmationHandler != nil { + s.opts.ConfirmationHandler(confirmation) + } + } + case <-s.done: + return + } + } +} + // Public API // Send an AMQP 1.0 message asynchronously. Messages are accumulated in a buffer, diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index 72872bbd..ec3041c4 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -2,7 +2,6 @@ package stream import ( "context" - "errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" @@ -101,68 +100,6 @@ var _ = Describe("Smart Producer", func() { Expect(p.SendBatch(context.Background(), msgs)).To(MatchError(ErrBatchTooLarge)) }) }) - - When("the pending confirmations + the batch is larger than max in flight", func() { - It("waits until some messages are confirmed", func(ctx context.Context) { - Skip("coordination is hard") - // setup - gomock.InOrder(fakeRawClient.EXPECT(). - Send( - gomock.AssignableToTypeOf(ctxType), - gomock.Eq(uint8(10)), - gomock.All( - gomock.Len(2), - gomock.AssignableToTypeOf([]common.PublishingMessager{}), - ), - ), - fakeRawClient.EXPECT(). - Send( - gomock.AssignableToTypeOf(ctxType), - gomock.Eq(uint8(10)), - gomock.All( - gomock.Len(1), - gomock.AssignableToTypeOf([]common.PublishingMessager{}), - ), - ). - DoAndReturn(func(_ context.Context, _ uint8, pMessages []common.PublishingMessager) error { - if p.opts.MaxInFlight >= 2 { - return errors.New("not permitted to send") - } - return nil - }), - ) - - const testMaxInFlight = 2 - p = newStandardProducer(10, fakeRawClient, &ProducerOptions{ - MaxInFlight: testMaxInFlight, - MaxBufferedMessages: 2, - BatchPublishingDelay: 0, - EnqueueTimeout: 0, - ConfirmationHandler: nil, - stream: "batch-send-test", - }) - - // act - m := amqp.Message{Data: []byte("i am a message!")} - Expect(p.SendBatch(context.Background(), []amqp.Message{m, m})).To(Succeed()) - - go func() { - p.confirmedPublish <- &publishConfirmOrError{ - publishingId: 0, - statusCode: 1, // confirmed đź‘Ť - } - }() - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer GinkgoRecover() - defer wg.Done() - Expect(p.SendBatch(ctx, []amqp.Message{m})).To(Succeed()) - }() - wg.Wait() - }, SpecTimeout(time.Second)) - }) }) Describe("send with ID", func() { @@ -268,10 +205,71 @@ var _ = Describe("Smart Producer", func() { }) - Context("message confirmations", func() { - It("calls the confirmation handler", func() { + }) + + Context("message confirmations", func() { + var ( + p *standardProducer + ) + + AfterEach(func() { + p.close() + }) + + It("calls the confirmation handler", func() { + // setup + pingBack := make(chan MessageConfirmation, 1) + wait := make(chan struct{}) + fakeRawClient.EXPECT(). + Send(gomock.AssignableToTypeOf(ctxType), + gomock.AssignableToTypeOf(uint8(42)), + gomock.AssignableToTypeOf([]common.PublishingMessager{}), + ).DoAndReturn(func(_ context.Context, _ uint8, _ []common.PublishingMessager) error { + close(wait) + return nil + }) + + p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ + MaxInFlight: 10, + MaxBufferedMessages: 10, + BatchPublishingDelay: time.Millisecond * 50, // we want fast batching + EnqueueTimeout: 0, + ConfirmationHandler: func(confirm *MessageConfirmation) { + pingBack <- *confirm + }, + stream: "test-stream", + }) + + // routines started, and should not be sending (there's nothing to send) + Consistently(pingBack).ShouldNot(Receive()) + + Expect( + p.Send(context.Background(), amqp.Message{Data: []byte("rabbitmq is awesome")}), + ).To(Succeed()) + select { + case <-wait: + case <-time.After(time.Second): + Fail("expected to be unblocked by the mock, but we are still waiting") + } + + // faking the producer manager receiving and forwarding a 'publish confirm' + p.confirmedPublish <- &publishConfirmOrError{ + publishingId: 0, + statusCode: 1, + } + + var mc MessageConfirmation + Eventually(pingBack).Should(Receive(&mc)) + Expect(mc.stream).To(Equal("test-stream")) + Expect(mc.publishingId).To(BeNumerically("==", 0)) + Expect(mc.status).To(Equal(Confirmed)) + Expect(mc.messages).To(HaveLen(1)) + Expect(mc.messages[0].Data).To(BeEquivalentTo("rabbitmq is awesome")) + }) + + When("the pending confirmations is greater or equal than max in flight", func() { + It("does not send the message and keeps the message in the message buffer", func() { // setup - pingBack := make(chan MessageConfirmation, 1) wait := make(chan struct{}) fakeRawClient.EXPECT(). Send(gomock.AssignableToTypeOf(ctxType), @@ -283,91 +281,74 @@ var _ = Describe("Smart Producer", func() { }) p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ - MaxInFlight: 10, - MaxBufferedMessages: 10, - BatchPublishingDelay: time.Millisecond * 50, // we want fast batching - EnqueueTimeout: 0, - ConfirmationHandler: func(confirm *MessageConfirmation) { - pingBack <- *confirm - }, - stream: "test-stream", + MaxInFlight: 3, + MaxBufferedMessages: 3, + BatchPublishingDelay: time.Millisecond * 1500, // we want to publish on max buffered + EnqueueTimeout: time.Millisecond * 10, // we want to time out quickly + ConfirmationHandler: nil, + stream: "test-stream", }) - // routines started, and should not be sending (there's nothing to send) - Consistently(pingBack).ShouldNot(Receive()) + // act + message := amqp.Message{Data: []byte("rabbitmq is the best messaging broker")} + Expect(p.Send(context.Background(), message)).To(Succeed()) + Expect(p.Send(context.Background(), message)).To(Succeed()) + Expect(p.Send(context.Background(), message)).To(Succeed()) - Expect( - p.Send(context.Background(), amqp.Message{Data: []byte("rabbitmq is awesome")}), - ).To(Succeed()) select { case <-wait: case <-time.After(time.Second): - Fail("expected to be unblocked by the mock, but we are still waiting") + Fail("time out waiting for the mock to unblock us") } - // faking the producer manager receiving and forwarding a 'publish confirm' - p.confirmedPublish <- &publishConfirmOrError{ - publishingId: 0, - statusCode: 1, - } + Expect(p.Send(context.Background(), message)).To(Succeed()) + Consistently(p.accumulator.messages). + WithPolling(time.Millisecond * 200). + Within(time.Millisecond * 1600). + Should(HaveLen(1)) // 3 messages were sent, 1 message is buffered + Expect(p.unconfirmedMessage.messages).To(HaveLen(3)) // 3 messages were sent, 3 confirmations are pending + }) + }) - var mc MessageConfirmation - Eventually(pingBack).Should(Receive(&mc)) - Expect(mc.stream).To(Equal("test-stream")) - Expect(mc.publishingId).To(BeNumerically("==", 0)) - Expect(mc.status).To(Equal(Confirmed)) - Expect(mc.messages).To(HaveLen(1)) - Expect(mc.messages[0].Data).To(BeEquivalentTo("rabbitmq is awesome")) + When("number of messages accumulated are greater than max in flight", func() { + It("does-something", func() { + Skip("TODO") }) + }) - When("the pending confirmations is greater or equal than max in flight", func() { - It("does not send the message and keeps the message in the message buffer", func() { - // setup - wait := make(chan struct{}) - fakeRawClient.EXPECT(). - Send(gomock.AssignableToTypeOf(ctxType), - gomock.AssignableToTypeOf(uint8(42)), - gomock.AssignableToTypeOf([]common.PublishingMessager{}), - ).DoAndReturn(func(_ context.Context, _ uint8, _ []common.PublishingMessager) error { - close(wait) - return nil - }) - - p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ - MaxInFlight: 3, - MaxBufferedMessages: 3, - BatchPublishingDelay: time.Millisecond * 1500, // we want to publish on max buffered - EnqueueTimeout: time.Millisecond * 10, // we want to time out quickly - ConfirmationHandler: nil, - stream: "test-stream", - }) - - // act - message := amqp.Message{Data: []byte("rabbitmq is the best messaging broker")} - Expect(p.Send(context.Background(), message)).To(Succeed()) - Expect(p.Send(context.Background(), message)).To(Succeed()) - Expect(p.Send(context.Background(), message)).To(Succeed()) - - select { - case <-wait: - case <-time.After(time.Second): - Fail("time out waiting for the mock to unblock us") - } + It("times out messages that do not receive a timely confirmation", func() { + // setup + wait := make(chan struct{}) + fakeRawClient.EXPECT(). + Send(gomock.AssignableToTypeOf(ctxType), + gomock.AssignableToTypeOf(uint8(42)), + gomock.AssignableToTypeOf([]common.PublishingMessager{})). + Times(2) - Expect(p.Send(context.Background(), message)).To(Succeed()) - Consistently(p.accumulator.messages). - WithPolling(time.Millisecond * 200). - Within(time.Millisecond * 1600). - Should(HaveLen(1)) // 3 messages were sent, 1 message is buffered - Expect(p.unconfirmedMessage.messages).To(HaveLen(3)) // 3 messages were sent, 3 confirmations are pending - }) + p = newStandardProducer(123, fakeRawClient, &ProducerOptions{ + MaxInFlight: 10, + MaxBufferedMessages: 10, + BatchPublishingDelay: time.Millisecond * 100, + EnqueueTimeout: time.Millisecond * 100, + ConfirmationHandler: func(c *MessageConfirmation) { + defer GinkgoRecover() + defer close(wait) + Expect(c.status).To(Equal(ClientTimeout)) + Expect(c.stream).To(Equal("some-stream")) + }, + ConfirmTimeout: time.Millisecond * 500, + stream: "some-stream", }) - When("number of messages accumulated are greater than max in flight", func() { - It("does-something", func() { - Skip("TODO") - }) - }) + // act + m := amqp.Message{Data: []byte("hello!")} + Expect(p.Send(context.Background(), m)).To(Succeed()) + Eventually(wait, "1100ms", "100ms").Should(BeClosed()) + + wait = make(chan struct{}) + Expect(p.SendBatch(context.Background(), []amqp.Message{m})) + Eventually(wait, "1100ms", "100ms").Should(BeClosed()) }) }) + }) From 76025f6128f794df5c8f90b5c1621ffae8a76d25 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 2 Nov 2023 14:22:01 +0000 Subject: [PATCH 16/24] Add Producer Manager component This component is responsible for keeping track of producers in the environment. This component also sends the relevant frame to the broker, so that the server registers a new publisher. Signed-off-by: Aitor Perez Cedres --- pkg/stream/environment.go | 16 +++ pkg/stream/producer.go | 13 ++- pkg/stream/producer_manager.go | 116 +++++++++++++++++++++ pkg/stream/producer_manager_test.go | 151 ++++++++++++++++++++++++++++ pkg/stream/producer_test.go | 16 +-- pkg/stream/types.go | 6 +- 6 files changed, 302 insertions(+), 16 deletions(-) create mode 100644 pkg/stream/producer_manager.go create mode 100644 pkg/stream/producer_manager_test.go diff --git a/pkg/stream/environment.go b/pkg/stream/environment.go index 02484ca8..d4a333c5 100644 --- a/pkg/stream/environment.go +++ b/pkg/stream/environment.go @@ -342,3 +342,19 @@ func (e *Environment) QuerySequence(ctx context.Context, reference, stream strin return uint64(0), lastError } + +func (e *Environment) CreateProducer(ctx context.Context, stream string, opts *ProducerOptions) (Producer, error) { + // TODO: calls the producer manager + /* + 1. locate leader for stream + 2. check if a producer manager is connected to leader + 2a. if exists and have capacity, fetch it + 2b. if not exists or not have capacity, continue + 2c. if not found, create a new producer manager + 3. create producer using publisher manager + 4. return producer + + Make a deep copy of ProducerOptions because we are going to modify the 'stream' attribute + */ + panic("implement me!") +} diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index 111c8443..fbc65cfd 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -103,12 +103,12 @@ type standardProducer struct { confirmedPublish chan *publishConfirmOrError } -func newStandardProducer(publisherId uint8, rawClient raw.Clienter, opts *ProducerOptions) *standardProducer { +func newStandardProducer(publisherId uint8, rawClient raw.Clienter, clientM *sync.Mutex, opts *ProducerOptions) *standardProducer { opts.validate() p := &standardProducer{ publisherId: publisherId, rawClient: rawClient, - rawClientMu: &sync.Mutex{}, // FIXME: this has to come as argument. this mutex must be shared among all users of the client + rawClientMu: clientM, publishingIdSeq: autoIncrementingSequence[uint64]{}, opts: opts, accumulator: newMessageAccumulator(opts.MaxBufferedMessages), @@ -349,11 +349,10 @@ func (s *standardProducer) SendBatch(ctx context.Context, messages []amqp.Messag // SendWithId always returns an error in the standard producer because the // publishing ID is tracked internally. +// +// TODO: revisit this. We could let users set their own publishing IDs, and very strongly +// +// advise to use one or the other, but not both func (s *standardProducer) SendWithId(_ context.Context, _ uint64, _ amqp.Message) error { return fmt.Errorf("%w: standard producer does not support sending with ID", ErrUnsupportedOperation) } - -func (s *standardProducer) GetLastPublishedId() uint64 { - //TODO implement me - panic("implement me") -} diff --git a/pkg/stream/producer_manager.go b/pkg/stream/producer_manager.go new file mode 100644 index 00000000..4bdd7f0e --- /dev/null +++ b/pkg/stream/producer_manager.go @@ -0,0 +1,116 @@ +package stream + +import ( + "context" + "errors" + "fmt" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" + "sync" +) + +var ( + errManagerFull = errors.New("producer manager is full") + errManagerClosed = errors.New("producer manager is closed") + errNegativeId = errors.New("negative id") + errIdOutOfBounds = errors.New("id outbounds producer list") +) + +const noReference = "" + +type producerManager struct { + m sync.Mutex + id int + config EnvironmentConfiguration + producers []internalProducer + client raw.Clienter + clientM *sync.Mutex + open bool +} + +func newProducerManager(id int, config EnvironmentConfiguration) *producerManager { + return &producerManager{ + m: sync.Mutex{}, + id: id, + config: config, + producers: make([]internalProducer, 0, config.MaxProducersByConnection), + client: nil, + clientM: &sync.Mutex{}, + open: false, + } +} + +func (p *producerManager) connect(ctx context.Context, uri string) error { + conf, err := raw.NewClientConfiguration(uri) + if err != nil { + return err + } + conf.SetConnectionName(fmt.Sprintf("%s-%d", "rabbitmq-stream-producer", p.id)) + + ctx2, cancel := maybeApplyDefaultTimeout(ctx) + defer cancel() + c, err := raw.DialConfig(ctx2, conf) + if err != nil { + return err + } + + p.client = c + p.open = true + return nil +} + +func (p *producerManager) createProducer(ctx context.Context, producerOpts *ProducerOptions) (Producer, error) { + if !p.open { + return nil, errManagerClosed + } + + var ( + publisherId uint8 + producer internalProducer + ) + if len(p.producers) == 0 { + publisherId = 0 + producer = newStandardProducer(publisherId, p.client, p.clientM, producerOpts) + p.producers = append(p.producers, producer) + } else { + for i := 0; i < len(p.producers); i++ { + if p.producers[i] == nil { + publisherId = uint8(i) + producer = newStandardProducer(publisherId, p.client, p.clientM, producerOpts) + p.producers[i] = producer + break + } + } + // no empty slots, manager at max capacity + if producer == nil && len(p.producers) == p.config.MaxProducersByConnection { + return nil, errManagerFull + } + // no empty slots, manager has capacity + if producer == nil { + publisherId = uint8(len(p.producers)) + producer = newStandardProducer(publisherId, p.client, p.clientM, producerOpts) + p.producers = append(p.producers, producer) + } + } + + ctx2, cancel := maybeApplyDefaultTimeout(ctx) + defer cancel() + p.clientM.Lock() + defer p.clientM.Unlock() + err := p.client.DeclarePublisher(ctx2, publisherId, noReference, producerOpts.stream) + if err != nil { + return nil, err + } + + return producer, nil +} + +func (p *producerManager) removeProducer(id int) error { + if id >= len(p.producers) { + return errIdOutOfBounds + } + if id < 0 { + return errNegativeId + } + p.producers[id] = nil + return nil +} diff --git a/pkg/stream/producer_manager_test.go b/pkg/stream/producer_manager_test.go new file mode 100644 index 00000000..ff2eb921 --- /dev/null +++ b/pkg/stream/producer_manager_test.go @@ -0,0 +1,151 @@ +package stream + +import ( + "context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + "reflect" +) + +var _ = Describe("ProducerManager", func() { + + var ( + mockCtrl *gomock.Controller + fakeRawClient *MockRawClient + ) + + BeforeEach(func() { + mockCtrl = gomock.NewController(GinkgoT()) + fakeRawClient = NewMockRawClient(mockCtrl) + }) + + It("creates new producers", func() { + prepareMockDeclarePublisher(fakeRawClient, 0) + prepareMockDeclarePublisher(fakeRawClient, 1) + prepareMockDeclarePublisher(fakeRawClient, 2) + + pm := newProducerManager(0, EnvironmentConfiguration{ + MaxProducersByConnection: 10, + }) + pm.client = fakeRawClient + pm.open = true + + p, err := pm.createProducer(context.Background(), &ProducerOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(p).ToNot(BeNil()) + + // the same raw client must be shared among all producers in this manager + Expect(p.(*standardProducer).rawClient).To(Equal(pm.client)) + Expect(p.(*standardProducer).rawClientMu).To(Equal(pm.clientM)) + Expect(p.(*standardProducer).publisherId).To(BeEquivalentTo(0)) + Expect(pm.producers).To(HaveLen(1)) + + p2, err := pm.createProducer(context.Background(), &ProducerOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(p2.(*standardProducer).rawClient).To(Equal(pm.client)) + Expect(p2.(*standardProducer).rawClientMu).To(Equal(pm.clientM)) + Expect(p2.(*standardProducer).publisherId).To(BeEquivalentTo(1)) + + p3, err := pm.createProducer(context.Background(), &ProducerOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(p3.(*standardProducer).rawClient).To(Equal(pm.client)) + Expect(p3.(*standardProducer).rawClientMu).To(Equal(pm.clientM)) + Expect(p3.(*standardProducer).publisherId).To(BeEquivalentTo(2)) + Expect(pm.producers).To(HaveLen(3)) + }) + + When("manager is full", func() { + It("errors in producer creation", func() { + prepareMockDeclarePublisher(fakeRawClient, 0) + pm := newProducerManager(0, EnvironmentConfiguration{ + MaxProducersByConnection: 1, + }) + pm.client = fakeRawClient + pm.open = true + + _, err := pm.createProducer(context.Background(), &ProducerOptions{}) + Expect(err).ToNot(HaveOccurred()) + + _, err = pm.createProducer(context.Background(), &ProducerOptions{}) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("producer manager is full"))) + + Expect(pm.producers).To(HaveLen(1)) + }) + }) + + When("manager is closed", func() { + It("does not create new producers", func() { + pm := newProducerManager(0, EnvironmentConfiguration{ + MaxProducersByConnection: 1, + }) + pm.client = fakeRawClient + pm.open = false + + _, err := pm.createProducer(context.Background(), &ProducerOptions{}) + Expect(err).To(MatchError(ContainSubstring("producer manager is closed"))) + }) + }) + + When("producers are added and removed", func() { + It("assigns the correct ID", func() { + prepareMockDeclarePublisher(fakeRawClient, 0) + prepareMockDeclarePublisher(fakeRawClient, 1) + prepareMockDeclarePublisher(fakeRawClient, 1) + prepareMockDeclarePublisher(fakeRawClient, 2) + pm := newProducerManager(0, EnvironmentConfiguration{ + MaxProducersByConnection: 5, + }) + pm.client = fakeRawClient + pm.open = true + + By("reusing publisher ID 1") + opts := &ProducerOptions{ + stream: "test-stream", + } + p, err := pm.createProducer(context.Background(), opts) + Expect(err).ToNot(HaveOccurred()) + Expect(p.(*standardProducer).publisherId).To(BeEquivalentTo(0)) + + p2, err := pm.createProducer(context.Background(), opts) + Expect(err).ToNot(HaveOccurred()) + Expect(p2.(*standardProducer).publisherId).To(BeEquivalentTo(1)) + + Expect(pm.removeProducer(1)).To(Succeed()) + + p3, err := pm.createProducer(context.Background(), opts) + Expect(err).ToNot(HaveOccurred()) + Expect(p3.(*standardProducer).publisherId).To(BeEquivalentTo(1)) + + By("reusing the producer slot #1") + Expect(pm.producers).To(HaveLen(2)) // this ensures we have not appended to the slice + + By("reusing publisher ID 2", func() { + pm.producers = append( + pm.producers, + &standardProducer{publisherId: 2}, + &standardProducer{publisherId: 3}, + &standardProducer{publisherId: 4}, + ) + }) + Expect(pm.removeProducer(2)).To(Succeed()) + p4, err := pm.createProducer(context.Background(), opts) + Expect(err).ToNot(HaveOccurred()) + Expect(p4.(*standardProducer).publisherId).To(BeEquivalentTo(2)) + + By("reusing the producer slot #1") + Expect(pm.producers).To(HaveLen(5)) + }) + }) +}) + +func prepareMockDeclarePublisher(m *MockRawClient, publisherId uint8) { + ctxType := reflect.TypeOf((*context.Context)(nil)).Elem() + m.EXPECT().DeclarePublisher( + gomock.AssignableToTypeOf(ctxType), + gomock.Eq(publisherId), + gomock.AssignableToTypeOf(""), + gomock.AssignableToTypeOf(""), + ) +} diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index ec3041c4..5888d7b9 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -36,7 +36,7 @@ var _ = Describe("Smart Producer", func() { When("the batch list is empty", func() { It("returns an error", func() { - p = newStandardProducer(0, fakeRawClient, &ProducerOptions{}) + p = newStandardProducer(0, fakeRawClient, &sync.Mutex{}, &ProducerOptions{}) Expect(p.SendBatch(context.Background(), []amqp.Message{})).To(MatchError("batch list is empty")) }) }) @@ -75,7 +75,7 @@ var _ = Describe("Smart Producer", func() { }), ) - p = newStandardProducer(1, fakeRawClient, &ProducerOptions{ + p = newStandardProducer(1, fakeRawClient, &sync.Mutex{}, &ProducerOptions{ MaxInFlight: 100, MaxBufferedMessages: 100, }) @@ -92,7 +92,7 @@ var _ = Describe("Smart Producer", func() { When("the batch list is larger than max in flight", func() { It("returns an error", func() { - p = newStandardProducer(0, fakeRawClient, &ProducerOptions{ + p = newStandardProducer(0, fakeRawClient, &sync.Mutex{}, &ProducerOptions{ MaxInFlight: 1, MaxBufferedMessages: 1, }) @@ -139,7 +139,7 @@ var _ = Describe("Smart Producer", func() { return nil }) - p = newStandardProducer(42, fakeRawClient, &ProducerOptions{ + p = newStandardProducer(42, fakeRawClient, &sync.Mutex{}, &ProducerOptions{ MaxInFlight: 5, MaxBufferedMessages: 5, BatchPublishingDelay: time.Millisecond * 200, // batching delay must be lower than Eventually's timeout @@ -177,7 +177,7 @@ var _ = Describe("Smart Producer", func() { }). Times(2) - p = newStandardProducer(42, fakeRawClient, &ProducerOptions{ + p = newStandardProducer(42, fakeRawClient, &sync.Mutex{}, &ProducerOptions{ MaxInFlight: 10, MaxBufferedMessages: 3, BatchPublishingDelay: time.Minute, // long batch delay so that publishing happens because buffer is full @@ -229,7 +229,7 @@ var _ = Describe("Smart Producer", func() { return nil }) - p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ + p = newStandardProducer(12, fakeRawClient, &sync.Mutex{}, &ProducerOptions{ MaxInFlight: 10, MaxBufferedMessages: 10, BatchPublishingDelay: time.Millisecond * 50, // we want fast batching @@ -280,7 +280,7 @@ var _ = Describe("Smart Producer", func() { return nil }) - p = newStandardProducer(12, fakeRawClient, &ProducerOptions{ + p = newStandardProducer(12, fakeRawClient, &sync.Mutex{}, &ProducerOptions{ MaxInFlight: 3, MaxBufferedMessages: 3, BatchPublishingDelay: time.Millisecond * 1500, // we want to publish on max buffered @@ -325,7 +325,7 @@ var _ = Describe("Smart Producer", func() { gomock.AssignableToTypeOf([]common.PublishingMessager{})). Times(2) - p = newStandardProducer(123, fakeRawClient, &ProducerOptions{ + p = newStandardProducer(123, fakeRawClient, &sync.Mutex{}, &ProducerOptions{ MaxInFlight: 10, MaxBufferedMessages: 10, BatchPublishingDelay: time.Millisecond * 100, diff --git a/pkg/stream/types.go b/pkg/stream/types.go index 35218915..52caa570 100644 --- a/pkg/stream/types.go +++ b/pkg/stream/types.go @@ -47,7 +47,11 @@ type Producer interface { Send(ctx context.Context, msg amqp.Message) error SendBatch(ctx context.Context, messages []amqp.Message) error SendWithId(ctx context.Context, publishingId uint64, msg amqp.Message) error - GetLastPublishedId() uint64 +} + +type internalProducer interface { + Producer + close() } type Message = common.Message From b0e35b3f91e1248e702652b8e63416d10d7e6cde Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 2 Nov 2023 14:24:53 +0000 Subject: [PATCH 17/24] Run go fmt Signed-off-by: Aitor Perez Cedres --- pkg/stream/environment.go | 6 +++--- pkg/stream/locator_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/stream/environment.go b/pkg/stream/environment.go index d4a333c5..2d9b305d 100644 --- a/pkg/stream/environment.go +++ b/pkg/stream/environment.go @@ -15,9 +15,9 @@ const ( ) type Environment struct { - configuration EnvironmentConfiguration - locators []*locator - retryPolicy backoffDurationFunc + configuration EnvironmentConfiguration + locators []*locator + retryPolicy backoffDurationFunc locatorSelectSequential bool } diff --git a/pkg/stream/locator_test.go b/pkg/stream/locator_test.go index 7b13498e..9d0e0343 100644 --- a/pkg/stream/locator_test.go +++ b/pkg/stream/locator_test.go @@ -31,7 +31,7 @@ var _ = Describe("Locator", func() { client: nil, isSet: true, clientClose: nil, - retryPolicy: backOffPolicy, + retryPolicy: backOffPolicy, } }) From 44828e535cefcf6314390d353bbf71ce86ff231f Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Thu, 2 Nov 2023 18:21:11 +0000 Subject: [PATCH 18/24] Work in Progress: Smart Producer - Producer status - Export Broker references from raw layer - Use metadata query to connect to the stream leader Signed-off-by: Aitor Perez Cedres --- internal/metadata.go | 36 +++++++ pkg/raw/client.go | 2 +- pkg/raw/client_test.go | 2 +- pkg/raw/client_types.go | 10 +- pkg/raw/client_types_test.go | 6 +- pkg/raw/stream_suite_test.go | 6 +- pkg/raw/test_helpers.go | 13 +++ pkg/raw/util.go | 2 +- pkg/stream/environment.go | 128 +++++++++++++++++++++++- pkg/stream/environment_internal_test.go | 83 +++++++++++++++ pkg/stream/environment_test.go | 1 + pkg/stream/locator.go | 19 +++- pkg/stream/producer.go | 61 ++++++++++- pkg/stream/producer_manager.go | 22 ++-- pkg/stream/producer_test.go | 57 ++++++++++- pkg/stream/system_test.go | 32 +++++- pkg/stream/types.go | 12 ++- 17 files changed, 457 insertions(+), 35 deletions(-) create mode 100644 pkg/stream/environment_internal_test.go diff --git a/internal/metadata.go b/internal/metadata.go index 80ae771e..09445ebb 100644 --- a/internal/metadata.go +++ b/internal/metadata.go @@ -79,12 +79,32 @@ type MetadataResponse struct { streamsMetadata []StreamMetadata } +func (m *MetadataResponse) Brokers() []Broker { + return m.brokers +} + +func (m *MetadataResponse) StreamsMetadata() []StreamMetadata { + return m.streamsMetadata +} + type Broker struct { reference uint16 host string port uint32 } +func (b *Broker) Reference() uint16 { + return b.reference +} + +func (b *Broker) Host() string { + return b.host +} + +func (b *Broker) Port() uint32 { + return b.port +} + type StreamMetadata struct { streamName string responseCode uint16 @@ -92,6 +112,22 @@ type StreamMetadata struct { replicasReferences []uint16 } +func (s *StreamMetadata) StreamName() string { + return s.streamName +} + +func (s *StreamMetadata) ResponseCode() uint16 { + return s.responseCode +} + +func (s *StreamMetadata) LeaderReference() uint16 { + return s.leaderReference +} + +func (s *StreamMetadata) ReplicasReferences() []uint16 { + return s.replicasReferences +} + func NewMetadataResponse(correlationId, port uint32, brokerReference, diff --git a/pkg/raw/client.go b/pkg/raw/client.go index 56ac861a..e12f1cf0 100644 --- a/pkg/raw/client.go +++ b/pkg/raw/client.go @@ -828,7 +828,7 @@ func DefaultDial(connectionTimeout time.Duration) func(network string, addr stri // Connect performs a Stream-protocol handshake to connect to RabbitMQ. On a // successful connect, it returns a nil error and starts listening to incoming -// frames from RabbitMQ. If more than 1 broker is defined in ClientConfiguration, +// frames from RabbitMQ. If more than 1 Broker is defined in ClientConfiguration, // they will be tried sequentially. // // It is recommended to establish a connection via DialConfig, instead of calling diff --git a/pkg/raw/client_test.go b/pkg/raw/client_test.go index 178f6376..a2d0d8df 100644 --- a/pkg/raw/client_test.go +++ b/pkg/raw/client_test.go @@ -128,7 +128,7 @@ var _ = Describe("Client", func() { Should(BeTrue(), "expected connection to be open") By("recording the server properties") - Expect(conf.RabbitmqBrokers().ServerProperties).To(HaveKeyWithValue("product", "mock-rabbitmq")) + Expect(conf.RabbitmqBroker().ServerProperties).To(HaveKeyWithValue("product", "mock-rabbitmq")) By("closing the client connection gracefully") go fakeRabbitMQ.fakeRabbitMQConnectionClose(itCtx) diff --git a/pkg/raw/client_types.go b/pkg/raw/client_types.go index 0e154593..2021eeac 100644 --- a/pkg/raw/client_types.go +++ b/pkg/raw/client_types.go @@ -25,7 +25,7 @@ const ( defaultConnectionTimeout = 30 * time.Second ) -type broker struct { +type Broker struct { Host string Port int Username string @@ -37,7 +37,7 @@ type broker struct { ServerProperties map[string]string } -var defaultBroker = broker{ +var defaultBroker = Broker{ Host: "localhost", Port: 5552, Username: "guest", @@ -49,7 +49,7 @@ var defaultBroker = broker{ } type ClientConfiguration struct { - rabbitmqBroker broker + rabbitmqBroker Broker clientMaxFrameSize uint32 clientHeartbeat uint32 authMechanism []string @@ -70,7 +70,7 @@ func (r *ClientConfiguration) SetDial(dial func(network, addr string) (net.Conn, r.dial = dial } -func (r *ClientConfiguration) RabbitmqBrokers() broker { +func (r *ClientConfiguration) RabbitmqBroker() Broker { return r.rabbitmqBroker } @@ -88,7 +88,7 @@ func (r *ClientConfiguration) SetConnectionName(connectionName string) { func NewClientConfiguration(rabbitmqUrl string) (*ClientConfiguration, error) { builder := &ClientConfiguration{ - rabbitmqBroker: broker{}, + rabbitmqBroker: Broker{}, clientHeartbeat: 60, clientMaxFrameSize: 1_048_576, } diff --git a/pkg/raw/client_types_test.go b/pkg/raw/client_types_test.go index e7dd8f73..5190f078 100644 --- a/pkg/raw/client_types_test.go +++ b/pkg/raw/client_types_test.go @@ -14,7 +14,7 @@ var _ = Describe("ClientTypes", func() { "rabbitmq-stream://foo:bar@localparty.com:4321/party-vhost") Expect(err).ToNot(HaveOccurred()) - brokers := clientConf.RabbitmqBrokers() + brokers := clientConf.RabbitmqBroker() Expect(brokers).To(MatchFields(IgnoreExtras, Fields{ "Host": Equal("localparty.com"), @@ -26,11 +26,11 @@ var _ = Describe("ClientTypes", func() { })) }) - It("accepts zero URLs and returns default broker", func() { + It("accepts zero URLs and returns default Broker", func() { conf, err := raw.NewClientConfiguration("") Expect(err).ToNot(HaveOccurred()) - broker := conf.RabbitmqBrokers() + broker := conf.RabbitmqBroker() Expect(broker).NotTo(BeNil()) Expect(broker).To(MatchFields(IgnoreExtras, Fields{ diff --git a/pkg/raw/stream_suite_test.go b/pkg/raw/stream_suite_test.go index 040abb92..dd401e29 100644 --- a/pkg/raw/stream_suite_test.go +++ b/pkg/raw/stream_suite_test.go @@ -676,9 +676,9 @@ func (rmq *fakeRabbitMQServer) fakeRabbitMQMetadataQuery(ctx context.Context, st frameSize := 4 + // header 4 + // correlationID 4 + // brokers length - 2 + // broker reference - 9 + // broker host - 4 + // broker port + 2 + // Broker reference + 9 + // Broker host + 4 + // Broker port 4 + // streamMetadata length 8 + // streamMetadata streamName 2 + // streamMetadata responseCode diff --git a/pkg/raw/test_helpers.go b/pkg/raw/test_helpers.go index 9d8b3fbd..6c15fe15 100644 --- a/pkg/raw/test_helpers.go +++ b/pkg/raw/test_helpers.go @@ -72,3 +72,16 @@ func (r *ClientConfiguration) SetServerProperties(keyValues ...string) { r.rabbitmqBroker.ServerProperties[keyValues[i]] = keyValues[i+1] } } + +func MetadataResponseForStream(name string) *MetadataResponse { + return internal.NewMetadataResponse( + 123, // correlation-id + 1234, // port + 0, // Broker reference + 1, // response code + 0, // leader reference + "fakehost", // host + name, // stream name + []uint16{0}, // replicas references + ) +} diff --git a/pkg/raw/util.go b/pkg/raw/util.go index cc1d0def..11905884 100644 --- a/pkg/raw/util.go +++ b/pkg/raw/util.go @@ -7,7 +7,7 @@ import ( "strings" ) -func parseURI(uri string) (broker, error) { +func parseURI(uri string) (Broker, error) { builder := defaultBroker if strings.Contains(uri, " ") { diff --git a/pkg/stream/environment.go b/pkg/stream/environment.go index 2d9b305d..a482856b 100644 --- a/pkg/stream/environment.go +++ b/pkg/stream/environment.go @@ -2,10 +2,12 @@ package stream import ( "context" + "errors" "fmt" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "log/slog" "math/rand" + "strings" "time" ) @@ -19,6 +21,7 @@ type Environment struct { locators []*locator retryPolicy backoffDurationFunc locatorSelectSequential bool + producerManagers []*producerManager } func NewEnvironment(ctx context.Context, configuration EnvironmentConfiguration) (*Environment, error) { @@ -356,5 +359,128 @@ func (e *Environment) CreateProducer(ctx context.Context, stream string, opts *P Make a deep copy of ProducerOptions because we are going to modify the 'stream' attribute */ - panic("implement me!") + logger := raw.LoggerFromCtxOrDiscard(ctx).WithGroup("CreateProducer") + rn := rand.Intn(100) + n := len(e.locators) + + // 1. locate leader for stream + var ( + lastError error + l *locator + metadata *raw.MetadataResponse + ) + for i := 0; i < n; i++ { + if e.locatorSelectSequential { + // round robin / sequential + l = e.locators[i] + } else { + // pick at random + l = e.pickLocator((i + rn) % n) + } + result := l.operationQueryStreamMetadata(ctx, []string{stream}) + if result[1] != nil { + lastError = result[1].(error) + if isNonRetryableError(lastError) { + return nil, lastError + } + logger.Error("locator operation failed", slog.Any("error", lastError)) + continue + } + metadata = result[0].(*raw.MetadataResponse) + lastError = nil + break + } + + if lastError != nil { + return nil, fmt.Errorf("locator operation failed: %w", lastError) + } + + //var found = false + //var pm *producerManager + //for _, streamMetadata := range metadata.StreamsMetadata() { + // if streamMetadata.StreamName() == stream { + // found = true + // leaderRef := streamMetadata.LeaderReference() + // brokerRef := metadata.Brokers()[leaderRef] + // + // for i := 0; i < len(e.producerManagers); i++ { + // if e.producerManagers[i].connectionEndpoint.Host == brokerRef.Host() && + // e.producerManagers[i].connectionEndpoint.Port == int(brokerRef.Port()) { + // pm = &e.producerManagers[i] + // break + // } + // } + // if pm != nil { + // + // } + // + // break + // } + //} + broker, err := findLeader(stream, metadata) + if err != nil { + return nil, err + } + + pm, err := e.createProducerManager(ctx, broker) + if err != nil { + return nil, err + } + o := opts.DeepCopy() + o.stream = stream + + // FIXME: check producer manager capacity before selecting a + return pm.createProducer(ctx, o) +} + +// initialises and connects a new producer manager +func (e *Environment) createProducerManager(ctx context.Context, broker raw.Broker) (*producerManager, error) { + logger := raw.LoggerFromCtxOrDiscard(ctx).WithGroup("createProducerManager") + // create new producer manager + pm := newProducerManager(len(e.producerManagers), e.configuration) + e.producerManagers = append(e.producerManagers, pm) + uri := uriFromConfigurationThatContains( + fmt.Sprintf("%s:%d", broker.Host, broker.Port), + &e.configuration, + ) + if len(uri) == 0 { + logger.Debug("did not find an URI for leader in configuration", slog.String("leader-host", broker.Host), slog.Int("leader-port", broker.Port)) + return nil, errors.New("empty uri in create producer manager") + } + + return pm, pm.connect(ctx, uri) +} + +func findLeader(stream string, metadata *raw.MetadataResponse) (raw.Broker, error) { + var leaderRef uint16 + var found = false + for _, streamMetadata := range metadata.StreamsMetadata() { + if streamMetadata.StreamName() == stream { + found = true + leaderRef = streamMetadata.LeaderReference() + break + } + } + if !found { + return raw.Broker{}, errors.New("stream leader not found") + } + + bb := raw.Broker{} + for _, broker := range metadata.Brokers() { + if broker.Reference() == leaderRef { + bb.Host = broker.Host() + bb.Port = int(broker.Port()) + return bb, nil + } + } + return raw.Broker{}, errors.New("broker hosting leader not found") +} + +func uriFromConfigurationThatContains(s string, c *EnvironmentConfiguration) string { + for _, uri := range c.Uris { + if strings.Contains(uri, s) { + return uri + } + } + return "" } diff --git a/pkg/stream/environment_internal_test.go b/pkg/stream/environment_internal_test.go new file mode 100644 index 00000000..55cbbe3a --- /dev/null +++ b/pkg/stream/environment_internal_test.go @@ -0,0 +1,83 @@ +//go:build rabbitmq.stream.test + +package stream + +import ( + "context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" + "go.uber.org/mock/gomock" + "time" +) + +var _ = Describe("Environment Internal Unit Test", Pending, func() { + var ( + env *Environment + fakeRawClient *MockRawClient + mockCtrl *gomock.Controller + backOffPolicyFn = func(_ int) time.Duration { + return time.Millisecond * 10 + } + ) + + BeforeEach(func() { + mockCtrl = gomock.NewController(GinkgoT()) + fakeRawClient = NewMockRawClient(mockCtrl) + + c := NewEnvironmentConfiguration( + WithLazyInitialization(true), + WithUri("rabbitmq-stream://foo:bar@fakehost:1234/my-vhost"), + ) + var err error + env, err = NewEnvironment(context.Background(), c) + Expect(err).ToNot(HaveOccurred()) + + env.AppendLocatorRawClient(fakeRawClient) + env.SetBackoffPolicy(backOffPolicyFn) + }) + + It("creates a new producer", func() { + By("locating the stream leader", func() { + fakeRawClient.EXPECT(). + IsOpen().Return(true) + fakeRawClient.EXPECT(). + MetadataQuery(gomock.Any(), gomock.AssignableToTypeOf([]string{"string-slice"})). + Return(raw.MetadataResponseForStream("some-stream"), nil) + fakeRawClient.EXPECT(). + DeclarePublisher( + gomock.Any(), + gomock.AssignableToTypeOf(uint8(1)), + gomock.AssignableToTypeOf("string"), + gomock.AssignableToTypeOf("string"), + ) + }) + p, err := env.CreateProducer(context.Background(), "some-stream", &ProducerOptions{}) + Expect(err).ToNot(HaveOccurred()) + + By("creating a producer manager connected to the leader") + Expect(env.producerManagers).To(HaveLen(1)) + Expect(env.producerManagers[0].config.Uris).To(ContainElement("rabbitmq-stream://foo:bar@fakehost:1234/my-vhost")) + + By("assigning a producer manager for the producer") + Expect(env.producerManagers[0].producers).To(ContainElement(p)) + }) + + It("returns an error for empty stream name", func() { + Skip("TODO") + }) +}) + +func prepareMockForDeclarePublish(mock *MockRawClient, pubId uint8) { + mock.EXPECT(). + IsOpen(). + Return(true) // from maybeInitializeLocator + + mock.EXPECT(). + DeclarePublisher( + gomock.Any(), + gomock.Eq(pubId), + gomock.AssignableToTypeOf("string"), + gomock.AssignableToTypeOf("string"), + ) +} diff --git a/pkg/stream/environment_test.go b/pkg/stream/environment_test.go index d3ff0644..c93c50dc 100644 --- a/pkg/stream/environment_test.go +++ b/pkg/stream/environment_test.go @@ -528,6 +528,7 @@ var _ = Describe("Environment", func() { }) }) + Context("query partitions", func() { BeforeEach(func() { mockRawClient.EXPECT(). diff --git a/pkg/stream/locator.go b/pkg/stream/locator.go index 6e5b9963..c3996f4a 100644 --- a/pkg/stream/locator.go +++ b/pkg/stream/locator.go @@ -32,8 +32,8 @@ func newLocator(c raw.ClientConfiguration, logger *slog.Logger) *locator { log: logger. WithGroup("locator"). With( - slog.String("host", c.RabbitmqBrokers().Host), - slog.Int("port", c.RabbitmqBrokers().Port), + slog.String("host", c.RabbitmqBroker().Host), + slog.Int("port", c.RabbitmqBroker().Port), ), rawClientConf: c, retryPolicy: defaultBackOffPolicy, @@ -128,7 +128,7 @@ func (l *locator) shutdownHandler() { const rabbitmqVersion311 = "v3.11" func (l *locator) isServer311orMore() bool { - v, ok := l.rawClientConf.RabbitmqBrokers().ServerProperties["version"] + v, ok := l.rawClientConf.RabbitmqBroker().ServerProperties["version"] if !ok { // version not found in server properties // returning false as we can't determine @@ -202,6 +202,7 @@ func (l *locator) operationQueryOffset(args ...any) []any { offset, err := l.client.QueryOffset(ctx, reference, stream) return []any{offset, err} } + func (l *locator) operationPartitions(args ...any) []any { ctx := args[0].(context.Context) superstream := args[1].(string) @@ -216,3 +217,15 @@ func (l *locator) operationQuerySequence(args ...any) []any { pubId, err := l.client.QueryPublisherSequence(ctx, reference, stream) return []any{pubId, err} } + +// Locator operation wrapper for MetadataQuery +// +// Requires context.Context and []string +// +// Returns *MetadataResponse and error +func (l *locator) operationQueryStreamMetadata(args ...any) []any { + ctx := args[0].(context.Context) + streamNames := args[1].([]string) + metadataResponse, err := l.client.MetadataQuery(ctx, streamNames) + return []any{metadataResponse, err} +} diff --git a/pkg/stream/producer.go b/pkg/stream/producer.go index fbc65cfd..8cc12745 100644 --- a/pkg/stream/producer.go +++ b/pkg/stream/producer.go @@ -82,8 +82,22 @@ func (p *ProducerOptions) validate() { } } +func (p *ProducerOptions) DeepCopy() *ProducerOptions { + r := new(ProducerOptions) + r.MaxInFlight = p.MaxInFlight + r.MaxBufferedMessages = p.MaxBufferedMessages + r.BatchPublishingDelay = p.BatchPublishingDelay + r.EnqueueTimeout = p.EnqueueTimeout + r.ConfirmationHandler = p.ConfirmationHandler + r.ConfirmTimeout = p.ConfirmTimeout + r.stream = p.stream + return r +} + type standardProducer struct { + m sync.Mutex publisherId uint8 + status status // shared Raw Client connection among all clients in the same manager rawClient raw.Clienter // this mutex must be shared among all components that have access to this @@ -97,6 +111,7 @@ type standardProducer struct { done chan struct{} cancel context.CancelFunc destructor sync.Once + closeCallback func(int) error unconfirmedMessage *confirmationTracker // this channel is used by the producer manager to send confirmation notifications // the end-user does not have access to this channel @@ -106,7 +121,9 @@ type standardProducer struct { func newStandardProducer(publisherId uint8, rawClient raw.Clienter, clientM *sync.Mutex, opts *ProducerOptions) *standardProducer { opts.validate() p := &standardProducer{ + m: sync.Mutex{}, publisherId: publisherId, + status: open, rawClient: rawClient, rawClientMu: clientM, publishingIdSeq: autoIncrementingSequence[uint64]{}, @@ -133,13 +150,17 @@ func newStandardProducer(publisherId uint8, rawClient raw.Clienter, clientM *syn return p } -func (s *standardProducer) close() { +func (s *standardProducer) shutdown() { s.cancel() s.destructor.Do(func() { close(s.done) }) } +func (s *standardProducer) setCloseCallback(f func(int) error) { + s.closeCallback = f +} + // synchronously sends the messages accumulated in the message buffer. If sending // is successful, it clears the message buffer. func (s *standardProducer) doSend(ctx context.Context) error { @@ -194,6 +215,12 @@ func (s *standardProducer) canSend() bool { return len(s.unconfirmedMessage.unconfirmedMessagesSemaphore) != s.opts.MaxInFlight } +func (s *standardProducer) isOpen() bool { + s.m.Lock() + defer s.m.Unlock() + return s.status == open +} + func (s *standardProducer) sendLoopAsync(ctx context.Context) { var publishingDelay time.Duration if s.opts.BatchPublishingDelay == 0 { @@ -276,7 +303,10 @@ func (s *standardProducer) confirmationTimeoutLoop() { // and sent after a delay, or when the buffer becomes full, whichever happens // first. func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { - //TODO implement me + if !s.isOpen() { + return ErrProducerClosed + } + var send bool if s.opts.EnqueueTimeout != 0 { var err error @@ -313,6 +343,10 @@ func (s *standardProducer) Send(ctx context.Context, msg amqp.Message) error { // by the broker) is greater than len(messages). This function will observe // context cancellation func (s *standardProducer) SendBatch(ctx context.Context, messages []amqp.Message) error { + if !s.isOpen() { + return ErrProducerClosed + } + if len(messages) == 0 { return ErrEmptyBatch } @@ -356,3 +390,26 @@ func (s *standardProducer) SendBatch(ctx context.Context, messages []amqp.Messag func (s *standardProducer) SendWithId(_ context.Context, _ uint64, _ amqp.Message) error { return fmt.Errorf("%w: standard producer does not support sending with ID", ErrUnsupportedOperation) } + +// Close the producer. After calling this function, the producer is considered finished +// and not expected to send anymore messages. Subsequent attempts to send messages will +// result in an error. +func (s *standardProducer) Close() { + // TODO: should we wait for pending confirmations? + s.m.Lock() + if s.status == closing || s.status == closed { + s.m.Unlock() + return + } + s.status = closing + s.m.Unlock() + + s.shutdown() + if s.closeCallback != nil { + _ = s.closeCallback(int(s.publisherId)) + // TODO: log error + } + s.m.Lock() + s.status = closed + s.m.Unlock() +} diff --git a/pkg/stream/producer_manager.go b/pkg/stream/producer_manager.go index 4bdd7f0e..a29c06b5 100644 --- a/pkg/stream/producer_manager.go +++ b/pkg/stream/producer_manager.go @@ -18,13 +18,14 @@ var ( const noReference = "" type producerManager struct { - m sync.Mutex - id int - config EnvironmentConfiguration - producers []internalProducer - client raw.Clienter - clientM *sync.Mutex - open bool + m sync.Mutex + id int + config EnvironmentConfiguration + connectionEndpoint raw.Broker + producers []internalProducer + client raw.Clienter + clientM *sync.Mutex + open bool } func newProducerManager(id int, config EnvironmentConfiguration) *producerManager { @@ -53,11 +54,15 @@ func (p *producerManager) connect(ctx context.Context, uri string) error { return err } + p.connectionEndpoint = conf.RabbitmqBroker() p.client = c p.open = true return nil } +// initialises a producer and sends a declare publisher frame. It returns +// an error if the manager does not have capacity, or if declare publisher +// fails func (p *producerManager) createProducer(ctx context.Context, producerOpts *ProducerOptions) (Producer, error) { if !p.open { return nil, errManagerClosed @@ -65,11 +70,12 @@ func (p *producerManager) createProducer(ctx context.Context, producerOpts *Prod var ( publisherId uint8 - producer internalProducer + producer *standardProducer ) if len(p.producers) == 0 { publisherId = 0 producer = newStandardProducer(publisherId, p.client, p.clientM, producerOpts) + producer.setCloseCallback(p.removeProducer) p.producers = append(p.producers, producer) } else { for i := 0; i < len(p.producers); i++ { diff --git a/pkg/stream/producer_test.go b/pkg/stream/producer_test.go index 5888d7b9..05db9dc6 100644 --- a/pkg/stream/producer_test.go +++ b/pkg/stream/producer_test.go @@ -31,7 +31,7 @@ var _ = Describe("Smart Producer", func() { ) AfterEach(func() { - p.close() + p.shutdown() }) When("the batch list is empty", func() { @@ -117,7 +117,7 @@ var _ = Describe("Smart Producer", func() { ) AfterEach(func() { - p.close() + p.shutdown() }) It("accumulates and sends messages", func() { @@ -204,7 +204,19 @@ var _ = Describe("Smart Producer", func() { )) }) + }) + + When("the producer is closed", func() { + It("does not send messages", func() { + p := newStandardProducer(42, fakeRawClient, &sync.Mutex{}, &ProducerOptions{stream: "some-stream"}) + p.Close() + message := amqp.Message{Data: []byte("not sending")} + Expect(p.Send(context.Background(), message)). + To(MatchError(ContainSubstring("producer is closed"))) + Expect(p.SendBatch(context.Background(), []amqp.Message{message})). + To(MatchError(ContainSubstring("producer is closed"))) + }) }) Context("message confirmations", func() { @@ -213,7 +225,7 @@ var _ = Describe("Smart Producer", func() { ) AfterEach(func() { - p.close() + p.shutdown() }) It("calls the confirmation handler", func() { @@ -351,4 +363,43 @@ var _ = Describe("Smart Producer", func() { }) }) + Describe("producer status", func() { + It("transitions the status", func(ctx context.Context) { + wait := make(chan struct{}) + p := newStandardProducer( + 0, + fakeRawClient, + &sync.Mutex{}, + &ProducerOptions{}, + ) + p.setCloseCallback(func(int) error { + <-time.After(time.Millisecond * 200) + close(wait) + return nil + }) + + By("setting status to open after initialisation") + Expect(p.status).To(Equal(open)) // it's open after creation + + By("setting status to closing after calling Close") + go func() { + defer GinkgoRecover() + Eventually(func() status { + p.m.Lock() + defer p.m.Unlock() + return p.status + }).Should(Equal(closing)) + }() + p.Close() + + By("setting status to closed after calling the callback") + select { + case <-wait: + // happy days! + case <-ctx.Done(): + Fail("expected to receive a callback in Close") + } + Expect(p.status).To(Equal(closed)) + }, SpecTimeout(time.Second)) + }) }) diff --git a/pkg/stream/system_test.go b/pkg/stream/system_test.go index eea05f45..8189ee5c 100644 --- a/pkg/stream/system_test.go +++ b/pkg/stream/system_test.go @@ -3,6 +3,7 @@ package stream_test import ( + "context" "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -11,7 +12,7 @@ import ( "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/common" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/stream" - "golang.org/x/exp/slog" + "log/slog" "sync" "time" ) @@ -24,8 +25,7 @@ var _ = Describe("stream package", func() { defaultRabbitmqUri = "rabbitmq-stream://guest:guest@localhost/%2F" ) It("can create and connect to a stream, publish and receive messages", func(ctx SpecContext) { - h := slog.HandlerOptions{Level: slog.LevelDebug}.NewTextHandler(GinkgoWriter) - debugLogger := slog.New(h) + debugLogger := slog.New(slog.NewTextHandler(GinkgoWriter, &slog.HandlerOptions{Level: slog.LevelDebug})) itCtx := raw.NewContextWithLogger(ctx, *debugLogger) By("creating a new environment") @@ -119,6 +119,32 @@ var _ = Describe("stream package", func() { By("deleting the stream") Expect(env.DeleteStream(itCtx, streamName)).To(Succeed()) }) + + It("connects to RabbitMQ", Focus, func() { + uri := "rabbitmq-stream://localhost:5552" + conf := stream.NewEnvironmentConfiguration(stream.WithUri(uri), stream.WithLazyInitialization(false)) + env, err := stream.NewEnvironment(context.Background(), conf) + Expect(err).ToNot(HaveOccurred()) + streamName := "test-stream" + Expect(env.CreateStream(context.Background(), streamName, stream.CreateStreamOptions{})).To(Succeed()) + // TODO + producer, err := env.CreateProducer(context.Background(), streamName, &stream.ProducerOptions{}) + Expect(err).ToNot(HaveOccurred()) + + DeferCleanup(func() { + producer.Close() + env.Close(context.Background()) + }) + + for i := 0; i < 1_000; i++ { + Expect(producer.Send(context.Background(), amqp.Message{Data: []byte(fmt.Sprintf("Message #%d", i))})).To(Succeed()) + } + + _, err = env.QueryStreamStats(context.Background(), streamName) + Expect(err).ToNot(HaveOccurred()) + + Expect(env.DeleteStream(context.Background(), streamName)).To(Succeed()) + }) }) func wrap[T any](v T) []T { diff --git a/pkg/stream/types.go b/pkg/stream/types.go index 52caa570..08758d4d 100644 --- a/pkg/stream/types.go +++ b/pkg/stream/types.go @@ -24,6 +24,7 @@ var ( ErrUntrackedConfirmation = errors.New("message confirmation not tracked") ErrEnqueueTimeout = errors.New("timed out queueing message") ErrMaxMessagesInFlight = errors.New("maximum number of messages in flight") + ErrProducerClosed = errors.New("producer is closed") ) type ByteCapacity uint64 @@ -47,12 +48,21 @@ type Producer interface { Send(ctx context.Context, msg amqp.Message) error SendBatch(ctx context.Context, messages []amqp.Message) error SendWithId(ctx context.Context, publishingId uint64, msg amqp.Message) error + Close() } type internalProducer interface { Producer - close() + shutdown() } type Message = common.Message type PublishingMessage = common.PublishingMessager + +type status int + +const ( + closed status = iota + closing + open +) From 5f195cee476af47b02910fad71dfd70692780b62 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Wed, 15 Nov 2023 16:34:36 +0000 Subject: [PATCH 19/24] Create Producer from Environment Signed-off-by: Aitor Perez Cedres --- main.go | 2 +- pkg/raw/client.go | 62 ++++++------ pkg/raw/client_types.go | 68 +++++-------- pkg/raw/client_types_test.go | 12 +-- pkg/raw/test_helpers.go | 8 +- pkg/raw/util.go | 7 +- pkg/stream/environment.go | 128 +++++++++++------------- pkg/stream/environment_configuration.go | 17 ++++ pkg/stream/environment_test.go | 2 + pkg/stream/locator.go | 53 ++++++---- pkg/stream/locator_test.go | 14 +-- pkg/stream/producer_manager.go | 14 ++- pkg/stream/system_test.go | 10 +- pkg/stream/types.go | 5 + 14 files changed, 209 insertions(+), 193 deletions(-) diff --git a/main.go b/main.go index 565e7f3d..a9b42502 100644 --- a/main.go +++ b/main.go @@ -73,7 +73,7 @@ func runRawClient() { panic(err) } - config.SetConnectionName("test-connection") + config.ConnectionName = "test-connection" ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() rabbitmqCtx := raw.NewContextWithLogger(ctx, *log) diff --git a/pkg/raw/client.go b/pkg/raw/client.go index e12f1cf0..ac6c9f8b 100644 --- a/pkg/raw/client.go +++ b/pkg/raw/client.go @@ -364,7 +364,6 @@ func (tc *Client) handleIncoming(ctx context.Context) error { case tc.publishErrorCh <- publishError: log.Debug("sent a publish error", "publisherId", publishError.PublisherId()) } - case internal.CommandMetadataUpdate: metadataUpdate := new(internal.MetadataUpdateResponse) err = metadataUpdate.Read(buffer) @@ -571,7 +570,7 @@ func (tc *Client) peerProperties(ctx context.Context) error { log := LoggerFromCtxOrDiscard(ctx).WithGroup("peer properties") log.Debug("starting peer properties") serverPropertiesResponse, err := tc.syncRequest(ctx, - internal.NewPeerPropertiesRequest(tc.configuration.connectionName)) + internal.NewPeerPropertiesRequest(tc.configuration.ConnectionName)) if err != nil { return err } @@ -584,7 +583,7 @@ func (tc *Client) peerProperties(ctx context.Context) error { "properties", response.ServerProperties, ) - tc.configuration.rabbitmqBroker.ServerProperties = response.ServerProperties + tc.configuration.RabbitmqAddr.ServerProperties = response.ServerProperties return streamErrorOrNil(response.ResponseCode()) } @@ -609,7 +608,7 @@ func (tc *Client) saslHandshake(ctx context.Context) error { return err } - tc.configuration.authMechanism = saslMechanismResponse.Mechanisms + tc.configuration.AuthMechanism = saslMechanismResponse.Mechanisms return streamErrorOrNil(saslMechanismResponse.ResponseCode()) } @@ -619,12 +618,12 @@ func (tc *Client) saslAuthenticate(ctx context.Context) error { log := LoggerFromCtxOrDiscard(ctx).WithGroup("sasl authenticate") log.Debug("starting SASL authenticate") - for _, mechanism := range tc.configuration.authMechanism { + for _, mechanism := range tc.configuration.AuthMechanism { if strings.EqualFold(mechanism, "PLAIN") { log.Debug("found PLAIN mechanism as supported") saslPlain := internal.NewSaslPlainMechanism( - tc.configuration.rabbitmqBroker.Username, - tc.configuration.rabbitmqBroker.Password, + tc.configuration.RabbitmqAddr.Username, + tc.configuration.RabbitmqAddr.Password, ) saslAuthReq := internal.NewSaslAuthenticateRequest(mechanism) err := saslAuthReq.SetChallengeResponse(saslPlain) @@ -650,7 +649,7 @@ func (tc *Client) open(ctx context.Context) error { log := LoggerFromCtxOrDiscard(ctx).WithGroup("open") log.Debug("starting open") - rabbit := tc.configuration.rabbitmqBroker + rabbit := tc.configuration.RabbitmqAddr openReq := internal.NewOpenRequest(rabbit.Vhost) openRespCommand, err := tc.syncRequest(ctx, openReq) if err != nil { @@ -754,14 +753,14 @@ func (tc *Client) handleClose(ctx context.Context, req *internal.CloseRequest) e // call Client.Connect() after this function. // // ClientConfiguration must not be nil. ClientConfiguration should be initialised -// using NewClientConfiguration(). A custom dial function can be set using +// using NewClientConfiguration(). A custom Dial function can be set using // ClientConfiguration.SetDial(). Check ClientConfiguration.SetDial() for more -// information. If dial function is not provided, DefaultDial is used with a +// information. If Dial function is not provided, DefaultDial is used with a // timeout of 30 seconds. DefaultDial uses net.Dial // -// If you want to implement a custom timeout or dial-retry mechanism, provide -// your own dial function using [raw.ClientConfiguration] SetDial(). A simple -// dial function could be as follows: +// If you want to implement a custom timeout or Dial-retry mechanism, provide +// your own Dial function using [raw.ClientConfiguration] SetDial(). A simple +// Dial function could be as follows: // // cfg.SetDial(func(network, addr string) (net.Conn, error) { // return net.DialTimeout(network, addr, time.Second) @@ -779,7 +778,7 @@ func DialConfig(ctx context.Context, config *ClientConfiguration) (Clienter, err var err error var conn net.Conn - dialer := config.dial + dialer := config.Dial if dialer == nil { log.Debug("no dial function provided, using default Dial") dialer = DefaultDial(defaultConnectionTimeout) @@ -792,7 +791,7 @@ func DialConfig(ctx context.Context, config *ClientConfiguration) (Clienter, err } // TODO: TLS if scheme is rabbitmq-stream+tls - addr := net.JoinHostPort(config.rabbitmqBroker.Host, strconv.FormatInt(int64(config.rabbitmqBroker.Port), 10)) + addr := net.JoinHostPort(config.RabbitmqAddr.Host, strconv.FormatInt(int64(config.RabbitmqAddr.Port), 10)) conn, err = dialer("tcp", addr) if err != nil { return nil, fmt.Errorf("failed to dial RabbitMQ '%s': %w", addr, err) @@ -828,7 +827,7 @@ func DefaultDial(connectionTimeout time.Duration) func(network string, addr stri // Connect performs a Stream-protocol handshake to connect to RabbitMQ. On a // successful connect, it returns a nil error and starts listening to incoming -// frames from RabbitMQ. If more than 1 Broker is defined in ClientConfiguration, +// frames from RabbitMQ. If more than 1 RabbitmqAddress is defined in ClientConfiguration, // they will be tried sequentially. // // It is recommended to establish a connection via DialConfig, instead of calling @@ -899,8 +898,8 @@ func (tc *Client) Connect(ctx context.Context) error { panic("could not polymorph SyncCommandRead into TuneRequest") } - desiredFrameSize := math.Min(float64(tuneReq.FrameMaxSize()), float64(tc.configuration.clientMaxFrameSize)) - desiredHeartbeat := math.Min(float64(tuneReq.HeartbeatPeriod()), float64(tc.configuration.clientHeartbeat)) + desiredFrameSize := math.Min(float64(tuneReq.FrameMaxSize()), float64(tc.configuration.ClientMaxFrameSize)) + desiredHeartbeat := math.Min(float64(tuneReq.HeartbeatPeriod()), float64(tc.configuration.ClientHeartbeat)) log.Debug( "desired tune options", @@ -1030,18 +1029,19 @@ func (tc *Client) Send(ctx context.Context, publisherId uint8, publishingMessage return tc.request(ctx, internal.NewPublishRequest(publisherId, uint32(len(publishingMessages)), buff.Bytes())) } -// SendSubEntryBatch aggregates, compress and publishes a batch of messages. -// Similar to Send, but it aggregates the messages for a single publishingId. -// For a single publishingId there could be multiple messages (max ushort). -// The compression is done via common.CompresserCodec interface see common.CompresserCodec for more details. -// The use case for SendSubEntryBatch is to compress a batch of messages for example sending logs. - -// SendSubEntryBatch is fire and forget, like the send function the client will receive the confirmation from the server. -// The server sends only the confirmation for the `publishingId`. +/* +SendSubEntryBatch aggregates, compress and publishes a batch of messages. +Similar to Send, but it aggregates the messages for a single publishingId. +For a single publishingId there could be multiple messages (max ushort). +The compression is done via common.CompresserCodec interface see common.CompresserCodec for more details. +The use case for SendSubEntryBatch is to compress a batch of messages for example sending logs. -// The publishingId should be the last id of []common.PublishingMessager but we leave the api open to allow -// the caller to choose the publishingId. +SendSubEntryBatch is fire and forget, like the send function the client will receive the confirmation from the server. +The server sends only the confirmation for the `publishingId`. +The publishingId should be the last id of []common.PublishingMessager but we leave the api open to allow +the caller to choose the publishingId. +*/ func (tc *Client) SendSubEntryBatch(ctx context.Context, publisherId uint8, publishingId uint64, compress common.CompresserCodec, messages []common.Message) error { if ctx == nil { @@ -1237,7 +1237,7 @@ func (tc *Client) Credit(ctx context.Context, subscriptionID uint8, credits uint // // MetadataResponse{ // correlationID uint32 -// brokers []Broker{ +// brokers []RabbitmqAddress{ // reference uint16 // host string // port uint32 @@ -1469,3 +1469,7 @@ func (tc *Client) ConsumerUpdateResponse(ctx context.Context, correlationId uint func (tc *Client) SendHeartbeat() error { return internal.WriteCommand(internal.NewHeartbeat(), tc.connection.GetWriter()) } + +func (tc *Client) ConnectionProperties() map[string]string { + return tc.connectionProperties +} diff --git a/pkg/raw/client_types.go b/pkg/raw/client_types.go index 2021eeac..2c46b233 100644 --- a/pkg/raw/client_types.go +++ b/pkg/raw/client_types.go @@ -25,76 +25,56 @@ const ( defaultConnectionTimeout = 30 * time.Second ) -type Broker struct { +type RabbitmqAddress struct { Host string Port int Username string Vhost string Password string Scheme string - AdvHost string - AdvPort string ServerProperties map[string]string } -var defaultBroker = Broker{ +var defaultBroker = RabbitmqAddress{ Host: "localhost", Port: 5552, Username: "guest", Vhost: "/", Password: "guest", Scheme: "rabbitmq-stream", - AdvHost: "", - AdvPort: "", } type ClientConfiguration struct { - rabbitmqBroker Broker - clientMaxFrameSize uint32 - clientHeartbeat uint32 - authMechanism []string - tlsConfig *tls.Config - connectionName string - dial func(network, addr string) (net.Conn, error) + RabbitmqAddr RabbitmqAddress + ClientMaxFrameSize uint32 + ClientHeartbeat uint32 + AuthMechanism []string + Tls *tls.Config + ConnectionName string + Dial func(network, addr string) (net.Conn, error) } -func (r *ClientConfiguration) TlsConfig() *tls.Config { - return r.tlsConfig -} - -func (r *ClientConfiguration) SetTlsConfig(tlsConfig *tls.Config) { - r.tlsConfig = tlsConfig -} - -func (r *ClientConfiguration) SetDial(dial func(network, addr string) (net.Conn, error)) { - r.dial = dial -} - -func (r *ClientConfiguration) RabbitmqBroker() Broker { - return r.rabbitmqBroker -} - -func (r *ClientConfiguration) SetClientMaxFrameSize(clientMaxFrameSize uint32) { - r.clientMaxFrameSize = clientMaxFrameSize -} - -func (r *ClientConfiguration) SetClientHeartbeat(clientHeartbeat uint32) { - r.clientHeartbeat = clientHeartbeat -} - -func (r *ClientConfiguration) SetConnectionName(connectionName string) { - r.connectionName = connectionName +func (r *ClientConfiguration) DeepCopy() *ClientConfiguration { + deepCopy := new(ClientConfiguration) + deepCopy.RabbitmqAddr = r.RabbitmqAddr + deepCopy.ClientMaxFrameSize = r.ClientMaxFrameSize + deepCopy.ClientHeartbeat = r.ClientHeartbeat + deepCopy.AuthMechanism = make([]string, len(r.AuthMechanism)) + copy(deepCopy.AuthMechanism, r.AuthMechanism) + deepCopy.Tls = r.Tls.Clone() + deepCopy.Dial = r.Dial + return deepCopy } func NewClientConfiguration(rabbitmqUrl string) (*ClientConfiguration, error) { builder := &ClientConfiguration{ - rabbitmqBroker: Broker{}, - clientHeartbeat: 60, - clientMaxFrameSize: 1_048_576, + RabbitmqAddr: RabbitmqAddress{}, + ClientHeartbeat: 60, + ClientMaxFrameSize: 1_048_576, } if len(rabbitmqUrl) == 0 { - builder.rabbitmqBroker = defaultBroker + builder.RabbitmqAddr = defaultBroker return builder, nil } @@ -103,7 +83,7 @@ func NewClientConfiguration(rabbitmqUrl string) (*ClientConfiguration, error) { return nil, err } - builder.rabbitmqBroker = broker + builder.RabbitmqAddr = broker return builder, nil } diff --git a/pkg/raw/client_types_test.go b/pkg/raw/client_types_test.go index 5190f078..b0483467 100644 --- a/pkg/raw/client_types_test.go +++ b/pkg/raw/client_types_test.go @@ -14,8 +14,8 @@ var _ = Describe("ClientTypes", func() { "rabbitmq-stream://foo:bar@localparty.com:4321/party-vhost") Expect(err).ToNot(HaveOccurred()) - brokers := clientConf.RabbitmqBroker() - Expect(brokers).To(MatchFields(IgnoreExtras, + addr := clientConf.RabbitmqAddr + Expect(addr).To(MatchFields(IgnoreExtras, Fields{ "Host": Equal("localparty.com"), "Port": BeNumerically("==", 4321), @@ -26,13 +26,13 @@ var _ = Describe("ClientTypes", func() { })) }) - It("accepts zero URLs and returns default Broker", func() { + It("accepts zero URLs and returns default RabbitmqAddress", func() { conf, err := raw.NewClientConfiguration("") Expect(err).ToNot(HaveOccurred()) - broker := conf.RabbitmqBroker() - Expect(broker).NotTo(BeNil()) - Expect(broker).To(MatchFields(IgnoreExtras, + addr := conf.RabbitmqAddr + Expect(addr).NotTo(BeNil()) + Expect(addr).To(MatchFields(IgnoreExtras, Fields{ "Host": Equal("localhost"), "Port": BeNumerically("==", 5552), diff --git a/pkg/raw/test_helpers.go b/pkg/raw/test_helpers.go index 6c15fe15..85ac87ba 100644 --- a/pkg/raw/test_helpers.go +++ b/pkg/raw/test_helpers.go @@ -57,19 +57,19 @@ func (tc *Client) ForceCloseConnectionSocket() { // If given only 1 argument, it sets the argument as key with empty string value. // Passing an odd number of arguments panics. func (r *ClientConfiguration) SetServerProperties(keyValues ...string) { - if r.rabbitmqBroker.ServerProperties == nil { - r.rabbitmqBroker.ServerProperties = make(map[string]string) + if r.RabbitmqAddr.ServerProperties == nil { + r.RabbitmqAddr.ServerProperties = make(map[string]string) } if len(keyValues) == 0 { return } if len(keyValues) == 1 { - r.rabbitmqBroker.ServerProperties[keyValues[0]] = "" + r.RabbitmqAddr.ServerProperties[keyValues[0]] = "" return } for i := 0; i < len(keyValues); i += 2 { - r.rabbitmqBroker.ServerProperties[keyValues[i]] = keyValues[i+1] + r.RabbitmqAddr.ServerProperties[keyValues[i]] = keyValues[i+1] } } diff --git a/pkg/raw/util.go b/pkg/raw/util.go index 11905884..a1971715 100644 --- a/pkg/raw/util.go +++ b/pkg/raw/util.go @@ -7,7 +7,7 @@ import ( "strings" ) -func parseURI(uri string) (Broker, error) { +func parseURI(uri string) (RabbitmqAddress, error) { builder := defaultBroker if strings.Contains(uri, " ") { @@ -68,11 +68,6 @@ func parseURI(uri string) (Broker, error) { } } - // see https://www.rabbitmq.com/uri-query-parameters.html - params := u.Query() - builder.AdvHost = params.Get("advHost") - builder.AdvPort = params.Get("advPort") - return builder, nil } diff --git a/pkg/stream/environment.go b/pkg/stream/environment.go index a482856b..7020446f 100644 --- a/pkg/stream/environment.go +++ b/pkg/stream/environment.go @@ -7,7 +7,7 @@ import ( "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "log/slog" "math/rand" - "strings" + "strconv" "time" ) @@ -22,6 +22,7 @@ type Environment struct { retryPolicy backoffDurationFunc locatorSelectSequential bool producerManagers []*producerManager + addressResolver AddressResolver } func NewEnvironment(ctx context.Context, configuration EnvironmentConfiguration) (*Environment, error) { @@ -48,7 +49,7 @@ func (e *Environment) start(ctx context.Context) error { return err } - c.SetConnectionName(fmt.Sprintf("%s-locator-%d", e.configuration.Id, i)) + c.ConnectionName = fmt.Sprintf("%s-locator-%d", e.configuration.Id, i) l := newLocator(*c, logger) // if ctx has a lower timeout, it will be used instead of DefaultTimeout @@ -168,15 +169,10 @@ func (e *Environment) DeleteStream(ctx context.Context, name string) error { // and awaits a confirmation response. If there's any error closing a connection, // the error is logged to a logger extracted from the context. func (e *Environment) Close(ctx context.Context) { - logger := raw.LoggerFromCtxOrDiscard(ctx).WithGroup("close") + //logger := raw.LoggerFromCtxOrDiscard(ctx).WithGroup("close") // TODO: shutdown producers/consumers for _, l := range e.locators { - if l.isSet { - err := l.client.Close(ctx) - if err != nil { - logger.Warn("error closing locator client", slog.Any("error", err)) - } - } + l.close() } } @@ -377,7 +373,7 @@ func (e *Environment) CreateProducer(ctx context.Context, stream string, opts *P // pick at random l = e.pickLocator((i + rn) % n) } - result := l.operationQueryStreamMetadata(ctx, []string{stream}) + result := l.locatorOperation((*locator).operationQueryStreamMetadata, ctx, []string{stream}) if result[1] != nil { lastError = result[1].(error) if isNonRetryableError(lastError) { @@ -395,63 +391,65 @@ func (e *Environment) CreateProducer(ctx context.Context, stream string, opts *P return nil, fmt.Errorf("locator operation failed: %w", lastError) } - //var found = false - //var pm *producerManager - //for _, streamMetadata := range metadata.StreamsMetadata() { - // if streamMetadata.StreamName() == stream { - // found = true - // leaderRef := streamMetadata.LeaderReference() - // brokerRef := metadata.Brokers()[leaderRef] - // - // for i := 0; i < len(e.producerManagers); i++ { - // if e.producerManagers[i].connectionEndpoint.Host == brokerRef.Host() && - // e.producerManagers[i].connectionEndpoint.Port == int(brokerRef.Port()) { - // pm = &e.producerManagers[i] - // break - // } - // } - // if pm != nil { - // - // } - // - // break - // } - //} - broker, err := findLeader(stream, metadata) + brokerLeader, err := findLeader(stream, metadata) if err != nil { return nil, err } - pm, err := e.createProducerManager(ctx, broker) - if err != nil { - return nil, err + // if there's an address resolver, always use the address resolver + var pmClient raw.Clienter + var rc *raw.ClientConfiguration + if e.addressResolver != nil { + rc = e.locators[0].rawClientConf.DeepCopy() + rc.RabbitmqAddr.Host, rc.RabbitmqAddr.Port = e.addressResolver(rc.RabbitmqAddr.Host, rc.RabbitmqAddr.Port) + // TODO max attempts variable, depending on number of stream replicas + for i := 0; i < 10; i++ { + client, err := raw.DialConfig(ctx, rc) + if err != nil { + logger.Warn("failed to dial", + slog.String("host", rc.RabbitmqAddr.Host), + slog.Int("port", rc.RabbitmqAddr.Port), + slog.Any("error", err), + ) + } + connProps := client.(*raw.Client).ConnectionProperties() + if connProps["advertised_host"] == brokerLeader.host && connProps["advertised_port"] == brokerLeader.port { + pmClient = client + break + } + + logger.Debug("connected to rabbit, but not to desired node", + slog.String("host", connProps["advertised_host"]), + slog.String("port", connProps["advertised_port"]), + ) + ctx2, cancel := maybeApplyDefaultTimeout(ctx) + _ = client.Close(ctx2) + cancel() + time.Sleep(time.Second) + } + if pmClient == nil { + panic("could not create a client for a new producer manager") + } + } else { + rc = e.locators[0].rawClientConf.DeepCopy() + // TODO: set connection name + c, err := raw.DialConfig(ctx, rc) + if err != nil { + panic(err) + } + pmClient = c } + + pm := newProducerManagerWithClient(0, e.configuration, pmClient, rc) + e.producerManagers = append(e.producerManagers, pm) o := opts.DeepCopy() o.stream = stream - // FIXME: check producer manager capacity before selecting a + // FIXME: check producer manager capacity before selecting a PM return pm.createProducer(ctx, o) } -// initialises and connects a new producer manager -func (e *Environment) createProducerManager(ctx context.Context, broker raw.Broker) (*producerManager, error) { - logger := raw.LoggerFromCtxOrDiscard(ctx).WithGroup("createProducerManager") - // create new producer manager - pm := newProducerManager(len(e.producerManagers), e.configuration) - e.producerManagers = append(e.producerManagers, pm) - uri := uriFromConfigurationThatContains( - fmt.Sprintf("%s:%d", broker.Host, broker.Port), - &e.configuration, - ) - if len(uri) == 0 { - logger.Debug("did not find an URI for leader in configuration", slog.String("leader-host", broker.Host), slog.Int("leader-port", broker.Port)) - return nil, errors.New("empty uri in create producer manager") - } - - return pm, pm.connect(ctx, uri) -} - -func findLeader(stream string, metadata *raw.MetadataResponse) (raw.Broker, error) { +func findLeader(stream string, metadata *raw.MetadataResponse) (hostPort, error) { var leaderRef uint16 var found = false for _, streamMetadata := range metadata.StreamsMetadata() { @@ -462,25 +460,13 @@ func findLeader(stream string, metadata *raw.MetadataResponse) (raw.Broker, erro } } if !found { - return raw.Broker{}, errors.New("stream leader not found") + return hostPort{}, errors.New("stream leader not found") } - bb := raw.Broker{} for _, broker := range metadata.Brokers() { if broker.Reference() == leaderRef { - bb.Host = broker.Host() - bb.Port = int(broker.Port()) - return bb, nil - } - } - return raw.Broker{}, errors.New("broker hosting leader not found") -} - -func uriFromConfigurationThatContains(s string, c *EnvironmentConfiguration) string { - for _, uri := range c.Uris { - if strings.Contains(uri, s) { - return uri + return hostPort{host: broker.Host(), port: strconv.Itoa(int(broker.Port()))}, nil } } - return "" + return hostPort{}, errors.New("broker hosting leader not found") } diff --git a/pkg/stream/environment_configuration.go b/pkg/stream/environment_configuration.go index fed93268..c664e104 100644 --- a/pkg/stream/environment_configuration.go +++ b/pkg/stream/environment_configuration.go @@ -20,6 +20,8 @@ const ( DefaultId = "rabbitmq-stream" ) +type AddressResolver func(host string, port int) (resolvedHost string, resolvedPort int) + type EnvironmentConfiguration struct { // The URI of the nodes to try to connect to (cluster). This takes precedence // over URI and Host + Port. @@ -56,6 +58,10 @@ type EnvironmentConfiguration struct { // // Default: "/" VirtualHost string + // TODO Docs + // + // Default: no-op + AddrResolver AddressResolver // The maximum number of `Producer` instances a single connection can maintain // before a new connection is open. The value must be between 1 and 255 // @@ -235,6 +241,17 @@ func WithId(id string) EnvironmentConfigurationOption { } } +// WithAddressResolver configures the environment to use resolver as +// AddressResolver. This is required when RabbitMQ is behind a load balancer, +// because the client will try to connect to a leader for publishing, and it +// needs a 'hint' to translate the broker host:port to the load balancer +// host:port. TODO: write an example +func WithAddressResolver(resolver AddressResolver) EnvironmentConfigurationOption { + return func(c *EnvironmentConfiguration) { + c.AddrResolver = resolver + } +} + func NewEnvironmentConfiguration(options ...EnvironmentConfigurationOption) EnvironmentConfiguration { e := &EnvironmentConfiguration{ Uris: []string{DefaultUri}, diff --git a/pkg/stream/environment_test.go b/pkg/stream/environment_test.go index c93c50dc..62b2abf2 100644 --- a/pkg/stream/environment_test.go +++ b/pkg/stream/environment_test.go @@ -1,3 +1,5 @@ +//go:build rabbitmq.stream.test + package stream_test import ( diff --git a/pkg/stream/locator.go b/pkg/stream/locator.go index c3996f4a..a20f728d 100644 --- a/pkg/stream/locator.go +++ b/pkg/stream/locator.go @@ -5,7 +5,6 @@ import ( "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "golang.org/x/mod/semver" "log/slog" - "net" "sync" "time" ) @@ -16,15 +15,14 @@ const ( type locator struct { sync.Mutex - log *slog.Logger - shutdownNotification chan struct{} - rawClientConf raw.ClientConfiguration - client raw.Clienter - isSet bool - clientClose <-chan error - retryPolicy backoffDurationFunc - addressResolver net.Addr // TODO: placeholder for address resolver - + log *slog.Logger + destructor *sync.Once + done chan struct{} + rawClientConf raw.ClientConfiguration + client raw.Clienter + isSet bool + clientClose <-chan error + retryPolicy backoffDurationFunc } func newLocator(c raw.ClientConfiguration, logger *slog.Logger) *locator { @@ -32,15 +30,30 @@ func newLocator(c raw.ClientConfiguration, logger *slog.Logger) *locator { log: logger. WithGroup("locator"). With( - slog.String("host", c.RabbitmqBroker().Host), - slog.Int("port", c.RabbitmqBroker().Port), + slog.String("host", c.RabbitmqAddr.Host), + slog.Int("port", c.RabbitmqAddr.Port), ), - rawClientConf: c, - retryPolicy: defaultBackOffPolicy, - client: nil, - isSet: false, - addressResolver: nil, - shutdownNotification: make(chan struct{}), + destructor: &sync.Once{}, + rawClientConf: c, + retryPolicy: defaultBackOffPolicy, + client: nil, + isSet: false, + done: make(chan struct{}), + } +} + +func (l *locator) close() { + logger := l.log.WithGroup("close") + l.destructor.Do(func() { + close(l.done) + }) + if l.isSet { + ctx, cancel := maybeApplyDefaultTimeout(context.Background()) + defer cancel() + if err := l.client.Close(ctx); err != nil { + logger.Warn("error closing locator client", slog.Any("error", err)) + } + l.isSet = false } } @@ -86,7 +99,7 @@ func (l *locator) shutdownHandler() { // TODO: we must close the shutdown notification channel before closing the client. // Or otherwise the clientClose case will proceed (incorrectly). select { - case <-l.shutdownNotification: + case <-l.done: log.Debug("locator shutdown") return case err := <-l.clientClose: @@ -128,7 +141,7 @@ func (l *locator) shutdownHandler() { const rabbitmqVersion311 = "v3.11" func (l *locator) isServer311orMore() bool { - v, ok := l.rawClientConf.RabbitmqBroker().ServerProperties["version"] + v, ok := l.rawClientConf.RabbitmqAddr.ServerProperties["version"] if !ok { // version not found in server properties // returning false as we can't determine diff --git a/pkg/stream/locator_test.go b/pkg/stream/locator_test.go index 9d0e0343..610828eb 100644 --- a/pkg/stream/locator_test.go +++ b/pkg/stream/locator_test.go @@ -25,13 +25,13 @@ var _ = Describe("Locator", func() { logger = slog.New(slog.NewTextHandler(GinkgoWriter, &slog.HandlerOptions{Level: slog.LevelDebug})) loc = &locator{ - log: logger, - shutdownNotification: make(chan struct{}), - rawClientConf: raw.ClientConfiguration{}, - client: nil, - isSet: true, - clientClose: nil, - retryPolicy: backOffPolicy, + log: logger, + done: make(chan struct{}), + rawClientConf: raw.ClientConfiguration{}, + client: nil, + isSet: true, + clientClose: nil, + retryPolicy: backOffPolicy, } }) diff --git a/pkg/stream/producer_manager.go b/pkg/stream/producer_manager.go index a29c06b5..cde5ffd8 100644 --- a/pkg/stream/producer_manager.go +++ b/pkg/stream/producer_manager.go @@ -21,7 +21,7 @@ type producerManager struct { m sync.Mutex id int config EnvironmentConfiguration - connectionEndpoint raw.Broker + connectionEndpoint raw.RabbitmqAddress producers []internalProducer client raw.Clienter clientM *sync.Mutex @@ -40,12 +40,20 @@ func newProducerManager(id int, config EnvironmentConfiguration) *producerManage } } +func newProducerManagerWithClient(id int, config EnvironmentConfiguration, client raw.Clienter, rc *raw.ClientConfiguration) (manager *producerManager) { + manager = newProducerManager(id, config) + manager.client = client + manager.open = true + manager.connectionEndpoint = rc.RabbitmqAddr + return manager +} + func (p *producerManager) connect(ctx context.Context, uri string) error { conf, err := raw.NewClientConfiguration(uri) if err != nil { return err } - conf.SetConnectionName(fmt.Sprintf("%s-%d", "rabbitmq-stream-producer", p.id)) + conf.ConnectionName = fmt.Sprintf("%s-%d", "rabbitmq-stream-producer", p.id) ctx2, cancel := maybeApplyDefaultTimeout(ctx) defer cancel() @@ -54,7 +62,7 @@ func (p *producerManager) connect(ctx context.Context, uri string) error { return err } - p.connectionEndpoint = conf.RabbitmqBroker() + p.connectionEndpoint = conf.RabbitmqAddr p.client = c p.open = true return nil diff --git a/pkg/stream/system_test.go b/pkg/stream/system_test.go index 8189ee5c..4eb12146 100644 --- a/pkg/stream/system_test.go +++ b/pkg/stream/system_test.go @@ -122,12 +122,18 @@ var _ = Describe("stream package", func() { It("connects to RabbitMQ", Focus, func() { uri := "rabbitmq-stream://localhost:5552" - conf := stream.NewEnvironmentConfiguration(stream.WithUri(uri), stream.WithLazyInitialization(false)) + conf := stream.NewEnvironmentConfiguration( + stream.WithUri(uri), + stream.WithLazyInitialization(false), + stream.WithAddressResolver(func(_ string, _ int) (_ string, _ int) { + return "localhost", 5552 + }), + stream.WithId("system-test"), + ) env, err := stream.NewEnvironment(context.Background(), conf) Expect(err).ToNot(HaveOccurred()) streamName := "test-stream" Expect(env.CreateStream(context.Background(), streamName, stream.CreateStreamOptions{})).To(Succeed()) - // TODO producer, err := env.CreateProducer(context.Background(), streamName, &stream.ProducerOptions{}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/stream/types.go b/pkg/stream/types.go index 08758d4d..77102856 100644 --- a/pkg/stream/types.go +++ b/pkg/stream/types.go @@ -66,3 +66,8 @@ const ( closing open ) + +type hostPort struct { + host string + port string +} From c2c019b147d713975d791ef85bf810d03c2e2269 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Fri, 17 Nov 2023 11:54:36 +0000 Subject: [PATCH 20/24] Fix tests after rebasing v2 Signed-off-by: Aitor Perez Cedres --- main.go | 65 +++++++--- pkg/raw/client_dial_test.go | 4 +- pkg/raw/client_test.go | 2 +- pkg/stream/environment.go | 3 + pkg/stream/environment_test.go | 7 +- pkg/stream/mock_producer_test.go | 179 ++++++++++++++++++++++++++++ pkg/stream/producer_manager.go | 97 +++++++++------ pkg/stream/producer_manager_test.go | 31 ++++- pkg/stream/stream_suite_test.go | 4 + pkg/stream/test_helpers.go | 3 + pkg/stream/types.go | 1 + 11 files changed, 334 insertions(+), 62 deletions(-) create mode 100644 pkg/stream/mock_producer_test.go diff --git a/main.go b/main.go index a9b42502..d3e87f1c 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "context" + "flag" "fmt" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/common" @@ -15,36 +16,72 @@ import ( ) func main() { - runSmartClient() - runRawClient() + runRawClientFlag := flag.Bool("run-raw-client", false, "set it to run raw client") + runSmartClientFlag := flag.Bool("run-smart-client", false, "set it to run raw client") + flag.Parse() + if *runRawClientFlag { + runRawClient() + } + if *runSmartClientFlag { + runSmartClient() + } } func runSmartClient() { slogOpts := &slog.HandlerOptions{ - Level: slog.LevelDebug, + Level: slog.LevelInfo, } log := slog.New(slog.NewTextHandler(os.Stdout, slogOpts)) + // FIXME: producer manager does not register notify publish ctx := raw.NewContextWithLogger(context.Background(), *log) c := stream.NewEnvironmentConfiguration( stream.WithLazyInitialization(false), stream.WithUri("rabbitmq-stream://localhost:5552"), + stream.WithAddressResolver(func(_ string, _ int) (_ string, _ int) { + return "localhost", 5552 + }), ) env, err := stream.NewEnvironment(ctx, c) if err != nil { panic(err) } - //defer env.Close() err = env.CreateStream(ctx, "my-stream", stream.CreateStreamOptions{}) if err != nil { panic(err) } + var nConfirm int + producer, err := env.CreateProducer(ctx, "my-stream", &stream.ProducerOptions{ + MaxInFlight: 1_000, + MaxBufferedMessages: 100, + ConfirmationHandler: func(c *stream.MessageConfirmation) { + if c.Status() == stream.Confirmed { + nConfirm += 1 + if nConfirm%1_000 == 0 { + log.Info("received confirmations", slog.Int("confirm-count", nConfirm)) + } + } else { + log.Warn("message not confirmed", slog.Int("confirm-status", int(c.Status()))) + } + }, + }) + + for i := 0; i < 1_000_000; i++ { + err = producer.Send(context.Background(), amqp.Message{Data: []byte(fmt.Sprintf("Message #%d", i))}) + if err != nil { + log.Warn("failed to send a message", slog.Int("message-n", i)) + } + if i%1_000 == 0 { + log.Info("sent messages", slog.Int("send-count", i)) + } + } + sc := bufio.NewScanner(os.Stdin) - fmt.Print("Close the connection and press enter") + fmt.Print("Press enter to continue and exit") sc.Scan() err = env.DeleteStream(ctx, "my-stream") @@ -52,17 +89,7 @@ func runSmartClient() { panic(err) } - err = env.CreateStream(ctx, "other-stream", stream.CreateStreamOptions{}) - if err != nil { - panic(err) - } - - fmt.Print("Life good! Press enter to exit") - sc.Scan() - err = env.DeleteStream(ctx, "other-stream") - if err != nil { - panic(err) - } + env.Close(ctx) } func runRawClient() { @@ -104,6 +131,12 @@ func runRawClient() { panic(err) } + metadata, err := streamClient.MetadataQuery(ctx, []string{streamName}) + if err != nil { + panic(err) + } + log.Info("metadata query success", slog.Any("metadata", *metadata)) + const batchSize = 100 const iterations = 1000 const totalMessages = iterations * batchSize diff --git a/pkg/raw/client_dial_test.go b/pkg/raw/client_dial_test.go index c7d370e9..dd17f90a 100644 --- a/pkg/raw/client_dial_test.go +++ b/pkg/raw/client_dial_test.go @@ -51,7 +51,7 @@ var _ = Describe("ClientDial", func() { <-time.After(time.Millisecond * 100) // have to introduce artificial delay for the background frame handler conf, _ := raw.NewClientConfiguration("") - conf.SetDial(func(_, _ string) (net.Conn, error) { + conf.Dial = func(_, _ string) (net.Conn, error) { c, err := net.DialTimeout("unix", serverSocketPath, time.Second) if err != nil { panic(err) @@ -59,7 +59,7 @@ var _ = Describe("ClientDial", func() { <-waiter go fakeServer.fakeRabbitMQConnectionOpen(ctx) return c, err - }) + } c, err := raw.DialConfig(ctx, conf) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/raw/client_test.go b/pkg/raw/client_test.go index a2d0d8df..5fcb9b4d 100644 --- a/pkg/raw/client_test.go +++ b/pkg/raw/client_test.go @@ -128,7 +128,7 @@ var _ = Describe("Client", func() { Should(BeTrue(), "expected connection to be open") By("recording the server properties") - Expect(conf.RabbitmqBroker().ServerProperties).To(HaveKeyWithValue("product", "mock-rabbitmq")) + Expect(conf.RabbitmqAddr.ServerProperties).To(HaveKeyWithValue("product", "mock-rabbitmq")) By("closing the client connection gracefully") go fakeRabbitMQ.fakeRabbitMQConnectionClose(itCtx) diff --git a/pkg/stream/environment.go b/pkg/stream/environment.go index 7020446f..2efcc6ed 100644 --- a/pkg/stream/environment.go +++ b/pkg/stream/environment.go @@ -413,6 +413,7 @@ func (e *Environment) CreateProducer(ctx context.Context, stream string, opts *P ) } connProps := client.(*raw.Client).ConnectionProperties() + // did I connect to the leader behind the load balancer? if connProps["advertised_host"] == brokerLeader.host && connProps["advertised_port"] == brokerLeader.port { pmClient = client break @@ -432,6 +433,8 @@ func (e *Environment) CreateProducer(ctx context.Context, stream string, opts *P } } else { rc = e.locators[0].rawClientConf.DeepCopy() + rc.RabbitmqAddr.Host = brokerLeader.host + rc.RabbitmqAddr.Port, _ = strconv.Atoi(brokerLeader.port) // TODO: set connection name c, err := raw.DialConfig(ctx, rc) if err != nil { diff --git a/pkg/stream/environment_test.go b/pkg/stream/environment_test.go index 62b2abf2..0a2dd729 100644 --- a/pkg/stream/environment_test.go +++ b/pkg/stream/environment_test.go @@ -281,7 +281,12 @@ var _ = Describe("Environment", func() { Return(errors.New("something went wrong")) }) - It("logs the error and moves on", func() { + It("logs the error and moves on", Pending, func() { + // TODO: the locator has its own logger, it does not use the logger + // from the context. This tests seems to make no sense, as the + // locator is appended by an utility function for tests, and + // this tests assumes that the locator uses the logger from context, + // which is not true. Revisit this test and its assumptions. logBuffer := gbytes.NewBuffer() logger := slog.New(slog.NewTextHandler(logBuffer, nil)) ctx := raw.NewContextWithLogger(context.Background(), *logger) diff --git a/pkg/stream/mock_producer_test.go b/pkg/stream/mock_producer_test.go new file mode 100644 index 00000000..fc7d92cb --- /dev/null +++ b/pkg/stream/mock_producer_test.go @@ -0,0 +1,179 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: types.go + +// Package stream is a generated GoMock package. +package stream + +import ( + context "context" + reflect "reflect" + + amqp "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/codecs/amqp" + gomock "go.uber.org/mock/gomock" +) + +// MockProducer is a mock of Producer interface. +type MockProducer struct { + ctrl *gomock.Controller + recorder *MockProducerMockRecorder +} + +// MockProducerMockRecorder is the mock recorder for MockProducer. +type MockProducerMockRecorder struct { + mock *MockProducer +} + +// NewMockProducer creates a new mock instance. +func NewMockProducer(ctrl *gomock.Controller) *MockProducer { + mock := &MockProducer{ctrl: ctrl} + mock.recorder = &MockProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProducer) EXPECT() *MockProducerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockProducer) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockProducerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockProducer)(nil).Close)) +} + +// Send mocks base method. +func (m *MockProducer) Send(ctx context.Context, msg amqp.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", ctx, msg) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockProducerMockRecorder) Send(ctx, msg interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockProducer)(nil).Send), ctx, msg) +} + +// SendBatch mocks base method. +func (m *MockProducer) SendBatch(ctx context.Context, messages []amqp.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendBatch", ctx, messages) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendBatch indicates an expected call of SendBatch. +func (mr *MockProducerMockRecorder) SendBatch(ctx, messages interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendBatch", reflect.TypeOf((*MockProducer)(nil).SendBatch), ctx, messages) +} + +// SendWithId mocks base method. +func (m *MockProducer) SendWithId(ctx context.Context, publishingId uint64, msg amqp.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendWithId", ctx, publishingId, msg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendWithId indicates an expected call of SendWithId. +func (mr *MockProducerMockRecorder) SendWithId(ctx, publishingId, msg interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithId", reflect.TypeOf((*MockProducer)(nil).SendWithId), ctx, publishingId, msg) +} + +// MockinternalProducer is a mock of internalProducer interface. +type MockinternalProducer struct { + ctrl *gomock.Controller + recorder *MockinternalProducerMockRecorder +} + +// MockinternalProducerMockRecorder is the mock recorder for MockinternalProducer. +type MockinternalProducerMockRecorder struct { + mock *MockinternalProducer +} + +// NewMockinternalProducer creates a new mock instance. +func NewMockinternalProducer(ctrl *gomock.Controller) *MockinternalProducer { + mock := &MockinternalProducer{ctrl: ctrl} + mock.recorder = &MockinternalProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockinternalProducer) EXPECT() *MockinternalProducerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockinternalProducer) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockinternalProducerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockinternalProducer)(nil).Close)) +} + +// Send mocks base method. +func (m *MockinternalProducer) Send(ctx context.Context, msg amqp.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", ctx, msg) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockinternalProducerMockRecorder) Send(ctx, msg interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockinternalProducer)(nil).Send), ctx, msg) +} + +// SendBatch mocks base method. +func (m *MockinternalProducer) SendBatch(ctx context.Context, messages []amqp.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendBatch", ctx, messages) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendBatch indicates an expected call of SendBatch. +func (mr *MockinternalProducerMockRecorder) SendBatch(ctx, messages interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendBatch", reflect.TypeOf((*MockinternalProducer)(nil).SendBatch), ctx, messages) +} + +// SendWithId mocks base method. +func (m *MockinternalProducer) SendWithId(ctx context.Context, publishingId uint64, msg amqp.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendWithId", ctx, publishingId, msg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendWithId indicates an expected call of SendWithId. +func (mr *MockinternalProducerMockRecorder) SendWithId(ctx, publishingId, msg interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithId", reflect.TypeOf((*MockinternalProducer)(nil).SendWithId), ctx, publishingId, msg) +} + +// shutdown mocks base method. +func (m *MockinternalProducer) shutdown() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "shutdown") +} + +// shutdown indicates an expected call of shutdown. +func (mr *MockinternalProducerMockRecorder) shutdown() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "shutdown", reflect.TypeOf((*MockinternalProducer)(nil).shutdown)) +} diff --git a/pkg/stream/producer_manager.go b/pkg/stream/producer_manager.go index cde5ffd8..0cd77b78 100644 --- a/pkg/stream/producer_manager.go +++ b/pkg/stream/producer_manager.go @@ -18,25 +18,30 @@ var ( const noReference = "" type producerManager struct { - m sync.Mutex - id int - config EnvironmentConfiguration - connectionEndpoint raw.RabbitmqAddress - producers []internalProducer - client raw.Clienter - clientM *sync.Mutex - open bool + m sync.Mutex + id int + config EnvironmentConfiguration + // records the endpoint that this manager is connected to + connectionEndpoint raw.RabbitmqAddress + producers []internalProducer + confirmationChannels []chan *publishConfirmOrError + // keeps track of how many producers are assigned to this manager + producerCount int + client raw.Clienter + clientM *sync.Mutex + open bool } func newProducerManager(id int, config EnvironmentConfiguration) *producerManager { return &producerManager{ - m: sync.Mutex{}, - id: id, - config: config, - producers: make([]internalProducer, 0, config.MaxProducersByConnection), - client: nil, - clientM: &sync.Mutex{}, - open: false, + m: sync.Mutex{}, + id: id, + config: config, + producers: make([]internalProducer, config.MaxProducersByConnection), + producerCount: 0, + client: nil, + clientM: &sync.Mutex{}, + open: false, } } @@ -65,9 +70,16 @@ func (p *producerManager) connect(ctx context.Context, uri string) error { p.connectionEndpoint = conf.RabbitmqAddr p.client = c p.open = true + + //confirmCh := c.NotifyPublish(make(chan *raw.PublishConfirm, DefaultMaxInFlight)) + return nil } +func (p *producerManager) publishConfirmationListener(ctx context.Context, c chan *raw.PublishConfirm) { + panic("implement me!") +} + // initialises a producer and sends a declare publisher frame. It returns // an error if the manager does not have capacity, or if declare publisher // fails @@ -76,35 +88,27 @@ func (p *producerManager) createProducer(ctx context.Context, producerOpts *Prod return nil, errManagerClosed } + // TODO: optimisation - check if producerCount == len(producers) -> return errManagerFull + var ( publisherId uint8 producer *standardProducer ) - if len(p.producers) == 0 { - publisherId = 0 - producer = newStandardProducer(publisherId, p.client, p.clientM, producerOpts) - producer.setCloseCallback(p.removeProducer) - p.producers = append(p.producers, producer) - } else { - for i := 0; i < len(p.producers); i++ { - if p.producers[i] == nil { - publisherId = uint8(i) - producer = newStandardProducer(publisherId, p.client, p.clientM, producerOpts) - p.producers[i] = producer - break - } - } - // no empty slots, manager at max capacity - if producer == nil && len(p.producers) == p.config.MaxProducersByConnection { - return nil, errManagerFull - } - // no empty slots, manager has capacity - if producer == nil { - publisherId = uint8(len(p.producers)) + for i := 0; i < len(p.producers); i++ { + if p.producers[i] == nil { + publisherId = uint8(i) producer = newStandardProducer(publisherId, p.client, p.clientM, producerOpts) - p.producers = append(p.producers, producer) + p.producers[i] = producer + p.producerCount += 1 + break } } + // no empty slots, manager at max capacity + if producer == nil { + return nil, errManagerFull + } + + producer.setCloseCallback(p.removeProducer) ctx2, cancel := maybeApplyDefaultTimeout(ctx) defer cancel() @@ -118,6 +122,12 @@ func (p *producerManager) createProducer(ctx context.Context, producerOpts *Prod return producer, nil } +/* +Removes a producer with index == id + +It closes the connection and sets the manager state to closed if the last +producer is removed. +*/ func (p *producerManager) removeProducer(id int) error { if id >= len(p.producers) { return errIdOutOfBounds @@ -126,5 +136,18 @@ func (p *producerManager) removeProducer(id int) error { return errNegativeId } p.producers[id] = nil + p.producerCount -= 1 + + if p.producerCount == 0 { + // last producer in this manager, closing connection + p.open = false + p.clientM.Lock() + defer p.clientM.Unlock() + ctx, cancel := maybeApplyDefaultTimeout(context.Background()) + defer cancel() + _ = p.client.Close(ctx) + // FIXME: have a logger and log error + } + return nil } diff --git a/pkg/stream/producer_manager_test.go b/pkg/stream/producer_manager_test.go index ff2eb921..7fa8bdd1 100644 --- a/pkg/stream/producer_manager_test.go +++ b/pkg/stream/producer_manager_test.go @@ -39,20 +39,21 @@ var _ = Describe("ProducerManager", func() { Expect(p.(*standardProducer).rawClient).To(Equal(pm.client)) Expect(p.(*standardProducer).rawClientMu).To(Equal(pm.clientM)) Expect(p.(*standardProducer).publisherId).To(BeEquivalentTo(0)) - Expect(pm.producers).To(HaveLen(1)) + Expect(pm.producers).To(ContainElement(p)) p2, err := pm.createProducer(context.Background(), &ProducerOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(p2.(*standardProducer).rawClient).To(Equal(pm.client)) Expect(p2.(*standardProducer).rawClientMu).To(Equal(pm.clientM)) Expect(p2.(*standardProducer).publisherId).To(BeEquivalentTo(1)) + Expect(pm.producers).To(ContainElement(p2)) p3, err := pm.createProducer(context.Background(), &ProducerOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(p3.(*standardProducer).rawClient).To(Equal(pm.client)) Expect(p3.(*standardProducer).rawClientMu).To(Equal(pm.clientM)) Expect(p3.(*standardProducer).publisherId).To(BeEquivalentTo(2)) - Expect(pm.producers).To(HaveLen(3)) + Expect(pm.producers).To(ContainElement(p3)) }) When("manager is full", func() { @@ -119,7 +120,7 @@ var _ = Describe("ProducerManager", func() { Expect(p3.(*standardProducer).publisherId).To(BeEquivalentTo(1)) By("reusing the producer slot #1") - Expect(pm.producers).To(HaveLen(2)) // this ensures we have not appended to the slice + Expect(pm.producers[1]).ToNot(BeNil()) By("reusing publisher ID 2", func() { pm.producers = append( @@ -134,8 +135,28 @@ var _ = Describe("ProducerManager", func() { Expect(err).ToNot(HaveOccurred()) Expect(p4.(*standardProducer).publisherId).To(BeEquivalentTo(2)) - By("reusing the producer slot #1") - Expect(pm.producers).To(HaveLen(5)) + By("reusing the producer slot #2") + Expect(pm.producers[2]).ToNot(BeNil()) + }) + }) + + When("last producer is closed", func() { + It("closes the connection", func() { + fakeRawClient.EXPECT().Close(gomock.Any()) + prepareMockDeclarePublisher(fakeRawClient, 0) + pm := newProducerManager(0, EnvironmentConfiguration{MaxProducersByConnection: 5}) + pm.client = fakeRawClient + pm.open = true + + producer, err := pm.createProducer(context.Background(), &ProducerOptions{}) + Expect(err).ToNot(HaveOccurred()) + producer.Close() + + Eventually(func() Producer { + pm.m.Lock() + defer pm.m.Unlock() + return pm.producers[0] + }, "1s", "200ms").Should(BeNil()) }) }) }) diff --git a/pkg/stream/stream_suite_test.go b/pkg/stream/stream_suite_test.go index 4de14863..53bfb4df 100644 --- a/pkg/stream/stream_suite_test.go +++ b/pkg/stream/stream_suite_test.go @@ -64,6 +64,10 @@ func (p *plainTextMessage) Body() string { var _ = SynchronizedBeforeSuite(func() { // Just once logger := log.New(GinkgoWriter, "[SBS] ", log.Ldate|log.Lmsgprefix) + if _, isSet := os.LookupEnv(SystemTestEnvVarName); !isSet { + logger.Println("System test variable to run system test not set. Skipping system tests...") + } + if _, isSet := os.LookupEnv(SystemTestSkipRabbitStart); isSet { logger.Println("System test variable to skip RabbitMQ start is set. Skipping...") return diff --git a/pkg/stream/test_helpers.go b/pkg/stream/test_helpers.go index 2059485e..02c7be58 100644 --- a/pkg/stream/test_helpers.go +++ b/pkg/stream/test_helpers.go @@ -6,6 +6,7 @@ import ( "context" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "log/slog" + "sync" "time" ) @@ -36,6 +37,8 @@ func (e *Environment) AppendLocatorRawClient(c raw.Clienter) { retryPolicy: func(int) time.Duration { return time.Millisecond * 10 }, + destructor: &sync.Once{}, + done: make(chan struct{}), }) } diff --git a/pkg/stream/types.go b/pkg/stream/types.go index 77102856..a4c7a5aa 100644 --- a/pkg/stream/types.go +++ b/pkg/stream/types.go @@ -51,6 +51,7 @@ type Producer interface { Close() } +//go:generate mockgen -source=types.go -destination=mock_producer_test.go -package stream type internalProducer interface { Producer shutdown() From 6ef17b7fdb2e5e4e1f55f3807a523326f762ac72 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Fri, 17 Nov 2023 13:30:06 +0000 Subject: [PATCH 21/24] Use stable version of Go in CI We started developing in Go 1.21 because slog package became GA in Go 1.21. The toolchain directive in go.mod is new and only available in Go 1.21. We have to temporarily bump the Go version in CI to 'stable'. We will provide compatibility to oldstable at a later date. Signed-off-by: Aitor Perez Cedres --- .github/workflows/build_test_linux.yml | 6 +- pkg/e2e/end_to_end_test.go | 2 +- pkg/stream/environment_internal_test.go | 83 ------------------------- pkg/stream/environment_test.go | 15 +++-- 4 files changed, 14 insertions(+), 92 deletions(-) delete mode 100644 pkg/stream/environment_internal_test.go diff --git a/.github/workflows/build_test_linux.yml b/.github/workflows/build_test_linux.yml index 9a62d379..6130c35b 100644 --- a/.github/workflows/build_test_linux.yml +++ b/.github/workflows/build_test_linux.yml @@ -17,7 +17,8 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: oldstable + go-version: stable +# go-version: oldstable - name: Build run: make build @@ -50,7 +51,8 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: oldstable + go-version: stable +# go-version: oldstable - name: E2E Tests env: diff --git a/pkg/e2e/end_to_end_test.go b/pkg/e2e/end_to_end_test.go index aef96bde..5532eeae 100644 --- a/pkg/e2e/end_to_end_test.go +++ b/pkg/e2e/end_to_end_test.go @@ -249,7 +249,7 @@ var _ = Describe("E2E", Serial, Label("e2e"), func() { streamClientConfiguration, err := raw.NewClientConfiguration(rabbitmqUri) Expect(err).ToNot(HaveOccurred()) connectionName := "notify-disconnection-test-1" - streamClientConfiguration.SetConnectionName(connectionName) + streamClientConfiguration.ConnectionName = connectionName By("preparing the environment") streamClient, err := raw.DialConfig(itCtx, streamClientConfiguration) diff --git a/pkg/stream/environment_internal_test.go b/pkg/stream/environment_internal_test.go deleted file mode 100644 index 55cbbe3a..00000000 --- a/pkg/stream/environment_internal_test.go +++ /dev/null @@ -1,83 +0,0 @@ -//go:build rabbitmq.stream.test - -package stream - -import ( - "context" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" - "go.uber.org/mock/gomock" - "time" -) - -var _ = Describe("Environment Internal Unit Test", Pending, func() { - var ( - env *Environment - fakeRawClient *MockRawClient - mockCtrl *gomock.Controller - backOffPolicyFn = func(_ int) time.Duration { - return time.Millisecond * 10 - } - ) - - BeforeEach(func() { - mockCtrl = gomock.NewController(GinkgoT()) - fakeRawClient = NewMockRawClient(mockCtrl) - - c := NewEnvironmentConfiguration( - WithLazyInitialization(true), - WithUri("rabbitmq-stream://foo:bar@fakehost:1234/my-vhost"), - ) - var err error - env, err = NewEnvironment(context.Background(), c) - Expect(err).ToNot(HaveOccurred()) - - env.AppendLocatorRawClient(fakeRawClient) - env.SetBackoffPolicy(backOffPolicyFn) - }) - - It("creates a new producer", func() { - By("locating the stream leader", func() { - fakeRawClient.EXPECT(). - IsOpen().Return(true) - fakeRawClient.EXPECT(). - MetadataQuery(gomock.Any(), gomock.AssignableToTypeOf([]string{"string-slice"})). - Return(raw.MetadataResponseForStream("some-stream"), nil) - fakeRawClient.EXPECT(). - DeclarePublisher( - gomock.Any(), - gomock.AssignableToTypeOf(uint8(1)), - gomock.AssignableToTypeOf("string"), - gomock.AssignableToTypeOf("string"), - ) - }) - p, err := env.CreateProducer(context.Background(), "some-stream", &ProducerOptions{}) - Expect(err).ToNot(HaveOccurred()) - - By("creating a producer manager connected to the leader") - Expect(env.producerManagers).To(HaveLen(1)) - Expect(env.producerManagers[0].config.Uris).To(ContainElement("rabbitmq-stream://foo:bar@fakehost:1234/my-vhost")) - - By("assigning a producer manager for the producer") - Expect(env.producerManagers[0].producers).To(ContainElement(p)) - }) - - It("returns an error for empty stream name", func() { - Skip("TODO") - }) -}) - -func prepareMockForDeclarePublish(mock *MockRawClient, pubId uint8) { - mock.EXPECT(). - IsOpen(). - Return(true) // from maybeInitializeLocator - - mock.EXPECT(). - DeclarePublisher( - gomock.Any(), - gomock.Eq(pubId), - gomock.AssignableToTypeOf("string"), - gomock.AssignableToTypeOf("string"), - ) -} diff --git a/pkg/stream/environment_test.go b/pkg/stream/environment_test.go index 0a2dd729..d29a4abe 100644 --- a/pkg/stream/environment_test.go +++ b/pkg/stream/environment_test.go @@ -281,12 +281,15 @@ var _ = Describe("Environment", func() { Return(errors.New("something went wrong")) }) - It("logs the error and moves on", Pending, func() { - // TODO: the locator has its own logger, it does not use the logger - // from the context. This tests seems to make no sense, as the - // locator is appended by an utility function for tests, and - // this tests assumes that the locator uses the logger from context, - // which is not true. Revisit this test and its assumptions. + It("logs the error and moves on", func() { + Skip("the locator has its own logger, it does not use the logger from the context. This tests seems to make no sense, as the locator is appended by an utility function for tests, and this tests assumes that the locator uses the logger from context, which is not true. Revisit this test and its assumptions") + /* + TODO: the locator has its own logger, it does not use the logger + from the context. This tests seems to make no sense, as the locator is appended + by an utility function for tests, and this tests assumes that the locator uses + the logger from context, which is not true. Revisit this test and its + assumptions. + */ logBuffer := gbytes.NewBuffer() logger := slog.New(slog.NewTextHandler(logBuffer, nil)) ctx := raw.NewContextWithLogger(context.Background(), *logger) From 3dcf74d222ae4df12f9852c902eca4ec286a03d7 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Fri, 17 Nov 2023 15:59:13 +0000 Subject: [PATCH 22/24] Fix: delete publisher when producer is removed from manager Signed-off-by: Aitor Perez Cedres --- pkg/stream/producer_manager.go | 6 ++++++ pkg/stream/producer_manager_test.go | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkg/stream/producer_manager.go b/pkg/stream/producer_manager.go index 0cd77b78..bdea170b 100644 --- a/pkg/stream/producer_manager.go +++ b/pkg/stream/producer_manager.go @@ -135,6 +135,12 @@ func (p *producerManager) removeProducer(id int) error { if id < 0 { return errNegativeId } + ctx, cancel := maybeApplyDefaultTimeout(context.Background()) + defer cancel() + p.clientM.Lock() + _ = p.client.DeletePublisher(ctx, uint8(id)) + // TODO log error if not nil + p.clientM.Unlock() p.producers[id] = nil p.producerCount -= 1 diff --git a/pkg/stream/producer_manager_test.go b/pkg/stream/producer_manager_test.go index 7fa8bdd1..e583f9f9 100644 --- a/pkg/stream/producer_manager_test.go +++ b/pkg/stream/producer_manager_test.go @@ -95,6 +95,8 @@ var _ = Describe("ProducerManager", func() { prepareMockDeclarePublisher(fakeRawClient, 1) prepareMockDeclarePublisher(fakeRawClient, 1) prepareMockDeclarePublisher(fakeRawClient, 2) + prepareMockDeletePublisher(fakeRawClient, 1) + prepareMockDeletePublisher(fakeRawClient, 2) pm := newProducerManager(0, EnvironmentConfiguration{ MaxProducersByConnection: 5, }) @@ -144,6 +146,7 @@ var _ = Describe("ProducerManager", func() { It("closes the connection", func() { fakeRawClient.EXPECT().Close(gomock.Any()) prepareMockDeclarePublisher(fakeRawClient, 0) + prepareMockDeletePublisher(fakeRawClient, 0) // called during producer.Close() callback pm := newProducerManager(0, EnvironmentConfiguration{MaxProducersByConnection: 5}) pm.client = fakeRawClient pm.open = true @@ -170,3 +173,11 @@ func prepareMockDeclarePublisher(m *MockRawClient, publisherId uint8) { gomock.AssignableToTypeOf(""), ) } + +func prepareMockDeletePublisher(m *MockRawClient, publisherId uint8) { + ctxType := reflect.TypeOf((*context.Context)(nil)).Elem() + m.EXPECT().DeletePublisher( + gomock.AssignableToTypeOf(ctxType), + gomock.Eq(publisherId), + ) +} From 3ccfcf2fc1c66493facd31b23ccddc04e92d52b3 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Fri, 15 Dec 2023 15:48:36 +0000 Subject: [PATCH 23/24] Fix staticcheck linter errors Signed-off-by: Aitor Perez Cedres --- .github/workflows/static-check.yaml | 2 +- go.mod | 22 +++++++-------- go.sum | 43 +++++++++++++++-------------- main.go | 3 ++ pkg/stream/message_accumulator.go | 4 --- pkg/stream/producer_manager.go | 7 +++-- pkg/stream/stream_suite_test.go | 9 ++---- pkg/stream/util.go | 4 --- 8 files changed, 44 insertions(+), 50 deletions(-) diff --git a/.github/workflows/static-check.yaml b/.github/workflows/static-check.yaml index 9a40b761..899e5dff 100644 --- a/.github/workflows/static-check.yaml +++ b/.github/workflows/static-check.yaml @@ -21,7 +21,7 @@ jobs: with: version: "2023.1.3" install-go: false - build-tags: "rabbitmq.stream.test,rabbitmq.stream.e2e" + build-tags: "rabbitmq.stream.test,rabbitmq.stream.e2e,rabbitmq.stream.system_test" format-check: name: Check Go Formatting diff --git a/go.mod b/go.mod index e90513fb..17f36f41 100644 --- a/go.mod +++ b/go.mod @@ -6,20 +6,20 @@ toolchain go1.21.0 require ( github.com/michaelklishin/rabbit-hole/v2 v2.15.0 - github.com/onsi/ginkgo/v2 v2.11.0 - github.com/onsi/gomega v1.27.10 - go.uber.org/mock v0.2.0 - golang.org/x/mod v0.12.0 + github.com/onsi/ginkgo/v2 v2.13.2 + github.com/onsi/gomega v1.30.0 + go.uber.org/mock v0.3.0 + golang.org/x/mod v0.14.0 ) require ( - github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/logr v1.3.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect - golang.org/x/tools v0.12.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20231212022811-ec68065c825e // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.16.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4459025f..019a2f52 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= @@ -30,11 +30,12 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c= -github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/pprof v0.0.0-20231212022811-ec68065c825e h1:bwOy7hAFd0C91URzMIEBfr6BAz29yk7Qj0cy6S7DJlU= +github.com/google/pprof v0.0.0-20231212022811-ec68065c825e/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -64,8 +65,8 @@ github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJ github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= +github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -80,8 +81,8 @@ github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfad github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA= @@ -97,8 +98,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU= -go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM= +go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= +go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -111,8 +112,8 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -131,8 +132,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -165,8 +166,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -184,8 +185,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -195,8 +196,8 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index d3e87f1c..3dc7dd76 100644 --- a/main.go +++ b/main.go @@ -69,6 +69,9 @@ func runSmartClient() { } }, }) + if err != nil { + panic(err) + } for i := 0; i < 1_000_000; i++ { err = producer.Send(context.Background(), amqp.Message{Data: []byte(fmt.Sprintf("Message #%d", i))}) diff --git a/pkg/stream/message_accumulator.go b/pkg/stream/message_accumulator.go index 9f86bf12..4b40e001 100644 --- a/pkg/stream/message_accumulator.go +++ b/pkg/stream/message_accumulator.go @@ -51,7 +51,3 @@ func (m *messageAccumulator) get() PublishingMessage { return nil } } - -func (m *messageAccumulator) isEmpty() bool { - return len(m.messages) == 0 -} diff --git a/pkg/stream/producer_manager.go b/pkg/stream/producer_manager.go index bdea170b..d7e880b8 100644 --- a/pkg/stream/producer_manager.go +++ b/pkg/stream/producer_manager.go @@ -22,8 +22,9 @@ type producerManager struct { id int config EnvironmentConfiguration // records the endpoint that this manager is connected to - connectionEndpoint raw.RabbitmqAddress - producers []internalProducer + connectionEndpoint raw.RabbitmqAddress + producers []internalProducer + //lint:ignore U1000 this is a WIP. It will be used next patches confirmationChannels []chan *publishConfirmOrError // keeps track of how many producers are assigned to this manager producerCount int @@ -53,6 +54,7 @@ func newProducerManagerWithClient(id int, config EnvironmentConfiguration, clien return manager } +//lint:ignore U1000 this is a WIP. It will be used next patches func (p *producerManager) connect(ctx context.Context, uri string) error { conf, err := raw.NewClientConfiguration(uri) if err != nil { @@ -76,6 +78,7 @@ func (p *producerManager) connect(ctx context.Context, uri string) error { return nil } +//lint:ignore U1000 this is a WIP. It will be used next patches func (p *producerManager) publishConfirmationListener(ctx context.Context, c chan *raw.PublishConfirm) { panic("implement me!") } diff --git a/pkg/stream/stream_suite_test.go b/pkg/stream/stream_suite_test.go index 53bfb4df..bd56cc35 100644 --- a/pkg/stream/stream_suite_test.go +++ b/pkg/stream/stream_suite_test.go @@ -22,9 +22,8 @@ func TestStream(t *testing.T) { } const ( - SystemTestSkipRabbitStart = "RABBITMQ_STREAM_SKIP_RABBIT_START" - SystemTestEnvVarName = "RABBITMQ_STREAM_RUN_SYSTEM_TEST" - containerName string = "rabbitmq-stream-go-client" + SystemTestEnvVarName = "RABBITMQ_STREAM_RUN_SYSTEM_TEST" + containerName string = "rabbitmq-stream-go-client" ) type plainTextMessage struct { @@ -66,10 +65,6 @@ var _ = SynchronizedBeforeSuite(func() { logger := log.New(GinkgoWriter, "[SBS] ", log.Ldate|log.Lmsgprefix) if _, isSet := os.LookupEnv(SystemTestEnvVarName); !isSet { logger.Println("System test variable to run system test not set. Skipping system tests...") - } - - if _, isSet := os.LookupEnv(SystemTestSkipRabbitStart); isSet { - logger.Println("System test variable to skip RabbitMQ start is set. Skipping...") return } diff --git a/pkg/stream/util.go b/pkg/stream/util.go index 51ea592a..87bccbce 100644 --- a/pkg/stream/util.go +++ b/pkg/stream/util.go @@ -76,10 +76,6 @@ type autoIncrementingSequence[T integer] struct { value T } -func newAutoIncrementingSequence[T integer](startingValue T) *autoIncrementingSequence[T] { - return &autoIncrementingSequence[T]{value: startingValue} -} - func (a *autoIncrementingSequence[T]) next() (next T) { a.Lock() defer a.Unlock() From dc522daf94531eb03443d48e0f33deff5833b563 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres Date: Tue, 2 Jan 2024 13:40:02 +0000 Subject: [PATCH 24/24] Fix system test before suite Skipping system tests were not actually skipping system tests. Signed-off-by: Aitor Perez Cedres --- Makefile | 9 +++++++++ go.mod | 6 +++--- go.sum | 12 ++++++------ pkg/stream/stream_suite_test.go | 13 ++++++++++--- pkg/stream/system_test.go | 12 ++++++++++-- 5 files changed, 38 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index c4787f4f..23979cfc 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,15 @@ tests-ci: --fail-on-pending \ --keep-going +.PHONY: system-tests +system-tests: ## Run system tests. It starts a rabbitmq container. To skip starting the rabbit container, use RABBITMQ_STREAM_SKIP_RABBIT_START="skip" + @printf "$(GREEN)Running system tests in parallel$(NORMAL)\n" + RABBITMQ_STREAM_RUN_SYSTEM_TEST="run" \ + $(GINKGO) $(GINKGO_RUN_SHARED_FLAGS) $(GINKGO_RUN_FLAGS) \ + --tags="rabbitmq.stream.test,rabbitmq.stream.system_test" \ + --focus 'System tests' \ + $(GINKGO_EXTRA) ./pkg/stream/ + #### e2e test suite accepts the flags -keep-rabbit-container=true and -rabbit-debug-log=true #### -keep-rabbit-container=true does not delete the rabbit container after the suite run. It is useful to examine rabbit logs after a test failure diff --git a/go.mod b/go.mod index 17f36f41..13729ceb 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,15 @@ require ( github.com/michaelklishin/rabbit-hole/v2 v2.15.0 github.com/onsi/ginkgo/v2 v2.13.2 github.com/onsi/gomega v1.30.0 - go.uber.org/mock v0.3.0 + go.uber.org/mock v0.4.0 golang.org/x/mod v0.14.0 ) require ( - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20231212022811-ec68065c825e // indirect + github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 019a2f52..9a400c2b 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= @@ -34,8 +34,8 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20231212022811-ec68065c825e h1:bwOy7hAFd0C91URzMIEBfr6BAz29yk7Qj0cy6S7DJlU= -github.com/google/pprof v0.0.0-20231212022811-ec68065c825e/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42 h1:dHLYa5D8/Ta0aLR2XcPsrkpAgGeFs6thhMcQK0oQ0n8= +github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -98,8 +98,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= -go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/pkg/stream/stream_suite_test.go b/pkg/stream/stream_suite_test.go index bd56cc35..b8c6b15e 100644 --- a/pkg/stream/stream_suite_test.go +++ b/pkg/stream/stream_suite_test.go @@ -22,8 +22,9 @@ func TestStream(t *testing.T) { } const ( - SystemTestEnvVarName = "RABBITMQ_STREAM_RUN_SYSTEM_TEST" - containerName string = "rabbitmq-stream-go-client" + SystemTestSkipRabbitStart = "RABBITMQ_STREAM_SKIP_RABBIT_START" + SystemTestEnvVarName = "RABBITMQ_STREAM_RUN_SYSTEM_TEST" + containerName string = "rabbitmq-stream-go-client" ) type plainTextMessage struct { @@ -64,7 +65,13 @@ var _ = SynchronizedBeforeSuite(func() { // Just once logger := log.New(GinkgoWriter, "[SBS] ", log.Ldate|log.Lmsgprefix) if _, isSet := os.LookupEnv(SystemTestEnvVarName); !isSet { - logger.Println("System test variable to run system test not set. Skipping system tests...") + // variable to run system tests is not set. We skip all the setup. + // Individual specs will be skipped in the system test specific nodes + return + } + + if _, isSet := os.LookupEnv(SystemTestSkipRabbitStart); isSet { + logger.Println("System test variable to skip RabbitMQ start is set. RabbitMQ container won't be started") return } diff --git a/pkg/stream/system_test.go b/pkg/stream/system_test.go index 4eb12146..3d65598a 100644 --- a/pkg/stream/system_test.go +++ b/pkg/stream/system_test.go @@ -13,17 +13,25 @@ import ( "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/raw" "github.com/rabbitmq/rabbitmq-stream-go-client/v2/pkg/stream" "log/slog" + "os" "sync" "time" ) -var _ = Describe("stream package", func() { +var _ = Describe("System tests", func() { const ( streamName = "stream-system-test" // 100 byte message messageBody = "Rabbitmq-is-awesomeRabbitmq-is-awesomeRabbitmq-is-awesomeRabbitmq-is-awesomeRabbitmq-is-awesome!!!!!" defaultRabbitmqUri = "rabbitmq-stream://guest:guest@localhost/%2F" ) + + BeforeEach(func() { + if _, isSet := os.LookupEnv(SystemTestEnvVarName); !isSet { + Skip("System test variable to run system test not set. Skipping system tests...") + } + }) + It("can create and connect to a stream, publish and receive messages", func(ctx SpecContext) { debugLogger := slog.New(slog.NewTextHandler(GinkgoWriter, &slog.HandlerOptions{Level: slog.LevelDebug})) itCtx := raw.NewContextWithLogger(ctx, *debugLogger) @@ -120,7 +128,7 @@ var _ = Describe("stream package", func() { Expect(env.DeleteStream(itCtx, streamName)).To(Succeed()) }) - It("connects to RabbitMQ", Focus, func() { + It("connects to RabbitMQ", func() { uri := "rabbitmq-stream://localhost:5552" conf := stream.NewEnvironmentConfiguration( stream.WithUri(uri),