From f906321db52b95ce4356734036556dc7938cf357 Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Sat, 19 Apr 2025 00:03:45 +0800 Subject: [PATCH 1/7] feat(sources/s3): migrate to AWS SDK v2 Closes https://github.com/trufflesecurity/trufflehog/issues/4054. Signed-off-by: Eng Zer Jun --- go.mod | 8 +- go.sum | 35 +++--- pkg/sources/s3/checkpointer.go | 4 +- pkg/sources/s3/checkpointer_test.go | 13 ++- pkg/sources/s3/s3.go | 175 ++++++++++++++-------------- 5 files changed, 115 insertions(+), 120 deletions(-) diff --git a/go.mod b/go.mod index 7feb4a7d596e..5078a511ba31 100644 --- a/go.mod +++ b/go.mod @@ -19,10 +19,11 @@ require ( github.com/adrg/strutil v0.3.1 github.com/alecthomas/kingpin/v2 v2.4.0 github.com/avast/apkparser v0.0.0-20250307094510-e2100ee9c0f5 - github.com/aws/aws-sdk-go v1.55.6 github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 github.com/aws/aws-sdk-go-v2/service/sns v1.34.4 github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 github.com/aws/smithy-go v1.22.3 @@ -150,12 +151,16 @@ require ( github.com/andybalholm/brotli v1.1.1 // indirect github.com/apache/arrow/go/v14 v14.0.2 // indirect github.com/atotto/clipboard v0.1.4 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect @@ -230,7 +235,6 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jpillora/s3 v1.1.4 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect diff --git a/go.sum b/go.sum index d67a04f7443f..17b73e368b24 100644 --- a/go.sum +++ b/go.sum @@ -116,44 +116,44 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/avast/apkparser v0.0.0-20250307094510-e2100ee9c0f5 h1:zRaiVswCgpvZ6lyeErxC94EJaZdSoj0mzUPYRBOnqCI= github.com/avast/apkparser v0.0.0-20250307094510-e2100ee9c0f5/go.mod h1:GNvprXNmXaDjpHmN3RFxz5QdK5VXTUvmQludCbjoBy4= -github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= -github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= -github.com/aws/aws-sdk-go-v2/config v1.29.13 h1:RgdPqWoE8nPpIekpVpDJsBckbqT4Liiaq9f35pbTh1Y= -github.com/aws/aws-sdk-go-v2/config v1.29.13/go.mod h1:NI28qs/IOUIRhsR7GQ/JdexoqRN9tDxkIrYZq0SOF44= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= -github.com/aws/aws-sdk-go-v2/credentials v1.17.66 h1:aKpEKaTy6n4CEJeYI1MNj97oSDLi4xro3UzQfwf5RWE= -github.com/aws/aws-sdk-go-v2/credentials v1.17.66/go.mod h1:xQ5SusDmHb/fy55wU0QqTy0yNfLqxzec59YcsRZB+rI= github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72 h1:PcKMOZfp+kNtJTw2HF2op6SjDvwPBYRvz0Y24PQLUR4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72/go.mod h1:vq7/m7dahFXcdzWVOvvjasDI9RcsD3RsTfHmDundJYg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= -github.com/aws/aws-sdk-go-v2/service/sns v1.34.3 h1:iJtp/KnPsgMO4TSGfjqi3oGr+R73W7xWqDXHCbqdnv8= -github.com/aws/aws-sdk-go-v2/service/sns v1.34.3/go.mod h1:PJtxxMdj747j8DeZENRTTYAz/lx/pADn/U0k7YNNiUY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 h1:tWUG+4wZqdMl/znThEk9tcCy8tTMxq8dW0JTgamohrY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= github.com/aws/aws-sdk-go-v2/service/sns v1.34.4 h1:ihddI5wufQQCJiujUgAvWRqZcfDmSKIfXlAuX7T95cg= github.com/aws/aws-sdk-go-v2/service/sns v1.34.4/go.mod h1:PJtxxMdj747j8DeZENRTTYAz/lx/pADn/U0k7YNNiUY= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 h1:xz7WvTMfSStb9Y8NpCT82FXLNC3QasqBfuAFHY4Pk5g= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.18/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= -github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= -github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E= @@ -310,8 +310,6 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= -github.com/getsentry/sentry-go v0.30.0 h1:lWUwDnY7sKHaVIoZ9wYqRHJ5iEmoc0pqcRqFkosKzBo= -github.com/getsentry/sentry-go v0.30.0/go.mod h1:WU9B9/1/sHDqeV8T+3VwwbjeR5MSXs/6aqG3mqZrezA= github.com/getsentry/sentry-go v0.32.0 h1:YKs+//QmwE3DcYtfKRH8/KyOOF/I6Qnx7qYGNHCGmCY= github.com/getsentry/sentry-go v0.32.0/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= @@ -506,10 +504,6 @@ github.com/jedib0t/go-pretty/v6 v6.6.7 h1:m+LbHpm0aIAPLzLbMfn8dc3Ht8MW7lsSO4MPIt github.com/jedib0t/go-pretty/v6 v6.6.7/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -614,9 +608,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= -github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -936,8 +929,6 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= -golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/pkg/sources/s3/checkpointer.go b/pkg/sources/s3/checkpointer.go index b47dcccca7bd..b2c957179633 100644 --- a/pkg/sources/s3/checkpointer.go +++ b/pkg/sources/s3/checkpointer.go @@ -5,7 +5,7 @@ import ( "fmt" "sync" - "github.com/aws/aws-sdk-go/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -153,7 +153,7 @@ func (p *Checkpointer) UpdateObjectCompletion( ctx context.Context, completedIdx int, bucket string, - pageContents []*s3.Object, + pageContents []s3types.Object, ) error { ctx = context.WithValues(ctx, "bucket", bucket, "completedIdx", completedIdx) ctx.Logger().V(5).Info("Updating progress") diff --git a/pkg/sources/s3/checkpointer_test.go b/pkg/sources/s3/checkpointer_test.go index 43dc84557d6e..137a7f0de5e3 100644 --- a/pkg/sources/s3/checkpointer_test.go +++ b/pkg/sources/s3/checkpointer_test.go @@ -5,7 +5,8 @@ import ( "fmt" "testing" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,11 +22,11 @@ func TestCheckpointerResumption(t *testing.T) { tracker := NewCheckpointer(ctx, initialProgress) firstPage := &s3.ListObjectsV2Output{ - Contents: make([]*s3.Object, 12), // Total of 12 objects + Contents: make([]s3types.Object, 12), // Total of 12 objects } for i := range 12 { key := fmt.Sprintf("key-%d", i) - firstPage.Contents[i] = &s3.Object{Key: &key} + firstPage.Contents[i] = s3types.Object{Key: &key} } // Process first 6 objects. @@ -134,7 +135,7 @@ func TestGetResumePoint(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - tracker := &Checkpointer{ progress: tt.progress} + tracker := &Checkpointer{progress: tt.progress} resumePoint, err := tracker.ResumePoint(context.Background()) if tt.expectError { @@ -223,10 +224,10 @@ func TestCheckpointerUpdate(t *testing.T) { lowestIncompleteIdx: 0, } - page := &s3.ListObjectsV2Output{Contents: make([]*s3.Object, tt.pageSize)} + page := &s3.ListObjectsV2Output{Contents: make([]s3types.Object, tt.pageSize)} for i := range tt.pageSize { key := fmt.Sprintf("key-%d", i) - page.Contents[i] = &s3.Object{Key: &key} + page.Contents[i] = s3types.Object{Key: &key} } // Setup pre-completed objects. diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index 4c13b988e1d3..1e93f95bc563 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -8,13 +8,14 @@ import ( "sync/atomic" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/aws/aws-sdk-go/service/sts" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/go-errors/errors" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" @@ -108,7 +109,7 @@ func (s *Source) Init( func (s *Source) Validate(ctx context.Context) []error { var errs []error - visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) error { + visitor := func(c context.Context, defaultRegionClient *s3.Client, roleArn string, buckets []string) error { roleErrs := s.validateBucketAccess(c, defaultRegionClient, roleArn, buckets) if len(roleErrs) > 0 { errs = append(errs, roleErrs...) @@ -134,14 +135,11 @@ func (s *Source) setMaxObjectSize(maxObjectSize int64) { } } -func (s *Source) newClient(region, roleArn string) (*s3.S3, error) { - cfg := aws.NewConfig() - cfg.CredentialsChainVerboseErrors = aws.Bool(true) - cfg.Region = aws.String(region) - +func (s *Source) newClient(ctx context.Context, region, roleArn string) (*s3.Client, error) { + var credsProvider aws.CredentialsProvider switch cred := s.conn.GetCredential().(type) { case *sourcespb.S3_SessionToken: - cfg.Credentials = credentials.NewStaticCredentials( + credsProvider = credentials.NewStaticCredentialsProvider( cred.SessionToken.GetKey(), cred.SessionToken.GetSecret(), cred.SessionToken.GetSessionToken(), @@ -149,36 +147,50 @@ func (s *Source) newClient(region, roleArn string) (*s3.S3, error) { log.RedactGlobally(cred.SessionToken.GetSecret()) log.RedactGlobally(cred.SessionToken.GetSessionToken()) case *sourcespb.S3_AccessKey: - cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.GetKey(), cred.AccessKey.GetSecret(), "") + credsProvider = credentials.NewStaticCredentialsProvider(cred.AccessKey.GetKey(), cred.AccessKey.GetSecret(), "") log.RedactGlobally(cred.AccessKey.GetSecret()) case *sourcespb.S3_Unauthenticated: - cfg.Credentials = credentials.AnonymousCredentials + credsProvider = aws.AnonymousCredentials{} default: // In all other cases, the AWS SDK will follow its normal waterfall logic to pick up credentials (i.e. they can // come from the environment or the credentials file or whatever else AWS gets up to). } if roleArn != "" { - sess, err := session.NewSession(cfg) + // A valid credentials is required to assume IAM role. aws.AnonymousCredentials is not a valid credentials. + // If the value of credsProvider is aws.AnonymousCredentials{} from the above switch-case, + // we will need to set credsProvider to nil to use SDK's default credential chain. + _, isUnauthenticated := credsProvider.(aws.AnonymousCredentials) + if !isUnauthenticated { + credsProvider = nil + } + + // The config loaded here will be used to retrieve and refresh temporary credentials from AssumeRole + cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credsProvider)) if err != nil { return nil, err } - stsClient := sts.New(sess) - cfg.Credentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) { - p.RoleSessionName = "trufflehog" + stsClient := sts.NewFromConfig(cfg) + provider := stscreds.NewAssumeRoleProvider(stsClient, roleArn, func(options *stscreds.AssumeRoleOptions) { + options.RoleSessionName = "trufflehog" }) + // From https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html#specify-credentials-programmatically: + // "If you explicitly configure a provider on aws.Config directly, + // you must also explicitly wrap the provider with this type using NewCredentialsCache" + credsProvider = aws.NewCredentialsCache(provider) } - sess, err := session.NewSessionWithOptions(session.Options{ - SharedConfigState: session.SharedConfigEnable, - Config: *cfg, - }) + cfg, err := config.LoadDefaultConfig( + ctx, + config.WithRegion(region), + config.WithCredentialsProvider(credsProvider), + ) if err != nil { return nil, err } - return s3.New(sess), nil + return s3.NewFromConfig(cfg), nil } // getBucketsToScan returns a list of S3 buckets to scan. @@ -188,7 +200,7 @@ func (s *Source) newClient(region, roleArn string) (*s3.S3, error) { // which allows resuming scanning from the same place if the scan is interrupted. // // Note: The IAM identity needs the s3:ListBuckets permission. -func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) { +func (s *Source) getBucketsToScan(ctx context.Context, client *s3.Client) ([]string, error) { if buckets := s.conn.GetBuckets(); len(buckets) > 0 { slices.Sort(buckets) return buckets, nil @@ -199,7 +211,7 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) { ignore[bucket] = struct{}{} } - res, err := client.ListBuckets(&s3.ListBucketsInput{}) + res, err := client.ListBuckets(ctx, &s3.ListBucketsInput{}) if err != nil { return nil, err } @@ -220,7 +232,7 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) { type pageMetadata struct { bucket string // The name of the S3 bucket being scanned pageNumber int // Current page number in the pagination sequence - client *s3.S3 // AWS S3 client configured for the appropriate region + client *s3.Client // AWS S3 client configured for the appropriate region page *s3.ListObjectsV2Output // Contains the list of S3 objects in this page } @@ -280,7 +292,7 @@ func determineResumePosition(ctx context.Context, tracker *Checkpointer, buckets func (s *Source) scanBuckets( ctx context.Context, - client *s3.S3, + client *s3.Client, role string, bucketsToScan []string, chunksChan chan *sources.Chunk, @@ -346,36 +358,34 @@ func (s *Source) scanBuckets( } pageNumber := 1 - err = regionalClient.ListObjectsV2PagesWithContext( - ctx, - input, - func(page *s3.ListObjectsV2Output, _ bool) bool { - pageMetadata := pageMetadata{ - bucket: bucket, - pageNumber: pageNumber, - client: regionalClient, - page: page, - } - processingState := processingState{ - errorCount: &errorCount, - objectCount: &objectCount, + paginator := s3.NewListObjectsV2Paginator(regionalClient, input) + for paginator.HasMorePages() { + output, err := paginator.NextPage(ctx) + if err != nil { + if role == "" { + ctx.Logger().Error(err, "could not list objects in bucket") + } else { + // Our documentation blesses specifying a role to assume without specifying buckets to scan, which will + // often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the + // account, but the role probably doesn't have access to all of them). This makes it expected behavior + // and therefore not an error. + ctx.Logger().V(3).Info("could not list objects in bucket", "err", err) } - s.pageChunker(ctx, pageMetadata, processingState, chunksChan) - - pageNumber++ - return true - }) - - if err != nil { - if role == "" { - ctx.Logger().Error(err, "could not list objects in bucket") - } else { - // Our documentation blesses specifying a role to assume without specifying buckets to scan, which will - // often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the - // account, but the role probably doesn't have access to all of them). This makes it expected behavior - // and therefore not an error. - ctx.Logger().V(3).Info("could not list objects in bucket", "err", err) + continue + } + pageMetadata := pageMetadata{ + bucket: bucket, + pageNumber: pageNumber, + client: regionalClient, + page: output, } + processingState := processingState{ + errorCount: &errorCount, + objectCount: &objectCount, + } + s.pageChunker(ctx, pageMetadata, processingState, chunksChan) + + pageNumber++ } } @@ -389,7 +399,7 @@ func (s *Source) scanBuckets( // Chunks emits chunks of bytes over a channel. func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error { - visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) error { + visitor := func(c context.Context, defaultRegionClient *s3.Client, roleArn string, buckets []string) error { s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan) return nil } @@ -399,20 +409,20 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . func (s *Source) getRegionalClientForBucket( ctx context.Context, - defaultRegionClient *s3.S3, + defaultRegionClient *s3.Client, role string, bucket string, -) (*s3.S3, error) { - region, err := s3manager.GetBucketRegionWithClient(ctx, defaultRegionClient, bucket) +) (*s3.Client, error) { + region, err := s3manager.GetBucketRegion(ctx, defaultRegionClient, bucket) if err != nil { - return nil, fmt.Errorf("could not get s3 region for bucket: %s", bucket) + return nil, fmt.Errorf("could not get s3 region for bucket: %s: %w", bucket, err) } if region == defaultAWSRegion { return defaultRegionClient, nil } - regionalClient, err := s.newClient(region, role) + regionalClient, err := s.newClient(ctx, region, role) if err != nil { return nil, fmt.Errorf("could not create regional s3 client for bucket %s: %w", bucket, err) } @@ -431,14 +441,6 @@ func (s *Source) pageChunker( ctx = context.WithValues(ctx, "bucket", metadata.bucket, "page_number", metadata.pageNumber) for objIdx, obj := range metadata.page.Contents { - if obj == nil { - s.metricsCollector.RecordObjectSkipped(metadata.bucket, "nil_object", 0) - if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil { - ctx.Logger().Error(err, "could not update progress for nil object") - } - continue - } - ctx = context.WithValues(ctx, "key", *obj.Key, "size", *obj.Size) if common.IsDone(ctx) { @@ -446,8 +448,8 @@ func (s *Source) pageChunker( } // Skip GLACIER and GLACIER_IR objects. - if obj.StorageClass == nil || strings.Contains(*obj.StorageClass, "GLACIER") { - ctx.Logger().V(5).Info("Skipping object in storage class", "storage_class", *obj.StorageClass) + if obj.StorageClass == s3types.ObjectStorageClassGlacier || obj.StorageClass == s3types.ObjectStorageClassGlacierIr { + ctx.Logger().V(5).Info("Skipping object in storage class", "storage_class", obj.StorageClass) s.metricsCollector.RecordObjectSkipped(metadata.bucket, "storage_class", float64(*obj.Size)) if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil { ctx.Logger().Error(err, "could not update progress for glacier object") @@ -514,7 +516,7 @@ func (s *Source) pageChunker( objCtx, cancel := context.WithTimeout(ctx, getObjectTimeout) defer cancel() - res, err := metadata.client.GetObjectWithContext(objCtx, &s3.GetObjectInput{ + res, err := metadata.client.GetObject(objCtx, &s3.GetObjectInput{ Bucket: &metadata.bucket, Key: obj.Key, }) @@ -567,7 +569,7 @@ func (s *Source) pageChunker( S3: &source_metadatapb.S3{ Bucket: metadata.bucket, File: sanitizer.UTF8(*obj.Key), - Link: sanitizer.UTF8(makeS3Link(metadata.bucket, *metadata.client.Config.Region, *obj.Key)), + Link: sanitizer.UTF8(makeS3Link(metadata.bucket, metadata.client.Options().Region, *obj.Key)), Email: sanitizer.UTF8(email), Timestamp: sanitizer.UTF8(modified), }, @@ -605,7 +607,7 @@ func (s *Source) pageChunker( _ = s.jobPool.Wait() } -func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleArn string, buckets []string) []error { +func (s *Source) validateBucketAccess(ctx context.Context, client *s3.Client, roleArn string, buckets []string) []error { shouldHaveAccessToAllBuckets := roleArn == "" wasAbleToListAnyBucket := false var errs []error @@ -621,7 +623,7 @@ func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleAr continue } - _, err = regionalClient.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket}) + _, err = regionalClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) if err == nil { wasAbleToListAnyBucket = true } else if shouldHaveAccessToAllBuckets { @@ -650,7 +652,7 @@ func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleAr // If no roles are configured, it will call the function with an empty role ARN. func (s *Source) visitRoles( ctx context.Context, - f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) error, + f func(c context.Context, defaultRegionClient *s3.Client, roleArn string, buckets []string) error, ) error { roles := s.conn.GetRoles() if len(roles) == 0 { @@ -660,12 +662,12 @@ func (s *Source) visitRoles( for _, role := range roles { s.metricsCollector.RecordRoleScanned(role) - client, err := s.newClient(defaultAWSRegion, role) + client, err := s.newClient(ctx, defaultAWSRegion, role) if err != nil { return fmt.Errorf("could not create s3 client: %w", err) } - bucketsToScan, err := s.getBucketsToScan(client) + bucketsToScan, err := s.getBucketsToScan(ctx, client) if err != nil { return fmt.Errorf("role %q could not list any s3 buckets for scanning: %w", role, err) } @@ -678,13 +680,10 @@ func (s *Source) visitRoles( return nil } -// S3 links currently have the general format of: -// https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key] +// makeS3Link creates a S3 virtual-hosted–style URIs. They have the format of: +// https://[bucket-name].s3.[region-code].amazonaws.com/[key-name] +// +// See https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access func makeS3Link(bucket, region, key string) string { - if region == defaultAWSRegion { - region = "" - } else { - region = "." + region - } - return fmt.Sprintf("https://%s.s3%s.amazonaws.com/%s", bucket, region, key) + return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", bucket, region, key) } From a5411bfff056301fcc764247627dbccb2d9d8045 Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Sat, 19 Apr 2025 17:26:49 +0800 Subject: [PATCH 2/7] Fix conditional typo Reference: https://github.com/trufflesecurity/trufflehog/pull/4069#discussion_r2050946321 Signed-off-by: Eng Zer Jun --- pkg/sources/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index 1e93f95bc563..f6fd1013485d 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -161,7 +161,7 @@ func (s *Source) newClient(ctx context.Context, region, roleArn string) (*s3.Cli // If the value of credsProvider is aws.AnonymousCredentials{} from the above switch-case, // we will need to set credsProvider to nil to use SDK's default credential chain. _, isUnauthenticated := credsProvider.(aws.AnonymousCredentials) - if !isUnauthenticated { + if isUnauthenticated { credsProvider = nil } From 2191e1898d291de777a99fb1267e7126745a53de Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Sat, 19 Apr 2025 20:11:09 +0800 Subject: [PATCH 3/7] Use no credentials for GetBucketRegion Reference: https://github.com/trufflesecurity/trufflehog/pull/4069#issuecomment-2816678647 Signed-off-by: Eng Zer Jun --- pkg/sources/s3/s3.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index f6fd1013485d..f64a2340d428 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -413,7 +413,12 @@ func (s *Source) getRegionalClientForBucket( role string, bucket string, ) (*s3.Client, error) { - region, err := s3manager.GetBucketRegion(ctx, defaultRegionClient, bucket) + region, err := s3manager.GetBucketRegion(ctx, defaultRegionClient, bucket, func(options *s3.Options) { + // GetBucketRegion can return the region through the X-Amz-Bucket-Region header with no credentials. + // However, when we provide an invalid credentials, GetBucketRegion return an error. + // See full explanation in https://github.com/trufflesecurity/trufflehog/pull/4069#issuecomment-2816678647. + options.Credentials = nil + }) if err != nil { return nil, fmt.Errorf("could not get s3 region for bucket: %s: %w", bucket, err) } From 8cce87ccb7d85f437d55f5edb3dffce0890f7a20 Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Fri, 25 Apr 2025 22:18:31 +0800 Subject: [PATCH 4/7] Fix infinite list objects loop when first page is error Signed-off-by: Eng Zer Jun --- pkg/sources/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index f64a2340d428..9a58bb5a8b94 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -371,7 +371,7 @@ func (s *Source) scanBuckets( // and therefore not an error. ctx.Logger().V(3).Info("could not list objects in bucket", "err", err) } - continue + break } pageMetadata := pageMetadata{ bucket: bucket, From bf28129af881191b95fbf81764a5622730883c0d Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Wed, 30 Apr 2025 23:14:32 +0800 Subject: [PATCH 5/7] Update go.mod Signed-off-by: Eng Zer Jun --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 587d130991f5..cbeda70c7cae 100644 --- a/go.mod +++ b/go.mod @@ -22,8 +22,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.73 - github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 github.com/aws/aws-sdk-go-v2/service/sns v1.34.4 github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 github.com/aws/smithy-go v1.22.3 @@ -158,7 +158,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect diff --git a/go.sum b/go.sum index 63a2c5297daf..925f098ee76c 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9 github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.73 h1:I91eIdOJMVK9oNiH2jvhp/AxMW+Gff8Rb5VjVHMhcJU= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.73/go.mod h1:vq7/m7dahFXcdzWVOvvjasDI9RcsD3RsTfHmDundJYg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74 h1:+1lc5oMFFHlVBclPXQf/POqlvdpBzjLaN2c3ujDCcZw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74/go.mod h1:EiskBoFr4SpYnFIbw8UM7DP7CacQXDHEmJqLI1xpRFI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -138,14 +138,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcu github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 h1:tWUG+4wZqdMl/znThEk9tcCy8tTMxq8dW0JTgamohrY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k= github.com/aws/aws-sdk-go-v2/service/sns v1.34.4 h1:ihddI5wufQQCJiujUgAvWRqZcfDmSKIfXlAuX7T95cg= github.com/aws/aws-sdk-go-v2/service/sns v1.34.4/go.mod h1:PJtxxMdj747j8DeZENRTTYAz/lx/pADn/U0k7YNNiUY= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= From f375a014a59ec88d61a553da22ff838ed805f1df Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Tue, 6 May 2025 09:24:22 +0800 Subject: [PATCH 6/7] Update s3/manager to v1.17.75 Fixes https://github.com/aws/aws-sdk-go-v2/issues/3077. Signed-off-by: Eng Zer Jun --- go.mod | 2 +- go.sum | 4 ++-- pkg/sources/s3/s3.go | 7 +------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 6696ac4b1e80..c24c57b21358 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 github.com/aws/aws-sdk-go-v2/service/sns v1.34.4 github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 diff --git a/go.sum b/go.sum index 925f098ee76c..fb6ecff5a3e1 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9 github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74 h1:+1lc5oMFFHlVBclPXQf/POqlvdpBzjLaN2c3ujDCcZw= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74/go.mod h1:EiskBoFr4SpYnFIbw8UM7DP7CacQXDHEmJqLI1xpRFI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index 9a58bb5a8b94..6799dfd19a36 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -413,12 +413,7 @@ func (s *Source) getRegionalClientForBucket( role string, bucket string, ) (*s3.Client, error) { - region, err := s3manager.GetBucketRegion(ctx, defaultRegionClient, bucket, func(options *s3.Options) { - // GetBucketRegion can return the region through the X-Amz-Bucket-Region header with no credentials. - // However, when we provide an invalid credentials, GetBucketRegion return an error. - // See full explanation in https://github.com/trufflesecurity/trufflehog/pull/4069#issuecomment-2816678647. - options.Credentials = nil - }) + region, err := s3manager.GetBucketRegion(ctx, defaultRegionClient, bucket) if err != nil { return nil, fmt.Errorf("could not get s3 region for bucket: %s: %w", bucket, err) } From 49a8209d9dfcc60880c6c44c4cb104cc6ec73385 Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Tue, 6 May 2025 22:44:26 +0800 Subject: [PATCH 7/7] Preserve V1 behavior for sourcespb.S3_Unauthenticated case Reference: https://github.com/trufflesecurity/trufflehog/pull/4069#discussion_r2075423056 Signed-off-by: Eng Zer Jun --- pkg/sources/s3/s3.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index 6799dfd19a36..5d20675cb889 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -157,14 +157,6 @@ func (s *Source) newClient(ctx context.Context, region, roleArn string) (*s3.Cli } if roleArn != "" { - // A valid credentials is required to assume IAM role. aws.AnonymousCredentials is not a valid credentials. - // If the value of credsProvider is aws.AnonymousCredentials{} from the above switch-case, - // we will need to set credsProvider to nil to use SDK's default credential chain. - _, isUnauthenticated := credsProvider.(aws.AnonymousCredentials) - if isUnauthenticated { - credsProvider = nil - } - // The config loaded here will be used to retrieve and refresh temporary credentials from AssumeRole cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credsProvider)) if err != nil {