From 7c8774d6dd45f5f1fb504824fdd98b5c66d2c976 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Mon, 10 Mar 2025 10:05:58 -0400 Subject: [PATCH 01/63] almost all passing --- go.sum | 210 ++++++++++ .../namespacemutation/webhookhandler_test.go | 1 + .../workloadmutation/webhookhandler_test.go | 5 +- main.go | 34 +- pkg/instrumentation/annotationmutator.go | 7 +- pkg/instrumentation/annotationtype.go | 4 + pkg/instrumentation/auto/annotation.go | 28 +- pkg/instrumentation/auto/annotation_test.go | 6 + pkg/instrumentation/auto/auto_monitor.go | 115 ++++++ .../auto/auto_monitor_config.go | 19 + pkg/instrumentation/auto/auto_monitor_test.go | 380 ++++++++++++++++++ 11 files changed, 796 insertions(+), 13 deletions(-) create mode 100644 pkg/instrumentation/auto/auto_monitor.go create mode 100644 pkg/instrumentation/auto/auto_monitor_config.go create mode 100644 pkg/instrumentation/auto/auto_monitor_test.go diff --git a/go.sum b/go.sum index da0ba5a13..f1a3020df 100644 --- a/go.sum +++ b/go.sum @@ -13,27 +13,142 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= +cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= +cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= +cloud.google.com/go/aiplatform v1.50.0/go.mod h1:IRc2b8XAMTa9ZmfJV1BCCQbieWWvDnP1A8znyz5N7y4= +cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= +cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= +cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= +cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= +cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= +cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= +cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= +cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= +cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= +cloud.google.com/go/baremetalsolution v1.2.0/go.mod h1:68wi9AwPYkEWIUT4SvSGS9UJwKzNpshjHsH4lzk8iOw= +cloud.google.com/go/batch v1.4.1/go.mod h1:KdBmDD61K0ovcxoRHGrN6GmOBWeAOyCgKD0Mugx4Fkk= +cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.55.0/go.mod h1:9Y5I3PN9kQWuid6183JFhOGOW3GcirA5LpsKCUn+2ec= +cloud.google.com/go/billing v1.17.0/go.mod h1:Z9+vZXEq+HwH7bhJkyI4OQcR6TSbeMrjlpEjO2vzY64= +cloud.google.com/go/binaryauthorization v1.7.0/go.mod h1:Zn+S6QqTMn6odcMU1zDZCJxPjU2tZPV1oDl45lWY154= +cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= +cloud.google.com/go/channel v1.17.0/go.mod h1:RpbhJsGi/lXWAUM1eF4IbQGbsfVlg2o8Iiy2/YLfVT0= +cloud.google.com/go/cloudbuild v1.14.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/clouddms v1.7.0/go.mod h1:MW1dC6SOtI/tPNCciTsXtsGNEM0i0OccykPvv3hiYeM= +cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= +cloud.google.com/go/container v1.26.0/go.mod h1:YJCmRet6+6jnYYRS000T6k0D0xUXQgBSaJ7VwI8FBj4= +cloud.google.com/go/containeranalysis v0.11.0/go.mod h1:4n2e99ZwpGxpNcz+YsFT1dfOHPQFGcAC8FN2M2/ne/U= +cloud.google.com/go/datacatalog v1.17.1/go.mod h1:nCSYFHgtxh2MiEktWIz71s/X+7ds/UT9kp0PC7waCzE= +cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= +cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= +cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= +cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= +cloud.google.com/go/dataplex v1.9.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataproc/v2 v2.2.0/go.mod h1:lZR7AQtwZPvmINx5J87DSOOpTfof9LVZju6/Qo4lmcY= +cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.14.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= +cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= +cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= +cloud.google.com/go/dialogflow v1.43.0/go.mod h1:pDUJdi4elL0MFmt1REMvFkdsUTYSHq+rTCS8wg0S3+M= +cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= +cloud.google.com/go/documentai v1.22.1/go.mod h1:LKs22aDHbJv7ufXuPypzRO7rG3ALLJxzdCXDPutw4Qc= +cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= +cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= +cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= +cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= +cloud.google.com/go/firestore v1.13.0/go.mod h1:QojqqOh8IntInDUSTAh0c8ZsPYAr68Ma8c5DWOy8xb8= +cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= +cloud.google.com/go/gkebackup v1.3.1/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= +cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= +cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= +cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= +cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iap v1.9.0/go.mod h1:01OFxd1R+NFrg78S+hoPV5PxEzv22HXaNqUUlmNHFuY= +cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= +cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= +cloud.google.com/go/kms v1.15.2/go.mod h1:3hopT4+7ooWRCjc2DxgnpESFxhIraaI2IpAVUEhbT/w= +cloud.google.com/go/language v1.11.0/go.mod h1:uDx+pFDdAKTY8ehpWbiXyQdz8tDSYLJbQcXsCkjYyvQ= +cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= +cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= +cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= +cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= +cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= +cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= +cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= +cloud.google.com/go/monitoring v1.16.0/go.mod h1:Ptp15HgAyM1fNICAojDMoNc/wUmn67mLHQfyqbw+poY= +cloud.google.com/go/networkconnectivity v1.13.0/go.mod h1:SAnGPes88pl7QRLUen2HmcBSE9AowVAcdug8c0RSBFk= +cloud.google.com/go/networkmanagement v1.9.0/go.mod h1:UTUaEU9YwbCAhhz3jEOHr+2/K/MrBk2XxOLS89LQzFw= +cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= +cloud.google.com/go/notebooks v1.10.0/go.mod h1:SOPYMZnttHxqot0SGSFSkRrwE29eqnKPBJFqgWmiK2k= +cloud.google.com/go/optimization v1.5.0/go.mod h1:evo1OvTxeBRBu6ydPlrIRizKY/LJKo/drDMMRKqGEUU= +cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= +cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= +cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= +cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= +cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= +cloud.google.com/go/policytroubleshooter v1.9.0/go.mod h1:+E2Lga7TycpeSTj2FsH4oXxTnrbHJGRlKhVZBLGgU64= +cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= +cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= +cloud.google.com/go/recommender v1.11.0/go.mod h1:kPiRQhPyTJ9kyXPCG6u/dlPLbYfFlkwHNRwdzPVAoII= +cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= +cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= +cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= +cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= +cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= +cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= +cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= +cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= +cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= +cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= +cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= +cloud.google.com/go/spanner v1.49.0/go.mod h1:eGj9mQGK8+hkgSVbHNQ06pQ4oS+cyc4tXXd6Dif1KoM= +cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= +cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= +cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= +cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= +cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= +cloud.google.com/go/translate v1.9.0/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/video v1.20.0/go.mod h1:U3G3FTnsvAGqglq9LxgqzOiBc/Nt8zis8S+850N2DUM= +cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= +cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= +cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= +cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= +cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= +cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= +cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= +cloud.google.com/go/workflows v1.12.0/go.mod h1:PYhSk2b6DhZ508tj8HXKaBh+OFe+xdl0dHF/tJdzPQM= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -61,10 +176,13 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -72,12 +190,14 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= @@ -85,6 +205,7 @@ github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4= github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -92,6 +213,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/brancz/kube-rbac-proxy v0.15.0/go.mod h1:ne9y1bsoAwZP4welF93ZcI3j2m/qX7gZRCYkh3fd0VY= github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU= github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= @@ -101,9 +223,11 @@ github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -115,16 +239,20 @@ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJ github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/digitalocean/godo v1.104.1 h1:SZNxjAsskM/su0YW9P8Wx3gU0W1Z13b6tZlYNpl5BnA= @@ -133,12 +261,14 @@ github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg= github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/efficientgo/core v1.0.0-rc.2 h1:7j62qHLnrZqO3V3UA0AqOGd5d5aXV3AX6m/NZBHp78I= @@ -157,6 +287,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -166,6 +298,7 @@ github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBd github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -174,11 +307,13 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= @@ -248,6 +383,7 @@ github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSM github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= @@ -278,12 +414,15 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -321,6 +460,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -337,6 +477,7 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -356,6 +497,7 @@ github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0Z github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -372,7 +514,11 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/consul/api v1.25.1 h1:CqrdhYzc8XZuPnhIYZWH45toM0LB9ZeYr/gvpLVI3PE= @@ -418,6 +564,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= @@ -428,6 +575,7 @@ github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hetznercloud/hcloud-go/v2 v2.4.0 h1:MqlAE+w125PLvJRCpAJmEwrIxoVdUdOyuFUhE/Ukbok= github.com/hetznercloud/hcloud-go/v2 v2.4.0/go.mod h1:l7fA5xsncFBzQTyw29/dw5Yr88yEGKKdc6BHf24ONS0= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -438,11 +586,13 @@ github.com/ionos-cloud/sdk-go/v6 v6.1.9 h1:Iq3VIXzeEbc8EbButuACgfLMiY5TPVWUPNrF+ github.com/ionos-cloud/sdk-go/v6 v6.1.9/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k= github.com/jarcoal/httpmock v1.3.0 h1:2RJ8GP0IIaWwcC9Fp2BmVi8Kog3v2Hn7VXM3fTd+nuc= github.com/jarcoal/httpmock v1.3.0/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= @@ -492,8 +642,11 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/linode/linodego v1.23.0 h1:s0ReCZtuN9Z1IoUN9w1RLeYO1dMZUGPwOQ/IBFsBHtU= github.com/linode/linodego v1.23.0/go.mod h1:0U7wj/UQOqBNbKv1FYTXiBUXueR8DY4HvIotwE0ENgg= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -517,6 +670,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a h1:0usWxe5SGXKQovz3p+BiQ81Jy845xSMu2CWKuXsXuUM= @@ -525,13 +680,17 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -539,6 +698,7 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -548,7 +708,9 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -556,6 +718,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= @@ -581,6 +744,7 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -623,8 +787,10 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI= github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= +github.com/prometheus/exporter-toolkit v0.10.0/go.mod h1:+sVFzuvV5JDyw+Ih6p3zFxZNVnKQa3x5qPmDSiPu4ZY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= @@ -639,6 +805,7 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ= @@ -647,6 +814,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shoenig/test v0.6.6 h1:Oe8TPH9wAbv++YPNDKJWUnI8Q4PPWCx3UbOfH+FxiMU= github.com/shoenig/test v0.6.6/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1qZoYLZzLnBw+QkPP9WZnjlSWihhxAJC1+/M= +github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -654,12 +823,15 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -679,7 +851,9 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/thanos-io/thanos v0.32.5-0.20231124114724-023faa2d67a3/go.mod h1:DzjNla690uSQazyHrLGK6FeOn3VfV2ReA57Pi7SgZ+8= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= @@ -694,6 +868,9 @@ github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3k github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -701,6 +878,14 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= +go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= +go.etcd.io/etcd/client/v2 v2.305.10/go.mod h1:m3CKZi69HzilhVqtPDcjhSGp+kA1OmbNn0qamH80xjA= +go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= +go.etcd.io/etcd/pkg/v3 v3.5.10/go.mod h1:TKTuCKKcF1zxmfKWDkfz5qqYaE3JncKKZPFf8c1nFUs= +go.etcd.io/etcd/raft/v3 v3.5.10/go.mod h1:odD6kr8XQXTy9oQnyMPBOr0TVe+gT0neQhElQ6jbGRc= +go.etcd.io/etcd/server/v3 v3.5.10/go.mod h1:gBplPHfs6YI0L+RpGkTQO7buDbHv5HJGG/Bst0/zIPo= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= @@ -717,12 +902,17 @@ go.opentelemetry.io/collector/confmap v0.101.0 h1:pGXZRBKnZqys1HgNECGSi8Pec5RBGa go.opentelemetry.io/collector/confmap v0.101.0/go.mod h1:BWKPIpYeUzSG6ZgCJMjF7xsLvyrvJCfYURl57E5vhiQ= go.opentelemetry.io/collector/featuregate v0.77.0 h1:m1/IzaXoQh6SgF6CM80vrBOCf5zSJ2GVISfA27fYzGU= go.opentelemetry.io/collector/featuregate v0.77.0/go.mod h1:/kVAsGUCyJXIDSgHftCN63QiwAEVHRLX2Kh/S+dqgHY= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0016/go.mod h1:OdN0alYOlYhHXu6BDlGehrZWgtBuiDsz/rlNeJeXiNg= +go.opentelemetry.io/collector/semconv v0.87.0/go.mod h1:j/8THcqVxFna1FpvA2zYIsUperEtOaRaqoLYIN4doWw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= @@ -733,9 +923,11 @@ go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8 go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= @@ -786,6 +978,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -925,6 +1118,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1003,6 +1197,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1064,6 +1259,7 @@ google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a h1:myvhA4is3vrit1a6NZCWBIwN0kNEnX21DJOJX/NvIfI= google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:SUBoKXbI1Efip18FClrQVGjWcyd0QZd8KkvdP34t7ww= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:itlFWGBbEyD32PUeJsTG8h8Wz7iJXfVK4gt1EJ+pAG0= google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1107,6 +1303,8 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/telebot.v3 v3.1.3/go.mod h1:GJKwwWqp9nSkIVN51eRKU78aB5f5OnQuWdwiIZfPbko= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1136,16 +1334,24 @@ k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3 k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= +k8s.io/apiserver v0.29.0/go.mod h1:31n78PsRKPmfpee7/l9NYEv67u6hOL6AfcE761HapDM= +k8s.io/cli-runtime v0.29.0/go.mod h1:VKudXp3X7wR45L+nER85YUzOQIru28HQpXr0mTdeCrk= k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= +k8s.io/code-generator v0.29.0/go.mod h1:5bqIZoCxs2zTRKMWNYqyQWW/bajc+ah4rh0tMY8zdGA= k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= +k8s.io/component-helpers v0.29.0/go.mod h1:j2coxVfmzTOXWSE6sta0MTgNSr572Dcx68F6DD+8fWc= +k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kms v0.29.0/go.mod h1:mB0f9HLxRXeXUfHfn1A7rpwOlzXI1gIWu86z6buNoYA= k8s.io/kube-openapi v0.0.0-20231129212854-f0671cc7e66a h1:ZeIPbyHHqahGIbeyLJJjAUhnxCKqXaDY+n89Ms8szyA= k8s.io/kube-openapi v0.0.0-20231129212854-f0671cc7e66a/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/kubectl v0.29.0 h1:Oqi48gXjikDhrBF67AYuZRTcJV4lg2l42GmvsP7FmYI= k8s.io/kubectl v0.29.0/go.mod h1:0jMjGWIcMIQzmUaMgAzhSELv5WtHo2a8pq67DtviAJs= +k8s.io/metrics v0.29.0/go.mod h1:UCuTT4dC/x/x6ODSk87IWIZQnuAfcwxOjb1gjWJdjMA= k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI= k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= @@ -1153,10 +1359,14 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= +sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3/go.mod h1:/d88dHCvoy7d0AKFT0yytezSGZKjsZBVs9YTkBHSGFk= +sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/internal/webhook/namespacemutation/webhookhandler_test.go b/internal/webhook/namespacemutation/webhookhandler_test.go index 7ea72bd10..03e3a5647 100644 --- a/internal/webhook/namespacemutation/webhookhandler_test.go +++ b/internal/webhook/namespacemutation/webhookhandler_test.go @@ -39,6 +39,7 @@ func TestHandle(t *testing.T) { logr.Logger{}, autoAnnotationConfig, instrumentation.NewTypeSet(instrumentation.TypeJava), + nil, ) h := NewWebhookHandler(decoder, mutators) for _, testCase := range []struct { diff --git a/internal/webhook/workloadmutation/webhookhandler_test.go b/internal/webhook/workloadmutation/webhookhandler_test.go index 35d7d319e..5bdc48926 100644 --- a/internal/webhook/workloadmutation/webhookhandler_test.go +++ b/internal/webhook/workloadmutation/webhookhandler_test.go @@ -6,10 +6,11 @@ package workloadmutation import ( "context" "encoding/json" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/go-logr/logr" "net/http" "testing" - "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" admv1 "k8s.io/api/admission/v1" @@ -21,7 +22,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" ) @@ -128,6 +128,7 @@ func TestHandle(t *testing.T) { logr.Logger{}, autoAnnotationConfig, instrumentation.NewTypeSet(instrumentation.TypeJava), + nil, ) injector := NewWebhookHandler(decoder, mutators) diff --git a/main.go b/main.go index 63b7fccf0..62329cc11 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,8 @@ import ( "encoding/json" "flag" "fmt" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "os" "runtime" "strings" @@ -292,23 +294,43 @@ func main() { if err = json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-annotation config") } else { + // TODO marshal/unmarshal monitor config + monitorConfig := auto.MonitorConfig{} + + supportedLanguages := instrumentation.NewTypeSet( + instrumentation.TypeJava, + instrumentation.TypePython, + instrumentation.TypeDotNet, + instrumentation.TypeNodeJS, + ) + if monitorConfig.Languages == nil { + monitorConfig.Languages = supportedLanguages + } + k8sConfig, err := rest.InClusterConfig() + if err != nil { + panic("TODO: Implement handling for failed clientset creation") + } + + clientSet, err := kubernetes.NewForConfig(k8sConfig) + if err != nil { + // TODO + panic("TODO: Implement handling for failed clientset creation") + } + monitor := auto.NewMonitor(ctx, logger, monitorConfig, clientSet) autoAnnotationMutators := auto.NewAnnotationMutators( mgr.GetClient(), mgr.GetAPIReader(), logger, autoAnnotationConfig, - instrumentation.NewTypeSet( - instrumentation.TypeJava, - instrumentation.TypePython, - instrumentation.TypeDotNet, - instrumentation.TypeNodeJS, - ), + supportedLanguages, + monitor, ) mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ Handler: workloadmutation.NewWebhookHandler(decoder, autoAnnotationMutators)}) mgr.GetWebhookServer().Register("/mutate-v1-namespace", &webhook.Admission{ Handler: namespacemutation.NewWebhookHandler(decoder, autoAnnotationMutators), }) + setupLog.Info("Auto-annotation is enabled") go waitForWebhookServerStart( ctx, diff --git a/pkg/instrumentation/annotationmutator.go b/pkg/instrumentation/annotationmutator.go index 134916072..4c5707b82 100644 --- a/pkg/instrumentation/annotationmutator.go +++ b/pkg/instrumentation/annotationmutator.go @@ -77,7 +77,7 @@ func NewAnnotationMutator(mutations []AnnotationMutation) AnnotationMutator { } // Mutate modifies the object's annotations based on the mutator's mutations. Returns all the mutated annotations. -func (m *AnnotationMutator) Mutate(obj metav1.Object) map[string]string { +func (m AnnotationMutator) Mutate(obj metav1.Object) map[string]string { annotations := obj.GetAnnotations() if annotations == nil { annotations = make(map[string]string) @@ -92,3 +92,8 @@ func (m *AnnotationMutator) Mutate(obj metav1.Object) map[string]string { obj.SetAnnotations(annotations) return allMutatedAnnotations } + +// ObjectAnnotationMutator responsible for getting, mutating, and setting the object's annotations map. Returns all the mutated annotations. +type ObjectAnnotationMutator interface { + Mutate(obj metav1.Object) map[string]string +} diff --git a/pkg/instrumentation/annotationtype.go b/pkg/instrumentation/annotationtype.go index 75ceb01f1..0e4b9246f 100644 --- a/pkg/instrumentation/annotationtype.go +++ b/pkg/instrumentation/annotationtype.go @@ -26,6 +26,10 @@ const ( TypeGo Type = "go" ) +func AllTypes() []Type { + return []Type{TypeJava, TypeNodeJS, TypePython, TypeDotNet, TypeGo} +} + // InjectAnnotationKey maps the instrumentation type to the inject annotation. func InjectAnnotationKey(instType Type) string { switch instType { diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 63c0d82ab..1ade95d04 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -29,6 +29,7 @@ const ( // AnnotationMutators contains functions that can be used to mutate annotations // on all supported objects based on the configured mutators. type AnnotationMutators struct { + monitor *Monitor clientWriter client.Writer clientReader client.Reader logger logr.Logger @@ -108,15 +109,32 @@ func (m *AnnotationMutators) rangeObjectList(ctx context.Context, list client.Ob } } -func (m *AnnotationMutators) mutate(name string, mutators map[string]instrumentation.AnnotationMutator, obj metav1.Object) (map[string]string, bool) { - mutator, ok := mutators[name] - if !ok { - mutator = m.defaultMutator +func (m *AnnotationMutators) mutate(namespacedName string, mutators map[string]instrumentation.AnnotationMutator, obj metav1.Object) (map[string]string, bool) { + // does autoAnnotateAutoInstrumentation or customSelector specify the namespaced name of the k8s object to annotate? + var mutator instrumentation.ObjectAnnotationMutator + mutator, specificMutatorExists := mutators[namespacedName] + + if !specificMutatorExists { + if m.monitor != nil && isWorkload(obj) && m.monitor.ShouldBeMonitored(obj) { + // Is the object is a workload and does a service selects the workload? + mutator = m.monitor + } else { + mutator = &m.defaultMutator + } } + mutatedAnnotations := mutator.Mutate(obj) return mutatedAnnotations, len(mutatedAnnotations) != 0 } +func isWorkload(obj metav1.Object) bool { + switch obj.(type) { + case *appsv1.Deployment, *appsv1.DaemonSet, *appsv1.StatefulSet: + return true + } + return false +} + func namespacedName(obj metav1.Object) string { return fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName()) } @@ -130,12 +148,14 @@ func NewAnnotationMutators( logger logr.Logger, cfg AnnotationConfig, typeSet instrumentation.TypeSet, + monitor *Monitor, ) *AnnotationMutators { builder := newMutatorBuilder(typeSet) return &AnnotationMutators{ clientWriter: clientWriter, clientReader: clientReader, logger: logger, + monitor: monitor, namespaceMutators: builder.buildMutators(getResources(cfg, typeSet, getNamespaces)), deploymentMutators: builder.buildMutators(getResources(cfg, typeSet, getDeployments)), daemonSetMutators: builder.buildMutators(getResources(cfg, typeSet, getDaemonSets)), diff --git a/pkg/instrumentation/auto/annotation_test.go b/pkg/instrumentation/auto/annotation_test.go index 72ac13ee5..ee830c195 100644 --- a/pkg/instrumentation/auto/annotation_test.go +++ b/pkg/instrumentation/auto/annotation_test.go @@ -116,6 +116,7 @@ func TestAnnotationMutators_Namespaces(t *testing.T) { logr.Logger{}, testCase.cfg, testCase.typeSet, + nil, ) mutators.MutateAndPatchAll(ctx) gotNamespaces := &corev1.NamespaceList{} @@ -200,6 +201,7 @@ func TestAnnotationMutators_Namespaces_Restart(t *testing.T) { logr.Logger{}, cfg, instrumentation.NewTypeSet(instrumentation.TypeJava), + nil, ) mutators.MutateAndPatchAll(context.Background()) ctx := context.Background() @@ -291,6 +293,7 @@ func TestAnnotationMutators_Deployments(t *testing.T) { logr.Logger{}, testCase.cfg, testCase.typeSet, + nil, ) mutators.MutateAndPatchAll(ctx) gotDeployments := &appsv1.DeploymentList{} @@ -359,6 +362,7 @@ func TestAnnotationMutators_DaemonSets(t *testing.T) { logr.Logger{}, testCase.cfg, testCase.typeSet, + nil, ) mutators.MutateAndPatchAll(ctx) gotDaemonSets := &appsv1.DaemonSetList{} @@ -427,6 +431,7 @@ func TestAnnotationMutators_StatefulSets(t *testing.T) { logr.Logger{}, testCase.cfg, testCase.typeSet, + nil, ) mutators.MutateAndPatchAll(ctx) gotStatefulSets := &appsv1.StatefulSetList{} @@ -493,6 +498,7 @@ func TestAnnotationMutators_ClientErrors(t *testing.T) { logr.Logger{}, cfg, instrumentation.NewTypeSet(instrumentation.TypeJava), + nil, ) mutators.MutateAndPatchAll(context.Background()) errClient.AssertCalled(t, "List", mock.Anything, mock.Anything, mock.Anything) diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go new file mode 100644 index 000000000..c773875ca --- /dev/null +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -0,0 +1,115 @@ +package auto + +import ( + "context" + "fmt" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/strings/slices" + "os" + "time" +) + +type Monitor struct { + serviceInformer cache.SharedIndexInformer + ctx context.Context + logger logr.Logger + config MonitorConfig +} + +func NewMonitor(ctx context.Context, logger logr.Logger, config MonitorConfig, k8sInterface kubernetes.Interface) *Monitor { + + factory := informers.NewSharedInformerFactory(k8sInterface, 10*time.Minute) + + serviceInformer := factory.Core().V1().Services().Informer() + factory.Start(ctx.Done()) // runs in background + synced := factory.WaitForCacheSync(ctx.Done()) + for v, ok := range synced { + if !ok { + _, _ = fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) + panic("TODO: handle bad cache sync") + } + } + + return &Monitor{serviceInformer: serviceInformer, ctx: ctx, logger: logger, config: config} +} + +// ShouldBeMonitored returns whether obj is selected by either custom selector or auto monitor +func (m Monitor) ShouldBeMonitored(obj metav1.Object) bool { + // TODO: check if in custom selector + // note: custom selector does not respect MonitorAllServices + if m.customSelected(obj) { + return true + } + + if !m.config.MonitorAllServices { + return false + } + + objectLabels := labels.Set(obj.GetLabels()) + // if object is not workload, return err + for _, informerObj := range m.serviceInformer.GetStore().List() { + service := informerObj.(*corev1.Service) + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + + m.logger.V(2).Info("AutoMonitor: testing serviceSelector", "serviceSelector", serviceSelector.String(), "objectLabels", objectLabels.String()) + if serviceSelector.Matches(objectLabels) { + m.logger.V(2).Info("AutoMonitor: matched!", "service", service, "object", obj.GetName()) + return true + } + + // remove if none matched, not in custom selector + } + return false +} + +// Mutate adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector +func (m Monitor) Mutate(obj metav1.Object) map[string]string { + // TODO: only create if automonitor enabled + annotations := obj.GetAnnotations() + allMutatedAnnotations := make(map[string]string) + for _, language := range instrumentation.AllTypes() { + insertMutation, removeMutation := buildMutations(language) + var mutatedAnnotations map[string]string + if _, ok := m.config.Languages[language]; ok { + mutatedAnnotations = insertMutation.Mutate(annotations) + + } else { + mutatedAnnotations = removeMutation.Mutate(annotations) + } + for k, v := range mutatedAnnotations { + allMutatedAnnotations[k] = v + } + } + obj.SetAnnotations(annotations) + return allMutatedAnnotations +} + +func (m Monitor) customSelected(obj metav1.Object) bool { + objName := namespacedName(obj) + var resourceList []string + + switch obj.(type) { + case *appsv1.Deployment: + for _, t := range instrumentation.AllTypes() { + resourceList = append(resourceList, m.config.CustomSelector.getResources(t).Deployments...) + } + case *appsv1.StatefulSet: + for _, t := range instrumentation.AllTypes() { + resourceList = append(resourceList, m.config.CustomSelector.getResources(t).StatefulSets...) + } + case *appsv1.DaemonSet: + for _, t := range instrumentation.AllTypes() { + resourceList = append(resourceList, m.config.CustomSelector.getResources(t).DaemonSets...) + } + } + + return slices.Contains(resourceList, objName) +} diff --git a/pkg/instrumentation/auto/auto_monitor_config.go b/pkg/instrumentation/auto/auto_monitor_config.go new file mode 100644 index 000000000..68cc9df4d --- /dev/null +++ b/pkg/instrumentation/auto/auto_monitor_config.go @@ -0,0 +1,19 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package auto + +import "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + +// AnnotationConfig details the resources that have enabled +// auto-annotation for each instrumentation type. +type MonitorConfig struct { + MonitorAllServices bool `json:"monitorAllServices"` + Languages instrumentation.TypeSet `json:"languages"` + AutoRestart bool `json:"autoRestart"` + Exclude struct { + Namespaces []string `json:"namespaces"` + Services []string `json:"services"` + } `json:"exclude"` + CustomSelector AnnotationConfig `json:"customSelector"` +} diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go new file mode 100644 index 000000000..862b63c11 --- /dev/null +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -0,0 +1,380 @@ +package auto + +import ( + "context" + "fmt" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "testing" +) + +var logger = logf.Log.WithName("auto_monitor_tests") + +func TestMonitor_Selected(t *testing.T) { + logger.Info("Starting testmonitor tests") + + allTypes := instrumentation.NewTypeSet(instrumentation.AllTypes()...) + tests := []struct { + name string + service corev1.Service + workload metav1.Object + config MonitorConfig + shouldMatch bool + }{ + { + name: "Should match Deployment with exact label match", + service: newTestService("svc-1", map[string]string{"app": "test"}), + workload: newTestDeployment("deploy-1", map[string]string{ + "app": "test", + }), + config: MonitorConfig{ + MonitorAllServices: true, + Languages: allTypes, + }, + shouldMatch: true, + }, + { + name: "Should not match Deployment with no labels", + service: newTestService("svc-2", map[string]string{"app": "test"}), + workload: newTestDeployment("deploy-2", map[string]string{}), + config: MonitorConfig{ + MonitorAllServices: true, + Languages: allTypes, + }, + shouldMatch: false, + }, + { + name: "Should not match when MonitorAllServices is false", + service: newTestService("svc-3", map[string]string{"app": "test"}), + workload: newTestDeployment("deploy-3", map[string]string{ + "app": "test", + }), + config: MonitorConfig{ + MonitorAllServices: false, + Languages: allTypes, + }, + shouldMatch: false, + }, + { + name: "Should match Deployment with multiple matching labels", + service: newTestService("svc-4", map[string]string{"app": "test", "env": "prod"}), + workload: newTestDeployment("deploy-4", map[string]string{ + "app": "test", + "env": "prod", + "extra": "label", + }), + config: MonitorConfig{ + MonitorAllServices: true, + Languages: allTypes, + }, + shouldMatch: true, + }, + { + name: "Should not match when service selector is empty", + service: newTestService("svc-5", map[string]string{}), + workload: newTestDeployment("deploy-5", map[string]string{ + "app": "test", + }), + config: MonitorConfig{ + MonitorAllServices: true, + Languages: allTypes, + }, + shouldMatch: false, + }, + { + name: "Should match StatefulSet with partial label match", + service: newTestService("svc-6", map[string]string{"app": "test"}), + workload: newTestStatefulSet("sts-1", map[string]string{ + "app": "test", + "other": "value", + }), + config: MonitorConfig{ + MonitorAllServices: true, + Languages: allTypes, + }, + shouldMatch: true, + }, + { + name: "Should not match when languages are empty", + service: newTestService("svc-7", map[string]string{"app": "test"}), + workload: newTestDeployment("deploy-7", map[string]string{ + "app": "test", + }), + config: MonitorConfig{ + MonitorAllServices: true, + }, + shouldMatch: false, + }, + { + name: "Should match DaemonSet with exact label match", + service: newTestService("svc-8", map[string]string{"app": "test"}), + workload: newTestDaemonSet("ds-1", map[string]string{ + "app": "test", + }), + config: MonitorConfig{ + MonitorAllServices: true, + Languages: allTypes, + }, + shouldMatch: true, + }, + { + name: "Should not match DaemonSet with mismatched labels", + service: newTestService("svc-9", map[string]string{"app": "test"}), + workload: newTestDaemonSet("ds-2", map[string]string{ + "app": "different", + }), + config: MonitorConfig{ + MonitorAllServices: true, + Languages: allTypes, + }, + shouldMatch: false, + }, + { + name: "Should match StatefulSet with exact label match", + service: newTestService("svc-10", map[string]string{"app": "test"}), + workload: newTestStatefulSet("sts-2", map[string]string{ + "app": "test", + }), + config: MonitorConfig{ + MonitorAllServices: true, + Languages: allTypes, + }, + shouldMatch: true, + }, + { + name: "Should not match StatefulSet with mismatched labels", + service: newTestService("svc-11", map[string]string{"app": "test"}), + workload: newTestStatefulSet("sts-3", map[string]string{ + "app": "different", + }), + config: MonitorConfig{ + MonitorAllServices: true, + Languages: allTypes, + }, + shouldMatch: false, + }, + { + name: "Should match Deployment in custom selector regardless of MonitorAllServices", + service: newTestService("svc-12", map[string]string{"app": "different"}), + workload: newTestDeployment("custom-deploy-1", map[string]string{ + "app": "different", + }), + config: MonitorConfig{ + MonitorAllServices: false, + Languages: allTypes, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Deployments: []string{"default/custom-deploy-1"}, + }, + }, + }, + shouldMatch: true, + }, + { + name: "Should match StatefulSet in custom selector", + service: newTestService("svc-13", map[string]string{"app": "different"}), + workload: newTestStatefulSet("custom-sts-1", map[string]string{ + "app": "different", + }), + config: MonitorConfig{ + MonitorAllServices: false, + Languages: allTypes, + CustomSelector: AnnotationConfig{ + Python: AnnotationResources{ + StatefulSets: []string{"default/custom-sts-1"}, + }, + }, + }, + shouldMatch: true, + }, + { + name: "Should match DaemonSet in custom selector", + service: newTestService("svc-14", map[string]string{"app": "different"}), + workload: newTestDaemonSet("custom-ds-1", map[string]string{ + "app": "different", + }), + config: MonitorConfig{ + MonitorAllServices: false, + Languages: allTypes, + CustomSelector: AnnotationConfig{ + NodeJS: AnnotationResources{ + DaemonSets: []string{"default/custom-ds-1"}, + }, + }, + }, + shouldMatch: true, + }, + { + name: "Should not match when workload not in custom selector", + service: newTestService("svc-15", map[string]string{"app": "different"}), + workload: newTestDeployment("non-custom-deploy", map[string]string{ + "app": "different", + }), + config: MonitorConfig{ + MonitorAllServices: false, + Languages: allTypes, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Deployments: []string{"default/different-deploy"}, + }, + }, + }, + shouldMatch: false, + }, + { + name: "Should match when workload in custom selector for multiple languages", + service: newTestService("svc-16", map[string]string{"app": "different"}), + workload: newTestDeployment("multi-lang-deploy", map[string]string{ + "app": "different", + }), + config: MonitorConfig{ + MonitorAllServices: false, + Languages: allTypes, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Deployments: []string{"default/multi-lang-deploy"}, + }, + Python: AnnotationResources{ + Deployments: []string{"default/multi-lang-deploy"}, + }, + }, + }, + shouldMatch: true, + }, + { + name: "Should match when workload in custom selector with multiple resource types", + service: newTestService("svc-17", map[string]string{"app": "different"}), + workload: newTestDeployment("mixed-resources-deploy", map[string]string{ + "app": "different", + }), + config: MonitorConfig{ + MonitorAllServices: false, + Languages: allTypes, + CustomSelector: AnnotationConfig{ + DotNet: AnnotationResources{ + Deployments: []string{"default/mixed-resources-deploy"}, + DaemonSets: []string{"default/some-ds"}, + StatefulSets: []string{"default/some-sts"}, + }, + }, + }, + shouldMatch: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + clientSet := fake.NewSimpleClientset() + + // Create service + _, err := clientSet.CoreV1().Services("default").Create(ctx, &tt.service, metav1.CreateOptions{}) + require.NoError(t, err) + + // Create workload + err = createWorkload(ctx, clientSet, tt.workload) + require.NoError(t, err) + + m := NewMonitor(ctx, logger, tt.config, clientSet) + matched := m.ShouldBeMonitored(tt.workload) + + assert.Equal(t, tt.shouldMatch, matched, + "Expected workload matching to be %v but got %v", tt.shouldMatch, matched) + }) + } +} + +// Helper functions to create test resources +func newTestService(name string, selector map[string]string) corev1.Service { + return corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Selector: selector, + }, + } +} + +func newTestDeployment(name string, labels map[string]string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + }, + }, + } +} + +func newTestDaemonSet(name string, labels map[string]string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: labels, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + }, + }, + } +} + +func newTestStatefulSet(name string, labels map[string]string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + }, + }, + } +} + +func createWorkload(ctx context.Context, clientSet *fake.Clientset, workload metav1.Object) error { + switch x := workload.(type) { + case *appsv1.Deployment: + _, err := clientSet.AppsV1().Deployments(x.GetNamespace()).Create(ctx, x, metav1.CreateOptions{}) + return err + case *appsv1.DaemonSet: + _, err := clientSet.AppsV1().DaemonSets(x.GetNamespace()).Create(ctx, x, metav1.CreateOptions{}) + return err + case *appsv1.StatefulSet: + _, err := clientSet.AppsV1().StatefulSets(x.GetNamespace()).Create(ctx, x, metav1.CreateOptions{}) + return err + default: + return fmt.Errorf("unsupported workload type: %T", workload) + } +} From 95b11c2063ce20454c33319b07ad64caa6013747 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Mon, 10 Mar 2025 10:23:44 -0400 Subject: [PATCH 02/63] passing --- pkg/instrumentation/auto/auto_monitor.go | 3 +++ pkg/instrumentation/auto/auto_monitor_test.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index c773875ca..6817b1d6b 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -57,6 +57,9 @@ func (m Monitor) ShouldBeMonitored(obj metav1.Object) bool { // if object is not workload, return err for _, informerObj := range m.serviceInformer.GetStore().List() { service := informerObj.(*corev1.Service) + if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 { + continue + } serviceSelector := labels.SelectorFromSet(service.Spec.Selector) m.logger.V(2).Info("AutoMonitor: testing serviceSelector", "serviceSelector", serviceSelector.String(), "objectLabels", objectLabels.String()) diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index 862b63c11..d71209953 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -101,7 +101,7 @@ func TestMonitor_Selected(t *testing.T) { shouldMatch: true, }, { - name: "Should not match when languages are empty", + name: "Should match when languages are empty", service: newTestService("svc-7", map[string]string{"app": "test"}), workload: newTestDeployment("deploy-7", map[string]string{ "app": "test", @@ -109,7 +109,7 @@ func TestMonitor_Selected(t *testing.T) { config: MonitorConfig{ MonitorAllServices: true, }, - shouldMatch: false, + shouldMatch: true, }, { name: "Should match DaemonSet with exact label match", From 2a6d9dfbc9762b682bdc23c1118336fff623a799 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Mon, 10 Mar 2025 17:14:43 -0400 Subject: [PATCH 03/63] check pod template labels instead of workload labels --- README.md | 2 +- main.go | 8 ++++-- pkg/instrumentation/auto/annotation.go | 1 - pkg/instrumentation/auto/auto_monitor.go | 26 +++++++++++++++++-- pkg/instrumentation/auto/auto_monitor_test.go | 12 --------- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 706926de2..d658cb0a3 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ kubectl get all ``` kubectl logs amazon-cloudwatch-agent-operator-controller-manager-66f67f47f78 -``` +```k You should see logs that look similar to below diff --git a/main.go b/main.go index 62329cc11..4f9bf23ee 100644 --- a/main.go +++ b/main.go @@ -295,7 +295,11 @@ func main() { setupLog.Error(err, "Unable to unmarshal auto-annotation config") } else { // TODO marshal/unmarshal monitor config - monitorConfig := auto.MonitorConfig{} + setupLog.Info("Test!") + monitorConfig := auto.MonitorConfig{ + // TODO: remove + MonitorAllServices: true, + } supportedLanguages := instrumentation.NewTypeSet( instrumentation.TypeJava, @@ -316,7 +320,7 @@ func main() { // TODO panic("TODO: Implement handling for failed clientset creation") } - monitor := auto.NewMonitor(ctx, logger, monitorConfig, clientSet) + monitor := auto.NewMonitor(ctx, setupLog, monitorConfig, clientSet) autoAnnotationMutators := auto.NewAnnotationMutators( mgr.GetClient(), mgr.GetAPIReader(), diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 1ade95d04..8717d9154 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -113,7 +113,6 @@ func (m *AnnotationMutators) mutate(namespacedName string, mutators map[string]i // does autoAnnotateAutoInstrumentation or customSelector specify the namespaced name of the k8s object to annotate? var mutator instrumentation.ObjectAnnotationMutator mutator, specificMutatorExists := mutators[namespacedName] - if !specificMutatorExists { if m.monitor != nil && isWorkload(obj) && m.monitor.ShouldBeMonitored(obj) { // Is the object is a workload and does a service selects the workload? diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index 6817b1d6b..b646b5e18 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -25,6 +25,7 @@ type Monitor struct { } func NewMonitor(ctx context.Context, logger logr.Logger, config MonitorConfig, k8sInterface kubernetes.Interface) *Monitor { + logger.Info("AutoMonitor starting...") factory := informers.NewSharedInformerFactory(k8sInterface, 10*time.Minute) @@ -37,12 +38,13 @@ func NewMonitor(ctx context.Context, logger logr.Logger, config MonitorConfig, k panic("TODO: handle bad cache sync") } } - + logger.Info("AutoMonitor enabled!") return &Monitor{serviceInformer: serviceInformer, ctx: ctx, logger: logger, config: config} } // ShouldBeMonitored returns whether obj is selected by either custom selector or auto monitor func (m Monitor) ShouldBeMonitored(obj metav1.Object) bool { + m.logger.Info("AutoMonitor shouldBeMonitored") // TODO: check if in custom selector // note: custom selector does not respect MonitorAllServices if m.customSelected(obj) { @@ -53,8 +55,11 @@ func (m Monitor) ShouldBeMonitored(obj metav1.Object) bool { return false } - objectLabels := labels.Set(obj.GetLabels()) + objectLabels := getTemplateSpecLabels(obj) // if object is not workload, return err + m.logger.Info(fmt.Sprintf("AutoMonitor labels: %v", objectLabels)) + m.logger.Info(fmt.Sprintf("AutoMonitor ListKeys: %v", m.serviceInformer.GetStore().ListKeys())) + m.logger.Info(fmt.Sprintf("AutoMonitor List: %v", m.serviceInformer.GetStore().List())) for _, informerObj := range m.serviceInformer.GetStore().List() { service := informerObj.(*corev1.Service) if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 { @@ -73,6 +78,23 @@ func (m Monitor) ShouldBeMonitored(obj metav1.Object) bool { return false } +func getTemplateSpecLabels(obj metav1.Object) labels.Set { + // Check if the object implements the type assertion for PodTemplateSpec + switch t := obj.(type) { + case *appsv1.Deployment: + return labels.Set(t.Spec.Template.Labels) + case *appsv1.StatefulSet: + return labels.Set(t.Spec.Template.Labels) + case *appsv1.DaemonSet: + return labels.Set(t.Spec.Template.Labels) + case *appsv1.ReplicaSet: + return labels.Set(t.Spec.Template.Labels) + default: + // Return empty labels.Set if the object type is not supported + return labels.Set{} + } +} + // Mutate adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector func (m Monitor) Mutate(obj metav1.Object) map[string]string { // TODO: only create if automonitor enabled diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index d71209953..31205cbbd 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -308,12 +308,8 @@ func newTestDeployment(name string, labels map[string]string) *appsv1.Deployment ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "default", - Labels: labels, }, Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -328,12 +324,8 @@ func newTestDaemonSet(name string, labels map[string]string) *appsv1.DaemonSet { ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "default", - Labels: labels, }, Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -348,12 +340,8 @@ func newTestStatefulSet(name string, labels map[string]string) *appsv1.StatefulS ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "default", - Labels: labels, }, Spec: appsv1.StatefulSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, From b50163c217137cb02485eafd82a694d677e0d19c Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Mon, 10 Mar 2025 17:40:19 -0400 Subject: [PATCH 04/63] Provide Monitor functions (except Mutate) workload instead of pod template ObjectMeta --- pkg/instrumentation/auto/annotation.go | 18 ++++++++++-------- pkg/instrumentation/auto/auto_monitor.go | 11 ++++++----- pkg/instrumentation/auto/auto_monitor_test.go | 3 ++- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 8717d9154..f3b047cf1 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -70,13 +70,13 @@ func (m *AnnotationMutators) MutateObject(obj client.Object) (any, bool) { func (m *AnnotationMutators) mutateObject(obj client.Object, _ any) (any, bool) { switch o := obj.(type) { case *corev1.Namespace: - return m.mutate(o.GetName(), m.namespaceMutators, o.GetObjectMeta()) + return m.mutate(o.GetName(), m.namespaceMutators, o.GetObjectMeta(), obj) case *appsv1.Deployment: - return m.mutate(namespacedName(o.GetObjectMeta()), m.deploymentMutators, o.Spec.Template.GetObjectMeta()) + return m.mutate(namespacedName(o.GetObjectMeta()), m.deploymentMutators, o.Spec.Template.GetObjectMeta(), obj) case *appsv1.DaemonSet: - return m.mutate(namespacedName(o.GetObjectMeta()), m.daemonSetMutators, o.Spec.Template.GetObjectMeta()) + return m.mutate(namespacedName(o.GetObjectMeta()), m.daemonSetMutators, o.Spec.Template.GetObjectMeta(), obj) case *appsv1.StatefulSet: - return m.mutate(namespacedName(o.GetObjectMeta()), m.statefulSetMutators, o.Spec.Template.GetObjectMeta()) + return m.mutate(namespacedName(o.GetObjectMeta()), m.statefulSetMutators, o.Spec.Template.GetObjectMeta(), obj) default: return nil, false } @@ -109,12 +109,14 @@ func (m *AnnotationMutators) rangeObjectList(ctx context.Context, list client.Ob } } -func (m *AnnotationMutators) mutate(namespacedName string, mutators map[string]instrumentation.AnnotationMutator, obj metav1.Object) (map[string]string, bool) { +func (m *AnnotationMutators) mutate(namespacedName string, mutators map[string]instrumentation.AnnotationMutator, podObj metav1.Object, workloadNamespaceObj client.Object) (map[string]string, bool) { // does autoAnnotateAutoInstrumentation or customSelector specify the namespaced name of the k8s object to annotate? var mutator instrumentation.ObjectAnnotationMutator + m.logger.Info("Trying mutate") mutator, specificMutatorExists := mutators[namespacedName] if !specificMutatorExists { - if m.monitor != nil && isWorkload(obj) && m.monitor.ShouldBeMonitored(obj) { + m.logger.Info(fmt.Sprintf("Annotation mutator '%s' does not exist", namespacedName)) + if m.monitor != nil && isWorkload(workloadNamespaceObj) && m.monitor.ShouldBeMonitored(workloadNamespaceObj) { // Is the object is a workload and does a service selects the workload? mutator = m.monitor } else { @@ -122,11 +124,11 @@ func (m *AnnotationMutators) mutate(namespacedName string, mutators map[string]i } } - mutatedAnnotations := mutator.Mutate(obj) + mutatedAnnotations := mutator.Mutate(podObj) return mutatedAnnotations, len(mutatedAnnotations) != 0 } -func isWorkload(obj metav1.Object) bool { +func isWorkload(obj client.Object) bool { switch obj.(type) { case *appsv1.Deployment, *appsv1.DaemonSet, *appsv1.StatefulSet: return true diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index b646b5e18..21edcdc17 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -14,6 +14,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/utils/strings/slices" "os" + "sigs.k8s.io/controller-runtime/pkg/client" "time" ) @@ -43,7 +44,7 @@ func NewMonitor(ctx context.Context, logger logr.Logger, config MonitorConfig, k } // ShouldBeMonitored returns whether obj is selected by either custom selector or auto monitor -func (m Monitor) ShouldBeMonitored(obj metav1.Object) bool { +func (m Monitor) ShouldBeMonitored(obj client.Object) bool { m.logger.Info("AutoMonitor shouldBeMonitored") // TODO: check if in custom selector // note: custom selector does not respect MonitorAllServices @@ -82,13 +83,13 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { // Check if the object implements the type assertion for PodTemplateSpec switch t := obj.(type) { case *appsv1.Deployment: - return labels.Set(t.Spec.Template.Labels) + return t.Spec.Template.Labels case *appsv1.StatefulSet: - return labels.Set(t.Spec.Template.Labels) + return t.Spec.Template.Labels case *appsv1.DaemonSet: - return labels.Set(t.Spec.Template.Labels) + return t.Spec.Template.Labels case *appsv1.ReplicaSet: - return labels.Set(t.Spec.Template.Labels) + return t.Spec.Template.Labels default: // Return empty labels.Set if the object type is not supported return labels.Set{} diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index 31205cbbd..20864f896 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -10,6 +10,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "testing" ) @@ -23,7 +24,7 @@ func TestMonitor_Selected(t *testing.T) { tests := []struct { name string service corev1.Service - workload metav1.Object + workload client.Object config MonitorConfig shouldMatch bool }{ From 3ce94cfc4dd628453579a002ff5054f8729ac3cf Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Tue, 11 Mar 2025 23:30:55 -0400 Subject: [PATCH 05/63] big progress --- .../namespacemutation/webhookhandler.go | 15 +- .../namespacemutation/webhookhandler_test.go | 2 +- .../workloadmutation/webhookhandler.go | 31 ++- main.go | 63 ++--- pkg/instrumentation/annotationtype.go | 17 +- pkg/instrumentation/auto/annotation.go | 47 ++-- pkg/instrumentation/auto/annotation_test.go | 6 - pkg/instrumentation/auto/auto_monitor.go | 215 +++++++++++++----- pkg/instrumentation/auto/auto_monitor_test.go | 19 +- pkg/instrumentation/auto/config.go | 45 +++- 10 files changed, 326 insertions(+), 134 deletions(-) diff --git a/internal/webhook/namespacemutation/webhookhandler.go b/internal/webhook/namespacemutation/webhookhandler.go index c290ac3b8..3966b9b5a 100644 --- a/internal/webhook/namespacemutation/webhookhandler.go +++ b/internal/webhook/namespacemutation/webhookhandler.go @@ -6,9 +6,8 @@ package namespacemutation import ( "context" "encoding/json" - "net/http" - corev1 "k8s.io/api/core/v1" + "net/http" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" @@ -21,12 +20,14 @@ var _ admission.Handler = (*handler)(nil) type handler struct { decoder *admission.Decoder annotationMutators *auto.AnnotationMutators + monitor *auto.Monitor } -func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators) admission.Handler { +func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators, monitor *auto.Monitor) admission.Handler { return &handler{ decoder: decoder, annotationMutators: annotationMutators, + monitor: monitor, } } @@ -36,7 +37,13 @@ func (h *handler) Handle(_ context.Context, req admission.Request) admission.Res if err != nil { return admission.Errored(http.StatusBadRequest, err) } - h.annotationMutators.MutateObject(namespace) + + if h.annotationMutators.IsMutated(namespace) && !h.monitor.AnyCustomSelectorDefined() { + h.annotationMutators.MutateObject(namespace) + } else { + // do not need to pass in oldObj because it's only used to check for workload pod template diff + h.monitor.MutateObject(nil, namespace) + } marshaledNamespace, err := json.Marshal(namespace) if err != nil { res := admission.Errored(http.StatusInternalServerError, err) diff --git a/internal/webhook/namespacemutation/webhookhandler_test.go b/internal/webhook/namespacemutation/webhookhandler_test.go index 03e3a5647..4124c0146 100644 --- a/internal/webhook/namespacemutation/webhookhandler_test.go +++ b/internal/webhook/namespacemutation/webhookhandler_test.go @@ -41,7 +41,7 @@ func TestHandle(t *testing.T) { instrumentation.NewTypeSet(instrumentation.TypeJava), nil, ) - h := NewWebhookHandler(decoder, mutators) + h := NewWebhookHandler(decoder, mutators, nil) for _, testCase := range []struct { req admission.Request name string diff --git a/internal/webhook/workloadmutation/webhookhandler.go b/internal/webhook/workloadmutation/webhookhandler.go index a2394596f..2b2d0c0be 100644 --- a/internal/webhook/workloadmutation/webhookhandler.go +++ b/internal/webhook/workloadmutation/webhookhandler.go @@ -8,9 +8,10 @@ import ( "context" "encoding/json" "errors" - "net/http" - + v1 "k8s.io/api/admission/v1" appsv1 "k8s.io/api/apps/v1" + "net/http" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -21,6 +22,7 @@ import ( // +kubebuilder:rbac:groups="apps",resources=daemonsets;deployments;statefulsets,verbs=get;list;watch var _ WebhookHandler = (*workloadMutationWebhook)(nil) +var logger = ctrl.Log.WithName("workload_webhook") // WebhookHandler is a webhook handler that analyzes new daemon-sets and injects appropriate annotations into it. type WebhookHandler interface { @@ -31,24 +33,29 @@ type WebhookHandler interface { type workloadMutationWebhook struct { decoder *admission.Decoder annotationMutators *auto.AnnotationMutators + monitor *auto.Monitor } // NewWebhookHandler creates a new WorkloadWebhookHandler. -func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators) WebhookHandler { +func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators, monitor *auto.Monitor) WebhookHandler { return &workloadMutationWebhook{ decoder: decoder, annotationMutators: annotationMutators, + monitor: monitor, } } func (p *workloadMutationWebhook) Handle(_ context.Context, req admission.Request) admission.Response { - var obj client.Object + var oldObj, obj client.Object switch objectKind := req.Kind.Kind; objectKind { case "DaemonSet": + oldObj = &appsv1.DaemonSet{} obj = &appsv1.DaemonSet{} case "Deployment": + oldObj = &appsv1.Deployment{} obj = &appsv1.Deployment{} case "StatefulSet": + oldObj = &appsv1.StatefulSet{} obj = &appsv1.StatefulSet{} default: return admission.Errored(http.StatusBadRequest, errors.New("failed to unmarshal request object")) @@ -58,7 +65,21 @@ func (p *workloadMutationWebhook) Handle(_ context.Context, req admission.Reques return admission.Errored(http.StatusBadRequest, err) } - p.annotationMutators.MutateObject(obj) + // populate old object + if req.Operation == v1.Update { + if err := p.decoder.DecodeRaw(req.OldObject, oldObj); err != nil { + logger.Error(err, "failed to unmarshal old object") + return admission.Errored(http.StatusBadRequest, err) + } + } + + // preserve backwards compatability. + if p.annotationMutators.IsMutated(obj) && !p.monitor.AnyCustomSelectorDefined() { + p.annotationMutators.MutateObject(obj) + } else { + p.monitor.MutateObject(oldObj, obj) + } + marshaledObject, err := json.Marshal(obj) if err != nil { res := admission.Errored(http.StatusInternalServerError, err) diff --git a/main.go b/main.go index 4f9bf23ee..ef9495647 100644 --- a/main.go +++ b/main.go @@ -130,6 +130,7 @@ func main() { autoInstrumentationDotNet string autoInstrumentationNodeJS string autoAnnotationConfigStr string + autoMonitorConfigStr string autoInstrumentationConfigStr string webhookPort int tlsOpt tlsConfig @@ -147,6 +148,7 @@ func main() { stringFlagOrEnv(&autoInstrumentationDotNet, "auto-instrumentation-dotnet-image", "RELATED_IMAGE_AUTO_INSTRUMENTATION_DOTNET", fmt.Sprintf("%s:%s", autoInstrumentationDotNetImageRepository, v.AutoInstrumentationDotNet), "The default OpenTelemetry Dotnet instrumentation image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&autoInstrumentationNodeJS, "auto-instrumentation-nodejs-image", "RELATED_IMAGE_AUTO_INSTRUMENTATION_NODEJS", fmt.Sprintf("%s:%s", autoInstrumentationNodeJSImageRepository, v.AutoInstrumentationNodeJS), "The default OpenTelemetry NodeJS instrumentation image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&autoAnnotationConfigStr, "auto-annotation-config", "AUTO_ANNOTATION_CONFIG", "", "The configuration for auto-annotation.") + pflag.StringVar(&autoMonitorConfigStr, "auto-monitor-config", "", "The configuration for auto-monitor.") pflag.StringVar(&autoInstrumentationConfigStr, "auto-instrumentation-config", "", "The configuration for auto-instrumentation.") stringFlagOrEnv(&dcgmExporterImage, "dcgm-exporter-image", "RELATED_IMAGE_DCGM_EXPORTER", fmt.Sprintf("%s:%s", dcgmExporterImageRepository, v.DcgmExporter), "The default DCGM Exporter image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&neuronMonitorImage, "neuron-monitor-image", "RELATED_IMAGE_NEURON_MONITOR", fmt.Sprintf("%s:%s", neuronMonitorImageRepository, v.NeuronMonitor), "The default Neuron monitor image. This image is used when no image is specified in the CustomResource.") @@ -290,49 +292,33 @@ func main() { if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" || autoAnnotationConfigStr == "" { setupLog.Info("Auto-annotation is disabled") } else { + + supportedLanguages := instrumentation.NewTypeSet( + instrumentation.TypeJava, + instrumentation.TypePython, + instrumentation.TypeDotNet, + instrumentation.TypeNodeJS, + ) + var autoAnnotationConfig auto.AnnotationConfig if err = json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-annotation config") } else { // TODO marshal/unmarshal monitor config setupLog.Info("Test!") - monitorConfig := auto.MonitorConfig{ - // TODO: remove - MonitorAllServices: true, - } - - supportedLanguages := instrumentation.NewTypeSet( - instrumentation.TypeJava, - instrumentation.TypePython, - instrumentation.TypeDotNet, - instrumentation.TypeNodeJS, - ) - if monitorConfig.Languages == nil { - monitorConfig.Languages = supportedLanguages - } - k8sConfig, err := rest.InClusterConfig() - if err != nil { - panic("TODO: Implement handling for failed clientset creation") - } + monitor := createMonitorFromConfig(autoMonitorConfigStr, ctx) - clientSet, err := kubernetes.NewForConfig(k8sConfig) - if err != nil { - // TODO - panic("TODO: Implement handling for failed clientset creation") - } - monitor := auto.NewMonitor(ctx, setupLog, monitorConfig, clientSet) autoAnnotationMutators := auto.NewAnnotationMutators( mgr.GetClient(), mgr.GetAPIReader(), logger, autoAnnotationConfig, supportedLanguages, - monitor, ) mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ - Handler: workloadmutation.NewWebhookHandler(decoder, autoAnnotationMutators)}) + Handler: workloadmutation.NewWebhookHandler(decoder, autoAnnotationMutators, monitor)}) mgr.GetWebhookServer().Register("/mutate-v1-namespace", &webhook.Admission{ - Handler: namespacemutation.NewWebhookHandler(decoder, autoAnnotationMutators), + Handler: namespacemutation.NewWebhookHandler(decoder, autoAnnotationMutators, monitor), }) setupLog.Info("Auto-annotation is enabled") @@ -384,6 +370,29 @@ func main() { } } +func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context) *auto.Monitor { + var monitorConfig *auto.MonitorConfig + var monitor *auto.Monitor + if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { + setupLog.Error(err, "Unable to unmarshal auto-monitor config") + } else { + if monitorConfig.Languages == nil || len(monitorConfig.Languages) == 0 { + monitorConfig.Languages = instrumentation.NewTypeSet(instrumentation.SupportedTypes()...) + } + k8sConfig, err := rest.InClusterConfig() + if err != nil { + panic("TODO: Implement handling for failed clientset creation") + } + + clientSet, err := kubernetes.NewForConfig(k8sConfig) + if err != nil { + panic("TODO: Implement handling for failed clientset creation") + } + monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet) + } + return monitor +} + func waitForWebhookServerStart(ctx context.Context, checker healthz.Checker, callback func(context.Context)) { ticker := time.NewTicker(time.Second) defer ticker.Stop() diff --git a/pkg/instrumentation/annotationtype.go b/pkg/instrumentation/annotationtype.go index 0e4b9246f..70798ea4f 100644 --- a/pkg/instrumentation/annotationtype.go +++ b/pkg/instrumentation/annotationtype.go @@ -3,12 +3,25 @@ package instrumentation +import ( + "encoding/json" +) + // Type is an enum for instrumentation types. type Type string // TypeSet is a map with Type keys. type TypeSet map[Type]any +func (s *TypeSet) UnmarshalJSON(data []byte) error { + var types []Type + if err := json.Unmarshal(data, &types); err != nil { + return err + } + *s = NewTypeSet(types...) + return nil +} + // NewTypeSet creates a new set of Type. func NewTypeSet(types ...Type) TypeSet { s := make(TypeSet, len(types)) @@ -26,8 +39,8 @@ const ( TypeGo Type = "go" ) -func AllTypes() []Type { - return []Type{TypeJava, TypeNodeJS, TypePython, TypeDotNet, TypeGo} +func SupportedTypes() []Type { + return []Type{TypeJava, TypeNodeJS, TypePython, TypeDotNet} } // InjectAnnotationKey maps the instrumentation type to the inject annotation. diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index f3b047cf1..da57102dc 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -29,7 +29,6 @@ const ( // AnnotationMutators contains functions that can be used to mutate annotations // on all supported objects based on the configured mutators. type AnnotationMutators struct { - monitor *Monitor clientWriter client.Writer clientReader client.Reader logger logr.Logger @@ -39,6 +38,13 @@ type AnnotationMutators struct { statefulSetMutators map[string]instrumentation.AnnotationMutator defaultMutator instrumentation.AnnotationMutator injectAnnotations map[string]struct{} + monitor *Monitor + cfg *AnnotationConfig +} + +// IsMutated returns if obj is specified in the config +func (m *AnnotationMutators) IsMutated(obj client.Object) bool { + return len(m.cfg.GetObjectLanguagesToAnnotate(obj)) > 0 } // RestartNamespace sets the restartedAtAnnotation for each of the namespace's supported resources and patches them. @@ -70,13 +76,13 @@ func (m *AnnotationMutators) MutateObject(obj client.Object) (any, bool) { func (m *AnnotationMutators) mutateObject(obj client.Object, _ any) (any, bool) { switch o := obj.(type) { case *corev1.Namespace: - return m.mutate(o.GetName(), m.namespaceMutators, o.GetObjectMeta(), obj) + return m.mutate(o.GetName(), m.namespaceMutators, o.GetObjectMeta()) case *appsv1.Deployment: - return m.mutate(namespacedName(o.GetObjectMeta()), m.deploymentMutators, o.Spec.Template.GetObjectMeta(), obj) + return m.mutate(namespacedName(o.GetObjectMeta()), m.deploymentMutators, o.Spec.Template.GetObjectMeta()) case *appsv1.DaemonSet: - return m.mutate(namespacedName(o.GetObjectMeta()), m.daemonSetMutators, o.Spec.Template.GetObjectMeta(), obj) + return m.mutate(namespacedName(o.GetObjectMeta()), m.daemonSetMutators, o.Spec.Template.GetObjectMeta()) case *appsv1.StatefulSet: - return m.mutate(namespacedName(o.GetObjectMeta()), m.statefulSetMutators, o.Spec.Template.GetObjectMeta(), obj) + return m.mutate(namespacedName(o.GetObjectMeta()), m.statefulSetMutators, o.Spec.Template.GetObjectMeta()) default: return nil, false } @@ -109,33 +115,15 @@ func (m *AnnotationMutators) rangeObjectList(ctx context.Context, list client.Ob } } -func (m *AnnotationMutators) mutate(namespacedName string, mutators map[string]instrumentation.AnnotationMutator, podObj metav1.Object, workloadNamespaceObj client.Object) (map[string]string, bool) { - // does autoAnnotateAutoInstrumentation or customSelector specify the namespaced name of the k8s object to annotate? - var mutator instrumentation.ObjectAnnotationMutator - m.logger.Info("Trying mutate") - mutator, specificMutatorExists := mutators[namespacedName] - if !specificMutatorExists { - m.logger.Info(fmt.Sprintf("Annotation mutator '%s' does not exist", namespacedName)) - if m.monitor != nil && isWorkload(workloadNamespaceObj) && m.monitor.ShouldBeMonitored(workloadNamespaceObj) { - // Is the object is a workload and does a service selects the workload? - mutator = m.monitor - } else { - mutator = &m.defaultMutator - } +func (m *AnnotationMutators) mutate(name string, mutators map[string]instrumentation.AnnotationMutator, obj metav1.Object) (map[string]string, bool) { + mutator, ok := mutators[name] + if !ok { + mutator = m.defaultMutator } - - mutatedAnnotations := mutator.Mutate(podObj) + mutatedAnnotations := mutator.Mutate(obj) return mutatedAnnotations, len(mutatedAnnotations) != 0 } -func isWorkload(obj client.Object) bool { - switch obj.(type) { - case *appsv1.Deployment, *appsv1.DaemonSet, *appsv1.StatefulSet: - return true - } - return false -} - func namespacedName(obj metav1.Object) string { return fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName()) } @@ -149,20 +137,19 @@ func NewAnnotationMutators( logger logr.Logger, cfg AnnotationConfig, typeSet instrumentation.TypeSet, - monitor *Monitor, ) *AnnotationMutators { builder := newMutatorBuilder(typeSet) return &AnnotationMutators{ clientWriter: clientWriter, clientReader: clientReader, logger: logger, - monitor: monitor, namespaceMutators: builder.buildMutators(getResources(cfg, typeSet, getNamespaces)), deploymentMutators: builder.buildMutators(getResources(cfg, typeSet, getDeployments)), daemonSetMutators: builder.buildMutators(getResources(cfg, typeSet, getDaemonSets)), statefulSetMutators: builder.buildMutators(getResources(cfg, typeSet, getStatefulSets)), defaultMutator: instrumentation.NewAnnotationMutator(maps.Values(builder.removeMutations)), injectAnnotations: buildInjectAnnotations(typeSet), + cfg: &cfg, } } diff --git a/pkg/instrumentation/auto/annotation_test.go b/pkg/instrumentation/auto/annotation_test.go index ee830c195..72ac13ee5 100644 --- a/pkg/instrumentation/auto/annotation_test.go +++ b/pkg/instrumentation/auto/annotation_test.go @@ -116,7 +116,6 @@ func TestAnnotationMutators_Namespaces(t *testing.T) { logr.Logger{}, testCase.cfg, testCase.typeSet, - nil, ) mutators.MutateAndPatchAll(ctx) gotNamespaces := &corev1.NamespaceList{} @@ -201,7 +200,6 @@ func TestAnnotationMutators_Namespaces_Restart(t *testing.T) { logr.Logger{}, cfg, instrumentation.NewTypeSet(instrumentation.TypeJava), - nil, ) mutators.MutateAndPatchAll(context.Background()) ctx := context.Background() @@ -293,7 +291,6 @@ func TestAnnotationMutators_Deployments(t *testing.T) { logr.Logger{}, testCase.cfg, testCase.typeSet, - nil, ) mutators.MutateAndPatchAll(ctx) gotDeployments := &appsv1.DeploymentList{} @@ -362,7 +359,6 @@ func TestAnnotationMutators_DaemonSets(t *testing.T) { logr.Logger{}, testCase.cfg, testCase.typeSet, - nil, ) mutators.MutateAndPatchAll(ctx) gotDaemonSets := &appsv1.DaemonSetList{} @@ -431,7 +427,6 @@ func TestAnnotationMutators_StatefulSets(t *testing.T) { logr.Logger{}, testCase.cfg, testCase.typeSet, - nil, ) mutators.MutateAndPatchAll(ctx) gotStatefulSets := &appsv1.StatefulSetList{} @@ -498,7 +493,6 @@ func TestAnnotationMutators_ClientErrors(t *testing.T) { logr.Logger{}, cfg, instrumentation.NewTypeSet(instrumentation.TypeJava), - nil, ) mutators.MutateAndPatchAll(context.Background()) errClient.AssertCalled(t, "List", mock.Anything, mock.Anything, mock.Anything) diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index 21edcdc17..eb9638cc3 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" - "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,23 +13,53 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/utils/strings/slices" "os" + "reflect" "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" "time" ) +var logger = logf.Log.WithName("auto_monitor") + type Monitor struct { serviceInformer cache.SharedIndexInformer ctx context.Context - logger logr.Logger config MonitorConfig } -func NewMonitor(ctx context.Context, logger logr.Logger, config MonitorConfig, k8sInterface kubernetes.Interface) *Monitor { +func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface) *Monitor { logger.Info("AutoMonitor starting...") factory := informers.NewSharedInformerFactory(k8sInterface, 10*time.Minute) serviceInformer := factory.Core().V1().Services().Informer() + m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config} + + _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: func(obj interface{}) { + if config.AutoRestart { + list, err := k8sInterface.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + logger.Error(err, "failed to list namespaces") + } + + // TODO: optimize this by trying to resolve workloads via endpoint slices + for _, namespace := range list.Items { + for _, deployment := range listAllDeployments(k8sInterface, namespace, ctx).Items { + m.MutateObject(nil, &deployment) + } + for _, deployment := range listAllStatefulSets(k8sInterface, namespace, ctx).Items { + m.MutateObject(nil, &deployment) + } + for _, deployment := range listAllDaemonSets(k8sInterface, namespace, ctx).Items { + m.MutateObject(nil, &deployment) + } + m.MutateObject(nil, &namespace) + } + } + }}) + if err != nil { + return nil + } factory.Start(ctx.Done()) // runs in background synced := factory.WaitForCacheSync(ctx.Done()) for v, ok := range synced { @@ -40,43 +69,32 @@ func NewMonitor(ctx context.Context, logger logr.Logger, config MonitorConfig, k } } logger.Info("AutoMonitor enabled!") - return &Monitor{serviceInformer: serviceInformer, ctx: ctx, logger: logger, config: config} + + return m } -// ShouldBeMonitored returns whether obj is selected by either custom selector or auto monitor -func (m Monitor) ShouldBeMonitored(obj client.Object) bool { - m.logger.Info("AutoMonitor shouldBeMonitored") - // TODO: check if in custom selector - // note: custom selector does not respect MonitorAllServices - if m.customSelected(obj) { - return true +func listAllDeployments(k8sInterface kubernetes.Interface, namespace corev1.Namespace, ctx context.Context) *appsv1.DeploymentList { + list, err := k8sInterface.AppsV1().Deployments(namespace.Name).List(ctx, metav1.ListOptions{}) + if err != nil { + logger.Error(err, "AutoMonitor failed to list deployments") } + return list +} - if !m.config.MonitorAllServices { - return false +func listAllStatefulSets(k8sInterface kubernetes.Interface, namespace corev1.Namespace, ctx context.Context) *appsv1.StatefulSetList { + list, err := k8sInterface.AppsV1().StatefulSets(namespace.Name).List(ctx, metav1.ListOptions{}) + if err != nil { + logger.Error(err, "AutoMonitor failed to list statefulsets") } + return list +} - objectLabels := getTemplateSpecLabels(obj) - // if object is not workload, return err - m.logger.Info(fmt.Sprintf("AutoMonitor labels: %v", objectLabels)) - m.logger.Info(fmt.Sprintf("AutoMonitor ListKeys: %v", m.serviceInformer.GetStore().ListKeys())) - m.logger.Info(fmt.Sprintf("AutoMonitor List: %v", m.serviceInformer.GetStore().List())) - for _, informerObj := range m.serviceInformer.GetStore().List() { - service := informerObj.(*corev1.Service) - if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 { - continue - } - serviceSelector := labels.SelectorFromSet(service.Spec.Selector) - - m.logger.V(2).Info("AutoMonitor: testing serviceSelector", "serviceSelector", serviceSelector.String(), "objectLabels", objectLabels.String()) - if serviceSelector.Matches(objectLabels) { - m.logger.V(2).Info("AutoMonitor: matched!", "service", service, "object", obj.GetName()) - return true - } - - // remove if none matched, not in custom selector +func listAllDaemonSets(k8sInterface kubernetes.Interface, namespace corev1.Namespace, ctx context.Context) *appsv1.DaemonSetList { + list, err := k8sInterface.AppsV1().DaemonSets(namespace.Name).List(ctx, metav1.ListOptions{}) + if err != nil { + logger.Error(err, "AutoMonitor failed to list DaemonSets") } - return false + return list } func getTemplateSpecLabels(obj metav1.Object) labels.Set { @@ -96,15 +114,62 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { } } -// Mutate adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector -func (m Monitor) Mutate(obj metav1.Object) map[string]string { - // TODO: only create if automonitor enabled - annotations := obj.GetAnnotations() +// MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector +func (m Monitor) MutateObject(oldObj client.Object, obj client.Object) map[string]string { + // todo: handle edge case where a workload is annotated because a service exposed it, and the service is removed. aka add to Service OnDelete + // continue only if restart is enabled or if workload pod template has been mutated + if !m.config.AutoRestart && isWorkload(obj) && !isWorkloadPodTemplateMutated(oldObj, obj) { + return nil + } + // custom selector takes precedence over service selector + if customSelectLanguages, selected := m.CustomSelected(obj); selected { + logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) + return mutate(obj, customSelectLanguages) + } + + if !m.config.MonitorAllServices { + return nil + } + + if m.excluded(obj) { + return nil + } + + objectLabels := getTemplateSpecLabels(obj) + // if object is not workload, return err + for _, informerObj := range m.serviceInformer.GetStore().List() { + service := informerObj.(*corev1.Service) + if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 { + continue + } + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + + if serviceSelector.Matches(objectLabels) { + logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is owned by service %s", obj.GetName(), m.config.Languages, service.Name)) + return mutate(obj, m.config.Languages) + } + } + return nil +} + +func mutate(obj client.Object, languagesToMonitor instrumentation.TypeSet) map[string]string { + podTemplate := getPodTemplate(obj) + var annotations map[string]string + + if podTemplate == nil { + annotations = obj.GetAnnotations() + } else { + annotations = podTemplate.GetAnnotations() + } + if annotations == nil { + annotations = make(map[string]string) + } + allMutatedAnnotations := make(map[string]string) - for _, language := range instrumentation.AllTypes() { + for _, language := range instrumentation.SupportedTypes() { insertMutation, removeMutation := buildMutations(language) var mutatedAnnotations map[string]string - if _, ok := m.config.Languages[language]; ok { + if _, ok := languagesToMonitor[language]; ok { mutatedAnnotations = insertMutation.Mutate(annotations) } else { @@ -118,24 +183,68 @@ func (m Monitor) Mutate(obj metav1.Object) map[string]string { return allMutatedAnnotations } -func (m Monitor) customSelected(obj metav1.Object) bool { - objName := namespacedName(obj) - var resourceList []string +func (m Monitor) CustomSelected(obj client.Object) (instrumentation.TypeSet, bool) { + languages := m.config.CustomSelector.GetObjectLanguagesToAnnotate(obj) + return languages, len(languages) > 0 +} - switch obj.(type) { - case *appsv1.Deployment: - for _, t := range instrumentation.AllTypes() { - resourceList = append(resourceList, m.config.CustomSelector.getResources(t).Deployments...) +// excluded returns whether a Namespace or a Service is excluded from AutoMonitor. +func (m Monitor) excluded(obj client.Object) bool { + switch obj.GetObjectKind().GroupVersionKind().Kind { + case "Namespace": + return slices.Contains(m.config.Exclude.Namespaces, obj.GetName()) + case "Service": + return slices.Contains(m.config.Exclude.Services, namespacedName(obj)) + } + return false +} + +func (m Monitor) Enabled() bool { + return m.config.MonitorAllServices +} + +func (m Monitor) AnyCustomSelectorDefined() bool { + for _, t := range instrumentation.SupportedTypes() { + resources := m.config.CustomSelector.getResources(t) + if len(resources.DaemonSets) > 0 { + return true } - case *appsv1.StatefulSet: - for _, t := range instrumentation.AllTypes() { - resourceList = append(resourceList, m.config.CustomSelector.getResources(t).StatefulSets...) + if len(resources.StatefulSets) > 0 { + return true } - case *appsv1.DaemonSet: - for _, t := range instrumentation.AllTypes() { - resourceList = append(resourceList, m.config.CustomSelector.getResources(t).DaemonSets...) + if len(resources.Deployments) > 0 { + return true + } + if len(resources.Namespaces) > 0 { + return true } } + return false +} + +func isWorkload(obj client.Object) bool { + kind := obj.GetObjectKind().GroupVersionKind().Kind + return kind == "Deployment" || kind == "StatefulSet" || kind == "DaemonSet" +} - return slices.Contains(resourceList, objName) +// isWorkloadPodTemplateMutated +func isWorkloadPodTemplateMutated(oldObject client.Object, object client.Object) bool { + oldTemplate, newTemplate := getPodTemplate(oldObject), getPodTemplate(object) + if oldTemplate == nil || newTemplate == nil { + return true + } + return !reflect.DeepEqual(oldTemplate, newTemplate) +} + +func getPodTemplate(obj client.Object) *corev1.PodTemplateSpec { + switch o := obj.(type) { + case *appsv1.Deployment: + return &o.Spec.Template + case *appsv1.StatefulSet: + return &o.Spec.Template + case *appsv1.DaemonSet: + return &o.Spec.Template + default: + return nil + } } diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index 20864f896..f7597bbe0 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -20,10 +20,11 @@ var logger = logf.Log.WithName("auto_monitor_tests") func TestMonitor_Selected(t *testing.T) { logger.Info("Starting testmonitor tests") - allTypes := instrumentation.NewTypeSet(instrumentation.AllTypes()...) + allTypes := instrumentation.NewTypeSet(instrumentation.SupportedTypes()...) tests := []struct { name string service corev1.Service + oldWorkload client.Object workload client.Object config MonitorConfig shouldMatch bool @@ -282,11 +283,11 @@ func TestMonitor_Selected(t *testing.T) { err = createWorkload(ctx, clientSet, tt.workload) require.NoError(t, err) - m := NewMonitor(ctx, logger, tt.config, clientSet) - matched := m.ShouldBeMonitored(tt.workload) + m := NewMonitor(ctx, tt.config, clientSet) + mutatedAnnotations := m.MutateObject(tt.workload, tt.workload) - assert.Equal(t, tt.shouldMatch, matched, - "Expected workload matching to be %v but got %v", tt.shouldMatch, matched) + assert.Equal(t, tt.shouldMatch, mutatedAnnotations, + "Expected workload matching to be %v but got %v", tt.shouldMatch, mutatedAnnotations) }) } } @@ -367,3 +368,11 @@ func createWorkload(ctx context.Context, clientSet *fake.Clientset, workload met return fmt.Errorf("unsupported workload type: %T", workload) } } + +func TestUnmarshal(t *testing.T) { + j := []byte(`["java", "nodejs", "python"]`) + set := instrumentation.TypeSet{} + err := set.UnmarshalJSON(j) + assert.NoError(t, err) + assert.Equal(t, instrumentation.TypeSet{instrumentation.TypeNodeJS: nil, instrumentation.TypeJava: nil, instrumentation.TypePython: nil}, set) +} diff --git a/pkg/instrumentation/auto/config.go b/pkg/instrumentation/auto/config.go index 12b5e9e6d..ac5741785 100644 --- a/pkg/instrumentation/auto/config.go +++ b/pkg/instrumentation/auto/config.go @@ -3,7 +3,13 @@ package auto -import "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" +import ( + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "slices" +) // AnnotationConfig details the resources that have enabled // auto-annotation for each instrumentation type. @@ -29,6 +35,43 @@ func (c AnnotationConfig) getResources(instType instrumentation.Type) Annotation } } +// GetObjectLanguagesToAnnotate get languages to annotate for an object +func (c AnnotationConfig) GetObjectLanguagesToAnnotate(obj client.Object) instrumentation.TypeSet { + objName := namespacedName(obj) + typesSelected := instrumentation.TypeSet{} + + types := instrumentation.SupportedTypes() + + switch obj.(type) { + case *appsv1.Deployment: + for _, t := range types { + if slices.Contains(c.getResources(t).Deployments, objName) { + typesSelected[t] = nil + } + } + case *appsv1.StatefulSet: + for _, t := range types { + if slices.Contains(c.getResources(t).StatefulSets, objName) { + typesSelected[t] = nil + } + } + case *appsv1.DaemonSet: + for _, t := range types { + if slices.Contains(c.getResources(t).DaemonSets, objName) { + typesSelected[t] = nil + } + } + case *corev1.Namespace: + for _, t := range types { + if slices.Contains(c.getResources(t).Namespaces, objName) { + typesSelected[t] = nil + } + } + } + + return typesSelected +} + // AnnotationResources contains slices of resource names for each // of the supported workloads. type AnnotationResources struct { From 97ee02bcbf96d889677d548134d70c50c4dad73f Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 12 Mar 2025 15:17:13 -0400 Subject: [PATCH 06/63] more progress --- .../validate_automonitor_daemonset_test.go | 41 ++ .../validate_automonitor_deployment_test.go | 212 +++++++++ .../validate_automonitor_methods.go | 445 +++++++++++++++++ .../validate_automonitor_namespace_test.go | 446 ++++++++++++++++++ .../validate_automonitor_statefulset_test.go | 191 ++++++++ .../namespacemutation/webhookhandler.go | 6 +- .../namespacemutation/webhookhandler_test.go | 4 +- .../workloadmutation/webhookhandler.go | 6 +- .../workloadmutation/webhookhandler_test.go | 3 +- main.go | 11 +- pkg/instrumentation/auto/annotation.go | 4 +- pkg/instrumentation/auto/auto_monitor.go | 41 +- pkg/instrumentation/auto/auto_monitor_test.go | 150 +++++- 13 files changed, 1527 insertions(+), 33 deletions(-) create mode 100644 integration-tests/manifests/automonitor/validate_automonitor_daemonset_test.go create mode 100644 integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go create mode 100644 integration-tests/manifests/automonitor/validate_automonitor_methods.go create mode 100644 integration-tests/manifests/automonitor/validate_automonitor_namespace_test.go create mode 100644 integration-tests/manifests/automonitor/validate_automonitor_statefulset_test.go diff --git a/integration-tests/manifests/automonitor/validate_automonitor_daemonset_test.go b/integration-tests/manifests/automonitor/validate_automonitor_daemonset_test.go new file mode 100644 index 000000000..bcd43ac0e --- /dev/null +++ b/integration-tests/manifests/automonitor/validate_automonitor_daemonset_test.go @@ -0,0 +1,41 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package annotations + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "math/big" + "path/filepath" + "testing" + "time" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" +) + +func TestAutoMonitorEnabled(t *testing.T) { + setupFunction(t, "auto-monitor-test", []string{sampleDeploymentYamlNameRelPath}) + clientSet := setupTest(t) + + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{filepath.Join(uniqueNamespace, daemonSetName)}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + } + updateAnnotationConfig(annotationConfig) + + if err := checkResourceAnnotations(t, clientSet, "daemonset", uniqueNamespace, daemonSetName, sampleDaemonsetYamlRelPath, startTime, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } + +} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go new file mode 100644 index 000000000..27697e61d --- /dev/null +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -0,0 +1,212 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package annotations + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "math/big" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" +) + +func TestAllLanguagesDeployment(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("deployment-namespace-all-languages-%d", randomNumber) + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + StatefulSets: []string{""}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + StatefulSets: []string{""}, + }, + DotNet: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + StatefulSets: []string{""}, + }, + NodeJS: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + assert.Nil(t, err) + + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + + if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation, injectPythonAnnotation, autoAnnotatePythonAnnotation, injectDotNetAnnotation, autoAnnotateDotNetAnnotation, injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } + +} + +func TestJavaOnlyDeployment(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("deployment-namespace-java-only-%d", randomNumber) + + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + StatefulSets: []string{""}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Errorf("Failed to marshal: %v\n", err) + } + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + + if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } +} + +func TestPythonOnlyDeployment(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("deployment-namespace-python-only-%d", randomNumber) + + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + if err != nil { + t.Errorf("Failed to get deployment app: %s", err.Error()) + } + + if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } + +} +func TestDotNetOnlyDeployment(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("deployment-namespace-dotnet-only-%d", randomNumber) + + annotationConfig := auto.AnnotationConfig{ + DotNet: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + if err != nil { + t.Errorf("Failed to get deployment app: %s", err.Error()) + } + + if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectDotNetAnnotation, autoAnnotateDotNetAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } + +} + +func TestNodeJSOnlyDeployment(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("deployment-namespace-nodejs-only-%d", randomNumber) + + annotationConfig := auto.AnnotationConfig{ + NodeJS: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + t.Error("Error:", err) + + } + + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + if err != nil { + t.Errorf("Failed to get deployment app: %s", err.Error()) + } + + if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } + +} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go new file mode 100644 index 000000000..b2dcf209c --- /dev/null +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -0,0 +1,445 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package annotations + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/uuid" + appsV1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/aws/amazon-cloudwatch-agent-operator/integration-tests/util" +) + +const ( + injectJavaAnnotation = "instrumentation.opentelemetry.io/inject-java" + autoAnnotateJavaAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-java" + injectPythonAnnotation = "instrumentation.opentelemetry.io/inject-python" + autoAnnotatePythonAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-python" + injectDotNetAnnotation = "instrumentation.opentelemetry.io/inject-dotnet" + autoAnnotateDotNetAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-dotnet" + + injectNodeJSAnnotation = "instrumentation.opentelemetry.io/inject-nodejs" + autoAnnotateNodeJSAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-nodejs" + + deploymentName = "sample-deployment" + nginxDeploymentName = "nginx" + statefulSetName = "sample-statefulset" + amazonCloudwatchNamespace = "amazon-cloudwatch" + + daemonSetName = "sample-daemonset" + + amazonControllerManager = "cloudwatch-controller-manager" + + sampleDaemonsetYamlRelPath = "../sample-daemonset.yaml" + sampleDeploymentYamlNameRelPath = "../sample-deployment.yaml" + sampleNginxAppYamlNameRelPath = "../../java/sample-deployment-java.yaml" + + sampleStatefulsetYamlNameRelPath = "../sample-statefulset.yaml" + timoutDuration = 2 * time.Minute + numberOfRetries = 10 + timeBetweenRetries = 5 * time.Second +) + +func applyYAMLWithKubectl(filename, namespace string) error { + cmd := exec.Command("kubectl", "apply", "-f", filename, "-n", namespace) + return cmd.Run() +} + +func createNamespaceAndApplyResources(t *testing.T, clientset *kubernetes.Clientset, name string, resourceFiles []string) error { + err := createNamespace(clientset, name) + if err != nil { + return err + } + + for _, file := range resourceFiles { + err = applyYAMLWithKubectl(file, name) + if err != nil { + t.Errorf("Could not apply resources %s/%s", name, file) + return err + } + } + return nil +} +func isNamespaceUpdated(clientset *kubernetes.Clientset, namespace string, startTime time.Time) bool { + //check if the namespace was updated + ns, err := clientset.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) + if err != nil { + fmt.Printf("Failed to get namespace %s: %v\n", namespace, err) + return false + } + if ns.CreationTimestamp.After(startTime) || ns.ResourceVersion != "" { + return true + } + + return false +} + +func deleteYAMLWithKubectl(filename, namespace string) error { + cmd := exec.Command("kubectl", "delete", "-f", filename, "-n", namespace) + return cmd.Run() +} +func deleteNamespaceAndResources(clientset *kubernetes.Clientset, name string, resourceFiles []string) error { + for _, file := range resourceFiles { + err := deleteYAMLWithKubectl(file, name) + if err != nil { + return err + } + } + + //delete Namespace + err := deleteNamespace(clientset, name) + return err +} + +// Check if name space exist and if it does not we create the namespace and wait until it is fully created +func createNamespace(clientSet *kubernetes.Clientset, name string) error { + _, err := clientSet.CoreV1().Namespaces().Get(context.Background(), name, metav1.GetOptions{}) + if err == nil { + return nil + } else if !errors.IsNotFound(err) { + return err + } + + namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}} + _, err = clientSet.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) + if err != nil { + return err + } + + startTime := time.Now() + for { + if time.Since(startTime) > timoutDuration { + return fmt.Errorf("timeout reached while waiting for namespace %s to be created", name) + } + + _, err := clientSet.CoreV1().Namespaces().Get(context.Background(), name, metav1.GetOptions{}) + if err == nil { + return nil + } else if !errors.IsNotFound(err) { //if any other error other than not found + return err + } + + time.Sleep(timeBetweenRetries) + } +} + +func deleteNamespace(clientset *kubernetes.Clientset, name string) error { + err := clientset.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{}) + return err +} + +// This function creates a namespace and checks it annotations and then deletes the namespace after check complete +func checkNameSpaceAnnotations(t *testing.T, clientSet *kubernetes.Clientset, expectedAnnotations []string, uniqueNamespace string, startTime time.Time) bool { + + if err := createNamespace(clientSet, uniqueNamespace); err != nil { + t.Fatalf("Failed to create/apply resoures on namespace: %v", err) + } + + defer func() { + if err := deleteNamespace(clientSet, uniqueNamespace); err != nil { + t.Fatalf("Failed to delete namespace: %v", err) + } + }() + + for { + if isNamespaceUpdated(clientSet, uniqueNamespace, startTime) { + fmt.Printf("Namespace %s has been updated.\n", uniqueNamespace) + break + } + elapsed := time.Since(startTime) + if elapsed >= timoutDuration { + fmt.Printf("Timeout reached while waiting for namespace %s to be updated.\n", uniqueNamespace) + break + } + } + + for i := 0; i < numberOfRetries; i++ { + correct := true + ns, err := clientSet.CoreV1().Namespaces().Get(context.TODO(), uniqueNamespace, metav1.GetOptions{}) + fmt.Printf("This is the loop iteration: %v\n, these are the annotation of ns %v", i, ns) + if err != nil { + fmt.Println("There was an error getting namespace, ", err) + return false + } + + for _, annotation := range expectedAnnotations { + fmt.Printf("\n This is the annotation: %v and its status %v, namespace name: %v, \n", ns.ObjectMeta.Annotations, ns.Status, ns.Name) + if ns.ObjectMeta.Annotations[annotation] != "true" { + time.Sleep(timeBetweenRetries) + correct = false + break + } + } + + if correct { + fmt.Println("Namespace annotations are correct!") + return true + } + } + return false +} + +// This function updates the operator and waits until it is ready +func updateOperator(t *testing.T, clientSet *kubernetes.Clientset, deployment *appsV1.Deployment, startTime time.Time) bool { + var err error + + args := deployment.Spec.Template.Spec.Containers[0].Args + + deployment, err = clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) + if err != nil { + t.Errorf("Failed to get deployment: %v\n", err) + return false + } + deployment.Spec.Template.Spec.Containers[0].Args = args + + _, err = clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + if err != nil { + t.Errorf("Failed to update deployment: %v\n", err) + return false + } + err = util.WaitForNewPodCreation(clientSet, deployment, startTime) + if err != nil { + fmt.Println("There was an error trying to wait for deployment available", err) + return false + } + fmt.Println("Deployment updated successfully!") + return true +} + +// check if the given pods have the expected annotations +func checkIfAnnotationExists(clientset *kubernetes.Clientset, pods *v1.PodList, expectedAnnotations []string) bool { + startTime := time.Now() + for { + if time.Since(startTime) > timoutDuration { + fmt.Println("Timeout reached while waiting for annotations.") + return false + } + + //This exist to check if any pods took too long to delete and we need to list pods again + currentPods, err := clientset.CoreV1().Pods(pods.Items[0].Namespace).List(context.TODO(), metav1.ListOptions{}) + + fmt.Println("Current pods len: ", len(currentPods.Items)) + if err != nil { + fmt.Printf("Failed to list pods: %v\n", err) + return false + } + + //check if all pods are in the Running phase + if !util.CheckIfPodsAreRunning(currentPods) { + + continue + } + + foundAllAnnotations := true + for _, pod := range currentPods.Items { + for _, annotation := range expectedAnnotations { + fmt.Printf("Checking pod %s for annotation %s in namespace %s\n", pod.Name, annotation, pod.Namespace) + + if value, exists := pod.Annotations[annotation]; !exists || value != "true" { + fmt.Printf("Pod %s does not have annotation %s with value 'true' in namespace %s\n", pod.Name, annotation, pod.Namespace) + foundAllAnnotations = false + break + } + } + if !foundAllAnnotations { + break + } + } + + if foundAllAnnotations { + fmt.Println("Annotations are correct!") + return true + } + + fmt.Println("Annotations not found in all pods or some pods are not in Running phase. Retrying...") + cmd := exec.Command("kubectl", "rollout", "restart", "deployment", amazonControllerManager, "-n", amazonCloudwatchNamespace) + + // Run the command and capture the output + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("Error restarting deployment: %v\n", err) + fmt.Printf("Output: %s\n", output) + } else { + fmt.Printf("Successfully deleted deployment: %s\n", output) + } + waitCmd := exec.Command("kubectl", "wait", "--for=condition=Available", "deployment/"+amazonControllerManager, "-n", amazonCloudwatchNamespace, "--timeout=300s") + + waitOutput, err := waitCmd.CombinedOutput() + if err != nil { + fmt.Printf("Error waiting for deployment: %v\n", err) + fmt.Printf("Output: %s\n", waitOutput) + } else { + fmt.Printf("Deployment is now available: %s\n", waitOutput) + } + time.Sleep(timeBetweenRetries) + } +} + +// Finds auto-annotation arg in operator and updates it, if not found it will be added to the end +func updateAnnotationConfig(deployment *appsV1.Deployment, jsonStr string) *appsV1.Deployment { + + args := deployment.Spec.Template.Spec.Containers[0].Args + indexOfAutoAnnotationConfigString := findIndexOfPrefix("--auto-annotation-config=", args) + if indexOfAutoAnnotationConfigString < 0 { + deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--auto-annotation-config="+jsonStr) + } else { + deployment.Spec.Template.Spec.Containers[0].Args[indexOfAutoAnnotationConfigString] = "--auto-annotation-config=" + jsonStr + } + return deployment +} +func findIndexOfPrefix(str string, strs []string) int { + for i, s := range strs { + if strings.HasPrefix(s, str) { + return i + } + } + return -1 +} + +// kubernetes setup +func setupTest(t *testing.T) *kubernetes.Clientset { + userHomeDir, err := os.UserHomeDir() + if err != nil { + t.Errorf("error getting user home dir: %v\n\n", err) + } + kubeConfigPath := filepath.Join(userHomeDir, ".kube", "config") + fmt.Printf("Using kubeconfig: %s\n\n", kubeConfigPath) + + kubeConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) + if err != nil { + t.Errorf("Error getting kubernetes config: %v\n\n", err) + } + + clientSet, err := kubernetes.NewForConfig(kubeConfig) + + if err != nil { + t.Errorf("error getting kubernetes config: %v\n\n", err) + } + return clientSet +} + +func updateTheOperator(t *testing.T, clientSet *kubernetes.Clientset, jsonStr string) { + deployment, err := clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) + if err != nil { + t.Errorf("Error getting deployment: %v\n\n", err) + } + deployment = updateAnnotationConfig(deployment, jsonStr) + + if !updateOperator(t, clientSet, deployment, time.Now().Add(-time.Second)) { + t.Error("Failed to update Operator", deployment, deployment.Name, deployment.Spec.Template.Spec.Containers[0].Args) + } +} + +func checkResourceAnnotations(t *testing.T, clientSet *kubernetes.Clientset, resourceType, uniqueNamespace, resourceName string, sampleAppYamlPath string, startTime time.Time, annotations []string, skipDelete bool) error { + if err := createNamespaceAndApplyResources(t, clientSet, uniqueNamespace, []string{sampleAppYamlPath}); err != nil { + t.Fatalf("Failed to create/apply resoures on namespace: %v", err) + return err + } + if !skipDelete { + t.Cleanup(func() { + if err := deleteNamespaceAndResources(clientSet, uniqueNamespace, []string{sampleAppYamlPath}); err != nil { + t.Fatalf("Failed to delete namespaces/resources: %v", err) + } + }) + } + + var resource interface{} + + switch resourceType { + case "deployment": + // Get deployment + deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment: %s", err.Error()) + } + resource = deployment + case "daemonset": + // Get daemonset + daemonset, err := clientSet.AppsV1().DaemonSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get daemonset: %s", err.Error()) + } + resource = daemonset + case "statefulset": + // Get statefulset + statefulset, err := clientSet.AppsV1().StatefulSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get statefulset: %s", err.Error()) + } + resource = statefulset + default: + return fmt.Errorf("unsupported resource type: %s", resourceType) + } + + // Wait for new pod creation + err := util.WaitForNewPodCreation(clientSet, resource, startTime) + if err != nil { + return fmt.Errorf("error waiting for pod creation: %s", err.Error()) + } + + // List resource pods + resourcePods, err := clientSet.CoreV1().Pods(uniqueNamespace).List(context.TODO(), metav1.ListOptions{}) + + if err != nil { + return fmt.Errorf("failed to list pods: %s", err.Error()) + } + + // Wait for pods to update + if !checkIfAnnotationExists(clientSet, resourcePods, annotations) { + return fmt.Errorf("missing annotations: %v", annotations) + } + + return nil +} +func annotationExists(annotations map[string]string, key string) bool { + _, exists := annotations[key] + return exists +} + +func setupFunction(t *testing.T, namespace string, apps []string) (*kubernetes.Clientset, string) { + t.Helper() + clientSet := setupTest(t) + newUUID := uuid.New() + uniqueNamespace := fmt.Sprintf(namespace+"-%s", newUUID.String()) + if err := createNamespaceAndApplyResources(t, clientSet, uniqueNamespace, apps); err != nil { + t.Fatalf("Failed to create/apply resoures on namespace: %v", err) + } + t.Cleanup(func() { + if err := deleteNamespaceAndResources(clientSet, uniqueNamespace, apps); err != nil { + t.Fatalf("Failed to delete namespaces/resources: %v", err) + } + }) + + return clientSet, uniqueNamespace +} + +func numberOfRevisions(deploymentName string, namespace string) int { + + numOfRevisions := 0 + i := 0 + for { + // Execute the kubectl rollout history command + cmd := exec.Command("kubectl", "rollout", "history", "deployment", deploymentName, "-n", namespace, "--revision", strconv.Itoa(i)) + err := cmd.Run() + if err != nil { + break + } + numOfRevisions++ + i++ + } + return numOfRevisions - 1 //don't want to count the first +} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_namespace_test.go b/integration-tests/manifests/automonitor/validate_automonitor_namespace_test.go new file mode 100644 index 000000000..b9dcbff36 --- /dev/null +++ b/integration-tests/manifests/automonitor/validate_automonitor_namespace_test.go @@ -0,0 +1,446 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package annotations + +import ( + "context" + "crypto/rand" + "encoding/json" + "fmt" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/uuid" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/amazon-cloudwatch-agent-operator/integration-tests/util" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" +) + +func TestAllLanguagesNamespace(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("namespace-all-languages-%d", randomNumber) + + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + DotNet: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + NodeJS: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + startTime := time.Now() + + updateTheOperator(t, clientSet, string(jsonStr)) + if !checkNameSpaceAnnotations(t, clientSet, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation, injectPythonAnnotation, autoAnnotatePythonAnnotation, injectDotNetAnnotation, autoAnnotateDotNetAnnotation, injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, uniqueNamespace, startTime) { + t.Error("Missing Languages annotations") + } +} + +func TestJavaOnlyNamespace(t *testing.T) { + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("namespace-java-only-%d", randomNumber) + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + if !checkNameSpaceAnnotations(t, clientSet, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, uniqueNamespace, startTime) { + t.Error("Missing Java annotations") + } +} + +func TestPythonOnlyNamespace(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("namespace-python-only-%d", randomNumber) + if err := createNamespace(clientSet, uniqueNamespace); err != nil { + t.Fatalf("Failed to create/apply resoures on namespace: %v", err) + } + + defer func() { + if err := deleteNamespace(clientSet, uniqueNamespace); err != nil { + t.Fatalf("Failed to delete namespace: %v", err) + } + }() + + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + startTime := time.Now() + + updateTheOperator(t, clientSet, string(jsonStr)) + + if !checkNameSpaceAnnotations(t, clientSet, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, uniqueNamespace, startTime) { + t.Error("Missing Python annotations") + } +} + +func TestDotNetOnlyNamespace(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("namespace-dotnet-only-%d", randomNumber) + if err := createNamespace(clientSet, uniqueNamespace); err != nil { + t.Fatalf("Failed to create/apply resoures on namespace: %v", err) + } + + defer func() { + if err := deleteNamespace(clientSet, uniqueNamespace); err != nil { + t.Fatalf("Failed to delete namespace: %v", err) + } + }() + + annotationConfig := auto.AnnotationConfig{ + DotNet: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + startTime := time.Now() + + updateTheOperator(t, clientSet, string(jsonStr)) + + if !checkNameSpaceAnnotations(t, clientSet, []string{injectDotNetAnnotation, autoAnnotateDotNetAnnotation}, uniqueNamespace, startTime) { + t.Error("Missing DotNet annotations") + } +} + +func TestNodeJSOnlyNamespace(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("namespace-nodejs-only-%d", randomNumber) + if err := createNamespace(clientSet, uniqueNamespace); err != nil { + t.Fatalf("Failed to create/apply resoures on namespace: %v", err) + } + + defer func() { + if err := deleteNamespace(clientSet, uniqueNamespace); err != nil { + t.Fatalf("Failed to delete namespace: %v", err) + } + }() + + annotationConfig := auto.AnnotationConfig{ + NodeJS: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + startTime := time.Now() + + updateTheOperator(t, clientSet, string(jsonStr)) + + if !checkNameSpaceAnnotations(t, clientSet, []string{injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, uniqueNamespace, startTime) { + t.Error("Missing nodejs annotations") + } +} + +// Multiple resources on the same namespace should all get annotations +func TestAnnotationsOnMultipleResources(t *testing.T) { + + clientSet := setupTest(t) + newUUID := uuid.New() + uniqueNamespace := fmt.Sprintf("multiple-resources-%s", newUUID.String()) + + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + DaemonSets: []string{filepath.Join(uniqueNamespace, daemonSetName)}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, + }, + Python: auto.AnnotationResources{}, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + if err != nil { + t.Errorf("Failed to get deployment app: %s", err.Error()) + } + + if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, true); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } + if err := checkResourceAnnotations(t, clientSet, "daemonset", uniqueNamespace, daemonSetName, sampleDaemonsetYamlRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, true); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } + if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } + +} + +func TestAutoAnnotationForManualAnnotationRemoval(t *testing.T) { + clientSet, uniqueNamespace := setupFunction(t, "manual-annotation-removal", []string{sampleDeploymentYamlNameRelPath}) + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + }, + Python: auto.AnnotationResources{}, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + if err != nil { + t.Errorf("Failed to get deployment app: %s", err.Error()) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + for { + deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + t.Fatalf("Deployment %s not found in namespace %s\n", deploymentName, uniqueNamespace) + } + t.Fatal("Error getting deployment") + } + + if deployment.Status.AvailableReplicas == *deployment.Spec.Replicas && deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas { + if deployment.Status.Replicas == deployment.Status.AvailableReplicas { + fmt.Println("All pods are fully ready and no pods are terminating.") + break + } + } + + // Sleep for a short interval before checking again + time.Sleep(5 * time.Second) + } + deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Error getting deployment: %v\n", err) + } + + //Removing all annotations + deployment.ObjectMeta.Annotations = nil + _, err = clientSet.AppsV1().Deployments(uniqueNamespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + if err != nil { + fmt.Printf("Error updating deployment: %v\n", err) + os.Exit(1) + } + + err = util.WaitForNewPodCreation(clientSet, deployment, startTime) + if err != nil { + t.Fatalf("Error waiting for pod creation: %v\n", err) + } + + deploymentPods, err := clientSet.CoreV1().Pods(uniqueNamespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("Error listing pods: %v\n", err) + } + //Check if operator has added back the annotations + checkIfAnnotationExists(clientSet, deploymentPods, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}) + +} + +// Creating two apps - First app is annotated +// Second app is not annotated but on the same namespace as the first app +// Annotate the namespace of the apps and make sure only the non annotated app was restarted +// Also tests if a resource is manually annotated and now its namespace is added for auto annotation +// the resource should not be modified and should not be restarted (auto-annotation annotation does not exist) +func TestOnlyNonAnnotatedAppsShouldBeRestarted(t *testing.T) { + + clientSet, uniqueNamespace := setupFunction(t, "non-annotated", []string{sampleDeploymentYamlNameRelPath, sampleNginxAppYamlNameRelPath}) + startTime := time.Now() + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + }, + Python: auto.AnnotationResources{}, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + updateTheOperator(t, clientSet, string(jsonStr)) + if err != nil { + t.Errorf("Failed to get deployment app: %s", err.Error()) + } + deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + if err != nil { + fmt.Printf("Error retrieving deployment: %v\n", err) + os.Exit(1) + } + nginxDeployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), nginxDeploymentName, metav1.GetOptions{}) + if err != nil { + fmt.Printf("Error retrieving deployment: %v\n", err) + os.Exit(1) + } + err = util.WaitForNewPodCreation(clientSet, deployment, startTime) + if err != nil { + t.Fatal("Error waiting for pod creation: ", err) + } + + if annotationExists(nginxDeployment.Annotations, autoAnnotateJavaAnnotation) { + t.Fatal("Auto-annotation annotation should not exist") + } + + numOfRevisions := numberOfRevisions(nginxDeploymentName, uniqueNamespace) + if numOfRevisions > 1 { + t.Fatal("Nginx was restarted") //should not be restarted since it already had annotations + } + numOfRevisions = numberOfRevisions(deploymentName, uniqueNamespace) + if numOfRevisions != 2 { + t.Fatal("Sample-deployment should have been restarted") //should not be restarted since it already had annotations + } + +} + +// Test if a resource is auto annotated and now its namespace is added for auto annotation +// the resource should not be restarted +func TestAlreadyAutoAnnotatedResourceShouldNotRestart(t *testing.T) { + + clientSet, uniqueNamespace := setupFunction(t, "already-annotated", []string{sampleDeploymentYamlNameRelPath}) + startTime := time.Now() + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + }, + Python: auto.AnnotationResources{}, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + updateTheOperator(t, clientSet, string(jsonStr)) + if err != nil { + t.Errorf("Failed to get deployment app: %s", err.Error()) + } + deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + if err != nil { + fmt.Printf("Error retrieving deployment: %v\n", err) + os.Exit(1) + } + + err = util.WaitForNewPodCreation(clientSet, deployment, startTime) + if err != nil { + t.Fatalf("Error waiting for pod creation: %v\n", err) + } + fmt.Println("Done checking deployment") + //adding deployment's namespace to get auto annotated + annotationConfig = auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{uniqueNamespace}, + Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, + }, + Python: auto.AnnotationResources{}, + } + jsonStr, err = json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + fmt.Println("Right before update operator", startTime) + updateTheOperator(t, clientSet, string(jsonStr)) + fmt.Println("Right after update operator", startTime) + + //number of revisions should not be greater than 2 + //first one is for creation second one is for the first operator change and third one should not exist (even with the second operator change) + numOfRevisions := numberOfRevisions(deploymentName, uniqueNamespace) + if numOfRevisions > 2 { + t.Fatal("Sample-deployment should not have been restarted after second operator update") //should not be restarted since it already had annotations + } + +} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_statefulset_test.go b/integration-tests/manifests/automonitor/validate_automonitor_statefulset_test.go new file mode 100644 index 000000000..4f8ccea04 --- /dev/null +++ b/integration-tests/manifests/automonitor/validate_automonitor_statefulset_test.go @@ -0,0 +1,191 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package annotations + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "math/big" + "path/filepath" + "testing" + "time" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" +) + +func TestAllLanguagesStatefulSet(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("statefulset-namespace-all-languages-%d", randomNumber) + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, + }, + DotNet: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, + }, + NodeJS: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation, injectPythonAnnotation, autoAnnotatePythonAnnotation, injectDotNetAnnotation, autoAnnotateDotNetAnnotation, injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } +} + +func TestJavaOnlyStatefulSet(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("statefulset-java-only-%d", randomNumber) + + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } +} + +func TestPythonOnlyStatefulSet(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("statefulset-namespace-python-only-%d", randomNumber) + annotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{""}, + }, + Python: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + + if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } +} + +func TestDotNetOnlyStatefulSet(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("statefulset-namespace-dotnet-only-%d", randomNumber) + annotationConfig := auto.AnnotationConfig{ + DotNet: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + + if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectDotNetAnnotation, autoAnnotateDotNetAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } +} +func TestNodeJSOnlyStatefulSet(t *testing.T) { + + clientSet := setupTest(t) + randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) + if err != nil { + panic(err) + } + randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace + uniqueNamespace := fmt.Sprintf("statefulset-namespace-nodejs-only-%d", randomNumber) + annotationConfig := auto.AnnotationConfig{ + NodeJS: auto.AnnotationResources{ + Namespaces: []string{""}, + DaemonSets: []string{""}, + Deployments: []string{""}, + StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, + }, + } + jsonStr, err := json.Marshal(annotationConfig) + if err != nil { + t.Error("Error:", err) + } + + startTime := time.Now() + updateTheOperator(t, clientSet, string(jsonStr)) + + if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { + t.Fatalf("Failed annotation check: %s", err.Error()) + } +} diff --git a/internal/webhook/namespacemutation/webhookhandler.go b/internal/webhook/namespacemutation/webhookhandler.go index 3966b9b5a..00d85e780 100644 --- a/internal/webhook/namespacemutation/webhookhandler.go +++ b/internal/webhook/namespacemutation/webhookhandler.go @@ -20,10 +20,10 @@ var _ admission.Handler = (*handler)(nil) type handler struct { decoder *admission.Decoder annotationMutators *auto.AnnotationMutators - monitor *auto.Monitor + monitor auto.MonitorInterface } -func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators, monitor *auto.Monitor) admission.Handler { +func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators, monitor auto.MonitorInterface) admission.Handler { return &handler{ decoder: decoder, annotationMutators: annotationMutators, @@ -38,7 +38,7 @@ func (h *handler) Handle(_ context.Context, req admission.Request) admission.Res return admission.Errored(http.StatusBadRequest, err) } - if h.annotationMutators.IsMutated(namespace) && !h.monitor.AnyCustomSelectorDefined() { + if h.annotationMutators.IsManaged(namespace) && !h.monitor.AnyCustomSelectorDefined() { h.annotationMutators.MutateObject(namespace) } else { // do not need to pass in oldObj because it's only used to check for workload pod template diff diff --git a/internal/webhook/namespacemutation/webhookhandler_test.go b/internal/webhook/namespacemutation/webhookhandler_test.go index 4124c0146..98b9a5e44 100644 --- a/internal/webhook/namespacemutation/webhookhandler_test.go +++ b/internal/webhook/namespacemutation/webhookhandler_test.go @@ -39,9 +39,8 @@ func TestHandle(t *testing.T) { logr.Logger{}, autoAnnotationConfig, instrumentation.NewTypeSet(instrumentation.TypeJava), - nil, ) - h := NewWebhookHandler(decoder, mutators, nil) + h := NewWebhookHandler(decoder, mutators, auto.NoopMonitor{}) for _, testCase := range []struct { req admission.Request name string @@ -76,6 +75,7 @@ func TestHandle(t *testing.T) { expected: http.StatusOK, allowed: true, }, + {}, } { t.Run(testCase.name, func(t *testing.T) { // test diff --git a/internal/webhook/workloadmutation/webhookhandler.go b/internal/webhook/workloadmutation/webhookhandler.go index 2b2d0c0be..09e39970a 100644 --- a/internal/webhook/workloadmutation/webhookhandler.go +++ b/internal/webhook/workloadmutation/webhookhandler.go @@ -33,11 +33,11 @@ type WebhookHandler interface { type workloadMutationWebhook struct { decoder *admission.Decoder annotationMutators *auto.AnnotationMutators - monitor *auto.Monitor + monitor auto.MonitorInterface } // NewWebhookHandler creates a new WorkloadWebhookHandler. -func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators, monitor *auto.Monitor) WebhookHandler { +func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators, monitor auto.MonitorInterface) WebhookHandler { return &workloadMutationWebhook{ decoder: decoder, annotationMutators: annotationMutators, @@ -74,7 +74,7 @@ func (p *workloadMutationWebhook) Handle(_ context.Context, req admission.Reques } // preserve backwards compatability. - if p.annotationMutators.IsMutated(obj) && !p.monitor.AnyCustomSelectorDefined() { + if p.annotationMutators.IsManaged(obj) && !p.monitor.AnyCustomSelectorDefined() { p.annotationMutators.MutateObject(obj) } else { p.monitor.MutateObject(oldObj, obj) diff --git a/internal/webhook/workloadmutation/webhookhandler_test.go b/internal/webhook/workloadmutation/webhookhandler_test.go index 5bdc48926..f1978eb7a 100644 --- a/internal/webhook/workloadmutation/webhookhandler_test.go +++ b/internal/webhook/workloadmutation/webhookhandler_test.go @@ -128,9 +128,8 @@ func TestHandle(t *testing.T) { logr.Logger{}, autoAnnotationConfig, instrumentation.NewTypeSet(instrumentation.TypeJava), - nil, ) - injector := NewWebhookHandler(decoder, mutators) + injector := NewWebhookHandler(decoder, mutators, auto.NoopMonitor{}) // test res := injector.Handle(context.Background(), tt.req) diff --git a/main.go b/main.go index ef9495647..d3e11fdde 100644 --- a/main.go +++ b/main.go @@ -289,6 +289,7 @@ func main() { decoder := admission.NewDecoder(mgr.GetScheme()) + // TODO handle case where auto monitor is enabled but auto annotation config is not specified if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" || autoAnnotationConfigStr == "" { setupLog.Info("Auto-annotation is disabled") } else { @@ -304,7 +305,7 @@ func main() { if err = json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-annotation config") } else { - // TODO marshal/unmarshal monitor config + // TODO handle case where auto monitor is enabled but auto annotation config is not specified setupLog.Info("Test!") monitor := createMonitorFromConfig(autoMonitorConfigStr, ctx) @@ -370,7 +371,7 @@ func main() { } } -func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context) *auto.Monitor { +func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context) auto.MonitorInterface { var monitorConfig *auto.MonitorConfig var monitor *auto.Monitor if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { @@ -381,12 +382,14 @@ func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context) * } k8sConfig, err := rest.InClusterConfig() if err != nil { - panic("TODO: Implement handling for failed clientset creation") + setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") + return auto.NoopMonitor{} } clientSet, err := kubernetes.NewForConfig(k8sConfig) if err != nil { - panic("TODO: Implement handling for failed clientset creation") + setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") + return auto.NoopMonitor{} } monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet) } diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index da57102dc..b0bcf9f94 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -42,8 +42,8 @@ type AnnotationMutators struct { cfg *AnnotationConfig } -// IsMutated returns if obj is specified in the config -func (m *AnnotationMutators) IsMutated(obj client.Object) bool { +// IsManaged returns if AnnotationMutators would ever mutate the object. +func (m *AnnotationMutators) IsManaged(obj client.Object) bool { return len(m.cfg.GetObjectLanguagesToAnnotate(obj)) > 0 } diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index eb9638cc3..a8e0f5250 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -21,12 +21,27 @@ import ( var logger = logf.Log.WithName("auto_monitor") +type MonitorInterface interface { + MutateObject(oldObj client.Object, obj client.Object) map[string]string + AnyCustomSelectorDefined() bool +} + type Monitor struct { serviceInformer cache.SharedIndexInformer ctx context.Context config MonitorConfig } +type NoopMonitor struct{} + +func (n NoopMonitor) MutateObject(oldObj client.Object, obj client.Object) map[string]string { + return map[string]string{} +} + +func (n NoopMonitor) AnyCustomSelectorDefined() bool { + return false +} + func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface) *Monitor { logger.Info("AutoMonitor starting...") @@ -60,11 +75,12 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet if err != nil { return nil } - factory.Start(ctx.Done()) // runs in background + factory.Start(ctx.Done()) synced := factory.WaitForCacheSync(ctx.Done()) for v, ok := range synced { if !ok { _, _ = fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) + // TODO: handle bad cache sync panic("TODO: handle bad cache sync") } } @@ -124,7 +140,7 @@ func (m Monitor) MutateObject(oldObj client.Object, obj client.Object) map[strin // custom selector takes precedence over service selector if customSelectLanguages, selected := m.CustomSelected(obj); selected { logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) - return mutate(obj, customSelectLanguages) + mutate(obj, customSelectLanguages) } if !m.config.MonitorAllServices { @@ -136,7 +152,6 @@ func (m Monitor) MutateObject(oldObj client.Object, obj client.Object) map[strin } objectLabels := getTemplateSpecLabels(obj) - // if object is not workload, return err for _, informerObj := range m.serviceInformer.GetStore().List() { service := informerObj.(*corev1.Service) if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 { @@ -152,15 +167,17 @@ func (m Monitor) MutateObject(oldObj client.Object, obj client.Object) map[strin return nil } -func mutate(obj client.Object, languagesToMonitor instrumentation.TypeSet) map[string]string { - podTemplate := getPodTemplate(obj) - var annotations map[string]string - - if podTemplate == nil { - annotations = obj.GetAnnotations() +// mutate obj. If object is a workload, mutate the pod template. otherwise, mutate the object's annotations itself. +func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet) map[string]string { + var obj metav1.Object + podTemplate := getPodTemplate(object) + if podTemplate != nil { + obj = podTemplate } else { - annotations = podTemplate.GetAnnotations() + obj = object } + + annotations := obj.GetAnnotations() if annotations == nil { annotations = make(map[string]string) } @@ -199,10 +216,6 @@ func (m Monitor) excluded(obj client.Object) bool { return false } -func (m Monitor) Enabled() bool { - return m.config.MonitorAllServices -} - func (m Monitor) AnyCustomSelectorDefined() bool { for _, t := range instrumentation.SupportedTypes() { resources := m.config.CustomSelector.getResources(t) diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index f7597bbe0..818506213 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -11,12 +11,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" "testing" ) -var logger = logf.Log.WithName("auto_monitor_tests") - func TestMonitor_Selected(t *testing.T) { logger.Info("Starting testmonitor tests") @@ -376,3 +373,150 @@ func TestUnmarshal(t *testing.T) { assert.NoError(t, err) assert.Equal(t, instrumentation.TypeSet{instrumentation.TypeNodeJS: nil, instrumentation.TypeJava: nil, instrumentation.TypePython: nil}, set) } + +func Test_isWorkloadPodTemplateMutated(t *testing.T) { + deploy := &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Image: "nginx:1.14.2"}}, + }, + }, + }, + } + tests := []struct { + name string + oldObject client.Object + object client.Object + want bool + }{ + {"nil objects", nil, nil, true}, + {"identical deployments", deploy.DeepCopy(), deploy.DeepCopy(), false}, + {"changed pod template", deploy.DeepCopy(), &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Image: "nginx:1.15.0"}}, + }, + }, + }, + }, true}, + {"non-workload", &corev1.ConfigMap{}, &corev1.ConfigMap{}, true}, + {"create (oldObject nil)", nil, deploy.DeepCopy(), true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isWorkloadPodTemplateMutated(tt.oldObject, tt.object)) + }) + } +} + +func Test_getPodTemplate(t *testing.T) { + template := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Image: "nginx"}}, + }, + } + + tests := []struct { + name string + obj client.Object + want *corev1.PodTemplateSpec + }{ + {"deployment", &appsv1.Deployment{Spec: appsv1.DeploymentSpec{Template: template}}, &template}, + {"statefulset", &appsv1.StatefulSet{Spec: appsv1.StatefulSetSpec{Template: template}}, &template}, + {"daemonset", &appsv1.DaemonSet{Spec: appsv1.DaemonSetSpec{Template: template}}, &template}, + {"other", &corev1.Pod{}, nil}, + {"nil", nil, nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, getPodTemplate(tt.obj)) + }) + } +} + +func Test_mutate(t *testing.T) { + tests := []struct { + name string + obj client.Object + languagesToMonitor instrumentation.TypeSet + wantObjAnnotations map[string]string + wantMutated map[string]string + }{ + { + name: "deployment - java only", + obj: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{}, + }, + }, + languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, + wantObjAnnotations: buildAnnotations("java"), + wantMutated: buildAnnotations("java"), + }, + { + name: "deployment - java and python", + obj: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{}, + }, + }, + languagesToMonitor: instrumentation.TypeSet{ + "java": struct{}{}, + "python": struct{}{}, + }, + wantObjAnnotations: mergeMaps(buildAnnotations("java"), buildAnnotations("python")), + wantMutated: mergeMaps(buildAnnotations("java"), buildAnnotations("python")), + }, + { + name: "remove python instrumentation", + obj: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: buildAnnotations("python"), + }, + }, + }, + }, + languagesToMonitor: instrumentation.TypeSet{}, + wantObjAnnotations: map[string]string{}, + wantMutated: buildAnnotations("python"), + }, + { + name: "remove one of two languages", + obj: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: mergeMaps(buildAnnotations("python"), buildAnnotations("java")), + }, + }, + }, + }, + languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, + wantObjAnnotations: buildAnnotations("java"), + wantMutated: buildAnnotations("python"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotMutated := mutate(tt.obj, tt.languagesToMonitor) + assert.Equal(t, tt.wantObjAnnotations, tt.obj.GetAnnotations()) + assert.Equal(t, tt.wantMutated, gotMutated) + }) + } +} +func mergeMaps(maps ...map[string]string) map[string]string { + result := make(map[string]string) + for _, m := range maps { + for k, v := range m { + result[k] = v + } + } + return result +} From 24c29cd2b97ee0e76f85a81a5e41c9d5a34e65c0 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 13 Mar 2025 19:06:43 -0400 Subject: [PATCH 07/63] works --- .../validate_annotation_methods.go | 1 - .../validate_automonitor_daemonset_test.go | 41 -- .../validate_automonitor_deployment_test.go | 440 ++++++++++------- .../validate_automonitor_methods.go | 439 ++++++++--------- .../validate_automonitor_namespace_test.go | 446 ------------------ .../validate_automonitor_statefulset_test.go | 191 -------- .../manifests/sample-deployment-service.yaml | 11 + integration-tests/util/util.go | 87 ++-- main.go | 2 +- pkg/instrumentation/annotationtype.go | 8 + pkg/instrumentation/auto/auto_monitor.go | 72 ++- pkg/instrumentation/auto/auto_monitor_test.go | 20 + 12 files changed, 633 insertions(+), 1125 deletions(-) delete mode 100644 integration-tests/manifests/automonitor/validate_automonitor_daemonset_test.go delete mode 100644 integration-tests/manifests/automonitor/validate_automonitor_namespace_test.go delete mode 100644 integration-tests/manifests/automonitor/validate_automonitor_statefulset_test.go create mode 100644 integration-tests/manifests/sample-deployment-service.yaml diff --git a/integration-tests/manifests/annotations/validate_annotation_methods.go b/integration-tests/manifests/annotations/validate_annotation_methods.go index b2dcf209c..7fdaaaf51 100644 --- a/integration-tests/manifests/annotations/validate_annotation_methods.go +++ b/integration-tests/manifests/annotations/validate_annotation_methods.go @@ -240,7 +240,6 @@ func checkIfAnnotationExists(clientset *kubernetes.Clientset, pods *v1.PodList, //check if all pods are in the Running phase if !util.CheckIfPodsAreRunning(currentPods) { - continue } diff --git a/integration-tests/manifests/automonitor/validate_automonitor_daemonset_test.go b/integration-tests/manifests/automonitor/validate_automonitor_daemonset_test.go deleted file mode 100644 index bcd43ac0e..000000000 --- a/integration-tests/manifests/automonitor/validate_automonitor_daemonset_test.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package annotations - -import ( - "crypto/rand" - "encoding/json" - "fmt" - "math/big" - "path/filepath" - "testing" - "time" - - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" -) - -func TestAutoMonitorEnabled(t *testing.T) { - setupFunction(t, "auto-monitor-test", []string{sampleDeploymentYamlNameRelPath}) - clientSet := setupTest(t) - - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{filepath.Join(uniqueNamespace, daemonSetName)}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - } - updateAnnotationConfig(annotationConfig) - - if err := checkResourceAnnotations(t, clientSet, "daemonset", uniqueNamespace, daemonSetName, sampleDaemonsetYamlRelPath, startTime, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } - -} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index 27697e61d..c0c41c7db 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -3,210 +3,298 @@ package annotations import ( - "crypto/rand" - "encoding/json" "fmt" - "math/big" - "path/filepath" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" + "github.com/stretchr/testify/assert" "testing" "time" - - "github.com/stretchr/testify/assert" - - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" ) -func TestAllLanguagesDeployment(t *testing.T) { +const sampleDeploymentServiceYaml = "../sample-deployment-service.yaml" - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("deployment-namespace-all-languages-%d", randomNumber) - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - StatefulSets: []string{""}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - StatefulSets: []string{""}, - }, - DotNet: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - StatefulSets: []string{""}, - }, - NodeJS: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - StatefulSets: []string{""}, - }, +var ( + defaultAnnotationConfig = auto.AnnotationConfig{ + Java: auto.AnnotationResources{}, + Python: auto.AnnotationResources{}, + DotNet: auto.AnnotationResources{}, + NodeJS: auto.AnnotationResources{}, } - jsonStr, err := json.Marshal(annotationConfig) - assert.Nil(t, err) - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) + none []string +) - if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation, injectPythonAnnotation, autoAnnotatePythonAnnotation, injectDotNetAnnotation, autoAnnotateDotNetAnnotation, injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } +func TestServiceThenDeployment(t *testing.T) { + helper := NewTestHelper(t, true) -} + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) -func TestJavaOnlyDeployment(t *testing.T) { + // Update operator + helper.UpdateAnnotationConfig(defaultAnnotationConfig) - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("deployment-namespace-java-only-%d", randomNumber) - - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - StatefulSets: []string{""}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Errorf("Failed to marshal: %v\n", err) - } - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) + helper.UpdateMonitorConfig(auto.MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + AutoRestart: false, + }) + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + assert.NoError(t, err) - if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, none) + assert.NoError(t, err) } -func TestPythonOnlyDeployment(t *testing.T) { +// create deployment, create service, should not annotate anything +func TestDeploymentThenServiceAutoRestartDisabled(t *testing.T) { + helper := NewTestHelper(t, true) - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("deployment-namespace-python-only-%d", randomNumber) - - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - StatefulSets: []string{""}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } + namespace := helper.Initialize("test-namespace", []string{}) - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - if err != nil { - t.Errorf("Failed to get deployment app: %s", err.Error()) - } + // Update operator + helper.UpdateAnnotationConfig(defaultAnnotationConfig) - if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } + helper.UpdateMonitorConfig(auto.MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + AutoRestart: false, + }) -} -func TestDotNetOnlyDeployment(t *testing.T) { + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("deployment-namespace-dotnet-only-%d", randomNumber) - - annotationConfig := auto.AnnotationConfig{ - DotNet: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - StatefulSets: []string{""}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } + // Check annotations + fmt.Println("Checking if sample-deployment exists") - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - if err != nil { - t.Errorf("Failed to get deployment app: %s", err.Error()) - } + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + assert.NoError(t, err) - if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectDotNetAnnotation, autoAnnotateDotNetAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } + err = helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) + assert.NoError(t, err) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + assert.NoError(t, err) } -func TestNodeJSOnlyDeployment(t *testing.T) { +func TestDeploymentThenServiceAutoRestartEnabled(t *testing.T) { + helper := NewTestHelper(t, true) - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("deployment-namespace-nodejs-only-%d", randomNumber) - - annotationConfig := auto.AnnotationConfig{ - NodeJS: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - StatefulSets: []string{""}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - t.Error("Error:", err) + namespace := helper.Initialize("test-namespace", []string{}) - } + // Update operator + helper.UpdateAnnotationConfig(defaultAnnotationConfig) - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - if err != nil { - t.Errorf("Failed to get deployment app: %s", err.Error()) - } + helper.UpdateMonitorConfig(auto.MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + AutoRestart: true, + }) - if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + assert.NoError(t, err) + + // Check annotations + fmt.Println("Checking if sample-deployment exists") + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + assert.NoError(t, err) + helper.startTime = time.Now() + err = helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) + assert.NoError(t, err) + + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, none) + assert.NoError(t, err) } + +//func TestAllLanguagesDeployment(t *testing.T) { +// +// clientSet := setupTest(t) +// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) +// if err != nil { +// panic(err) +// } +// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace +// uniqueNamespace := fmt.Sprintf("deployment-namespace-all-languages-%d", randomNumber) +// annotationConfig := auto.AnnotationConfig{ +// Java: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, +// StatefulSets: []string{""}, +// }, +// Python: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, +// StatefulSets: []string{""}, +// }, +// DotNet: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, +// StatefulSets: []string{""}, +// }, +// NodeJS: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, +// StatefulSets: []string{""}, +// }, +// } +// jsonStr, err := json.Marshal(annotationConfig) +// assert.Nil(t, err) +// +// startTime := time.Now() +// updateOperatorConfig(t, clientSet, string(jsonStr)) +// +// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation, injectPythonAnnotation, autoAnnotatePythonAnnotation, injectDotNetAnnotation, autoAnnotateDotNetAnnotation, injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { +// t.Fatalf("Failed annotation check: %s", err.Error()) +// } +// +//} +// +//func TestJavaOnlyDeployment(t *testing.T) { +// +// clientSet := setupTest(t) +// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) +// if err != nil { +// panic(err) +// } +// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace +// uniqueNamespace := fmt.Sprintf("deployment-namespace-java-only-%d", randomNumber) +// +// annotationConfig := auto.AnnotationConfig{ +// Java: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, +// StatefulSets: []string{""}, +// }, +// Python: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{""}, +// StatefulSets: []string{""}, +// }, +// } +// jsonStr, err := json.Marshal(annotationConfig) +// if err != nil { +// t.Errorf("Failed to marshal: %v\n", err) +// } +// startTime := time.Now() +// updateOperatorConfig(t, clientSet, string(jsonStr)) +// +// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, false); err != nil { +// t.Fatalf("Failed annotation check: %s", err.Error()) +// } +//} +// +//func TestPythonOnlyDeployment(t *testing.T) { +// +// clientSet := setupTest(t) +// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) +// if err != nil { +// panic(err) +// } +// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace +// uniqueNamespace := fmt.Sprintf("deployment-namespace-python-only-%d", randomNumber) +// +// annotationConfig := auto.AnnotationConfig{ +// Java: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{""}, +// StatefulSets: []string{""}, +// }, +// Python: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, +// StatefulSets: []string{""}, +// }, +// } +// jsonStr, err := json.Marshal(annotationConfig) +// if err != nil { +// t.Error("Error:", err) +// } +// +// startTime := time.Now() +// updateOperatorConfig(t, clientSet, string(jsonStr)) +// if err != nil { +// t.Errorf("Failed to get deployment app: %s", err.Error()) +// } +// +// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, false); err != nil { +// t.Fatalf("Failed annotation check: %s", err.Error()) +// } +// +//} +//func TestDotNetOnlyDeployment(t *testing.T) { +// +// clientSet := setupTest(t) +// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) +// if err != nil { +// panic(err) +// } +// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace +// uniqueNamespace := fmt.Sprintf("deployment-namespace-dotnet-only-%d", randomNumber) +// +// annotationConfig := auto.AnnotationConfig{ +// DotNet: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, +// StatefulSets: []string{""}, +// }, +// } +// jsonStr, err := json.Marshal(annotationConfig) +// if err != nil { +// t.Error("Error:", err) +// } +// +// startTime := time.Now() +// updateOperatorConfig(t, clientSet, string(jsonStr)) +// if err != nil { +// t.Errorf("Failed to get deployment app: %s", err.Error()) +// } +// +// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectDotNetAnnotation, autoAnnotateDotNetAnnotation}, false); err != nil { +// t.Fatalf("Failed annotation check: %s", err.Error()) +// } +// +//} +// +//func TestNodeJSOnlyDeployment(t *testing.T) { +// +// clientSet := setupTest(t) +// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) +// if err != nil { +// panic(err) +// } +// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace +// uniqueNamespace := fmt.Sprintf("deployment-namespace-nodejs-only-%d", randomNumber) +// +// annotationConfig := auto.AnnotationConfig{ +// NodeJS: auto.AnnotationResources{ +// Namespaces: []string{""}, +// DaemonSets: []string{""}, +// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, +// StatefulSets: []string{""}, +// }, +// } +// jsonStr, err := json.Marshal(annotationConfig) +// if err != nil { +// t.Error("Error:", err) +// t.Error("Error:", err) +// +// } +// +// startTime := time.Now() +// updateOperatorConfig(t, clientSet, string(jsonStr)) +// if err != nil { +// t.Errorf("Failed to get deployment app: %s", err.Error()) +// } +// +// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { +// t.Fatalf("Failed annotation check: %s", err.Error()) +// } +// +//} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go index b2dcf209c..d85dbe9c7 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -4,7 +4,10 @@ package annotations import ( "context" + "encoding/json" "fmt" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" + "github.com/stretchr/testify/assert" "os" "os/exec" "path/filepath" @@ -31,7 +34,6 @@ const ( autoAnnotatePythonAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-python" injectDotNetAnnotation = "instrumentation.opentelemetry.io/inject-dotnet" autoAnnotateDotNetAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-dotnet" - injectNodeJSAnnotation = "instrumentation.opentelemetry.io/inject-nodejs" autoAnnotateNodeJSAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-nodejs" @@ -39,75 +41,110 @@ const ( nginxDeploymentName = "nginx" statefulSetName = "sample-statefulset" amazonCloudwatchNamespace = "amazon-cloudwatch" + daemonSetName = "sample-daemonset" + amazonControllerManager = "amazon-cloudwatch-observability-controller-manager" - daemonSetName = "sample-daemonset" + sampleDaemonsetYamlRelPath = "../sample-daemonset.yaml" + sampleDeploymentYamlNameRelPath = "../sample-deployment.yaml" + sampleNginxAppYamlNameRelPath = "../../java/sample-deployment-java.yaml" + sampleStatefulsetYamlNameRelPath = "../sample-statefulset.yaml" - amazonControllerManager = "cloudwatch-controller-manager" + timoutDuration = 2 * time.Minute + numberOfRetries = 10 + timeBetweenRetries = 5 * time.Second +) - sampleDaemonsetYamlRelPath = "../sample-daemonset.yaml" - sampleDeploymentYamlNameRelPath = "../sample-deployment.yaml" - sampleNginxAppYamlNameRelPath = "../../java/sample-deployment-java.yaml" +type TestHelper struct { + clientSet *kubernetes.Clientset + t *testing.T + startTime time.Time + skipDelete bool +} - sampleStatefulsetYamlNameRelPath = "../sample-statefulset.yaml" - timoutDuration = 2 * time.Minute - numberOfRetries = 10 - timeBetweenRetries = 5 * time.Second -) +func NewTestHelper(t *testing.T, skipDelete bool) *TestHelper { + return &TestHelper{ + clientSet: setupTest(t), + t: t, + skipDelete: skipDelete, + } +} -func applyYAMLWithKubectl(filename, namespace string) error { +func setupTest(t *testing.T) *kubernetes.Clientset { + userHomeDir, err := os.UserHomeDir() + if err != nil { + t.Errorf("error getting user home dir: %v\n\n", err) + } + kubeConfigPath := filepath.Join(userHomeDir, ".kube", "config") + fmt.Printf("Using kubeconfig: %s\n\n", kubeConfigPath) + + kubeConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) + if err != nil { + t.Errorf("Error getting kubernetes config: %v\n\n", err) + } + + clientSet, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + t.Errorf("error getting kubernetes config: %v\n\n", err) + } + return clientSet +} + +func (h *TestHelper) ApplyYAMLWithKubectl(filename, namespace string) error { cmd := exec.Command("kubectl", "apply", "-f", filename, "-n", namespace) + fmt.Printf("Applying YAML with kubectl %s\n", cmd) return cmd.Run() } -func createNamespaceAndApplyResources(t *testing.T, clientset *kubernetes.Clientset, name string, resourceFiles []string) error { - err := createNamespace(clientset, name) +func (h *TestHelper) CreateNamespaceAndApplyResources(namespace string, resourceFiles []string) error { + fmt.Printf("Creating namespace %s\n", namespace) + err := h.CreateNamespace(namespace) if err != nil { return err } for _, file := range resourceFiles { - err = applyYAMLWithKubectl(file, name) + err = h.ApplyYAMLWithKubectl(file, namespace) if err != nil { - t.Errorf("Could not apply resources %s/%s", name, file) + h.t.Errorf("Could not apply resources %s/%s\n", namespace, file) return err } } + + for _, file := range resourceFiles { + err = h.WaitYamlWithKubectl(file, namespace) + if err != nil { + h.t.Errorf("Could not wait resources %s/%s\n", namespace, file) + } + } return nil } -func isNamespaceUpdated(clientset *kubernetes.Clientset, namespace string, startTime time.Time) bool { - //check if the namespace was updated - ns, err := clientset.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) + +func (h *TestHelper) IsNamespaceUpdated(namespace string) bool { + ns, err := h.clientSet.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) if err != nil { fmt.Printf("Failed to get namespace %s: %v\n", namespace, err) return false } - if ns.CreationTimestamp.After(startTime) || ns.ResourceVersion != "" { - return true - } - - return false + return ns.CreationTimestamp.After(h.startTime) || ns.ResourceVersion != "" } -func deleteYAMLWithKubectl(filename, namespace string) error { +func (h *TestHelper) DeleteYAMLWithKubectl(filename, namespace string) error { cmd := exec.Command("kubectl", "delete", "-f", filename, "-n", namespace) return cmd.Run() } -func deleteNamespaceAndResources(clientset *kubernetes.Clientset, name string, resourceFiles []string) error { + +func (h *TestHelper) DeleteNamespaceAndResources(name string, resourceFiles []string) error { for _, file := range resourceFiles { - err := deleteYAMLWithKubectl(file, name) - if err != nil { + if err := h.DeleteYAMLWithKubectl(file, name); err != nil { return err } } - - //delete Namespace - err := deleteNamespace(clientset, name) - return err + return h.DeleteNamespace(name) } -// Check if name space exist and if it does not we create the namespace and wait until it is fully created -func createNamespace(clientSet *kubernetes.Clientset, name string) error { - _, err := clientSet.CoreV1().Namespaces().Get(context.Background(), name, metav1.GetOptions{}) +// CreateNamespace if not already created +func (h *TestHelper) CreateNamespace(name string) error { + _, err := h.clientSet.CoreV1().Namespaces().Get(context.Background(), name, metav1.GetOptions{}) if err == nil { return nil } else if !errors.IsNotFound(err) { @@ -115,7 +152,7 @@ func createNamespace(clientSet *kubernetes.Clientset, name string) error { } namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}} - _, err = clientSet.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) + _, err = h.clientSet.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) if err != nil { return err } @@ -126,10 +163,10 @@ func createNamespace(clientSet *kubernetes.Clientset, name string) error { return fmt.Errorf("timeout reached while waiting for namespace %s to be created", name) } - _, err := clientSet.CoreV1().Namespaces().Get(context.Background(), name, metav1.GetOptions{}) + _, err := h.clientSet.CoreV1().Namespaces().Get(context.Background(), name, metav1.GetOptions{}) if err == nil { return nil - } else if !errors.IsNotFound(err) { //if any other error other than not found + } else if !errors.IsNotFound(err) { return err } @@ -137,30 +174,27 @@ func createNamespace(clientSet *kubernetes.Clientset, name string) error { } } -func deleteNamespace(clientset *kubernetes.Clientset, name string) error { - err := clientset.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{}) - return err +func (h *TestHelper) DeleteNamespace(name string) error { + return h.clientSet.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{}) } -// This function creates a namespace and checks it annotations and then deletes the namespace after check complete -func checkNameSpaceAnnotations(t *testing.T, clientSet *kubernetes.Clientset, expectedAnnotations []string, uniqueNamespace string, startTime time.Time) bool { - - if err := createNamespace(clientSet, uniqueNamespace); err != nil { - t.Fatalf("Failed to create/apply resoures on namespace: %v", err) +func (h *TestHelper) CheckNameSpaceAnnotations(expectedAnnotations []string, uniqueNamespace string) bool { + if err := h.CreateNamespace(uniqueNamespace); err != nil { + h.t.Fatalf("Failed to create/apply resources on namespace: %v", err) } defer func() { - if err := deleteNamespace(clientSet, uniqueNamespace); err != nil { - t.Fatalf("Failed to delete namespace: %v", err) + if err := h.DeleteNamespace(uniqueNamespace); err != nil { + h.t.Fatalf("Failed to delete namespace: %v", err) } }() for { - if isNamespaceUpdated(clientSet, uniqueNamespace, startTime) { + if h.IsNamespaceUpdated(uniqueNamespace) { fmt.Printf("Namespace %s has been updated.\n", uniqueNamespace) break } - elapsed := time.Since(startTime) + elapsed := time.Since(h.startTime) if elapsed >= timoutDuration { fmt.Printf("Timeout reached while waiting for namespace %s to be updated.\n", uniqueNamespace) break @@ -169,15 +203,13 @@ func checkNameSpaceAnnotations(t *testing.T, clientSet *kubernetes.Clientset, ex for i := 0; i < numberOfRetries; i++ { correct := true - ns, err := clientSet.CoreV1().Namespaces().Get(context.TODO(), uniqueNamespace, metav1.GetOptions{}) - fmt.Printf("This is the loop iteration: %v\n, these are the annotation of ns %v", i, ns) + ns, err := h.clientSet.CoreV1().Namespaces().Get(context.TODO(), uniqueNamespace, metav1.GetOptions{}) if err != nil { fmt.Println("There was an error getting namespace, ", err) return false } for _, annotation := range expectedAnnotations { - fmt.Printf("\n This is the annotation: %v and its status %v, namespace name: %v, \n", ns.ObjectMeta.Annotations, ns.Status, ns.Name) if ns.ObjectMeta.Annotations[annotation] != "true" { time.Sleep(timeBetweenRetries) correct = false @@ -193,115 +225,100 @@ func checkNameSpaceAnnotations(t *testing.T, clientSet *kubernetes.Clientset, ex return false } -// This function updates the operator and waits until it is ready -func updateOperator(t *testing.T, clientSet *kubernetes.Clientset, deployment *appsV1.Deployment, startTime time.Time) bool { - var err error - +func (h *TestHelper) UpdateOperator(deployment *appsV1.Deployment) bool { args := deployment.Spec.Template.Spec.Containers[0].Args - - deployment, err = clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) + now := time.Now() + deployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) if err != nil { - t.Errorf("Failed to get deployment: %v\n", err) + h.t.Errorf("Failed to get deployment: %v\n", err) return false } deployment.Spec.Template.Spec.Containers[0].Args = args + forceRestart(deployment) - _, err = clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + _, err = h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) if err != nil { - t.Errorf("Failed to update deployment: %v\n", err) + h.t.Errorf("Failed to update deployment: %v\n", err) return false } - err = util.WaitForNewPodCreation(clientSet, deployment, startTime) + + err = util.WaitForNewPodCreation(h.clientSet, deployment, now) if err != nil { fmt.Println("There was an error trying to wait for deployment available", err) return false } - fmt.Println("Deployment updated successfully!") + fmt.Println("Operator updated successfully!") return true } -// check if the given pods have the expected annotations -func checkIfAnnotationExists(clientset *kubernetes.Clientset, pods *v1.PodList, expectedAnnotations []string) bool { - startTime := time.Now() - for { - if time.Since(startTime) > timoutDuration { - fmt.Println("Timeout reached while waiting for annotations.") - return false - } - - //This exist to check if any pods took too long to delete and we need to list pods again - currentPods, err := clientset.CoreV1().Pods(pods.Items[0].Namespace).List(context.TODO(), metav1.ListOptions{}) - - fmt.Println("Current pods len: ", len(currentPods.Items)) - if err != nil { - fmt.Printf("Failed to list pods: %v\n", err) - return false - } +func forceRestart(deployment *appsV1.Deployment) { + annotations := deployment.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations["test-restart"] = time.Now().String() + deployment.Spec.Template.SetAnnotations(annotations) +} - //check if all pods are in the Running phase - if !util.CheckIfPodsAreRunning(currentPods) { +func (h *TestHelper) PodsAnnotationsValid(namespace string, shouldExistAnnotations []string, shouldNotExistAnnotations []string) bool { + currentPods, err := h.clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + fmt.Printf("Failed to list pods: %v\n", err) + return false + } + validAnnotations := true + for _, pod := range currentPods.Items { + fmt.Printf("Pod %s is in phase %s\n", pod.Name, pod.Status.Phase) + fmt.Println("Pod ", pod.GetAnnotations()) + if pod.Status.Phase != v1.PodRunning { continue } - - foundAllAnnotations := true - for _, pod := range currentPods.Items { - for _, annotation := range expectedAnnotations { - fmt.Printf("Checking pod %s for annotation %s in namespace %s\n", pod.Name, annotation, pod.Namespace) - - if value, exists := pod.Annotations[annotation]; !exists || value != "true" { - fmt.Printf("Pod %s does not have annotation %s with value 'true' in namespace %s\n", pod.Name, annotation, pod.Namespace) - foundAllAnnotations = false - break - } - } - if !foundAllAnnotations { + for _, annotation := range shouldExistAnnotations { + if value, exists := pod.Annotations[annotation]; !exists || value != "true" { + fmt.Println("Pod", pod.Namespace, pod.Name, " does not have annotation ", annotation) + validAnnotations = false break } } - - if foundAllAnnotations { - fmt.Println("Annotations are correct!") - return true + for _, annotation := range shouldNotExistAnnotations { + if _, exists := pod.Annotations[annotation]; exists { + fmt.Println("Pod", pod.Namespace, pod.Name, " shouldn't have annotation ", annotation) + validAnnotations = false + break + } } - - fmt.Println("Annotations not found in all pods or some pods are not in Running phase. Retrying...") - cmd := exec.Command("kubectl", "rollout", "restart", "deployment", amazonControllerManager, "-n", amazonCloudwatchNamespace) - - // Run the command and capture the output - output, err := cmd.CombinedOutput() - if err != nil { - fmt.Printf("Error restarting deployment: %v\n", err) - fmt.Printf("Output: %s\n", output) - } else { - fmt.Printf("Successfully deleted deployment: %s\n", output) + if !validAnnotations { + break } - waitCmd := exec.Command("kubectl", "wait", "--for=condition=Available", "deployment/"+amazonControllerManager, "-n", amazonCloudwatchNamespace, "--timeout=300s") + } - waitOutput, err := waitCmd.CombinedOutput() - if err != nil { - fmt.Printf("Error waiting for deployment: %v\n", err) - fmt.Printf("Output: %s\n", waitOutput) - } else { - fmt.Printf("Deployment is now available: %s\n", waitOutput) - } - time.Sleep(timeBetweenRetries) + if validAnnotations { + return true } + return false } -// Finds auto-annotation arg in operator and updates it, if not found it will be added to the end -func updateAnnotationConfig(deployment *appsV1.Deployment, jsonStr string) *appsV1.Deployment { +func (h *TestHelper) restartOperator() { + cmd := exec.Command("kubectl", "rollout", "restart", "deployment", amazonControllerManager, "-n", amazonCloudwatchNamespace) + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("Error restarting deployment: %v\nOutput: %s\n", err, output) + return + } - args := deployment.Spec.Template.Spec.Containers[0].Args - indexOfAutoAnnotationConfigString := findIndexOfPrefix("--auto-annotation-config=", args) - if indexOfAutoAnnotationConfigString < 0 { - deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--auto-annotation-config="+jsonStr) - } else { - deployment.Spec.Template.Spec.Containers[0].Args[indexOfAutoAnnotationConfigString] = "--auto-annotation-config=" + jsonStr + waitCmd := exec.Command("kubectl", "wait", "--for=condition=Available", + "deployment/"+amazonControllerManager, + "-n", amazonCloudwatchNamespace, + "--timeout=300s") + + waitOutput, err := waitCmd.CombinedOutput() + if err != nil { + fmt.Printf("Error waiting for deployment: %v\nOutput: %s\n", err, waitOutput) } - return deployment } -func findIndexOfPrefix(str string, strs []string) int { + +func (h *TestHelper) findIndexOfPrefix(str string, strs []string) int { for i, s := range strs { if strings.HasPrefix(s, str) { return i @@ -310,136 +327,124 @@ func findIndexOfPrefix(str string, strs []string) int { return -1 } -// kubernetes setup -func setupTest(t *testing.T) *kubernetes.Clientset { - userHomeDir, err := os.UserHomeDir() - if err != nil { - t.Errorf("error getting user home dir: %v\n\n", err) - } - kubeConfigPath := filepath.Join(userHomeDir, ".kube", "config") - fmt.Printf("Using kubeconfig: %s\n\n", kubeConfigPath) +func (h *TestHelper) UpdateMonitorConfig(config auto.MonitorConfig) time.Time { + jsonStr, err := json.Marshal(config) + assert.Nil(h.t, err) - kubeConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) - if err != nil { - t.Errorf("Error getting kubernetes config: %v\n\n", err) - } - - clientSet, err := kubernetes.NewForConfig(kubeConfig) + startTime := time.Now() + h.updateOperatorConfig(string(jsonStr), "--auto-monitor-config=") + return startTime +} - if err != nil { - t.Errorf("error getting kubernetes config: %v\n\n", err) - } - return clientSet +func (h *TestHelper) UpdateAnnotationConfig(config auto.AnnotationConfig) { + jsonStr, err := json.Marshal(config) + assert.Nil(h.t, err) + h.updateOperatorConfig(string(jsonStr), "--auto-annotation-config=") } -func updateTheOperator(t *testing.T, clientSet *kubernetes.Clientset, jsonStr string) { - deployment, err := clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) +func (h *TestHelper) updateOperatorConfig(jsonStr string, flag string) { + deployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) if err != nil { - t.Errorf("Error getting deployment: %v\n\n", err) + h.t.Errorf("Error getting deployment: %v\n\n", err) + return } - deployment = updateAnnotationConfig(deployment, jsonStr) - - if !updateOperator(t, clientSet, deployment, time.Now().Add(-time.Second)) { - t.Error("Failed to update Operator", deployment, deployment.Name, deployment.Spec.Template.Spec.Containers[0].Args) + args := deployment.Spec.Template.Spec.Containers[0].Args + indexOfAutoAnnotationConfigString := h.findIndexOfPrefix(flag, args) + if indexOfAutoAnnotationConfigString < 0 { + deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, flag+jsonStr) + } else { + deployment.Spec.Template.Spec.Containers[0].Args[indexOfAutoAnnotationConfigString] = flag + jsonStr } -} -func checkResourceAnnotations(t *testing.T, clientSet *kubernetes.Clientset, resourceType, uniqueNamespace, resourceName string, sampleAppYamlPath string, startTime time.Time, annotations []string, skipDelete bool) error { - if err := createNamespaceAndApplyResources(t, clientSet, uniqueNamespace, []string{sampleAppYamlPath}); err != nil { - t.Fatalf("Failed to create/apply resoures on namespace: %v", err) - return err - } - if !skipDelete { - t.Cleanup(func() { - if err := deleteNamespaceAndResources(clientSet, uniqueNamespace, []string{sampleAppYamlPath}); err != nil { - t.Fatalf("Failed to delete namespaces/resources: %v", err) - } - }) + if !h.UpdateOperator(deployment) { + h.t.Error("Failed to update Operator", deployment, deployment.Name, deployment.Spec.Template.Spec.Containers[0].Args) } +} +func (h *TestHelper) ValidateWorkloadAnnotations(resourceType, uniqueNamespace, resourceName string, shouldExist []string, shouldNotExist []string) error { var resource interface{} + var err error switch resourceType { case "deployment": - // Get deployment - deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get deployment: %s", err.Error()) - } - resource = deployment + resource, err = h.clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) case "daemonset": - // Get daemonset - daemonset, err := clientSet.AppsV1().DaemonSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get daemonset: %s", err.Error()) - } - resource = daemonset + resource, err = h.clientSet.AppsV1().DaemonSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) case "statefulset": - // Get statefulset - statefulset, err := clientSet.AppsV1().StatefulSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get statefulset: %s", err.Error()) - } - resource = statefulset + resource, err = h.clientSet.AppsV1().StatefulSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) default: return fmt.Errorf("unsupported resource type: %s", resourceType) } - // Wait for new pod creation - err := util.WaitForNewPodCreation(clientSet, resource, startTime) if err != nil { - return fmt.Errorf("error waiting for pod creation: %s", err.Error()) + return fmt.Errorf("failed to get %s: %s", resourceType, err.Error()) } - // List resource pods - resourcePods, err := clientSet.CoreV1().Pods(uniqueNamespace).List(context.TODO(), metav1.ListOptions{}) + if err := util.WaitForNewPodCreation(h.clientSet, resource, h.startTime); err != nil { + return fmt.Errorf("error waiting for pod creation: %s", err.Error()) + } - if err != nil { - return fmt.Errorf("failed to list pods: %s", err.Error()) + if h.PodsAnnotationsValid(uniqueNamespace, shouldExist, shouldNotExist) { + return nil + } else { + return fmt.Errorf("A pod has invalid annotations") } +} - // Wait for pods to update - if !checkIfAnnotationExists(clientSet, resourcePods, annotations) { - return fmt.Errorf("missing annotations: %v", annotations) +func (h *TestHelper) CreateResource(uniqueNamespace string, sampleAppYamlPath string, skipDelete bool) error { + if err := h.CreateNamespaceAndApplyResources(uniqueNamespace, []string{sampleAppYamlPath}); err != nil { + return fmt.Errorf("failed to create/apply resources on namespace: %v", err) } + if !skipDelete { + h.t.Cleanup(func() { + if err := h.DeleteNamespaceAndResources(uniqueNamespace, []string{sampleAppYamlPath}); err != nil { + h.t.Fatalf("Failed to delete namespaces/resources: %v", err) + } + }) + } return nil } -func annotationExists(annotations map[string]string, key string) bool { - _, exists := annotations[key] - return exists -} -func setupFunction(t *testing.T, namespace string, apps []string) (*kubernetes.Clientset, string) { - t.Helper() - clientSet := setupTest(t) +func (h *TestHelper) Initialize(namespace string, apps []string) string { newUUID := uuid.New() - uniqueNamespace := fmt.Sprintf(namespace+"-%s", newUUID.String()) - if err := createNamespaceAndApplyResources(t, clientSet, uniqueNamespace, apps); err != nil { - t.Fatalf("Failed to create/apply resoures on namespace: %v", err) + uniqueNamespace := fmt.Sprintf("%s-%s", namespace, newUUID.String()) + + h.UpdateMonitorConfig(auto.MonitorConfig{MonitorAllServices: false}) + h.UpdateAnnotationConfig(auto.AnnotationConfig{}) + h.startTime = time.Now() + if err := h.CreateNamespaceAndApplyResources(uniqueNamespace, apps); err != nil { + h.t.Fatalf("Failed to create/apply resources on namespace: %v", err) } - t.Cleanup(func() { - if err := deleteNamespaceAndResources(clientSet, uniqueNamespace, apps); err != nil { - t.Fatalf("Failed to delete namespaces/resources: %v", err) - } - }) - return clientSet, uniqueNamespace -} + if !h.skipDelete { + h.t.Cleanup(func() { + fmt.Printf("Deleting namespace %s and resources %s", uniqueNamespace, apps) + if err := h.DeleteNamespaceAndResources(uniqueNamespace, apps); err != nil { + h.t.Fatalf("Failed to delete namespaces/resources: %v", err) + } + }) + } -func numberOfRevisions(deploymentName string, namespace string) int { + return uniqueNamespace +} +func (h *TestHelper) NumberOfRevisions(deploymentName string, namespace string) int { numOfRevisions := 0 i := 0 for { - // Execute the kubectl rollout history command cmd := exec.Command("kubectl", "rollout", "history", "deployment", deploymentName, "-n", namespace, "--revision", strconv.Itoa(i)) - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { break } numOfRevisions++ i++ } - return numOfRevisions - 1 //don't want to count the first + return numOfRevisions - 1 +} + +func (h *TestHelper) WaitYamlWithKubectl(filename string, namespace string) error { + cmd := exec.Command("kubectl", "wait", "--for=create", "-f", filename, "-n", namespace) + fmt.Printf("Waiting YAML with kubectl %s\n", cmd) + return cmd.Run() } diff --git a/integration-tests/manifests/automonitor/validate_automonitor_namespace_test.go b/integration-tests/manifests/automonitor/validate_automonitor_namespace_test.go deleted file mode 100644 index b9dcbff36..000000000 --- a/integration-tests/manifests/automonitor/validate_automonitor_namespace_test.go +++ /dev/null @@ -1,446 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package annotations - -import ( - "context" - "crypto/rand" - "encoding/json" - "fmt" - "math/big" - "os" - "path/filepath" - "testing" - "time" - - "github.com/google/uuid" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/aws/amazon-cloudwatch-agent-operator/integration-tests/util" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" -) - -func TestAllLanguagesNamespace(t *testing.T) { - - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("namespace-all-languages-%d", randomNumber) - - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - DotNet: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - NodeJS: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - startTime := time.Now() - - updateTheOperator(t, clientSet, string(jsonStr)) - if !checkNameSpaceAnnotations(t, clientSet, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation, injectPythonAnnotation, autoAnnotatePythonAnnotation, injectDotNetAnnotation, autoAnnotateDotNetAnnotation, injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, uniqueNamespace, startTime) { - t.Error("Missing Languages annotations") - } -} - -func TestJavaOnlyNamespace(t *testing.T) { - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("namespace-java-only-%d", randomNumber) - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - if !checkNameSpaceAnnotations(t, clientSet, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, uniqueNamespace, startTime) { - t.Error("Missing Java annotations") - } -} - -func TestPythonOnlyNamespace(t *testing.T) { - - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("namespace-python-only-%d", randomNumber) - if err := createNamespace(clientSet, uniqueNamespace); err != nil { - t.Fatalf("Failed to create/apply resoures on namespace: %v", err) - } - - defer func() { - if err := deleteNamespace(clientSet, uniqueNamespace); err != nil { - t.Fatalf("Failed to delete namespace: %v", err) - } - }() - - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - startTime := time.Now() - - updateTheOperator(t, clientSet, string(jsonStr)) - - if !checkNameSpaceAnnotations(t, clientSet, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, uniqueNamespace, startTime) { - t.Error("Missing Python annotations") - } -} - -func TestDotNetOnlyNamespace(t *testing.T) { - - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("namespace-dotnet-only-%d", randomNumber) - if err := createNamespace(clientSet, uniqueNamespace); err != nil { - t.Fatalf("Failed to create/apply resoures on namespace: %v", err) - } - - defer func() { - if err := deleteNamespace(clientSet, uniqueNamespace); err != nil { - t.Fatalf("Failed to delete namespace: %v", err) - } - }() - - annotationConfig := auto.AnnotationConfig{ - DotNet: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - startTime := time.Now() - - updateTheOperator(t, clientSet, string(jsonStr)) - - if !checkNameSpaceAnnotations(t, clientSet, []string{injectDotNetAnnotation, autoAnnotateDotNetAnnotation}, uniqueNamespace, startTime) { - t.Error("Missing DotNet annotations") - } -} - -func TestNodeJSOnlyNamespace(t *testing.T) { - - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("namespace-nodejs-only-%d", randomNumber) - if err := createNamespace(clientSet, uniqueNamespace); err != nil { - t.Fatalf("Failed to create/apply resoures on namespace: %v", err) - } - - defer func() { - if err := deleteNamespace(clientSet, uniqueNamespace); err != nil { - t.Fatalf("Failed to delete namespace: %v", err) - } - }() - - annotationConfig := auto.AnnotationConfig{ - NodeJS: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - startTime := time.Now() - - updateTheOperator(t, clientSet, string(jsonStr)) - - if !checkNameSpaceAnnotations(t, clientSet, []string{injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, uniqueNamespace, startTime) { - t.Error("Missing nodejs annotations") - } -} - -// Multiple resources on the same namespace should all get annotations -func TestAnnotationsOnMultipleResources(t *testing.T) { - - clientSet := setupTest(t) - newUUID := uuid.New() - uniqueNamespace := fmt.Sprintf("multiple-resources-%s", newUUID.String()) - - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - DaemonSets: []string{filepath.Join(uniqueNamespace, daemonSetName)}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, - }, - Python: auto.AnnotationResources{}, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - if err != nil { - t.Errorf("Failed to get deployment app: %s", err.Error()) - } - - if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, true); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } - if err := checkResourceAnnotations(t, clientSet, "daemonset", uniqueNamespace, daemonSetName, sampleDaemonsetYamlRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, true); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } - if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } - -} - -func TestAutoAnnotationForManualAnnotationRemoval(t *testing.T) { - clientSet, uniqueNamespace := setupFunction(t, "manual-annotation-removal", []string{sampleDeploymentYamlNameRelPath}) - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - }, - Python: auto.AnnotationResources{}, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - if err != nil { - t.Errorf("Failed to get deployment app: %s", err.Error()) - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - for { - deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(ctx, deploymentName, metav1.GetOptions{}) - if err != nil { - if errors.IsNotFound(err) { - t.Fatalf("Deployment %s not found in namespace %s\n", deploymentName, uniqueNamespace) - } - t.Fatal("Error getting deployment") - } - - if deployment.Status.AvailableReplicas == *deployment.Spec.Replicas && deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas { - if deployment.Status.Replicas == deployment.Status.AvailableReplicas { - fmt.Println("All pods are fully ready and no pods are terminating.") - break - } - } - - // Sleep for a short interval before checking again - time.Sleep(5 * time.Second) - } - deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(ctx, deploymentName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Error getting deployment: %v\n", err) - } - - //Removing all annotations - deployment.ObjectMeta.Annotations = nil - _, err = clientSet.AppsV1().Deployments(uniqueNamespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - if err != nil { - fmt.Printf("Error updating deployment: %v\n", err) - os.Exit(1) - } - - err = util.WaitForNewPodCreation(clientSet, deployment, startTime) - if err != nil { - t.Fatalf("Error waiting for pod creation: %v\n", err) - } - - deploymentPods, err := clientSet.CoreV1().Pods(uniqueNamespace).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - t.Fatalf("Error listing pods: %v\n", err) - } - //Check if operator has added back the annotations - checkIfAnnotationExists(clientSet, deploymentPods, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}) - -} - -// Creating two apps - First app is annotated -// Second app is not annotated but on the same namespace as the first app -// Annotate the namespace of the apps and make sure only the non annotated app was restarted -// Also tests if a resource is manually annotated and now its namespace is added for auto annotation -// the resource should not be modified and should not be restarted (auto-annotation annotation does not exist) -func TestOnlyNonAnnotatedAppsShouldBeRestarted(t *testing.T) { - - clientSet, uniqueNamespace := setupFunction(t, "non-annotated", []string{sampleDeploymentYamlNameRelPath, sampleNginxAppYamlNameRelPath}) - startTime := time.Now() - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - }, - Python: auto.AnnotationResources{}, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - updateTheOperator(t, clientSet, string(jsonStr)) - if err != nil { - t.Errorf("Failed to get deployment app: %s", err.Error()) - } - deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) - if err != nil { - fmt.Printf("Error retrieving deployment: %v\n", err) - os.Exit(1) - } - nginxDeployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), nginxDeploymentName, metav1.GetOptions{}) - if err != nil { - fmt.Printf("Error retrieving deployment: %v\n", err) - os.Exit(1) - } - err = util.WaitForNewPodCreation(clientSet, deployment, startTime) - if err != nil { - t.Fatal("Error waiting for pod creation: ", err) - } - - if annotationExists(nginxDeployment.Annotations, autoAnnotateJavaAnnotation) { - t.Fatal("Auto-annotation annotation should not exist") - } - - numOfRevisions := numberOfRevisions(nginxDeploymentName, uniqueNamespace) - if numOfRevisions > 1 { - t.Fatal("Nginx was restarted") //should not be restarted since it already had annotations - } - numOfRevisions = numberOfRevisions(deploymentName, uniqueNamespace) - if numOfRevisions != 2 { - t.Fatal("Sample-deployment should have been restarted") //should not be restarted since it already had annotations - } - -} - -// Test if a resource is auto annotated and now its namespace is added for auto annotation -// the resource should not be restarted -func TestAlreadyAutoAnnotatedResourceShouldNotRestart(t *testing.T) { - - clientSet, uniqueNamespace := setupFunction(t, "already-annotated", []string{sampleDeploymentYamlNameRelPath}) - startTime := time.Now() - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - }, - Python: auto.AnnotationResources{}, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - updateTheOperator(t, clientSet, string(jsonStr)) - if err != nil { - t.Errorf("Failed to get deployment app: %s", err.Error()) - } - deployment, err := clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) - if err != nil { - fmt.Printf("Error retrieving deployment: %v\n", err) - os.Exit(1) - } - - err = util.WaitForNewPodCreation(clientSet, deployment, startTime) - if err != nil { - t.Fatalf("Error waiting for pod creation: %v\n", err) - } - fmt.Println("Done checking deployment") - //adding deployment's namespace to get auto annotated - annotationConfig = auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{uniqueNamespace}, - Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, - }, - Python: auto.AnnotationResources{}, - } - jsonStr, err = json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - fmt.Println("Right before update operator", startTime) - updateTheOperator(t, clientSet, string(jsonStr)) - fmt.Println("Right after update operator", startTime) - - //number of revisions should not be greater than 2 - //first one is for creation second one is for the first operator change and third one should not exist (even with the second operator change) - numOfRevisions := numberOfRevisions(deploymentName, uniqueNamespace) - if numOfRevisions > 2 { - t.Fatal("Sample-deployment should not have been restarted after second operator update") //should not be restarted since it already had annotations - } - -} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_statefulset_test.go b/integration-tests/manifests/automonitor/validate_automonitor_statefulset_test.go deleted file mode 100644 index 4f8ccea04..000000000 --- a/integration-tests/manifests/automonitor/validate_automonitor_statefulset_test.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package annotations - -import ( - "crypto/rand" - "encoding/json" - "fmt" - "math/big" - "path/filepath" - "testing" - "time" - - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" -) - -func TestAllLanguagesStatefulSet(t *testing.T) { - - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("statefulset-namespace-all-languages-%d", randomNumber) - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, - }, - DotNet: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, - }, - NodeJS: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation, injectPythonAnnotation, autoAnnotatePythonAnnotation, injectDotNetAnnotation, autoAnnotateDotNetAnnotation, injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } -} - -func TestJavaOnlyStatefulSet(t *testing.T) { - - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("statefulset-java-only-%d", randomNumber) - - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } -} - -func TestPythonOnlyStatefulSet(t *testing.T) { - - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("statefulset-namespace-python-only-%d", randomNumber) - annotationConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{""}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - - if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } -} - -func TestDotNetOnlyStatefulSet(t *testing.T) { - - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("statefulset-namespace-dotnet-only-%d", randomNumber) - annotationConfig := auto.AnnotationConfig{ - DotNet: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - - if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectDotNetAnnotation, autoAnnotateDotNetAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } -} -func TestNodeJSOnlyStatefulSet(t *testing.T) { - - clientSet := setupTest(t) - randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) - if err != nil { - panic(err) - } - randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace - uniqueNamespace := fmt.Sprintf("statefulset-namespace-nodejs-only-%d", randomNumber) - annotationConfig := auto.AnnotationConfig{ - NodeJS: auto.AnnotationResources{ - Namespaces: []string{""}, - DaemonSets: []string{""}, - Deployments: []string{""}, - StatefulSets: []string{filepath.Join(uniqueNamespace, statefulSetName)}, - }, - } - jsonStr, err := json.Marshal(annotationConfig) - if err != nil { - t.Error("Error:", err) - } - - startTime := time.Now() - updateTheOperator(t, clientSet, string(jsonStr)) - - if err := checkResourceAnnotations(t, clientSet, "statefulset", uniqueNamespace, statefulSetName, sampleStatefulsetYamlNameRelPath, startTime, []string{injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { - t.Fatalf("Failed annotation check: %s", err.Error()) - } -} diff --git a/integration-tests/manifests/sample-deployment-service.yaml b/integration-tests/manifests/sample-deployment-service.yaml new file mode 100644 index 000000000..2af8e448b --- /dev/null +++ b/integration-tests/manifests/sample-deployment-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: sample-deployment-service +spec: + selector: + app: nginx-app + ports: + - protocol: TCP + port: 80 + targetPort: 80 \ No newline at end of file diff --git a/integration-tests/util/util.go b/integration-tests/util/util.go index 616ee224c..d9ccffaa7 100644 --- a/integration-tests/util/util.go +++ b/integration-tests/util/util.go @@ -6,6 +6,7 @@ package util import ( "context" "fmt" + "k8s.io/apimachinery/pkg/util/wait" "time" appsV1 "k8s.io/api/apps/v1" @@ -18,44 +19,72 @@ import ( const TimoutDuration = 2 * time.Minute const TimeBetweenRetries = 2 * time.Second -// WaitForNewPodCreation takes in a resource either Deployment, DaemonSet, or StatefulSet wait until it is in running stage func WaitForNewPodCreation(clientSet *kubernetes.Clientset, resource interface{}, startTime time.Time) error { - start := time.Now() - for { - if time.Since(start) > TimoutDuration { - return fmt.Errorf("timed out waiting for new pod creation") - } - namespace := "" - labelSelector := "" - switch r := resource.(type) { - case *appsV1.Deployment: - namespace = r.Namespace - labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() - case *appsV1.DaemonSet: - namespace = r.Namespace - labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() - case *appsV1.StatefulSet: - namespace = r.Namespace - labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() - default: - return fmt.Errorf("unsupported resource type") - } - - newPods, _ := clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ + // 1. Use wait.PollImmediate instead of manual polling + // 2. Move type switch outside the polling loop + fmt.Println("start time: ", startTime) + namespace := "" + labelSelector := "" + switch r := resource.(type) { + case *appsV1.Deployment: + namespace = r.Namespace + labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() + case *appsV1.DaemonSet: + namespace = r.Namespace + labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() + case *appsV1.StatefulSet: + namespace = r.Namespace + labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() + default: + return fmt.Errorf("unsupported resource type") + } + return wait.PollImmediate(TimeBetweenRetries, TimoutDuration, func() (bool, error) { + // 3. Handle list error + newPods, err := clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: labelSelector, }) + // get list of pod names + //podNames := make([]string, 0) + //for _, pod := range newPods.Items { + // podNames = append(podNames, pod.Name) + //} + // + //fmt.Println(podNames) + if err != nil { + return false, fmt.Errorf("failed to list pods: %v", err) + } + // 4. Check for pod readiness, not just running for _, pod := range newPods.Items { - if pod.CreationTimestamp.Time.After(startTime) && pod.Status.Phase == v1.PodRunning { - fmt.Printf("Operator pod %s created after start time and is running\n", pod.Name) - return nil - } else if pod.CreationTimestamp.Time.After(startTime) { - fmt.Printf("Operator pod %s created after start time but is not in running stage\n", pod.Name) + fmt.Println("Pod Name: ", pod.Name, ", pod.Creation: ", pod.CreationTimestamp) + if pod.CreationTimestamp.Time.After(startTime.Add(-time.Second)) { + if pod.Status.Phase == v1.PodRunning { + // 5. Check if pod is ready + isReady := isPodReady(&pod) + if isReady { + fmt.Printf("pod %s created after start time and is ready\n", pod.Name) + return true, nil + } + fmt.Printf("pod %s is running but not ready\n", pod.Name) + } else { + fmt.Printf("pod %s created after start time but is in %s state\n", + pod.Name, pod.Status.Phase) + } } } - time.Sleep(TimeBetweenRetries) + return false, nil + }) +} + +// Helper function to check if pod is ready +func isPodReady(pod *v1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == v1.PodReady { + return condition.Status == v1.ConditionTrue + } } + return false } func CheckIfPodsAreRunning(pods *v1.PodList) bool { diff --git a/main.go b/main.go index d3e11fdde..6c90045dc 100644 --- a/main.go +++ b/main.go @@ -373,7 +373,7 @@ func main() { func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context) auto.MonitorInterface { var monitorConfig *auto.MonitorConfig - var monitor *auto.Monitor + var monitor auto.MonitorInterface = auto.NoopMonitor{} if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-monitor config") } else { diff --git a/pkg/instrumentation/annotationtype.go b/pkg/instrumentation/annotationtype.go index 70798ea4f..0f3d2d42d 100644 --- a/pkg/instrumentation/annotationtype.go +++ b/pkg/instrumentation/annotationtype.go @@ -22,6 +22,14 @@ func (s *TypeSet) UnmarshalJSON(data []byte) error { return nil } +func (s TypeSet) MarshalJSON() ([]byte, error) { + var types []Type + for t := range s { + types = append(types, t) + } + return json.Marshal(types) +} + // NewTypeSet creates a new set of Type. func NewTypeSet(types ...Type) TypeSet { s := make(TypeSet, len(types)) diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index a8e0f5250..8f3831edf 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -34,7 +34,7 @@ type Monitor struct { type NoopMonitor struct{} -func (n NoopMonitor) MutateObject(oldObj client.Object, obj client.Object) map[string]string { +func (n NoopMonitor) MutateObject(_ client.Object, _ client.Object) map[string]string { return map[string]string{} } @@ -52,23 +52,51 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: func(obj interface{}) { if config.AutoRestart { - list, err := k8sInterface.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + namespaces, err := k8sInterface.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { - logger.Error(err, "failed to list namespaces") + logger.Error(err, "failed to namespaces namespaces") } // TODO: optimize this by trying to resolve workloads via endpoint slices - for _, namespace := range list.Items { - for _, deployment := range listAllDeployments(k8sInterface, namespace, ctx).Items { - m.MutateObject(nil, &deployment) + for _, namespace := range namespaces.Items { + for _, resource := range listAllDeployments(k8sInterface, namespace, ctx).Items { + mutatedAnnotations := m.MutateObject(nil, &resource) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := k8sInterface.AppsV1().Deployments(namespace.Name).Update(ctx, &resource, metav1.UpdateOptions{}) + if err != nil { + logger.Error(err, "failed to update deployment") + } } - for _, deployment := range listAllStatefulSets(k8sInterface, namespace, ctx).Items { - m.MutateObject(nil, &deployment) + for _, resource := range listAllStatefulSets(k8sInterface, namespace, ctx).Items { + mutatedAnnotations := m.MutateObject(nil, &resource) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := k8sInterface.AppsV1().StatefulSets(namespace.Name).Update(ctx, &resource, metav1.UpdateOptions{}) + if err != nil { + logger.Error(err, "failed to update statefulset") + } } - for _, deployment := range listAllDaemonSets(k8sInterface, namespace, ctx).Items { - m.MutateObject(nil, &deployment) + for _, resource := range listAllDaemonSets(k8sInterface, namespace, ctx).Items { + mutatedAnnotations := m.MutateObject(nil, &resource) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := k8sInterface.AppsV1().DaemonSets(namespace.Name).Update(ctx, &resource, metav1.UpdateOptions{}) + if err != nil { + logger.Error(err, "failed to update daemonset") + } + } + mutatedAnnotations := m.MutateObject(nil, &namespace) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := k8sInterface.CoreV1().Namespaces().Update(ctx, &namespace, metav1.UpdateOptions{}) + if err != nil { + logger.Error(err, "failed to update namespace") } - m.MutateObject(nil, &namespace) } } }}) @@ -133,28 +161,29 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { // MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector func (m Monitor) MutateObject(oldObj client.Object, obj client.Object) map[string]string { // todo: handle edge case where a workload is annotated because a service exposed it, and the service is removed. aka add to Service OnDelete - // continue only if restart is enabled or if workload pod template has been mutated - if !m.config.AutoRestart && isWorkload(obj) && !isWorkloadPodTemplateMutated(oldObj, obj) { - return nil - } // custom selector takes precedence over service selector if customSelectLanguages, selected := m.CustomSelected(obj); selected { logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) - mutate(obj, customSelectLanguages) + return mutate(obj, customSelectLanguages) + } + + // continue only if restart is enabled or if workload pod template has been mutated + if isWorkload(obj) && !isWorkloadPodTemplateMutated(oldObj, obj) { + return map[string]string{} } if !m.config.MonitorAllServices { - return nil + return map[string]string{} } if m.excluded(obj) { - return nil + return map[string]string{} } objectLabels := getTemplateSpecLabels(obj) for _, informerObj := range m.serviceInformer.GetStore().List() { service := informerObj.(*corev1.Service) - if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 { + if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 || service.GetNamespace() != obj.GetNamespace() { continue } serviceSelector := labels.SelectorFromSet(service.Spec.Selector) @@ -164,7 +193,7 @@ func (m Monitor) MutateObject(oldObj client.Object, obj client.Object) map[strin return mutate(obj, m.config.Languages) } } - return nil + return map[string]string{} } // mutate obj. If object is a workload, mutate the pod template. otherwise, mutate the object's annotations itself. @@ -243,9 +272,6 @@ func isWorkload(obj client.Object) bool { // isWorkloadPodTemplateMutated func isWorkloadPodTemplateMutated(oldObject client.Object, object client.Object) bool { oldTemplate, newTemplate := getPodTemplate(oldObject), getPodTemplate(object) - if oldTemplate == nil || newTemplate == nil { - return true - } return !reflect.DeepEqual(oldTemplate, newTemplate) } diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index 818506213..52dd588c0 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -2,6 +2,7 @@ package auto import ( "context" + "encoding/json" "fmt" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/stretchr/testify/assert" @@ -374,6 +375,25 @@ func TestUnmarshal(t *testing.T) { assert.Equal(t, instrumentation.TypeSet{instrumentation.TypeNodeJS: nil, instrumentation.TypeJava: nil, instrumentation.TypePython: nil}, set) } +func TestMarshal(t *testing.T) { + types := instrumentation.TypeSet{instrumentation.TypeNodeJS: nil, instrumentation.TypePython: nil} + res, err := types.MarshalJSON() + assert.NoError(t, err) + AssertJsonEqual(t, []byte(`["nodejs","python"]`), res) +} + +func AssertJsonEqual(t *testing.T, expectedJson []byte, actualJson []byte) { + var obj1, obj2 interface{} + + err := json.Unmarshal(expectedJson, &obj1) + assert.NoError(t, err) + + err = json.Unmarshal(actualJson, &obj2) + assert.NoError(t, err) + + assert.Equal(t, obj1, obj2) +} + func Test_isWorkloadPodTemplateMutated(t *testing.T) { deploy := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ From b2d6c1c0dbd193c30429116f22ad9e7ada415bcf Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 19 Mar 2025 10:36:08 -0400 Subject: [PATCH 08/63] blah --- .../validate_automonitor_deployment_test.go | 330 ++++++++---------- .../validate_automonitor_methods.go | 96 ++++- integration-tests/util/util.go | 14 +- main.go | 7 +- pkg/instrumentation/auto/auto_monitor.go | 240 +++++++++---- 5 files changed, 395 insertions(+), 292 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index c0c41c7db..cd008aae8 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -104,197 +104,139 @@ func TestDeploymentThenServiceAutoRestartEnabled(t *testing.T) { assert.NoError(t, err) } -//func TestAllLanguagesDeployment(t *testing.T) { -// -// clientSet := setupTest(t) -// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) -// if err != nil { -// panic(err) -// } -// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace -// uniqueNamespace := fmt.Sprintf("deployment-namespace-all-languages-%d", randomNumber) -// annotationConfig := auto.AnnotationConfig{ -// Java: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, -// StatefulSets: []string{""}, -// }, -// Python: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, -// StatefulSets: []string{""}, -// }, -// DotNet: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, -// StatefulSets: []string{""}, -// }, -// NodeJS: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, -// StatefulSets: []string{""}, -// }, -// } -// jsonStr, err := json.Marshal(annotationConfig) -// assert.Nil(t, err) -// -// startTime := time.Now() -// updateOperatorConfig(t, clientSet, string(jsonStr)) -// -// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation, injectPythonAnnotation, autoAnnotatePythonAnnotation, injectDotNetAnnotation, autoAnnotateDotNetAnnotation, injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { -// t.Fatalf("Failed annotation check: %s", err.Error()) -// } -// -//} -// -//func TestJavaOnlyDeployment(t *testing.T) { -// -// clientSet := setupTest(t) -// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) -// if err != nil { -// panic(err) -// } -// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace -// uniqueNamespace := fmt.Sprintf("deployment-namespace-java-only-%d", randomNumber) -// -// annotationConfig := auto.AnnotationConfig{ -// Java: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, -// StatefulSets: []string{""}, -// }, -// Python: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{""}, -// StatefulSets: []string{""}, -// }, -// } -// jsonStr, err := json.Marshal(annotationConfig) -// if err != nil { -// t.Errorf("Failed to marshal: %v\n", err) -// } -// startTime := time.Now() -// updateOperatorConfig(t, clientSet, string(jsonStr)) -// -// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectJavaAnnotation, autoAnnotateJavaAnnotation}, false); err != nil { -// t.Fatalf("Failed annotation check: %s", err.Error()) -// } -//} -// -//func TestPythonOnlyDeployment(t *testing.T) { -// -// clientSet := setupTest(t) -// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) -// if err != nil { -// panic(err) -// } -// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace -// uniqueNamespace := fmt.Sprintf("deployment-namespace-python-only-%d", randomNumber) -// -// annotationConfig := auto.AnnotationConfig{ -// Java: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{""}, -// StatefulSets: []string{""}, -// }, -// Python: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, -// StatefulSets: []string{""}, -// }, -// } -// jsonStr, err := json.Marshal(annotationConfig) -// if err != nil { -// t.Error("Error:", err) -// } -// -// startTime := time.Now() -// updateOperatorConfig(t, clientSet, string(jsonStr)) -// if err != nil { -// t.Errorf("Failed to get deployment app: %s", err.Error()) -// } -// -// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}, false); err != nil { -// t.Fatalf("Failed annotation check: %s", err.Error()) -// } -// -//} -//func TestDotNetOnlyDeployment(t *testing.T) { -// -// clientSet := setupTest(t) -// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) -// if err != nil { -// panic(err) -// } -// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace -// uniqueNamespace := fmt.Sprintf("deployment-namespace-dotnet-only-%d", randomNumber) -// -// annotationConfig := auto.AnnotationConfig{ -// DotNet: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, -// StatefulSets: []string{""}, -// }, -// } -// jsonStr, err := json.Marshal(annotationConfig) -// if err != nil { -// t.Error("Error:", err) -// } -// -// startTime := time.Now() -// updateOperatorConfig(t, clientSet, string(jsonStr)) -// if err != nil { -// t.Errorf("Failed to get deployment app: %s", err.Error()) -// } -// -// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectDotNetAnnotation, autoAnnotateDotNetAnnotation}, false); err != nil { -// t.Fatalf("Failed annotation check: %s", err.Error()) -// } -// -//} -// -//func TestNodeJSOnlyDeployment(t *testing.T) { -// -// clientSet := setupTest(t) -// randomNumber, err := rand.Int(rand.Reader, big.NewInt(9000)) -// if err != nil { -// panic(err) -// } -// randomNumber.Add(randomNumber, big.NewInt(1000)) //adding a hash to namespace -// uniqueNamespace := fmt.Sprintf("deployment-namespace-nodejs-only-%d", randomNumber) -// -// annotationConfig := auto.AnnotationConfig{ -// NodeJS: auto.AnnotationResources{ -// Namespaces: []string{""}, -// DaemonSets: []string{""}, -// Deployments: []string{filepath.Join(uniqueNamespace, deploymentName)}, -// StatefulSets: []string{""}, -// }, -// } -// jsonStr, err := json.Marshal(annotationConfig) -// if err != nil { -// t.Error("Error:", err) -// t.Error("Error:", err) -// -// } -// -// startTime := time.Now() -// updateOperatorConfig(t, clientSet, string(jsonStr)) -// if err != nil { -// t.Errorf("Failed to get deployment app: %s", err.Error()) -// } -// -// if err := checkResourceAnnotations(t, clientSet, "deployment", uniqueNamespace, deploymentName, sampleDeploymentYamlNameRelPath, startTime, []string{injectNodeJSAnnotation, autoAnnotateNodeJSAnnotation}, false); err != nil { -// t.Fatalf("Failed annotation check: %s", err.Error()) -// } -// -//} +func TestDeploymentWithCustomSelector(t *testing.T) { + helper := NewTestHelper(t, true) + + namespace := helper.Initialize("test-namespace", []string{}) + + // Set up custom selector config + customSelectorConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Deployments: []string{"sample-deployment"}, + }, + Python: auto.AnnotationResources{ + Deployments: []string{"sample-deployment"}, + }, + } + + // Update operator with auto monitor disabled and custom selector + helper.UpdateMonitorConfig(auto.MonitorConfig{ + MonitorAllServices: false, + Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + AutoRestart: false, + CustomSelector: customSelectorConfig, + }) + + // Create deployment + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + assert.NoError(t, err) + + // Validate annotations are present + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation}, + []string{autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + assert.NoError(t, err) +} + +func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { + helper := NewTestHelper(t, true) + + namespace := helper.Initialize("test-namespace", []string{}) + + // Update operator with auto monitor disabled + helper.UpdateMonitorConfig(auto.MonitorConfig{ + MonitorAllServices: false, + Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + AutoRestart: false, + }) + + // Create deployment + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + assert.NoError(t, err) + + // Validate no annotations present + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", + none, + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + assert.NoError(t, err) + + // Update operator with custom selector + customSelectorConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Deployments: []string{"sample-deployment"}, + }, + Python: auto.AnnotationResources{ + Deployments: []string{"sample-deployment"}, + }, + DotNet: auto.AnnotationResources{ + Deployments: []string{"sample-deployment"}, + }, + NodeJS: auto.AnnotationResources{ + Deployments: []string{"sample-deployment"}, + }, + } + + helper.UpdateMonitorConfig(auto.MonitorConfig{ + MonitorAllServices: false, + Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + AutoRestart: false, + CustomSelector: customSelectorConfig, + }) + + // Validate annotations are present + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, + none) + assert.NoError(t, err) +} + +func TestDeploymentWithExcludedThenIncludedService(t *testing.T) { + helper := NewTestHelper(t, true) + + namespace := helper.Initialize("test-namespace", []string{}) + + // Set up config with service exclusion + monitorConfig := auto.MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + Exclude: struct { + Namespaces []string `json:"namespaces"` + Services []string `json:"services"` + }{ + Services: []string{namespace + "/sample-deployment-service"}, // assuming this is the service name in sampleDeploymentServiceYaml + }, + } + + // Update operator config + helper.UpdateAnnotationConfig(defaultAnnotationConfig) + helper.UpdateMonitorConfig(monitorConfig) + + // Create service first + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) + assert.NoError(t, err) + + // Create deployment + err = helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + assert.NoError(t, err) + + fmt.Println("Sleeping!") + time.Sleep(1 * time.Minute) + // Validate that deployment has no annotations + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", + none, + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + assert.NoError(t, err) + + // Update config to remove exclusion + monitorConfig.Exclude.Services = []string{} + helper.UpdateMonitorConfig(monitorConfig) + err = helper.RestartDeployment(namespace, "sample-deployment") + if err != nil { + return + } + // Validate that deployment now has annotations + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, + none) + assert.NoError(t, err) +} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go index d85dbe9c7..280024e83 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -8,6 +8,8 @@ import ( "fmt" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" "os" "os/exec" "path/filepath" @@ -228,26 +230,36 @@ func (h *TestHelper) CheckNameSpaceAnnotations(expectedAnnotations []string, uni func (h *TestHelper) UpdateOperator(deployment *appsV1.Deployment) bool { args := deployment.Spec.Template.Spec.Containers[0].Args now := time.Now() - deployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) - if err != nil { - h.t.Errorf("Failed to get deployment: %v\n", err) - return false - } - deployment.Spec.Template.Spec.Containers[0].Args = args - forceRestart(deployment) - _, err = h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - if err != nil { - h.t.Errorf("Failed to update deployment: %v\n", err) + retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + // Get the latest version of the deployment + currentDeployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) + if err != nil { + return err + } + + // Apply your changes to the latest version + currentDeployment.Spec.Template.Spec.Containers[0].Args = args + forceRestart(currentDeployment) + + // Try to update + _, updateErr := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Update(context.TODO(), currentDeployment, metav1.UpdateOptions{}) + return updateErr + }) + + if retryErr != nil { + h.t.Errorf("Failed to update deployment after retries: %v\n", retryErr) return false } - err = util.WaitForNewPodCreation(h.clientSet, deployment, now) + err := util.WaitForNewPodCreation(h.clientSet, deployment, now) if err != nil { fmt.Println("There was an error trying to wait for deployment available", err) return false } + fmt.Println("Operator updated successfully!") + fmt.Println(args) return true } @@ -327,18 +339,20 @@ func (h *TestHelper) findIndexOfPrefix(str string, strs []string) int { return -1 } -func (h *TestHelper) UpdateMonitorConfig(config auto.MonitorConfig) time.Time { +func (h *TestHelper) UpdateMonitorConfig(config auto.MonitorConfig) { jsonStr, err := json.Marshal(config) assert.Nil(h.t, err) - startTime := time.Now() + fmt.Println("Setting monitor config to:") + util.PrettyPrint(config) h.updateOperatorConfig(string(jsonStr), "--auto-monitor-config=") - return startTime } func (h *TestHelper) UpdateAnnotationConfig(config auto.AnnotationConfig) { jsonStr, err := json.Marshal(config) assert.Nil(h.t, err) + fmt.Println("Setting annotation config to:") + util.PrettyPrint(config) h.updateOperatorConfig(string(jsonStr), "--auto-annotation-config=") } @@ -359,6 +373,7 @@ func (h *TestHelper) updateOperatorConfig(jsonStr string, flag string) { if !h.UpdateOperator(deployment) { h.t.Error("Failed to update Operator", deployment, deployment.Name, deployment.Spec.Template.Spec.Containers[0].Args) } + time.Sleep(5 * time.Second) } func (h *TestHelper) ValidateWorkloadAnnotations(resourceType, uniqueNamespace, resourceName string, shouldExist []string, shouldNotExist []string) error { @@ -448,3 +463,56 @@ func (h *TestHelper) WaitYamlWithKubectl(filename string, namespace string) erro fmt.Printf("Waiting YAML with kubectl %s\n", cmd) return cmd.Run() } + +func (h *TestHelper) RestartDeployment(namespace string, deploymentName string) error { + retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + // Get the latest version of the deployment + deployment, err := h.clientSet.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment: %v", err) + } + + // Add or update restart annotation + if deployment.Spec.Template.Annotations == nil { + deployment.Spec.Template.Annotations = make(map[string]string) + } + deployment.Spec.Template.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339) + + // Update the deployment + _, updateErr := h.clientSet.AppsV1().Deployments(namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + return updateErr + }) + + if retryErr != nil { + return fmt.Errorf("failed to restart deployment %s: %v", deploymentName, retryErr) + } + + // Wait for rollout to complete + err := h.WaitForDeploymentRollout(namespace, deploymentName) + if err != nil { + return fmt.Errorf("failed to wait for deployment rollout: %v", err) + } + + fmt.Printf("Successfully restarted deployment %s in namespace %s\n", deploymentName, namespace) + return nil +} + +// WaitForDeploymentRollout waits for the deployment to complete its rollout +func (h *TestHelper) WaitForDeploymentRollout(namespace string, deploymentName string) error { + return wait.PollImmediate(time.Second*2, time.Minute*5, func() (bool, error) { + deployment, err := h.clientSet.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + if err != nil { + return false, err + } + + // Check if the rollout is complete + if deployment.Generation <= deployment.Status.ObservedGeneration && + deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas && + deployment.Status.Replicas == *deployment.Spec.Replicas && + deployment.Status.AvailableReplicas == *deployment.Spec.Replicas { + return true, nil + } + + return false, nil + }) +} diff --git a/integration-tests/util/util.go b/integration-tests/util/util.go index d9ccffaa7..5818d417e 100644 --- a/integration-tests/util/util.go +++ b/integration-tests/util/util.go @@ -5,6 +5,7 @@ package util import ( "context" + "encoding/json" "fmt" "k8s.io/apimachinery/pkg/util/wait" "time" @@ -56,7 +57,7 @@ func WaitForNewPodCreation(clientSet *kubernetes.Clientset, resource interface{} // 4. Check for pod readiness, not just running for _, pod := range newPods.Items { - fmt.Println("Pod Name: ", pod.Name, ", pod.Creation: ", pod.CreationTimestamp) + //fmt.Println("Pod Name: ", pod.Name, ", pod.Creation: ", pod.CreationTimestamp) if pod.CreationTimestamp.Time.After(startTime.Add(-time.Second)) { if pod.Status.Phase == v1.PodRunning { // 5. Check if pod is ready @@ -65,10 +66,10 @@ func WaitForNewPodCreation(clientSet *kubernetes.Clientset, resource interface{} fmt.Printf("pod %s created after start time and is ready\n", pod.Name) return true, nil } - fmt.Printf("pod %s is running but not ready\n", pod.Name) + //fmt.Printf("pod %s is running but not ready\n", pod.Name) } else { - fmt.Printf("pod %s created after start time but is in %s state\n", - pod.Name, pod.Status.Phase) + //fmt.Printf("pod %s created after start time but is in %s state\n", + // pod.Name, pod.Status.Phase) } } } @@ -101,3 +102,8 @@ func CheckIfPodsAreRunning(pods *v1.PodList) bool { fmt.Println("All pods are in the Running phase") return true } + +func PrettyPrint(data interface{}) { + b, _ := json.MarshalIndent(data, "", " ") + fmt.Println(string(b)) +} diff --git a/main.go b/main.go index 6c90045dc..fa750abec 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,7 @@ import ( "k8s.io/client-go/rest" "os" "runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "strings" "time" @@ -307,7 +308,7 @@ func main() { } else { // TODO handle case where auto monitor is enabled but auto annotation config is not specified setupLog.Info("Test!") - monitor := createMonitorFromConfig(autoMonitorConfigStr, ctx) + monitor := createMonitorFromConfig(autoMonitorConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) autoAnnotationMutators := auto.NewAnnotationMutators( mgr.GetClient(), @@ -371,7 +372,7 @@ func main() { } } -func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context) auto.MonitorInterface { +func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context, client client.Client, reader client.Reader) auto.MonitorInterface { var monitorConfig *auto.MonitorConfig var monitor auto.MonitorInterface = auto.NoopMonitor{} if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { @@ -391,7 +392,7 @@ func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context) a setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") return auto.NoopMonitor{} } - monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet) + monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader) } return monitor } diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index 8f3831edf..7bd10b7dd 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -16,6 +16,7 @@ import ( "reflect" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" + "strings" "time" ) @@ -30,6 +31,8 @@ type Monitor struct { serviceInformer cache.SharedIndexInformer ctx context.Context config MonitorConfig + k8sInterface kubernetes.Interface + customSelectors *AnnotationMutators } type NoopMonitor struct{} @@ -42,65 +45,23 @@ func (n NoopMonitor) AnyCustomSelectorDefined() bool { return false } -func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface) *Monitor { +func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, c client.Client, r client.Reader) *Monitor { logger.Info("AutoMonitor starting...") - - factory := informers.NewSharedInformerFactory(k8sInterface, 10*time.Minute) - + // todo, throw warning if exclude config service is not namespaced (doesn't contain `/`) + // todo: informers.WithTransform() as option to only store what parts of service are needed + factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute) serviceInformer := factory.Core().V1().Services().Informer() - m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config} + m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: NewAnnotationMutators(c, r, logger, config.CustomSelector, config.Languages)} _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: func(obj interface{}) { - if config.AutoRestart { - namespaces, err := k8sInterface.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - if err != nil { - logger.Error(err, "failed to namespaces namespaces") - } - - // TODO: optimize this by trying to resolve workloads via endpoint slices - for _, namespace := range namespaces.Items { - for _, resource := range listAllDeployments(k8sInterface, namespace, ctx).Items { - mutatedAnnotations := m.MutateObject(nil, &resource) - if len(mutatedAnnotations) == 0 { - continue - } - _, err := k8sInterface.AppsV1().Deployments(namespace.Name).Update(ctx, &resource, metav1.UpdateOptions{}) - if err != nil { - logger.Error(err, "failed to update deployment") - } - } - for _, resource := range listAllStatefulSets(k8sInterface, namespace, ctx).Items { - mutatedAnnotations := m.MutateObject(nil, &resource) - if len(mutatedAnnotations) == 0 { - continue - } - _, err := k8sInterface.AppsV1().StatefulSets(namespace.Name).Update(ctx, &resource, metav1.UpdateOptions{}) - if err != nil { - logger.Error(err, "failed to update statefulset") - } - } - for _, resource := range listAllDaemonSets(k8sInterface, namespace, ctx).Items { - mutatedAnnotations := m.MutateObject(nil, &resource) - if len(mutatedAnnotations) == 0 { - continue - } - _, err := k8sInterface.AppsV1().DaemonSets(namespace.Name).Update(ctx, &resource, metav1.UpdateOptions{}) - if err != nil { - logger.Error(err, "failed to update daemonset") - } - } - mutatedAnnotations := m.MutateObject(nil, &namespace) - if len(mutatedAnnotations) == 0 { - continue - } - _, err := k8sInterface.CoreV1().Namespaces().Update(ctx, &namespace, metav1.UpdateOptions{}) - if err != nil { - logger.Error(err, "failed to update namespace") - } - } + if serviceInformer.HasSynced() { + m.onServiceAdd(obj) + } else { + logger.Info(fmt.Sprintf("Service %v has not synced yet, this is first sync. skipping annotation", obj)) } }}) if err != nil { + logger.Error(err, "failed to start auto monitor") return nil } factory.Start(ctx.Done()) @@ -112,29 +73,95 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet panic("TODO: handle bad cache sync") } } - logger.Info("AutoMonitor enabled!") + logger.Info("Enabled!") + + if m.config.AutoRestart { + logger.Info("Auto restarting custom selector resources") + m.customSelectors.MutateAndPatchAll(ctx) + // update all existing services + logger.Info("Auto restarting service resources, except for excludedServices or services in excludedNamespaces", "excludedServices", m.config.Exclude.Services, "excludedNamespaces", m.config.Exclude.Namespaces) + list, err := k8sInterface.CoreV1().Services("").List(ctx, metav1.ListOptions{}) + if err != nil { + return nil + } + for _, service := range list.Items { + m.onServiceAdd(service) + } + } else { + logger.Info("Auto restart disabled. To instrument workloads, restart the workloads exposed by a service.") + } + logger.Info("Initialization complete!") return m } -func listAllDeployments(k8sInterface kubernetes.Interface, namespace corev1.Namespace, ctx context.Context) *appsv1.DeploymentList { - list, err := k8sInterface.AppsV1().Deployments(namespace.Name).List(ctx, metav1.ListOptions{}) +func (m *Monitor) onServiceAdd(obj interface{}) { + service := obj.(*corev1.Service) + if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 { + return + } + + // we should not execute this code on start up because it needs to iterate over all services in MutateObject, + if !m.config.AutoRestart || m.excludedService(service) { + return + } + namespace := service.GetNamespace() + for _, resource := range listServiceDeployments(m.k8sInterface, service, m.ctx).Items { + mutatedAnnotations := m.MutateServiceWorkload(&resource, service) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := m.k8sInterface.AppsV1().Deployments(namespace).Update(m.ctx, &resource, metav1.UpdateOptions{}) + if err != nil { + logger.Error(err, "failed to update deployment") + } + } + for _, resource := range listServiceStatefulSets(m.k8sInterface, service, m.ctx).Items { + mutatedAnnotations := m.MutateServiceWorkload(&resource, service) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := m.k8sInterface.AppsV1().StatefulSets(namespace).Update(m.ctx, &resource, metav1.UpdateOptions{}) + if err != nil { + logger.Error(err, "failed to update statefulset") + } + } + for _, resource := range listServiceDaemonSets(m.k8sInterface, service, m.ctx).Items { + mutatedAnnotations := m.MutateServiceWorkload(&resource, service) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := m.k8sInterface.AppsV1().DaemonSets(namespace).Update(m.ctx, &resource, metav1.UpdateOptions{}) + if err != nil { + logger.Error(err, "failed to update daemonset") + } + } +} + +func listServiceDeployments(k8sInterface kubernetes.Interface, service *corev1.Service, ctx context.Context) *appsv1.DeploymentList { + list, err := k8sInterface.AppsV1().Deployments(service.GetNamespace()).List(ctx, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(service.Spec.Selector), + }) if err != nil { logger.Error(err, "AutoMonitor failed to list deployments") } return list } -func listAllStatefulSets(k8sInterface kubernetes.Interface, namespace corev1.Namespace, ctx context.Context) *appsv1.StatefulSetList { - list, err := k8sInterface.AppsV1().StatefulSets(namespace.Name).List(ctx, metav1.ListOptions{}) +func listServiceStatefulSets(k8sInterface kubernetes.Interface, service *corev1.Service, ctx context.Context) *appsv1.StatefulSetList { + list, err := k8sInterface.AppsV1().StatefulSets(service.GetNamespace()).List(ctx, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(service.Spec.Selector), + }) if err != nil { logger.Error(err, "AutoMonitor failed to list statefulsets") } return list } -func listAllDaemonSets(k8sInterface kubernetes.Interface, namespace corev1.Namespace, ctx context.Context) *appsv1.DaemonSetList { - list, err := k8sInterface.AppsV1().DaemonSets(namespace.Name).List(ctx, metav1.ListOptions{}) +func listServiceDaemonSets(k8sInterface kubernetes.Interface, service *corev1.Service, ctx context.Context) *appsv1.DaemonSetList { + list, err := k8sInterface.AppsV1().DaemonSets(service.GetNamespace()).List(ctx, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(service.Spec.Selector), + }) if err != nil { logger.Error(err, "AutoMonitor failed to list DaemonSets") } @@ -158,8 +185,31 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { } } +// MutateServiceWorkload mutates the annotations of the workload's object without iterating over +// TODO: remove? +func (m *Monitor) MutateServiceWorkload(obj client.Object, service *corev1.Service) map[string]string { + if customSelectLanguages, selected := m.CustomSelected(obj); selected { + logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) + return mutate(obj, customSelectLanguages) + } + + if !m.config.MonitorAllServices { + return map[string]string{} + } + if m.excludedNamespace(obj.GetNamespace()) { + return map[string]string{} + } + + if m.excludedService(service) { + return map[string]string{} + } + + logger.Info(fmt.Sprintf("start up: setting %s instrumentation annotations to %s because it is owned by service %s", obj.GetName(), m.config.Languages, service.Name)) + return mutate(obj, m.config.Languages) +} + // MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector -func (m Monitor) MutateObject(oldObj client.Object, obj client.Object) map[string]string { +func (m *Monitor) MutateObject(oldObj client.Object, obj client.Object) map[string]string { // todo: handle edge case where a workload is annotated because a service exposed it, and the service is removed. aka add to Service OnDelete // custom selector takes precedence over service selector if customSelectLanguages, selected := m.CustomSelected(obj); selected { @@ -167,8 +217,7 @@ func (m Monitor) MutateObject(oldObj client.Object, obj client.Object) map[strin return mutate(obj, customSelectLanguages) } - // continue only if restart is enabled or if workload pod template has been mutated - if isWorkload(obj) && !isWorkloadPodTemplateMutated(oldObj, obj) { + if !allowedToMutate(oldObj, obj, m.config.AutoRestart) { return map[string]string{} } @@ -176,13 +225,16 @@ func (m Monitor) MutateObject(oldObj client.Object, obj client.Object) map[strin return map[string]string{} } - if m.excluded(obj) { + if m.excludedNamespace(obj.GetNamespace()) { return map[string]string{} } objectLabels := getTemplateSpecLabels(obj) for _, informerObj := range m.serviceInformer.GetStore().List() { service := informerObj.(*corev1.Service) + if m.excludedService(service) { + continue + } if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 || service.GetNamespace() != obj.GetNamespace() { continue } @@ -229,23 +281,29 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet) ma return allMutatedAnnotations } -func (m Monitor) CustomSelected(obj client.Object) (instrumentation.TypeSet, bool) { +func (m *Monitor) CustomSelected(obj client.Object) (instrumentation.TypeSet, bool) { languages := m.config.CustomSelector.GetObjectLanguagesToAnnotate(obj) return languages, len(languages) > 0 } -// excluded returns whether a Namespace or a Service is excluded from AutoMonitor. -func (m Monitor) excluded(obj client.Object) bool { - switch obj.GetObjectKind().GroupVersionKind().Kind { - case "Namespace": - return slices.Contains(m.config.Exclude.Namespaces, obj.GetName()) - case "Service": - return slices.Contains(m.config.Exclude.Services, namespacedName(obj)) +// excludedService returns whether a Namespace or a Service is excludedService from AutoMonitor. +func (m *Monitor) excludedService(obj client.Object) bool { + excluded := slices.Contains(m.config.Exclude.Services, namespacedName(obj)) || m.excludedNamespace(obj.GetNamespace()) + logger.Info(fmt.Sprintf("%s excluded? %v", namespacedName(obj), excluded)) + return excluded +} + +func (m *Monitor) excludedNamespace(namespace string) bool { + if strings.HasPrefix(namespace, "kube-") { + return false } - return false + if strings.EqualFold(namespace, "amazon-cloudwatch") { + return false + } + return slices.Contains(m.config.Exclude.Namespaces, namespace) } -func (m Monitor) AnyCustomSelectorDefined() bool { +func (m *Monitor) AnyCustomSelectorDefined() bool { for _, t := range instrumentation.SupportedTypes() { resources := m.config.CustomSelector.getResources(t) if len(resources.DaemonSets) > 0 { @@ -265,16 +323,44 @@ func (m Monitor) AnyCustomSelectorDefined() bool { } func isWorkload(obj client.Object) bool { - kind := obj.GetObjectKind().GroupVersionKind().Kind - return kind == "Deployment" || kind == "StatefulSet" || kind == "DaemonSet" + switch obj.(type) { + case *appsv1.Deployment: + return true + case *appsv1.StatefulSet: + return true + case *appsv1.DaemonSet: + return true + default: + return false + } } -// isWorkloadPodTemplateMutated -func isWorkloadPodTemplateMutated(oldObject client.Object, object client.Object) bool { +// allowedToMutate returns if object is already being mutated or if auto restart is enabled. does not guarantee that the object will be mutated. +func allowedToMutate(oldObject client.Object, object client.Object, autoRestart bool) bool { + // mutating a namespace is always safe + if isNamespace(object) { + return true + } + // should only mutate workloads or namespaces + if !isWorkload(object) { + return false + } + + if autoRestart { + return true + } oldTemplate, newTemplate := getPodTemplate(oldObject), getPodTemplate(object) return !reflect.DeepEqual(oldTemplate, newTemplate) } +func isNamespace(object client.Object) bool { + switch object.(type) { + case *corev1.Namespace: + return true + } + return false +} + func getPodTemplate(obj client.Object) *corev1.PodTemplateSpec { switch o := obj.(type) { case *appsv1.Deployment: From c70f7f820cea35a97c5be8a823c1497e6f39b7eb Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 19 Mar 2025 10:36:08 -0400 Subject: [PATCH 09/63] blah --- pkg/instrumentation/auto/auto_monitor_test.go | 702 ++++++++---------- 1 file changed, 314 insertions(+), 388 deletions(-) diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index 52dd588c0..de8035c12 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -3,370 +3,17 @@ package auto import ( "context" "encoding/json" - "fmt" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/controller-runtime/pkg/client" + fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" ) -func TestMonitor_Selected(t *testing.T) { - logger.Info("Starting testmonitor tests") - - allTypes := instrumentation.NewTypeSet(instrumentation.SupportedTypes()...) - tests := []struct { - name string - service corev1.Service - oldWorkload client.Object - workload client.Object - config MonitorConfig - shouldMatch bool - }{ - { - name: "Should match Deployment with exact label match", - service: newTestService("svc-1", map[string]string{"app": "test"}), - workload: newTestDeployment("deploy-1", map[string]string{ - "app": "test", - }), - config: MonitorConfig{ - MonitorAllServices: true, - Languages: allTypes, - }, - shouldMatch: true, - }, - { - name: "Should not match Deployment with no labels", - service: newTestService("svc-2", map[string]string{"app": "test"}), - workload: newTestDeployment("deploy-2", map[string]string{}), - config: MonitorConfig{ - MonitorAllServices: true, - Languages: allTypes, - }, - shouldMatch: false, - }, - { - name: "Should not match when MonitorAllServices is false", - service: newTestService("svc-3", map[string]string{"app": "test"}), - workload: newTestDeployment("deploy-3", map[string]string{ - "app": "test", - }), - config: MonitorConfig{ - MonitorAllServices: false, - Languages: allTypes, - }, - shouldMatch: false, - }, - { - name: "Should match Deployment with multiple matching labels", - service: newTestService("svc-4", map[string]string{"app": "test", "env": "prod"}), - workload: newTestDeployment("deploy-4", map[string]string{ - "app": "test", - "env": "prod", - "extra": "label", - }), - config: MonitorConfig{ - MonitorAllServices: true, - Languages: allTypes, - }, - shouldMatch: true, - }, - { - name: "Should not match when service selector is empty", - service: newTestService("svc-5", map[string]string{}), - workload: newTestDeployment("deploy-5", map[string]string{ - "app": "test", - }), - config: MonitorConfig{ - MonitorAllServices: true, - Languages: allTypes, - }, - shouldMatch: false, - }, - { - name: "Should match StatefulSet with partial label match", - service: newTestService("svc-6", map[string]string{"app": "test"}), - workload: newTestStatefulSet("sts-1", map[string]string{ - "app": "test", - "other": "value", - }), - config: MonitorConfig{ - MonitorAllServices: true, - Languages: allTypes, - }, - shouldMatch: true, - }, - { - name: "Should match when languages are empty", - service: newTestService("svc-7", map[string]string{"app": "test"}), - workload: newTestDeployment("deploy-7", map[string]string{ - "app": "test", - }), - config: MonitorConfig{ - MonitorAllServices: true, - }, - shouldMatch: true, - }, - { - name: "Should match DaemonSet with exact label match", - service: newTestService("svc-8", map[string]string{"app": "test"}), - workload: newTestDaemonSet("ds-1", map[string]string{ - "app": "test", - }), - config: MonitorConfig{ - MonitorAllServices: true, - Languages: allTypes, - }, - shouldMatch: true, - }, - { - name: "Should not match DaemonSet with mismatched labels", - service: newTestService("svc-9", map[string]string{"app": "test"}), - workload: newTestDaemonSet("ds-2", map[string]string{ - "app": "different", - }), - config: MonitorConfig{ - MonitorAllServices: true, - Languages: allTypes, - }, - shouldMatch: false, - }, - { - name: "Should match StatefulSet with exact label match", - service: newTestService("svc-10", map[string]string{"app": "test"}), - workload: newTestStatefulSet("sts-2", map[string]string{ - "app": "test", - }), - config: MonitorConfig{ - MonitorAllServices: true, - Languages: allTypes, - }, - shouldMatch: true, - }, - { - name: "Should not match StatefulSet with mismatched labels", - service: newTestService("svc-11", map[string]string{"app": "test"}), - workload: newTestStatefulSet("sts-3", map[string]string{ - "app": "different", - }), - config: MonitorConfig{ - MonitorAllServices: true, - Languages: allTypes, - }, - shouldMatch: false, - }, - { - name: "Should match Deployment in custom selector regardless of MonitorAllServices", - service: newTestService("svc-12", map[string]string{"app": "different"}), - workload: newTestDeployment("custom-deploy-1", map[string]string{ - "app": "different", - }), - config: MonitorConfig{ - MonitorAllServices: false, - Languages: allTypes, - CustomSelector: AnnotationConfig{ - Java: AnnotationResources{ - Deployments: []string{"default/custom-deploy-1"}, - }, - }, - }, - shouldMatch: true, - }, - { - name: "Should match StatefulSet in custom selector", - service: newTestService("svc-13", map[string]string{"app": "different"}), - workload: newTestStatefulSet("custom-sts-1", map[string]string{ - "app": "different", - }), - config: MonitorConfig{ - MonitorAllServices: false, - Languages: allTypes, - CustomSelector: AnnotationConfig{ - Python: AnnotationResources{ - StatefulSets: []string{"default/custom-sts-1"}, - }, - }, - }, - shouldMatch: true, - }, - { - name: "Should match DaemonSet in custom selector", - service: newTestService("svc-14", map[string]string{"app": "different"}), - workload: newTestDaemonSet("custom-ds-1", map[string]string{ - "app": "different", - }), - config: MonitorConfig{ - MonitorAllServices: false, - Languages: allTypes, - CustomSelector: AnnotationConfig{ - NodeJS: AnnotationResources{ - DaemonSets: []string{"default/custom-ds-1"}, - }, - }, - }, - shouldMatch: true, - }, - { - name: "Should not match when workload not in custom selector", - service: newTestService("svc-15", map[string]string{"app": "different"}), - workload: newTestDeployment("non-custom-deploy", map[string]string{ - "app": "different", - }), - config: MonitorConfig{ - MonitorAllServices: false, - Languages: allTypes, - CustomSelector: AnnotationConfig{ - Java: AnnotationResources{ - Deployments: []string{"default/different-deploy"}, - }, - }, - }, - shouldMatch: false, - }, - { - name: "Should match when workload in custom selector for multiple languages", - service: newTestService("svc-16", map[string]string{"app": "different"}), - workload: newTestDeployment("multi-lang-deploy", map[string]string{ - "app": "different", - }), - config: MonitorConfig{ - MonitorAllServices: false, - Languages: allTypes, - CustomSelector: AnnotationConfig{ - Java: AnnotationResources{ - Deployments: []string{"default/multi-lang-deploy"}, - }, - Python: AnnotationResources{ - Deployments: []string{"default/multi-lang-deploy"}, - }, - }, - }, - shouldMatch: true, - }, - { - name: "Should match when workload in custom selector with multiple resource types", - service: newTestService("svc-17", map[string]string{"app": "different"}), - workload: newTestDeployment("mixed-resources-deploy", map[string]string{ - "app": "different", - }), - config: MonitorConfig{ - MonitorAllServices: false, - Languages: allTypes, - CustomSelector: AnnotationConfig{ - DotNet: AnnotationResources{ - Deployments: []string{"default/mixed-resources-deploy"}, - DaemonSets: []string{"default/some-ds"}, - StatefulSets: []string{"default/some-sts"}, - }, - }, - }, - shouldMatch: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - clientSet := fake.NewSimpleClientset() - - // Create service - _, err := clientSet.CoreV1().Services("default").Create(ctx, &tt.service, metav1.CreateOptions{}) - require.NoError(t, err) - - // Create workload - err = createWorkload(ctx, clientSet, tt.workload) - require.NoError(t, err) - - m := NewMonitor(ctx, tt.config, clientSet) - mutatedAnnotations := m.MutateObject(tt.workload, tt.workload) - - assert.Equal(t, tt.shouldMatch, mutatedAnnotations, - "Expected workload matching to be %v but got %v", tt.shouldMatch, mutatedAnnotations) - }) - } -} - -// Helper functions to create test resources -func newTestService(name string, selector map[string]string) corev1.Service { - return corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Selector: selector, - }, - } -} - -func newTestDeployment(name string, labels map[string]string) *appsv1.Deployment { - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "default", - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - }, - }, - } -} - -func newTestDaemonSet(name string, labels map[string]string) *appsv1.DaemonSet { - return &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "default", - }, - Spec: appsv1.DaemonSetSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - }, - }, - } -} - -func newTestStatefulSet(name string, labels map[string]string) *appsv1.StatefulSet { - return &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "default", - }, - Spec: appsv1.StatefulSetSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - }, - }, - } -} - -func createWorkload(ctx context.Context, clientSet *fake.Clientset, workload metav1.Object) error { - switch x := workload.(type) { - case *appsv1.Deployment: - _, err := clientSet.AppsV1().Deployments(x.GetNamespace()).Create(ctx, x, metav1.CreateOptions{}) - return err - case *appsv1.DaemonSet: - _, err := clientSet.AppsV1().DaemonSets(x.GetNamespace()).Create(ctx, x, metav1.CreateOptions{}) - return err - case *appsv1.StatefulSet: - _, err := clientSet.AppsV1().StatefulSets(x.GetNamespace()).Create(ctx, x, metav1.CreateOptions{}) - return err - default: - return fmt.Errorf("unsupported workload type: %T", workload) - } -} - func TestUnmarshal(t *testing.T) { j := []byte(`["java", "nodejs", "python"]`) set := instrumentation.TypeSet{} @@ -379,55 +26,53 @@ func TestMarshal(t *testing.T) { types := instrumentation.TypeSet{instrumentation.TypeNodeJS: nil, instrumentation.TypePython: nil} res, err := types.MarshalJSON() assert.NoError(t, err) - AssertJsonEqual(t, []byte(`["nodejs","python"]`), res) -} - -func AssertJsonEqual(t *testing.T, expectedJson []byte, actualJson []byte) { - var obj1, obj2 interface{} - - err := json.Unmarshal(expectedJson, &obj1) - assert.NoError(t, err) - - err = json.Unmarshal(actualJson, &obj2) - assert.NoError(t, err) - - assert.Equal(t, obj1, obj2) + assertJsonEqual(t, []byte(`["nodejs","python"]`), res) } -func Test_isWorkloadPodTemplateMutated(t *testing.T) { +func Test_allowedToMutate(t *testing.T) { deploy := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Image: "nginx:1.14.2"}}, + Containers: []corev1.Container{{Image: "nginx:1"}}, }, }, }, } - tests := []struct { - name string - oldObject client.Object - object client.Object - want bool - }{ - {"nil objects", nil, nil, true}, - {"identical deployments", deploy.DeepCopy(), deploy.DeepCopy(), false}, - {"changed pod template", deploy.DeepCopy(), &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Image: "nginx:1.15.0"}}, - }, + namespace := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + }, + } + mutatedDeploy := appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Image: "nginx:2"}}, }, }, - }, true}, - {"non-workload", &corev1.ConfigMap{}, &corev1.ConfigMap{}, true}, - {"create (oldObject nil)", nil, deploy.DeepCopy(), true}, + }, + } + tests := []struct { + name string + oldObject client.Object + object client.Object + autoRestart bool + want bool + }{ + {"identical deployments", deploy.DeepCopy(), deploy.DeepCopy(), false, false}, + {"identical deployments, auto restart", deploy.DeepCopy(), deploy.DeepCopy(), true, true}, //should try and mutate in case deployment should no longer have annotations and mutators need to run to remove annotations + {"changed pod template", deploy.DeepCopy(), &mutatedDeploy, false, true}, + {"non-workload", &corev1.ConfigMap{}, &corev1.ConfigMap{}, false, false}, + {"non-workload, auto restart", &corev1.ConfigMap{}, &corev1.ConfigMap{Data: map[string]string{"test": "test"}}, true, false}, + {"create (oldObject nil)", nil, deploy.DeepCopy(), false, true}, + {"namespace, auto restart false", nil, &namespace, false, true}, + {"namespace, auto restart true", nil, &namespace, true, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, isWorkloadPodTemplateMutated(tt.oldObject, tt.object)) + assert.Equal(t, tt.want, allowedToMutate(tt.oldObject, tt.object, tt.autoRestart)) }) } } @@ -458,6 +103,137 @@ func Test_getPodTemplate(t *testing.T) { } } +func TestMonitor_MutateObject(t *testing.T) { + tests := []struct { + name string + config MonitorConfig + deploymentNs string + serviceNs string + deploymentSelector map[string]string + serviceSelector map[string]string + expectedWorkloadAnnotations map[string]string + }{ + { + name: "same namespace, same selector, monitorallservices true, not excluded", + config: createConfig(true, nil, nil, false), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "same"}, + serviceSelector: map[string]string{"app": "same"}, + expectedWorkloadAnnotations: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "different namespace, same selector, monitorallservices true, not excluded", + config: createConfig(true, nil, nil, false), + deploymentNs: "namespace-2", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "same"}, + serviceSelector: map[string]string{"app": "same"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "same namespace, different selector, monitorallservices true, not excluded", + config: createConfig(true, nil, nil, false), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-2"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "same namespace, same selector, monitorallservices false, not excluded", + config: createConfig(false, nil, nil, false), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-1"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "same namespace, same selector, monitorallservices true, excluded namespace", + config: createConfig(false, []string{"namespace-1"}, nil, false), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-1"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "same namespace, same selector, monitorallservices true, excluded service", + config: createConfig(false, nil, []string{"namespace-1/svc-16"}, false), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-1"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test each workload type + workloadTypes := []struct { + name string + create func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) + }{ + { + name: "Deployment", + create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { + deployment := newTestDeployment("workload-16", ns, selector) + return clientset.AppsV1().Deployments(ns).Create(ctx, deployment, metav1.CreateOptions{}) + }, + }, + { + name: "StatefulSet", + create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { + statefulset := newTestStatefulSet("workload-16", ns, selector) + return clientset.AppsV1().StatefulSets(ns).Create(ctx, statefulset, metav1.CreateOptions{}) + }, + }, + { + name: "DaemonSet", + create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { + daemonset := newTestDaemonSet("workload-16", ns, selector) + return clientset.AppsV1().DaemonSets(ns).Create(ctx, daemonset, metav1.CreateOptions{}) + }, + }, + } + + for _, workload := range workloadTypes { + t.Run(workload.name, func(t *testing.T) { + // Setup fresh clients for each workload test + fakeClient := fake2.NewFakeClient() + clientset := fake.NewSimpleClientset() + ctx := context.TODO() + + monitor := NewMonitor(ctx, tt.config, clientset, fakeClient, fakeClient) + + // Create service + service := newTestService("svc-16", tt.serviceNs, tt.serviceSelector) + + // Setup test environment + serviceNamespace := createNamespace(t, clientset, ctx, service.Namespace) + if tt.deploymentNs != serviceNamespace.Name { + createNamespace(t, clientset, ctx, tt.deploymentNs) + } + + // Create service + _, err := clientset.CoreV1().Services(service.Namespace).Create(ctx, service, metav1.CreateOptions{}) + assert.NoError(t, err) + + // Create workload + workloadObj, err := workload.create(clientset, ctx, tt.deploymentNs, tt.deploymentSelector) + assert.NoError(t, err) + + // Test + mutatedAnnotations := monitor.MutateObject(nil, workloadObj) + assert.Equal(t, tt.expectedWorkloadAnnotations, mutatedAnnotations) + }) + } + }) + } +} + func Test_mutate(t *testing.T) { tests := []struct { name string @@ -521,16 +297,150 @@ func Test_mutate(t *testing.T) { wantObjAnnotations: buildAnnotations("java"), wantMutated: buildAnnotations("python"), }, + { + name: "manually specified annotation is not touched", + obj: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{instrumentation.InjectAnnotationKey(instrumentation.TypeJava): defaultAnnotationValue}, + }, + }, + }, + }, + languagesToMonitor: instrumentation.TypeSet{}, + wantObjAnnotations: map[string]string{instrumentation.InjectAnnotationKey(instrumentation.TypeJava): defaultAnnotationValue}, + wantMutated: map[string]string{}, + }, + { + name: "remove all", + obj: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: buildAnnotations("java"), + }, + }, + }, + }, + languagesToMonitor: instrumentation.TypeSet{}, + wantObjAnnotations: map[string]string{}, + wantMutated: buildAnnotations("java"), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { gotMutated := mutate(tt.obj, tt.languagesToMonitor) - assert.Equal(t, tt.wantObjAnnotations, tt.obj.GetAnnotations()) + switch tt.obj.(type) { + case *appsv1.Deployment, *appsv1.DaemonSet, *appsv1.StatefulSet: + assert.Equal(t, tt.wantObjAnnotations, getPodTemplate(tt.obj).GetAnnotations()) + default: + assert.Equal(t, tt.wantObjAnnotations, tt.obj.GetAnnotations()) + } assert.Equal(t, tt.wantMutated, gotMutated) }) } } + +func Test_StartupMutateObject(t *testing.T) { + testService := newTestService("service-1", "default", map[string]string{"test": "test"}) + testDeployment := newTestDeployment("deployment-1", "default", map[string]string{"test": "test"}) + notMatchingService := newTestService("service-2", "default", map[string]string{"test2": "test2"}) + config := createConfig(true, nil, nil, true) + clientset := fake.NewSimpleClientset(testService, testDeployment, notMatchingService) + _ = NewMonitor(context.TODO(), config, clientset, fake2.NewFakeClient(), fake2.NewFakeClient()) + // todo finish +} + +func assertJsonEqual(t *testing.T, expectedJson []byte, actualJson []byte) { + var obj1, obj2 interface{} + + err := json.Unmarshal(expectedJson, &obj1) + assert.NoError(t, err) + + err = json.Unmarshal(actualJson, &obj2) + assert.NoError(t, err) + + assert.Equal(t, obj1, obj2) +} + +func createNamespace(t *testing.T, clientset *fake.Clientset, ctx context.Context, namespaceName string) *corev1.Namespace { + serviceNamespace, err := clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, metav1.CreateOptions{}) + assert.NoError(t, err) + return serviceNamespace +} + +func newTestService(name string, namespace string, selector map[string]string) *corev1.Service { + service := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: selector, + }, + } + return service.DeepCopy() +} + +func newTestDeployment(name string, namespace string, labels map[string]string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + }, + }, + } +} + +func newTestStatefulSet(name, namespace string, selector map[string]string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: selector, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selector, + }, + }, + }, + } +} + +func newTestDaemonSet(name, namespace string, selector map[string]string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: selector, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selector, + }, + }, + }, + } +} + func mergeMaps(maps ...map[string]string) map[string]string { result := make(map[string]string) for _, m := range maps { @@ -540,3 +450,19 @@ func mergeMaps(maps ...map[string]string) map[string]string { } return result } + +func createConfig(monitorAll bool, excludedNs, excludedSvcs []string, autoRestart bool) MonitorConfig { + return MonitorConfig{ + MonitorAllServices: monitorAll, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + AutoRestart: autoRestart, + Exclude: struct { + Namespaces []string `json:"namespaces"` + Services []string `json:"services"` + }{ + Namespaces: excludedNs, + Services: excludedSvcs, + }, + CustomSelector: AnnotationConfig{}, + } +} From e47e85a7feccb7d7b36e5a32251da33a1253727c Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 19 Mar 2025 15:37:33 -0400 Subject: [PATCH 10/63] fix logging, add more testing --- main.go | 2 +- pkg/instrumentation/auto/annotation.go | 1 - pkg/instrumentation/auto/auto_monitor.go | 89 +++--- pkg/instrumentation/auto/auto_monitor_test.go | 260 +++++++++++------- 4 files changed, 206 insertions(+), 146 deletions(-) diff --git a/main.go b/main.go index fa750abec..2c062cea1 100644 --- a/main.go +++ b/main.go @@ -392,7 +392,7 @@ func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context, c setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") return auto.NoopMonitor{} } - monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader) + monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, setupLog) } return monitor } diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index b0bcf9f94..172c8081d 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -38,7 +38,6 @@ type AnnotationMutators struct { statefulSetMutators map[string]instrumentation.AnnotationMutator defaultMutator instrumentation.AnnotationMutator injectAnnotations map[string]struct{} - monitor *Monitor cfg *AnnotationConfig } diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index 7bd10b7dd..e4c2ae3ab 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -11,17 +12,14 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "k8s.io/utils/strings/slices" "os" "reflect" "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" + "slices" "strings" "time" ) -var logger = logf.Log.WithName("auto_monitor") - type MonitorInterface interface { MutateObject(oldObj client.Object, obj client.Object) map[string]string AnyCustomSelectorDefined() bool @@ -33,6 +31,7 @@ type Monitor struct { config MonitorConfig k8sInterface kubernetes.Interface customSelectors *AnnotationMutators + logger logr.Logger } type NoopMonitor struct{} @@ -45,17 +44,22 @@ func (n NoopMonitor) AnyCustomSelectorDefined() bool { return false } -func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, c client.Client, r client.Reader) *Monitor { +func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, c client.Client, r client.Reader, logger logr.Logger) *Monitor { logger.Info("AutoMonitor starting...") // todo, throw warning if exclude config service is not namespaced (doesn't contain `/`) // todo: informers.WithTransform() as option to only store what parts of service are needed factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute) serviceInformer := factory.Core().V1().Services().Informer() - m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: NewAnnotationMutators(c, r, logger, config.CustomSelector, config.Languages)} + m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: NewAnnotationMutators(c, r, logger, config.CustomSelector, instrumentation.NewTypeSet(instrumentation.SupportedTypes()...))} _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: func(obj interface{}) { if serviceInformer.HasSynced() { - m.onServiceAdd(obj) + service, ok := obj.(*corev1.Service) + if !ok { + logger.Error(nil, "Service informer is unable to cast obj to (*corev1.Service)") + panic("AHHHHH!!!!") + } + m.onServiceAdd(service) } else { logger.Info(fmt.Sprintf("Service %v has not synced yet, this is first sync. skipping annotation", obj)) } @@ -84,9 +88,8 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet if err != nil { return nil } - for _, service := range list.Items { - m.onServiceAdd(service) + m.onServiceAdd(&service) } } else { logger.Info("Auto restart disabled. To instrument workloads, restart the workloads exposed by a service.") @@ -95,8 +98,7 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet return m } -func (m *Monitor) onServiceAdd(obj interface{}) { - service := obj.(*corev1.Service) +func (m *Monitor) onServiceAdd(service *corev1.Service) { if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 { return } @@ -106,66 +108,69 @@ func (m *Monitor) onServiceAdd(obj interface{}) { return } namespace := service.GetNamespace() - for _, resource := range listServiceDeployments(m.k8sInterface, service, m.ctx).Items { + for _, resource := range m.listServiceDeployments(service, m.ctx) { mutatedAnnotations := m.MutateServiceWorkload(&resource, service) if len(mutatedAnnotations) == 0 { continue } _, err := m.k8sInterface.AppsV1().Deployments(namespace).Update(m.ctx, &resource, metav1.UpdateOptions{}) if err != nil { - logger.Error(err, "failed to update deployment") + m.logger.Error(err, "failed to update deployment") } } - for _, resource := range listServiceStatefulSets(m.k8sInterface, service, m.ctx).Items { + for _, resource := range m.listServiceStatefulSets(service, m.ctx) { mutatedAnnotations := m.MutateServiceWorkload(&resource, service) if len(mutatedAnnotations) == 0 { continue } _, err := m.k8sInterface.AppsV1().StatefulSets(namespace).Update(m.ctx, &resource, metav1.UpdateOptions{}) if err != nil { - logger.Error(err, "failed to update statefulset") + m.logger.Error(err, "failed to update statefulset") } } - for _, resource := range listServiceDaemonSets(m.k8sInterface, service, m.ctx).Items { + for _, resource := range m.listServiceDaemonSets(service, m.ctx) { mutatedAnnotations := m.MutateServiceWorkload(&resource, service) if len(mutatedAnnotations) == 0 { continue } _, err := m.k8sInterface.AppsV1().DaemonSets(namespace).Update(m.ctx, &resource, metav1.UpdateOptions{}) if err != nil { - logger.Error(err, "failed to update daemonset") + m.logger.Error(err, "failed to update daemonset") } } } -func listServiceDeployments(k8sInterface kubernetes.Interface, service *corev1.Service, ctx context.Context) *appsv1.DeploymentList { - list, err := k8sInterface.AppsV1().Deployments(service.GetNamespace()).List(ctx, metav1.ListOptions{ - LabelSelector: labels.FormatLabels(service.Spec.Selector), - }) +func (m *Monitor) listServiceDeployments(service *corev1.Service, ctx context.Context) []appsv1.Deployment { + list, err := m.k8sInterface.AppsV1().Deployments(service.GetNamespace()).List(ctx, metav1.ListOptions{}) if err != nil { - logger.Error(err, "AutoMonitor failed to list deployments") + m.logger.Error(err, "AutoMonitor failed to list deployments") } - return list + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + return slices.DeleteFunc(list.Items, func(deployment appsv1.Deployment) bool { + return !serviceSelector.Matches(getTemplateSpecLabels(&deployment)) + }) } -func listServiceStatefulSets(k8sInterface kubernetes.Interface, service *corev1.Service, ctx context.Context) *appsv1.StatefulSetList { - list, err := k8sInterface.AppsV1().StatefulSets(service.GetNamespace()).List(ctx, metav1.ListOptions{ - LabelSelector: labels.FormatLabels(service.Spec.Selector), - }) +func (m *Monitor) listServiceStatefulSets(service *corev1.Service, ctx context.Context) []appsv1.StatefulSet { + list, err := m.k8sInterface.AppsV1().StatefulSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) if err != nil { - logger.Error(err, "AutoMonitor failed to list statefulsets") + m.logger.Error(err, "AutoMonitor failed to list statefulsets") } - return list + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + return slices.DeleteFunc(list.Items, func(daemonSet appsv1.StatefulSet) bool { + return !serviceSelector.Matches(getTemplateSpecLabels(&daemonSet)) + }) } -func listServiceDaemonSets(k8sInterface kubernetes.Interface, service *corev1.Service, ctx context.Context) *appsv1.DaemonSetList { - list, err := k8sInterface.AppsV1().DaemonSets(service.GetNamespace()).List(ctx, metav1.ListOptions{ - LabelSelector: labels.FormatLabels(service.Spec.Selector), - }) +func (m *Monitor) listServiceDaemonSets(service *corev1.Service, ctx context.Context) []appsv1.DaemonSet { + list, err := m.k8sInterface.AppsV1().DaemonSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) if err != nil { - logger.Error(err, "AutoMonitor failed to list DaemonSets") + m.logger.Error(err, "AutoMonitor failed to list DaemonSets") } - return list + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + return slices.DeleteFunc(list.Items, func(daemonSet appsv1.DaemonSet) bool { + return !serviceSelector.Matches(getTemplateSpecLabels(&daemonSet)) + }) } func getTemplateSpecLabels(obj metav1.Object) labels.Set { @@ -189,7 +194,7 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { // TODO: remove? func (m *Monitor) MutateServiceWorkload(obj client.Object, service *corev1.Service) map[string]string { if customSelectLanguages, selected := m.CustomSelected(obj); selected { - logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) + m.logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) return mutate(obj, customSelectLanguages) } @@ -204,7 +209,7 @@ func (m *Monitor) MutateServiceWorkload(obj client.Object, service *corev1.Servi return map[string]string{} } - logger.Info(fmt.Sprintf("start up: setting %s instrumentation annotations to %s because it is owned by service %s", obj.GetName(), m.config.Languages, service.Name)) + m.logger.Info(fmt.Sprintf("start up: setting %s instrumentation annotations to %s because it is owned by service %s", obj.GetName(), m.config.Languages, service.Name)) return mutate(obj, m.config.Languages) } @@ -213,7 +218,7 @@ func (m *Monitor) MutateObject(oldObj client.Object, obj client.Object) map[stri // todo: handle edge case where a workload is annotated because a service exposed it, and the service is removed. aka add to Service OnDelete // custom selector takes precedence over service selector if customSelectLanguages, selected := m.CustomSelected(obj); selected { - logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) + m.logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) return mutate(obj, customSelectLanguages) } @@ -241,7 +246,7 @@ func (m *Monitor) MutateObject(oldObj client.Object, obj client.Object) map[stri serviceSelector := labels.SelectorFromSet(service.Spec.Selector) if serviceSelector.Matches(objectLabels) { - logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is owned by service %s", obj.GetName(), m.config.Languages, service.Name)) + m.logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is owned by service %s", obj.GetName(), m.config.Languages, service.Name)) return mutate(obj, m.config.Languages) } } @@ -260,10 +265,10 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet) ma annotations := obj.GetAnnotations() if annotations == nil { - annotations = make(map[string]string) + annotations = map[string]string{} } - allMutatedAnnotations := make(map[string]string) + allMutatedAnnotations := map[string]string{} for _, language := range instrumentation.SupportedTypes() { insertMutation, removeMutation := buildMutations(language) var mutatedAnnotations map[string]string @@ -289,7 +294,7 @@ func (m *Monitor) CustomSelected(obj client.Object) (instrumentation.TypeSet, bo // excludedService returns whether a Namespace or a Service is excludedService from AutoMonitor. func (m *Monitor) excludedService(obj client.Object) bool { excluded := slices.Contains(m.config.Exclude.Services, namespacedName(obj)) || m.excludedNamespace(obj.GetNamespace()) - logger.Info(fmt.Sprintf("%s excluded? %v", namespacedName(obj), excluded)) + m.logger.Info(fmt.Sprintf("%s excluded? %v", namespacedName(obj), excluded)) return excluded } diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index de8035c12..33409410c 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -4,14 +4,18 @@ import ( "context" "encoding/json" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/go-logr/logr/testr" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/controller-runtime/pkg/client" fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" + "time" ) func TestUnmarshal(t *testing.T) { @@ -26,6 +30,7 @@ func TestMarshal(t *testing.T) { types := instrumentation.TypeSet{instrumentation.TypeNodeJS: nil, instrumentation.TypePython: nil} res, err := types.MarshalJSON() assert.NoError(t, err) + // todo test irregardless of order assertJsonEqual(t, []byte(`["nodejs","python"]`), res) } @@ -169,44 +174,44 @@ func TestMonitor_MutateObject(t *testing.T) { }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test each workload type - workloadTypes := []struct { - name string - create func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) - }{ - { - name: "Deployment", - create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { - deployment := newTestDeployment("workload-16", ns, selector) - return clientset.AppsV1().Deployments(ns).Create(ctx, deployment, metav1.CreateOptions{}) - }, - }, - { - name: "StatefulSet", - create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { - statefulset := newTestStatefulSet("workload-16", ns, selector) - return clientset.AppsV1().StatefulSets(ns).Create(ctx, statefulset, metav1.CreateOptions{}) - }, - }, - { - name: "DaemonSet", - create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { - daemonset := newTestDaemonSet("workload-16", ns, selector) - return clientset.AppsV1().DaemonSets(ns).Create(ctx, daemonset, metav1.CreateOptions{}) - }, - }, - } + workloadTypes := []struct { + name string + create func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) + }{ + { + name: "Deployment", + create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { + deployment := newTestDeployment("workload-16", ns, selector) + return clientset.AppsV1().Deployments(ns).Create(ctx, deployment, metav1.CreateOptions{}) + }, + }, + { + name: "StatefulSet", + create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { + statefulset := newTestStatefulSet("workload-16", ns, selector) + return clientset.AppsV1().StatefulSets(ns).Create(ctx, statefulset, metav1.CreateOptions{}) + }, + }, + { + name: "DaemonSet", + create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { + daemonset := newTestDaemonSet("workload-16", ns, selector) + return clientset.AppsV1().DaemonSets(ns).Create(ctx, daemonset, metav1.CreateOptions{}) + }, + }, + } - for _, workload := range workloadTypes { - t.Run(workload.name, func(t *testing.T) { + for _, workload := range workloadTypes { + t.Run(workload.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { // Setup fresh clients for each workload test fakeClient := fake2.NewFakeClient() clientset := fake.NewSimpleClientset() ctx := context.TODO() - monitor := NewMonitor(ctx, tt.config, clientset, fakeClient, fakeClient) + logger := testr.New(t) + monitor := NewMonitor(ctx, tt.config, clientset, fakeClient, fakeClient, logger) // Create service service := newTestService("svc-16", tt.serviceNs, tt.serviceSelector) @@ -224,6 +229,11 @@ func TestMonitor_MutateObject(t *testing.T) { // Create workload workloadObj, err := workload.create(clientset, ctx, tt.deploymentNs, tt.deploymentSelector) assert.NoError(t, err) + // need to wait until service informer is updated + err = wait.PollImmediate(1*time.Millisecond, 5*time.Millisecond, func() (bool, error) { + return len(monitor.serviceInformer.GetStore().ListKeys()) > 0, nil + }) + assert.NoError(t, err) // Test mutatedAnnotations := monitor.MutateObject(nil, workloadObj) @@ -237,29 +247,21 @@ func TestMonitor_MutateObject(t *testing.T) { func Test_mutate(t *testing.T) { tests := []struct { name string - obj client.Object + podAnnotations map[string]string languagesToMonitor instrumentation.TypeSet wantObjAnnotations map[string]string wantMutated map[string]string }{ { - name: "deployment - java only", - obj: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{}, - }, - }, + name: "java only", + podAnnotations: nil, languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, wantObjAnnotations: buildAnnotations("java"), wantMutated: buildAnnotations("java"), }, { - name: "deployment - java and python", - obj: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{}, - }, - }, + name: "java and python", + podAnnotations: nil, languagesToMonitor: instrumentation.TypeSet{ "java": struct{}{}, "python": struct{}{}, @@ -268,91 +270,141 @@ func Test_mutate(t *testing.T) { wantMutated: mergeMaps(buildAnnotations("java"), buildAnnotations("python")), }, { - name: "remove python instrumentation", - obj: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: buildAnnotations("python"), - }, - }, - }, - }, + name: "remove python instrumentation", + podAnnotations: buildAnnotations("python"), languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{}, wantMutated: buildAnnotations("python"), }, { - name: "remove one of two languages", - obj: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: mergeMaps(buildAnnotations("python"), buildAnnotations("java")), - }, - }, - }, - }, + name: "remove one of two languages", + podAnnotations: mergeMaps(buildAnnotations("python"), buildAnnotations("java")), languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, wantObjAnnotations: buildAnnotations("java"), wantMutated: buildAnnotations("python"), }, { - name: "manually specified annotation is not touched", - obj: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{instrumentation.InjectAnnotationKey(instrumentation.TypeJava): defaultAnnotationValue}, - }, - }, - }, - }, + name: "manually specified annotation is not touched", + podAnnotations: map[string]string{instrumentation.InjectAnnotationKey(instrumentation.TypeJava): defaultAnnotationValue}, languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{instrumentation.InjectAnnotationKey(instrumentation.TypeJava): defaultAnnotationValue}, wantMutated: map[string]string{}, }, { - name: "remove all", - obj: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: buildAnnotations("java"), - }, - }, - }, - }, + name: "remove all", + podAnnotations: buildAnnotations("java"), languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{}, wantMutated: buildAnnotations("java"), }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotMutated := mutate(tt.obj, tt.languagesToMonitor) - switch tt.obj.(type) { - case *appsv1.Deployment, *appsv1.DaemonSet, *appsv1.StatefulSet: - assert.Equal(t, tt.wantObjAnnotations, getPodTemplate(tt.obj).GetAnnotations()) - default: - assert.Equal(t, tt.wantObjAnnotations, tt.obj.GetAnnotations()) + workloadTypes := []struct { + name string + create func(annotations map[string]string) client.Object + }{ + { + name: "Deployment", + create: func(annotations map[string]string) client.Object { + return &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, + }, + }, + } + }, + }, + { + name: "StatefulSet", + create: func(annotations map[string]string) client.Object { + return &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, + }, + }, + } + }, + }, + { + name: "DaemonSet", + create: func(annotations map[string]string) client.Object { + return &appsv1.DaemonSet{ + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, + }, + }, + } + }, + }, + } + + for _, workload := range workloadTypes { + t.Run(workload.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + obj := workload.create(tt.podAnnotations).DeepCopyObject().(client.Object) + gotMutated := mutate(obj, tt.languagesToMonitor) + assert.Equal(t, tt.wantObjAnnotations, getPodTemplate(obj).GetAnnotations()) + assert.Equal(t, tt.wantMutated, gotMutated) + }) } - assert.Equal(t, tt.wantMutated, gotMutated) }) } } -func Test_StartupMutateObject(t *testing.T) { +func Test_StartupAutoRestart(t *testing.T) { + service := newTestService("service-1", "default", map[string]string{"test": "test"}) + matchingDeployment := newTestDeployment("deployment-1", "default", map[string]string{"test": "test"}) + nonMatchingDeployment := newTestDeployment("deployment-2", "default", map[string]string{}) + customSelectedDeployment := newTestDeployment("deployment-3", "default", map[string]string{}) + config := MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + AutoRestart: true, + CustomSelector: AnnotationConfig{ + AnnotationResources{}, AnnotationResources{Deployments: []string{namespacedName(customSelectedDeployment)}}, + AnnotationResources{}, AnnotationResources{}, + }, + } + objs := []runtime.Object{service, matchingDeployment, nonMatchingDeployment, customSelectedDeployment} + clientset := fake.NewSimpleClientset(objs...) + fakeClient := fake2.NewFakeClient(objs...) + m := NewMonitor(context.TODO(), config, clientset, fakeClient, fakeClient, testr.New(t)) + + updatedMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments("default").Get(context.TODO(), matchingDeployment.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, buildAnnotations(instrumentation.TypeJava), updatedMatchingDeployment.Spec.Template.GetAnnotations()) + updatedNonMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments("default").Get(context.TODO(), nonMatchingDeployment.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Empty(t, updatedNonMatchingDeployment.Spec.Template.GetAnnotations()) + err = fakeClient.Get(context.TODO(), client.ObjectKeyFromObject(customSelectedDeployment), customSelectedDeployment) + assert.NoError(t, err) + assert.Equal(t, buildAnnotations(instrumentation.TypePython), customSelectedDeployment.Spec.Template.GetAnnotations()) +} + +func Test_listServiceDeployments(t *testing.T) { testService := newTestService("service-1", "default", map[string]string{"test": "test"}) testDeployment := newTestDeployment("deployment-1", "default", map[string]string{"test": "test"}) notMatchingService := newTestService("service-2", "default", map[string]string{"test2": "test2"}) - config := createConfig(true, nil, nil, true) clientset := fake.NewSimpleClientset(testService, testDeployment, notMatchingService) - _ = NewMonitor(context.TODO(), config, clientset, fake2.NewFakeClient(), fake2.NewFakeClient()) - // todo finish + m := Monitor{k8sInterface: clientset, logger: testr.New(t)} + matchingServiceDeployments := m.listServiceDeployments(testService, context.TODO()) + assert.Len(t, matchingServiceDeployments, 1) + notMatchingServiceDeployments := m.listServiceDeployments(notMatchingService, context.TODO()) + assert.Len(t, notMatchingServiceDeployments, 0) } +// Helper functions + func assertJsonEqual(t *testing.T, expectedJson []byte, actualJson []byte) { var obj1, obj2 interface{} @@ -366,7 +418,8 @@ func assertJsonEqual(t *testing.T, expectedJson []byte, actualJson []byte) { } func createNamespace(t *testing.T, clientset *fake.Clientset, ctx context.Context, namespaceName string) *corev1.Namespace { - serviceNamespace, err := clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, metav1.CreateOptions{}) + namespace := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}} + serviceNamespace, err := clientset.CoreV1().Namespaces().Create(ctx, &namespace, metav1.CreateOptions{}) assert.NoError(t, err) return serviceNamespace } @@ -385,7 +438,7 @@ func newTestService(name string, namespace string, selector map[string]string) * } func newTestDeployment(name string, namespace string, labels map[string]string) *appsv1.Deployment { - return &appsv1.Deployment{ + deployment := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, @@ -401,10 +454,11 @@ func newTestDeployment(name string, namespace string, labels map[string]string) }, }, } + return deployment.DeepCopy() } func newTestStatefulSet(name, namespace string, selector map[string]string) *appsv1.StatefulSet { - return &appsv1.StatefulSet{ + statefulSet := appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, @@ -420,10 +474,11 @@ func newTestStatefulSet(name, namespace string, selector map[string]string) *app }, }, } + return statefulSet.DeepCopy() } func newTestDaemonSet(name, namespace string, selector map[string]string) *appsv1.DaemonSet { - return &appsv1.DaemonSet{ + daemonSet := appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, @@ -439,6 +494,7 @@ func newTestDaemonSet(name, namespace string, selector map[string]string) *appsv }, }, } + return daemonSet.DeepCopy() } func mergeMaps(maps ...map[string]string) map[string]string { From 6ab7980c10c59c16714a1407ebec064985642794 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 19 Mar 2025 15:48:18 -0400 Subject: [PATCH 11/63] remove unnecessary HasSynced --- pkg/instrumentation/auto/auto_monitor.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index e4c2ae3ab..1d28d83a5 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -53,16 +53,8 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: NewAnnotationMutators(c, r, logger, config.CustomSelector, instrumentation.NewTypeSet(instrumentation.SupportedTypes()...))} _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: func(obj interface{}) { - if serviceInformer.HasSynced() { - service, ok := obj.(*corev1.Service) - if !ok { - logger.Error(nil, "Service informer is unable to cast obj to (*corev1.Service)") - panic("AHHHHH!!!!") - } - m.onServiceAdd(service) - } else { - logger.Info(fmt.Sprintf("Service %v has not synced yet, this is first sync. skipping annotation", obj)) - } + m.onServiceAdd(obj.(*corev1.Service)) + }}) if err != nil { logger.Error(err, "failed to start auto monitor") @@ -143,7 +135,7 @@ func (m *Monitor) onServiceAdd(service *corev1.Service) { func (m *Monitor) listServiceDeployments(service *corev1.Service, ctx context.Context) []appsv1.Deployment { list, err := m.k8sInterface.AppsV1().Deployments(service.GetNamespace()).List(ctx, metav1.ListOptions{}) if err != nil { - m.logger.Error(err, "AutoMonitor failed to list deployments") + m.logger.Error(err, "failed to list deployments") } serviceSelector := labels.SelectorFromSet(service.Spec.Selector) return slices.DeleteFunc(list.Items, func(deployment appsv1.Deployment) bool { @@ -154,7 +146,7 @@ func (m *Monitor) listServiceDeployments(service *corev1.Service, ctx context.Co func (m *Monitor) listServiceStatefulSets(service *corev1.Service, ctx context.Context) []appsv1.StatefulSet { list, err := m.k8sInterface.AppsV1().StatefulSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) if err != nil { - m.logger.Error(err, "AutoMonitor failed to list statefulsets") + m.logger.Error(err, "failed to list statefulsets") } serviceSelector := labels.SelectorFromSet(service.Spec.Selector) return slices.DeleteFunc(list.Items, func(daemonSet appsv1.StatefulSet) bool { @@ -165,7 +157,7 @@ func (m *Monitor) listServiceStatefulSets(service *corev1.Service, ctx context.C func (m *Monitor) listServiceDaemonSets(service *corev1.Service, ctx context.Context) []appsv1.DaemonSet { list, err := m.k8sInterface.AppsV1().DaemonSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) if err != nil { - m.logger.Error(err, "AutoMonitor failed to list DaemonSets") + m.logger.Error(err, "failed to list DaemonSets") } serviceSelector := labels.SelectorFromSet(service.Spec.Selector) return slices.DeleteFunc(list.Items, func(daemonSet appsv1.DaemonSet) bool { @@ -182,8 +174,6 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { return t.Spec.Template.Labels case *appsv1.DaemonSet: return t.Spec.Template.Labels - case *appsv1.ReplicaSet: - return t.Spec.Template.Labels default: // Return empty labels.Set if the object type is not supported return labels.Set{} From 82750e1b524774db4f5d2c35795da16b5a3032bb Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 20 Mar 2025 12:24:42 -0400 Subject: [PATCH 12/63] Handle service deletion, service update. test service deletion --- pkg/instrumentation/auto/auto_monitor.go | 167 +++++++++--------- pkg/instrumentation/auto/auto_monitor_test.go | 144 ++++++++++++--- 2 files changed, 208 insertions(+), 103 deletions(-) diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index 1d28d83a5..2529ec6d9 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -52,10 +52,17 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet serviceInformer := factory.Core().V1().Services().Informer() m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: NewAnnotationMutators(c, r, logger, config.CustomSelector, instrumentation.NewTypeSet(instrumentation.SupportedTypes()...))} - _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: func(obj interface{}) { - m.onServiceAdd(obj.(*corev1.Service)) - - }}) + _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + m.onServiceEvent(nil, obj.(*corev1.Service)) + }, + UpdateFunc: func(oldObj, obj interface{}) { + m.onServiceEvent(oldObj.(*corev1.Service), obj.(*corev1.Service)) + }, + DeleteFunc: func(obj interface{}) { + m.onServiceEvent(obj.(*corev1.Service), nil) + }, + }) if err != nil { logger.Error(err, "failed to start auto monitor") return nil @@ -81,7 +88,7 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet return nil } for _, service := range list.Items { - m.onServiceAdd(&service) + m.onServiceEvent(nil, &service) } } else { logger.Info("Auto restart disabled. To instrument workloads, restart the workloads exposed by a service.") @@ -90,79 +97,97 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet return m } -func (m *Monitor) onServiceAdd(service *corev1.Service) { - if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 { +func (m *Monitor) onServiceEvent(oldService *corev1.Service, service *corev1.Service) { + if !m.config.AutoRestart { return } - - // we should not execute this code on start up because it needs to iterate over all services in MutateObject, - if !m.config.AutoRestart || m.excludedService(service) { - return - } - namespace := service.GetNamespace() - for _, resource := range m.listServiceDeployments(service, m.ctx) { - mutatedAnnotations := m.MutateServiceWorkload(&resource, service) + for _, resource := range m.listServiceDeployments(m.ctx, oldService, service) { + mutatedAnnotations := m.MutateObject(&resource, &resource) if len(mutatedAnnotations) == 0 { continue } - _, err := m.k8sInterface.AppsV1().Deployments(namespace).Update(m.ctx, &resource, metav1.UpdateOptions{}) + _, err := m.k8sInterface.AppsV1().Deployments(resource.GetNamespace()).Update(m.ctx, &resource, metav1.UpdateOptions{}) if err != nil { m.logger.Error(err, "failed to update deployment") } } - for _, resource := range m.listServiceStatefulSets(service, m.ctx) { - mutatedAnnotations := m.MutateServiceWorkload(&resource, service) + for _, resource := range m.listServiceStatefulSets(m.ctx, oldService, service) { + mutatedAnnotations := m.MutateObject(&resource, &resource) if len(mutatedAnnotations) == 0 { continue } - _, err := m.k8sInterface.AppsV1().StatefulSets(namespace).Update(m.ctx, &resource, metav1.UpdateOptions{}) + _, err := m.k8sInterface.AppsV1().StatefulSets(resource.GetNamespace()).Update(m.ctx, &resource, metav1.UpdateOptions{}) if err != nil { m.logger.Error(err, "failed to update statefulset") } } - for _, resource := range m.listServiceDaemonSets(service, m.ctx) { - mutatedAnnotations := m.MutateServiceWorkload(&resource, service) + for _, resource := range m.listServiceDaemonSets(m.ctx, oldService, service) { + mutatedAnnotations := m.MutateObject(&resource, &resource) if len(mutatedAnnotations) == 0 { continue } - _, err := m.k8sInterface.AppsV1().DaemonSets(namespace).Update(m.ctx, &resource, metav1.UpdateOptions{}) + _, err := m.k8sInterface.AppsV1().DaemonSets(resource.GetNamespace()).Update(m.ctx, &resource, metav1.UpdateOptions{}) if err != nil { m.logger.Error(err, "failed to update daemonset") } } } -func (m *Monitor) listServiceDeployments(service *corev1.Service, ctx context.Context) []appsv1.Deployment { - list, err := m.k8sInterface.AppsV1().Deployments(service.GetNamespace()).List(ctx, metav1.ListOptions{}) - if err != nil { - m.logger.Error(err, "failed to list deployments") +func (m *Monitor) listServiceDeployments(ctx context.Context, services ...*corev1.Service) []appsv1.Deployment { + var deployments []appsv1.Deployment + for _, service := range services { + if service == nil { + continue + } + list, err := m.k8sInterface.AppsV1().Deployments(service.GetNamespace()).List(ctx, metav1.ListOptions{}) + if err != nil { + m.logger.Error(err, "failed to list deployments") + } + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + trimmed := slices.DeleteFunc(list.Items, func(deployment appsv1.Deployment) bool { + return !serviceSelector.Matches(getTemplateSpecLabels(&deployment)) + }) + deployments = append(deployments, trimmed...) } - serviceSelector := labels.SelectorFromSet(service.Spec.Selector) - return slices.DeleteFunc(list.Items, func(deployment appsv1.Deployment) bool { - return !serviceSelector.Matches(getTemplateSpecLabels(&deployment)) - }) + return deployments } -func (m *Monitor) listServiceStatefulSets(service *corev1.Service, ctx context.Context) []appsv1.StatefulSet { - list, err := m.k8sInterface.AppsV1().StatefulSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) - if err != nil { - m.logger.Error(err, "failed to list statefulsets") +func (m *Monitor) listServiceStatefulSets(ctx context.Context, services ...*corev1.Service) []appsv1.StatefulSet { + var statefulSets []appsv1.StatefulSet + for _, service := range services { + if service == nil { + continue + } + list, err := m.k8sInterface.AppsV1().StatefulSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) + if err != nil { + m.logger.Error(err, "failed to list statefulsets") + } + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + trimmed := slices.DeleteFunc(list.Items, func(statefulSet appsv1.StatefulSet) bool { + return !serviceSelector.Matches(getTemplateSpecLabels(&statefulSet)) + }) + statefulSets = append(statefulSets, trimmed...) } - serviceSelector := labels.SelectorFromSet(service.Spec.Selector) - return slices.DeleteFunc(list.Items, func(daemonSet appsv1.StatefulSet) bool { - return !serviceSelector.Matches(getTemplateSpecLabels(&daemonSet)) - }) + return statefulSets } -func (m *Monitor) listServiceDaemonSets(service *corev1.Service, ctx context.Context) []appsv1.DaemonSet { - list, err := m.k8sInterface.AppsV1().DaemonSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) - if err != nil { - m.logger.Error(err, "failed to list DaemonSets") +func (m *Monitor) listServiceDaemonSets(ctx context.Context, services ...*corev1.Service) []appsv1.DaemonSet { + var daemonSets []appsv1.DaemonSet + for _, service := range services { + if service == nil { + continue + } + list, err := m.k8sInterface.AppsV1().DaemonSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) + if err != nil { + m.logger.Error(err, "failed to list DaemonSets") + } + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + trimmed := slices.DeleteFunc(list.Items, func(daemonSet appsv1.DaemonSet) bool { + return !serviceSelector.Matches(getTemplateSpecLabels(&daemonSet)) + }) + daemonSets = append(daemonSets, trimmed...) } - serviceSelector := labels.SelectorFromSet(service.Spec.Selector) - return slices.DeleteFunc(list.Items, func(daemonSet appsv1.DaemonSet) bool { - return !serviceSelector.Matches(getTemplateSpecLabels(&daemonSet)) - }) + return daemonSets } func getTemplateSpecLabels(obj metav1.Object) labels.Set { @@ -180,50 +205,32 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { } } -// MutateServiceWorkload mutates the annotations of the workload's object without iterating over -// TODO: remove? -func (m *Monitor) MutateServiceWorkload(obj client.Object, service *corev1.Service) map[string]string { - if customSelectLanguages, selected := m.CustomSelected(obj); selected { - m.logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) - return mutate(obj, customSelectLanguages) - } - - if !m.config.MonitorAllServices { - return map[string]string{} - } - if m.excludedNamespace(obj.GetNamespace()) { - return map[string]string{} - } - - if m.excludedService(service) { - return map[string]string{} - } - - m.logger.Info(fmt.Sprintf("start up: setting %s instrumentation annotations to %s because it is owned by service %s", obj.GetName(), m.config.Languages, service.Name)) - return mutate(obj, m.config.Languages) -} - // MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector -func (m *Monitor) MutateObject(oldObj client.Object, obj client.Object) map[string]string { +func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { // todo: handle edge case where a workload is annotated because a service exposed it, and the service is removed. aka add to Service OnDelete // custom selector takes precedence over service selector if customSelectLanguages, selected := m.CustomSelected(obj); selected { m.logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) - return mutate(obj, customSelectLanguages) + return mutate(obj, customSelectLanguages, true) } - if !allowedToMutate(oldObj, obj, m.config.AutoRestart) { + if !safeToMutate(oldObj, obj, m.config.AutoRestart) { return map[string]string{} } + return mutate(obj, m.config.Languages, m.shouldInsert(obj)) +} + +func (m *Monitor) shouldInsert(obj client.Object) bool { if !m.config.MonitorAllServices { - return map[string]string{} + return false } if m.excludedNamespace(obj.GetNamespace()) { - return map[string]string{} + return false } + // determine if the object is currently selected by a service objectLabels := getTemplateSpecLabels(obj) for _, informerObj := range m.serviceInformer.GetStore().List() { service := informerObj.(*corev1.Service) @@ -237,14 +244,14 @@ func (m *Monitor) MutateObject(oldObj client.Object, obj client.Object) map[stri if serviceSelector.Matches(objectLabels) { m.logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is owned by service %s", obj.GetName(), m.config.Languages, service.Name)) - return mutate(obj, m.config.Languages) + return true } } - return map[string]string{} + return false } // mutate obj. If object is a workload, mutate the pod template. otherwise, mutate the object's annotations itself. -func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet) map[string]string { +func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet, shouldInsert bool) map[string]string { var obj metav1.Object podTemplate := getPodTemplate(object) if podTemplate != nil { @@ -262,7 +269,7 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet) ma for _, language := range instrumentation.SupportedTypes() { insertMutation, removeMutation := buildMutations(language) var mutatedAnnotations map[string]string - if _, ok := languagesToMonitor[language]; ok { + if _, ok := languagesToMonitor[language]; ok && shouldInsert { mutatedAnnotations = insertMutation.Mutate(annotations) } else { @@ -330,8 +337,8 @@ func isWorkload(obj client.Object) bool { } } -// allowedToMutate returns if object is already being mutated or if auto restart is enabled. does not guarantee that the object will be mutated. -func allowedToMutate(oldObject client.Object, object client.Object, autoRestart bool) bool { +// safeToMutate returns if object is already being mutated or if auto restart is enabled. does not guarantee that the object will be mutated. +func safeToMutate(oldObject client.Object, object client.Object, autoRestart bool) bool { // mutating a namespace is always safe if isNamespace(object) { return true diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index 33409410c..b2bf419de 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -18,6 +18,8 @@ import ( "time" ) +const defaultNs = "default" + func TestUnmarshal(t *testing.T) { j := []byte(`["java", "nodejs", "python"]`) set := instrumentation.TypeSet{} @@ -31,7 +33,11 @@ func TestMarshal(t *testing.T) { res, err := types.MarshalJSON() assert.NoError(t, err) // todo test irregardless of order - assertJsonEqual(t, []byte(`["nodejs","python"]`), res) + var s []string + err = json.Unmarshal(res, &s) + assert.NoError(t, err) + + assert.ElementsMatch(t, []string{"nodejs", "python"}, s) } func Test_allowedToMutate(t *testing.T) { @@ -77,7 +83,7 @@ func Test_allowedToMutate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, allowedToMutate(tt.oldObject, tt.object, tt.autoRestart)) + assert.Equal(t, tt.want, safeToMutate(tt.oldObject, tt.object, tt.autoRestart)) }) } } @@ -156,7 +162,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, same selector, monitorallservices true, excluded namespace", - config: createConfig(false, []string{"namespace-1"}, nil, false), + config: createConfig(true, []string{"namespace-1"}, nil, false), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -165,7 +171,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, same selector, monitorallservices true, excluded service", - config: createConfig(false, nil, []string{"namespace-1/svc-16"}, false), + config: createConfig(true, nil, []string{"namespace-1/svc-16"}, false), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -181,7 +187,7 @@ func TestMonitor_MutateObject(t *testing.T) { { name: "Deployment", create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { - deployment := newTestDeployment("workload-16", ns, selector) + deployment := newTestDeployment("workload-16", ns, selector, nil) return clientset.AppsV1().Deployments(ns).Create(ctx, deployment, metav1.CreateOptions{}) }, }, @@ -205,6 +211,7 @@ func TestMonitor_MutateObject(t *testing.T) { t.Run(workload.name, func(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() // Setup fresh clients for each workload test fakeClient := fake2.NewFakeClient() clientset := fake.NewSimpleClientset() @@ -223,16 +230,14 @@ func TestMonitor_MutateObject(t *testing.T) { } // Create service - _, err := clientset.CoreV1().Services(service.Namespace).Create(ctx, service, metav1.CreateOptions{}) + _, err := monitor.k8sInterface.CoreV1().Services(service.Namespace).Create(ctx, service, metav1.CreateOptions{}) assert.NoError(t, err) // Create workload workloadObj, err := workload.create(clientset, ctx, tt.deploymentNs, tt.deploymentSelector) assert.NoError(t, err) // need to wait until service informer is updated - err = wait.PollImmediate(1*time.Millisecond, 5*time.Millisecond, func() (bool, error) { - return len(monitor.serviceInformer.GetStore().ListKeys()) > 0, nil - }) + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys > 0 }) assert.NoError(t, err) // Test @@ -244,6 +249,90 @@ func TestMonitor_MutateObject(t *testing.T) { } } +func waitForInformerUpdate(monitor *Monitor, isValid func(int) bool) error { + return wait.PollImmediate(1*time.Millisecond, 5*time.Millisecond, func() (bool, error) { + return isValid(len(monitor.serviceInformer.GetStore().ListKeys())), nil + }) +} + +func Test_OptOutByRemovingService(t *testing.T) { + t.Run("auto restart true, delete and then restart operator", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + deployment := newTestDeployment("deployment", defaultNs, nil, annotations) + objs := []runtime.Object{ + deployment, + } + + clientset := fake.NewSimpleClientset(objs...) + c := fake2.NewFakeClient(objs...) + NewMonitor(context.TODO(), createConfig(true, nil, nil, true), clientset, c, c, testr.New(t)) + err := c.Get(context.TODO(), client.ObjectKeyFromObject(deployment), deployment) + assert.NoError(t, err) + assert.Equal(t, userAnnotations, deployment.Spec.Template.Annotations) + }) + + t.Run("auto restart true, delete while operator running", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + labels := map[string]string{"app": "test"} + service := newTestService("service", defaultNs, labels) + deployment := newTestDeployment("deployment", defaultNs, labels, annotations) + objs := []runtime.Object{ + service, + deployment, + } + clientset := fake.NewSimpleClientset(objs...) + c := fake2.NewFakeClient(objs...) + monitor := NewMonitor(context.TODO(), createConfig(true, nil, nil, true), clientset, c, c, testr.New(t)) + err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys == 0 }) + assert.NoError(t, err) + updatedDeployment, err := clientset.AppsV1().Deployments(defaultNs).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, userAnnotations, updatedDeployment.Spec.Template.Annotations) + }) + + t.Run("auto restart false, delete and then restart operator", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + originalAnnotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + deployment := newTestDeployment("deployment", defaultNs, nil, originalAnnotations) + objs := []runtime.Object{ + deployment, + } + + clientset := fake.NewSimpleClientset(objs...) + c := fake2.NewFakeClient(objs...) + NewMonitor(context.TODO(), createConfig(true, nil, nil, false), clientset, c, c, testr.New(t)) + err := c.Get(context.TODO(), client.ObjectKeyFromObject(deployment), deployment) + assert.NoError(t, err) + assert.Equal(t, originalAnnotations, deployment.Spec.Template.Annotations) + }) + + t.Run("auto restart false, delete while operator running", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + originalAnnotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + labels := map[string]string{"app": "test"} + service := newTestService("service", defaultNs, labels) + deployment := newTestDeployment("deployment", defaultNs, labels, originalAnnotations) + objs := []runtime.Object{ + service, + deployment, + } + clientset := fake.NewSimpleClientset(objs...) + c := fake2.NewFakeClient(objs...) + monitor := NewMonitor(context.TODO(), createConfig(true, nil, nil, false), clientset, c, c, testr.New(t)) + err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys == 0 }) + assert.NoError(t, err) + updatedDeployment, err := clientset.AppsV1().Deployments(defaultNs).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, originalAnnotations, updatedDeployment.Spec.Template.Annotations) + }) +} + func Test_mutate(t *testing.T) { tests := []struct { name string @@ -297,6 +386,13 @@ func Test_mutate(t *testing.T) { wantObjAnnotations: map[string]string{}, wantMutated: buildAnnotations("java"), }, + { + name: "remove only language annotations", + podAnnotations: mergeAnnotations(buildAnnotations("java"), map[string]string{"test": "test"}), + languagesToMonitor: instrumentation.TypeSet{}, + wantObjAnnotations: map[string]string{"test": "test"}, + wantMutated: buildAnnotations("java"), + }, } workloadTypes := []struct { @@ -352,7 +448,8 @@ func Test_mutate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { obj := workload.create(tt.podAnnotations).DeepCopyObject().(client.Object) - gotMutated := mutate(obj, tt.languagesToMonitor) + // TODO test different shouldInsert values + gotMutated := mutate(obj, tt.languagesToMonitor, true) assert.Equal(t, tt.wantObjAnnotations, getPodTemplate(obj).GetAnnotations()) assert.Equal(t, tt.wantMutated, gotMutated) }) @@ -362,10 +459,10 @@ func Test_mutate(t *testing.T) { } func Test_StartupAutoRestart(t *testing.T) { - service := newTestService("service-1", "default", map[string]string{"test": "test"}) - matchingDeployment := newTestDeployment("deployment-1", "default", map[string]string{"test": "test"}) - nonMatchingDeployment := newTestDeployment("deployment-2", "default", map[string]string{}) - customSelectedDeployment := newTestDeployment("deployment-3", "default", map[string]string{}) + service := newTestService("service-1", defaultNs, map[string]string{"test": "test"}) + matchingDeployment := newTestDeployment("deployment-1", defaultNs, map[string]string{"test": "test"}, nil) + nonMatchingDeployment := newTestDeployment("deployment-2", defaultNs, map[string]string{}, nil) + customSelectedDeployment := newTestDeployment("deployment-3", defaultNs, map[string]string{}, nil) config := MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), @@ -380,10 +477,10 @@ func Test_StartupAutoRestart(t *testing.T) { fakeClient := fake2.NewFakeClient(objs...) m := NewMonitor(context.TODO(), config, clientset, fakeClient, fakeClient, testr.New(t)) - updatedMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments("default").Get(context.TODO(), matchingDeployment.Name, metav1.GetOptions{}) + updatedMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments(defaultNs).Get(context.TODO(), matchingDeployment.Name, metav1.GetOptions{}) assert.NoError(t, err) assert.Equal(t, buildAnnotations(instrumentation.TypeJava), updatedMatchingDeployment.Spec.Template.GetAnnotations()) - updatedNonMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments("default").Get(context.TODO(), nonMatchingDeployment.Name, metav1.GetOptions{}) + updatedNonMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments(defaultNs).Get(context.TODO(), nonMatchingDeployment.Name, metav1.GetOptions{}) assert.NoError(t, err) assert.Empty(t, updatedNonMatchingDeployment.Spec.Template.GetAnnotations()) err = fakeClient.Get(context.TODO(), client.ObjectKeyFromObject(customSelectedDeployment), customSelectedDeployment) @@ -392,14 +489,14 @@ func Test_StartupAutoRestart(t *testing.T) { } func Test_listServiceDeployments(t *testing.T) { - testService := newTestService("service-1", "default", map[string]string{"test": "test"}) - testDeployment := newTestDeployment("deployment-1", "default", map[string]string{"test": "test"}) - notMatchingService := newTestService("service-2", "default", map[string]string{"test2": "test2"}) + testService := newTestService("service-1", defaultNs, map[string]string{"test": "test"}) + testDeployment := newTestDeployment("deployment-1", defaultNs, map[string]string{"test": "test"}, nil) + notMatchingService := newTestService("service-2", defaultNs, map[string]string{"test2": "test2"}) clientset := fake.NewSimpleClientset(testService, testDeployment, notMatchingService) m := Monitor{k8sInterface: clientset, logger: testr.New(t)} - matchingServiceDeployments := m.listServiceDeployments(testService, context.TODO()) + matchingServiceDeployments := m.listServiceDeployments(context.TODO(), testService) assert.Len(t, matchingServiceDeployments, 1) - notMatchingServiceDeployments := m.listServiceDeployments(notMatchingService, context.TODO()) + notMatchingServiceDeployments := m.listServiceDeployments(context.TODO(), notMatchingService) assert.Len(t, notMatchingServiceDeployments, 0) } @@ -437,7 +534,7 @@ func newTestService(name string, namespace string, selector map[string]string) * return service.DeepCopy() } -func newTestDeployment(name string, namespace string, labels map[string]string) *appsv1.Deployment { +func newTestDeployment(name string, namespace string, labels map[string]string, annotations map[string]string) *appsv1.Deployment { deployment := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -449,7 +546,8 @@ func newTestDeployment(name string, namespace string, labels map[string]string) }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + Labels: labels, + Annotations: annotations, }, }, }, From a886b9a987dc02ddb3a7e70cc92c415a83fd56c3 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 21 Mar 2025 13:31:19 -0400 Subject: [PATCH 13/63] Improve backwards compatability, make MutateAndPatchAll work with both AutoMonitor and AnnotationMutators, check if safe to mutate before applying custom selector, run opt out tests for each type of workload, test opt out by disabling monitorallservices --- .../namespacemutation/webhookhandler.go | 21 +- .../namespacemutation/webhookhandler_test.go | 3 +- .../workloadmutation/webhookhandler.go | 19 +- .../workloadmutation/webhookhandler_test.go | 2 +- main.go | 78 +++--- pkg/instrumentation/auto/annotation.go | 49 ++-- pkg/instrumentation/auto/annotation_test.go | 14 +- pkg/instrumentation/auto/auto_monitor.go | 55 ++-- pkg/instrumentation/auto/auto_monitor_test.go | 249 +++++++++++------- pkg/instrumentation/auto/callback.go | 50 +++- 10 files changed, 317 insertions(+), 223 deletions(-) diff --git a/internal/webhook/namespacemutation/webhookhandler.go b/internal/webhook/namespacemutation/webhookhandler.go index 00d85e780..3b58bec5e 100644 --- a/internal/webhook/namespacemutation/webhookhandler.go +++ b/internal/webhook/namespacemutation/webhookhandler.go @@ -18,16 +18,14 @@ import ( var _ admission.Handler = (*handler)(nil) type handler struct { - decoder *admission.Decoder - annotationMutators *auto.AnnotationMutators - monitor auto.MonitorInterface + decoder *admission.Decoder + monitor auto.MonitorInterface } -func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators, monitor auto.MonitorInterface) admission.Handler { +func NewWebhookHandler(decoder *admission.Decoder, monitor auto.MonitorInterface) admission.Handler { return &handler{ - decoder: decoder, - annotationMutators: annotationMutators, - monitor: monitor, + decoder: decoder, + monitor: monitor, } } @@ -38,12 +36,9 @@ func (h *handler) Handle(_ context.Context, req admission.Request) admission.Res return admission.Errored(http.StatusBadRequest, err) } - if h.annotationMutators.IsManaged(namespace) && !h.monitor.AnyCustomSelectorDefined() { - h.annotationMutators.MutateObject(namespace) - } else { - // do not need to pass in oldObj because it's only used to check for workload pod template diff - h.monitor.MutateObject(nil, namespace) - } + // do not need to pass in oldObj because it's only used to check for workload pod template diff + h.monitor.MutateObject(nil, namespace) + marshaledNamespace, err := json.Marshal(namespace) if err != nil { res := admission.Errored(http.StatusInternalServerError, err) diff --git a/internal/webhook/namespacemutation/webhookhandler_test.go b/internal/webhook/namespacemutation/webhookhandler_test.go index 98b9a5e44..7ea72bd10 100644 --- a/internal/webhook/namespacemutation/webhookhandler_test.go +++ b/internal/webhook/namespacemutation/webhookhandler_test.go @@ -40,7 +40,7 @@ func TestHandle(t *testing.T) { autoAnnotationConfig, instrumentation.NewTypeSet(instrumentation.TypeJava), ) - h := NewWebhookHandler(decoder, mutators, auto.NoopMonitor{}) + h := NewWebhookHandler(decoder, mutators) for _, testCase := range []struct { req admission.Request name string @@ -75,7 +75,6 @@ func TestHandle(t *testing.T) { expected: http.StatusOK, allowed: true, }, - {}, } { t.Run(testCase.name, func(t *testing.T) { // test diff --git a/internal/webhook/workloadmutation/webhookhandler.go b/internal/webhook/workloadmutation/webhookhandler.go index 09e39970a..30b20558e 100644 --- a/internal/webhook/workloadmutation/webhookhandler.go +++ b/internal/webhook/workloadmutation/webhookhandler.go @@ -31,17 +31,15 @@ type WebhookHandler interface { // the implementation. type workloadMutationWebhook struct { - decoder *admission.Decoder - annotationMutators *auto.AnnotationMutators - monitor auto.MonitorInterface + decoder *admission.Decoder + monitor auto.MonitorInterface } // NewWebhookHandler creates a new WorkloadWebhookHandler. -func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators, monitor auto.MonitorInterface) WebhookHandler { +func NewWebhookHandler(decoder *admission.Decoder, monitor auto.MonitorInterface) WebhookHandler { return &workloadMutationWebhook{ - decoder: decoder, - annotationMutators: annotationMutators, - monitor: monitor, + decoder: decoder, + monitor: monitor, } } @@ -73,12 +71,7 @@ func (p *workloadMutationWebhook) Handle(_ context.Context, req admission.Reques } } - // preserve backwards compatability. - if p.annotationMutators.IsManaged(obj) && !p.monitor.AnyCustomSelectorDefined() { - p.annotationMutators.MutateObject(obj) - } else { - p.monitor.MutateObject(oldObj, obj) - } + p.monitor.MutateObject(oldObj, obj) marshaledObject, err := json.Marshal(obj) if err != nil { diff --git a/internal/webhook/workloadmutation/webhookhandler_test.go b/internal/webhook/workloadmutation/webhookhandler_test.go index f1978eb7a..37d89c8a2 100644 --- a/internal/webhook/workloadmutation/webhookhandler_test.go +++ b/internal/webhook/workloadmutation/webhookhandler_test.go @@ -129,7 +129,7 @@ func TestHandle(t *testing.T) { autoAnnotationConfig, instrumentation.NewTypeSet(instrumentation.TypeJava), ) - injector := NewWebhookHandler(decoder, mutators, auto.NoopMonitor{}) + injector := NewWebhookHandler(decoder, mutators) // test res := injector.Handle(context.Background(), tt.req) diff --git a/main.go b/main.go index 2c062cea1..cb4bd5487 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,8 @@ import ( "encoding/json" "flag" "fmt" + "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/namespacemutation" + "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/workloadmutation" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "os" @@ -37,9 +39,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent-operator/controllers" "github.com/aws/amazon-cloudwatch-agent-operator/internal/config" "github.com/aws/amazon-cloudwatch-agent-operator/internal/version" - "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/namespacemutation" "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/podmutation" - "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/workloadmutation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/featuregate" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" @@ -291,50 +291,49 @@ func main() { decoder := admission.NewDecoder(mgr.GetScheme()) // TODO handle case where auto monitor is enabled but auto annotation config is not specified + var autoAnnotationConfig auto.AnnotationConfig + var autoAnnotationMutators *auto.AnnotationMutators + supportedLanguages := instrumentation.NewTypeSet(instrumentation.SupportedTypes()...) + if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" || autoAnnotationConfigStr == "" { setupLog.Info("Auto-annotation is disabled") } else { - - supportedLanguages := instrumentation.NewTypeSet( - instrumentation.TypeJava, - instrumentation.TypePython, - instrumentation.TypeDotNet, - instrumentation.TypeNodeJS, - ) - - var autoAnnotationConfig auto.AnnotationConfig if err = json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-annotation config") } else { - // TODO handle case where auto monitor is enabled but auto annotation config is not specified - setupLog.Info("Test!") - monitor := createMonitorFromConfig(autoMonitorConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) - - autoAnnotationMutators := auto.NewAnnotationMutators( + // TODO: detect empty + autoAnnotationMutators = auto.NewAnnotationMutators( mgr.GetClient(), mgr.GetAPIReader(), logger, autoAnnotationConfig, supportedLanguages, ) - mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ - Handler: workloadmutation.NewWebhookHandler(decoder, autoAnnotationMutators, monitor)}) - mgr.GetWebhookServer().Register("/mutate-v1-namespace", &webhook.Admission{ - Handler: namespacemutation.NewWebhookHandler(decoder, autoAnnotationMutators, monitor), - }) - - setupLog.Info("Auto-annotation is enabled") - go waitForWebhookServerStart( - ctx, - mgr.GetWebhookServer().StartedChecker(), - func(ctx context.Context) { - setupLog.Info("Applying auto-annotation") - autoAnnotationMutators.MutateAndPatchAll(ctx) - }, - ) } } + monitor := createMonitorFromConfig(autoMonitorConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader(), autoAnnotationMutators) + + if monitor != nil { + mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ + Handler: workloadmutation.NewWebhookHandler(decoder, monitor)}) + mgr.GetWebhookServer().Register("/mutate-v1-namespace", &webhook.Admission{ + Handler: namespacemutation.NewWebhookHandler(decoder, monitor), + }) + + setupLog.Info("Auto-annotation is enabled") + go waitForWebhookServerStart( + ctx, + mgr.GetWebhookServer().StartedChecker(), + func(ctx context.Context) { + setupLog.Info("Applying auto-annotation") + auto.MutateAndPatchAll(monitor, ctx) + }, + ) + } else { + setupLog.Info("Auto-annotation and Auto Monitor is disabled") + } + if os.Getenv("ENABLE_WEBHOOKS") != "false" { if err = otelv1alpha1.SetupCollectorWebhook(mgr, cfg); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "AmazonCloudWatchAgent") @@ -372,11 +371,12 @@ func main() { } } -func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context, client client.Client, reader client.Reader) auto.MonitorInterface { +func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context, client client.Client, reader client.Reader, autoAnnotationMutators *auto.AnnotationMutators) auto.MonitorInterface { var monitorConfig *auto.MonitorConfig - var monitor auto.MonitorInterface = auto.NoopMonitor{} + var monitor auto.MonitorInterface = autoAnnotationMutators if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { - setupLog.Error(err, "Unable to unmarshal auto-monitor config") + setupLog.Error(err, "Unable to unmarshal auto-monitor config, disabling AutoMonitor") + return monitor } else { if monitorConfig.Languages == nil || len(monitorConfig.Languages) == 0 { monitorConfig.Languages = instrumentation.NewTypeSet(instrumentation.SupportedTypes()...) @@ -384,15 +384,19 @@ func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context, c k8sConfig, err := rest.InClusterConfig() if err != nil { setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") - return auto.NoopMonitor{} + return monitor } clientSet, err := kubernetes.NewForConfig(k8sConfig) if err != nil { setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") - return auto.NoopMonitor{} + return monitor + } + if autoAnnotationMutators != nil { + monitor = auto.NewMonitorWithExistingAnnotationMutator(ctx, *monitorConfig, clientSet, client, reader, setupLog, autoAnnotationMutators) + } else { + monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, setupLog) } - monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, setupLog) } return monitor } diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 172c8081d..46e8c3b18 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -41,34 +41,35 @@ type AnnotationMutators struct { cfg *AnnotationConfig } -// IsManaged returns if AnnotationMutators would ever mutate the object. -func (m *AnnotationMutators) IsManaged(obj client.Object) bool { - return len(m.cfg.GetObjectLanguagesToAnnotate(obj)) > 0 +func (m *AnnotationMutators) GetAnnotationMutators() *AnnotationMutators { + return m +} + +func (m *AnnotationMutators) GetLogger() logr.Logger { + return m.logger +} + +func (m *AnnotationMutators) GetReader() client.Reader { + return m.clientReader } -// RestartNamespace sets the restartedAtAnnotation for each of the namespace's supported resources and patches them. -func (m *AnnotationMutators) RestartNamespace(ctx context.Context, namespace *corev1.Namespace, mutatedAnnotations map[string]string) { - m.rangeObjectList(ctx, &appsv1.DeploymentList{}, client.InNamespace(namespace.Name), - chainCallbacks(m.shouldRestartFunc(mutatedAnnotations), m.patchFunc(ctx, setRestartAnnotation))) - m.rangeObjectList(ctx, &appsv1.DaemonSetList{}, client.InNamespace(namespace.Name), - chainCallbacks(m.shouldRestartFunc(mutatedAnnotations), m.patchFunc(ctx, setRestartAnnotation))) - m.rangeObjectList(ctx, &appsv1.StatefulSetList{}, client.InNamespace(namespace.Name), - chainCallbacks(m.shouldRestartFunc(mutatedAnnotations), m.patchFunc(ctx, setRestartAnnotation))) +func (m *AnnotationMutators) GetWriter() client.Writer { + return m.clientWriter } -// MutateAndPatchAll runs the mutators for each of the supported resources and patches them. -func (m *AnnotationMutators) MutateAndPatchAll(ctx context.Context) { - m.rangeObjectList(ctx, &appsv1.DeploymentList{}, &client.ListOptions{}, m.patchFunc(ctx, m.mutateObject)) - m.rangeObjectList(ctx, &appsv1.DaemonSetList{}, &client.ListOptions{}, m.patchFunc(ctx, m.mutateObject)) - m.rangeObjectList(ctx, &appsv1.StatefulSetList{}, &client.ListOptions{}, m.patchFunc(ctx, m.mutateObject)) - m.rangeObjectList(ctx, &corev1.NamespaceList{}, &client.ListOptions{}, - chainCallbacks(m.patchFunc(ctx, m.mutateObject), m.restartNamespaceFunc(ctx)), - ) +// IsManaged returns if AnnotationMutators would ever mutate the object. +func (m *AnnotationMutators) IsManaged(obj client.Object) bool { + return len(m.cfg.GetObjectLanguagesToAnnotate(obj)) > 0 } // MutateObject modifies annotations for a single object using the configured mutators. -func (m *AnnotationMutators) MutateObject(obj client.Object) (any, bool) { - return m.mutateObject(obj, nil) +func (m *AnnotationMutators) MutateObject(_ client.Object, obj client.Object) map[string]string { + mutatedAnnotations, _ := m.mutateObject(obj, nil) + annotations, ok := mutatedAnnotations.(map[string]string) + if !ok { + m.logger.Error(nil, "could not cast annotations to map") + } + return annotations } // mutateObject modifies annotations for a single object using the configured mutators. @@ -87,9 +88,9 @@ func (m *AnnotationMutators) mutateObject(obj client.Object, _ any) (any, bool) } } -func (m *AnnotationMutators) rangeObjectList(ctx context.Context, list client.ObjectList, option client.ListOption, fn objectCallbackFunc) { - if err := m.clientReader.List(ctx, list, option); err != nil { - m.logger.Error(err, "Unable to list objects", +func rangeObjectList(m MonitorInterface, ctx context.Context, list client.ObjectList, option client.ListOption, fn objectCallbackFunc) { + if err := m.GetReader().List(ctx, list, option); err != nil { + m.GetLogger().Error(err, "Unable to list objects", "kind", fmt.Sprintf("%T", list), ) return diff --git a/pkg/instrumentation/auto/annotation_test.go b/pkg/instrumentation/auto/annotation_test.go index 72ac13ee5..e8f6c82f4 100644 --- a/pkg/instrumentation/auto/annotation_test.go +++ b/pkg/instrumentation/auto/annotation_test.go @@ -117,7 +117,7 @@ func TestAnnotationMutators_Namespaces(t *testing.T) { testCase.cfg, testCase.typeSet, ) - mutators.MutateAndPatchAll(ctx) + MutateAndPatchAll(mutators, ctx) gotNamespaces := &corev1.NamespaceList{} require.NoError(t, fakeClient.List(ctx, gotNamespaces)) for _, gotNamespace := range gotNamespaces.Items { @@ -201,7 +201,7 @@ func TestAnnotationMutators_Namespaces_Restart(t *testing.T) { cfg, instrumentation.NewTypeSet(instrumentation.TypeJava), ) - mutators.MutateAndPatchAll(context.Background()) + MutateAndPatchAll(mutators, context.Background()) ctx := context.Background() for _, namespacedResource := range namespacedRestartExpectedResources { assert.NoError(t, fakeClient.Get(ctx, client.ObjectKeyFromObject(namespacedResource), namespacedResource)) @@ -292,7 +292,7 @@ func TestAnnotationMutators_Deployments(t *testing.T) { testCase.cfg, testCase.typeSet, ) - mutators.MutateAndPatchAll(ctx) + MutateAndPatchAll(mutators, ctx) gotDeployments := &appsv1.DeploymentList{} require.NoError(t, fakeClient.List(ctx, gotDeployments)) for _, gotDeployment := range gotDeployments.Items { @@ -360,7 +360,7 @@ func TestAnnotationMutators_DaemonSets(t *testing.T) { testCase.cfg, testCase.typeSet, ) - mutators.MutateAndPatchAll(ctx) + MutateAndPatchAll(mutators, ctx) gotDaemonSets := &appsv1.DaemonSetList{} require.NoError(t, fakeClient.List(ctx, gotDaemonSets)) for _, gotDaemonSet := range gotDaemonSets.Items { @@ -428,7 +428,7 @@ func TestAnnotationMutators_StatefulSets(t *testing.T) { testCase.cfg, testCase.typeSet, ) - mutators.MutateAndPatchAll(ctx) + MutateAndPatchAll(mutators, ctx) gotStatefulSets := &appsv1.StatefulSetList{} require.NoError(t, fakeClient.List(ctx, gotStatefulSets)) for _, gotStatefulSet := range gotStatefulSets.Items { @@ -494,11 +494,11 @@ func TestAnnotationMutators_ClientErrors(t *testing.T) { cfg, instrumentation.NewTypeSet(instrumentation.TypeJava), ) - mutators.MutateAndPatchAll(context.Background()) + MutateAndPatchAll(mutators, context.Background()) errClient.AssertCalled(t, "List", mock.Anything, mock.Anything, mock.Anything) mutators.clientWriter = errClient mutators.clientReader = fakeClient - mutators.MutateAndPatchAll(context.Background()) + MutateAndPatchAll(mutators, context.Background()) errClient.AssertCalled(t, "Patch", mock.Anything, mock.Anything, mock.Anything, mock.Anything) } diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index 2529ec6d9..f4fc53193 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -22,7 +22,10 @@ import ( type MonitorInterface interface { MutateObject(oldObj client.Object, obj client.Object) map[string]string - AnyCustomSelectorDefined() bool + GetAnnotationMutators() *AnnotationMutators + GetLogger() logr.Logger + GetReader() client.Reader + GetWriter() client.Writer } type Monitor struct { @@ -30,11 +33,31 @@ type Monitor struct { ctx context.Context config MonitorConfig k8sInterface kubernetes.Interface + clientReader client.Reader + clientWriter client.Writer customSelectors *AnnotationMutators logger logr.Logger } -type NoopMonitor struct{} +func (m *Monitor) GetAnnotationMutators() *AnnotationMutators { + return m.customSelectors +} + +func (m *Monitor) GetLogger() logr.Logger { + return m.logger +} + +func (m *Monitor) GetReader() client.Reader { + return m.clientReader +} + +func (m *Monitor) GetWriter() client.Writer { + return m.clientWriter +} + +type NoopMonitor struct { + customSelectors *AnnotationMutators +} func (n NoopMonitor) MutateObject(_ client.Object, _ client.Object) map[string]string { return map[string]string{} @@ -44,14 +67,14 @@ func (n NoopMonitor) AnyCustomSelectorDefined() bool { return false } -func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, c client.Client, r client.Reader, logger logr.Logger) *Monitor { +func NewMonitorWithExistingAnnotationMutator(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger, mutators *AnnotationMutators) *Monitor { logger.Info("AutoMonitor starting...") // todo, throw warning if exclude config service is not namespaced (doesn't contain `/`) // todo: informers.WithTransform() as option to only store what parts of service are needed factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute) serviceInformer := factory.Core().V1().Services().Informer() - m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: NewAnnotationMutators(c, r, logger, config.CustomSelector, instrumentation.NewTypeSet(instrumentation.SupportedTypes()...))} + m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutators, clientReader: r, clientWriter: w} _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { m.onServiceEvent(nil, obj.(*corev1.Service)) @@ -79,17 +102,7 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet logger.Info("Enabled!") if m.config.AutoRestart { - logger.Info("Auto restarting custom selector resources") - m.customSelectors.MutateAndPatchAll(ctx) - // update all existing services - logger.Info("Auto restarting service resources, except for excludedServices or services in excludedNamespaces", "excludedServices", m.config.Exclude.Services, "excludedNamespaces", m.config.Exclude.Namespaces) - list, err := k8sInterface.CoreV1().Services("").List(ctx, metav1.ListOptions{}) - if err != nil { - return nil - } - for _, service := range list.Items { - m.onServiceEvent(nil, &service) - } + MutateAndPatchAll(m, ctx) } else { logger.Info("Auto restart disabled. To instrument workloads, restart the workloads exposed by a service.") } @@ -97,6 +110,10 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet return m } +func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { + return NewMonitorWithExistingAnnotationMutator(ctx, config, k8sInterface, w, r, logger, NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.NewTypeSet(instrumentation.SupportedTypes()...))) +} + func (m *Monitor) onServiceEvent(oldService *corev1.Service, service *corev1.Service) { if !m.config.AutoRestart { return @@ -207,6 +224,10 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { // MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { + if !safeToMutate(oldObj, obj, m.config.AutoRestart) { + return map[string]string{} + } + // todo: handle edge case where a workload is annotated because a service exposed it, and the service is removed. aka add to Service OnDelete // custom selector takes precedence over service selector if customSelectLanguages, selected := m.CustomSelected(obj); selected { @@ -214,10 +235,6 @@ func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { return mutate(obj, customSelectLanguages, true) } - if !safeToMutate(oldObj, obj, m.config.AutoRestart) { - return map[string]string{} - } - return mutate(obj, m.config.Languages, m.shouldInsert(obj)) } diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index b2bf419de..68c5599b3 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/controller-runtime/pkg/client" fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -20,6 +21,56 @@ import ( const defaultNs = "default" +var workloadTypes = []struct { + name string + create func(name, namespace string, labels, annotations map[string]string) client.Object + get func(clientset kubernetes.Interface, namespace, name string) (client.Object, error) + getWithClient func(c client.Reader, ns, name string) (client.Object, error) +}{ + { + name: "Deployment", + create: func(name, ns string, labels, annotations map[string]string) client.Object { + return newTestDeployment(name, ns, labels, annotations) + }, + get: func(c kubernetes.Interface, ns, name string) (client.Object, error) { + return c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) + }, + getWithClient: func(c client.Reader, ns, name string) (client.Object, error) { + obj := &appsv1.Deployment{} + err := c.Get(context.TODO(), client.ObjectKey{Namespace: ns, Name: name}, obj) + return obj, err + }, + }, + { + name: "StatefulSet", + create: func(name, ns string, labels, annotations map[string]string) client.Object { + return newTestStatefulSet(name, ns, labels, annotations) + }, + get: func(c kubernetes.Interface, ns, name string) (client.Object, error) { + return c.AppsV1().StatefulSets(ns).Get(context.TODO(), name, metav1.GetOptions{}) + }, + getWithClient: func(c client.Reader, ns, name string) (client.Object, error) { + obj := &appsv1.StatefulSet{} + err := c.Get(context.TODO(), client.ObjectKey{Namespace: ns, Name: name}, obj) + return obj, err + }, + }, + { + name: "DaemonSet", + create: func(name, ns string, labels, annotations map[string]string) client.Object { + return newTestDaemonSet(name, ns, labels, annotations) + }, + get: func(c kubernetes.Interface, ns, name string) (client.Object, error) { + return c.AppsV1().DaemonSets(ns).Get(context.TODO(), name, metav1.GetOptions{}) + }, + getWithClient: func(c client.Reader, ns, name string) (client.Object, error) { + obj := &appsv1.DaemonSet{} + err := c.Get(context.TODO(), client.ObjectKey{Namespace: ns, Name: name}, obj) + return obj, err + }, + }, +} + func TestUnmarshal(t *testing.T) { j := []byte(`["java", "nodejs", "python"]`) set := instrumentation.TypeSet{} @@ -194,14 +245,14 @@ func TestMonitor_MutateObject(t *testing.T) { { name: "StatefulSet", create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { - statefulset := newTestStatefulSet("workload-16", ns, selector) + statefulset := newTestStatefulSet("workload-16", ns, selector, nil) return clientset.AppsV1().StatefulSets(ns).Create(ctx, statefulset, metav1.CreateOptions{}) }, }, { name: "DaemonSet", create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { - daemonset := newTestDaemonSet("workload-16", ns, selector) + daemonset := newTestDaemonSet("workload-16", ns, selector, nil) return clientset.AppsV1().DaemonSets(ns).Create(ctx, daemonset, metav1.CreateOptions{}) }, }, @@ -256,81 +307,101 @@ func waitForInformerUpdate(monitor *Monitor, isValid func(int) bool) error { } func Test_OptOutByRemovingService(t *testing.T) { - t.Run("auto restart true, delete and then restart operator", func(t *testing.T) { - userAnnotations := map[string]string{"test": "blah"} - annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) - deployment := newTestDeployment("deployment", defaultNs, nil, annotations) - objs := []runtime.Object{ - deployment, - } - - clientset := fake.NewSimpleClientset(objs...) - c := fake2.NewFakeClient(objs...) - NewMonitor(context.TODO(), createConfig(true, nil, nil, true), clientset, c, c, testr.New(t)) - err := c.Get(context.TODO(), client.ObjectKeyFromObject(deployment), deployment) - assert.NoError(t, err) - assert.Equal(t, userAnnotations, deployment.Spec.Template.Annotations) - }) - - t.Run("auto restart true, delete while operator running", func(t *testing.T) { - userAnnotations := map[string]string{"test": "blah"} - annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) - labels := map[string]string{"app": "test"} - service := newTestService("service", defaultNs, labels) - deployment := newTestDeployment("deployment", defaultNs, labels, annotations) - objs := []runtime.Object{ - service, - deployment, - } - clientset := fake.NewSimpleClientset(objs...) - c := fake2.NewFakeClient(objs...) - monitor := NewMonitor(context.TODO(), createConfig(true, nil, nil, true), clientset, c, c, testr.New(t)) - err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) - assert.NoError(t, err) - err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys == 0 }) - assert.NoError(t, err) - updatedDeployment, err := clientset.AppsV1().Deployments(defaultNs).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) - assert.NoError(t, err) - assert.Equal(t, userAnnotations, updatedDeployment.Spec.Template.Annotations) - }) - - t.Run("auto restart false, delete and then restart operator", func(t *testing.T) { - userAnnotations := map[string]string{"test": "blah"} - originalAnnotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) - deployment := newTestDeployment("deployment", defaultNs, nil, originalAnnotations) - objs := []runtime.Object{ - deployment, - } - - clientset := fake.NewSimpleClientset(objs...) - c := fake2.NewFakeClient(objs...) - NewMonitor(context.TODO(), createConfig(true, nil, nil, false), clientset, c, c, testr.New(t)) - err := c.Get(context.TODO(), client.ObjectKeyFromObject(deployment), deployment) - assert.NoError(t, err) - assert.Equal(t, originalAnnotations, deployment.Spec.Template.Annotations) - }) + for _, wt := range workloadTypes { + t.Run(wt.name, func(t *testing.T) { + t.Run("auto restart true, delete and then restart operator", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + workload := wt.create("deployment", defaultNs, nil, annotations) + + clientset := fake.NewSimpleClientset(workload) + c := fake2.NewFakeClient(workload) + NewMonitor(context.TODO(), createConfig(true, nil, nil, true), clientset, c, c, testr.New(t)) + updatedWorkload, err := wt.getWithClient(c, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, userAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + }) + + t.Run("auto restart true, delete while operator running", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + labels := map[string]string{"app": "test"} + service := newTestService("service", defaultNs, labels) + workload := wt.create("workload", defaultNs, labels, annotations) + + clientset := fake.NewSimpleClientset(service, workload) + c := fake2.NewFakeClient(service, workload) + monitor := NewMonitor(context.TODO(), createConfig(true, nil, nil, true), clientset, c, c, testr.New(t)) + + err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys == 0 }) + assert.NoError(t, err) + + updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, userAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + }) + + t.Run("auto restart false, delete and then restart operator", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + originalAnnotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + workload := wt.create("workload", defaultNs, nil, originalAnnotations) + + clientset := fake.NewSimpleClientset(workload) + c := fake2.NewFakeClient(workload) + NewMonitor(context.TODO(), createConfig(true, nil, nil, false), clientset, c, c, testr.New(t)) + + updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, originalAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + }) + + t.Run("auto restart false, delete while operator running", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + originalAnnotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + labels := map[string]string{"app": "test"} + service := newTestService("service", defaultNs, labels) + workload := wt.create("workload", defaultNs, labels, originalAnnotations) + + clientset := fake.NewSimpleClientset(service, workload) + c := fake2.NewFakeClient(service, workload) + monitor := NewMonitor(context.TODO(), createConfig(true, nil, nil, false), clientset, c, c, testr.New(t)) + + err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys == 0 }) + assert.NoError(t, err) + + updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, originalAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + }) + }) + } +} - t.Run("auto restart false, delete while operator running", func(t *testing.T) { - userAnnotations := map[string]string{"test": "blah"} - originalAnnotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) - labels := map[string]string{"app": "test"} - service := newTestService("service", defaultNs, labels) - deployment := newTestDeployment("deployment", defaultNs, labels, originalAnnotations) - objs := []runtime.Object{ - service, - deployment, - } - clientset := fake.NewSimpleClientset(objs...) - c := fake2.NewFakeClient(objs...) - monitor := NewMonitor(context.TODO(), createConfig(true, nil, nil, false), clientset, c, c, testr.New(t)) - err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) - assert.NoError(t, err) - err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys == 0 }) - assert.NoError(t, err) - updatedDeployment, err := clientset.AppsV1().Deployments(defaultNs).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) - assert.NoError(t, err) - assert.Equal(t, originalAnnotations, updatedDeployment.Spec.Template.Annotations) - }) +func Test_OptOutByDisablingMonitorAllServices(t *testing.T) { + for _, wt := range workloadTypes { + t.Run(wt.name, func(t *testing.T) { + t.Run("auto restart true", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + labels := map[string]string{"app": "test"} + service := newTestService("service", defaultNs, labels) + workload := wt.create("workload", defaultNs, labels, annotations) + + clientset := fake.NewSimpleClientset(service, workload) + c := fake2.NewFakeClient(service, workload) + NewMonitor(context.TODO(), createConfig(false, nil, nil, true), clientset, c, c, testr.New(t)) + + updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, userAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + + }) + }) + } } func Test_mutate(t *testing.T) { @@ -502,18 +573,6 @@ func Test_listServiceDeployments(t *testing.T) { // Helper functions -func assertJsonEqual(t *testing.T, expectedJson []byte, actualJson []byte) { - var obj1, obj2 interface{} - - err := json.Unmarshal(expectedJson, &obj1) - assert.NoError(t, err) - - err = json.Unmarshal(actualJson, &obj2) - assert.NoError(t, err) - - assert.Equal(t, obj1, obj2) -} - func createNamespace(t *testing.T, clientset *fake.Clientset, ctx context.Context, namespaceName string) *corev1.Namespace { namespace := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}} serviceNamespace, err := clientset.CoreV1().Namespaces().Create(ctx, &namespace, metav1.CreateOptions{}) @@ -555,7 +614,7 @@ func newTestDeployment(name string, namespace string, labels map[string]string, return deployment.DeepCopy() } -func newTestStatefulSet(name, namespace string, selector map[string]string) *appsv1.StatefulSet { +func newTestStatefulSet(name string, namespace string, labels map[string]string, annotations map[string]string) *appsv1.StatefulSet { statefulSet := appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -563,11 +622,12 @@ func newTestStatefulSet(name, namespace string, selector map[string]string) *app }, Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: selector, + MatchLabels: labels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: selector, + Labels: labels, + Annotations: annotations, }, }, }, @@ -575,7 +635,7 @@ func newTestStatefulSet(name, namespace string, selector map[string]string) *app return statefulSet.DeepCopy() } -func newTestDaemonSet(name, namespace string, selector map[string]string) *appsv1.DaemonSet { +func newTestDaemonSet(name string, namespace string, labels map[string]string, annotations map[string]string) *appsv1.DaemonSet { daemonSet := appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -583,11 +643,12 @@ func newTestDaemonSet(name, namespace string, selector map[string]string) *appsv }, Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: selector, + MatchLabels: labels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: selector, + Labels: labels, + Annotations: annotations, }, }, }, diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index 34e0ee0d3..06f70d2d2 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -7,10 +7,10 @@ import ( "context" "encoding/json" "fmt" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" "sigs.k8s.io/controller-runtime/pkg/client" @@ -66,11 +66,11 @@ func createPatch(obj client.Object) (client.Patch, error) { return &basicPatch{originalJSON: originalJSON}, nil } -func (m *AnnotationMutators) patchFunc(ctx context.Context, callback objectCallbackFunc) objectCallbackFunc { +func patchFunc(m MonitorInterface, ctx context.Context, callback objectCallbackFunc) objectCallbackFunc { return func(obj client.Object, _ any) (any, bool) { patch, err := createPatch(obj) if err != nil { - m.logger.Error(err, "Unable to create patch", + m.GetLogger().Error(err, "Unable to create patch", "kind", fmt.Sprintf("%T", obj), "name", obj.GetName(), "namespace", obj.GetNamespace(), @@ -81,8 +81,8 @@ func (m *AnnotationMutators) patchFunc(ctx context.Context, callback objectCallb if !ok { return ret, false } - if err = m.clientWriter.Patch(ctx, obj, patch); err != nil { - m.logger.Error(err, "Unable to send patch", + if err = m.GetWriter().Patch(ctx, obj, patch); err != nil { + m.GetLogger().Error(err, "Unable to send patch", "kind", fmt.Sprintf("%T", obj), "name", obj.GetName(), "namespace", obj.GetNamespace(), @@ -93,7 +93,7 @@ func (m *AnnotationMutators) patchFunc(ctx context.Context, callback objectCallb } } -func (m *AnnotationMutators) restartNamespaceFunc(ctx context.Context) objectCallbackFunc { +func restartNamespaceFunc(m MonitorInterface, ctx context.Context) objectCallbackFunc { return func(obj client.Object, previousResult any) (any, bool) { mutatedAnnotations, ok := previousResult.(map[string]string) if !ok { @@ -103,21 +103,21 @@ func (m *AnnotationMutators) restartNamespaceFunc(ctx context.Context) objectCal if !ok { return nil, false } - m.RestartNamespace(ctx, namespace, mutatedAnnotations) + RestartNamespace(m, ctx, namespace, mutatedAnnotations) return nil, true } } // shouldRestartFunc returns a func that determines if a resource should be restarted -func (m *AnnotationMutators) shouldRestartFunc(namespaceMutatedAnnotations map[string]string) objectCallbackFunc { +func shouldRestartFunc(m MonitorInterface, namespaceMutatedAnnotations map[string]string) objectCallbackFunc { return func(obj client.Object, _ any) (any, bool) { switch o := obj.(type) { case *appsv1.Deployment: - return nil, m.shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) + return nil, shouldRestartResource(m, namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) case *appsv1.DaemonSet: - return nil, m.shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) + return nil, shouldRestartResource(m, namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) case *appsv1.StatefulSet: - return nil, m.shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) + return nil, shouldRestartResource(m, namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) default: return nil, false } @@ -125,13 +125,13 @@ func (m *AnnotationMutators) shouldRestartFunc(namespaceMutatedAnnotations map[s } // shouldRestartResource returns true if a resource requires a restart corresponding to the mutated annotations on its namespace -func (m *AnnotationMutators) shouldRestartResource(namespaceMutatedAnnotations map[string]string, obj metav1.Object) bool { +func shouldRestartResource(m MonitorInterface, namespaceMutatedAnnotations map[string]string, obj metav1.Object) bool { var shouldRestart bool if resourceAnnotations := obj.GetAnnotations(); resourceAnnotations != nil { // For each of the namespace mutated annotations, for namespaceMutatedAnnotation, namespaceMutatedAnnotationValue := range namespaceMutatedAnnotations { - if _, ok := m.injectAnnotations[namespaceMutatedAnnotation]; !ok { + if _, ok := m.GetAnnotationMutators().injectAnnotations[namespaceMutatedAnnotation]; !ok { // If it is not an inject-* annotation, we can ignore it continue } @@ -152,3 +152,27 @@ func (m *AnnotationMutators) shouldRestartResource(namespaceMutatedAnnotations m return shouldRestart } + +// RestartNamespace sets the restartedAtAnnotation for each of the namespace's supported resources and patches them. +func RestartNamespace(m MonitorInterface, ctx context.Context, namespace *corev1.Namespace, mutatedAnnotations map[string]string) { + callbackFunc := patchFunc(m, ctx, setRestartAnnotation) + rangeObjectList(m, ctx, &appsv1.DeploymentList{}, client.InNamespace(namespace.Name), chainCallbacks(shouldRestartFunc(m, mutatedAnnotations), callbackFunc)) + rangeObjectList(m, ctx, &appsv1.DaemonSetList{}, client.InNamespace(namespace.Name), chainCallbacks(shouldRestartFunc(m, mutatedAnnotations), callbackFunc)) + rangeObjectList(m, ctx, &appsv1.StatefulSetList{}, client.InNamespace(namespace.Name), chainCallbacks(shouldRestartFunc(m, mutatedAnnotations), callbackFunc)) +} + +// MutateAndPatchAll runs the mutators for each of the supported resources and patches them. +func MutateAndPatchAll(m MonitorInterface, ctx context.Context) { + f := getMutateObjectFunc(m) + callbackFunc := patchFunc(m, ctx, f) + rangeObjectList(m, ctx, &appsv1.DeploymentList{}, &client.ListOptions{}, callbackFunc) + rangeObjectList(m, ctx, &appsv1.DaemonSetList{}, &client.ListOptions{}, callbackFunc) + rangeObjectList(m, ctx, &appsv1.StatefulSetList{}, &client.ListOptions{}, callbackFunc) + rangeObjectList(m, ctx, &corev1.NamespaceList{}, &client.ListOptions{}, chainCallbacks(callbackFunc, restartNamespaceFunc(m, ctx))) +} + +func getMutateObjectFunc(m MonitorInterface) objectCallbackFunc { + return func(obj client.Object, _ any) (any, bool) { + return m.MutateObject(nil, obj), true + } +} From 9d6b4a77e814e33c8b872936a2446d9b39106a81 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 21 Mar 2025 17:21:09 -0400 Subject: [PATCH 14/63] Maintain legacy autoAnnotateAutoInstrumentation behavior. CustomSelectors should obey autoRestart, autoAnnotateAutoInstrumentation shouldn't. Rename MonitorInterface to InstrumentationAnnotator. Add docs. --- .../server/server_test.go | 2 +- .../namespacemutation/webhookhandler.go | 4 +- .../workloadmutation/webhookhandler.go | 4 +- main.go | 14 +- pkg/instrumentation/auto/annotation.go | 15 +- pkg/instrumentation/auto/auto_monitor.go | 147 +++++++++--------- pkg/instrumentation/auto/auto_monitor_test.go | 137 ++++++++-------- pkg/instrumentation/auto/callback.go | 14 +- pkg/instrumentation/auto/config.go | 23 ++- pkg/instrumentation/sdk_test.go | 2 +- 10 files changed, 187 insertions(+), 175 deletions(-) diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/server/server_test.go b/cmd/amazon-cloudwatch-agent-target-allocator/server/server_test.go index 2f41e2798..0ade38d21 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/server/server_test.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/server/server_test.go @@ -81,7 +81,7 @@ func TestServer_TargetsHandler(t *testing.T) { want want }{ { - name: "Empty target map", + name: "AnyResourcesConfigured target map", args: args{ collector: "test-collector", job: "test-job", diff --git a/internal/webhook/namespacemutation/webhookhandler.go b/internal/webhook/namespacemutation/webhookhandler.go index 3b58bec5e..c85c307e2 100644 --- a/internal/webhook/namespacemutation/webhookhandler.go +++ b/internal/webhook/namespacemutation/webhookhandler.go @@ -19,10 +19,10 @@ var _ admission.Handler = (*handler)(nil) type handler struct { decoder *admission.Decoder - monitor auto.MonitorInterface + monitor auto.InstrumentationAnnotator } -func NewWebhookHandler(decoder *admission.Decoder, monitor auto.MonitorInterface) admission.Handler { +func NewWebhookHandler(decoder *admission.Decoder, monitor auto.InstrumentationAnnotator) admission.Handler { return &handler{ decoder: decoder, monitor: monitor, diff --git a/internal/webhook/workloadmutation/webhookhandler.go b/internal/webhook/workloadmutation/webhookhandler.go index 30b20558e..e1ca071df 100644 --- a/internal/webhook/workloadmutation/webhookhandler.go +++ b/internal/webhook/workloadmutation/webhookhandler.go @@ -32,11 +32,11 @@ type WebhookHandler interface { // the implementation. type workloadMutationWebhook struct { decoder *admission.Decoder - monitor auto.MonitorInterface + monitor auto.InstrumentationAnnotator } // NewWebhookHandler creates a new WorkloadWebhookHandler. -func NewWebhookHandler(decoder *admission.Decoder, monitor auto.MonitorInterface) WebhookHandler { +func NewWebhookHandler(decoder *admission.Decoder, monitor auto.InstrumentationAnnotator) WebhookHandler { return &workloadMutationWebhook{ decoder: decoder, monitor: monitor, diff --git a/main.go b/main.go index cb4bd5487..ca6f50851 100644 --- a/main.go +++ b/main.go @@ -312,7 +312,7 @@ func main() { } } - monitor := createMonitorFromConfig(autoMonitorConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader(), autoAnnotationMutators) + monitor := createInstrumentationAnnotator(autoMonitorConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader(), autoAnnotationMutators) if monitor != nil { mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ @@ -371,16 +371,13 @@ func main() { } } -func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context, client client.Client, reader client.Reader, autoAnnotationMutators *auto.AnnotationMutators) auto.MonitorInterface { +func createInstrumentationAnnotator(autoMonitorConfigStr string, ctx context.Context, client client.Client, reader client.Reader, autoAnnotationMutators *auto.AnnotationMutators) auto.InstrumentationAnnotator { var monitorConfig *auto.MonitorConfig - var monitor auto.MonitorInterface = autoAnnotationMutators + var monitor auto.InstrumentationAnnotator = autoAnnotationMutators if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-monitor config, disabling AutoMonitor") return monitor } else { - if monitorConfig.Languages == nil || len(monitorConfig.Languages) == 0 { - monitorConfig.Languages = instrumentation.NewTypeSet(instrumentation.SupportedTypes()...) - } k8sConfig, err := rest.InClusterConfig() if err != nil { setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") @@ -392,8 +389,9 @@ func createMonitorFromConfig(autoMonitorConfigStr string, ctx context.Context, c setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") return monitor } - if autoAnnotationMutators != nil { - monitor = auto.NewMonitorWithExistingAnnotationMutator(ctx, *monitorConfig, clientSet, client, reader, setupLog, autoAnnotationMutators) + + if monitorConfig.CustomSelector.Empty() && !autoAnnotationMutators.Empty() { + monitor = auto.NewMonitorWithLegacyMutator(ctx, *monitorConfig, clientSet, client, reader, setupLog, autoAnnotationMutators) } else { monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, setupLog) } diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 46e8c3b18..0e96bd601 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -38,7 +38,7 @@ type AnnotationMutators struct { statefulSetMutators map[string]instrumentation.AnnotationMutator defaultMutator instrumentation.AnnotationMutator injectAnnotations map[string]struct{} - cfg *AnnotationConfig + cfg AnnotationConfig } func (m *AnnotationMutators) GetAnnotationMutators() *AnnotationMutators { @@ -57,11 +57,6 @@ func (m *AnnotationMutators) GetWriter() client.Writer { return m.clientWriter } -// IsManaged returns if AnnotationMutators would ever mutate the object. -func (m *AnnotationMutators) IsManaged(obj client.Object) bool { - return len(m.cfg.GetObjectLanguagesToAnnotate(obj)) > 0 -} - // MutateObject modifies annotations for a single object using the configured mutators. func (m *AnnotationMutators) MutateObject(_ client.Object, obj client.Object) map[string]string { mutatedAnnotations, _ := m.mutateObject(obj, nil) @@ -88,7 +83,7 @@ func (m *AnnotationMutators) mutateObject(obj client.Object, _ any) (any, bool) } } -func rangeObjectList(m MonitorInterface, ctx context.Context, list client.ObjectList, option client.ListOption, fn objectCallbackFunc) { +func rangeObjectList(m InstrumentationAnnotator, ctx context.Context, list client.ObjectList, option client.ListOption, fn objectCallbackFunc) { if err := m.GetReader().List(ctx, list, option); err != nil { m.GetLogger().Error(err, "Unable to list objects", "kind", fmt.Sprintf("%T", list), @@ -124,6 +119,10 @@ func (m *AnnotationMutators) mutate(name string, mutators map[string]instrumenta return mutatedAnnotations, len(mutatedAnnotations) != 0 } +func (m *AnnotationMutators) Empty() bool { + return m.cfg.Empty() +} + func namespacedName(obj metav1.Object) string { return fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName()) } @@ -149,7 +148,7 @@ func NewAnnotationMutators( statefulSetMutators: builder.buildMutators(getResources(cfg, typeSet, getStatefulSets)), defaultMutator: instrumentation.NewAnnotationMutator(maps.Values(builder.removeMutations)), injectAnnotations: buildInjectAnnotations(typeSet), - cfg: &cfg, + cfg: cfg, } } diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index f4fc53193..077c1c116 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -20,7 +20,8 @@ import ( "time" ) -type MonitorInterface interface { +// InstrumentationAnnotator is the highest level abstraction used to annotate kubernetes resources for instrumentation +type InstrumentationAnnotator interface { MutateObject(oldObj client.Object, obj client.Object) map[string]string GetAnnotationMutators() *AnnotationMutators GetLogger() logr.Logger @@ -29,14 +30,15 @@ type MonitorInterface interface { } type Monitor struct { - serviceInformer cache.SharedIndexInformer - ctx context.Context - config MonitorConfig - k8sInterface kubernetes.Interface - clientReader client.Reader - clientWriter client.Writer - customSelectors *AnnotationMutators - logger logr.Logger + serviceInformer cache.SharedIndexInformer + ctx context.Context + config MonitorConfig + k8sInterface kubernetes.Interface + clientReader client.Reader + clientWriter client.Writer + customSelectors *AnnotationMutators + logger logr.Logger + autoRestartCustomSelectors bool // Deprecated, do not use. } func (m *Monitor) GetAnnotationMutators() *AnnotationMutators { @@ -67,14 +69,27 @@ func (n NoopMonitor) AnyCustomSelectorDefined() bool { return false } -func NewMonitorWithExistingAnnotationMutator(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger, mutators *AnnotationMutators) *Monitor { +// NewMonitorWithLegacyMutator is used to create an InstrumentationMutator that supports AutoMonitor and the legacy autoAnnotateAutoInstrumentation +// +// *Deprecated, do not use directly.* +func NewMonitorWithLegacyMutator(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger, legacyMutator *AnnotationMutators) *Monitor { logger.Info("AutoMonitor starting...") // todo, throw warning if exclude config service is not namespaced (doesn't contain `/`) // todo: informers.WithTransform() as option to only store what parts of service are needed factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute) serviceInformer := factory.Core().V1().Services().Informer() + legacy := legacyMutator != nil + + var mutator *AnnotationMutators + if legacy { + // TODO: Add link to migration guide + logger.Error(nil, "Warning: Please switch from autoAnnotateAutoConfiguration to customSelector syntax. autoAnnotateAutoConfiguration is deprecated, and will be removed. For more info, visit here: ") + mutator = legacyMutator + } else { + mutator = NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.NewTypeSet(instrumentation.SupportedTypes()...)) + } - m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutators, clientReader: r, clientWriter: w} + m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutator, clientReader: r, clientWriter: w, autoRestartCustomSelectors: legacy} _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { m.onServiceEvent(nil, obj.(*corev1.Service)) @@ -99,19 +114,14 @@ func NewMonitorWithExistingAnnotationMutator(ctx context.Context, config Monitor panic("TODO: handle bad cache sync") } } - logger.Info("Enabled!") - if m.config.AutoRestart { - MutateAndPatchAll(m, ctx) - } else { - logger.Info("Auto restart disabled. To instrument workloads, restart the workloads exposed by a service.") - } logger.Info("Initialization complete!") return m } +// NewMonitor is used to create an InstrumentationMutator that supports AutoMonitor. func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { - return NewMonitorWithExistingAnnotationMutator(ctx, config, k8sInterface, w, r, logger, NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.NewTypeSet(instrumentation.SupportedTypes()...))) + return NewMonitorWithLegacyMutator(ctx, config, k8sInterface, w, r, logger, nil) } func (m *Monitor) onServiceEvent(oldService *corev1.Service, service *corev1.Service) { @@ -224,21 +234,25 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { // MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { - if !safeToMutate(oldObj, obj, m.config.AutoRestart) { + if !safeToMutate(oldObj, obj, m.config.AutoRestart, m.autoRestartCustomSelectors) { return map[string]string{} } - // todo: handle edge case where a workload is annotated because a service exposed it, and the service is removed. aka add to Service OnDelete - // custom selector takes precedence over service selector - if customSelectLanguages, selected := m.CustomSelected(obj); selected { - m.logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is specified in custom selector", obj.GetName(), customSelectLanguages)) - return mutate(obj, customSelectLanguages, true) + // custom selectors override default mutate logic + customSelectedLanguages := m.customSelectors.cfg.LanguagesOf(obj) + if len(customSelectedLanguages) > 0 { + return mutate(obj, customSelectedLanguages, true) } - return mutate(obj, m.config.Languages, m.shouldInsert(obj)) + return mutate(obj, m.config.Languages, m.isWorkloadAutoMonitored(obj)) } -func (m *Monitor) shouldInsert(obj client.Object) bool { +// returns if workload is auto monitored +func (m *Monitor) isWorkloadAutoMonitored(obj client.Object) bool { + if isNamespace(obj) { + return false + } + if !m.config.MonitorAllServices { return false } @@ -267,8 +281,8 @@ func (m *Monitor) shouldInsert(obj client.Object) bool { return false } -// mutate obj. If object is a workload, mutate the pod template. otherwise, mutate the object's annotations itself. -func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet, shouldInsert bool) map[string]string { +// mutate if object is a workload, mutate the pod template. otherwise, mutate the object's annotations itself. It will add annotations if needsInstrumentation is true. Otherwise, it will remove instrumentation annotations. +func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet, needsInstrumentation bool) map[string]string { var obj metav1.Object podTemplate := getPodTemplate(object) if podTemplate != nil { @@ -286,9 +300,8 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet, sh for _, language := range instrumentation.SupportedTypes() { insertMutation, removeMutation := buildMutations(language) var mutatedAnnotations map[string]string - if _, ok := languagesToMonitor[language]; ok && shouldInsert { + if _, ok := languagesToMonitor[language]; ok && needsInstrumentation { mutatedAnnotations = insertMutation.Mutate(annotations) - } else { mutatedAnnotations = removeMutation.Mutate(annotations) } @@ -300,11 +313,6 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet, sh return allMutatedAnnotations } -func (m *Monitor) CustomSelected(obj client.Object) (instrumentation.TypeSet, bool) { - languages := m.config.CustomSelector.GetObjectLanguagesToAnnotate(obj) - return languages, len(languages) > 0 -} - // excludedService returns whether a Namespace or a Service is excludedService from AutoMonitor. func (m *Monitor) excludedService(obj client.Object) bool { excluded := slices.Contains(m.config.Exclude.Services, namespacedName(obj)) || m.excludedNamespace(obj.GetNamespace()) @@ -322,26 +330,39 @@ func (m *Monitor) excludedNamespace(namespace string) bool { return slices.Contains(m.config.Exclude.Namespaces, namespace) } -func (m *Monitor) AnyCustomSelectorDefined() bool { - for _, t := range instrumentation.SupportedTypes() { - resources := m.config.CustomSelector.getResources(t) - if len(resources.DaemonSets) > 0 { - return true - } - if len(resources.StatefulSets) > 0 { - return true - } - if len(resources.Deployments) > 0 { - return true - } - if len(resources.Namespaces) > 0 { - return true - } +// safeToMutate returns whether the customer consents to the operator updating their workload's pods. The user consents if any of the following conditions are true: +// +// 1. Auto restart enabled. +// +// 2. preserve existing autoAnnotateAutoConfiguration behavior by mutating if no custom selectors are defined in the MonitorConfig, AND customSelectors mutates the provided workload +// +// 3. MonitorAllServices is enabled AND the workload is already going to restart (aka, the pod template is already modified) +func safeToMutate(oldWorkload client.Object, workload client.Object, autoRestart bool, autoRestartCustomSelectors bool) bool { + // always ok to mutate namespace + if isNamespace(workload) { + return true } - return false + // should only mutate workloads or namespaces + if !isMutableType(workload) { + return false + } + + if autoRestart { + return true + } + + if autoRestartCustomSelectors { + return true + } + + oldTemplate, newTemplate := getPodTemplate(oldWorkload), getPodTemplate(workload) + return !reflect.DeepEqual(oldTemplate, newTemplate) } -func isWorkload(obj client.Object) bool { +func isMutableType(obj client.Object) bool { + if isNamespace(obj) { + return true + } switch obj.(type) { case *appsv1.Deployment: return true @@ -354,26 +375,8 @@ func isWorkload(obj client.Object) bool { } } -// safeToMutate returns if object is already being mutated or if auto restart is enabled. does not guarantee that the object will be mutated. -func safeToMutate(oldObject client.Object, object client.Object, autoRestart bool) bool { - // mutating a namespace is always safe - if isNamespace(object) { - return true - } - // should only mutate workloads or namespaces - if !isWorkload(object) { - return false - } - - if autoRestart { - return true - } - oldTemplate, newTemplate := getPodTemplate(oldObject), getPodTemplate(object) - return !reflect.DeepEqual(oldTemplate, newTemplate) -} - -func isNamespace(object client.Object) bool { - switch object.(type) { +func isNamespace(obj client.Object) bool { + switch obj.(type) { case *corev1.Namespace: return true } diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index 68c5599b3..9f4c7d9cf 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/go-logr/logr" "github.com/go-logr/logr/testr" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" @@ -91,7 +92,7 @@ func TestMarshal(t *testing.T) { assert.ElementsMatch(t, []string{"nodejs", "python"}, s) } -func Test_allowedToMutate(t *testing.T) { +func Test_safeToMutate(t *testing.T) { deploy := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ @@ -116,25 +117,27 @@ func Test_allowedToMutate(t *testing.T) { }, } tests := []struct { - name string - oldObject client.Object - object client.Object - autoRestart bool - want bool + name string + oldObject client.Object + object client.Object + autoRestart bool + autoRestartCustomSelectors bool + want bool }{ - {"identical deployments", deploy.DeepCopy(), deploy.DeepCopy(), false, false}, - {"identical deployments, auto restart", deploy.DeepCopy(), deploy.DeepCopy(), true, true}, //should try and mutate in case deployment should no longer have annotations and mutators need to run to remove annotations - {"changed pod template", deploy.DeepCopy(), &mutatedDeploy, false, true}, - {"non-workload", &corev1.ConfigMap{}, &corev1.ConfigMap{}, false, false}, - {"non-workload, auto restart", &corev1.ConfigMap{}, &corev1.ConfigMap{Data: map[string]string{"test": "test"}}, true, false}, - {"create (oldObject nil)", nil, deploy.DeepCopy(), false, true}, - {"namespace, auto restart false", nil, &namespace, false, true}, - {"namespace, auto restart true", nil, &namespace, true, true}, + {"identical deployments", deploy.DeepCopy(), deploy.DeepCopy(), false, false, false}, + {"identical deployments, auto restart", deploy.DeepCopy(), deploy.DeepCopy(), true, false, true}, //should try and mutate in case deployment should no longer have annotations and mutators need to run to remove annotations + {"changed pod template", deploy.DeepCopy(), &mutatedDeploy, false, false, true}, + {"non-workload", &corev1.ConfigMap{}, &corev1.ConfigMap{}, false, false, false}, + {"non-workload, auto restart", &corev1.ConfigMap{}, &corev1.ConfigMap{Data: map[string]string{"test": "test"}}, true, false, false}, + {"create (oldObject nil)", nil, deploy.DeepCopy(), false, false, true}, + {"namespace, auto restart false", nil, &namespace, false, false, true}, + {"namespace, auto restart true", nil, &namespace, true, false, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, safeToMutate(tt.oldObject, tt.object, tt.autoRestart)) + //noinspection GoDeprecation + assert.Equal(t, tt.want, safeToMutate(tt.oldObject, tt.object, tt.autoRestart, tt.autoRestartCustomSelectors)) }) } } @@ -262,7 +265,7 @@ func TestMonitor_MutateObject(t *testing.T) { t.Run(workload.name, func(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - t.Parallel() + //t.Parallel() // Setup fresh clients for each workload test fakeClient := fake2.NewFakeClient() clientset := fake.NewSimpleClientset() @@ -316,7 +319,11 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(workload) c := fake2.NewFakeClient(workload) - NewMonitor(context.TODO(), createConfig(true, nil, nil, true), clientset, c, c, testr.New(t)) + var config MonitorConfig = createConfig(true, nil, nil, true) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + MutateAndPatchAll(monitor, context.TODO()) updatedWorkload, err := wt.getWithClient(c, defaultNs, workload.GetName()) assert.NoError(t, err) assert.Equal(t, userAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) @@ -331,7 +338,11 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(service, workload) c := fake2.NewFakeClient(service, workload) - monitor := NewMonitor(context.TODO(), createConfig(true, nil, nil, true), clientset, c, c, testr.New(t)) + var config MonitorConfig = createConfig(true, nil, nil, true) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + MutateAndPatchAll(monitor, context.TODO()) err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) assert.NoError(t, err) @@ -350,7 +361,11 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(workload) c := fake2.NewFakeClient(workload) - NewMonitor(context.TODO(), createConfig(true, nil, nil, false), clientset, c, c, testr.New(t)) + var config MonitorConfig = createConfig(true, nil, nil, false) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + MutateAndPatchAll(monitor, context.TODO()) updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) assert.NoError(t, err) @@ -366,7 +381,11 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(service, workload) c := fake2.NewFakeClient(service, workload) - monitor := NewMonitor(context.TODO(), createConfig(true, nil, nil, false), clientset, c, c, testr.New(t)) + var config MonitorConfig = createConfig(true, nil, nil, false) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + MutateAndPatchAll(monitor, context.TODO()) err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) assert.NoError(t, err) @@ -393,7 +412,11 @@ func Test_OptOutByDisablingMonitorAllServices(t *testing.T) { clientset := fake.NewSimpleClientset(service, workload) c := fake2.NewFakeClient(service, workload) - NewMonitor(context.TODO(), createConfig(false, nil, nil, true), clientset, c, c, testr.New(t)) + var config MonitorConfig = createConfig(false, nil, nil, true) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + MutateAndPatchAll(monitor, context.TODO()) updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) assert.NoError(t, err) @@ -411,6 +434,7 @@ func Test_mutate(t *testing.T) { languagesToMonitor instrumentation.TypeSet wantObjAnnotations map[string]string wantMutated map[string]string + shouldInsert bool }{ { name: "java only", @@ -418,6 +442,7 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, wantObjAnnotations: buildAnnotations("java"), wantMutated: buildAnnotations("java"), + shouldInsert: true, }, { name: "java and python", @@ -428,6 +453,7 @@ func Test_mutate(t *testing.T) { }, wantObjAnnotations: mergeMaps(buildAnnotations("java"), buildAnnotations("python")), wantMutated: mergeMaps(buildAnnotations("java"), buildAnnotations("python")), + shouldInsert: true, }, { name: "remove python instrumentation", @@ -435,6 +461,7 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{}, wantMutated: buildAnnotations("python"), + shouldInsert: true, }, { name: "remove one of two languages", @@ -442,6 +469,7 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, wantObjAnnotations: buildAnnotations("java"), wantMutated: buildAnnotations("python"), + shouldInsert: true, }, { name: "manually specified annotation is not touched", @@ -449,6 +477,7 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{instrumentation.InjectAnnotationKey(instrumentation.TypeJava): defaultAnnotationValue}, wantMutated: map[string]string{}, + shouldInsert: true, }, { name: "remove all", @@ -456,6 +485,7 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{}, wantMutated: buildAnnotations("java"), + shouldInsert: true, }, { name: "remove only language annotations", @@ -463,54 +493,15 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{"test": "test"}, wantMutated: buildAnnotations("java"), - }, - } - - workloadTypes := []struct { - name string - create func(annotations map[string]string) client.Object - }{ - { - name: "Deployment", - create: func(annotations map[string]string) client.Object { - return &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: annotations, - }, - }, - }, - } - }, + shouldInsert: true, }, { - name: "StatefulSet", - create: func(annotations map[string]string) client.Object { - return &appsv1.StatefulSet{ - Spec: appsv1.StatefulSetSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: annotations, - }, - }, - }, - } - }, - }, - { - name: "DaemonSet", - create: func(annotations map[string]string) client.Object { - return &appsv1.DaemonSet{ - Spec: appsv1.DaemonSetSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: annotations, - }, - }, - }, - } - }, + name: "respects isWorkloadAutoMonitored", + podAnnotations: mergeAnnotations(buildAnnotations("python"), map[string]string{"test": "test"}), + languagesToMonitor: instrumentation.TypeSet{"python": struct{}{}, "java": struct{}{}}, + wantObjAnnotations: map[string]string{"test": "test"}, + wantMutated: buildAnnotations("python"), + shouldInsert: false, }, } @@ -518,9 +509,9 @@ func Test_mutate(t *testing.T) { t.Run(workload.name, func(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - obj := workload.create(tt.podAnnotations).DeepCopyObject().(client.Object) - // TODO test different shouldInsert values - gotMutated := mutate(obj, tt.languagesToMonitor, true) + obj := workload.create("workload", "default", nil, tt.podAnnotations).DeepCopyObject().(client.Object) + // TODO test different isWorkloadAutoMonitored values + gotMutated := mutate(obj, tt.languagesToMonitor, tt.shouldInsert) assert.Equal(t, tt.wantObjAnnotations, getPodTemplate(obj).GetAnnotations()) assert.Equal(t, tt.wantMutated, gotMutated) }) @@ -546,8 +537,10 @@ func Test_StartupAutoRestart(t *testing.T) { objs := []runtime.Object{service, matchingDeployment, nonMatchingDeployment, customSelectedDeployment} clientset := fake.NewSimpleClientset(objs...) fakeClient := fake2.NewFakeClient(objs...) - m := NewMonitor(context.TODO(), config, clientset, fakeClient, fakeClient, testr.New(t)) - + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + m := NewMonitor(context.TODO(), config, k8sInterface, fakeClient, fakeClient, logger) + MutateAndPatchAll(m, context.TODO()) updatedMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments(defaultNs).Get(context.TODO(), matchingDeployment.Name, metav1.GetOptions{}) assert.NoError(t, err) assert.Equal(t, buildAnnotations(instrumentation.TypeJava), updatedMatchingDeployment.Spec.Template.GetAnnotations()) diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index 06f70d2d2..143ab202b 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -66,7 +66,7 @@ func createPatch(obj client.Object) (client.Patch, error) { return &basicPatch{originalJSON: originalJSON}, nil } -func patchFunc(m MonitorInterface, ctx context.Context, callback objectCallbackFunc) objectCallbackFunc { +func patchFunc(m InstrumentationAnnotator, ctx context.Context, callback objectCallbackFunc) objectCallbackFunc { return func(obj client.Object, _ any) (any, bool) { patch, err := createPatch(obj) if err != nil { @@ -93,7 +93,7 @@ func patchFunc(m MonitorInterface, ctx context.Context, callback objectCallbackF } } -func restartNamespaceFunc(m MonitorInterface, ctx context.Context) objectCallbackFunc { +func restartNamespaceFunc(m InstrumentationAnnotator, ctx context.Context) objectCallbackFunc { return func(obj client.Object, previousResult any) (any, bool) { mutatedAnnotations, ok := previousResult.(map[string]string) if !ok { @@ -109,7 +109,7 @@ func restartNamespaceFunc(m MonitorInterface, ctx context.Context) objectCallbac } // shouldRestartFunc returns a func that determines if a resource should be restarted -func shouldRestartFunc(m MonitorInterface, namespaceMutatedAnnotations map[string]string) objectCallbackFunc { +func shouldRestartFunc(m InstrumentationAnnotator, namespaceMutatedAnnotations map[string]string) objectCallbackFunc { return func(obj client.Object, _ any) (any, bool) { switch o := obj.(type) { case *appsv1.Deployment: @@ -125,7 +125,7 @@ func shouldRestartFunc(m MonitorInterface, namespaceMutatedAnnotations map[strin } // shouldRestartResource returns true if a resource requires a restart corresponding to the mutated annotations on its namespace -func shouldRestartResource(m MonitorInterface, namespaceMutatedAnnotations map[string]string, obj metav1.Object) bool { +func shouldRestartResource(m InstrumentationAnnotator, namespaceMutatedAnnotations map[string]string, obj metav1.Object) bool { var shouldRestart bool if resourceAnnotations := obj.GetAnnotations(); resourceAnnotations != nil { @@ -154,7 +154,7 @@ func shouldRestartResource(m MonitorInterface, namespaceMutatedAnnotations map[s } // RestartNamespace sets the restartedAtAnnotation for each of the namespace's supported resources and patches them. -func RestartNamespace(m MonitorInterface, ctx context.Context, namespace *corev1.Namespace, mutatedAnnotations map[string]string) { +func RestartNamespace(m InstrumentationAnnotator, ctx context.Context, namespace *corev1.Namespace, mutatedAnnotations map[string]string) { callbackFunc := patchFunc(m, ctx, setRestartAnnotation) rangeObjectList(m, ctx, &appsv1.DeploymentList{}, client.InNamespace(namespace.Name), chainCallbacks(shouldRestartFunc(m, mutatedAnnotations), callbackFunc)) rangeObjectList(m, ctx, &appsv1.DaemonSetList{}, client.InNamespace(namespace.Name), chainCallbacks(shouldRestartFunc(m, mutatedAnnotations), callbackFunc)) @@ -162,7 +162,7 @@ func RestartNamespace(m MonitorInterface, ctx context.Context, namespace *corev1 } // MutateAndPatchAll runs the mutators for each of the supported resources and patches them. -func MutateAndPatchAll(m MonitorInterface, ctx context.Context) { +func MutateAndPatchAll(m InstrumentationAnnotator, ctx context.Context) { f := getMutateObjectFunc(m) callbackFunc := patchFunc(m, ctx, f) rangeObjectList(m, ctx, &appsv1.DeploymentList{}, &client.ListOptions{}, callbackFunc) @@ -171,7 +171,7 @@ func MutateAndPatchAll(m MonitorInterface, ctx context.Context) { rangeObjectList(m, ctx, &corev1.NamespaceList{}, &client.ListOptions{}, chainCallbacks(callbackFunc, restartNamespaceFunc(m, ctx))) } -func getMutateObjectFunc(m MonitorInterface) objectCallbackFunc { +func getMutateObjectFunc(m InstrumentationAnnotator) objectCallbackFunc { return func(obj client.Object, _ any) (any, bool) { return m.MutateObject(nil, obj), true } diff --git a/pkg/instrumentation/auto/config.go b/pkg/instrumentation/auto/config.go index ac5741785..c11ca8344 100644 --- a/pkg/instrumentation/auto/config.go +++ b/pkg/instrumentation/auto/config.go @@ -35,8 +35,8 @@ func (c AnnotationConfig) getResources(instType instrumentation.Type) Annotation } } -// GetObjectLanguagesToAnnotate get languages to annotate for an object -func (c AnnotationConfig) GetObjectLanguagesToAnnotate(obj client.Object) instrumentation.TypeSet { +// LanguagesOf get languages to annotate for an object +func (c AnnotationConfig) LanguagesOf(obj client.Object) instrumentation.TypeSet { objName := namespacedName(obj) typesSelected := instrumentation.TypeSet{} @@ -72,6 +72,25 @@ func (c AnnotationConfig) GetObjectLanguagesToAnnotate(obj client.Object) instru return typesSelected } +func (c AnnotationConfig) Empty() bool { + for _, t := range instrumentation.SupportedTypes() { + resources := c.getResources(t) + if len(resources.DaemonSets) > 0 { + return false + } + if len(resources.StatefulSets) > 0 { + return false + } + if len(resources.Deployments) > 0 { + return false + } + if len(resources.Namespaces) > 0 { + return false + } + } + return true +} + // AnnotationResources contains slices of resource names for each // of the supported workloads. type AnnotationResources struct { diff --git a/pkg/instrumentation/sdk_test.go b/pkg/instrumentation/sdk_test.go index d77d1271d..0cec74c1d 100644 --- a/pkg/instrumentation/sdk_test.go +++ b/pkg/instrumentation/sdk_test.go @@ -298,7 +298,7 @@ func TestSDKInjection(t *testing.T) { }, }, { - name: "Empty instrumentation spec", + name: "AnyResourcesConfigured instrumentation spec", inst: v1alpha1.Instrumentation{ Spec: v1alpha1.InstrumentationSpec{}, }, From 1216808d735f7dbec42107a01444b192bdcad55d Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Mon, 24 Mar 2025 14:10:58 -0400 Subject: [PATCH 15/63] Clean up --- Dockerfile | 7 +++++-- README.md | 2 +- .../server/server_test.go | 2 +- .../annotations/validate_annotation_methods.go | 1 + .../validate_automonitor_deployment_test.go | 14 +++++++------- .../webhook/workloadmutation/webhookhandler.go | 4 +--- main.go | 11 ++++++----- pkg/instrumentation/annotationmutator.go | 7 +------ pkg/instrumentation/annotationtype.go | 4 ++-- pkg/instrumentation/auto/auto_monitor.go | 4 ++-- pkg/instrumentation/auto/config.go | 10 +++++----- 11 files changed, 32 insertions(+), 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7bf26e26b..c03e24aeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,7 @@ COPY go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN go mod download +RUN go install github.com/go-delve/delve/cmd/dlv@latest # Copy the go source COPY main.go main.go @@ -37,9 +38,11 @@ RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -ldflags="-X ${VERSION_PKG} # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +#FROM gcr.io/distroless/static:nonroot +FROM ubuntu:latest WORKDIR / COPY --from=builder /workspace/manager . +COPY --from=builder /go/bin/dlv . USER 65532:65532 -ENTRYPOINT ["/manager"] \ No newline at end of file +ENTRYPOINT ["/dlv", "--listen=:2345", "--headless=true", "--api-version=2", "exec", "/manager"] \ No newline at end of file diff --git a/README.md b/README.md index d658cb0a3..706926de2 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ kubectl get all ``` kubectl logs amazon-cloudwatch-agent-operator-controller-manager-66f67f47f78 -```k +``` You should see logs that look similar to below diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/server/server_test.go b/cmd/amazon-cloudwatch-agent-target-allocator/server/server_test.go index 0ade38d21..2f41e2798 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/server/server_test.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/server/server_test.go @@ -81,7 +81,7 @@ func TestServer_TargetsHandler(t *testing.T) { want want }{ { - name: "AnyResourcesConfigured target map", + name: "Empty target map", args: args{ collector: "test-collector", job: "test-job", diff --git a/integration-tests/manifests/annotations/validate_annotation_methods.go b/integration-tests/manifests/annotations/validate_annotation_methods.go index 7fdaaaf51..b2dcf209c 100644 --- a/integration-tests/manifests/annotations/validate_annotation_methods.go +++ b/integration-tests/manifests/annotations/validate_annotation_methods.go @@ -240,6 +240,7 @@ func checkIfAnnotationExists(clientset *kubernetes.Clientset, pods *v1.PodList, //check if all pods are in the Running phase if !util.CheckIfPodsAreRunning(currentPods) { + continue } diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index cd008aae8..88a1f70a3 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -34,7 +34,7 @@ func TestServiceThenDeployment(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + Languages: instrumentation.SupportedTypes(), AutoRestart: false, }) err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) @@ -55,7 +55,7 @@ func TestDeploymentThenServiceAutoRestartDisabled(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + Languages: instrumentation.SupportedTypes(), AutoRestart: false, }) @@ -84,7 +84,7 @@ func TestDeploymentThenServiceAutoRestartEnabled(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + Languages: instrumentation.SupportedTypes(), AutoRestart: true, }) @@ -122,7 +122,7 @@ func TestDeploymentWithCustomSelector(t *testing.T) { // Update operator with auto monitor disabled and custom selector helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: false, - Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + Languages: instrumentation.SupportedTypes(), AutoRestart: false, CustomSelector: customSelectorConfig, }) @@ -146,7 +146,7 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { // Update operator with auto monitor disabled helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: false, - Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + Languages: instrumentation.SupportedTypes(), AutoRestart: false, }) @@ -178,7 +178,7 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: false, - Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + Languages: instrumentation.SupportedTypes(), AutoRestart: false, CustomSelector: customSelectorConfig, }) @@ -198,7 +198,7 @@ func TestDeploymentWithExcludedThenIncludedService(t *testing.T) { // Set up config with service exclusion monitorConfig := auto.MonitorConfig{ MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.SupportedTypes()...), + Languages: instrumentation.SupportedTypes(), Exclude: struct { Namespaces []string `json:"namespaces"` Services []string `json:"services"` diff --git a/internal/webhook/workloadmutation/webhookhandler.go b/internal/webhook/workloadmutation/webhookhandler.go index e1ca071df..36d9da84b 100644 --- a/internal/webhook/workloadmutation/webhookhandler.go +++ b/internal/webhook/workloadmutation/webhookhandler.go @@ -11,7 +11,6 @@ import ( v1 "k8s.io/api/admission/v1" appsv1 "k8s.io/api/apps/v1" "net/http" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -22,7 +21,6 @@ import ( // +kubebuilder:rbac:groups="apps",resources=daemonsets;deployments;statefulsets,verbs=get;list;watch var _ WebhookHandler = (*workloadMutationWebhook)(nil) -var logger = ctrl.Log.WithName("workload_webhook") // WebhookHandler is a webhook handler that analyzes new daemon-sets and injects appropriate annotations into it. type WebhookHandler interface { @@ -66,7 +64,7 @@ func (p *workloadMutationWebhook) Handle(_ context.Context, req admission.Reques // populate old object if req.Operation == v1.Update { if err := p.decoder.DecodeRaw(req.OldObject, oldObj); err != nil { - logger.Error(err, "failed to unmarshal old object") + p.monitor.GetLogger().WithName("workload_webhook").Error(err, "failed to unmarshal old object") return admission.Errored(http.StatusBadRequest, err) } } diff --git a/main.go b/main.go index ca6f50851..484052f4d 100644 --- a/main.go +++ b/main.go @@ -293,7 +293,7 @@ func main() { // TODO handle case where auto monitor is enabled but auto annotation config is not specified var autoAnnotationConfig auto.AnnotationConfig var autoAnnotationMutators *auto.AnnotationMutators - supportedLanguages := instrumentation.NewTypeSet(instrumentation.SupportedTypes()...) + supportedLanguages := instrumentation.SupportedTypes() if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" || autoAnnotationConfigStr == "" { setupLog.Info("Auto-annotation is disabled") @@ -316,7 +316,8 @@ func main() { if monitor != nil { mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ - Handler: workloadmutation.NewWebhookHandler(decoder, monitor)}) + Handler: workloadmutation.NewWebhookHandler(decoder, monitor), + }) mgr.GetWebhookServer().Register("/mutate-v1-namespace", &webhook.Admission{ Handler: namespacemutation.NewWebhookHandler(decoder, monitor), }) @@ -389,11 +390,11 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, ctx context.Con setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") return monitor } - + logger := ctrl.Log.WithName("auto_monitor") if monitorConfig.CustomSelector.Empty() && !autoAnnotationMutators.Empty() { - monitor = auto.NewMonitorWithLegacyMutator(ctx, *monitorConfig, clientSet, client, reader, setupLog, autoAnnotationMutators) + monitor = auto.NewMonitorWithLegacyMutator(ctx, *monitorConfig, clientSet, client, reader, logger, autoAnnotationMutators) } else { - monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, setupLog) + monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, logger) } } return monitor diff --git a/pkg/instrumentation/annotationmutator.go b/pkg/instrumentation/annotationmutator.go index 4c5707b82..134916072 100644 --- a/pkg/instrumentation/annotationmutator.go +++ b/pkg/instrumentation/annotationmutator.go @@ -77,7 +77,7 @@ func NewAnnotationMutator(mutations []AnnotationMutation) AnnotationMutator { } // Mutate modifies the object's annotations based on the mutator's mutations. Returns all the mutated annotations. -func (m AnnotationMutator) Mutate(obj metav1.Object) map[string]string { +func (m *AnnotationMutator) Mutate(obj metav1.Object) map[string]string { annotations := obj.GetAnnotations() if annotations == nil { annotations = make(map[string]string) @@ -92,8 +92,3 @@ func (m AnnotationMutator) Mutate(obj metav1.Object) map[string]string { obj.SetAnnotations(annotations) return allMutatedAnnotations } - -// ObjectAnnotationMutator responsible for getting, mutating, and setting the object's annotations map. Returns all the mutated annotations. -type ObjectAnnotationMutator interface { - Mutate(obj metav1.Object) map[string]string -} diff --git a/pkg/instrumentation/annotationtype.go b/pkg/instrumentation/annotationtype.go index 0f3d2d42d..ae90127fb 100644 --- a/pkg/instrumentation/annotationtype.go +++ b/pkg/instrumentation/annotationtype.go @@ -47,8 +47,8 @@ const ( TypeGo Type = "go" ) -func SupportedTypes() []Type { - return []Type{TypeJava, TypeNodeJS, TypePython, TypeDotNet} +func SupportedTypes() TypeSet { + return NewTypeSet(TypeJava, TypeNodeJS, TypePython, TypeDotNet) } // InjectAnnotationKey maps the instrumentation type to the inject annotation. diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/auto_monitor.go index 077c1c116..7ebc59e4a 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/auto_monitor.go @@ -86,7 +86,7 @@ func NewMonitorWithLegacyMutator(ctx context.Context, config MonitorConfig, k8sI logger.Error(nil, "Warning: Please switch from autoAnnotateAutoConfiguration to customSelector syntax. autoAnnotateAutoConfiguration is deprecated, and will be removed. For more info, visit here: ") mutator = legacyMutator } else { - mutator = NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.NewTypeSet(instrumentation.SupportedTypes()...)) + mutator = NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.SupportedTypes()) } m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutator, clientReader: r, clientWriter: w, autoRestartCustomSelectors: legacy} @@ -297,7 +297,7 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet, ne } allMutatedAnnotations := map[string]string{} - for _, language := range instrumentation.SupportedTypes() { + for language := range instrumentation.SupportedTypes() { insertMutation, removeMutation := buildMutations(language) var mutatedAnnotations map[string]string if _, ok := languagesToMonitor[language]; ok && needsInstrumentation { diff --git a/pkg/instrumentation/auto/config.go b/pkg/instrumentation/auto/config.go index c11ca8344..3b024fa37 100644 --- a/pkg/instrumentation/auto/config.go +++ b/pkg/instrumentation/auto/config.go @@ -44,25 +44,25 @@ func (c AnnotationConfig) LanguagesOf(obj client.Object) instrumentation.TypeSet switch obj.(type) { case *appsv1.Deployment: - for _, t := range types { + for t := range types { if slices.Contains(c.getResources(t).Deployments, objName) { typesSelected[t] = nil } } case *appsv1.StatefulSet: - for _, t := range types { + for t := range types { if slices.Contains(c.getResources(t).StatefulSets, objName) { typesSelected[t] = nil } } case *appsv1.DaemonSet: - for _, t := range types { + for t := range types { if slices.Contains(c.getResources(t).DaemonSets, objName) { typesSelected[t] = nil } } case *corev1.Namespace: - for _, t := range types { + for t := range types { if slices.Contains(c.getResources(t).Namespaces, objName) { typesSelected[t] = nil } @@ -73,7 +73,7 @@ func (c AnnotationConfig) LanguagesOf(obj client.Object) instrumentation.TypeSet } func (c AnnotationConfig) Empty() bool { - for _, t := range instrumentation.SupportedTypes() { + for t := range instrumentation.SupportedTypes() { resources := c.getResources(t) if len(resources.DaemonSets) > 0 { return false From b5a0e968fcfc9b1c6147dd9f42b1d1af321f98aa Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Mon, 24 Mar 2025 14:12:17 -0400 Subject: [PATCH 16/63] undo dockerfile update --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index c03e24aeb..7bf26e26b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,6 @@ COPY go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN go mod download -RUN go install github.com/go-delve/delve/cmd/dlv@latest # Copy the go source COPY main.go main.go @@ -38,11 +37,9 @@ RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -ldflags="-X ${VERSION_PKG} # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -#FROM gcr.io/distroless/static:nonroot -FROM ubuntu:latest +FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/manager . -COPY --from=builder /go/bin/dlv . USER 65532:65532 -ENTRYPOINT ["/dlv", "--listen=:2345", "--headless=true", "--api-version=2", "exec", "/manager"] \ No newline at end of file +ENTRYPOINT ["/manager"] \ No newline at end of file From e1cf759444b28e0274585b208dbbc9882ffed1ea Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Tue, 25 Mar 2025 14:26:54 -0400 Subject: [PATCH 17/63] Fix integration tests --- .../validate_automonitor_deployment_test.go | 49 +++++---- .../validate_automonitor_methods.go | 84 ++++++++++----- integration-tests/util/util.go | 25 +---- pkg/instrumentation/auto/auto_monitor_test.go | 101 +++++++++++++----- 4 files changed, 157 insertions(+), 102 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index 88a1f70a3..87dcb491f 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -3,7 +3,6 @@ package annotations import ( - "fmt" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" "github.com/stretchr/testify/assert" @@ -37,7 +36,7 @@ func TestServiceThenDeployment(t *testing.T) { Languages: instrumentation.SupportedTypes(), AutoRestart: false, }) - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) assert.NoError(t, err) err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, none) @@ -59,10 +58,7 @@ func TestDeploymentThenServiceAutoRestartDisabled(t *testing.T) { AutoRestart: false, }) - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) - - // Check annotations - fmt.Println("Checking if sample-deployment exists") + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) assert.NoError(t, err) @@ -88,18 +84,16 @@ func TestDeploymentThenServiceAutoRestartEnabled(t *testing.T) { AutoRestart: true, }) - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) assert.NoError(t, err) - // Check annotations - fmt.Println("Checking if sample-deployment exists") err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) assert.NoError(t, err) helper.startTime = time.Now() err = helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) assert.NoError(t, err) - + time.Sleep(1 * time.Second) err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, none) assert.NoError(t, err) } @@ -112,10 +106,10 @@ func TestDeploymentWithCustomSelector(t *testing.T) { // Set up custom selector config customSelectorConfig := auto.AnnotationConfig{ Java: auto.AnnotationResources{ - Deployments: []string{"sample-deployment"}, + Deployments: []string{namespace + "/sample-deployment"}, }, Python: auto.AnnotationResources{ - Deployments: []string{"sample-deployment"}, + Deployments: []string{namespace + "/sample-deployment"}, }, } @@ -128,7 +122,7 @@ func TestDeploymentWithCustomSelector(t *testing.T) { }) // Create deployment - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) assert.NoError(t, err) // Validate annotations are present @@ -151,7 +145,7 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { }) // Create deployment - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) assert.NoError(t, err) // Validate no annotations present @@ -161,18 +155,19 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { assert.NoError(t, err) // Update operator with custom selector + namespacedDeployment := namespace + "sample-deployment" customSelectorConfig := auto.AnnotationConfig{ Java: auto.AnnotationResources{ - Deployments: []string{"sample-deployment"}, + Deployments: []string{namespacedDeployment}, }, Python: auto.AnnotationResources{ - Deployments: []string{"sample-deployment"}, + Deployments: []string{namespacedDeployment}, }, DotNet: auto.AnnotationResources{ - Deployments: []string{"sample-deployment"}, + Deployments: []string{namespacedDeployment}, }, NodeJS: auto.AnnotationResources{ - Deployments: []string{"sample-deployment"}, + Deployments: []string{namespacedDeployment}, }, } @@ -183,10 +178,16 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { CustomSelector: customSelectorConfig, }) - // Validate annotations are present - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, - none) + // Validate annotations are not present + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + assert.NoError(t, err) + + err = helper.RestartDeployment(namespace, "sample-deployment") + assert.NoError(t, err) + + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) assert.NoError(t, err) } @@ -216,11 +217,9 @@ func TestDeploymentWithExcludedThenIncludedService(t *testing.T) { assert.NoError(t, err) // Create deployment - err = helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYamlNameRelPath}) + err = helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) assert.NoError(t, err) - fmt.Println("Sleeping!") - time.Sleep(1 * time.Minute) // Validate that deployment has no annotations err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go index 280024e83..ea5043051 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -7,6 +7,8 @@ import ( "encoding/json" "fmt" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" + "github.com/go-logr/logr" + "github.com/go-logr/logr/testr" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" @@ -47,7 +49,7 @@ const ( amazonControllerManager = "amazon-cloudwatch-observability-controller-manager" sampleDaemonsetYamlRelPath = "../sample-daemonset.yaml" - sampleDeploymentYamlNameRelPath = "../sample-deployment.yaml" + sampleDeploymentYaml = "../sample-deployment.yaml" sampleNginxAppYamlNameRelPath = "../../java/sample-deployment-java.yaml" sampleStatefulsetYamlNameRelPath = "../sample-statefulset.yaml" @@ -61,23 +63,27 @@ type TestHelper struct { t *testing.T startTime time.Time skipDelete bool + logger logr.Logger } func NewTestHelper(t *testing.T, skipDelete bool) *TestHelper { + logger := testr.New(t) return &TestHelper{ - clientSet: setupTest(t), + clientSet: setupTest(t, logger), t: t, skipDelete: skipDelete, + logger: logger, } } -func setupTest(t *testing.T) *kubernetes.Clientset { +func setupTest(t *testing.T, logger logr.Logger) *kubernetes.Clientset { userHomeDir, err := os.UserHomeDir() + if err != nil { t.Errorf("error getting user home dir: %v\n\n", err) } kubeConfigPath := filepath.Join(userHomeDir, ".kube", "config") - fmt.Printf("Using kubeconfig: %s\n\n", kubeConfigPath) + logger.Info(fmt.Sprintf("Using kubeconfig: %s\n\n", kubeConfigPath)) kubeConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) if err != nil { @@ -93,12 +99,12 @@ func setupTest(t *testing.T) *kubernetes.Clientset { func (h *TestHelper) ApplyYAMLWithKubectl(filename, namespace string) error { cmd := exec.Command("kubectl", "apply", "-f", filename, "-n", namespace) - fmt.Printf("Applying YAML with kubectl %s\n", cmd) + h.logger.Info(fmt.Sprintf("Applying YAML with kubectl %s\n", cmd)) return cmd.Run() } func (h *TestHelper) CreateNamespaceAndApplyResources(namespace string, resourceFiles []string) error { - fmt.Printf("Creating namespace %s\n", namespace) + h.logger.Info(fmt.Sprintf("Creating namespace %s\n", namespace)) err := h.CreateNamespace(namespace) if err != nil { return err @@ -116,7 +122,10 @@ func (h *TestHelper) CreateNamespaceAndApplyResources(namespace string, resource err = h.WaitYamlWithKubectl(file, namespace) if err != nil { h.t.Errorf("Could not wait resources %s/%s\n", namespace, file) + return err } + // ignore error because it could run on resource which doesn't rollout (aka a service) + _ = h.RolloutWaitYamlWithKubectl(file, namespace) } return nil } @@ -124,7 +133,7 @@ func (h *TestHelper) CreateNamespaceAndApplyResources(namespace string, resource func (h *TestHelper) IsNamespaceUpdated(namespace string) bool { ns, err := h.clientSet.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) if err != nil { - fmt.Printf("Failed to get namespace %s: %v\n", namespace, err) + h.logger.Info(fmt.Sprintf("Failed to get namespace %s: %v\n", namespace, err)) return false } return ns.CreationTimestamp.After(h.startTime) || ns.ResourceVersion != "" @@ -193,12 +202,12 @@ func (h *TestHelper) CheckNameSpaceAnnotations(expectedAnnotations []string, uni for { if h.IsNamespaceUpdated(uniqueNamespace) { - fmt.Printf("Namespace %s has been updated.\n", uniqueNamespace) + h.logger.Info(fmt.Sprintf("Namespace %s has been updated.\n", uniqueNamespace)) break } elapsed := time.Since(h.startTime) if elapsed >= timoutDuration { - fmt.Printf("Timeout reached while waiting for namespace %s to be updated.\n", uniqueNamespace) + h.logger.Info(fmt.Sprintf("Timeout reached while waiting for namespace %s to be updated.\n", uniqueNamespace)) break } } @@ -207,7 +216,7 @@ func (h *TestHelper) CheckNameSpaceAnnotations(expectedAnnotations []string, uni correct := true ns, err := h.clientSet.CoreV1().Namespaces().Get(context.TODO(), uniqueNamespace, metav1.GetOptions{}) if err != nil { - fmt.Println("There was an error getting namespace, ", err) + h.logger.Error(err, "There was an error getting namespace, ") return false } @@ -220,7 +229,7 @@ func (h *TestHelper) CheckNameSpaceAnnotations(expectedAnnotations []string, uni } if correct { - fmt.Println("Namespace annotations are correct!") + h.logger.Info("Namespace annotations are correct!") return true } } @@ -254,12 +263,11 @@ func (h *TestHelper) UpdateOperator(deployment *appsV1.Deployment) bool { err := util.WaitForNewPodCreation(h.clientSet, deployment, now) if err != nil { - fmt.Println("There was an error trying to wait for deployment available", err) + h.logger.Error(err, "There was an error trying to wait for deployment available") return false } - fmt.Println("Operator updated successfully!") - fmt.Println(args) + h.logger.Info("Operator updated successfully!", "args", args) return true } @@ -272,30 +280,31 @@ func forceRestart(deployment *appsV1.Deployment) { deployment.Spec.Template.SetAnnotations(annotations) } +// TODO: should check deployment template spec? func (h *TestHelper) PodsAnnotationsValid(namespace string, shouldExistAnnotations []string, shouldNotExistAnnotations []string) bool { currentPods, err := h.clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) if err != nil { - fmt.Printf("Failed to list pods: %v\n", err) + h.logger.Info(fmt.Sprintf("Failed to list pods: %v\n", err)) return false } validAnnotations := true for _, pod := range currentPods.Items { - fmt.Printf("Pod %s is in phase %s\n", pod.Name, pod.Status.Phase) - fmt.Println("Pod ", pod.GetAnnotations()) + h.logger.Info(fmt.Sprintf("Pod %s is in phase %s\n", pod.Name, pod.Status.Phase)) + h.logger.Info("Pod ", pod.GetAnnotations()) if pod.Status.Phase != v1.PodRunning { continue } for _, annotation := range shouldExistAnnotations { if value, exists := pod.Annotations[annotation]; !exists || value != "true" { - fmt.Println("Pod", pod.Namespace, pod.Name, " does not have annotation ", annotation) + h.logger.Info("Pod", pod.Namespace, pod.Name, " does not have annotation ", annotation) validAnnotations = false break } } for _, annotation := range shouldNotExistAnnotations { if _, exists := pod.Annotations[annotation]; exists { - fmt.Println("Pod", pod.Namespace, pod.Name, " shouldn't have annotation ", annotation) + h.logger.Info("Pod", pod.Namespace, pod.Name, " shouldn't have annotation ", annotation) validAnnotations = false break } @@ -315,7 +324,7 @@ func (h *TestHelper) restartOperator() { cmd := exec.Command("kubectl", "rollout", "restart", "deployment", amazonControllerManager, "-n", amazonCloudwatchNamespace) output, err := cmd.CombinedOutput() if err != nil { - fmt.Printf("Error restarting deployment: %v\nOutput: %s\n", err, output) + h.logger.Info(fmt.Sprintf("Error restarting deployment: %v\nOutput: %s\n", err, output)) return } @@ -326,7 +335,7 @@ func (h *TestHelper) restartOperator() { waitOutput, err := waitCmd.CombinedOutput() if err != nil { - fmt.Printf("Error waiting for deployment: %v\nOutput: %s\n", err, waitOutput) + h.logger.Info(fmt.Sprintf("Error waiting for deployment: %v\nOutput: %s\n", err, waitOutput)) } } @@ -343,7 +352,7 @@ func (h *TestHelper) UpdateMonitorConfig(config auto.MonitorConfig) { jsonStr, err := json.Marshal(config) assert.Nil(h.t, err) - fmt.Println("Setting monitor config to:") + h.logger.Info("Setting monitor config to:") util.PrettyPrint(config) h.updateOperatorConfig(string(jsonStr), "--auto-monitor-config=") } @@ -351,7 +360,7 @@ func (h *TestHelper) UpdateMonitorConfig(config auto.MonitorConfig) { func (h *TestHelper) UpdateAnnotationConfig(config auto.AnnotationConfig) { jsonStr, err := json.Marshal(config) assert.Nil(h.t, err) - fmt.Println("Setting annotation config to:") + h.logger.Info("Setting annotation config to:") util.PrettyPrint(config) h.updateOperatorConfig(string(jsonStr), "--auto-annotation-config=") } @@ -377,6 +386,14 @@ func (h *TestHelper) updateOperatorConfig(jsonStr string, flag string) { } func (h *TestHelper) ValidateWorkloadAnnotations(resourceType, uniqueNamespace, resourceName string, shouldExist []string, shouldNotExist []string) error { + return retry.OnError(retry.DefaultBackoff, func(err error) bool { + return err != nil + }, func() error { + return h.ValidateWorkloadAnnotationsA(resourceType, uniqueNamespace, resourceName, shouldExist, shouldNotExist) + }) +} + +func (h *TestHelper) ValidateWorkloadAnnotationsA(resourceType, uniqueNamespace, resourceName string, shouldExist []string, shouldNotExist []string) error { var resource interface{} var err error @@ -434,7 +451,7 @@ func (h *TestHelper) Initialize(namespace string, apps []string) string { if !h.skipDelete { h.t.Cleanup(func() { - fmt.Printf("Deleting namespace %s and resources %s", uniqueNamespace, apps) + h.logger.Info(fmt.Sprintf("Deleting namespace %s and resources %s", uniqueNamespace, apps)) if err := h.DeleteNamespaceAndResources(uniqueNamespace, apps); err != nil { h.t.Fatalf("Failed to delete namespaces/resources: %v", err) } @@ -460,7 +477,13 @@ func (h *TestHelper) NumberOfRevisions(deploymentName string, namespace string) func (h *TestHelper) WaitYamlWithKubectl(filename string, namespace string) error { cmd := exec.Command("kubectl", "wait", "--for=create", "-f", filename, "-n", namespace) - fmt.Printf("Waiting YAML with kubectl %s\n", cmd) + h.logger.Info(fmt.Sprintf("Waiting YAML with kubectl %s\n", cmd)) + return cmd.Run() +} + +func (h *TestHelper) RolloutWaitYamlWithKubectl(filename string, namespace string) error { + cmd := exec.Command("kubectl", "rollout", "status", "-f", filename, "-n", namespace) + h.logger.Info(fmt.Sprintf("Waiting YAML with kubectl %s\n", cmd)) return cmd.Run() } @@ -480,7 +503,14 @@ func (h *TestHelper) RestartDeployment(namespace string, deploymentName string) // Update the deployment _, updateErr := h.clientSet.AppsV1().Deployments(namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return updateErr + if updateErr != nil { + return fmt.Errorf("failed to update deployment: %v", updateErr) + } + + // wait + cmd := exec.Command("kubectl", "rollout", "status", "deployment/"+deploymentName, "-n", namespace) + h.logger.Info(fmt.Sprintf("Waiting YAML with kubectl %s\n", cmd)) + return cmd.Run() }) if retryErr != nil { @@ -493,7 +523,7 @@ func (h *TestHelper) RestartDeployment(namespace string, deploymentName string) return fmt.Errorf("failed to wait for deployment rollout: %v", err) } - fmt.Printf("Successfully restarted deployment %s in namespace %s\n", deploymentName, namespace) + h.logger.Info(fmt.Sprintf("Successfully restarted deployment %s in namespace %s\n", deploymentName, namespace)) return nil } diff --git a/integration-tests/util/util.go b/integration-tests/util/util.go index 5818d417e..1408561a8 100644 --- a/integration-tests/util/util.go +++ b/integration-tests/util/util.go @@ -17,13 +17,10 @@ import ( "k8s.io/client-go/kubernetes" ) -const TimoutDuration = 2 * time.Minute +const TimoutDuration = 1 * time.Minute const TimeBetweenRetries = 2 * time.Second func WaitForNewPodCreation(clientSet *kubernetes.Clientset, resource interface{}, startTime time.Time) error { - // 1. Use wait.PollImmediate instead of manual polling - // 2. Move type switch outside the polling loop - fmt.Println("start time: ", startTime) namespace := "" labelSelector := "" switch r := resource.(type) { @@ -39,41 +36,23 @@ func WaitForNewPodCreation(clientSet *kubernetes.Clientset, resource interface{} default: return fmt.Errorf("unsupported resource type") } - return wait.PollImmediate(TimeBetweenRetries, TimoutDuration, func() (bool, error) { - // 3. Handle list error + return wait.PollUntilContextTimeout(context.TODO(), TimeBetweenRetries, TimoutDuration, true, func(ctx context.Context) (bool, error) { newPods, err := clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: labelSelector, }) - // get list of pod names - //podNames := make([]string, 0) - //for _, pod := range newPods.Items { - // podNames = append(podNames, pod.Name) - //} - // - //fmt.Println(podNames) if err != nil { return false, fmt.Errorf("failed to list pods: %v", err) } - - // 4. Check for pod readiness, not just running for _, pod := range newPods.Items { - //fmt.Println("Pod Name: ", pod.Name, ", pod.Creation: ", pod.CreationTimestamp) if pod.CreationTimestamp.Time.After(startTime.Add(-time.Second)) { if pod.Status.Phase == v1.PodRunning { - // 5. Check if pod is ready isReady := isPodReady(&pod) if isReady { - fmt.Printf("pod %s created after start time and is ready\n", pod.Name) return true, nil } - //fmt.Printf("pod %s is running but not ready\n", pod.Name) - } else { - //fmt.Printf("pod %s created after start time but is in %s state\n", - // pod.Name, pod.Status.Phase) } } } - return false, nil }) } diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/auto_monitor_test.go index 9f4c7d9cf..f2829fdbe 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/auto_monitor_test.go @@ -168,28 +168,33 @@ func Test_getPodTemplate(t *testing.T) { } } +type MutateObjectTest struct { + name string + config MonitorConfig + deploymentNs string + serviceNs string + deploymentSelector map[string]string + serviceSelector map[string]string + expectedWorkloadAnnotations map[string]string +} + +var none = AnnotationConfig{} + func TestMonitor_MutateObject(t *testing.T) { - tests := []struct { - name string - config MonitorConfig - deploymentNs string - serviceNs string - deploymentSelector map[string]string - serviceSelector map[string]string - expectedWorkloadAnnotations map[string]string - }{ + annotated := buildAnnotations(instrumentation.TypeJava) + tests := []MutateObjectTest{ { name: "same namespace, same selector, monitorallservices true, not excluded", - config: createConfig(true, nil, nil, false), + config: simpleConfig(true, nil, nil, false, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "same"}, serviceSelector: map[string]string{"app": "same"}, - expectedWorkloadAnnotations: buildAnnotations(instrumentation.TypeJava), + expectedWorkloadAnnotations: annotated, }, { name: "different namespace, same selector, monitorallservices true, not excluded", - config: createConfig(true, nil, nil, false), + config: simpleConfig(true, nil, nil, false, none), deploymentNs: "namespace-2", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "same"}, @@ -198,7 +203,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, different selector, monitorallservices true, not excluded", - config: createConfig(true, nil, nil, false), + config: simpleConfig(true, nil, nil, false, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -207,7 +212,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, same selector, monitorallservices false, not excluded", - config: createConfig(false, nil, nil, false), + config: simpleConfig(false, nil, nil, false, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -216,7 +221,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, same selector, monitorallservices true, excluded namespace", - config: createConfig(true, []string{"namespace-1"}, nil, false), + config: simpleConfig(true, []string{"namespace-1"}, nil, false, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -225,13 +230,55 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, same selector, monitorallservices true, excluded service", - config: createConfig(true, nil, []string{"namespace-1/svc-16"}, false), + config: simpleConfig(true, nil, []string{"namespace-1/svc"}, false, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, serviceSelector: map[string]string{"app": "different-1"}, expectedWorkloadAnnotations: map[string]string{}, }, + { + name: "different namespace, different selector, monitorallservices false, custom selected workload", + config: simpleConfig(true, nil, nil, false, AnnotationConfig{Java: AnnotationResources{ + Namespaces: nil, + Deployments: []string{"namespace-1/workload"}, + DaemonSets: []string{"namespace-1/workload"}, + StatefulSets: []string{"namespace-1/workload"}, + }}), + deploymentNs: "namespace-1", + serviceNs: "namespace-2", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-2"}, + expectedWorkloadAnnotations: annotated, + }, + { + name: "different namespace, different selector, monitorallservices false, custom selected namespace of workload", + config: simpleConfig(true, nil, nil, false, AnnotationConfig{Java: AnnotationResources{ + Namespaces: []string{"namespace-1"}, + Deployments: nil, + DaemonSets: nil, + StatefulSets: nil, + }}), + deploymentNs: "namespace-1", + serviceNs: "namespace-2", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-2"}, + expectedWorkloadAnnotations: map[string]string{}, // empty because even though it should be custom selected, it is modified on the pod level for namespaces, so the pod template is not updated + }, + { + name: "different namespace, different selector, monitorallservices false, custom selected namespace of service, not workload", + config: simpleConfig(true, nil, nil, false, AnnotationConfig{Java: AnnotationResources{ + Namespaces: []string{"namespace-2"}, + Deployments: nil, + DaemonSets: nil, + StatefulSets: nil, + }}), + deploymentNs: "namespace-1", + serviceNs: "namespace-2", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-2"}, + expectedWorkloadAnnotations: map[string]string{}, // empty because even though it should be custom selected, it is modified on the pod level for namespaces, so the pod template is not updated + }, } workloadTypes := []struct { @@ -241,21 +288,21 @@ func TestMonitor_MutateObject(t *testing.T) { { name: "Deployment", create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { - deployment := newTestDeployment("workload-16", ns, selector, nil) + deployment := newTestDeployment("workload", ns, selector, nil) return clientset.AppsV1().Deployments(ns).Create(ctx, deployment, metav1.CreateOptions{}) }, }, { name: "StatefulSet", create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { - statefulset := newTestStatefulSet("workload-16", ns, selector, nil) + statefulset := newTestStatefulSet("workload", ns, selector, nil) return clientset.AppsV1().StatefulSets(ns).Create(ctx, statefulset, metav1.CreateOptions{}) }, }, { name: "DaemonSet", create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { - daemonset := newTestDaemonSet("workload-16", ns, selector, nil) + daemonset := newTestDaemonSet("workload", ns, selector, nil) return clientset.AppsV1().DaemonSets(ns).Create(ctx, daemonset, metav1.CreateOptions{}) }, }, @@ -275,7 +322,7 @@ func TestMonitor_MutateObject(t *testing.T) { monitor := NewMonitor(ctx, tt.config, clientset, fakeClient, fakeClient, logger) // Create service - service := newTestService("svc-16", tt.serviceNs, tt.serviceSelector) + service := newTestService("svc", tt.serviceNs, tt.serviceSelector) // Setup test environment serviceNamespace := createNamespace(t, clientset, ctx, service.Namespace) @@ -319,7 +366,7 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(workload) c := fake2.NewFakeClient(workload) - var config MonitorConfig = createConfig(true, nil, nil, true) + var config MonitorConfig = simpleConfig(true, nil, nil, true, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -338,7 +385,7 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(service, workload) c := fake2.NewFakeClient(service, workload) - var config MonitorConfig = createConfig(true, nil, nil, true) + var config MonitorConfig = simpleConfig(true, nil, nil, true, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -361,7 +408,7 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(workload) c := fake2.NewFakeClient(workload) - var config MonitorConfig = createConfig(true, nil, nil, false) + var config MonitorConfig = simpleConfig(true, nil, nil, false, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -381,7 +428,7 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(service, workload) c := fake2.NewFakeClient(service, workload) - var config MonitorConfig = createConfig(true, nil, nil, false) + var config MonitorConfig = simpleConfig(true, nil, nil, false, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -412,7 +459,7 @@ func Test_OptOutByDisablingMonitorAllServices(t *testing.T) { clientset := fake.NewSimpleClientset(service, workload) c := fake2.NewFakeClient(service, workload) - var config MonitorConfig = createConfig(false, nil, nil, true) + var config MonitorConfig = simpleConfig(false, nil, nil, true, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -659,7 +706,7 @@ func mergeMaps(maps ...map[string]string) map[string]string { return result } -func createConfig(monitorAll bool, excludedNs, excludedSvcs []string, autoRestart bool) MonitorConfig { +func simpleConfig(monitorAll bool, excludedNs, excludedSvcs []string, autoRestart bool, customSelector AnnotationConfig) MonitorConfig { return MonitorConfig{ MonitorAllServices: monitorAll, Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), @@ -671,6 +718,6 @@ func createConfig(monitorAll bool, excludedNs, excludedSvcs []string, autoRestar Namespaces: excludedNs, Services: excludedSvcs, }, - CustomSelector: AnnotationConfig{}, + CustomSelector: customSelector, } } From 33df3e8993b9d39a8441986cf0b154e2f2cde9a1 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 26 Mar 2025 16:02:25 -0400 Subject: [PATCH 18/63] Rename auto_monitor*.go files to monitor*.go, rename AutoRestart to RestartPods --- .../validate_automonitor_deployment_test.go | 16 ++++----- .../auto/{auto_monitor.go => monitor.go} | 34 ++++++++----------- ...to_monitor_config.go => monitor_config.go} | 2 +- .../{auto_monitor_test.go => monitor_test.go} | 14 ++++---- 4 files changed, 31 insertions(+), 35 deletions(-) rename pkg/instrumentation/auto/{auto_monitor.go => monitor.go} (94%) rename pkg/instrumentation/auto/{auto_monitor_config.go => monitor_config.go} (91%) rename pkg/instrumentation/auto/{auto_monitor_test.go => monitor_test.go} (99%) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index 87dcb491f..9f19351a8 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -34,7 +34,7 @@ func TestServiceThenDeployment(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.SupportedTypes(), - AutoRestart: false, + RestartPods: false, }) err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) assert.NoError(t, err) @@ -44,7 +44,7 @@ func TestServiceThenDeployment(t *testing.T) { } // create deployment, create service, should not annotate anything -func TestDeploymentThenServiceAutoRestartDisabled(t *testing.T) { +func TestDeploymentThenServiceRestartPodsDisabled(t *testing.T) { helper := NewTestHelper(t, true) namespace := helper.Initialize("test-namespace", []string{}) @@ -55,7 +55,7 @@ func TestDeploymentThenServiceAutoRestartDisabled(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.SupportedTypes(), - AutoRestart: false, + RestartPods: false, }) err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) @@ -70,7 +70,7 @@ func TestDeploymentThenServiceAutoRestartDisabled(t *testing.T) { assert.NoError(t, err) } -func TestDeploymentThenServiceAutoRestartEnabled(t *testing.T) { +func TestDeploymentThenServiceRestartPodsEnabled(t *testing.T) { helper := NewTestHelper(t, true) namespace := helper.Initialize("test-namespace", []string{}) @@ -81,7 +81,7 @@ func TestDeploymentThenServiceAutoRestartEnabled(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.SupportedTypes(), - AutoRestart: true, + RestartPods: true, }) err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) @@ -117,7 +117,7 @@ func TestDeploymentWithCustomSelector(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: false, Languages: instrumentation.SupportedTypes(), - AutoRestart: false, + RestartPods: false, CustomSelector: customSelectorConfig, }) @@ -141,7 +141,7 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: false, Languages: instrumentation.SupportedTypes(), - AutoRestart: false, + RestartPods: false, }) // Create deployment @@ -174,7 +174,7 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { helper.UpdateMonitorConfig(auto.MonitorConfig{ MonitorAllServices: false, Languages: instrumentation.SupportedTypes(), - AutoRestart: false, + RestartPods: false, CustomSelector: customSelectorConfig, }) diff --git a/pkg/instrumentation/auto/auto_monitor.go b/pkg/instrumentation/auto/monitor.go similarity index 94% rename from pkg/instrumentation/auto/auto_monitor.go rename to pkg/instrumentation/auto/monitor.go index 7ebc59e4a..524b23eac 100644 --- a/pkg/instrumentation/auto/auto_monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -30,15 +30,15 @@ type InstrumentationAnnotator interface { } type Monitor struct { - serviceInformer cache.SharedIndexInformer - ctx context.Context - config MonitorConfig - k8sInterface kubernetes.Interface - clientReader client.Reader - clientWriter client.Writer - customSelectors *AnnotationMutators - logger logr.Logger - autoRestartCustomSelectors bool // Deprecated, do not use. + serviceInformer cache.SharedIndexInformer + ctx context.Context + config MonitorConfig + k8sInterface kubernetes.Interface + clientReader client.Reader + clientWriter client.Writer + customSelectors *AnnotationMutators + logger logr.Logger + restartCustomSelectors bool // Deprecated, do not use. } func (m *Monitor) GetAnnotationMutators() *AnnotationMutators { @@ -65,10 +65,6 @@ func (n NoopMonitor) MutateObject(_ client.Object, _ client.Object) map[string]s return map[string]string{} } -func (n NoopMonitor) AnyCustomSelectorDefined() bool { - return false -} - // NewMonitorWithLegacyMutator is used to create an InstrumentationMutator that supports AutoMonitor and the legacy autoAnnotateAutoInstrumentation // // *Deprecated, do not use directly.* @@ -89,7 +85,7 @@ func NewMonitorWithLegacyMutator(ctx context.Context, config MonitorConfig, k8sI mutator = NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.SupportedTypes()) } - m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutator, clientReader: r, clientWriter: w, autoRestartCustomSelectors: legacy} + m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutator, clientReader: r, clientWriter: w, restartCustomSelectors: legacy} _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { m.onServiceEvent(nil, obj.(*corev1.Service)) @@ -125,7 +121,7 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet } func (m *Monitor) onServiceEvent(oldService *corev1.Service, service *corev1.Service) { - if !m.config.AutoRestart { + if !m.config.RestartPods { return } for _, resource := range m.listServiceDeployments(m.ctx, oldService, service) { @@ -234,7 +230,7 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { // MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { - if !safeToMutate(oldObj, obj, m.config.AutoRestart, m.autoRestartCustomSelectors) { + if !safeToMutate(oldObj, obj, m.config.RestartPods, m.restartCustomSelectors) { return map[string]string{} } @@ -337,7 +333,7 @@ func (m *Monitor) excludedNamespace(namespace string) bool { // 2. preserve existing autoAnnotateAutoConfiguration behavior by mutating if no custom selectors are defined in the MonitorConfig, AND customSelectors mutates the provided workload // // 3. MonitorAllServices is enabled AND the workload is already going to restart (aka, the pod template is already modified) -func safeToMutate(oldWorkload client.Object, workload client.Object, autoRestart bool, autoRestartCustomSelectors bool) bool { +func safeToMutate(oldWorkload client.Object, workload client.Object, restartPods bool, restartCustomSelectors bool) bool { // always ok to mutate namespace if isNamespace(workload) { return true @@ -347,11 +343,11 @@ func safeToMutate(oldWorkload client.Object, workload client.Object, autoRestart return false } - if autoRestart { + if restartPods { return true } - if autoRestartCustomSelectors { + if restartCustomSelectors { return true } diff --git a/pkg/instrumentation/auto/auto_monitor_config.go b/pkg/instrumentation/auto/monitor_config.go similarity index 91% rename from pkg/instrumentation/auto/auto_monitor_config.go rename to pkg/instrumentation/auto/monitor_config.go index 68cc9df4d..86957e13c 100644 --- a/pkg/instrumentation/auto/auto_monitor_config.go +++ b/pkg/instrumentation/auto/monitor_config.go @@ -10,7 +10,7 @@ import "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" type MonitorConfig struct { MonitorAllServices bool `json:"monitorAllServices"` Languages instrumentation.TypeSet `json:"languages"` - AutoRestart bool `json:"autoRestart"` + RestartPods bool `json:"restartPods"` Exclude struct { Namespaces []string `json:"namespaces"` Services []string `json:"services"` diff --git a/pkg/instrumentation/auto/auto_monitor_test.go b/pkg/instrumentation/auto/monitor_test.go similarity index 99% rename from pkg/instrumentation/auto/auto_monitor_test.go rename to pkg/instrumentation/auto/monitor_test.go index f2829fdbe..9fed9fb69 100644 --- a/pkg/instrumentation/auto/auto_monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -120,8 +120,8 @@ func Test_safeToMutate(t *testing.T) { name string oldObject client.Object object client.Object - autoRestart bool - autoRestartCustomSelectors bool + restartPods bool + restartPodsCustomSelectors bool want bool }{ {"identical deployments", deploy.DeepCopy(), deploy.DeepCopy(), false, false, false}, @@ -137,7 +137,7 @@ func Test_safeToMutate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { //noinspection GoDeprecation - assert.Equal(t, tt.want, safeToMutate(tt.oldObject, tt.object, tt.autoRestart, tt.autoRestartCustomSelectors)) + assert.Equal(t, tt.want, safeToMutate(tt.oldObject, tt.object, tt.restartPods, tt.restartPodsCustomSelectors)) }) } } @@ -567,7 +567,7 @@ func Test_mutate(t *testing.T) { } } -func Test_StartupAutoRestart(t *testing.T) { +func Test_StartupRestartPods(t *testing.T) { service := newTestService("service-1", defaultNs, map[string]string{"test": "test"}) matchingDeployment := newTestDeployment("deployment-1", defaultNs, map[string]string{"test": "test"}, nil) nonMatchingDeployment := newTestDeployment("deployment-2", defaultNs, map[string]string{}, nil) @@ -575,7 +575,7 @@ func Test_StartupAutoRestart(t *testing.T) { config := MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), - AutoRestart: true, + RestartPods: true, CustomSelector: AnnotationConfig{ AnnotationResources{}, AnnotationResources{Deployments: []string{namespacedName(customSelectedDeployment)}}, AnnotationResources{}, AnnotationResources{}, @@ -706,11 +706,11 @@ func mergeMaps(maps ...map[string]string) map[string]string { return result } -func simpleConfig(monitorAll bool, excludedNs, excludedSvcs []string, autoRestart bool, customSelector AnnotationConfig) MonitorConfig { +func simpleConfig(monitorAll bool, excludedNs, excludedSvcs []string, restartPods bool, customSelector AnnotationConfig) MonitorConfig { return MonitorConfig{ MonitorAllServices: monitorAll, Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), - AutoRestart: autoRestart, + RestartPods: restartPods, Exclude: struct { Namespaces []string `json:"namespaces"` Services []string `json:"services"` From 734966fbda1f4923fe8d0a4aad4fd599b9388890 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 27 Mar 2025 10:27:00 -0400 Subject: [PATCH 19/63] Enable multi instrumentation feature flag, enable skip multi instrumentation container validation by default. This is required for automonitor to work, because it tries to apply all init containers to each selected pod. --- pkg/featuregate/featuregate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/featuregate/featuregate.go b/pkg/featuregate/featuregate.go index 6ae36fc37..f6a1e6f74 100644 --- a/pkg/featuregate/featuregate.go +++ b/pkg/featuregate/featuregate.go @@ -59,7 +59,7 @@ var ( EnableMultiInstrumentationSupport = featuregate.GlobalRegistry().MustRegister( "operator.autoinstrumentation.multi-instrumentation", - featuregate.StageAlpha, + featuregate.StageBeta, featuregate.WithRegisterFromVersion("0.86.0"), featuregate.WithRegisterDescription("controls whether the operator supports multi instrumentation")) @@ -86,7 +86,7 @@ var ( // annotations from being used. SkipMultiInstrumentationContainerValidation = featuregate.GlobalRegistry().MustRegister( "operator.autoinstrumentation.multi-instrumentation.skip-container-validation", - featuregate.StageAlpha, + featuregate.StageBeta, featuregate.WithRegisterDescription("controls whether the operator validates the container annotations when multi-instrumentation is enabled")) ) From ed696e5f4b9f022f3fff4db8bc34cd188758f648 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 9 Apr 2025 10:11:46 -0400 Subject: [PATCH 20/63] come up with actual commit message --- main.go | 54 ++++++------ pkg/instrumentation/auto/config.go | 6 ++ pkg/instrumentation/auto/monitor.go | 96 ++++++---------------- pkg/instrumentation/auto/monitor_config.go | 7 +- pkg/instrumentation/auto/monitor_test.go | 50 +++++------ 5 files changed, 80 insertions(+), 133 deletions(-) diff --git a/main.go b/main.go index 484052f4d..201de8ff9 100644 --- a/main.go +++ b/main.go @@ -290,29 +290,7 @@ func main() { decoder := admission.NewDecoder(mgr.GetScheme()) - // TODO handle case where auto monitor is enabled but auto annotation config is not specified - var autoAnnotationConfig auto.AnnotationConfig - var autoAnnotationMutators *auto.AnnotationMutators - supportedLanguages := instrumentation.SupportedTypes() - - if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" || autoAnnotationConfigStr == "" { - setupLog.Info("Auto-annotation is disabled") - } else { - if err = json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { - setupLog.Error(err, "Unable to unmarshal auto-annotation config") - } else { - // TODO: detect empty - autoAnnotationMutators = auto.NewAnnotationMutators( - mgr.GetClient(), - mgr.GetAPIReader(), - logger, - autoAnnotationConfig, - supportedLanguages, - ) - } - } - - monitor := createInstrumentationAnnotator(autoMonitorConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader(), autoAnnotationMutators) + monitor := createInstrumentationAnnotator(autoMonitorConfigStr, autoInstrumentationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) if monitor != nil { mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ @@ -372,7 +350,29 @@ func main() { } } -func createInstrumentationAnnotator(autoMonitorConfigStr string, ctx context.Context, client client.Client, reader client.Reader, autoAnnotationMutators *auto.AnnotationMutators) auto.InstrumentationAnnotator { +func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, client client.Client, reader client.Reader) auto.InstrumentationAnnotator { + var autoAnnotationConfig auto.AnnotationConfig + var autoAnnotationMutators *auto.AnnotationMutators + supportedLanguages := instrumentation.SupportedTypes() + + if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" { + setupLog.Info("Auto-annotation is disabled") + } else if autoAnnotationConfigStr != "" { + if err := json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { + setupLog.Error(err, "Unable to unmarshal auto-annotation config") + } else { + // TODO: detect empty + setupLog.Info("WARNING: Using deprecated autoAnnotateAutoInstrumentation config. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") + return auto.NewAnnotationMutators( + client, + reader, + setupLog, + autoAnnotationConfig, + supportedLanguages, + ) + } + } + var monitorConfig *auto.MonitorConfig var monitor auto.InstrumentationAnnotator = autoAnnotationMutators if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { @@ -391,11 +391,7 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, ctx context.Con return monitor } logger := ctrl.Log.WithName("auto_monitor") - if monitorConfig.CustomSelector.Empty() && !autoAnnotationMutators.Empty() { - monitor = auto.NewMonitorWithLegacyMutator(ctx, *monitorConfig, clientSet, client, reader, logger, autoAnnotationMutators) - } else { - monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, logger) - } + monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, logger) } return monitor } diff --git a/pkg/instrumentation/auto/config.go b/pkg/instrumentation/auto/config.go index 3b024fa37..4b5e8d350 100644 --- a/pkg/instrumentation/auto/config.go +++ b/pkg/instrumentation/auto/config.go @@ -42,6 +42,12 @@ func (c AnnotationConfig) LanguagesOf(obj client.Object) instrumentation.TypeSet types := instrumentation.SupportedTypes() + for t := range types { + if slices.Contains(c.getResources(t).Namespaces, obj.GetNamespace()) { + typesSelected[t] = nil + } + } + switch obj.(type) { case *appsv1.Deployment: for t := range types { diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 524b23eac..a659a4a89 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -16,7 +16,6 @@ import ( "reflect" "sigs.k8s.io/controller-runtime/pkg/client" "slices" - "strings" "time" ) @@ -30,15 +29,14 @@ type InstrumentationAnnotator interface { } type Monitor struct { - serviceInformer cache.SharedIndexInformer - ctx context.Context - config MonitorConfig - k8sInterface kubernetes.Interface - clientReader client.Reader - clientWriter client.Writer - customSelectors *AnnotationMutators - logger logr.Logger - restartCustomSelectors bool // Deprecated, do not use. + serviceInformer cache.SharedIndexInformer + ctx context.Context + config MonitorConfig + k8sInterface kubernetes.Interface + clientReader client.Reader + clientWriter client.Writer + customSelectors *AnnotationMutators + logger logr.Logger } func (m *Monitor) GetAnnotationMutators() *AnnotationMutators { @@ -65,27 +63,17 @@ func (n NoopMonitor) MutateObject(_ client.Object, _ client.Object) map[string]s return map[string]string{} } -// NewMonitorWithLegacyMutator is used to create an InstrumentationMutator that supports AutoMonitor and the legacy autoAnnotateAutoInstrumentation -// -// *Deprecated, do not use directly.* -func NewMonitorWithLegacyMutator(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger, legacyMutator *AnnotationMutators) *Monitor { +// NewMonitor is used to create an InstrumentationMutator that supports AutoMonitor. +func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { logger.Info("AutoMonitor starting...") // todo, throw warning if exclude config service is not namespaced (doesn't contain `/`) // todo: informers.WithTransform() as option to only store what parts of service are needed factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute) serviceInformer := factory.Core().V1().Services().Informer() - legacy := legacyMutator != nil var mutator *AnnotationMutators - if legacy { - // TODO: Add link to migration guide - logger.Error(nil, "Warning: Please switch from autoAnnotateAutoConfiguration to customSelector syntax. autoAnnotateAutoConfiguration is deprecated, and will be removed. For more info, visit here: ") - mutator = legacyMutator - } else { - mutator = NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.SupportedTypes()) - } - m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutator, clientReader: r, clientWriter: w, restartCustomSelectors: legacy} + m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutator, clientReader: r, clientWriter: w} _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { m.onServiceEvent(nil, obj.(*corev1.Service)) @@ -115,11 +103,6 @@ func NewMonitorWithLegacyMutator(ctx context.Context, config MonitorConfig, k8sI return m } -// NewMonitor is used to create an InstrumentationMutator that supports AutoMonitor. -func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { - return NewMonitorWithLegacyMutator(ctx, config, k8sInterface, w, r, logger, nil) -} - func (m *Monitor) onServiceEvent(oldService *corev1.Service, service *corev1.Service) { if !m.config.RestartPods { return @@ -230,17 +213,20 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { // MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { - if !safeToMutate(oldObj, obj, m.config.RestartPods, m.restartCustomSelectors) { + if !safeToMutate(oldObj, obj, m.config.RestartPods) { return map[string]string{} } - - // custom selectors override default mutate logic - customSelectedLanguages := m.customSelectors.cfg.LanguagesOf(obj) - if len(customSelectedLanguages) > 0 { - return mutate(obj, customSelectedLanguages, true) + languagesToAnnotate := m.customSelectors.cfg.LanguagesOf(obj) + if m.isWorkloadAutoMonitored(obj) { + for l := range m.config.Languages { + languagesToAnnotate[l] = nil + } + } + for l := range m.config.Exclude.LanguagesOf(obj) { + delete(languagesToAnnotate, l) } - return mutate(obj, m.config.Languages, m.isWorkloadAutoMonitored(obj)) + return mutate(obj, languagesToAnnotate) } // returns if workload is auto monitored @@ -253,17 +239,10 @@ func (m *Monitor) isWorkloadAutoMonitored(obj client.Object) bool { return false } - if m.excludedNamespace(obj.GetNamespace()) { - return false - } - // determine if the object is currently selected by a service objectLabels := getTemplateSpecLabels(obj) for _, informerObj := range m.serviceInformer.GetStore().List() { service := informerObj.(*corev1.Service) - if m.excludedService(service) { - continue - } if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 || service.GetNamespace() != obj.GetNamespace() { continue } @@ -278,7 +257,7 @@ func (m *Monitor) isWorkloadAutoMonitored(obj client.Object) bool { } // mutate if object is a workload, mutate the pod template. otherwise, mutate the object's annotations itself. It will add annotations if needsInstrumentation is true. Otherwise, it will remove instrumentation annotations. -func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet, needsInstrumentation bool) map[string]string { +func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet) map[string]string { var obj metav1.Object podTemplate := getPodTemplate(object) if podTemplate != nil { @@ -296,7 +275,7 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet, ne for language := range instrumentation.SupportedTypes() { insertMutation, removeMutation := buildMutations(language) var mutatedAnnotations map[string]string - if _, ok := languagesToMonitor[language]; ok && needsInstrumentation { + if _, ok := languagesToMonitor[language]; ok { mutatedAnnotations = insertMutation.Mutate(annotations) } else { mutatedAnnotations = removeMutation.Mutate(annotations) @@ -309,31 +288,11 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet, ne return allMutatedAnnotations } -// excludedService returns whether a Namespace or a Service is excludedService from AutoMonitor. -func (m *Monitor) excludedService(obj client.Object) bool { - excluded := slices.Contains(m.config.Exclude.Services, namespacedName(obj)) || m.excludedNamespace(obj.GetNamespace()) - m.logger.Info(fmt.Sprintf("%s excluded? %v", namespacedName(obj), excluded)) - return excluded -} - -func (m *Monitor) excludedNamespace(namespace string) bool { - if strings.HasPrefix(namespace, "kube-") { - return false - } - if strings.EqualFold(namespace, "amazon-cloudwatch") { - return false - } - return slices.Contains(m.config.Exclude.Namespaces, namespace) -} - // safeToMutate returns whether the customer consents to the operator updating their workload's pods. The user consents if any of the following conditions are true: // // 1. Auto restart enabled. -// -// 2. preserve existing autoAnnotateAutoConfiguration behavior by mutating if no custom selectors are defined in the MonitorConfig, AND customSelectors mutates the provided workload -// -// 3. MonitorAllServices is enabled AND the workload is already going to restart (aka, the pod template is already modified) -func safeToMutate(oldWorkload client.Object, workload client.Object, restartPods bool, restartCustomSelectors bool) bool { +// 2. MonitorAllServices is enabled AND the workload is already going to restart (aka, the pod template is already modified) +func safeToMutate(oldWorkload client.Object, workload client.Object, restartPods bool) bool { // always ok to mutate namespace if isNamespace(workload) { return true @@ -346,11 +305,6 @@ func safeToMutate(oldWorkload client.Object, workload client.Object, restartPods if restartPods { return true } - - if restartCustomSelectors { - return true - } - oldTemplate, newTemplate := getPodTemplate(oldWorkload), getPodTemplate(workload) return !reflect.DeepEqual(oldTemplate, newTemplate) } diff --git a/pkg/instrumentation/auto/monitor_config.go b/pkg/instrumentation/auto/monitor_config.go index 86957e13c..64d6f9bd8 100644 --- a/pkg/instrumentation/auto/monitor_config.go +++ b/pkg/instrumentation/auto/monitor_config.go @@ -11,9 +11,6 @@ type MonitorConfig struct { MonitorAllServices bool `json:"monitorAllServices"` Languages instrumentation.TypeSet `json:"languages"` RestartPods bool `json:"restartPods"` - Exclude struct { - Namespaces []string `json:"namespaces"` - Services []string `json:"services"` - } `json:"exclude"` - CustomSelector AnnotationConfig `json:"customSelector"` + Exclude AnnotationConfig `json:"exclude"` + CustomSelector AnnotationConfig `json:"customSelector"` } diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index 9fed9fb69..447900279 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -137,7 +137,7 @@ func Test_safeToMutate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { //noinspection GoDeprecation - assert.Equal(t, tt.want, safeToMutate(tt.oldObject, tt.object, tt.restartPods, tt.restartPodsCustomSelectors)) + assert.Equal(t, tt.want, safeToMutate(tt.oldObject, tt.object, tt.restartPods)) }) } } @@ -185,7 +185,7 @@ func TestMonitor_MutateObject(t *testing.T) { tests := []MutateObjectTest{ { name: "same namespace, same selector, monitorallservices true, not excluded", - config: simpleConfig(true, nil, nil, false, none), + config: simpleConfig(true, false, none, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "same"}, @@ -194,7 +194,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "different namespace, same selector, monitorallservices true, not excluded", - config: simpleConfig(true, nil, nil, false, none), + config: simpleConfig(true, false, none, none), deploymentNs: "namespace-2", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "same"}, @@ -203,7 +203,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, different selector, monitorallservices true, not excluded", - config: simpleConfig(true, nil, nil, false, none), + config: simpleConfig(true, false, none, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -212,7 +212,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, same selector, monitorallservices false, not excluded", - config: simpleConfig(false, nil, nil, false, none), + config: simpleConfig(false, false, none, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -221,7 +221,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, same selector, monitorallservices true, excluded namespace", - config: simpleConfig(true, []string{"namespace-1"}, nil, false, none), + config: simpleConfig(true, false, none, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -230,7 +230,7 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, same selector, monitorallservices true, excluded service", - config: simpleConfig(true, nil, []string{"namespace-1/svc"}, false, none), + config: simpleConfig(true, false, none, none), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -239,12 +239,12 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "different namespace, different selector, monitorallservices false, custom selected workload", - config: simpleConfig(true, nil, nil, false, AnnotationConfig{Java: AnnotationResources{ + config: simpleConfig(true, false, AnnotationConfig{Java: AnnotationResources{ Namespaces: nil, Deployments: []string{"namespace-1/workload"}, DaemonSets: []string{"namespace-1/workload"}, StatefulSets: []string{"namespace-1/workload"}, - }}), + }}, none), deploymentNs: "namespace-1", serviceNs: "namespace-2", deploymentSelector: map[string]string{"app": "different-1"}, @@ -253,12 +253,12 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "different namespace, different selector, monitorallservices false, custom selected namespace of workload", - config: simpleConfig(true, nil, nil, false, AnnotationConfig{Java: AnnotationResources{ + config: simpleConfig(true, false, AnnotationConfig{Java: AnnotationResources{ Namespaces: []string{"namespace-1"}, Deployments: nil, DaemonSets: nil, StatefulSets: nil, - }}), + }}, none), deploymentNs: "namespace-1", serviceNs: "namespace-2", deploymentSelector: map[string]string{"app": "different-1"}, @@ -267,12 +267,12 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "different namespace, different selector, monitorallservices false, custom selected namespace of service, not workload", - config: simpleConfig(true, nil, nil, false, AnnotationConfig{Java: AnnotationResources{ + config: simpleConfig(true, false, AnnotationConfig{Java: AnnotationResources{ Namespaces: []string{"namespace-2"}, Deployments: nil, DaemonSets: nil, StatefulSets: nil, - }}), + }}, none), deploymentNs: "namespace-1", serviceNs: "namespace-2", deploymentSelector: map[string]string{"app": "different-1"}, @@ -366,7 +366,7 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(workload) c := fake2.NewFakeClient(workload) - var config MonitorConfig = simpleConfig(true, nil, nil, true, none) + var config MonitorConfig = simpleConfig(true, true, none, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -385,7 +385,7 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(service, workload) c := fake2.NewFakeClient(service, workload) - var config MonitorConfig = simpleConfig(true, nil, nil, true, none) + var config MonitorConfig = simpleConfig(true, true, none, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -408,7 +408,7 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(workload) c := fake2.NewFakeClient(workload) - var config MonitorConfig = simpleConfig(true, nil, nil, false, none) + var config MonitorConfig = simpleConfig(true, false, none, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -428,7 +428,7 @@ func Test_OptOutByRemovingService(t *testing.T) { clientset := fake.NewSimpleClientset(service, workload) c := fake2.NewFakeClient(service, workload) - var config MonitorConfig = simpleConfig(true, nil, nil, false, none) + var config MonitorConfig = simpleConfig(true, false, none, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -459,7 +459,7 @@ func Test_OptOutByDisablingMonitorAllServices(t *testing.T) { clientset := fake.NewSimpleClientset(service, workload) c := fake2.NewFakeClient(service, workload) - var config MonitorConfig = simpleConfig(false, nil, nil, true, none) + var config MonitorConfig = simpleConfig(false, true, none, none) var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) @@ -558,7 +558,7 @@ func Test_mutate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { obj := workload.create("workload", "default", nil, tt.podAnnotations).DeepCopyObject().(client.Object) // TODO test different isWorkloadAutoMonitored values - gotMutated := mutate(obj, tt.languagesToMonitor, tt.shouldInsert) + gotMutated := mutate(obj, tt.languagesToMonitor) assert.Equal(t, tt.wantObjAnnotations, getPodTemplate(obj).GetAnnotations()) assert.Equal(t, tt.wantMutated, gotMutated) }) @@ -706,18 +706,12 @@ func mergeMaps(maps ...map[string]string) map[string]string { return result } -func simpleConfig(monitorAll bool, excludedNs, excludedSvcs []string, restartPods bool, customSelector AnnotationConfig) MonitorConfig { +func simpleConfig(monitorAll bool, restartPods bool, customSelector AnnotationConfig, excluded AnnotationConfig) MonitorConfig { return MonitorConfig{ MonitorAllServices: monitorAll, Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), RestartPods: restartPods, - Exclude: struct { - Namespaces []string `json:"namespaces"` - Services []string `json:"services"` - }{ - Namespaces: excludedNs, - Services: excludedSvcs, - }, - CustomSelector: customSelector, + Exclude: excluded, + CustomSelector: customSelector, } } From 319e18a479942faadc655c684c46ae5815827132 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 9 Apr 2025 10:55:17 -0400 Subject: [PATCH 21/63] Update tests to expect `namespace` in exclude and `customSelector` to update pod templates in workloads. --- pkg/instrumentation/auto/monitor.go | 2 +- pkg/instrumentation/auto/monitor_test.go | 32 ++++++++---------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index a659a4a89..67bf7a009 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -71,7 +71,7 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute) serviceInformer := factory.Core().V1().Services().Informer() - var mutator *AnnotationMutators + mutator := NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.SupportedTypes()) m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutator, clientReader: r, clientWriter: w} _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index 447900279..9fec878ca 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -220,8 +220,10 @@ func TestMonitor_MutateObject(t *testing.T) { expectedWorkloadAnnotations: map[string]string{}, }, { - name: "same namespace, same selector, monitorallservices true, excluded namespace", - config: simpleConfig(true, false, none, none), + name: "same namespace, same selector, monitorallservices true, excluded namespace", + config: simpleConfig(true, false, none, AnnotationConfig{Java: AnnotationResources{ + Namespaces: []string{"namespace-1"}, + }}), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -229,8 +231,12 @@ func TestMonitor_MutateObject(t *testing.T) { expectedWorkloadAnnotations: map[string]string{}, }, { - name: "same namespace, same selector, monitorallservices true, excluded service", - config: simpleConfig(true, false, none, none), + name: "same namespace, same selector, monitorallservices true, excluded service", + config: simpleConfig(true, false, none, AnnotationConfig{Java: AnnotationResources{ + Deployments: []string{"namespace-1/workload"}, + DaemonSets: []string{"namespace-1/workload"}, + StatefulSets: []string{"namespace-1/workload"}, + }}), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -263,7 +269,7 @@ func TestMonitor_MutateObject(t *testing.T) { serviceNs: "namespace-2", deploymentSelector: map[string]string{"app": "different-1"}, serviceSelector: map[string]string{"app": "different-2"}, - expectedWorkloadAnnotations: map[string]string{}, // empty because even though it should be custom selected, it is modified on the pod level for namespaces, so the pod template is not updated + expectedWorkloadAnnotations: annotated, }, { name: "different namespace, different selector, monitorallservices false, custom selected namespace of service, not workload", @@ -481,7 +487,6 @@ func Test_mutate(t *testing.T) { languagesToMonitor instrumentation.TypeSet wantObjAnnotations map[string]string wantMutated map[string]string - shouldInsert bool }{ { name: "java only", @@ -489,7 +494,6 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, wantObjAnnotations: buildAnnotations("java"), wantMutated: buildAnnotations("java"), - shouldInsert: true, }, { name: "java and python", @@ -500,7 +504,6 @@ func Test_mutate(t *testing.T) { }, wantObjAnnotations: mergeMaps(buildAnnotations("java"), buildAnnotations("python")), wantMutated: mergeMaps(buildAnnotations("java"), buildAnnotations("python")), - shouldInsert: true, }, { name: "remove python instrumentation", @@ -508,7 +511,6 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{}, wantMutated: buildAnnotations("python"), - shouldInsert: true, }, { name: "remove one of two languages", @@ -516,7 +518,6 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, wantObjAnnotations: buildAnnotations("java"), wantMutated: buildAnnotations("python"), - shouldInsert: true, }, { name: "manually specified annotation is not touched", @@ -524,7 +525,6 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{instrumentation.InjectAnnotationKey(instrumentation.TypeJava): defaultAnnotationValue}, wantMutated: map[string]string{}, - shouldInsert: true, }, { name: "remove all", @@ -532,7 +532,6 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{}, wantMutated: buildAnnotations("java"), - shouldInsert: true, }, { name: "remove only language annotations", @@ -540,15 +539,6 @@ func Test_mutate(t *testing.T) { languagesToMonitor: instrumentation.TypeSet{}, wantObjAnnotations: map[string]string{"test": "test"}, wantMutated: buildAnnotations("java"), - shouldInsert: true, - }, - { - name: "respects isWorkloadAutoMonitored", - podAnnotations: mergeAnnotations(buildAnnotations("python"), map[string]string{"test": "test"}), - languagesToMonitor: instrumentation.TypeSet{"python": struct{}{}, "java": struct{}{}}, - wantObjAnnotations: map[string]string{"test": "test"}, - wantMutated: buildAnnotations("python"), - shouldInsert: false, }, } From 42f6e23614f28e0b7dd5da19a8d666825fb2d37f Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 9 Apr 2025 15:49:29 -0400 Subject: [PATCH 22/63] add transformer, warning function --- pkg/instrumentation/auto/annotation.go | 22 ++++++++++++++++++++++ pkg/instrumentation/auto/monitor.go | 20 +++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 0e96bd601..9c8dfabe1 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -137,6 +137,7 @@ func NewAnnotationMutators( cfg AnnotationConfig, typeSet instrumentation.TypeSet, ) *AnnotationMutators { + warnNonNamespacedNames(typeSet, cfg, logger) builder := newMutatorBuilder(typeSet) return &AnnotationMutators{ clientWriter: clientWriter, @@ -152,6 +153,27 @@ func NewAnnotationMutators( } } +func warnNonNamespacedNames(typeSet instrumentation.TypeSet, cfg AnnotationConfig, logger logr.Logger) { + for t := range typeSet { + resources := cfg.getResources(t) + for _, deployment := range resources.Deployments { + if !strings.Contains(deployment, "/") { + logger.Info("invalid deployment name, needs to be namespaced", "deployment", deployment) + } + } + for _, daemonSet := range resources.DaemonSets { + if !strings.Contains(daemonSet, "/") { + logger.Info("invalid daemonSet name, needs to be namespaced", "daemonSet", daemonSet) + } + } + for _, statefulSet := range resources.StatefulSets { + if !strings.Contains(statefulSet, "/") { + logger.Info("invalid statefulSet name, needs to be namespaced", "statefulSet", statefulSet) + } + } + } +} + func getResources( cfg AnnotationConfig, typeSet instrumentation.TypeSet, diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 67bf7a009..139ef487d 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -2,6 +2,7 @@ package auto import ( "context" + "errors" "fmt" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/go-logr/logr" @@ -66,9 +67,22 @@ func (n NoopMonitor) MutateObject(_ client.Object, _ client.Object) map[string]s // NewMonitor is used to create an InstrumentationMutator that supports AutoMonitor. func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { logger.Info("AutoMonitor starting...") - // todo, throw warning if exclude config service is not namespaced (doesn't contain `/`) - // todo: informers.WithTransform() as option to only store what parts of service are needed - factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute) + factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute, informers.WithTransform(func(obj interface{}) (interface{}, error) { + svc, ok := obj.(*corev1.Service) + if !ok { + return obj, errors.New(fmt.Sprintf("error transforming service: %s not a service", obj)) + } + // Return only the fields we need + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svc.Name, + Namespace: svc.Namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: svc.Spec.Selector, + }, + }, nil + })) serviceInformer := factory.Core().V1().Services().Informer() mutator := NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.SupportedTypes()) From d35ed593586716b5c0339541ea09d4c74e933cd2 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 10 Apr 2025 11:25:12 -0400 Subject: [PATCH 23/63] fix integration tests to work with new exclude syntax --- .../validate_automonitor_deployment_test.go | 19 +++++++++++-------- .../validate_automonitor_methods.go | 5 +---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index 9f19351a8..9f69a419a 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -155,7 +155,7 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { assert.NoError(t, err) // Update operator with custom selector - namespacedDeployment := namespace + "sample-deployment" + namespacedDeployment := namespace + "/sample-deployment" customSelectorConfig := auto.AnnotationConfig{ Java: auto.AnnotationResources{ Deployments: []string{namespacedDeployment}, @@ -196,15 +196,18 @@ func TestDeploymentWithExcludedThenIncludedService(t *testing.T) { namespace := helper.Initialize("test-namespace", []string{}) - // Set up config with service exclusion + // Set up config with exclusion + resources := auto.AnnotationResources{ + Deployments: []string{namespace + "/sample-deployment"}, + } monitorConfig := auto.MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.SupportedTypes(), - Exclude: struct { - Namespaces []string `json:"namespaces"` - Services []string `json:"services"` - }{ - Services: []string{namespace + "/sample-deployment-service"}, // assuming this is the service name in sampleDeploymentServiceYaml + Exclude: auto.AnnotationConfig{ + Java: resources, + Python: resources, + DotNet: resources, + NodeJS: resources, }, } @@ -227,7 +230,7 @@ func TestDeploymentWithExcludedThenIncludedService(t *testing.T) { assert.NoError(t, err) // Update config to remove exclusion - monitorConfig.Exclude.Services = []string{} + monitorConfig.Exclude = auto.AnnotationConfig{} helper.UpdateMonitorConfig(monitorConfig) err = helper.RestartDeployment(namespace, "sample-deployment") if err != nil { diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go index ea5043051..786c061fb 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -314,10 +314,7 @@ func (h *TestHelper) PodsAnnotationsValid(namespace string, shouldExistAnnotatio } } - if validAnnotations { - return true - } - return false + return validAnnotations } func (h *TestHelper) restartOperator() { From fbafe30064d3701c551366e752c55aee7410229e Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 11 Apr 2025 13:16:27 -0400 Subject: [PATCH 24/63] namespace-level exclude should update pod template, but namespace-level custom selector should not update pod template --- .../validate_automonitor_deployment_test.go | 12 ++++++------ pkg/instrumentation/auto/config.go | 10 ++++++---- pkg/instrumentation/auto/monitor.go | 4 ++-- pkg/instrumentation/auto/monitor_test.go | 7 +++---- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index 9f69a419a..1777f8f98 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -149,9 +149,10 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { assert.NoError(t, err) // Validate no annotations present + all := []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + all) assert.NoError(t, err) // Update operator with custom selector @@ -178,16 +179,15 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { CustomSelector: customSelectorConfig, }) - // Validate annotations are not present - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + // Validate annotations are not present (custom selector obeys restart pods) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, all) assert.NoError(t, err) err = helper.RestartDeployment(namespace, "sample-deployment") assert.NoError(t, err) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + // validate annotations are present + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", all, none) assert.NoError(t, err) } diff --git a/pkg/instrumentation/auto/config.go b/pkg/instrumentation/auto/config.go index 4b5e8d350..410b27d1d 100644 --- a/pkg/instrumentation/auto/config.go +++ b/pkg/instrumentation/auto/config.go @@ -36,15 +36,17 @@ func (c AnnotationConfig) getResources(instType instrumentation.Type) Annotation } // LanguagesOf get languages to annotate for an object -func (c AnnotationConfig) LanguagesOf(obj client.Object) instrumentation.TypeSet { +func (c AnnotationConfig) LanguagesOf(obj client.Object, checkNamespace bool) instrumentation.TypeSet { objName := namespacedName(obj) typesSelected := instrumentation.TypeSet{} types := instrumentation.SupportedTypes() - for t := range types { - if slices.Contains(c.getResources(t).Namespaces, obj.GetNamespace()) { - typesSelected[t] = nil + if checkNamespace { + for t := range types { + if slices.Contains(c.getResources(t).Namespaces, obj.GetNamespace()) { + typesSelected[t] = nil + } } } diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 139ef487d..96a286bd4 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -230,13 +230,13 @@ func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { if !safeToMutate(oldObj, obj, m.config.RestartPods) { return map[string]string{} } - languagesToAnnotate := m.customSelectors.cfg.LanguagesOf(obj) + languagesToAnnotate := m.customSelectors.cfg.LanguagesOf(obj, false) if m.isWorkloadAutoMonitored(obj) { for l := range m.config.Languages { languagesToAnnotate[l] = nil } } - for l := range m.config.Exclude.LanguagesOf(obj) { + for l := range m.config.Exclude.LanguagesOf(obj, true) { delete(languagesToAnnotate, l) } diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index 9fec878ca..0fb9d964e 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -221,9 +221,8 @@ func TestMonitor_MutateObject(t *testing.T) { }, { name: "same namespace, same selector, monitorallservices true, excluded namespace", - config: simpleConfig(true, false, none, AnnotationConfig{Java: AnnotationResources{ - Namespaces: []string{"namespace-1"}, - }}), + config: simpleConfig(true, false, none, + AnnotationConfig{Java: AnnotationResources{Namespaces: []string{"namespace-1"}}}), deploymentNs: "namespace-1", serviceNs: "namespace-1", deploymentSelector: map[string]string{"app": "different-1"}, @@ -269,7 +268,7 @@ func TestMonitor_MutateObject(t *testing.T) { serviceNs: "namespace-2", deploymentSelector: map[string]string{"app": "different-1"}, serviceSelector: map[string]string{"app": "different-2"}, - expectedWorkloadAnnotations: annotated, + expectedWorkloadAnnotations: map[string]string{}, // empty because even though it should be custom selected, it is modified on the pod level for namespaces, so the pod template is not updated }, { name: "different namespace, different selector, monitorallservices false, custom selected namespace of service, not workload", From d65c9f00e41bdc944b106dfd34a549f1e62199fc Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 23 Apr 2025 11:09:44 -0400 Subject: [PATCH 25/63] Use correct autoAnnotationConfigStr to create Instrumentation Annotator --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 201de8ff9..35dfeb7c2 100644 --- a/main.go +++ b/main.go @@ -290,7 +290,7 @@ func main() { decoder := admission.NewDecoder(mgr.GetScheme()) - monitor := createInstrumentationAnnotator(autoMonitorConfigStr, autoInstrumentationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) + monitor := createInstrumentationAnnotator(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) if monitor != nil { mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ From eecdc1478287abf6e354c35b27e20a16b5ea810d Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 23 Apr 2025 16:28:58 -0400 Subject: [PATCH 26/63] Restart namespace for AutoMonitor based on `restartPods`, always restart namespace for autoAnnotateAutoInstrumentation. --- .../validate_automonitor_deployment_test.go | 26 ++++++------------- main.go | 2 +- pkg/instrumentation/auto/annotation.go | 4 +++ pkg/instrumentation/auto/annotation_test.go | 14 +++++----- pkg/instrumentation/auto/callback.go | 12 ++++++--- pkg/instrumentation/auto/monitor.go | 14 ++++++++++ pkg/instrumentation/auto/monitor_test.go | 12 ++++----- 7 files changed, 48 insertions(+), 36 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index 1777f8f98..258c8ac09 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -28,10 +28,7 @@ func TestServiceThenDeployment(t *testing.T) { namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - // Update operator - helper.UpdateAnnotationConfig(defaultAnnotationConfig) - - helper.UpdateMonitorConfig(auto.MonitorConfig{ + helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.SupportedTypes(), RestartPods: false, @@ -49,10 +46,7 @@ func TestDeploymentThenServiceRestartPodsDisabled(t *testing.T) { namespace := helper.Initialize("test-namespace", []string{}) - // Update operator - helper.UpdateAnnotationConfig(defaultAnnotationConfig) - - helper.UpdateMonitorConfig(auto.MonitorConfig{ + helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.SupportedTypes(), RestartPods: false, @@ -75,10 +69,7 @@ func TestDeploymentThenServiceRestartPodsEnabled(t *testing.T) { namespace := helper.Initialize("test-namespace", []string{}) - // Update operator - helper.UpdateAnnotationConfig(defaultAnnotationConfig) - - helper.UpdateMonitorConfig(auto.MonitorConfig{ + helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.SupportedTypes(), RestartPods: true, @@ -114,7 +105,7 @@ func TestDeploymentWithCustomSelector(t *testing.T) { } // Update operator with auto monitor disabled and custom selector - helper.UpdateMonitorConfig(auto.MonitorConfig{ + helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: false, Languages: instrumentation.SupportedTypes(), RestartPods: false, @@ -138,7 +129,7 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { namespace := helper.Initialize("test-namespace", []string{}) // Update operator with auto monitor disabled - helper.UpdateMonitorConfig(auto.MonitorConfig{ + helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: false, Languages: instrumentation.SupportedTypes(), RestartPods: false, @@ -172,7 +163,7 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { }, } - helper.UpdateMonitorConfig(auto.MonitorConfig{ + helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: false, Languages: instrumentation.SupportedTypes(), RestartPods: false, @@ -212,8 +203,7 @@ func TestDeploymentWithExcludedThenIncludedService(t *testing.T) { } // Update operator config - helper.UpdateAnnotationConfig(defaultAnnotationConfig) - helper.UpdateMonitorConfig(monitorConfig) + helper.UpdateMonitorConfig(&monitorConfig) // Create service first err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) @@ -231,7 +221,7 @@ func TestDeploymentWithExcludedThenIncludedService(t *testing.T) { // Update config to remove exclusion monitorConfig.Exclude = auto.AnnotationConfig{} - helper.UpdateMonitorConfig(monitorConfig) + helper.UpdateMonitorConfig(&monitorConfig) err = helper.RestartDeployment(namespace, "sample-deployment") if err != nil { return diff --git a/main.go b/main.go index 35dfeb7c2..3b2490ede 100644 --- a/main.go +++ b/main.go @@ -306,7 +306,7 @@ func main() { mgr.GetWebhookServer().StartedChecker(), func(ctx context.Context) { setupLog.Info("Applying auto-annotation") - auto.MutateAndPatchAll(monitor, ctx) + monitor.MutateAndPatchAll(ctx) }, ) } else { diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 9c8dfabe1..3e1a17594 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -41,6 +41,10 @@ type AnnotationMutators struct { cfg AnnotationConfig } +func (m *AnnotationMutators) MutateAndPatchAll(ctx context.Context) { + MutateAndPatchAll(m, ctx, true) +} + func (m *AnnotationMutators) GetAnnotationMutators() *AnnotationMutators { return m } diff --git a/pkg/instrumentation/auto/annotation_test.go b/pkg/instrumentation/auto/annotation_test.go index e8f6c82f4..2850f7fae 100644 --- a/pkg/instrumentation/auto/annotation_test.go +++ b/pkg/instrumentation/auto/annotation_test.go @@ -117,7 +117,7 @@ func TestAnnotationMutators_Namespaces(t *testing.T) { testCase.cfg, testCase.typeSet, ) - MutateAndPatchAll(mutators, ctx) + MutateAndPatchAll(mutators, ctx, false) gotNamespaces := &corev1.NamespaceList{} require.NoError(t, fakeClient.List(ctx, gotNamespaces)) for _, gotNamespace := range gotNamespaces.Items { @@ -201,7 +201,7 @@ func TestAnnotationMutators_Namespaces_Restart(t *testing.T) { cfg, instrumentation.NewTypeSet(instrumentation.TypeJava), ) - MutateAndPatchAll(mutators, context.Background()) + mutators.MutateAndPatchAll(context.Background()) ctx := context.Background() for _, namespacedResource := range namespacedRestartExpectedResources { assert.NoError(t, fakeClient.Get(ctx, client.ObjectKeyFromObject(namespacedResource), namespacedResource)) @@ -292,7 +292,7 @@ func TestAnnotationMutators_Deployments(t *testing.T) { testCase.cfg, testCase.typeSet, ) - MutateAndPatchAll(mutators, ctx) + MutateAndPatchAll(mutators, ctx, false) gotDeployments := &appsv1.DeploymentList{} require.NoError(t, fakeClient.List(ctx, gotDeployments)) for _, gotDeployment := range gotDeployments.Items { @@ -360,7 +360,7 @@ func TestAnnotationMutators_DaemonSets(t *testing.T) { testCase.cfg, testCase.typeSet, ) - MutateAndPatchAll(mutators, ctx) + MutateAndPatchAll(mutators, ctx, false) gotDaemonSets := &appsv1.DaemonSetList{} require.NoError(t, fakeClient.List(ctx, gotDaemonSets)) for _, gotDaemonSet := range gotDaemonSets.Items { @@ -428,7 +428,7 @@ func TestAnnotationMutators_StatefulSets(t *testing.T) { testCase.cfg, testCase.typeSet, ) - MutateAndPatchAll(mutators, ctx) + MutateAndPatchAll(mutators, ctx, false) gotStatefulSets := &appsv1.StatefulSetList{} require.NoError(t, fakeClient.List(ctx, gotStatefulSets)) for _, gotStatefulSet := range gotStatefulSets.Items { @@ -494,11 +494,11 @@ func TestAnnotationMutators_ClientErrors(t *testing.T) { cfg, instrumentation.NewTypeSet(instrumentation.TypeJava), ) - MutateAndPatchAll(mutators, context.Background()) + MutateAndPatchAll(mutators, context.Background(), false) errClient.AssertCalled(t, "List", mock.Anything, mock.Anything, mock.Anything) mutators.clientWriter = errClient mutators.clientReader = fakeClient - MutateAndPatchAll(mutators, context.Background()) + MutateAndPatchAll(mutators, context.Background(), false) errClient.AssertCalled(t, "Patch", mock.Anything, mock.Anything, mock.Anything, mock.Anything) } diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index 143ab202b..add923dce 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -93,8 +93,11 @@ func patchFunc(m InstrumentationAnnotator, ctx context.Context, callback objectC } } -func restartNamespaceFunc(m InstrumentationAnnotator, ctx context.Context) objectCallbackFunc { +func restartNamespaceFunc(m InstrumentationAnnotator, ctx context.Context, shouldRestartNamespace bool) objectCallbackFunc { return func(obj client.Object, previousResult any) (any, bool) { + if !shouldRestartNamespace { + return nil, false + } mutatedAnnotations, ok := previousResult.(map[string]string) if !ok { return nil, false @@ -162,17 +165,18 @@ func RestartNamespace(m InstrumentationAnnotator, ctx context.Context, namespace } // MutateAndPatchAll runs the mutators for each of the supported resources and patches them. -func MutateAndPatchAll(m InstrumentationAnnotator, ctx context.Context) { +func MutateAndPatchAll(m InstrumentationAnnotator, ctx context.Context, restartNamespace bool) { f := getMutateObjectFunc(m) callbackFunc := patchFunc(m, ctx, f) rangeObjectList(m, ctx, &appsv1.DeploymentList{}, &client.ListOptions{}, callbackFunc) rangeObjectList(m, ctx, &appsv1.DaemonSetList{}, &client.ListOptions{}, callbackFunc) rangeObjectList(m, ctx, &appsv1.StatefulSetList{}, &client.ListOptions{}, callbackFunc) - rangeObjectList(m, ctx, &corev1.NamespaceList{}, &client.ListOptions{}, chainCallbacks(callbackFunc, restartNamespaceFunc(m, ctx))) + rangeObjectList(m, ctx, &corev1.NamespaceList{}, &client.ListOptions{}, chainCallbacks(callbackFunc, restartNamespaceFunc(m, ctx, restartNamespace))) } func getMutateObjectFunc(m InstrumentationAnnotator) objectCallbackFunc { return func(obj client.Object, _ any) (any, bool) { - return m.MutateObject(nil, obj), true + mutatedAnnotations := m.MutateObject(nil, obj) + return mutatedAnnotations, true } } diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 96a286bd4..86deac7fa 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -27,6 +27,7 @@ type InstrumentationAnnotator interface { GetLogger() logr.Logger GetReader() client.Reader GetWriter() client.Writer + MutateAndPatchAll(ctx context.Context) } type Monitor struct { @@ -40,6 +41,13 @@ type Monitor struct { logger logr.Logger } +func (m *Monitor) MutateAndPatchAll(ctx context.Context) { + if m.config.RestartPods { + MutateAndPatchAll(m, ctx, false) + } + // todo: what to do about updating namespace annotations? maybe update them here? or pass in variable to MutateAndPatchAll? +} + func (m *Monitor) GetAnnotationMutators() *AnnotationMutators { return m.customSelectors } @@ -66,6 +74,12 @@ func (n NoopMonitor) MutateObject(_ client.Object, _ client.Object) map[string]s // NewMonitor is used to create an InstrumentationMutator that supports AutoMonitor. func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { + // Config default values + if len(config.Languages) == 0 { + logger.Info("Setting languages to default") + config.Languages = instrumentation.SupportedTypes() + } + logger.Info("AutoMonitor starting...") factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute, informers.WithTransform(func(obj interface{}) (interface{}, error) { svc, ok := obj.(*corev1.Service) diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index 0fb9d964e..c48e8c747 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -375,7 +375,7 @@ func Test_OptOutByRemovingService(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO()) + MutateAndPatchAll(monitor, context.TODO(), false) updatedWorkload, err := wt.getWithClient(c, defaultNs, workload.GetName()) assert.NoError(t, err) assert.Equal(t, userAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) @@ -394,7 +394,7 @@ func Test_OptOutByRemovingService(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO()) + MutateAndPatchAll(monitor, context.TODO(), false) err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) assert.NoError(t, err) @@ -417,7 +417,7 @@ func Test_OptOutByRemovingService(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO()) + MutateAndPatchAll(monitor, context.TODO(), false) updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) assert.NoError(t, err) @@ -437,7 +437,7 @@ func Test_OptOutByRemovingService(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO()) + MutateAndPatchAll(monitor, context.TODO(), false) err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) assert.NoError(t, err) @@ -468,7 +468,7 @@ func Test_OptOutByDisablingMonitorAllServices(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO()) + MutateAndPatchAll(monitor, context.TODO(), false) updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) assert.NoError(t, err) @@ -576,7 +576,7 @@ func Test_StartupRestartPods(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) m := NewMonitor(context.TODO(), config, k8sInterface, fakeClient, fakeClient, logger) - MutateAndPatchAll(m, context.TODO()) + MutateAndPatchAll(m, context.TODO(), false) updatedMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments(defaultNs).Get(context.TODO(), matchingDeployment.Name, metav1.GetOptions{}) assert.NoError(t, err) assert.Equal(t, buildAnnotations(instrumentation.TypeJava), updatedMatchingDeployment.Spec.Template.GetAnnotations()) From 185ff811fee8eea4789ad375f998dd549c55956f Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 23 Apr 2025 16:29:09 -0400 Subject: [PATCH 27/63] omit languages, exclude, and customselector when empty --- pkg/instrumentation/auto/monitor_config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/instrumentation/auto/monitor_config.go b/pkg/instrumentation/auto/monitor_config.go index 64d6f9bd8..e151632c7 100644 --- a/pkg/instrumentation/auto/monitor_config.go +++ b/pkg/instrumentation/auto/monitor_config.go @@ -9,8 +9,8 @@ import "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" // auto-annotation for each instrumentation type. type MonitorConfig struct { MonitorAllServices bool `json:"monitorAllServices"` - Languages instrumentation.TypeSet `json:"languages"` + Languages instrumentation.TypeSet `json:"languages,omitempty"` RestartPods bool `json:"restartPods"` - Exclude AnnotationConfig `json:"exclude"` - CustomSelector AnnotationConfig `json:"customSelector"` + Exclude AnnotationConfig `json:"exclude,omitempty"` + CustomSelector AnnotationConfig `json:"customSelector,omitempty"` } From c28093a9772a8cbee9c9c52342a87b7a0661b706 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 23 Apr 2025 16:29:25 -0400 Subject: [PATCH 28/63] add more integ tests for AutoMonitor --- .../validate_automonitor_methods.go | 103 ++-- .../automonitor/validate_automonitor_test.go | 476 ++++++++++++++++++ 2 files changed, 536 insertions(+), 43 deletions(-) create mode 100644 integration-tests/manifests/automonitor/validate_automonitor_test.go diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go index 786c061fb..40c16843d 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -15,6 +15,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "testing" @@ -280,41 +281,32 @@ func forceRestart(deployment *appsV1.Deployment) { deployment.Spec.Template.SetAnnotations(annotations) } -// TODO: should check deployment template spec? func (h *TestHelper) PodsAnnotationsValid(namespace string, shouldExistAnnotations []string, shouldNotExistAnnotations []string) bool { currentPods, err := h.clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) if err != nil { h.logger.Info(fmt.Sprintf("Failed to list pods: %v\n", err)) return false } - - validAnnotations := true for _, pod := range currentPods.Items { h.logger.Info(fmt.Sprintf("Pod %s is in phase %s\n", pod.Name, pod.Status.Phase)) - h.logger.Info("Pod ", pod.GetAnnotations()) if pod.Status.Phase != v1.PodRunning { continue } for _, annotation := range shouldExistAnnotations { if value, exists := pod.Annotations[annotation]; !exists || value != "true" { - h.logger.Info("Pod", pod.Namespace, pod.Name, " does not have annotation ", annotation) - validAnnotations = false - break + h.logger.Info(fmt.Sprintf("%s/%s should have annotation %s", pod.Namespace, pod.Name, annotation)) + return false } } for _, annotation := range shouldNotExistAnnotations { if _, exists := pod.Annotations[annotation]; exists { - h.logger.Info("Pod", pod.Namespace, pod.Name, " shouldn't have annotation ", annotation) - validAnnotations = false - break + h.logger.Info(fmt.Sprintf("%s/%s should not have annotation %s", pod.Namespace, pod.Name, annotation)) + return false } } - if !validAnnotations { - break - } } - return validAnnotations + return true } func (h *TestHelper) restartOperator() { @@ -345,7 +337,7 @@ func (h *TestHelper) findIndexOfPrefix(str string, strs []string) int { return -1 } -func (h *TestHelper) UpdateMonitorConfig(config auto.MonitorConfig) { +func (h *TestHelper) UpdateMonitorConfig(config *auto.MonitorConfig) { jsonStr, err := json.Marshal(config) assert.Nil(h.t, err) @@ -354,12 +346,13 @@ func (h *TestHelper) UpdateMonitorConfig(config auto.MonitorConfig) { h.updateOperatorConfig(string(jsonStr), "--auto-monitor-config=") } -func (h *TestHelper) UpdateAnnotationConfig(config auto.AnnotationConfig) { - jsonStr, err := json.Marshal(config) - assert.Nil(h.t, err) - h.logger.Info("Setting annotation config to:") - util.PrettyPrint(config) - h.updateOperatorConfig(string(jsonStr), "--auto-annotation-config=") +func (h *TestHelper) UpdateAnnotationConfig(config *auto.AnnotationConfig) { + var jsonStr = "" + if marshalledConfig, err := json.Marshal(config); config != nil && assert.Nil(h.t, err) { + jsonStr = string(marshalledConfig) + } + h.logger.Info("Setting annotation config to ", "jsonStr", jsonStr) + h.updateOperatorConfig(jsonStr, "--auto-annotation-config=") } func (h *TestHelper) updateOperatorConfig(jsonStr string, flag string) { @@ -370,10 +363,17 @@ func (h *TestHelper) updateOperatorConfig(jsonStr string, flag string) { } args := deployment.Spec.Template.Spec.Containers[0].Args indexOfAutoAnnotationConfigString := h.findIndexOfPrefix(flag, args) + shouldDelete := len(jsonStr) == 0 if indexOfAutoAnnotationConfigString < 0 { - deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, flag+jsonStr) + if !shouldDelete { + deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, flag+jsonStr) + } } else { - deployment.Spec.Template.Spec.Containers[0].Args[indexOfAutoAnnotationConfigString] = flag + jsonStr + if shouldDelete { + deployment.Spec.Template.Spec.Containers[0].Args = slices.Delete(deployment.Spec.Template.Spec.Containers[0].Args, indexOfAutoAnnotationConfigString, indexOfAutoAnnotationConfigString+1) + } else { + deployment.Spec.Template.Spec.Containers[0].Args[indexOfAutoAnnotationConfigString] = flag + jsonStr + } } if !h.UpdateOperator(deployment) { @@ -391,33 +391,49 @@ func (h *TestHelper) ValidateWorkloadAnnotations(resourceType, uniqueNamespace, } func (h *TestHelper) ValidateWorkloadAnnotationsA(resourceType, uniqueNamespace, resourceName string, shouldExist []string, shouldNotExist []string) error { - var resource interface{} - var err error - + var annotations map[string]string switch resourceType { case "deployment": - resource, err = h.clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) + resource, err := h.clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) + if err != nil { + return err + } + annotations = resource.Spec.Template.Annotations case "daemonset": - resource, err = h.clientSet.AppsV1().DaemonSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) + resource, err := h.clientSet.AppsV1().DaemonSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) + if err != nil { + return err + } + annotations = resource.Spec.Template.Annotations case "statefulset": - resource, err = h.clientSet.AppsV1().StatefulSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) + resource, err := h.clientSet.AppsV1().StatefulSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) + if err != nil { + return err + } + annotations = resource.Spec.Template.Annotations default: return fmt.Errorf("unsupported resource type: %s", resourceType) } - - if err != nil { - return fmt.Errorf("failed to get %s: %s", resourceType, err.Error()) - } - - if err := util.WaitForNewPodCreation(h.clientSet, resource, h.startTime); err != nil { - return fmt.Errorf("error waiting for pod creation: %s", err.Error()) + for _, shouldExistAnnotation := range shouldExist { + if _, ok := annotations[shouldExistAnnotation]; !ok { + return fmt.Errorf("annotation should be present: %s", shouldExistAnnotation) + } } - - if h.PodsAnnotationsValid(uniqueNamespace, shouldExist, shouldNotExist) { - return nil - } else { - return fmt.Errorf("A pod has invalid annotations") + for _, shouldNotExistAnnotation := range shouldNotExist { + if _, ok := annotations[shouldNotExistAnnotation]; ok { + return fmt.Errorf("annotation should not be present: %s", shouldNotExistAnnotation) + } } + return nil + //if err := util.WaitForNewPodCreation(h.clientSet, resource, h.startTime); err != nil { + // return fmt.Errorf("error waiting for pod creation: %s", err.Error()) + //} + // + //if h.PodsAnnotationsValid(uniqueNamespace, shouldExist, shouldNotExist) { + // return nil + //} else { + // return fmt.Errorf("A pod has invalid annotations") + //} } func (h *TestHelper) CreateResource(uniqueNamespace string, sampleAppYamlPath string, skipDelete bool) error { @@ -439,8 +455,8 @@ func (h *TestHelper) Initialize(namespace string, apps []string) string { newUUID := uuid.New() uniqueNamespace := fmt.Sprintf("%s-%s", namespace, newUUID.String()) - h.UpdateMonitorConfig(auto.MonitorConfig{MonitorAllServices: false}) - h.UpdateAnnotationConfig(auto.AnnotationConfig{}) + h.UpdateMonitorConfig(&auto.MonitorConfig{MonitorAllServices: false}) + h.UpdateAnnotationConfig(nil) h.startTime = time.Now() if err := h.CreateNamespaceAndApplyResources(uniqueNamespace, apps); err != nil { h.t.Fatalf("Failed to create/apply resources on namespace: %v", err) @@ -485,6 +501,7 @@ func (h *TestHelper) RolloutWaitYamlWithKubectl(filename string, namespace strin } func (h *TestHelper) RestartDeployment(namespace string, deploymentName string) error { + h.logger.Info(fmt.Sprintf("Restarting %s/%s...", namespace, deploymentName)) retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { // Get the latest version of the deployment deployment, err := h.clientSet.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_test.go b/integration-tests/manifests/automonitor/validate_automonitor_test.go new file mode 100644 index 000000000..aaa906405 --- /dev/null +++ b/integration-tests/manifests/automonitor/validate_automonitor_test.go @@ -0,0 +1,476 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package annotations + +import ( + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +const ( + sampleDeploymentWithoutServiceYaml = "../sample-deployment-without-service.yaml" + customerServiceYaml = "../customer-service.yaml" + frontendAppYaml = "../frontend-app.yaml" + adminDashboardYaml = "../admin-dashboard.yaml" + conflictingDeploymentYaml = "../conflicting-deployment.yaml" +) + +var ( + allLanguages = []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} + javaPythonOnly = []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation} + dotnetNodejsOnly = []string{autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} +) + +// Permutation 1 [HIGH]: Enable monitoring for all services without auto restarts +func TestPermutation1_MonitorAllServicesNoAutoRestarts(t *testing.T) { + helper := NewTestHelper(t, true) + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentYaml}) + + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + RestartPods: false, + }) + + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) + assert.NoError(t, err) + + // Verify no annotations without restart + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allLanguages) + assert.NoError(t, err) + + // Manually restart and verify annotations + err = helper.RestartDeployment(namespace, "sample-deployment") + assert.NoError(t, err) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) +} + +// Permutation 2 [HIGH]: Disable automatic monitoring for all services +func TestPermutation2_DisableMonitoringNoAutoRestarts(t *testing.T) { + helper := NewTestHelper(t, true) + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) + + // First enable monitoring with auto-restart + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + RestartPods: true, + }) + + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + + // Verify initial annotations + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) + + // Disable monitoring without auto-restart + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: false, + RestartPods: false, + }) + + // Verify annotations still present + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) + + // Manually restart and verify annotations removed + err = helper.RestartDeployment(namespace, "sample-deployment") + assert.NoError(t, err) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allLanguages) + assert.NoError(t, err) +} + +// Permutation 3 [HIGH]: Monitor all services with pod restarts enabled +func TestPermutation3_MonitorAllServicesWithAutoRestarts(t *testing.T) { + helper := NewTestHelper(t, true) + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) + + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + + // Verify no initial annotations + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allLanguages) + assert.NoError(t, err) + + // Enable monitoring with auto-restart + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + RestartPods: true, + }) + + // Verify annotations automatically added + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) +} + +// Permutation 4 [MED]: Disable monitoring but allow pod restarts +func TestPermutation4_DisableMonitoringWithAutoRestarts(t *testing.T) { + helper := NewTestHelper(t, true) + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) + + // Start with monitoring enabled + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + RestartPods: true, + }) + + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + + // Verify initial annotations + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) + + // Disable monitoring with auto-restart + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: false, + RestartPods: true, + }) + + // Verify annotations automatically removed + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allLanguages) + assert.NoError(t, err) +} + +// Permutation 5 [HIGH]: Monitor only Java and Python services without pod restarts +func TestPermutation5_MonitorSelectedLanguagesNoAutoRestarts(t *testing.T) { + helper := NewTestHelper(t, true) + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml, sampleDeploymentYaml}) + // Start with all languages enabled + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + RestartPods: true, + }) + + time.Sleep(time.Second * 5) + + // Verify all annotations present + err := helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) + + // Update to Java and Python only without auto-restart + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), RestartPods: false, + }) + + // Verify annotations unchanged without restart + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) + + // Manually restart and verify only Java/Python remain + err = helper.RestartDeployment(namespace, "sample-deployment") + assert.NoError(t, err) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", javaPythonOnly, dotnetNodejsOnly) + assert.NoError(t, err) +} + +// Permutation 6 [MED]: Monitor Java and Python with pod restarts enabled +func TestPermutation6_MonitorSelectedLanguagesWithAutoRestarts(t *testing.T) { + helper := NewTestHelper(t, true) + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) + + // Start with all languages enabled + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + RestartPods: true, + }) + + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + + // Verify all annotations present + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) + + // Update to Java and Python only with auto-restart + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), RestartPods: true, + }) + + // Verify only Java/Python annotations remain + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", javaPythonOnly, dotnetNodejsOnly) + assert.NoError(t, err) +} + +// Permutation 9 [HIGH]: Monitor all services but exclude specific Java workloads +func TestPermutation9_MonitorWithExclusionsNoAutoRestarts(t *testing.T) { + helper := NewTestHelper(t, true) + + // Create two namespaces + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) + kubeSystemNS := helper.Initialize("kube-system", []string{sampleDeploymentServiceYaml}) + + // Set up exclusions + excludeConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{"kube-system"}, + Deployments: []string{namespace + "/customer-service"}, + }, + } + + // Update config with exclusions + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + RestartPods: false, + Exclude: excludeConfig, + }) + + // Create deployments in both namespaces + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml, customerServiceYaml}) + assert.NoError(t, err) + err = helper.CreateNamespaceAndApplyResources(kubeSystemNS, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + + // Manually restart deployments + err = helper.RestartDeployment(namespace, "sample-deployment") + assert.NoError(t, err) + err = helper.RestartDeployment(namespace, "customer-service") + assert.NoError(t, err) + err = helper.RestartDeployment(kubeSystemNS, "sample-deployment") + assert.NoError(t, err) + + // Verify regular deployment has all annotations + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) + + // Verify excluded customer-service has no Java annotations + nonJavaAnnotations := []string{autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", nonJavaAnnotations, []string{autoAnnotateJavaAnnotation}) + assert.NoError(t, err) + + // Verify kube-system deployment has no Java annotations + err = helper.ValidateWorkloadAnnotations("deployment", kubeSystemNS, "sample-deployment", nonJavaAnnotations, []string{autoAnnotateJavaAnnotation}) + assert.NoError(t, err) +} + +// Permutation 10 [HIGH]: Monitor all services with auto-restarts but exclude specific Java workloads +func TestPermutation10_MonitorWithExclusionsWithAutoRestarts(t *testing.T) { + helper := NewTestHelper(t, true) + + // Create two namespaces + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) + kubeSystemNS := helper.Initialize("kube-system", []string{sampleDeploymentServiceYaml}) + + // Create deployments before enabling monitoring + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml, customerServiceYaml}) + assert.NoError(t, err) + err = helper.CreateNamespaceAndApplyResources(kubeSystemNS, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + + // Set up exclusions and enable monitoring with auto-restart + excludeConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{"kube-system"}, + Deployments: []string{namespace + "/customer-service"}, + }, + } + + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + RestartPods: true, + Exclude: excludeConfig, + }) + + // Verify regular deployment has all annotations + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + assert.NoError(t, err) + + // Verify excluded customer-service has no Java annotations + nonJavaAnnotations := []string{autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", nonJavaAnnotations, []string{autoAnnotateJavaAnnotation}) + assert.NoError(t, err) + + // Verify kube-system deployment has no Java annotations + err = helper.ValidateWorkloadAnnotations("deployment", kubeSystemNS, "sample-deployment", nonJavaAnnotations, []string{autoAnnotateJavaAnnotation}) + assert.NoError(t, err) +} + +// Permutation 18 [HIGH]: Monitor all services with customSelector and specific languages +func TestPermutation18_MonitorWithCustomSelectorAndAutoRestarts(t *testing.T) { + helper := NewTestHelper(t, true) + + defaultNS := helper.Initialize("default", []string{sampleDeploymentServiceYaml}) + testNS := helper.Initialize("test", []string{}) + + // Create deployments + err := helper.CreateNamespaceAndApplyResources(defaultNS, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + err = helper.CreateNamespaceAndApplyResources(testNS, []string{sampleDeploymentWithoutServiceYaml}) + assert.NoError(t, err) + + // Set up custom selector config + customSelectorConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{"default"}, + }, + Python: auto.AnnotationResources{ + Deployments: []string{testNS + "/sample-deployment-without-service"}, + }, + } + + // Update config with custom selector + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeDotNet), RestartPods: true, + CustomSelector: customSelectorConfig, + }) + + // Verify default namespace deployment has dotnet annotation + err = helper.ValidateWorkloadAnnotations("deployment", defaultNS, "sample-deployment", + []string{autoAnnotateDotNetAnnotation}, + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}) + assert.NoError(t, err) + + // Verify test namespace deployment has python annotation + err = helper.ValidateWorkloadAnnotations("deployment", testNS, "sample-deployment-without-service", + []string{autoAnnotatePythonAnnotation}, + []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + assert.NoError(t, err) +} + +// Permutation 19 [HIGH++]: Complex scenario with exclusions and customSelector +func TestPermutation19_ComplexMonitoringWithExclusionsAndCustomSelector(t *testing.T) { + helper := NewTestHelper(t, true) + + defaultNS := helper.Initialize("default", []string{sampleDeploymentServiceYaml}) + testNS := helper.Initialize("test", []string{}) + kubeSystemNS := helper.Initialize("kube-system", []string{sampleDeploymentServiceYaml}) + + // Create deployments in all namespaces + err := helper.CreateNamespaceAndApplyResources(defaultNS, []string{sampleDeploymentYaml, customerServiceYaml}) + assert.NoError(t, err) + err = helper.CreateNamespaceAndApplyResources(testNS, []string{conflictingDeploymentYaml, sampleDeploymentYaml}) + assert.NoError(t, err) + err = helper.CreateNamespaceAndApplyResources(kubeSystemNS, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + + // Set up complex config + excludeConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{"kube-system", "test"}, + Deployments: []string{defaultNS + "/customer-service"}, + }, + } + + customSelectorConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{"default"}, + Deployments: []string{testNS + "/conflicting-deployment"}, + }, + Python: auto.AnnotationResources{ + Deployments: []string{testNS + "/conflicting-deployment"}, + }, + DotNet: auto.AnnotationResources{ + Namespaces: []string{"test"}, + }, + } + + // Update operator config + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeNodeJS), RestartPods: true, + Exclude: excludeConfig, + CustomSelector: customSelectorConfig, + }) + + // Verify test/conflicting-deployment + err = helper.ValidateWorkloadAnnotations("deployment", testNS, "conflicting-deployment", + []string{autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, + []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation}) + assert.NoError(t, err) + + // Verify default/customer-service + err = helper.ValidateWorkloadAnnotations("deployment", defaultNS, "customer-service", + []string{autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, + []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation}) + assert.NoError(t, err) + + // Verify default/sample-deployment + err = helper.ValidateWorkloadAnnotations("deployment", defaultNS, "sample-deployment", + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, + []string{autoAnnotateDotNetAnnotation}) + assert.NoError(t, err) + + // Verify kube-system/sample-deployment + err = helper.ValidateWorkloadAnnotations("deployment", kubeSystemNS, "sample-deployment", + []string{autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, + []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation}) + assert.NoError(t, err) +} + +// Permutation 20 [HIGH]: Disable general monitoring but enable specific instrumentation +func TestPermutation20_SelectiveMonitoringWithCustomSelector(t *testing.T) { + helper := NewTestHelper(t, true) + + webNS := helper.Initialize("web", []string{}) + analyticsNS := helper.Initialize("analytics", []string{}) + dataScienceNS := helper.Initialize("data-science", []string{}) + + // Create deployments + err := helper.CreateNamespaceAndApplyResources(webNS, []string{frontendAppYaml, adminDashboardYaml, sampleDeploymentYaml}) + assert.NoError(t, err) + err = helper.CreateNamespaceAndApplyResources(analyticsNS, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + err = helper.CreateNamespaceAndApplyResources(dataScienceNS, []string{sampleDeploymentYaml}) + assert.NoError(t, err) + + // Set up custom selector config + customSelectorConfig := auto.AnnotationConfig{ + Python: auto.AnnotationResources{ + Namespaces: []string{"analytics", "data-science"}, + }, + NodeJS: auto.AnnotationResources{ + Deployments: []string{webNS + "/frontend-app", webNS + "/admin-dashboard"}, + }, + } + + // Update operator config + helper.UpdateAnnotationConfig(nil) + helper.UpdateMonitorConfig(&auto.MonitorConfig{ + MonitorAllServices: false, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), RestartPods: true, + CustomSelector: customSelectorConfig, + }) + + // Verify frontend-app and admin-dashboard have NodeJS only + err = helper.ValidateWorkloadAnnotations("deployment", webNS, "frontend-app", + []string{autoAnnotateNodeJSAnnotation}, + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation}) + assert.NoError(t, err) + + err = helper.ValidateWorkloadAnnotations("deployment", webNS, "admin-dashboard", + []string{autoAnnotateNodeJSAnnotation}, + []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation}) + assert.NoError(t, err) + + // Verify sample-deployment in web namespace has no annotations + err = helper.ValidateWorkloadAnnotations("deployment", webNS, "sample-deployment", + none, + allLanguages) + assert.NoError(t, err) + + // Verify analytics/data-science deployments have no pod template annotations + err = helper.ValidateWorkloadAnnotations("deployment", analyticsNS, "sample-deployment", + none, + allLanguages) + assert.NoError(t, err) + + err = helper.ValidateWorkloadAnnotations("deployment", dataScienceNS, "sample-deployment", + none, + allLanguages) + assert.NoError(t, err) +} From 707856b92f6a2f5e70fde2b8b195f7c8455c37db Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 24 Apr 2025 10:18:11 -0400 Subject: [PATCH 29/63] fix permutation 9 (don't use kube-system) --- .../automonitor/validate_automonitor_test.go | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_test.go b/integration-tests/manifests/automonitor/validate_automonitor_test.go index aaa906405..6d9cff691 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_test.go @@ -206,16 +206,16 @@ func TestPermutation6_MonitorSelectedLanguagesWithAutoRestarts(t *testing.T) { func TestPermutation9_MonitorWithExclusionsNoAutoRestarts(t *testing.T) { helper := NewTestHelper(t, true) - // Create two namespaces namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - kubeSystemNS := helper.Initialize("kube-system", []string{sampleDeploymentServiceYaml}) // Set up exclusions excludeConfig := auto.AnnotationConfig{ Java: auto.AnnotationResources{ - Namespaces: []string{"kube-system"}, Deployments: []string{namespace + "/customer-service"}, }, + Python: auto.AnnotationResources{ + Namespaces: []string{namespace}, + }, } // Update config with exclusions @@ -226,31 +226,26 @@ func TestPermutation9_MonitorWithExclusionsNoAutoRestarts(t *testing.T) { Exclude: excludeConfig, }) - // Create deployments in both namespaces err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml, customerServiceYaml}) assert.NoError(t, err) - err = helper.CreateNamespaceAndApplyResources(kubeSystemNS, []string{sampleDeploymentYaml}) - assert.NoError(t, err) // Manually restart deployments err = helper.RestartDeployment(namespace, "sample-deployment") assert.NoError(t, err) err = helper.RestartDeployment(namespace, "customer-service") assert.NoError(t, err) - err = helper.RestartDeployment(kubeSystemNS, "sample-deployment") - assert.NoError(t, err) - // Verify regular deployment has all annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + nonPythonAnnotations := []string{autoAnnotateJavaAnnotation, injectJavaAnnotation, autoAnnotateDotNetAnnotation, injectDotNetAnnotation, autoAnnotateNodeJSAnnotation, injectNodeJSAnnotation} + // Verify regular deployment has all annotations except python + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", nonPythonAnnotations, []string{autoAnnotatePythonAnnotation}) assert.NoError(t, err) // Verify excluded customer-service has no Java annotations - nonJavaAnnotations := []string{autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", nonJavaAnnotations, []string{autoAnnotateJavaAnnotation}) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", []string{autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation}) assert.NoError(t, err) // Verify kube-system deployment has no Java annotations - err = helper.ValidateWorkloadAnnotations("deployment", kubeSystemNS, "sample-deployment", nonJavaAnnotations, []string{autoAnnotateJavaAnnotation}) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", nonPythonAnnotations, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}) assert.NoError(t, err) } From a702b6a80438844671c3f68d5c703387ccb21330 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 09:20:20 -0400 Subject: [PATCH 30/63] improve comments --- pkg/instrumentation/auto/monitor.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 86deac7fa..876b3c2e1 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -244,12 +244,14 @@ func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { if !safeToMutate(oldObj, obj, m.config.RestartPods) { return map[string]string{} } + languagesToAnnotate := m.customSelectors.cfg.LanguagesOf(obj, false) if m.isWorkloadAutoMonitored(obj) { for l := range m.config.Languages { languagesToAnnotate[l] = nil } } + for l := range m.config.Exclude.LanguagesOf(obj, true) { delete(languagesToAnnotate, l) } @@ -257,7 +259,7 @@ func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { return mutate(obj, languagesToAnnotate) } -// returns if workload is auto monitored +// returns if workload is auto monitored (does not include custom selector) func (m *Monitor) isWorkloadAutoMonitored(obj client.Object) bool { if isNamespace(obj) { return false From d9c3a00eab5c2cad7b44055cc9b9bde05d54ab29 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 10:33:37 -0400 Subject: [PATCH 31/63] Fix integ tests --- .../validate_automonitor_deployment_test.go | 12 +- .../automonitor/validate_automonitor_test.go | 171 ++++++++---------- 2 files changed, 82 insertions(+), 101 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index 258c8ac09..de9801dad 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -24,7 +24,7 @@ var ( ) func TestServiceThenDeployment(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) @@ -42,7 +42,7 @@ func TestServiceThenDeployment(t *testing.T) { // create deployment, create service, should not annotate anything func TestDeploymentThenServiceRestartPodsDisabled(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{}) @@ -65,7 +65,7 @@ func TestDeploymentThenServiceRestartPodsDisabled(t *testing.T) { } func TestDeploymentThenServiceRestartPodsEnabled(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{}) @@ -90,7 +90,7 @@ func TestDeploymentThenServiceRestartPodsEnabled(t *testing.T) { } func TestDeploymentWithCustomSelector(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{}) @@ -124,7 +124,7 @@ func TestDeploymentWithCustomSelector(t *testing.T) { } func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{}) @@ -183,7 +183,7 @@ func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { } func TestDeploymentWithExcludedThenIncludedService(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{}) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_test.go b/integration-tests/manifests/automonitor/validate_automonitor_test.go index 6d9cff691..e951f3afd 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_test.go @@ -26,7 +26,7 @@ var ( // Permutation 1 [HIGH]: Enable monitoring for all services without auto restarts func TestPermutation1_MonitorAllServicesNoAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentYaml}) helper.UpdateAnnotationConfig(nil) @@ -51,7 +51,7 @@ func TestPermutation1_MonitorAllServicesNoAutoRestarts(t *testing.T) { // Permutation 2 [HIGH]: Disable automatic monitoring for all services func TestPermutation2_DisableMonitoringNoAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) // First enable monitoring with auto-restart @@ -87,7 +87,7 @@ func TestPermutation2_DisableMonitoringNoAutoRestarts(t *testing.T) { // Permutation 3 [HIGH]: Monitor all services with pod restarts enabled func TestPermutation3_MonitorAllServicesWithAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) @@ -110,7 +110,7 @@ func TestPermutation3_MonitorAllServicesWithAutoRestarts(t *testing.T) { // Permutation 4 [MED]: Disable monitoring but allow pod restarts func TestPermutation4_DisableMonitoringWithAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) // Start with monitoring enabled @@ -140,7 +140,7 @@ func TestPermutation4_DisableMonitoringWithAutoRestarts(t *testing.T) { // Permutation 5 [HIGH]: Monitor only Java and Python services without pod restarts func TestPermutation5_MonitorSelectedLanguagesNoAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml, sampleDeploymentYaml}) // Start with all languages enabled helper.UpdateAnnotationConfig(nil) @@ -174,7 +174,7 @@ func TestPermutation5_MonitorSelectedLanguagesNoAutoRestarts(t *testing.T) { // Permutation 6 [MED]: Monitor Java and Python with pod restarts enabled func TestPermutation6_MonitorSelectedLanguagesWithAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) // Start with all languages enabled @@ -204,7 +204,7 @@ func TestPermutation6_MonitorSelectedLanguagesWithAutoRestarts(t *testing.T) { // Permutation 9 [HIGH]: Monitor all services but exclude specific Java workloads func TestPermutation9_MonitorWithExclusionsNoAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) @@ -251,22 +251,18 @@ func TestPermutation9_MonitorWithExclusionsNoAutoRestarts(t *testing.T) { // Permutation 10 [HIGH]: Monitor all services with auto-restarts but exclude specific Java workloads func TestPermutation10_MonitorWithExclusionsWithAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) - // Create two namespaces + // Create single namespace namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - kubeSystemNS := helper.Initialize("kube-system", []string{sampleDeploymentServiceYaml}) // Create deployments before enabling monitoring err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml, customerServiceYaml}) assert.NoError(t, err) - err = helper.CreateNamespaceAndApplyResources(kubeSystemNS, []string{sampleDeploymentYaml}) - assert.NoError(t, err) // Set up exclusions and enable monitoring with auto-restart excludeConfig := auto.AnnotationConfig{ Java: auto.AnnotationResources{ - Namespaces: []string{"kube-system"}, Deployments: []string{namespace + "/customer-service"}, }, } @@ -286,32 +282,26 @@ func TestPermutation10_MonitorWithExclusionsWithAutoRestarts(t *testing.T) { nonJavaAnnotations := []string{autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", nonJavaAnnotations, []string{autoAnnotateJavaAnnotation}) assert.NoError(t, err) - - // Verify kube-system deployment has no Java annotations - err = helper.ValidateWorkloadAnnotations("deployment", kubeSystemNS, "sample-deployment", nonJavaAnnotations, []string{autoAnnotateJavaAnnotation}) - assert.NoError(t, err) } // Permutation 18 [HIGH]: Monitor all services with customSelector and specific languages func TestPermutation18_MonitorWithCustomSelectorAndAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) - defaultNS := helper.Initialize("default", []string{sampleDeploymentServiceYaml}) - testNS := helper.Initialize("test", []string{}) + // Create single namespace + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) // Create deployments - err := helper.CreateNamespaceAndApplyResources(defaultNS, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - err = helper.CreateNamespaceAndApplyResources(testNS, []string{sampleDeploymentWithoutServiceYaml}) + err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml, sampleDeploymentWithoutServiceYaml}) assert.NoError(t, err) // Set up custom selector config customSelectorConfig := auto.AnnotationConfig{ Java: auto.AnnotationResources{ - Namespaces: []string{"default"}, + Namespaces: []string{namespace}, }, Python: auto.AnnotationResources{ - Deployments: []string{testNS + "/sample-deployment-without-service"}, + Deployments: []string{namespace + "/sample-deployment-without-service"}, }, } @@ -319,117 +309,112 @@ func TestPermutation18_MonitorWithCustomSelectorAndAutoRestarts(t *testing.T) { helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.TypeDotNet), RestartPods: true, - CustomSelector: customSelectorConfig, + Languages: instrumentation.NewTypeSet(instrumentation.TypeDotNet), + RestartPods: true, + CustomSelector: customSelectorConfig, }) - // Verify default namespace deployment has dotnet annotation - err = helper.ValidateWorkloadAnnotations("deployment", defaultNS, "sample-deployment", + // Verify service-selected deployment has dotnet annotation + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", []string{autoAnnotateDotNetAnnotation}, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}) assert.NoError(t, err) - // Verify test namespace deployment has python annotation - err = helper.ValidateWorkloadAnnotations("deployment", testNS, "sample-deployment-without-service", + // Verify non-service deployment has python annotation + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment-without-service", []string{autoAnnotatePythonAnnotation}, []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) assert.NoError(t, err) } -// Permutation 19 [HIGH++]: Complex scenario with exclusions and customSelector -func TestPermutation19_ComplexMonitoringWithExclusionsAndCustomSelector(t *testing.T) { - helper := NewTestHelper(t, true) +// Permutation 19 [HIGH++]: Test that exclude takes precedence over all +func TestPermutation19_ConflictingCustomSelectorExclude(t *testing.T) { + helper := NewTestHelper(t, false) - defaultNS := helper.Initialize("default", []string{sampleDeploymentServiceYaml}) - testNS := helper.Initialize("test", []string{}) - kubeSystemNS := helper.Initialize("kube-system", []string{sampleDeploymentServiceYaml}) + // Create single namespace + namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - // Create deployments in all namespaces - err := helper.CreateNamespaceAndApplyResources(defaultNS, []string{sampleDeploymentYaml, customerServiceYaml}) - assert.NoError(t, err) - err = helper.CreateNamespaceAndApplyResources(testNS, []string{conflictingDeploymentYaml, sampleDeploymentYaml}) - assert.NoError(t, err) - err = helper.CreateNamespaceAndApplyResources(kubeSystemNS, []string{sampleDeploymentYaml}) + // Create deployments in the namespace + err := helper.CreateNamespaceAndApplyResources(namespace, []string{ + sampleDeploymentYaml, + customerServiceYaml, + conflictingDeploymentYaml, + }) assert.NoError(t, err) - // Set up complex config + // exclude config without namespace-level exclusion, will add later excludeConfig := auto.AnnotationConfig{ Java: auto.AnnotationResources{ - Namespaces: []string{"kube-system", "test"}, - Deployments: []string{defaultNS + "/customer-service"}, + Deployments: []string{namespace + "/customer-service"}, }, } customSelectorConfig := auto.AnnotationConfig{ Java: auto.AnnotationResources{ - Namespaces: []string{"default"}, - Deployments: []string{testNS + "/conflicting-deployment"}, + Namespaces: []string{namespace}, }, Python: auto.AnnotationResources{ - Deployments: []string{testNS + "/conflicting-deployment"}, + Deployments: []string{namespace + "/conflicting-deployment"}, }, DotNet: auto.AnnotationResources{ - Namespaces: []string{"test"}, + Namespaces: []string{namespace}, }, } // Update operator config helper.UpdateAnnotationConfig(nil) - helper.UpdateMonitorConfig(&auto.MonitorConfig{ + monitorConfig := &auto.MonitorConfig{ MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeNodeJS), RestartPods: true, - Exclude: excludeConfig, - CustomSelector: customSelectorConfig, - }) + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeNodeJS), + RestartPods: true, + Exclude: excludeConfig, + CustomSelector: customSelectorConfig, + } - // Verify test/conflicting-deployment - err = helper.ValidateWorkloadAnnotations("deployment", testNS, "conflicting-deployment", - []string{autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, - []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation}) + helper.UpdateMonitorConfig(monitorConfig) + + // Verify conflicting-deployment has Python and NodeJS + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "conflicting-deployment", + []string{autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation, autoAnnotateJavaAnnotation}, + []string{autoAnnotateDotNetAnnotation}) assert.NoError(t, err) - // Verify default/customer-service - err = helper.ValidateWorkloadAnnotations("deployment", defaultNS, "customer-service", + // Verify customer-service has Python and NodeJS (Java excluded) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", []string{autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation}) assert.NoError(t, err) - // Verify default/sample-deployment - err = helper.ValidateWorkloadAnnotations("deployment", defaultNS, "sample-deployment", + // Verify sample-deployment has Java, Python, and NodeJS + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, []string{autoAnnotateDotNetAnnotation}) assert.NoError(t, err) - - // Verify kube-system/sample-deployment - err = helper.ValidateWorkloadAnnotations("deployment", kubeSystemNS, "sample-deployment", - []string{autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, - []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation}) - assert.NoError(t, err) } // Permutation 20 [HIGH]: Disable general monitoring but enable specific instrumentation func TestPermutation20_SelectiveMonitoringWithCustomSelector(t *testing.T) { - helper := NewTestHelper(t, true) + helper := NewTestHelper(t, false) - webNS := helper.Initialize("web", []string{}) - analyticsNS := helper.Initialize("analytics", []string{}) - dataScienceNS := helper.Initialize("data-science", []string{}) + // Create single namespace + namespace := helper.Initialize("test-namespace", []string{}) // Create deployments - err := helper.CreateNamespaceAndApplyResources(webNS, []string{frontendAppYaml, adminDashboardYaml, sampleDeploymentYaml}) - assert.NoError(t, err) - err = helper.CreateNamespaceAndApplyResources(analyticsNS, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - err = helper.CreateNamespaceAndApplyResources(dataScienceNS, []string{sampleDeploymentYaml}) + err := helper.CreateNamespaceAndApplyResources(namespace, []string{ + frontendAppYaml, + adminDashboardYaml, + sampleDeploymentYaml, + sampleDeploymentWithoutServiceYaml, + }) assert.NoError(t, err) // Set up custom selector config customSelectorConfig := auto.AnnotationConfig{ Python: auto.AnnotationResources{ - Namespaces: []string{"analytics", "data-science"}, + Deployments: []string{namespace + "/sample-deployment-without-service"}, }, NodeJS: auto.AnnotationResources{ - Deployments: []string{webNS + "/frontend-app", webNS + "/admin-dashboard"}, + Deployments: []string{namespace + "/frontend-app", namespace + "/admin-dashboard"}, }, } @@ -437,35 +422,31 @@ func TestPermutation20_SelectiveMonitoringWithCustomSelector(t *testing.T) { helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: false, - Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), RestartPods: true, - CustomSelector: customSelectorConfig, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + CustomSelector: customSelectorConfig, }) // Verify frontend-app and admin-dashboard have NodeJS only - err = helper.ValidateWorkloadAnnotations("deployment", webNS, "frontend-app", + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "frontend-app", []string{autoAnnotateNodeJSAnnotation}, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation}) assert.NoError(t, err) - err = helper.ValidateWorkloadAnnotations("deployment", webNS, "admin-dashboard", + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "admin-dashboard", []string{autoAnnotateNodeJSAnnotation}, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation}) assert.NoError(t, err) - // Verify sample-deployment in web namespace has no annotations - err = helper.ValidateWorkloadAnnotations("deployment", webNS, "sample-deployment", - none, - allLanguages) - assert.NoError(t, err) - - // Verify analytics/data-science deployments have no pod template annotations - err = helper.ValidateWorkloadAnnotations("deployment", analyticsNS, "sample-deployment", + // Verify sample-deployment has no annotations + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allLanguages) assert.NoError(t, err) - err = helper.ValidateWorkloadAnnotations("deployment", dataScienceNS, "sample-deployment", - none, - allLanguages) + // Verify sample-deployment-without-service has Python + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment-without-service", + []string{autoAnnotatePythonAnnotation}, + []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) assert.NoError(t, err) } From 20b463c9f3c09a1a3fb52591a9592622bd5e8130 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 10:39:10 -0400 Subject: [PATCH 32/63] add `DISABLE_AUTO_MONITOR` environment variable --- main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 3b2490ede..5b67d084c 100644 --- a/main.go +++ b/main.go @@ -310,7 +310,7 @@ func main() { }, ) } else { - setupLog.Info("Auto-annotation and Auto Monitor is disabled") + setupLog.Info("Auto-annotation / Auto Monitor is disabled") } if os.Getenv("ENABLE_WEBHOOKS") != "false" { @@ -373,6 +373,11 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationC } } + if os.Getenv("DISABLE_AUTO_MONITOR") == "true" { + setupLog.Info("Auto-monitor is disabled") + return nil + } + var monitorConfig *auto.MonitorConfig var monitor auto.InstrumentationAnnotator = autoAnnotationMutators if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { From 47e3b80ea856ea045a35663979af38f5b03ef136 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 10:43:54 -0400 Subject: [PATCH 33/63] update github workflow to run new integration tests --- .../workflows/operator-integration-test.yml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index 3ec2b4b20..702a98d68 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -281,6 +281,48 @@ jobs: sleep 5 go test -v -run TestAnnotationsOnMultipleResources ./integration-tests/manifests/annotations -timeout 30m + AutoMonitorTest: + name: AutoMonitorTest + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Start minikube + uses: medyagh/setup-minikube@master + + - name: Deploy cert-manager to minikube + run: + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml + + - name: Verify minikube and cert-manager + run: | + sleep 10 + kubectl get pods -A + + - name: Build image + run: | + eval $(minikube docker-env) + make container + docker images + + - name: Deploy operator to minikube + run: | + make deploy + + - name: Test AutoMonitor-created annotations + run: | + kubectl get pods -A + kubectl describe pods -n default + sleep 10 + go test -v ./integration-tests/manifests/automonitor -timeout 30m + sleep 10 + DaemonsetAnnotationsTest: name: DaemonsetAnnotationsTest runs-on: ubuntu-latest From 936f0c28cb0ffaaa0392fb159af7d23a6a333382 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 11:09:49 -0400 Subject: [PATCH 34/63] Add controller manager name flag for minikube deployment --- .github/workflows/operator-integration-test.yml | 2 +- .../manifests/automonitor/validate_automonitor_methods.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index 926629af3..4e504844a 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -320,7 +320,7 @@ jobs: kubectl get pods -A kubectl describe pods -n default sleep 10 - go test -v ./integration-tests/manifests/automonitor -timeout 30m + go test -v ./integration-tests/manifests/automonitor -timeout 30m -controllerManagerName 'cloudwatch-controller-manager' sleep 10 DaemonsetAnnotationsTest: diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go index 40c16843d..8b6a77d74 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -5,6 +5,7 @@ package annotations import ( "context" "encoding/json" + "flag" "fmt" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" "github.com/go-logr/logr" @@ -47,7 +48,6 @@ const ( statefulSetName = "sample-statefulset" amazonCloudwatchNamespace = "amazon-cloudwatch" daemonSetName = "sample-daemonset" - amazonControllerManager = "amazon-cloudwatch-observability-controller-manager" sampleDaemonsetYamlRelPath = "../sample-daemonset.yaml" sampleDeploymentYaml = "../sample-deployment.yaml" @@ -59,6 +59,10 @@ const ( timeBetweenRetries = 5 * time.Second ) +var ( + amazonControllerManager = *flag.String("controllerManagerName", "amazon-cloudwatch-observability-controller-manager", "short") +) + type TestHelper struct { clientSet *kubernetes.Clientset t *testing.T From 4a86baf7ea830ca96429b07d130298e48a6c72b0 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 11:33:11 -0400 Subject: [PATCH 35/63] Add controller manager name flag for minikube deployment --- .../automonitor/validate_automonitor_methods.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go index 8b6a77d74..0afafdd64 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -60,7 +60,7 @@ const ( ) var ( - amazonControllerManager = *flag.String("controllerManagerName", "amazon-cloudwatch-observability-controller-manager", "short") + amazonControllerManager = flag.String("controllerManagerName", "amazon-cloudwatch-observability-controller-manager", "short") ) type TestHelper struct { @@ -247,7 +247,7 @@ func (h *TestHelper) UpdateOperator(deployment *appsV1.Deployment) bool { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { // Get the latest version of the deployment - currentDeployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) + currentDeployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), *amazonControllerManager, metav1.GetOptions{}) if err != nil { return err } @@ -314,7 +314,7 @@ func (h *TestHelper) PodsAnnotationsValid(namespace string, shouldExistAnnotatio } func (h *TestHelper) restartOperator() { - cmd := exec.Command("kubectl", "rollout", "restart", "deployment", amazonControllerManager, "-n", amazonCloudwatchNamespace) + cmd := exec.Command("kubectl", "rollout", "restart", "deployment", *amazonControllerManager, "-n", amazonCloudwatchNamespace) output, err := cmd.CombinedOutput() if err != nil { h.logger.Info(fmt.Sprintf("Error restarting deployment: %v\nOutput: %s\n", err, output)) @@ -322,7 +322,7 @@ func (h *TestHelper) restartOperator() { } waitCmd := exec.Command("kubectl", "wait", "--for=condition=Available", - "deployment/"+amazonControllerManager, + "deployment/"+*amazonControllerManager, "-n", amazonCloudwatchNamespace, "--timeout=300s") @@ -360,7 +360,7 @@ func (h *TestHelper) UpdateAnnotationConfig(config *auto.AnnotationConfig) { } func (h *TestHelper) updateOperatorConfig(jsonStr string, flag string) { - deployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), amazonControllerManager, metav1.GetOptions{}) + deployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), *amazonControllerManager, metav1.GetOptions{}) if err != nil { h.t.Errorf("Error getting deployment: %v\n\n", err) return From cabba78d142fbc44a4c13e573b7e2dbf611d1351 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 12:26:07 -0400 Subject: [PATCH 36/63] Add manifest files --- .../manifests/admin-dashboard.yaml | 32 +++++++++++++++++++ .../manifests/conflicting-deployment.yaml | 32 +++++++++++++++++++ .../manifests/customer-service.yaml | 32 +++++++++++++++++++ integration-tests/manifests/frontend-app.yaml | 32 +++++++++++++++++++ .../sample-deployment-without-service.yaml | 21 ++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 integration-tests/manifests/admin-dashboard.yaml create mode 100644 integration-tests/manifests/conflicting-deployment.yaml create mode 100644 integration-tests/manifests/customer-service.yaml create mode 100644 integration-tests/manifests/frontend-app.yaml create mode 100644 integration-tests/manifests/sample-deployment-without-service.yaml diff --git a/integration-tests/manifests/admin-dashboard.yaml b/integration-tests/manifests/admin-dashboard.yaml new file mode 100644 index 000000000..eb69de81c --- /dev/null +++ b/integration-tests/manifests/admin-dashboard.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin-dashboard + labels: + app: admin-dashboard +spec: + replicas: 1 + selector: + matchLabels: + app: admin-dashboard + template: + metadata: + labels: + app: admin-dashboard + spec: + containers: + - name: admin-dashboard-container + image: nginx:latest + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: admin-dashboard-service +spec: + selector: + app: admin-dashboard + ports: + - port: 80 + targetPort: 80 diff --git a/integration-tests/manifests/conflicting-deployment.yaml b/integration-tests/manifests/conflicting-deployment.yaml new file mode 100644 index 000000000..ece71b497 --- /dev/null +++ b/integration-tests/manifests/conflicting-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: conflicting-deployment + labels: + app: conflicting-app +spec: + replicas: 1 + selector: + matchLabels: + app: conflicting-app + template: + metadata: + labels: + app: conflicting-app + spec: + containers: + - name: conflicting-container + image: nginx:latest + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: conflicting-service +spec: + selector: + app: conflicting-app + ports: + - port: 80 + targetPort: 80 diff --git a/integration-tests/manifests/customer-service.yaml b/integration-tests/manifests/customer-service.yaml new file mode 100644 index 000000000..d4af762b1 --- /dev/null +++ b/integration-tests/manifests/customer-service.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: customer-service + labels: + app: customer-service +spec: + replicas: 1 + selector: + matchLabels: + app: customer-service + template: + metadata: + labels: + app: customer-service + spec: + containers: + - name: customer-service-container + image: nginx:latest + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: customer-service +spec: + selector: + app: customer-service + ports: + - port: 80 + targetPort: 80 diff --git a/integration-tests/manifests/frontend-app.yaml b/integration-tests/manifests/frontend-app.yaml new file mode 100644 index 000000000..c833a1358 --- /dev/null +++ b/integration-tests/manifests/frontend-app.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend-app + labels: + app: frontend +spec: + replicas: 1 + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + spec: + containers: + - name: frontend-container + image: nginx:latest + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend-service +spec: + selector: + app: frontend + ports: + - port: 80 + targetPort: 80 diff --git a/integration-tests/manifests/sample-deployment-without-service.yaml b/integration-tests/manifests/sample-deployment-without-service.yaml new file mode 100644 index 000000000..e58e35eb1 --- /dev/null +++ b/integration-tests/manifests/sample-deployment-without-service.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sample-deployment-without-service + labels: + app: sample-app-without-service +spec: + replicas: 1 + selector: + matchLabels: + app: sample-app-without-service + template: + metadata: + labels: + app: sample-app-without-service + spec: + containers: + - name: sample-container + image: nginx:latest + ports: + - containerPort: 80 From 955e643dde47e6d42e41b71c1cb52693c318972e Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 14:05:38 -0400 Subject: [PATCH 37/63] temp: add log for cloudwatch controller manager for instrumentation test. Please revert --- .github/workflows/operator-integration-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index 4e504844a..9aab705d5 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -78,6 +78,7 @@ jobs: kubectl wait --for=condition=Ready pod --all -n default kubectl get pods -A kubectl describe pods -n default + kubectl logs deployment/cloudwatch-controller-manager --all-containers=true go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/java/default_instrumentation_java_env_variables.json app_signals - name: Test for defined instrumentation resources for Java From a2007b03c75eee37d35fac591607cb089e23c6c4 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 14:12:10 -0400 Subject: [PATCH 38/63] temp: add log for cloudwatch controller manager for instrumentation test. Please revert --- .github/workflows/operator-integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index 9aab705d5..7221bfec2 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -78,7 +78,7 @@ jobs: kubectl wait --for=condition=Ready pod --all -n default kubectl get pods -A kubectl describe pods -n default - kubectl logs deployment/cloudwatch-controller-manager --all-containers=true + kubectl logs deployment/cloudwatch-controller-manager --all-containers=true -n amazon-cloudwatch go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/java/default_instrumentation_java_env_variables.json app_signals - name: Test for defined instrumentation resources for Java From d55e3cffa01f55887f6b14c7d4656197c6c2b43f Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 14:24:50 -0400 Subject: [PATCH 39/63] Correctly return nil when automonitor and autoannotation are disabled --- main.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 5b67d084c..741a8ef74 100644 --- a/main.go +++ b/main.go @@ -352,7 +352,6 @@ func main() { func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, client client.Client, reader client.Reader) auto.InstrumentationAnnotator { var autoAnnotationConfig auto.AnnotationConfig - var autoAnnotationMutators *auto.AnnotationMutators supportedLanguages := instrumentation.SupportedTypes() if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" { @@ -362,7 +361,7 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationC setupLog.Error(err, "Unable to unmarshal auto-annotation config") } else { // TODO: detect empty - setupLog.Info("WARNING: Using deprecated autoAnnotateAutoInstrumentation config. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") + setupLog.Info("WARNING: Using deprecated autoAnnotateAutoInstrumentation config, Disabling AutoMonitor. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") return auto.NewAnnotationMutators( client, reader, @@ -379,26 +378,24 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationC } var monitorConfig *auto.MonitorConfig - var monitor auto.InstrumentationAnnotator = autoAnnotationMutators if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-monitor config, disabling AutoMonitor") - return monitor + return nil } else { k8sConfig, err := rest.InClusterConfig() if err != nil { setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") - return monitor + return nil } clientSet, err := kubernetes.NewForConfig(k8sConfig) if err != nil { setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") - return monitor + return nil } logger := ctrl.Log.WithName("auto_monitor") - monitor = auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, logger) + return auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, logger) } - return monitor } func waitForWebhookServerStart(ctx context.Context, checker healthz.Checker, callback func(context.Context)) { From aae6fd509ce4ec8067eef6db574efd70ea630cd8 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 14:56:16 -0400 Subject: [PATCH 40/63] Remove redundant annotation config clears --- .../manifests/automonitor/validate_automonitor_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_test.go b/integration-tests/manifests/automonitor/validate_automonitor_test.go index e951f3afd..baf9f979e 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_test.go @@ -29,7 +29,6 @@ func TestPermutation1_MonitorAllServicesNoAutoRestarts(t *testing.T) { helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentYaml}) - helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, RestartPods: false, @@ -55,7 +54,6 @@ func TestPermutation2_DisableMonitoringNoAutoRestarts(t *testing.T) { namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) // First enable monitoring with auto-restart - helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, RestartPods: true, @@ -114,7 +112,6 @@ func TestPermutation4_DisableMonitoringWithAutoRestarts(t *testing.T) { namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) // Start with monitoring enabled - helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, RestartPods: true, @@ -143,7 +140,6 @@ func TestPermutation5_MonitorSelectedLanguagesNoAutoRestarts(t *testing.T) { helper := NewTestHelper(t, false) namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml, sampleDeploymentYaml}) // Start with all languages enabled - helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, RestartPods: true, @@ -178,7 +174,6 @@ func TestPermutation6_MonitorSelectedLanguagesWithAutoRestarts(t *testing.T) { namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) // Start with all languages enabled - helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, RestartPods: true, @@ -219,7 +214,6 @@ func TestPermutation9_MonitorWithExclusionsNoAutoRestarts(t *testing.T) { } // Update config with exclusions - helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, RestartPods: false, @@ -267,7 +261,6 @@ func TestPermutation10_MonitorWithExclusionsWithAutoRestarts(t *testing.T) { }, } - helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, RestartPods: true, @@ -306,7 +299,6 @@ func TestPermutation18_MonitorWithCustomSelectorAndAutoRestarts(t *testing.T) { } // Update config with custom selector - helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.NewTypeSet(instrumentation.TypeDotNet), @@ -362,7 +354,6 @@ func TestPermutation19_ConflictingCustomSelectorExclude(t *testing.T) { } // Update operator config - helper.UpdateAnnotationConfig(nil) monitorConfig := &auto.MonitorConfig{ MonitorAllServices: true, Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeNodeJS), @@ -419,7 +410,6 @@ func TestPermutation20_SelectiveMonitoringWithCustomSelector(t *testing.T) { } // Update operator config - helper.UpdateAnnotationConfig(nil) helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: false, Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), From e60317598a51e8715e4b8ac1d656c9bfe7710d31 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 15:12:10 -0400 Subject: [PATCH 41/63] Refactor annotation functions --- .../automonitor/validate_automonitor_test.go | 127 +++++++++++------- 1 file changed, 80 insertions(+), 47 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_test.go b/integration-tests/manifests/automonitor/validate_automonitor_test.go index baf9f979e..78877c806 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_test.go @@ -6,6 +6,8 @@ import ( "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" "github.com/stretchr/testify/assert" + "maps" + "slices" "testing" "time" ) @@ -18,11 +20,30 @@ const ( conflictingDeploymentYaml = "../conflicting-deployment.yaml" ) -var ( - allLanguages = []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} - javaPythonOnly = []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation} - dotnetNodejsOnly = []string{autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} -) +var all = slices.Collect(maps.Keys(instrumentation.SupportedTypes())) +var allAnnotations = getAnnotations(all...) + +// getAnnotations returns both auto and inject annotations for the specified language types +func getAnnotations(types ...instrumentation.Type) []string { + var annotations []string + for _, t := range types { + switch t { + case instrumentation.TypeJava: + annotations = append(annotations, autoAnnotateJavaAnnotation, injectJavaAnnotation) + case instrumentation.TypePython: + annotations = append(annotations, autoAnnotatePythonAnnotation, injectPythonAnnotation) + case instrumentation.TypeDotNet: + annotations = append(annotations, autoAnnotateDotNetAnnotation, injectDotNetAnnotation) + case instrumentation.TypeNodeJS: + annotations = append(annotations, autoAnnotateNodeJSAnnotation, injectNodeJSAnnotation) + } + } + return annotations +} + +// getAllTypes returns all supported language types + +// getNoAnnotations returns an empty annotation list // Permutation 1 [HIGH]: Enable monitoring for all services without auto restarts func TestPermutation1_MonitorAllServicesNoAutoRestarts(t *testing.T) { @@ -38,13 +59,13 @@ func TestPermutation1_MonitorAllServicesNoAutoRestarts(t *testing.T) { assert.NoError(t, err) // Verify no annotations without restart - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allLanguages) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allAnnotations) assert.NoError(t, err) // Manually restart and verify annotations err = helper.RestartDeployment(namespace, "sample-deployment") assert.NoError(t, err) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) assert.NoError(t, err) } @@ -63,7 +84,7 @@ func TestPermutation2_DisableMonitoringNoAutoRestarts(t *testing.T) { assert.NoError(t, err) // Verify initial annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) assert.NoError(t, err) // Disable monitoring without auto-restart @@ -73,13 +94,13 @@ func TestPermutation2_DisableMonitoringNoAutoRestarts(t *testing.T) { }) // Verify annotations still present - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) assert.NoError(t, err) // Manually restart and verify annotations removed err = helper.RestartDeployment(namespace, "sample-deployment") assert.NoError(t, err) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allLanguages) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allAnnotations) assert.NoError(t, err) } @@ -92,7 +113,7 @@ func TestPermutation3_MonitorAllServicesWithAutoRestarts(t *testing.T) { assert.NoError(t, err) // Verify no initial annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allLanguages) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allAnnotations) assert.NoError(t, err) // Enable monitoring with auto-restart @@ -102,7 +123,7 @@ func TestPermutation3_MonitorAllServicesWithAutoRestarts(t *testing.T) { }) // Verify annotations automatically added - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) assert.NoError(t, err) } @@ -121,7 +142,7 @@ func TestPermutation4_DisableMonitoringWithAutoRestarts(t *testing.T) { assert.NoError(t, err) // Verify initial annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) assert.NoError(t, err) // Disable monitoring with auto-restart @@ -131,7 +152,7 @@ func TestPermutation4_DisableMonitoringWithAutoRestarts(t *testing.T) { }) // Verify annotations automatically removed - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allLanguages) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allAnnotations) assert.NoError(t, err) } @@ -148,23 +169,26 @@ func TestPermutation5_MonitorSelectedLanguagesNoAutoRestarts(t *testing.T) { time.Sleep(time.Second * 5) // Verify all annotations present - err := helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + err := helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) assert.NoError(t, err) // Update to Java and Python only without auto-restart helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), RestartPods: false, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), + RestartPods: false, }) // Verify annotations unchanged without restart - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) assert.NoError(t, err) // Manually restart and verify only Java/Python remain err = helper.RestartDeployment(namespace, "sample-deployment") assert.NoError(t, err) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", javaPythonOnly, dotnetNodejsOnly) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", + getAnnotations(instrumentation.TypeJava, instrumentation.TypePython), + getAnnotations(instrumentation.TypeDotNet, instrumentation.TypeNodeJS)) assert.NoError(t, err) } @@ -183,17 +207,20 @@ func TestPermutation6_MonitorSelectedLanguagesWithAutoRestarts(t *testing.T) { assert.NoError(t, err) // Verify all annotations present - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) assert.NoError(t, err) // Update to Java and Python only with auto-restart helper.UpdateMonitorConfig(&auto.MonitorConfig{ MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), RestartPods: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), + RestartPods: true, }) // Verify only Java/Python annotations remain - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", javaPythonOnly, dotnetNodejsOnly) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", + getAnnotations(instrumentation.TypeJava, instrumentation.TypePython), + getAnnotations(instrumentation.TypeDotNet, instrumentation.TypeNodeJS)) assert.NoError(t, err) } @@ -229,17 +256,22 @@ func TestPermutation9_MonitorWithExclusionsNoAutoRestarts(t *testing.T) { err = helper.RestartDeployment(namespace, "customer-service") assert.NoError(t, err) - nonPythonAnnotations := []string{autoAnnotateJavaAnnotation, injectJavaAnnotation, autoAnnotateDotNetAnnotation, injectDotNetAnnotation, autoAnnotateNodeJSAnnotation, injectNodeJSAnnotation} // Verify regular deployment has all annotations except python - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", nonPythonAnnotations, []string{autoAnnotatePythonAnnotation}) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", + getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet, instrumentation.TypeNodeJS), + getAnnotations(instrumentation.TypePython)) assert.NoError(t, err) // Verify excluded customer-service has no Java annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", []string{autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation}) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", + getAnnotations(instrumentation.TypeDotNet, instrumentation.TypeNodeJS), + getAnnotations(instrumentation.TypeJava, instrumentation.TypePython)) assert.NoError(t, err) - // Verify kube-system deployment has no Java annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", nonPythonAnnotations, []string{injectPythonAnnotation, autoAnnotatePythonAnnotation}) + // Verify kube-system deployment has no Python annotations + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", + getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet, instrumentation.TypeNodeJS), + getAnnotations(instrumentation.TypePython)) assert.NoError(t, err) } @@ -268,12 +300,13 @@ func TestPermutation10_MonitorWithExclusionsWithAutoRestarts(t *testing.T) { }) // Verify regular deployment has all annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allLanguages, none) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) assert.NoError(t, err) // Verify excluded customer-service has no Java annotations - nonJavaAnnotations := []string{autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", nonJavaAnnotations, []string{autoAnnotateJavaAnnotation}) + err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", + getAnnotations(instrumentation.TypePython, instrumentation.TypeDotNet, instrumentation.TypeNodeJS), + getAnnotations(instrumentation.TypeJava)) assert.NoError(t, err) } @@ -308,14 +341,14 @@ func TestPermutation18_MonitorWithCustomSelectorAndAutoRestarts(t *testing.T) { // Verify service-selected deployment has dotnet annotation err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - []string{autoAnnotateDotNetAnnotation}, - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}) + getAnnotations(instrumentation.TypeDotNet), + getAnnotations(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeNodeJS)) assert.NoError(t, err) // Verify non-service deployment has python annotation err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment-without-service", - []string{autoAnnotatePythonAnnotation}, - []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + getAnnotations(instrumentation.TypePython), + getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet, instrumentation.TypeNodeJS)) assert.NoError(t, err) } @@ -364,22 +397,22 @@ func TestPermutation19_ConflictingCustomSelectorExclude(t *testing.T) { helper.UpdateMonitorConfig(monitorConfig) - // Verify conflicting-deployment has Python and NodeJS + // Verify conflicting-deployment has Python, NodeJS and Java err = helper.ValidateWorkloadAnnotations("deployment", namespace, "conflicting-deployment", - []string{autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation, autoAnnotateJavaAnnotation}, - []string{autoAnnotateDotNetAnnotation}) + getAnnotations(instrumentation.TypePython, instrumentation.TypeNodeJS, instrumentation.TypeJava), + getAnnotations(instrumentation.TypeDotNet)) assert.NoError(t, err) // Verify customer-service has Python and NodeJS (Java excluded) err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", - []string{autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, - []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation}) + getAnnotations(instrumentation.TypePython, instrumentation.TypeNodeJS), + getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet)) assert.NoError(t, err) // Verify sample-deployment has Java, Python, and NodeJS err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateNodeJSAnnotation}, - []string{autoAnnotateDotNetAnnotation}) + getAnnotations(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeNodeJS), + getAnnotations(instrumentation.TypeDotNet)) assert.NoError(t, err) } @@ -419,24 +452,24 @@ func TestPermutation20_SelectiveMonitoringWithCustomSelector(t *testing.T) { // Verify frontend-app and admin-dashboard have NodeJS only err = helper.ValidateWorkloadAnnotations("deployment", namespace, "frontend-app", - []string{autoAnnotateNodeJSAnnotation}, - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation}) + getAnnotations(instrumentation.TypeNodeJS), + getAnnotations(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeDotNet)) assert.NoError(t, err) err = helper.ValidateWorkloadAnnotations("deployment", namespace, "admin-dashboard", - []string{autoAnnotateNodeJSAnnotation}, - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation}) + getAnnotations(instrumentation.TypeNodeJS), + getAnnotations(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeDotNet)) assert.NoError(t, err) // Verify sample-deployment has no annotations err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, - allLanguages) + allAnnotations) assert.NoError(t, err) // Verify sample-deployment-without-service has Python err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment-without-service", - []string{autoAnnotatePythonAnnotation}, - []string{autoAnnotateJavaAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) + getAnnotations(instrumentation.TypePython), + getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet, instrumentation.TypeNodeJS)) assert.NoError(t, err) } From 3bfe07c472772a058e2236dbda2aebbd13cc62c6 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Fri, 25 Apr 2025 18:35:52 -0400 Subject: [PATCH 42/63] disable autoannotation if autoAnnotationConfig is empty and AutoMonitor is not empty. --- .gitignore | 3 ++- main.go | 24 ++++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 2fdb4be34..1220c9aae 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ build .DS_Store .idea .attach_pid* -bin \ No newline at end of file +bin +/amazon-cloudwatch-agent-operator \ No newline at end of file diff --git a/main.go b/main.go index 741a8ef74..aea4958d7 100644 --- a/main.go +++ b/main.go @@ -360,20 +360,24 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationC if err := json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-annotation config") } else { - // TODO: detect empty - setupLog.Info("WARNING: Using deprecated autoAnnotateAutoInstrumentation config, Disabling AutoMonitor. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") - return auto.NewAnnotationMutators( - client, - reader, - setupLog, - autoAnnotationConfig, - supportedLanguages, - ) + // todo: technically a breaking change, because previously an empty autoAnnotationConfig would clear all annotations, but automonitor does not reproduce this behavior by default because it requires restartPods to be enabled. + if autoAnnotationConfig.Empty() && autoMonitorConfigStr != "" { + setupLog.Info("Auto-annotation is disabled because it is empty and the AutoMonitor config is not empty. Trying AutoMonitor...") + } else { + setupLog.Info("WARNING: Using deprecated autoAnnotateAutoInstrumentation config, Disabling AutoMonitor. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") + return auto.NewAnnotationMutators( + client, + reader, + setupLog, + autoAnnotationConfig, + supportedLanguages, + ) + } } } if os.Getenv("DISABLE_AUTO_MONITOR") == "true" { - setupLog.Info("Auto-monitor is disabled") + setupLog.Info("Auto-monitor is disabled due to DISABLE_AUTO_MONITOR environment variable") return nil } From 0ffe221af6dd53b67efafb75f3acdfb87f292f9e Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 1 May 2025 11:58:57 -0400 Subject: [PATCH 43/63] Add namespace MutateObject tests, fix namespace mutation. a namespaced name of a "namespace-a" is just "namespace-a", not "namespace-a/namespace-a" --- pkg/instrumentation/auto/annotation.go | 3 + pkg/instrumentation/auto/monitor_test.go | 187 +++++++++++++++++++++++ 2 files changed, 190 insertions(+) diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 3e1a17594..dc96979fa 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -128,6 +128,9 @@ func (m *AnnotationMutators) Empty() bool { } func namespacedName(obj metav1.Object) string { + if _, ok := obj.(*corev1.Namespace); ok { + return obj.GetName() + } return fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName()) } diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index c48e8c747..fa31748b2 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -355,6 +355,193 @@ func TestMonitor_MutateObject(t *testing.T) { } } +func TestMonitor_MutateObject_Namespace(t *testing.T) { + tests := []struct { + name string + config MonitorConfig + namespace string + existingAnnotations map[string]string + expectedNamespaceAnnotations map[string]string + expectedMutated map[string]string + }{ + { + name: "namespace included in custom selector java", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: buildAnnotations(instrumentation.TypeJava), + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "namespace included in custom selector python", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypePython), + CustomSelector: AnnotationConfig{ + Python: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: buildAnnotations(instrumentation.TypePython), + expectedMutated: buildAnnotations(instrumentation.TypePython), + }, + { + name: "namespace excluded", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + Exclude: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: map[string]string{}, + expectedMutated: map[string]string{}, + }, + { + name: "namespace not in any selection", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: map[string]string{}, + expectedMutated: map[string]string{}, + }, + { + name: "multiple languages in custom selector", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + Python: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: mergeMaps( + buildAnnotations(instrumentation.TypeJava), + buildAnnotations(instrumentation.TypePython), + ), + expectedMutated: mergeMaps( + buildAnnotations(instrumentation.TypeJava), + buildAnnotations(instrumentation.TypePython), + ), + }, + { + name: "remove java when no longer selected", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + }, + namespace: "test-namespace", + existingAnnotations: buildAnnotations(instrumentation.TypeJava), + expectedNamespaceAnnotations: map[string]string{}, + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "preserve existing non-instrumentation annotations", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: map[string]string{"custom-key": "custom-value"}, + expectedNamespaceAnnotations: mergeMaps( + buildAnnotations(instrumentation.TypeJava), + map[string]string{"custom-key": "custom-value"}, + ), + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "restart pods has no effect on namespace selection", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: buildAnnotations(instrumentation.TypeJava), + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "add annotation to namespace with existing annotations", + config: MonitorConfig{ + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + Python: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: buildAnnotations(instrumentation.TypePython), + expectedNamespaceAnnotations: mergeMaps( + buildAnnotations(instrumentation.TypeJava), + buildAnnotations(instrumentation.TypePython), + ), + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup test environment + clientset := fake.NewSimpleClientset() + fakeClient := fake2.NewFakeClient() + ctx := context.TODO() + + // Create namespace with existing annotations if any + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.namespace, + Annotations: tt.existingAnnotations, + }, + } + + // Create monitor + logger := testr.New(t) + monitor := NewMonitor(ctx, tt.config, clientset, fakeClient, fakeClient, logger) + + // Test + mutatedAnnotations := monitor.MutateObject(nil, namespace) + assert.Equal(t, tt.expectedMutated, mutatedAnnotations, "Mutated annotations don't match expected") + + assert.Equal(t, tt.expectedNamespaceAnnotations, namespace.GetAnnotations(), "Namespace annotations don't match expected") + }) + } +} + func waitForInformerUpdate(monitor *Monitor, isValid func(int) bool) error { return wait.PollImmediate(1*time.Millisecond, 5*time.Millisecond, func() (bool, error) { return isValid(len(monitor.serviceInformer.GetStore().ListKeys())), nil From 3b7cb97dba297c3aacc0f5b66ac28d35eeaf1d43 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 1 May 2025 14:20:12 -0400 Subject: [PATCH 44/63] Add default exclusion for kube-system and amazon-cloudwatch --- pkg/instrumentation/auto/monitor.go | 5 + pkg/instrumentation/auto/monitor_test.go | 122 +++++++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 876b3c2e1..371cc28f2 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -259,6 +259,8 @@ func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { return mutate(obj, languagesToAnnotate) } +var excludedNamespaces = []string{"kube-system", "amazon-cloudwatch"} + // returns if workload is auto monitored (does not include custom selector) func (m *Monitor) isWorkloadAutoMonitored(obj client.Object) bool { if isNamespace(obj) { @@ -269,6 +271,9 @@ func (m *Monitor) isWorkloadAutoMonitored(obj client.Object) bool { return false } + if slices.Contains(excludedNamespaces, obj.GetNamespace()) { + return false + } // determine if the object is currently selected by a service objectLabels := getTemplateSpecLabels(obj) for _, informerObj := range m.serviceInformer.GetStore().List() { diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index fa31748b2..646a0dc3e 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -355,6 +355,128 @@ func TestMonitor_MutateObject(t *testing.T) { } } +func TestSystemNamespaceExclusion(t *testing.T) { + tests := []struct { + name string + namespace string + config MonitorConfig + expectedWorkloadAnnotations map[string]string + }{ + { + name: "kube-system namespace is excluded by default", + namespace: "kube-system", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + }, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "amazon-cloudwatch namespace is excluded by default", + namespace: "amazon-cloudwatch", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + }, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "kube-system can be included via customSelector", + namespace: "kube-system", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + DaemonSets: []string{"kube-system/workload"}, + Deployments: []string{"kube-system/workload"}, + StatefulSets: []string{"kube-system/workload"}, + }, + }, + }, + expectedWorkloadAnnotations: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "amazon-cloudwatch can be included via customSelector", + namespace: "amazon-cloudwatch", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + DaemonSets: []string{"amazon-cloudwatch/workload"}, + Deployments: []string{"amazon-cloudwatch/workload"}, + StatefulSets: []string{"amazon-cloudwatch/workload"}, + }, + }, + }, + expectedWorkloadAnnotations: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "regular namespace is not excluded by default", + namespace: "test-namespace", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + }, + expectedWorkloadAnnotations: buildAnnotations(instrumentation.TypeJava), + }, + } + + for _, workloadType := range workloadTypes { + t.Run(workloadType.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup fresh clients for each test + clientset := fake.NewSimpleClientset() + fakeClient := fake2.NewFakeClient() + ctx := context.TODO() + logger := testr.New(t) + + // Create namespace + namespace := createNamespace(t, clientset, ctx, tt.namespace) + + // Create a monitor + monitor := NewMonitor(ctx, tt.config, clientset, fakeClient, fakeClient, logger) + + // Create service + labels := map[string]string{"app": "test"} + service := newTestService("service", namespace.Name, labels) + _, err := clientset.CoreV1().Services(namespace.Name).Create(ctx, service, metav1.CreateOptions{}) + assert.NoError(t, err) + + // Create workload object + workloadObj := workloadType.create("workload", namespace.Name, labels, nil) + + // Add workload to clientset based on its type + switch obj := workloadObj.(type) { + case *appsv1.Deployment: + _, err = clientset.AppsV1().Deployments(namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + case *appsv1.StatefulSet: + _, err = clientset.AppsV1().StatefulSets(namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + case *appsv1.DaemonSet: + _, err = clientset.AppsV1().DaemonSets(namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + } + assert.NoError(t, err) + + // Wait for service informer to be updated + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys > 0 }) + assert.NoError(t, err) + + // Test mutation + mutatedAnnotations := monitor.MutateObject(nil, workloadObj) + assert.Equal(t, tt.expectedWorkloadAnnotations, mutatedAnnotations) + }) + } + }) + } +} + func TestMonitor_MutateObject_Namespace(t *testing.T) { tests := []struct { name string From 68101a0b76a38cdf80ba3300741e57143bf2d4ce Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 1 May 2025 14:33:11 -0400 Subject: [PATCH 45/63] Split MutateAndPatchAll into MutateAndPatchWorklads and MutateAndPatchNamespaces so Monitor's MutateAndPatchAll can trigger different behavior separately based on RestartPods --- pkg/instrumentation/auto/annotation.go | 4 +++- pkg/instrumentation/auto/annotation_test.go | 12 ++++++------ pkg/instrumentation/auto/callback.go | 8 ++++++-- pkg/instrumentation/auto/monitor.go | 3 ++- pkg/instrumentation/auto/monitor_test.go | 12 ++++++------ 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index dc96979fa..4a36b5458 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -42,7 +42,9 @@ type AnnotationMutators struct { } func (m *AnnotationMutators) MutateAndPatchAll(ctx context.Context) { - MutateAndPatchAll(m, ctx, true) + var m2 InstrumentationAnnotator = m + MutateAndPatchWorkloads(m2, ctx) + MutateAndPatchNamespaces(m2, ctx, true) } func (m *AnnotationMutators) GetAnnotationMutators() *AnnotationMutators { diff --git a/pkg/instrumentation/auto/annotation_test.go b/pkg/instrumentation/auto/annotation_test.go index 2850f7fae..72ac13ee5 100644 --- a/pkg/instrumentation/auto/annotation_test.go +++ b/pkg/instrumentation/auto/annotation_test.go @@ -117,7 +117,7 @@ func TestAnnotationMutators_Namespaces(t *testing.T) { testCase.cfg, testCase.typeSet, ) - MutateAndPatchAll(mutators, ctx, false) + mutators.MutateAndPatchAll(ctx) gotNamespaces := &corev1.NamespaceList{} require.NoError(t, fakeClient.List(ctx, gotNamespaces)) for _, gotNamespace := range gotNamespaces.Items { @@ -292,7 +292,7 @@ func TestAnnotationMutators_Deployments(t *testing.T) { testCase.cfg, testCase.typeSet, ) - MutateAndPatchAll(mutators, ctx, false) + mutators.MutateAndPatchAll(ctx) gotDeployments := &appsv1.DeploymentList{} require.NoError(t, fakeClient.List(ctx, gotDeployments)) for _, gotDeployment := range gotDeployments.Items { @@ -360,7 +360,7 @@ func TestAnnotationMutators_DaemonSets(t *testing.T) { testCase.cfg, testCase.typeSet, ) - MutateAndPatchAll(mutators, ctx, false) + mutators.MutateAndPatchAll(ctx) gotDaemonSets := &appsv1.DaemonSetList{} require.NoError(t, fakeClient.List(ctx, gotDaemonSets)) for _, gotDaemonSet := range gotDaemonSets.Items { @@ -428,7 +428,7 @@ func TestAnnotationMutators_StatefulSets(t *testing.T) { testCase.cfg, testCase.typeSet, ) - MutateAndPatchAll(mutators, ctx, false) + mutators.MutateAndPatchAll(ctx) gotStatefulSets := &appsv1.StatefulSetList{} require.NoError(t, fakeClient.List(ctx, gotStatefulSets)) for _, gotStatefulSet := range gotStatefulSets.Items { @@ -494,11 +494,11 @@ func TestAnnotationMutators_ClientErrors(t *testing.T) { cfg, instrumentation.NewTypeSet(instrumentation.TypeJava), ) - MutateAndPatchAll(mutators, context.Background(), false) + mutators.MutateAndPatchAll(context.Background()) errClient.AssertCalled(t, "List", mock.Anything, mock.Anything, mock.Anything) mutators.clientWriter = errClient mutators.clientReader = fakeClient - MutateAndPatchAll(mutators, context.Background(), false) + mutators.MutateAndPatchAll(context.Background()) errClient.AssertCalled(t, "Patch", mock.Anything, mock.Anything, mock.Anything, mock.Anything) } diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index add923dce..3e0b3643e 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -165,13 +165,17 @@ func RestartNamespace(m InstrumentationAnnotator, ctx context.Context, namespace } // MutateAndPatchAll runs the mutators for each of the supported resources and patches them. -func MutateAndPatchAll(m InstrumentationAnnotator, ctx context.Context, restartNamespace bool) { + +func MutateAndPatchWorkloads(m InstrumentationAnnotator, ctx context.Context) { f := getMutateObjectFunc(m) callbackFunc := patchFunc(m, ctx, f) rangeObjectList(m, ctx, &appsv1.DeploymentList{}, &client.ListOptions{}, callbackFunc) rangeObjectList(m, ctx, &appsv1.DaemonSetList{}, &client.ListOptions{}, callbackFunc) rangeObjectList(m, ctx, &appsv1.StatefulSetList{}, &client.ListOptions{}, callbackFunc) - rangeObjectList(m, ctx, &corev1.NamespaceList{}, &client.ListOptions{}, chainCallbacks(callbackFunc, restartNamespaceFunc(m, ctx, restartNamespace))) +} + +func MutateAndPatchNamespaces(m InstrumentationAnnotator, ctx context.Context, restartNamespace bool) { + rangeObjectList(m, ctx, &corev1.NamespaceList{}, &client.ListOptions{}, chainCallbacks(patchFunc(m, ctx, getMutateObjectFunc(m)), restartNamespaceFunc(m, ctx, restartNamespace))) } func getMutateObjectFunc(m InstrumentationAnnotator) objectCallbackFunc { diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 371cc28f2..514bc0146 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -43,8 +43,9 @@ type Monitor struct { func (m *Monitor) MutateAndPatchAll(ctx context.Context) { if m.config.RestartPods { - MutateAndPatchAll(m, ctx, false) + MutateAndPatchWorkloads(m, ctx) } + MutateAndPatchNamespaces(m, ctx, m.config.RestartPods) // todo: what to do about updating namespace annotations? maybe update them here? or pass in variable to MutateAndPatchAll? } diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index 646a0dc3e..5c27e298b 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -684,7 +684,7 @@ func Test_OptOutByRemovingService(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO(), false) + monitor.MutateAndPatchAll(context.TODO()) updatedWorkload, err := wt.getWithClient(c, defaultNs, workload.GetName()) assert.NoError(t, err) assert.Equal(t, userAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) @@ -703,7 +703,7 @@ func Test_OptOutByRemovingService(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO(), false) + monitor.MutateAndPatchAll(context.TODO()) err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) assert.NoError(t, err) @@ -726,7 +726,7 @@ func Test_OptOutByRemovingService(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO(), false) + monitor.MutateAndPatchAll(context.TODO()) updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) assert.NoError(t, err) @@ -746,7 +746,7 @@ func Test_OptOutByRemovingService(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO(), false) + monitor.MutateAndPatchAll(context.TODO()) err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) assert.NoError(t, err) @@ -777,7 +777,7 @@ func Test_OptOutByDisablingMonitorAllServices(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) - MutateAndPatchAll(monitor, context.TODO(), false) + monitor.MutateAndPatchAll(context.TODO()) updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) assert.NoError(t, err) @@ -885,7 +885,7 @@ func Test_StartupRestartPods(t *testing.T) { var k8sInterface kubernetes.Interface = clientset var logger logr.Logger = testr.New(t) m := NewMonitor(context.TODO(), config, k8sInterface, fakeClient, fakeClient, logger) - MutateAndPatchAll(m, context.TODO(), false) + m.MutateAndPatchAll(context.TODO()) updatedMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments(defaultNs).Get(context.TODO(), matchingDeployment.Name, metav1.GetOptions{}) assert.NoError(t, err) assert.Equal(t, buildAnnotations(instrumentation.TypeJava), updatedMatchingDeployment.Spec.Template.GetAnnotations()) From efbe7c321b8f2f737610cb70650f48423c07221b Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 1 May 2025 14:43:32 -0400 Subject: [PATCH 46/63] Add license --- pkg/instrumentation/auto/monitor.go | 3 +++ pkg/instrumentation/auto/monitor_test.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 514bc0146..3c966bce2 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + package auto import ( diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index 5c27e298b..2e3f85df1 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + package auto import ( From 88ade73e1b24a4471c0eee1e62f21e5659394064 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 1 May 2025 15:30:31 -0400 Subject: [PATCH 47/63] Do not modify feature flag values, instead override at podmutator level if automonitor is enabled. this prevents multi instrumentation unit tests from failing --- Dockerfile | 2 +- .../validate_automonitor_deployment_test.go | 8 ++-- .../validate_automonitor_methods.go | 14 +++--- .../automonitor/validate_automonitor_test.go | 8 ++-- integration-tests/util/util.go | 3 +- .../namespacemutation/webhookhandler.go | 3 +- .../workloadmutation/webhookhandler.go | 3 +- .../workloadmutation/webhookhandler_test.go | 6 ++- main.go | 30 +++++++------ pkg/featuregate/featuregate.go | 4 +- pkg/instrumentation/auto/callback.go | 1 + pkg/instrumentation/auto/config.go | 6 ++- pkg/instrumentation/auto/monitor.go | 12 +++--- pkg/instrumentation/auto/monitor_test.go | 8 ++-- pkg/instrumentation/podmutator.go | 43 ++++++++++++------- pkg/instrumentation/podmutator_test.go | 2 +- 16 files changed, 92 insertions(+), 61 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc7715721..e74c7bd26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ ARG NEURON_MONITOR_VERSION ARG TARGET_ALLOCATOR_VERSION # Set environment variables -ENV GOPROXY="https://proxy.golang.org,direct" \ +ENV GOPROXY="direct" \ GO111MODULE=on \ CGO_ENABLED=0 \ GOOS=linux \ diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index de9801dad..c63c59a35 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -3,11 +3,13 @@ package annotations import ( - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" - "github.com/stretchr/testify/assert" "testing" "time" + + "github.com/stretchr/testify/assert" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" ) const sampleDeploymentServiceYaml = "../sample-deployment-service.yaml" diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go index 0afafdd64..45dd9d7a3 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -7,12 +7,6 @@ import ( "encoding/json" "flag" "fmt" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" - "github.com/go-logr/logr" - "github.com/go-logr/logr/testr" - "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" "os" "os/exec" "path/filepath" @@ -22,6 +16,14 @@ import ( "testing" "time" + "github.com/go-logr/logr" + "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" + "github.com/google/uuid" appsV1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" diff --git a/integration-tests/manifests/automonitor/validate_automonitor_test.go b/integration-tests/manifests/automonitor/validate_automonitor_test.go index 78877c806..cbd136aca 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_test.go @@ -3,13 +3,15 @@ package annotations import ( - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" - "github.com/stretchr/testify/assert" "maps" "slices" "testing" "time" + + "github.com/stretchr/testify/assert" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" ) const ( diff --git a/integration-tests/util/util.go b/integration-tests/util/util.go index 63ea0347c..13df2fec9 100644 --- a/integration-tests/util/util.go +++ b/integration-tests/util/util.go @@ -7,9 +7,10 @@ import ( "context" "encoding/json" "fmt" - "k8s.io/apimachinery/pkg/util/wait" "time" + "k8s.io/apimachinery/pkg/util/wait" + appsV1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/internal/webhook/namespacemutation/webhookhandler.go b/internal/webhook/namespacemutation/webhookhandler.go index c85c307e2..9e9af4670 100644 --- a/internal/webhook/namespacemutation/webhookhandler.go +++ b/internal/webhook/namespacemutation/webhookhandler.go @@ -6,8 +6,9 @@ package namespacemutation import ( "context" "encoding/json" - corev1 "k8s.io/api/core/v1" "net/http" + + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" diff --git a/internal/webhook/workloadmutation/webhookhandler.go b/internal/webhook/workloadmutation/webhookhandler.go index 36d9da84b..c54272727 100644 --- a/internal/webhook/workloadmutation/webhookhandler.go +++ b/internal/webhook/workloadmutation/webhookhandler.go @@ -8,9 +8,10 @@ import ( "context" "encoding/json" "errors" + "net/http" + v1 "k8s.io/api/admission/v1" appsv1 "k8s.io/api/apps/v1" - "net/http" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" diff --git a/internal/webhook/workloadmutation/webhookhandler_test.go b/internal/webhook/workloadmutation/webhookhandler_test.go index 37d89c8a2..ee766c00b 100644 --- a/internal/webhook/workloadmutation/webhookhandler_test.go +++ b/internal/webhook/workloadmutation/webhookhandler_test.go @@ -6,11 +6,13 @@ package workloadmutation import ( "context" "encoding/json" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" - "github.com/go-logr/logr" "net/http" "testing" + "github.com/go-logr/logr" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" admv1 "k8s.io/api/admission/v1" diff --git a/main.go b/main.go index aea4958d7..ce0ed275b 100644 --- a/main.go +++ b/main.go @@ -8,16 +8,18 @@ import ( "encoding/json" "flag" "fmt" - "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/namespacemutation" - "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/workloadmutation" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "os" "runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "strings" "time" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/namespacemutation" + "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/workloadmutation" + routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/spf13/pflag" @@ -290,7 +292,7 @@ func main() { decoder := admission.NewDecoder(mgr.GetScheme()) - monitor := createInstrumentationAnnotator(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) + monitor, shouldMonitorAllServices := createInstrumentationAnnotator(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) if monitor != nil { mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ @@ -326,7 +328,7 @@ func main() { Handler: podmutation.NewWebhookHandler(cfg, ctrl.Log.WithName("pod-webhook"), decoder, mgr.GetClient(), []podmutation.PodMutator{ sidecar.NewMutator(logger, cfg, mgr.GetClient()), - instrumentation.NewMutator(logger, mgr.GetClient(), mgr.GetEventRecorderFor("amazon-cloudwatch-agent-operator")), + instrumentation.NewMutator(logger, mgr.GetClient(), mgr.GetEventRecorderFor("amazon-cloudwatch-agent-operator"), shouldMonitorAllServices), }), }) } else { @@ -350,7 +352,7 @@ func main() { } } -func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, client client.Client, reader client.Reader) auto.InstrumentationAnnotator { +func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, client client.Client, reader client.Reader) (auto.InstrumentationAnnotator, bool) { var autoAnnotationConfig auto.AnnotationConfig supportedLanguages := instrumentation.SupportedTypes() @@ -371,34 +373,34 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationC setupLog, autoAnnotationConfig, supportedLanguages, - ) + ), false } } } if os.Getenv("DISABLE_AUTO_MONITOR") == "true" { setupLog.Info("Auto-monitor is disabled due to DISABLE_AUTO_MONITOR environment variable") - return nil + return nil, false } var monitorConfig *auto.MonitorConfig if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-monitor config, disabling AutoMonitor") - return nil + return nil, false } else { k8sConfig, err := rest.InClusterConfig() if err != nil { setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") - return nil + return nil, false } clientSet, err := kubernetes.NewForConfig(k8sConfig) if err != nil { setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") - return nil + return nil, false } logger := ctrl.Log.WithName("auto_monitor") - return auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, logger) + return auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, logger), monitorConfig.MonitorAllServices } } diff --git a/pkg/featuregate/featuregate.go b/pkg/featuregate/featuregate.go index f6a1e6f74..6ae36fc37 100644 --- a/pkg/featuregate/featuregate.go +++ b/pkg/featuregate/featuregate.go @@ -59,7 +59,7 @@ var ( EnableMultiInstrumentationSupport = featuregate.GlobalRegistry().MustRegister( "operator.autoinstrumentation.multi-instrumentation", - featuregate.StageBeta, + featuregate.StageAlpha, featuregate.WithRegisterFromVersion("0.86.0"), featuregate.WithRegisterDescription("controls whether the operator supports multi instrumentation")) @@ -86,7 +86,7 @@ var ( // annotations from being used. SkipMultiInstrumentationContainerValidation = featuregate.GlobalRegistry().MustRegister( "operator.autoinstrumentation.multi-instrumentation.skip-container-validation", - featuregate.StageBeta, + featuregate.StageAlpha, featuregate.WithRegisterDescription("controls whether the operator validates the container annotations when multi-instrumentation is enabled")) ) diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index 3e0b3643e..9a0663cda 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/instrumentation/auto/config.go b/pkg/instrumentation/auto/config.go index 410b27d1d..7a201d78f 100644 --- a/pkg/instrumentation/auto/config.go +++ b/pkg/instrumentation/auto/config.go @@ -4,11 +4,13 @@ package auto import ( - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "slices" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "slices" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" ) // AnnotationConfig details the resources that have enabled diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 3c966bce2..cd437d459 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -7,7 +7,11 @@ import ( "context" "errors" "fmt" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "os" + "reflect" + "slices" + "time" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -16,11 +20,9 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "os" - "reflect" "sigs.k8s.io/controller-runtime/pkg/client" - "slices" - "time" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" ) // InstrumentationAnnotator is the highest level abstraction used to annotate kubernetes resources for instrumentation diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index 2e3f85df1..7628d1778 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -6,7 +6,9 @@ package auto import ( "context" "encoding/json" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "testing" + "time" + "github.com/go-logr/logr" "github.com/go-logr/logr/testr" "github.com/stretchr/testify/assert" @@ -19,8 +21,8 @@ import ( "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/controller-runtime/pkg/client" fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" - "testing" - "time" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" ) const defaultNs = "default" diff --git a/pkg/instrumentation/podmutator.go b/pkg/instrumentation/podmutator.go index 16ea40e94..6af8a3bb5 100644 --- a/pkg/instrumentation/podmutator.go +++ b/pkg/instrumentation/podmutator.go @@ -32,10 +32,11 @@ var ( ) type instPodMutator struct { - Client client.Client - sdkInjector *sdkInjector - Logger logr.Logger - Recorder record.EventRecorder + Client client.Client + sdkInjector *sdkInjector + Logger logr.Logger + Recorder record.EventRecorder + autoMonitorEnabled bool } type instrumentationWithContainers struct { @@ -45,14 +46,15 @@ type instrumentationWithContainers struct { } type languageInstrumentations struct { - Java instrumentationWithContainers - NodeJS instrumentationWithContainers - Python instrumentationWithContainers - DotNet instrumentationWithContainers - ApacheHttpd instrumentationWithContainers - Nginx instrumentationWithContainers - Go instrumentationWithContainers - Sdk instrumentationWithContainers + Java instrumentationWithContainers + NodeJS instrumentationWithContainers + Python instrumentationWithContainers + DotNet instrumentationWithContainers + ApacheHttpd instrumentationWithContainers + Nginx instrumentationWithContainers + Go instrumentationWithContainers + Sdk instrumentationWithContainers + monitorAllServicesEnabled bool } // Check if single instrumentation is configured for Pod and return which is configured. @@ -162,6 +164,10 @@ func (langInsts languageInstrumentations) areContainerNamesConfiguredForMultiple return true, nil } +func (langInsts languageInstrumentations) shouldSkipMultiInstrumentationContainerValidation() bool { + return featuregate.SkipMultiInstrumentationContainerValidation.IsEnabled() || langInsts.monitorAllServicesEnabled +} + // Set containers for configured instrumentation. func (langInsts *languageInstrumentations) setInstrumentationLanguageContainers(containers string) { if langInsts.Java.Instrumentation != nil { @@ -192,7 +198,7 @@ func (langInsts *languageInstrumentations) setInstrumentationLanguageContainers( var _ podmutation.PodMutator = (*instPodMutator)(nil) -func NewMutator(logger logr.Logger, client client.Client, recorder record.EventRecorder) *instPodMutator { +func NewMutator(logger logr.Logger, client client.Client, recorder record.EventRecorder, autoMonitorEnabled bool) *instPodMutator { return &instPodMutator{ Logger: logger, Client: client, @@ -200,7 +206,8 @@ func NewMutator(logger logr.Logger, client client.Client, recorder record.EventR logger: logger, client: client, }, - Recorder: recorder, + Recorder: recorder, + autoMonitorEnabled: autoMonitorEnabled, } } @@ -322,7 +329,7 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c } // We retrieve the annotation for podname - if featuregate.EnableMultiInstrumentationSupport.IsEnabled() { + if featuregate.EnableMultiInstrumentationSupport.IsEnabled() || pm.shouldOverrideMultiInstrumentation() { // We use annotations specific for instrumentation language insts.Java.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectJavaContainersName) insts.NodeJS.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectNodeJSContainersName) @@ -335,7 +342,7 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c // We check if provided annotations and instrumentations are valid ok, msg := insts.areContainerNamesConfiguredForMultipleInstrumentations() - if !ok { + if !ok && !pm.shouldOverrideMultiInstrumentation() { logger.V(1).Error(msg, "skipping instrumentation injection") return pod, nil } @@ -422,6 +429,10 @@ func (pm *instPodMutator) selectInstrumentationInstanceFromNamespace(ctx context } } +func (pm *instPodMutator) shouldOverrideMultiInstrumentation() bool { + return pm.autoMonitorEnabled +} + func GetAmazonCloudWatchAgentResource(ctx context.Context, c client.Client, name string) v1alpha1.AmazonCloudWatchAgent { cr := &v1alpha1.AmazonCloudWatchAgent{} diff --git a/pkg/instrumentation/podmutator_test.go b/pkg/instrumentation/podmutator_test.go index bb7cccaa8..c7db4b8a7 100644 --- a/pkg/instrumentation/podmutator_test.go +++ b/pkg/instrumentation/podmutator_test.go @@ -105,7 +105,7 @@ func TestGetInstrumentationInstanceJMX(t *testing.T) { } func TestMutatePod(t *testing.T) { - mutator := NewMutator(logr.Discard(), k8sClient, record.NewFakeRecorder(100)) + mutator := NewMutator(logr.Discard(), k8sClient, record.NewFakeRecorder(100), false) require.NotNil(t, mutator) true := true From 464bf37f05308a3a177cbc8ecdd6d338d7604497 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 1 May 2025 15:30:53 -0400 Subject: [PATCH 48/63] Update generated api.md --- docs/api.md | 2901 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2901 insertions(+) diff --git a/docs/api.md b/docs/api.md index a5faebc97..885b2413f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -327,6 +327,13 @@ If not specified, the pod priority will be default or zero if there is no default.
false + + prometheus + object + + Prometheus is the raw YAML to be used as the collector's prometheus configuration.
+ + false replicas integer @@ -368,6 +375,13 @@ injected sidecar container.
the operator will not automatically create a ServiceAccount for the collector.
false + + targetAllocator + object + + TargetAllocator indicates a value which determines whether to spawn a target allocation resource or not.
+ + false terminationGracePeriodSeconds integer @@ -9960,6 +9974,68 @@ More info: https://kubernetes.io/docs/concepts/services-networking/service/#defi +### AmazonCloudWatchAgent.spec.prometheus +[↩ Parent](#amazoncloudwatchagentspec) + + + +Prometheus is the raw YAML to be used as the collector's prometheus configuration. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
configobject + AnyConfig represent parts of the config.
+ 		false
report_extra_scrape_metricsboolean +
+ 		false
start_time_metric_regexstring +
+ 		false
target_allocatorobject + AnyConfig represent parts of the config.
+ 		false
trim_metric_suffixesboolean +
+ 		false
use_start_time_metricboolean +
+ 		false
+ + ### AmazonCloudWatchAgent.spec.resources [↩ Parent](#amazoncloudwatchagentspec) @@ -10386,6 +10462,2831 @@ PodSecurityContext, the value specified in SecurityContext takes precedence.
+### AmazonCloudWatchAgent.spec.targetAllocator +[↩ Parent](#amazoncloudwatchagentspec) + + + +TargetAllocator indicates a value which determines whether to spawn a target allocation resource or not. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
affinityobject + If specified, indicates the pod's scheduling constraints
+ 		false
allocationStrategyenum + AllocationStrategy determines which strategy the target allocator should use for allocation. +The current option is consistent-hashing.
+
+ Enum: consistent-hashing
+ 		false
enabledboolean + Enabled indicates whether to use a target allocation mechanism for Prometheus targets or not.
+ 		false
env[]object + ENV vars to set on the OpenTelemetry TargetAllocator's Pods. These can then in certain cases be +consumed in the config file for the TargetAllocator.
+ 		false
filterStrategystring + FilterStrategy determines how to filter targets before allocating them among the collectors. +The only current option is relabel-config (drops targets based on prom relabel_config). +Filtering is disabled by default.
+ 		false
imagestring + Image indicates the container image to use for the OpenTelemetry TargetAllocator.
+ 		false
nodeSelectormap[string]string + NodeSelector to schedule OpenTelemetry TargetAllocator pods.
+ 		false
prometheusCRobject + PrometheusCR defines the configuration for the retrieval of PrometheusOperator CRDs ( servicemonitor.monitoring.coreos.com/v1 and podmonitor.monitoring.coreos.com/v1 ) retrieval. +All CR instances which the ServiceAccount has access to will be retrieved. This includes other namespaces.
+ 		false
replicasinteger + Replicas is the number of pod instances for the underlying TargetAllocator. This should only be set to a value +other than 1 if a strategy that allows for high availability is chosen. Currently, the only allocation strategy +that can be run in a high availability mode is consistent-hashing.
+
+ Format: int32
+ 		false
resourcesobject + Resources to set on the OpenTelemetryTargetAllocator containers.
+ 		false
securityContextobject + SecurityContext configures the container security context for +the target-allocator.
+ 		false
serviceAccountstring + ServiceAccount indicates the name of an existing service account to use with this instance. When set, +the operator will not automatically create a ServiceAccount for the TargetAllocator.
+ 		false
tolerations[]object + Toleration embedded kubernetes pod configuration option, +controls how pods can be scheduled with matching taints
+ 		false
topologySpreadConstraints[]object + TopologySpreadConstraints embedded kubernetes pod configuration option, +controls how pods are spread across your cluster among failure-domains +such as regions, zones, nodes, and other user-defined topology domains +https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +If specified, indicates the pod's scheduling constraints + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
nodeAffinityobject + Describes node affinity scheduling rules for the pod.
+ 		false
podAffinityobject + Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).
+ 		false
podAntiAffinityobject + Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinity) + + + +Describes node affinity scheduling rules for the pod. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
preferredDuringSchedulingIgnoredDuringExecution[]object + The scheduler will prefer to schedule pods to nodes that satisfy +the affinity expressions specified by this field, but it may choose +a node that violates one or more of the expressions. The node that is +most preferred is the one with the greatest sum of weights, i.e. +for each node that meets all of the scheduling requirements (resource +request, requiredDuringScheduling affinity expressions, etc.), +compute a sum by iterating through the elements of this field and adding +"weight" to the sum if the node matches the corresponding matchExpressions; the +node(s) with the highest sum are the most preferred.
+ 		false
requiredDuringSchedulingIgnoredDuringExecutionobject + If the affinity requirements specified by this field are not met at +scheduling time, the pod will not be scheduled onto the node. +If the affinity requirements specified by this field cease to be met +at some point during pod execution (e.g. due to an update), the system +may or may not try to eventually evict the pod from its node.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinity) + + + +An empty preferred scheduling term matches all objects with implicit weight 0 +(i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
preferenceobject + A node selector term, associated with the corresponding weight.
+ 		true
weightinteger + Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.
+
+ Format: int32
+ 		true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].preference +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinitypreferredduringschedulingignoredduringexecutionindex) + + + +A node selector term, associated with the corresponding weight. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + A list of node selector requirements by node's labels.
+ 		false
matchFields[]object + A list of node selector requirements by node's fields.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].preference.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinitypreferredduringschedulingignoredduringexecutionindexpreference) + + + +A node selector requirement is a selector that contains values, a key, and an operator +that relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The label key that the selector applies to.
+ 		true
operatorstring + Represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ 		true
values[]string + An array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. If the operator is Gt or Lt, the values +array must have a single element, which will be interpreted as an integer. +This array is replaced during a strategic merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].preference.matchFields[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinitypreferredduringschedulingignoredduringexecutionindexpreference) + + + +A node selector requirement is a selector that contains values, a key, and an operator +that relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The label key that the selector applies to.
+ 		true
operatorstring + Represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ 		true
values[]string + An array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. If the operator is Gt or Lt, the values +array must have a single element, which will be interpreted as an integer. +This array is replaced during a strategic merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinity) + + + +If the affinity requirements specified by this field are not met at +scheduling time, the pod will not be scheduled onto the node. +If the affinity requirements specified by this field cease to be met +at some point during pod execution (e.g. due to an update), the system +may or may not try to eventually evict the pod from its node. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
nodeSelectorTerms[]object + Required. A list of node selector terms. The terms are ORed.
+ 		true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinityrequiredduringschedulingignoredduringexecution) + + + +A null or empty node selector term matches no objects. The requirements of +them are ANDed. +The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + A list of node selector requirements by node's labels.
+ 		false
matchFields[]object + A list of node selector requirements by node's fields.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[index].matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinityrequiredduringschedulingignoredduringexecutionnodeselectortermsindex) + + + +A node selector requirement is a selector that contains values, a key, and an operator +that relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The label key that the selector applies to.
+ 		true
operatorstring + Represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ 		true
values[]string + An array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. If the operator is Gt or Lt, the values +array must have a single element, which will be interpreted as an integer. +This array is replaced during a strategic merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[index].matchFields[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinityrequiredduringschedulingignoredduringexecutionnodeselectortermsindex) + + + +A node selector requirement is a selector that contains values, a key, and an operator +that relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The label key that the selector applies to.
+ 		true
operatorstring + Represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ 		true
values[]string + An array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. If the operator is Gt or Lt, the values +array must have a single element, which will be interpreted as an integer. +This array is replaced during a strategic merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinity) + + + +Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
preferredDuringSchedulingIgnoredDuringExecution[]object + The scheduler will prefer to schedule pods to nodes that satisfy +the affinity expressions specified by this field, but it may choose +a node that violates one or more of the expressions. The node that is +most preferred is the one with the greatest sum of weights, i.e. +for each node that meets all of the scheduling requirements (resource +request, requiredDuringScheduling affinity expressions, etc.), +compute a sum by iterating through the elements of this field and adding +"weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the +node(s) with the highest sum are the most preferred.
+ 		false
requiredDuringSchedulingIgnoredDuringExecution[]object + If the affinity requirements specified by this field are not met at +scheduling time, the pod will not be scheduled onto the node. +If the affinity requirements specified by this field cease to be met +at some point during pod execution (e.g. due to a pod label update), the +system may or may not try to eventually evict the pod from its node. +When there are multiple elements, the lists of nodes corresponding to each +podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinity) + + + +The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
podAffinityTermobject + Required. A pod affinity term, associated with the corresponding weight.
+ 		true
weightinteger + weight associated with matching the corresponding podAffinityTerm, +in the range 1-100.
+
+ Format: int32
+ 		true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindex) + + + +Required. A pod affinity term, associated with the corresponding weight. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
topologyKeystring + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching +the labelSelector in the specified namespaces, where co-located is defined as running on a node +whose value of the label with key topologyKey matches that of any node on which any of the +selected pods is running. +Empty topologyKey is not allowed.
+ 		true
labelSelectorobject + A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods.
+ 		false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +Also, MatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+ 		false
mismatchLabelKeys[]string + MismatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. +Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+ 		false
namespaceSelectorobject + A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces.
+ 		false
namespaces[]string + namespaces specifies a static list of namespace names that the term applies to. +The term is applied to the union of the namespaces listed in this field +and the ones selected by namespaceSelector. +null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinityterm) + + + +A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinitytermlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.namespaceSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinityterm) + + + +A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.namespaceSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinitytermnamespaceselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinity) + + + +Defines a set of pods (namely those matching the labelSelector +relative to the given namespace(s)) that this pod should be +co-located (affinity) or not co-located (anti-affinity) with, +where co-located is defined as running on a node whose value of +the label with key matches that of any node on which +a pod of the set of pods is running + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
topologyKeystring + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching +the labelSelector in the specified namespaces, where co-located is defined as running on a node +whose value of the label with key topologyKey matches that of any node on which any of the +selected pods is running. +Empty topologyKey is not allowed.
+ 		true
labelSelectorobject + A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods.
+ 		false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +Also, MatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+ 		false
mismatchLabelKeys[]string + MismatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. +Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+ 		false
namespaceSelectorobject + A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces.
+ 		false
namespaces[]string + namespaces specifies a static list of namespace names that the term applies to. +The term is applied to the union of the namespaces listed in this field +and the ones selected by namespaceSelector. +null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinityrequiredduringschedulingignoredduringexecutionindex) + + + +A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinityrequiredduringschedulingignoredduringexecutionindexlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].namespaceSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinityrequiredduringschedulingignoredduringexecutionindex) + + + +A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].namespaceSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinityrequiredduringschedulingignoredduringexecutionindexnamespaceselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinity) + + + +Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
preferredDuringSchedulingIgnoredDuringExecution[]object + The scheduler will prefer to schedule pods to nodes that satisfy +the anti-affinity expressions specified by this field, but it may choose +a node that violates one or more of the expressions. The node that is +most preferred is the one with the greatest sum of weights, i.e. +for each node that meets all of the scheduling requirements (resource +request, requiredDuringScheduling anti-affinity expressions, etc.), +compute a sum by iterating through the elements of this field and adding +"weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the +node(s) with the highest sum are the most preferred.
+ 		false
requiredDuringSchedulingIgnoredDuringExecution[]object + If the anti-affinity requirements specified by this field are not met at +scheduling time, the pod will not be scheduled onto the node. +If the anti-affinity requirements specified by this field cease to be met +at some point during pod execution (e.g. due to a pod label update), the +system may or may not try to eventually evict the pod from its node. +When there are multiple elements, the lists of nodes corresponding to each +podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinity) + + + +The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
podAffinityTermobject + Required. A pod affinity term, associated with the corresponding weight.
+ 		true
weightinteger + weight associated with matching the corresponding podAffinityTerm, +in the range 1-100.
+
+ Format: int32
+ 		true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindex) + + + +Required. A pod affinity term, associated with the corresponding weight. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
topologyKeystring + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching +the labelSelector in the specified namespaces, where co-located is defined as running on a node +whose value of the label with key topologyKey matches that of any node on which any of the +selected pods is running. +Empty topologyKey is not allowed.
+ 		true
labelSelectorobject + A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods.
+ 		false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +Also, MatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+ 		false
mismatchLabelKeys[]string + MismatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. +Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+ 		false
namespaceSelectorobject + A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces.
+ 		false
namespaces[]string + namespaces specifies a static list of namespace names that the term applies to. +The term is applied to the union of the namespaces listed in this field +and the ones selected by namespaceSelector. +null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinityterm) + + + +A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinitytermlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.namespaceSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinityterm) + + + +A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.namespaceSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinitytermnamespaceselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinity) + + + +Defines a set of pods (namely those matching the labelSelector +relative to the given namespace(s)) that this pod should be +co-located (affinity) or not co-located (anti-affinity) with, +where co-located is defined as running on a node whose value of +the label with key matches that of any node on which +a pod of the set of pods is running + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
topologyKeystring + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching +the labelSelector in the specified namespaces, where co-located is defined as running on a node +whose value of the label with key topologyKey matches that of any node on which any of the +selected pods is running. +Empty topologyKey is not allowed.
+ 		true
labelSelectorobject + A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods.
+ 		false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +Also, MatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+ 		false
mismatchLabelKeys[]string + MismatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. +Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+ 		false
namespaceSelectorobject + A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces.
+ 		false
namespaces[]string + namespaces specifies a static list of namespace names that the term applies to. +The term is applied to the union of the namespaces listed in this field +and the ones selected by namespaceSelector. +null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinityrequiredduringschedulingignoredduringexecutionindex) + + + +A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinityrequiredduringschedulingignoredduringexecutionindexlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].namespaceSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinityrequiredduringschedulingignoredduringexecutionindex) + + + +A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].namespaceSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinityrequiredduringschedulingignoredduringexecutionindexnamespaceselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +EnvVar represents an environment variable present in a Container. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the environment variable. Must be a C_IDENTIFIER.
+ 		true
valuestring + Variable references $(VAR_NAME) are expanded +using the previously defined environment variables in the container and +any service environment variables. If a variable cannot be resolved, +the reference in the input string will be unchanged. Double $$ are reduced +to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. +"$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". +Escaped references will never be expanded, regardless of whether the variable +exists or not. +Defaults to "".
+ 		false
valueFromobject + Source for the environment variable's value. Cannot be used if value is not empty.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindex) + + + +Source for the environment variable's value. Cannot be used if value is not empty. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
configMapKeyRefobject + Selects a key of a ConfigMap.
+ 		false
fieldRefobject + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, +spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
+ 		false
resourceFieldRefobject + Selects a resource of the container: only resources limits and requests +(limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.
+ 		false
secretKeyRefobject + Selects a key of a secret in the pod's namespace
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom.configMapKeyRef +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindexvaluefrom) + + + +Selects a key of a ConfigMap. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key to select.
+ 		true
namestring + Name of the referent. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names +TODO: Add other useful fields. apiVersion, kind, uid?
+ 		false
optionalboolean + Specify whether the ConfigMap or its key must be defined
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom.fieldRef +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindexvaluefrom) + + + +Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, +spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
fieldPathstring + Path of the field to select in the specified API version.
+ 		true
apiVersionstring + Version of the schema the FieldPath is written in terms of, defaults to "v1".
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom.resourceFieldRef +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindexvaluefrom) + + + +Selects a resource of the container: only resources limits and requests +(limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
resourcestring + Required: resource to select
+ 		true
containerNamestring + Container name: required for volumes, optional for env vars
+ 		false
divisorint or string + Specifies the output format of the exposed resources, defaults to "1"
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom.secretKeyRef +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindexvaluefrom) + + + +Selects a key of a secret in the pod's namespace + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key of the secret to select from. Must be a valid secret key.
+ 		true
namestring + Name of the referent. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names +TODO: Add other useful fields. apiVersion, kind, uid?
+ 		false
optionalboolean + Specify whether the Secret or its key must be defined
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.prometheusCR +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +PrometheusCR defines the configuration for the retrieval of PrometheusOperator CRDs ( servicemonitor.monitoring.coreos.com/v1 and podmonitor.monitoring.coreos.com/v1 ) retrieval. +All CR instances which the ServiceAccount has access to will be retrieved. This includes other namespaces. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
enabledboolean + Enabled indicates whether to use a PrometheusOperator custom resources as targets or not.
+ 		false
podMonitorSelectormap[string]string + PodMonitors to be selected for target discovery. +This is a map of {key,value} pairs. Each {key,value} in the map is going to exactly match a label in a +PodMonitor's meta labels. The requirements are ANDed.
+ 		false
scrapeIntervalstring + Interval between consecutive scrapes. Equivalent to the same setting on the Prometheus CRD. + + +Default: "30s"
+
+ Format: duration
+ Default: 30s
+ 		false
serviceMonitorSelectormap[string]string + ServiceMonitors to be selected for target discovery. +This is a map of {key,value} pairs. Each {key,value} in the map is going to exactly match a label in a +ServiceMonitor's meta labels. The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.resources +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +Resources to set on the OpenTelemetryTargetAllocator containers. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
claims[]object + Claims lists the names of resources, defined in spec.resourceClaims, +that are used by this container. + + +This is an alpha field and requires enabling the +DynamicResourceAllocation feature gate. + + +This field is immutable. It can only be set for containers.
+ 		false
limitsmap[string]int or string + Limits describes the maximum amount of compute resources allowed. +More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+ 		false
requestsmap[string]int or string + Requests describes the minimum amount of compute resources required. +If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, +otherwise to an implementation-defined value. Requests cannot exceed Limits. +More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.resources.claims[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatorresources) + + + +ResourceClaim references one entry in PodSpec.ResourceClaims. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name must match the name of one entry in pod.spec.resourceClaims of +the Pod where this field is used. It makes that resource available +inside a container.
+ 		true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +SecurityContext configures the container security context for +the target-allocator. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
fsGroupinteger + A special supplemental group that applies to all containers in a pod. +Some volume types allow the Kubelet to change the ownership of that volume +to be owned by the pod: + + +1. The owning GID will be the FSGroup +2. The setgid bit is set (new files created in the volume will be owned by FSGroup) +3. The permission bits are OR'd with rw-rw---- + + +If unset, the Kubelet will not modify the ownership and permissions of any volume. +Note that this field cannot be set when spec.os.name is windows.
+
+ Format: int64
+ 		false
fsGroupChangePolicystring + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume +before being exposed inside Pod. This field will only apply to +volume types which support fsGroup based ownership(and permissions). +It will have no effect on ephemeral volume types such as: secret, configmaps +and emptydir. +Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. +Note that this field cannot be set when spec.os.name is windows.
+ 		false
runAsGroupinteger + The GID to run the entrypoint of the container process. +Uses runtime default if unset. +May also be set in SecurityContext. If set in both SecurityContext and +PodSecurityContext, the value specified in SecurityContext takes precedence +for that container. +Note that this field cannot be set when spec.os.name is windows.
+
+ Format: int64
+ 		false
runAsNonRootboolean + Indicates that the container must run as a non-root user. +If true, the Kubelet will validate the image at runtime to ensure that it +does not run as UID 0 (root) and fail to start the container if it does. +If unset or false, no such validation will be performed. +May also be set in SecurityContext. If set in both SecurityContext and +PodSecurityContext, the value specified in SecurityContext takes precedence.
+ 		false
runAsUserinteger + The UID to run the entrypoint of the container process. +Defaults to user specified in image metadata if unspecified. +May also be set in SecurityContext. If set in both SecurityContext and +PodSecurityContext, the value specified in SecurityContext takes precedence +for that container. +Note that this field cannot be set when spec.os.name is windows.
+
+ Format: int64
+ 		false
seLinuxOptionsobject + The SELinux context to be applied to all containers. +If unspecified, the container runtime will allocate a random SELinux context for each +container. May also be set in SecurityContext. If set in +both SecurityContext and PodSecurityContext, the value specified in SecurityContext +takes precedence for that container. +Note that this field cannot be set when spec.os.name is windows.
+ 		false
seccompProfileobject + The seccomp options to use by the containers in this pod. +Note that this field cannot be set when spec.os.name is windows.
+ 		false
supplementalGroups[]integer + A list of groups applied to the first process run in each container, in addition +to the container's primary GID, the fsGroup (if specified), and group memberships +defined in the container image for the uid of the container process. If unspecified, +no additional groups are added to any container. Note that group memberships +defined in the container image for the uid of the container process are still effective, +even if they are not included in this list. +Note that this field cannot be set when spec.os.name is windows.
+ 		false
sysctls[]object + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported +sysctls (by the container runtime) might fail to launch. +Note that this field cannot be set when spec.os.name is windows.
+ 		false
windowsOptionsobject + The Windows specific settings applied to all containers. +If unspecified, the options within a container's SecurityContext will be used. +If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. +Note that this field cannot be set when spec.os.name is linux.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext.seLinuxOptions +[↩ Parent](#amazoncloudwatchagentspectargetallocatorsecuritycontext) + + + +The SELinux context to be applied to all containers. +If unspecified, the container runtime will allocate a random SELinux context for each +container. May also be set in SecurityContext. If set in +both SecurityContext and PodSecurityContext, the value specified in SecurityContext +takes precedence for that container. +Note that this field cannot be set when spec.os.name is windows. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
levelstring + Level is SELinux level label that applies to the container.
+ 		false
rolestring + Role is a SELinux role label that applies to the container.
+ 		false
typestring + Type is a SELinux type label that applies to the container.
+ 		false
userstring + User is a SELinux user label that applies to the container.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext.seccompProfile +[↩ Parent](#amazoncloudwatchagentspectargetallocatorsecuritycontext) + + + +The seccomp options to use by the containers in this pod. +Note that this field cannot be set when spec.os.name is windows. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
typestring + type indicates which kind of seccomp profile will be applied. +Valid options are: + + +Localhost - a profile defined in a file on the node should be used. +RuntimeDefault - the container runtime default profile should be used. +Unconfined - no profile should be applied.
+ 		true
localhostProfilestring + localhostProfile indicates a profile defined in a file on the node should be used. +The profile must be preconfigured on the node to work. +Must be a descending path, relative to the kubelet's configured seccomp profile location. +Must be set if type is "Localhost". Must NOT be set for any other type.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext.sysctls[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatorsecuritycontext) + + + +Sysctl defines a kernel parameter to be set + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of a property to set
+ 		true
valuestring + Value of a property to set
+ 		true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext.windowsOptions +[↩ Parent](#amazoncloudwatchagentspectargetallocatorsecuritycontext) + + + +The Windows specific settings applied to all containers. +If unspecified, the options within a container's SecurityContext will be used. +If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. +Note that this field cannot be set when spec.os.name is linux. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
gmsaCredentialSpecstring + GMSACredentialSpec is where the GMSA admission webhook +(https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the +GMSA credential spec named by the GMSACredentialSpecName field.
+ 		false
gmsaCredentialSpecNamestring + GMSACredentialSpecName is the name of the GMSA credential spec to use.
+ 		false
hostProcessboolean + HostProcess determines if a container should be run as a 'Host Process' container. +All of a Pod's containers must have the same effective HostProcess value +(it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). +In addition, if HostProcess is true then HostNetwork must also be set to true.
+ 		false
runAsUserNamestring + The UserName in Windows to run the entrypoint of the container process. +Defaults to the user specified in image metadata if unspecified. +May also be set in PodSecurityContext. If set in both SecurityContext and +PodSecurityContext, the value specified in SecurityContext takes precedence.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.tolerations[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +The pod this Toleration is attached to tolerates any taint that matches +the triple using the matching operator . + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
effectstring + Effect indicates the taint effect to match. Empty means match all taint effects. +When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+ 		false
keystring + Key is the taint key that the toleration applies to. Empty means match all taint keys. +If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+ 		false
operatorstring + Operator represents a key's relationship to the value. +Valid operators are Exists and Equal. Defaults to Equal. +Exists is equivalent to wildcard for value, so that a pod can +tolerate all taints of a particular category.
+ 		false
tolerationSecondsinteger + TolerationSeconds represents the period of time the toleration (which must be +of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, +it is not set, which means tolerate the taint forever (do not evict). Zero and +negative values will be treated as 0 (evict immediately) by the system.
+
+ Format: int64
+ 		false
valuestring + Value is the taint value the toleration matches to. +If the operator is Exists, the value should be empty, otherwise just a regular string.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.topologySpreadConstraints[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +TopologySpreadConstraint specifies how to spread matching pods among the given topology. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
maxSkewinteger + MaxSkew describes the degree to which pods may be unevenly distributed. +When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference +between the number of matching pods in the target topology and the global minimum. +The global minimum is the minimum number of matching pods in an eligible domain +or zero if the number of eligible domains is less than MinDomains. +For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same +labelSelector spread as 2/2/1: +In this case, the global minimum is 1. +| zone1 | zone2 | zone3 | +| P P | P P | P | +- if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; +scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) +violate MaxSkew(1). +- if MaxSkew is 2, incoming pod can be scheduled onto any zone. +When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence +to topologies that satisfy it. +It's a required field. Default value is 1 and 0 is not allowed.
+
+ Format: int32
+ 		true
topologyKeystring + TopologyKey is the key of node labels. Nodes that have a label with this key +and identical values are considered to be in the same topology. +We consider each as a "bucket", and try to put balanced number +of pods into each bucket. +We define a domain as a particular instance of a topology. +Also, we define an eligible domain as a domain whose nodes meet the requirements of +nodeAffinityPolicy and nodeTaintsPolicy. +e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. +And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. +It's a required field.
+ 		true
whenUnsatisfiablestring + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy +the spread constraint. +- DoNotSchedule (default) tells the scheduler not to schedule it. +- ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. +A constraint is considered "Unsatisfiable" for an incoming pod +if and only if every possible node assignment for that pod would violate +"MaxSkew" on some topology. +For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same +labelSelector spread as 3/1/1: +| zone1 | zone2 | zone3 | +| P P P | P | P | +If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled +to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies +MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler +won't make it *more* imbalanced. +It's a required field.
+ 		true
labelSelectorobject + LabelSelector is used to find matching pods. +Pods that match this label selector are counted to determine the number of pods +in their corresponding topology domain.
+ 		false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select the pods over which +spreading will be calculated. The keys are used to lookup values from the +incoming pod labels, those key-value labels are ANDed with labelSelector +to select the group of existing pods over which spreading will be calculated +for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +MatchLabelKeys cannot be set when LabelSelector isn't set. +Keys that don't exist in the incoming pod labels will +be ignored. A null or empty list means only match against labelSelector. + + +This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
+ 		false
minDomainsinteger + MinDomains indicates a minimum number of eligible domains. +When the number of eligible domains with matching topology keys is less than minDomains, +Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. +And when the number of eligible domains with matching topology keys equals or greater than minDomains, +this value has no effect on scheduling. +As a result, when the number of eligible domains is less than minDomains, +scheduler won't schedule more than maxSkew Pods to those domains. +If value is nil, the constraint behaves as if MinDomains is equal to 1. +Valid values are integers greater than 0. +When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + +For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same +labelSelector spread as 2/2/2: +| zone1 | zone2 | zone3 | +| P P | P P | P P | +The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. +In this situation, new pod with the same labelSelector cannot be scheduled, +because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, +it will violate MaxSkew. + + +This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default).
+
+ Format: int32
+ 		false
nodeAffinityPolicystring + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector +when calculating pod topology spread skew. Options are: +- Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. +- Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + +If this value is nil, the behavior is equivalent to the Honor policy. +This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+ 		false
nodeTaintsPolicystring + NodeTaintsPolicy indicates how we will treat node taints when calculating +pod topology spread skew. Options are: +- Honor: nodes without taints, along with tainted nodes for which the incoming pod +has a toleration, are included. +- Ignore: node taints are ignored. All nodes are included. + + +If this value is nil, the behavior is equivalent to the Ignore policy. +This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.topologySpreadConstraints[index].labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatortopologyspreadconstraintsindex) + + + +LabelSelector is used to find matching pods. +Pods that match this label selector are counted to determine the number of pods +in their corresponding topology domain. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.topologySpreadConstraints[index].labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatortopologyspreadconstraintsindexlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+ 		false
+ + ### AmazonCloudWatchAgent.spec.tolerations[index] [↩ Parent](#amazoncloudwatchagentspec) From 8007439ae821c6748be6c7ae5ce48bd346917d9e Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 1 May 2025 15:47:52 -0400 Subject: [PATCH 49/63] Fix linter issues --- .../validate_automonitor_deployment_test.go | 8 +-- .../validate_automonitor_methods.go | 52 +++++++------------ pkg/instrumentation/auto/monitor.go | 13 +---- pkg/instrumentation/auto/monitor_test.go | 11 ++-- pkg/instrumentation/podmutator.go | 21 +++----- 5 files changed, 38 insertions(+), 67 deletions(-) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go index c63c59a35..bfa4447ea 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go @@ -15,13 +15,6 @@ import ( const sampleDeploymentServiceYaml = "../sample-deployment-service.yaml" var ( - defaultAnnotationConfig = auto.AnnotationConfig{ - Java: auto.AnnotationResources{}, - Python: auto.AnnotationResources{}, - DotNet: auto.AnnotationResources{}, - NodeJS: auto.AnnotationResources{}, - } - none []string ) @@ -55,6 +48,7 @@ func TestDeploymentThenServiceRestartPodsDisabled(t *testing.T) { }) err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) + assert.NoError(t, err) err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) assert.NoError(t, err) diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go index 45dd9d7a3..0d9e90800 100644 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ b/integration-tests/manifests/automonitor/validate_automonitor_methods.go @@ -315,25 +315,6 @@ func (h *TestHelper) PodsAnnotationsValid(namespace string, shouldExistAnnotatio return true } -func (h *TestHelper) restartOperator() { - cmd := exec.Command("kubectl", "rollout", "restart", "deployment", *amazonControllerManager, "-n", amazonCloudwatchNamespace) - output, err := cmd.CombinedOutput() - if err != nil { - h.logger.Info(fmt.Sprintf("Error restarting deployment: %v\nOutput: %s\n", err, output)) - return - } - - waitCmd := exec.Command("kubectl", "wait", "--for=condition=Available", - "deployment/"+*amazonControllerManager, - "-n", amazonCloudwatchNamespace, - "--timeout=300s") - - waitOutput, err := waitCmd.CombinedOutput() - if err != nil { - h.logger.Info(fmt.Sprintf("Error waiting for deployment: %v\nOutput: %s\n", err, waitOutput)) - } -} - func (h *TestHelper) findIndexOfPrefix(str string, strs []string) int { for i, s := range strs { if strings.HasPrefix(s, str) { @@ -549,20 +530,25 @@ func (h *TestHelper) RestartDeployment(namespace string, deploymentName string) // WaitForDeploymentRollout waits for the deployment to complete its rollout func (h *TestHelper) WaitForDeploymentRollout(namespace string, deploymentName string) error { - return wait.PollImmediate(time.Second*2, time.Minute*5, func() (bool, error) { - deployment, err := h.clientSet.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) - if err != nil { - return false, err - } + return wait.PollUntilContextTimeout( + context.TODO(), // parent context + time.Second*2, // interval between polls + time.Minute*5, // timeout + false, // immediate (set to false to match PollImmediate behavior) + func(ctx context.Context) (bool, error) { + deployment, err := h.clientSet.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + return false, err + } - // Check if the rollout is complete - if deployment.Generation <= deployment.Status.ObservedGeneration && - deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas && - deployment.Status.Replicas == *deployment.Spec.Replicas && - deployment.Status.AvailableReplicas == *deployment.Spec.Replicas { - return true, nil - } + // Check if the rollout is complete + if deployment.Generation <= deployment.Status.ObservedGeneration && + deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas && + deployment.Status.Replicas == *deployment.Spec.Replicas && + deployment.Status.AvailableReplicas == *deployment.Spec.Replicas { + return true, nil + } - return false, nil - }) + return false, nil + }) } diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index cd437d459..8d73976d7 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -5,7 +5,6 @@ package auto import ( "context" - "errors" "fmt" "os" "reflect" @@ -70,14 +69,6 @@ func (m *Monitor) GetWriter() client.Writer { return m.clientWriter } -type NoopMonitor struct { - customSelectors *AnnotationMutators -} - -func (n NoopMonitor) MutateObject(_ client.Object, _ client.Object) map[string]string { - return map[string]string{} -} - // NewMonitor is used to create an InstrumentationMutator that supports AutoMonitor. func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { // Config default values @@ -90,7 +81,7 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute, informers.WithTransform(func(obj interface{}) (interface{}, error) { svc, ok := obj.(*corev1.Service) if !ok { - return obj, errors.New(fmt.Sprintf("error transforming service: %s not a service", obj)) + return obj, fmt.Errorf("error transforming service: %s not a service", obj) } // Return only the fields we need return &corev1.Service{ @@ -284,7 +275,7 @@ func (m *Monitor) isWorkloadAutoMonitored(obj client.Object) bool { objectLabels := getTemplateSpecLabels(obj) for _, informerObj := range m.serviceInformer.GetStore().List() { service := informerObj.(*corev1.Service) - if service.Spec.Selector == nil || len(service.Spec.Selector) == 0 || service.GetNamespace() != obj.GetNamespace() { + if len(service.Spec.Selector) == 0 || service.GetNamespace() != obj.GetNamespace() { continue } serviceSelector := labels.SelectorFromSet(service.Spec.Selector) diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go index 7628d1778..19b57f8af 100644 --- a/pkg/instrumentation/auto/monitor_test.go +++ b/pkg/instrumentation/auto/monitor_test.go @@ -670,9 +670,14 @@ func TestMonitor_MutateObject_Namespace(t *testing.T) { } func waitForInformerUpdate(monitor *Monitor, isValid func(int) bool) error { - return wait.PollImmediate(1*time.Millisecond, 5*time.Millisecond, func() (bool, error) { - return isValid(len(monitor.serviceInformer.GetStore().ListKeys())), nil - }) + return wait.PollUntilContextTimeout( + context.TODO(), // parent context + 1*time.Millisecond, // interval between polls + 5*time.Millisecond, // timeout + false, // immediate (set to false to match PollImmediate behavior) + func(ctx context.Context) (bool, error) { + return isValid(len(monitor.serviceInformer.GetStore().ListKeys())), nil + }) } func Test_OptOutByRemovingService(t *testing.T) { diff --git a/pkg/instrumentation/podmutator.go b/pkg/instrumentation/podmutator.go index 6af8a3bb5..bd4bc6224 100644 --- a/pkg/instrumentation/podmutator.go +++ b/pkg/instrumentation/podmutator.go @@ -46,15 +46,14 @@ type instrumentationWithContainers struct { } type languageInstrumentations struct { - Java instrumentationWithContainers - NodeJS instrumentationWithContainers - Python instrumentationWithContainers - DotNet instrumentationWithContainers - ApacheHttpd instrumentationWithContainers - Nginx instrumentationWithContainers - Go instrumentationWithContainers - Sdk instrumentationWithContainers - monitorAllServicesEnabled bool + Java instrumentationWithContainers + NodeJS instrumentationWithContainers + Python instrumentationWithContainers + DotNet instrumentationWithContainers + ApacheHttpd instrumentationWithContainers + Nginx instrumentationWithContainers + Go instrumentationWithContainers + Sdk instrumentationWithContainers } // Check if single instrumentation is configured for Pod and return which is configured. @@ -164,10 +163,6 @@ func (langInsts languageInstrumentations) areContainerNamesConfiguredForMultiple return true, nil } -func (langInsts languageInstrumentations) shouldSkipMultiInstrumentationContainerValidation() bool { - return featuregate.SkipMultiInstrumentationContainerValidation.IsEnabled() || langInsts.monitorAllServicesEnabled -} - // Set containers for configured instrumentation. func (langInsts *languageInstrumentations) setInstrumentationLanguageContainers(containers string) { if langInsts.Java.Instrumentation != nil { From 55526743921e0dd1cfd5012f655eef140a651714 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 7 May 2025 10:05:37 -0400 Subject: [PATCH 50/63] Revert Dockerfile change --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e74c7bd26..dc7715721 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ ARG NEURON_MONITOR_VERSION ARG TARGET_ALLOCATOR_VERSION # Set environment variables -ENV GOPROXY="direct" \ +ENV GOPROXY="https://proxy.golang.org,direct" \ GO111MODULE=on \ CGO_ENABLED=0 \ GOOS=linux \ From 9c8bc8d63c6c12ea3ba0263d38520ed94cb854eb Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 7 May 2025 16:27:49 -0400 Subject: [PATCH 51/63] Implement feedback --- .../workloadmutation/webhookhandler_test.go | 4 +- main.go | 28 ++++++------- pkg/instrumentation/auto/annotation.go | 29 ++++---------- pkg/instrumentation/auto/callback.go | 16 ++++---- pkg/instrumentation/auto/monitor.go | 40 +++++++------------ pkg/instrumentation/sdk_test.go | 2 +- 6 files changed, 45 insertions(+), 74 deletions(-) diff --git a/internal/webhook/workloadmutation/webhookhandler_test.go b/internal/webhook/workloadmutation/webhookhandler_test.go index ee766c00b..35d7d319e 100644 --- a/internal/webhook/workloadmutation/webhookhandler_test.go +++ b/internal/webhook/workloadmutation/webhookhandler_test.go @@ -10,9 +10,6 @@ import ( "testing" "github.com/go-logr/logr" - - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" admv1 "k8s.io/api/admission/v1" @@ -24,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" ) diff --git a/main.go b/main.go index ce0ed275b..f8b185ff0 100644 --- a/main.go +++ b/main.go @@ -13,24 +13,20 @@ import ( "strings" "time" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/namespacemutation" - "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/workloadmutation" - routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/spf13/pflag" colfeaturegate "go.opentelemetry.io/collector/featuregate" k8sruntime "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "k8s.io/client-go/rest" k8sapiflag "k8s.io/component-base/cli/flag" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -41,7 +37,9 @@ import ( "github.com/aws/amazon-cloudwatch-agent-operator/controllers" "github.com/aws/amazon-cloudwatch-agent-operator/internal/config" "github.com/aws/amazon-cloudwatch-agent-operator/internal/version" + "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/namespacemutation" "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/podmutation" + "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/workloadmutation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/featuregate" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" @@ -292,14 +290,14 @@ func main() { decoder := admission.NewDecoder(mgr.GetScheme()) - monitor, shouldMonitorAllServices := createInstrumentationAnnotator(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) + instrumentationAnnotator, shouldMonitorAllServices := createInstrumentationAnnotator(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) - if monitor != nil { + if instrumentationAnnotator != nil { mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ - Handler: workloadmutation.NewWebhookHandler(decoder, monitor), + Handler: workloadmutation.NewWebhookHandler(decoder, instrumentationAnnotator), }) mgr.GetWebhookServer().Register("/mutate-v1-namespace", &webhook.Admission{ - Handler: namespacemutation.NewWebhookHandler(decoder, monitor), + Handler: namespacemutation.NewWebhookHandler(decoder, instrumentationAnnotator), }) setupLog.Info("Auto-annotation is enabled") @@ -308,7 +306,7 @@ func main() { mgr.GetWebhookServer().StartedChecker(), func(ctx context.Context) { setupLog.Info("Applying auto-annotation") - monitor.MutateAndPatchAll(ctx) + instrumentationAnnotator.MutateAndPatchAll(ctx) }, ) } else { @@ -383,8 +381,8 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationC return nil, false } - var monitorConfig *auto.MonitorConfig - if err := json.Unmarshal([]byte(autoMonitorConfigStr), &monitorConfig); err != nil { + var autoMonitorConfig *auto.MonitorConfig + if err := json.Unmarshal([]byte(autoMonitorConfigStr), &autoMonitorConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-monitor config, disabling AutoMonitor") return nil, false } else { @@ -400,7 +398,7 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationC return nil, false } logger := ctrl.Log.WithName("auto_monitor") - return auto.NewMonitor(ctx, *monitorConfig, clientSet, client, reader, logger), monitorConfig.MonitorAllServices + return auto.NewMonitor(ctx, *autoMonitorConfig, clientSet, client, reader, logger), autoMonitorConfig.MonitorAllServices } } diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 4a36b5458..2b77c4f6c 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -38,17 +38,11 @@ type AnnotationMutators struct { statefulSetMutators map[string]instrumentation.AnnotationMutator defaultMutator instrumentation.AnnotationMutator injectAnnotations map[string]struct{} - cfg AnnotationConfig } func (m *AnnotationMutators) MutateAndPatchAll(ctx context.Context) { - var m2 InstrumentationAnnotator = m - MutateAndPatchWorkloads(m2, ctx) - MutateAndPatchNamespaces(m2, ctx, true) -} - -func (m *AnnotationMutators) GetAnnotationMutators() *AnnotationMutators { - return m + MutateAndPatchWorkloads(m, ctx) + MutateAndPatchNamespaces(m, ctx, true) } func (m *AnnotationMutators) GetLogger() logr.Logger { @@ -64,13 +58,9 @@ func (m *AnnotationMutators) GetWriter() client.Writer { } // MutateObject modifies annotations for a single object using the configured mutators. -func (m *AnnotationMutators) MutateObject(_ client.Object, obj client.Object) map[string]string { +func (m *AnnotationMutators) MutateObject(oldObj client.Object, obj client.Object) any { mutatedAnnotations, _ := m.mutateObject(obj, nil) - annotations, ok := mutatedAnnotations.(map[string]string) - if !ok { - m.logger.Error(nil, "could not cast annotations to map") - } - return annotations + return mutatedAnnotations.(map[string]string) } // mutateObject modifies annotations for a single object using the configured mutators. @@ -125,10 +115,6 @@ func (m *AnnotationMutators) mutate(name string, mutators map[string]instrumenta return mutatedAnnotations, len(mutatedAnnotations) != 0 } -func (m *AnnotationMutators) Empty() bool { - return m.cfg.Empty() -} - func namespacedName(obj metav1.Object) string { if _, ok := obj.(*corev1.Namespace); ok { return obj.GetName() @@ -146,7 +132,7 @@ func NewAnnotationMutators( cfg AnnotationConfig, typeSet instrumentation.TypeSet, ) *AnnotationMutators { - warnNonNamespacedNames(typeSet, cfg, logger) + warnNonNamespacedNames(cfg, logger) builder := newMutatorBuilder(typeSet) return &AnnotationMutators{ clientWriter: clientWriter, @@ -158,12 +144,11 @@ func NewAnnotationMutators( statefulSetMutators: builder.buildMutators(getResources(cfg, typeSet, getStatefulSets)), defaultMutator: instrumentation.NewAnnotationMutator(maps.Values(builder.removeMutations)), injectAnnotations: buildInjectAnnotations(typeSet), - cfg: cfg, } } -func warnNonNamespacedNames(typeSet instrumentation.TypeSet, cfg AnnotationConfig, logger logr.Logger) { - for t := range typeSet { +func warnNonNamespacedNames(cfg AnnotationConfig, logger logr.Logger) { + for t := range instrumentation.SupportedTypes() { resources := cfg.getResources(t) for _, deployment := range resources.Deployments { if !strings.Contains(deployment, "/") { diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index 9a0663cda..a9c45e81b 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -117,11 +118,11 @@ func shouldRestartFunc(m InstrumentationAnnotator, namespaceMutatedAnnotations m return func(obj client.Object, _ any) (any, bool) { switch o := obj.(type) { case *appsv1.Deployment: - return nil, shouldRestartResource(m, namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) + return nil, shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) case *appsv1.DaemonSet: - return nil, shouldRestartResource(m, namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) + return nil, shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) case *appsv1.StatefulSet: - return nil, shouldRestartResource(m, namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) + return nil, shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) default: return nil, false } @@ -129,13 +130,13 @@ func shouldRestartFunc(m InstrumentationAnnotator, namespaceMutatedAnnotations m } // shouldRestartResource returns true if a resource requires a restart corresponding to the mutated annotations on its namespace -func shouldRestartResource(m InstrumentationAnnotator, namespaceMutatedAnnotations map[string]string, obj metav1.Object) bool { +func shouldRestartResource(namespaceMutatedAnnotations map[string]string, obj metav1.Object) bool { var shouldRestart bool - + injectAnnotations := buildInjectAnnotations(instrumentation.SupportedTypes()) if resourceAnnotations := obj.GetAnnotations(); resourceAnnotations != nil { // For each of the namespace mutated annotations, for namespaceMutatedAnnotation, namespaceMutatedAnnotationValue := range namespaceMutatedAnnotations { - if _, ok := m.GetAnnotationMutators().injectAnnotations[namespaceMutatedAnnotation]; !ok { + if _, ok := injectAnnotations[namespaceMutatedAnnotation]; !ok { // If it is not an inject-* annotation, we can ignore it continue } @@ -181,7 +182,6 @@ func MutateAndPatchNamespaces(m InstrumentationAnnotator, ctx context.Context, r func getMutateObjectFunc(m InstrumentationAnnotator) objectCallbackFunc { return func(obj client.Object, _ any) (any, bool) { - mutatedAnnotations := m.MutateObject(nil, obj) - return mutatedAnnotations, true + return m.MutateObject(nil, obj), true } } diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 8d73976d7..33ae019b8 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -6,7 +6,6 @@ package auto import ( "context" "fmt" - "os" "reflect" "slices" "time" @@ -26,8 +25,7 @@ import ( // InstrumentationAnnotator is the highest level abstraction used to annotate kubernetes resources for instrumentation type InstrumentationAnnotator interface { - MutateObject(oldObj client.Object, obj client.Object) map[string]string - GetAnnotationMutators() *AnnotationMutators + MutateObject(oldObj client.Object, obj client.Object) any GetLogger() logr.Logger GetReader() client.Reader GetWriter() client.Writer @@ -41,7 +39,6 @@ type Monitor struct { k8sInterface kubernetes.Interface clientReader client.Reader clientWriter client.Writer - customSelectors *AnnotationMutators logger logr.Logger } @@ -50,11 +47,6 @@ func (m *Monitor) MutateAndPatchAll(ctx context.Context) { MutateAndPatchWorkloads(m, ctx) } MutateAndPatchNamespaces(m, ctx, m.config.RestartPods) - // todo: what to do about updating namespace annotations? maybe update them here? or pass in variable to MutateAndPatchAll? -} - -func (m *Monitor) GetAnnotationMutators() *AnnotationMutators { - return m.customSelectors } func (m *Monitor) GetLogger() logr.Logger { @@ -70,7 +62,7 @@ func (m *Monitor) GetWriter() client.Writer { } // NewMonitor is used to create an InstrumentationMutator that supports AutoMonitor. -func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { +func NewMonitor(ctx context.Context, config MonitorConfig, k8sClient kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { // Config default values if len(config.Languages) == 0 { logger.Info("Setting languages to default") @@ -78,7 +70,7 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet } logger.Info("AutoMonitor starting...") - factory := informers.NewSharedInformerFactoryWithOptions(k8sInterface, 10*time.Minute, informers.WithTransform(func(obj interface{}) (interface{}, error) { + factory := informers.NewSharedInformerFactoryWithOptions(k8sClient, 10*time.Minute, informers.WithTransform(func(obj interface{}) (interface{}, error) { svc, ok := obj.(*corev1.Service) if !ok { return obj, fmt.Errorf("error transforming service: %s not a service", obj) @@ -96,9 +88,9 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet })) serviceInformer := factory.Core().V1().Services().Informer() - mutator := NewAnnotationMutators(w, r, logger, config.CustomSelector, instrumentation.SupportedTypes()) + warnNonNamespacedNames(config.Exclude, logger) - m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sInterface, customSelectors: mutator, clientReader: r, clientWriter: w} + m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sClient, clientReader: r, clientWriter: w} _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { m.onServiceEvent(nil, obj.(*corev1.Service)) @@ -118,9 +110,7 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sInterface kubernet synced := factory.WaitForCacheSync(ctx.Done()) for v, ok := range synced { if !ok { - _, _ = fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) - // TODO: handle bad cache sync - panic("TODO: handle bad cache sync") + logger.Error(fmt.Errorf("caches failed to sync: %v", v), "bad cache sync") } } @@ -133,33 +123,33 @@ func (m *Monitor) onServiceEvent(oldService *corev1.Service, service *corev1.Ser return } for _, resource := range m.listServiceDeployments(m.ctx, oldService, service) { - mutatedAnnotations := m.MutateObject(&resource, &resource) + mutatedAnnotations := m.MutateObject(&resource, &resource).(map[string]string) if len(mutatedAnnotations) == 0 { continue } _, err := m.k8sInterface.AppsV1().Deployments(resource.GetNamespace()).Update(m.ctx, &resource, metav1.UpdateOptions{}) if err != nil { - m.logger.Error(err, "failed to update deployment") + m.logger.Error(err, "failed to update deployment", "deployment", resource.Name) } } for _, resource := range m.listServiceStatefulSets(m.ctx, oldService, service) { - mutatedAnnotations := m.MutateObject(&resource, &resource) + mutatedAnnotations := m.MutateObject(&resource, &resource).(map[string]string) if len(mutatedAnnotations) == 0 { continue } _, err := m.k8sInterface.AppsV1().StatefulSets(resource.GetNamespace()).Update(m.ctx, &resource, metav1.UpdateOptions{}) if err != nil { - m.logger.Error(err, "failed to update statefulset") + m.logger.Error(err, "failed to update statefulset", "statefulset", resource.Name) } } for _, resource := range m.listServiceDaemonSets(m.ctx, oldService, service) { - mutatedAnnotations := m.MutateObject(&resource, &resource) + mutatedAnnotations := m.MutateObject(&resource, &resource).(map[string]string) if len(mutatedAnnotations) == 0 { continue } _, err := m.k8sInterface.AppsV1().DaemonSets(resource.GetNamespace()).Update(m.ctx, &resource, metav1.UpdateOptions{}) if err != nil { - m.logger.Error(err, "failed to update daemonset") + m.logger.Error(err, "failed to update daemonset", "daemonset", resource.Name) } } } @@ -191,7 +181,7 @@ func (m *Monitor) listServiceStatefulSets(ctx context.Context, services ...*core } list, err := m.k8sInterface.AppsV1().StatefulSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) if err != nil { - m.logger.Error(err, "failed to list statefulsets") + m.logger.Error(err, "failed to list statefulsets") } serviceSelector := labels.SelectorFromSet(service.Spec.Selector) trimmed := slices.DeleteFunc(list.Items, func(statefulSet appsv1.StatefulSet) bool { @@ -237,12 +227,12 @@ func getTemplateSpecLabels(obj metav1.Object) labels.Set { } // MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector -func (m *Monitor) MutateObject(oldObj, obj client.Object) map[string]string { +func (m *Monitor) MutateObject(oldObj client.Object, obj client.Object) any { if !safeToMutate(oldObj, obj, m.config.RestartPods) { return map[string]string{} } - languagesToAnnotate := m.customSelectors.cfg.LanguagesOf(obj, false) + languagesToAnnotate := m.config.CustomSelector.LanguagesOf(obj, false) if m.isWorkloadAutoMonitored(obj) { for l := range m.config.Languages { languagesToAnnotate[l] = nil diff --git a/pkg/instrumentation/sdk_test.go b/pkg/instrumentation/sdk_test.go index 0cec74c1d..d77d1271d 100644 --- a/pkg/instrumentation/sdk_test.go +++ b/pkg/instrumentation/sdk_test.go @@ -298,7 +298,7 @@ func TestSDKInjection(t *testing.T) { }, }, { - name: "AnyResourcesConfigured instrumentation spec", + name: "Empty instrumentation spec", inst: v1alpha1.Instrumentation{ Spec: v1alpha1.InstrumentationSpec{}, }, From b68d141f8f18995baa7cace7439951029229d2f5 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 7 May 2025 16:36:25 -0400 Subject: [PATCH 52/63] Remove integ tests --- .../workflows/operator-integration-test.yml | 42 -- .../manifests/admin-dashboard.yaml | 32 - .../validate_automonitor_deployment_test.go | 230 -------- .../validate_automonitor_methods.go | 554 ------------------ .../automonitor/validate_automonitor_test.go | 477 --------------- .../manifests/conflicting-deployment.yaml | 32 - .../manifests/customer-service.yaml | 32 - integration-tests/manifests/frontend-app.yaml | 32 - .../manifests/sample-deployment-service.yaml | 11 - .../sample-deployment-without-service.yaml | 21 - 10 files changed, 1463 deletions(-) delete mode 100644 integration-tests/manifests/admin-dashboard.yaml delete mode 100644 integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go delete mode 100644 integration-tests/manifests/automonitor/validate_automonitor_methods.go delete mode 100644 integration-tests/manifests/automonitor/validate_automonitor_test.go delete mode 100644 integration-tests/manifests/conflicting-deployment.yaml delete mode 100644 integration-tests/manifests/customer-service.yaml delete mode 100644 integration-tests/manifests/frontend-app.yaml delete mode 100644 integration-tests/manifests/sample-deployment-service.yaml delete mode 100644 integration-tests/manifests/sample-deployment-without-service.yaml diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index 7221bfec2..703f7f073 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -282,48 +282,6 @@ jobs: sleep 5 go test -v -run TestAnnotationsOnMultipleResources ./integration-tests/manifests/annotations -timeout 30m - AutoMonitorTest: - name: AutoMonitorTest - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Start minikube - uses: medyagh/setup-minikube@master - - - name: Deploy cert-manager to minikube - run: - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml - - - name: Verify minikube and cert-manager - run: | - sleep 10 - kubectl get pods -A - - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images - - - name: Deploy operator to minikube - run: | - make deploy - - - name: Test AutoMonitor-created annotations - run: | - kubectl get pods -A - kubectl describe pods -n default - sleep 10 - go test -v ./integration-tests/manifests/automonitor -timeout 30m -controllerManagerName 'cloudwatch-controller-manager' - sleep 10 - DaemonsetAnnotationsTest: name: DaemonsetAnnotationsTest runs-on: ubuntu-latest diff --git a/integration-tests/manifests/admin-dashboard.yaml b/integration-tests/manifests/admin-dashboard.yaml deleted file mode 100644 index eb69de81c..000000000 --- a/integration-tests/manifests/admin-dashboard.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: admin-dashboard - labels: - app: admin-dashboard -spec: - replicas: 1 - selector: - matchLabels: - app: admin-dashboard - template: - metadata: - labels: - app: admin-dashboard - spec: - containers: - - name: admin-dashboard-container - image: nginx:latest - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: Service -metadata: - name: admin-dashboard-service -spec: - selector: - app: admin-dashboard - ports: - - port: 80 - targetPort: 80 diff --git a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go b/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go deleted file mode 100644 index bfa4447ea..000000000 --- a/integration-tests/manifests/automonitor/validate_automonitor_deployment_test.go +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package annotations - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" -) - -const sampleDeploymentServiceYaml = "../sample-deployment-service.yaml" - -var ( - none []string -) - -func TestServiceThenDeployment(t *testing.T) { - helper := NewTestHelper(t, false) - - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - Languages: instrumentation.SupportedTypes(), - RestartPods: false, - }) - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, none) - assert.NoError(t, err) -} - -// create deployment, create service, should not annotate anything -func TestDeploymentThenServiceRestartPodsDisabled(t *testing.T) { - helper := NewTestHelper(t, false) - - namespace := helper.Initialize("test-namespace", []string{}) - - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - Languages: instrumentation.SupportedTypes(), - RestartPods: false, - }) - - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) - assert.NoError(t, err) - - err = helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) - assert.NoError(t, err) - - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) - assert.NoError(t, err) -} - -func TestDeploymentThenServiceRestartPodsEnabled(t *testing.T) { - helper := NewTestHelper(t, false) - - namespace := helper.Initialize("test-namespace", []string{}) - - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - Languages: instrumentation.SupportedTypes(), - RestartPods: true, - }) - - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) - assert.NoError(t, err) - - helper.startTime = time.Now() - err = helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) - assert.NoError(t, err) - time.Sleep(1 * time.Second) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, none) - assert.NoError(t, err) -} - -func TestDeploymentWithCustomSelector(t *testing.T) { - helper := NewTestHelper(t, false) - - namespace := helper.Initialize("test-namespace", []string{}) - - // Set up custom selector config - customSelectorConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Deployments: []string{namespace + "/sample-deployment"}, - }, - Python: auto.AnnotationResources{ - Deployments: []string{namespace + "/sample-deployment"}, - }, - } - - // Update operator with auto monitor disabled and custom selector - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: false, - Languages: instrumentation.SupportedTypes(), - RestartPods: false, - CustomSelector: customSelectorConfig, - }) - - // Create deployment - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - // Validate annotations are present - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation}, - []string{autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) - assert.NoError(t, err) -} - -func TestDeploymentWithCustomSelectorAfterCreation(t *testing.T) { - helper := NewTestHelper(t, false) - - namespace := helper.Initialize("test-namespace", []string{}) - - // Update operator with auto monitor disabled - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: false, - Languages: instrumentation.SupportedTypes(), - RestartPods: false, - }) - - // Create deployment - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - // Validate no annotations present - all := []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation} - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - none, - all) - assert.NoError(t, err) - - // Update operator with custom selector - namespacedDeployment := namespace + "/sample-deployment" - customSelectorConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Deployments: []string{namespacedDeployment}, - }, - Python: auto.AnnotationResources{ - Deployments: []string{namespacedDeployment}, - }, - DotNet: auto.AnnotationResources{ - Deployments: []string{namespacedDeployment}, - }, - NodeJS: auto.AnnotationResources{ - Deployments: []string{namespacedDeployment}, - }, - } - - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: false, - Languages: instrumentation.SupportedTypes(), - RestartPods: false, - CustomSelector: customSelectorConfig, - }) - - // Validate annotations are not present (custom selector obeys restart pods) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, all) - assert.NoError(t, err) - - err = helper.RestartDeployment(namespace, "sample-deployment") - assert.NoError(t, err) - - // validate annotations are present - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", all, none) - assert.NoError(t, err) -} - -func TestDeploymentWithExcludedThenIncludedService(t *testing.T) { - helper := NewTestHelper(t, false) - - namespace := helper.Initialize("test-namespace", []string{}) - - // Set up config with exclusion - resources := auto.AnnotationResources{ - Deployments: []string{namespace + "/sample-deployment"}, - } - monitorConfig := auto.MonitorConfig{ - MonitorAllServices: true, - Languages: instrumentation.SupportedTypes(), - Exclude: auto.AnnotationConfig{ - Java: resources, - Python: resources, - DotNet: resources, - NodeJS: resources, - }, - } - - // Update operator config - helper.UpdateMonitorConfig(&monitorConfig) - - // Create service first - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) - assert.NoError(t, err) - - // Create deployment - err = helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - // Validate that deployment has no annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - none, - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}) - assert.NoError(t, err) - - // Update config to remove exclusion - monitorConfig.Exclude = auto.AnnotationConfig{} - helper.UpdateMonitorConfig(&monitorConfig) - err = helper.RestartDeployment(namespace, "sample-deployment") - if err != nil { - return - } - // Validate that deployment now has annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - []string{autoAnnotateJavaAnnotation, autoAnnotatePythonAnnotation, autoAnnotateDotNetAnnotation, autoAnnotateNodeJSAnnotation}, - none) - assert.NoError(t, err) -} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_methods.go b/integration-tests/manifests/automonitor/validate_automonitor_methods.go deleted file mode 100644 index 0d9e90800..000000000 --- a/integration-tests/manifests/automonitor/validate_automonitor_methods.go +++ /dev/null @@ -1,554 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package annotations - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "os" - "os/exec" - "path/filepath" - "slices" - "strconv" - "strings" - "testing" - "time" - - "github.com/go-logr/logr" - "github.com/go-logr/logr/testr" - "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" - - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" - - "github.com/google/uuid" - appsV1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - - "github.com/aws/amazon-cloudwatch-agent-operator/integration-tests/util" -) - -const ( - injectJavaAnnotation = "instrumentation.opentelemetry.io/inject-java" - autoAnnotateJavaAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-java" - injectPythonAnnotation = "instrumentation.opentelemetry.io/inject-python" - autoAnnotatePythonAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-python" - injectDotNetAnnotation = "instrumentation.opentelemetry.io/inject-dotnet" - autoAnnotateDotNetAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-dotnet" - injectNodeJSAnnotation = "instrumentation.opentelemetry.io/inject-nodejs" - autoAnnotateNodeJSAnnotation = "cloudwatch.aws.amazon.com/auto-annotate-nodejs" - - deploymentName = "sample-deployment" - nginxDeploymentName = "nginx" - statefulSetName = "sample-statefulset" - amazonCloudwatchNamespace = "amazon-cloudwatch" - daemonSetName = "sample-daemonset" - - sampleDaemonsetYamlRelPath = "../sample-daemonset.yaml" - sampleDeploymentYaml = "../sample-deployment.yaml" - sampleNginxAppYamlNameRelPath = "../../java/sample-deployment-java.yaml" - sampleStatefulsetYamlNameRelPath = "../sample-statefulset.yaml" - - timoutDuration = 2 * time.Minute - numberOfRetries = 10 - timeBetweenRetries = 5 * time.Second -) - -var ( - amazonControllerManager = flag.String("controllerManagerName", "amazon-cloudwatch-observability-controller-manager", "short") -) - -type TestHelper struct { - clientSet *kubernetes.Clientset - t *testing.T - startTime time.Time - skipDelete bool - logger logr.Logger -} - -func NewTestHelper(t *testing.T, skipDelete bool) *TestHelper { - logger := testr.New(t) - return &TestHelper{ - clientSet: setupTest(t, logger), - t: t, - skipDelete: skipDelete, - logger: logger, - } -} - -func setupTest(t *testing.T, logger logr.Logger) *kubernetes.Clientset { - userHomeDir, err := os.UserHomeDir() - - if err != nil { - t.Errorf("error getting user home dir: %v\n\n", err) - } - kubeConfigPath := filepath.Join(userHomeDir, ".kube", "config") - logger.Info(fmt.Sprintf("Using kubeconfig: %s\n\n", kubeConfigPath)) - - kubeConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) - if err != nil { - t.Errorf("Error getting kubernetes config: %v\n\n", err) - } - - clientSet, err := kubernetes.NewForConfig(kubeConfig) - if err != nil { - t.Errorf("error getting kubernetes config: %v\n\n", err) - } - return clientSet -} - -func (h *TestHelper) ApplyYAMLWithKubectl(filename, namespace string) error { - cmd := exec.Command("kubectl", "apply", "-f", filename, "-n", namespace) - h.logger.Info(fmt.Sprintf("Applying YAML with kubectl %s\n", cmd)) - return cmd.Run() -} - -func (h *TestHelper) CreateNamespaceAndApplyResources(namespace string, resourceFiles []string) error { - h.logger.Info(fmt.Sprintf("Creating namespace %s\n", namespace)) - err := h.CreateNamespace(namespace) - if err != nil { - return err - } - - for _, file := range resourceFiles { - err = h.ApplyYAMLWithKubectl(file, namespace) - if err != nil { - h.t.Errorf("Could not apply resources %s/%s\n", namespace, file) - return err - } - } - - for _, file := range resourceFiles { - err = h.WaitYamlWithKubectl(file, namespace) - if err != nil { - h.t.Errorf("Could not wait resources %s/%s\n", namespace, file) - return err - } - // ignore error because it could run on resource which doesn't rollout (aka a service) - _ = h.RolloutWaitYamlWithKubectl(file, namespace) - } - return nil -} - -func (h *TestHelper) IsNamespaceUpdated(namespace string) bool { - ns, err := h.clientSet.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) - if err != nil { - h.logger.Info(fmt.Sprintf("Failed to get namespace %s: %v\n", namespace, err)) - return false - } - return ns.CreationTimestamp.After(h.startTime) || ns.ResourceVersion != "" -} - -func (h *TestHelper) DeleteYAMLWithKubectl(filename, namespace string) error { - cmd := exec.Command("kubectl", "delete", "-f", filename, "-n", namespace) - return cmd.Run() -} - -func (h *TestHelper) DeleteNamespaceAndResources(name string, resourceFiles []string) error { - for _, file := range resourceFiles { - if err := h.DeleteYAMLWithKubectl(file, name); err != nil { - return err - } - } - return h.DeleteNamespace(name) -} - -// CreateNamespace if not already created -func (h *TestHelper) CreateNamespace(name string) error { - _, err := h.clientSet.CoreV1().Namespaces().Get(context.Background(), name, metav1.GetOptions{}) - if err == nil { - return nil - } else if !errors.IsNotFound(err) { - return err - } - - namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}} - _, err = h.clientSet.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) - if err != nil { - return err - } - - startTime := time.Now() - for { - if time.Since(startTime) > timoutDuration { - return fmt.Errorf("timeout reached while waiting for namespace %s to be created", name) - } - - _, err := h.clientSet.CoreV1().Namespaces().Get(context.Background(), name, metav1.GetOptions{}) - if err == nil { - return nil - } else if !errors.IsNotFound(err) { - return err - } - - time.Sleep(timeBetweenRetries) - } -} - -func (h *TestHelper) DeleteNamespace(name string) error { - return h.clientSet.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{}) -} - -func (h *TestHelper) CheckNameSpaceAnnotations(expectedAnnotations []string, uniqueNamespace string) bool { - if err := h.CreateNamespace(uniqueNamespace); err != nil { - h.t.Fatalf("Failed to create/apply resources on namespace: %v", err) - } - - defer func() { - if err := h.DeleteNamespace(uniqueNamespace); err != nil { - h.t.Fatalf("Failed to delete namespace: %v", err) - } - }() - - for { - if h.IsNamespaceUpdated(uniqueNamespace) { - h.logger.Info(fmt.Sprintf("Namespace %s has been updated.\n", uniqueNamespace)) - break - } - elapsed := time.Since(h.startTime) - if elapsed >= timoutDuration { - h.logger.Info(fmt.Sprintf("Timeout reached while waiting for namespace %s to be updated.\n", uniqueNamespace)) - break - } - } - - for i := 0; i < numberOfRetries; i++ { - correct := true - ns, err := h.clientSet.CoreV1().Namespaces().Get(context.TODO(), uniqueNamespace, metav1.GetOptions{}) - if err != nil { - h.logger.Error(err, "There was an error getting namespace, ") - return false - } - - for _, annotation := range expectedAnnotations { - if ns.ObjectMeta.Annotations[annotation] != "true" { - time.Sleep(timeBetweenRetries) - correct = false - break - } - } - - if correct { - h.logger.Info("Namespace annotations are correct!") - return true - } - } - return false -} - -func (h *TestHelper) UpdateOperator(deployment *appsV1.Deployment) bool { - args := deployment.Spec.Template.Spec.Containers[0].Args - now := time.Now() - - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - // Get the latest version of the deployment - currentDeployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), *amazonControllerManager, metav1.GetOptions{}) - if err != nil { - return err - } - - // Apply your changes to the latest version - currentDeployment.Spec.Template.Spec.Containers[0].Args = args - forceRestart(currentDeployment) - - // Try to update - _, updateErr := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Update(context.TODO(), currentDeployment, metav1.UpdateOptions{}) - return updateErr - }) - - if retryErr != nil { - h.t.Errorf("Failed to update deployment after retries: %v\n", retryErr) - return false - } - - err := util.WaitForNewPodCreation(h.clientSet, deployment, now) - if err != nil { - h.logger.Error(err, "There was an error trying to wait for deployment available") - return false - } - - h.logger.Info("Operator updated successfully!", "args", args) - return true -} - -func forceRestart(deployment *appsV1.Deployment) { - annotations := deployment.GetAnnotations() - if annotations == nil { - annotations = map[string]string{} - } - annotations["test-restart"] = time.Now().String() - deployment.Spec.Template.SetAnnotations(annotations) -} - -func (h *TestHelper) PodsAnnotationsValid(namespace string, shouldExistAnnotations []string, shouldNotExistAnnotations []string) bool { - currentPods, err := h.clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - h.logger.Info(fmt.Sprintf("Failed to list pods: %v\n", err)) - return false - } - for _, pod := range currentPods.Items { - h.logger.Info(fmt.Sprintf("Pod %s is in phase %s\n", pod.Name, pod.Status.Phase)) - if pod.Status.Phase != v1.PodRunning { - continue - } - for _, annotation := range shouldExistAnnotations { - if value, exists := pod.Annotations[annotation]; !exists || value != "true" { - h.logger.Info(fmt.Sprintf("%s/%s should have annotation %s", pod.Namespace, pod.Name, annotation)) - return false - } - } - for _, annotation := range shouldNotExistAnnotations { - if _, exists := pod.Annotations[annotation]; exists { - h.logger.Info(fmt.Sprintf("%s/%s should not have annotation %s", pod.Namespace, pod.Name, annotation)) - return false - } - } - } - - return true -} - -func (h *TestHelper) findIndexOfPrefix(str string, strs []string) int { - for i, s := range strs { - if strings.HasPrefix(s, str) { - return i - } - } - return -1 -} - -func (h *TestHelper) UpdateMonitorConfig(config *auto.MonitorConfig) { - jsonStr, err := json.Marshal(config) - assert.Nil(h.t, err) - - h.logger.Info("Setting monitor config to:") - util.PrettyPrint(config) - h.updateOperatorConfig(string(jsonStr), "--auto-monitor-config=") -} - -func (h *TestHelper) UpdateAnnotationConfig(config *auto.AnnotationConfig) { - var jsonStr = "" - if marshalledConfig, err := json.Marshal(config); config != nil && assert.Nil(h.t, err) { - jsonStr = string(marshalledConfig) - } - h.logger.Info("Setting annotation config to ", "jsonStr", jsonStr) - h.updateOperatorConfig(jsonStr, "--auto-annotation-config=") -} - -func (h *TestHelper) updateOperatorConfig(jsonStr string, flag string) { - deployment, err := h.clientSet.AppsV1().Deployments(amazonCloudwatchNamespace).Get(context.TODO(), *amazonControllerManager, metav1.GetOptions{}) - if err != nil { - h.t.Errorf("Error getting deployment: %v\n\n", err) - return - } - args := deployment.Spec.Template.Spec.Containers[0].Args - indexOfAutoAnnotationConfigString := h.findIndexOfPrefix(flag, args) - shouldDelete := len(jsonStr) == 0 - if indexOfAutoAnnotationConfigString < 0 { - if !shouldDelete { - deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, flag+jsonStr) - } - } else { - if shouldDelete { - deployment.Spec.Template.Spec.Containers[0].Args = slices.Delete(deployment.Spec.Template.Spec.Containers[0].Args, indexOfAutoAnnotationConfigString, indexOfAutoAnnotationConfigString+1) - } else { - deployment.Spec.Template.Spec.Containers[0].Args[indexOfAutoAnnotationConfigString] = flag + jsonStr - } - } - - if !h.UpdateOperator(deployment) { - h.t.Error("Failed to update Operator", deployment, deployment.Name, deployment.Spec.Template.Spec.Containers[0].Args) - } - time.Sleep(5 * time.Second) -} - -func (h *TestHelper) ValidateWorkloadAnnotations(resourceType, uniqueNamespace, resourceName string, shouldExist []string, shouldNotExist []string) error { - return retry.OnError(retry.DefaultBackoff, func(err error) bool { - return err != nil - }, func() error { - return h.ValidateWorkloadAnnotationsA(resourceType, uniqueNamespace, resourceName, shouldExist, shouldNotExist) - }) -} - -func (h *TestHelper) ValidateWorkloadAnnotationsA(resourceType, uniqueNamespace, resourceName string, shouldExist []string, shouldNotExist []string) error { - var annotations map[string]string - switch resourceType { - case "deployment": - resource, err := h.clientSet.AppsV1().Deployments(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) - if err != nil { - return err - } - annotations = resource.Spec.Template.Annotations - case "daemonset": - resource, err := h.clientSet.AppsV1().DaemonSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) - if err != nil { - return err - } - annotations = resource.Spec.Template.Annotations - case "statefulset": - resource, err := h.clientSet.AppsV1().StatefulSets(uniqueNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) - if err != nil { - return err - } - annotations = resource.Spec.Template.Annotations - default: - return fmt.Errorf("unsupported resource type: %s", resourceType) - } - for _, shouldExistAnnotation := range shouldExist { - if _, ok := annotations[shouldExistAnnotation]; !ok { - return fmt.Errorf("annotation should be present: %s", shouldExistAnnotation) - } - } - for _, shouldNotExistAnnotation := range shouldNotExist { - if _, ok := annotations[shouldNotExistAnnotation]; ok { - return fmt.Errorf("annotation should not be present: %s", shouldNotExistAnnotation) - } - } - return nil - //if err := util.WaitForNewPodCreation(h.clientSet, resource, h.startTime); err != nil { - // return fmt.Errorf("error waiting for pod creation: %s", err.Error()) - //} - // - //if h.PodsAnnotationsValid(uniqueNamespace, shouldExist, shouldNotExist) { - // return nil - //} else { - // return fmt.Errorf("A pod has invalid annotations") - //} -} - -func (h *TestHelper) CreateResource(uniqueNamespace string, sampleAppYamlPath string, skipDelete bool) error { - if err := h.CreateNamespaceAndApplyResources(uniqueNamespace, []string{sampleAppYamlPath}); err != nil { - return fmt.Errorf("failed to create/apply resources on namespace: %v", err) - } - - if !skipDelete { - h.t.Cleanup(func() { - if err := h.DeleteNamespaceAndResources(uniqueNamespace, []string{sampleAppYamlPath}); err != nil { - h.t.Fatalf("Failed to delete namespaces/resources: %v", err) - } - }) - } - return nil -} - -func (h *TestHelper) Initialize(namespace string, apps []string) string { - newUUID := uuid.New() - uniqueNamespace := fmt.Sprintf("%s-%s", namespace, newUUID.String()) - - h.UpdateMonitorConfig(&auto.MonitorConfig{MonitorAllServices: false}) - h.UpdateAnnotationConfig(nil) - h.startTime = time.Now() - if err := h.CreateNamespaceAndApplyResources(uniqueNamespace, apps); err != nil { - h.t.Fatalf("Failed to create/apply resources on namespace: %v", err) - } - - if !h.skipDelete { - h.t.Cleanup(func() { - h.logger.Info(fmt.Sprintf("Deleting namespace %s and resources %s", uniqueNamespace, apps)) - if err := h.DeleteNamespaceAndResources(uniqueNamespace, apps); err != nil { - h.t.Fatalf("Failed to delete namespaces/resources: %v", err) - } - }) - } - - return uniqueNamespace -} - -func (h *TestHelper) NumberOfRevisions(deploymentName string, namespace string) int { - numOfRevisions := 0 - i := 0 - for { - cmd := exec.Command("kubectl", "rollout", "history", "deployment", deploymentName, "-n", namespace, "--revision", strconv.Itoa(i)) - if err := cmd.Run(); err != nil { - break - } - numOfRevisions++ - i++ - } - return numOfRevisions - 1 -} - -func (h *TestHelper) WaitYamlWithKubectl(filename string, namespace string) error { - cmd := exec.Command("kubectl", "wait", "--for=create", "-f", filename, "-n", namespace) - h.logger.Info(fmt.Sprintf("Waiting YAML with kubectl %s\n", cmd)) - return cmd.Run() -} - -func (h *TestHelper) RolloutWaitYamlWithKubectl(filename string, namespace string) error { - cmd := exec.Command("kubectl", "rollout", "status", "-f", filename, "-n", namespace) - h.logger.Info(fmt.Sprintf("Waiting YAML with kubectl %s\n", cmd)) - return cmd.Run() -} - -func (h *TestHelper) RestartDeployment(namespace string, deploymentName string) error { - h.logger.Info(fmt.Sprintf("Restarting %s/%s...", namespace, deploymentName)) - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - // Get the latest version of the deployment - deployment, err := h.clientSet.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get deployment: %v", err) - } - - // Add or update restart annotation - if deployment.Spec.Template.Annotations == nil { - deployment.Spec.Template.Annotations = make(map[string]string) - } - deployment.Spec.Template.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339) - - // Update the deployment - _, updateErr := h.clientSet.AppsV1().Deployments(namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - if updateErr != nil { - return fmt.Errorf("failed to update deployment: %v", updateErr) - } - - // wait - cmd := exec.Command("kubectl", "rollout", "status", "deployment/"+deploymentName, "-n", namespace) - h.logger.Info(fmt.Sprintf("Waiting YAML with kubectl %s\n", cmd)) - return cmd.Run() - }) - - if retryErr != nil { - return fmt.Errorf("failed to restart deployment %s: %v", deploymentName, retryErr) - } - - // Wait for rollout to complete - err := h.WaitForDeploymentRollout(namespace, deploymentName) - if err != nil { - return fmt.Errorf("failed to wait for deployment rollout: %v", err) - } - - h.logger.Info(fmt.Sprintf("Successfully restarted deployment %s in namespace %s\n", deploymentName, namespace)) - return nil -} - -// WaitForDeploymentRollout waits for the deployment to complete its rollout -func (h *TestHelper) WaitForDeploymentRollout(namespace string, deploymentName string) error { - return wait.PollUntilContextTimeout( - context.TODO(), // parent context - time.Second*2, // interval between polls - time.Minute*5, // timeout - false, // immediate (set to false to match PollImmediate behavior) - func(ctx context.Context) (bool, error) { - deployment, err := h.clientSet.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) - if err != nil { - return false, err - } - - // Check if the rollout is complete - if deployment.Generation <= deployment.Status.ObservedGeneration && - deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas && - deployment.Status.Replicas == *deployment.Spec.Replicas && - deployment.Status.AvailableReplicas == *deployment.Spec.Replicas { - return true, nil - } - - return false, nil - }) -} diff --git a/integration-tests/manifests/automonitor/validate_automonitor_test.go b/integration-tests/manifests/automonitor/validate_automonitor_test.go deleted file mode 100644 index cbd136aca..000000000 --- a/integration-tests/manifests/automonitor/validate_automonitor_test.go +++ /dev/null @@ -1,477 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package annotations - -import ( - "maps" - "slices" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" - "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" -) - -const ( - sampleDeploymentWithoutServiceYaml = "../sample-deployment-without-service.yaml" - customerServiceYaml = "../customer-service.yaml" - frontendAppYaml = "../frontend-app.yaml" - adminDashboardYaml = "../admin-dashboard.yaml" - conflictingDeploymentYaml = "../conflicting-deployment.yaml" -) - -var all = slices.Collect(maps.Keys(instrumentation.SupportedTypes())) -var allAnnotations = getAnnotations(all...) - -// getAnnotations returns both auto and inject annotations for the specified language types -func getAnnotations(types ...instrumentation.Type) []string { - var annotations []string - for _, t := range types { - switch t { - case instrumentation.TypeJava: - annotations = append(annotations, autoAnnotateJavaAnnotation, injectJavaAnnotation) - case instrumentation.TypePython: - annotations = append(annotations, autoAnnotatePythonAnnotation, injectPythonAnnotation) - case instrumentation.TypeDotNet: - annotations = append(annotations, autoAnnotateDotNetAnnotation, injectDotNetAnnotation) - case instrumentation.TypeNodeJS: - annotations = append(annotations, autoAnnotateNodeJSAnnotation, injectNodeJSAnnotation) - } - } - return annotations -} - -// getAllTypes returns all supported language types - -// getNoAnnotations returns an empty annotation list - -// Permutation 1 [HIGH]: Enable monitoring for all services without auto restarts -func TestPermutation1_MonitorAllServicesNoAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, false) - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentYaml}) - - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - RestartPods: false, - }) - - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentServiceYaml}) - assert.NoError(t, err) - - // Verify no annotations without restart - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allAnnotations) - assert.NoError(t, err) - - // Manually restart and verify annotations - err = helper.RestartDeployment(namespace, "sample-deployment") - assert.NoError(t, err) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) - assert.NoError(t, err) -} - -// Permutation 2 [HIGH]: Disable automatic monitoring for all services -func TestPermutation2_DisableMonitoringNoAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, false) - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - - // First enable monitoring with auto-restart - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - RestartPods: true, - }) - - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - // Verify initial annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) - assert.NoError(t, err) - - // Disable monitoring without auto-restart - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: false, - RestartPods: false, - }) - - // Verify annotations still present - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) - assert.NoError(t, err) - - // Manually restart and verify annotations removed - err = helper.RestartDeployment(namespace, "sample-deployment") - assert.NoError(t, err) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allAnnotations) - assert.NoError(t, err) -} - -// Permutation 3 [HIGH]: Monitor all services with pod restarts enabled -func TestPermutation3_MonitorAllServicesWithAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, false) - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - // Verify no initial annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allAnnotations) - assert.NoError(t, err) - - // Enable monitoring with auto-restart - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - RestartPods: true, - }) - - // Verify annotations automatically added - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) - assert.NoError(t, err) -} - -// Permutation 4 [MED]: Disable monitoring but allow pod restarts -func TestPermutation4_DisableMonitoringWithAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, false) - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - - // Start with monitoring enabled - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - RestartPods: true, - }) - - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - // Verify initial annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) - assert.NoError(t, err) - - // Disable monitoring with auto-restart - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: false, - RestartPods: true, - }) - - // Verify annotations automatically removed - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", none, allAnnotations) - assert.NoError(t, err) -} - -// Permutation 5 [HIGH]: Monitor only Java and Python services without pod restarts -func TestPermutation5_MonitorSelectedLanguagesNoAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, false) - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml, sampleDeploymentYaml}) - // Start with all languages enabled - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - RestartPods: true, - }) - - time.Sleep(time.Second * 5) - - // Verify all annotations present - err := helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) - assert.NoError(t, err) - - // Update to Java and Python only without auto-restart - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), - RestartPods: false, - }) - - // Verify annotations unchanged without restart - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) - assert.NoError(t, err) - - // Manually restart and verify only Java/Python remain - err = helper.RestartDeployment(namespace, "sample-deployment") - assert.NoError(t, err) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - getAnnotations(instrumentation.TypeJava, instrumentation.TypePython), - getAnnotations(instrumentation.TypeDotNet, instrumentation.TypeNodeJS)) - assert.NoError(t, err) -} - -// Permutation 6 [MED]: Monitor Java and Python with pod restarts enabled -func TestPermutation6_MonitorSelectedLanguagesWithAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, false) - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - - // Start with all languages enabled - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - RestartPods: true, - }) - - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml}) - assert.NoError(t, err) - - // Verify all annotations present - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) - assert.NoError(t, err) - - // Update to Java and Python only with auto-restart - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), - RestartPods: true, - }) - - // Verify only Java/Python annotations remain - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - getAnnotations(instrumentation.TypeJava, instrumentation.TypePython), - getAnnotations(instrumentation.TypeDotNet, instrumentation.TypeNodeJS)) - assert.NoError(t, err) -} - -// Permutation 9 [HIGH]: Monitor all services but exclude specific Java workloads -func TestPermutation9_MonitorWithExclusionsNoAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, false) - - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - - // Set up exclusions - excludeConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Deployments: []string{namespace + "/customer-service"}, - }, - Python: auto.AnnotationResources{ - Namespaces: []string{namespace}, - }, - } - - // Update config with exclusions - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - RestartPods: false, - Exclude: excludeConfig, - }) - - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml, customerServiceYaml}) - assert.NoError(t, err) - - // Manually restart deployments - err = helper.RestartDeployment(namespace, "sample-deployment") - assert.NoError(t, err) - err = helper.RestartDeployment(namespace, "customer-service") - assert.NoError(t, err) - - // Verify regular deployment has all annotations except python - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet, instrumentation.TypeNodeJS), - getAnnotations(instrumentation.TypePython)) - assert.NoError(t, err) - - // Verify excluded customer-service has no Java annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", - getAnnotations(instrumentation.TypeDotNet, instrumentation.TypeNodeJS), - getAnnotations(instrumentation.TypeJava, instrumentation.TypePython)) - assert.NoError(t, err) - - // Verify kube-system deployment has no Python annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet, instrumentation.TypeNodeJS), - getAnnotations(instrumentation.TypePython)) - assert.NoError(t, err) -} - -// Permutation 10 [HIGH]: Monitor all services with auto-restarts but exclude specific Java workloads -func TestPermutation10_MonitorWithExclusionsWithAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, false) - - // Create single namespace - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - - // Create deployments before enabling monitoring - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml, customerServiceYaml}) - assert.NoError(t, err) - - // Set up exclusions and enable monitoring with auto-restart - excludeConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Deployments: []string{namespace + "/customer-service"}, - }, - } - - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - RestartPods: true, - Exclude: excludeConfig, - }) - - // Verify regular deployment has all annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", allAnnotations, none) - assert.NoError(t, err) - - // Verify excluded customer-service has no Java annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", - getAnnotations(instrumentation.TypePython, instrumentation.TypeDotNet, instrumentation.TypeNodeJS), - getAnnotations(instrumentation.TypeJava)) - assert.NoError(t, err) -} - -// Permutation 18 [HIGH]: Monitor all services with customSelector and specific languages -func TestPermutation18_MonitorWithCustomSelectorAndAutoRestarts(t *testing.T) { - helper := NewTestHelper(t, false) - - // Create single namespace - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - - // Create deployments - err := helper.CreateNamespaceAndApplyResources(namespace, []string{sampleDeploymentYaml, sampleDeploymentWithoutServiceYaml}) - assert.NoError(t, err) - - // Set up custom selector config - customSelectorConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{namespace}, - }, - Python: auto.AnnotationResources{ - Deployments: []string{namespace + "/sample-deployment-without-service"}, - }, - } - - // Update config with custom selector - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.TypeDotNet), - RestartPods: true, - CustomSelector: customSelectorConfig, - }) - - // Verify service-selected deployment has dotnet annotation - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - getAnnotations(instrumentation.TypeDotNet), - getAnnotations(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeNodeJS)) - assert.NoError(t, err) - - // Verify non-service deployment has python annotation - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment-without-service", - getAnnotations(instrumentation.TypePython), - getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet, instrumentation.TypeNodeJS)) - assert.NoError(t, err) -} - -// Permutation 19 [HIGH++]: Test that exclude takes precedence over all -func TestPermutation19_ConflictingCustomSelectorExclude(t *testing.T) { - helper := NewTestHelper(t, false) - - // Create single namespace - namespace := helper.Initialize("test-namespace", []string{sampleDeploymentServiceYaml}) - - // Create deployments in the namespace - err := helper.CreateNamespaceAndApplyResources(namespace, []string{ - sampleDeploymentYaml, - customerServiceYaml, - conflictingDeploymentYaml, - }) - assert.NoError(t, err) - - // exclude config without namespace-level exclusion, will add later - excludeConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Deployments: []string{namespace + "/customer-service"}, - }, - } - - customSelectorConfig := auto.AnnotationConfig{ - Java: auto.AnnotationResources{ - Namespaces: []string{namespace}, - }, - Python: auto.AnnotationResources{ - Deployments: []string{namespace + "/conflicting-deployment"}, - }, - DotNet: auto.AnnotationResources{ - Namespaces: []string{namespace}, - }, - } - - // Update operator config - monitorConfig := &auto.MonitorConfig{ - MonitorAllServices: true, - Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeNodeJS), - RestartPods: true, - Exclude: excludeConfig, - CustomSelector: customSelectorConfig, - } - - helper.UpdateMonitorConfig(monitorConfig) - - // Verify conflicting-deployment has Python, NodeJS and Java - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "conflicting-deployment", - getAnnotations(instrumentation.TypePython, instrumentation.TypeNodeJS, instrumentation.TypeJava), - getAnnotations(instrumentation.TypeDotNet)) - assert.NoError(t, err) - - // Verify customer-service has Python and NodeJS (Java excluded) - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "customer-service", - getAnnotations(instrumentation.TypePython, instrumentation.TypeNodeJS), - getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet)) - assert.NoError(t, err) - - // Verify sample-deployment has Java, Python, and NodeJS - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - getAnnotations(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeNodeJS), - getAnnotations(instrumentation.TypeDotNet)) - assert.NoError(t, err) -} - -// Permutation 20 [HIGH]: Disable general monitoring but enable specific instrumentation -func TestPermutation20_SelectiveMonitoringWithCustomSelector(t *testing.T) { - helper := NewTestHelper(t, false) - - // Create single namespace - namespace := helper.Initialize("test-namespace", []string{}) - - // Create deployments - err := helper.CreateNamespaceAndApplyResources(namespace, []string{ - frontendAppYaml, - adminDashboardYaml, - sampleDeploymentYaml, - sampleDeploymentWithoutServiceYaml, - }) - assert.NoError(t, err) - - // Set up custom selector config - customSelectorConfig := auto.AnnotationConfig{ - Python: auto.AnnotationResources{ - Deployments: []string{namespace + "/sample-deployment-without-service"}, - }, - NodeJS: auto.AnnotationResources{ - Deployments: []string{namespace + "/frontend-app", namespace + "/admin-dashboard"}, - }, - } - - // Update operator config - helper.UpdateMonitorConfig(&auto.MonitorConfig{ - MonitorAllServices: false, - Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), - RestartPods: true, - CustomSelector: customSelectorConfig, - }) - - // Verify frontend-app and admin-dashboard have NodeJS only - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "frontend-app", - getAnnotations(instrumentation.TypeNodeJS), - getAnnotations(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeDotNet)) - assert.NoError(t, err) - - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "admin-dashboard", - getAnnotations(instrumentation.TypeNodeJS), - getAnnotations(instrumentation.TypeJava, instrumentation.TypePython, instrumentation.TypeDotNet)) - assert.NoError(t, err) - - // Verify sample-deployment has no annotations - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment", - none, - allAnnotations) - assert.NoError(t, err) - - // Verify sample-deployment-without-service has Python - err = helper.ValidateWorkloadAnnotations("deployment", namespace, "sample-deployment-without-service", - getAnnotations(instrumentation.TypePython), - getAnnotations(instrumentation.TypeJava, instrumentation.TypeDotNet, instrumentation.TypeNodeJS)) - assert.NoError(t, err) -} diff --git a/integration-tests/manifests/conflicting-deployment.yaml b/integration-tests/manifests/conflicting-deployment.yaml deleted file mode 100644 index ece71b497..000000000 --- a/integration-tests/manifests/conflicting-deployment.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: conflicting-deployment - labels: - app: conflicting-app -spec: - replicas: 1 - selector: - matchLabels: - app: conflicting-app - template: - metadata: - labels: - app: conflicting-app - spec: - containers: - - name: conflicting-container - image: nginx:latest - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: Service -metadata: - name: conflicting-service -spec: - selector: - app: conflicting-app - ports: - - port: 80 - targetPort: 80 diff --git a/integration-tests/manifests/customer-service.yaml b/integration-tests/manifests/customer-service.yaml deleted file mode 100644 index d4af762b1..000000000 --- a/integration-tests/manifests/customer-service.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: customer-service - labels: - app: customer-service -spec: - replicas: 1 - selector: - matchLabels: - app: customer-service - template: - metadata: - labels: - app: customer-service - spec: - containers: - - name: customer-service-container - image: nginx:latest - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: Service -metadata: - name: customer-service -spec: - selector: - app: customer-service - ports: - - port: 80 - targetPort: 80 diff --git a/integration-tests/manifests/frontend-app.yaml b/integration-tests/manifests/frontend-app.yaml deleted file mode 100644 index c833a1358..000000000 --- a/integration-tests/manifests/frontend-app.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: frontend-app - labels: - app: frontend -spec: - replicas: 1 - selector: - matchLabels: - app: frontend - template: - metadata: - labels: - app: frontend - spec: - containers: - - name: frontend-container - image: nginx:latest - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: Service -metadata: - name: frontend-service -spec: - selector: - app: frontend - ports: - - port: 80 - targetPort: 80 diff --git a/integration-tests/manifests/sample-deployment-service.yaml b/integration-tests/manifests/sample-deployment-service.yaml deleted file mode 100644 index 2af8e448b..000000000 --- a/integration-tests/manifests/sample-deployment-service.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: sample-deployment-service -spec: - selector: - app: nginx-app - ports: - - protocol: TCP - port: 80 - targetPort: 80 \ No newline at end of file diff --git a/integration-tests/manifests/sample-deployment-without-service.yaml b/integration-tests/manifests/sample-deployment-without-service.yaml deleted file mode 100644 index e58e35eb1..000000000 --- a/integration-tests/manifests/sample-deployment-without-service.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sample-deployment-without-service - labels: - app: sample-app-without-service -spec: - replicas: 1 - selector: - matchLabels: - app: sample-app-without-service - template: - metadata: - labels: - app: sample-app-without-service - spec: - containers: - - name: sample-container - image: nginx:latest - ports: - - containerPort: 80 From 86672142c5e7f180f88a6338de199dcd6fb245c2 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 7 May 2025 16:41:09 -0400 Subject: [PATCH 53/63] Remove integ tests --- .github/workflows/operator-integration-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index 703f7f073..d46630e11 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -78,7 +78,6 @@ jobs: kubectl wait --for=condition=Ready pod --all -n default kubectl get pods -A kubectl describe pods -n default - kubectl logs deployment/cloudwatch-controller-manager --all-containers=true -n amazon-cloudwatch go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/java/default_instrumentation_java_env_variables.json app_signals - name: Test for defined instrumentation resources for Java From 3fb6a0ffcb86d417b3a921eb54e126567616282b Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Wed, 7 May 2025 16:43:41 -0400 Subject: [PATCH 54/63] go format --- pkg/instrumentation/auto/callback.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index a9c45e81b..fd5236d0d 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" appsv1 "k8s.io/api/apps/v1" From 910a042327242ad9a23f118dd691be9101e2a7b7 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 8 May 2025 11:31:41 -0400 Subject: [PATCH 55/63] finish rolling back integration test changes --- integration-tests/util/util.go | 75 ++++++++++++++-------------------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/integration-tests/util/util.go b/integration-tests/util/util.go index 13df2fec9..f107bba9a 100644 --- a/integration-tests/util/util.go +++ b/integration-tests/util/util.go @@ -5,12 +5,9 @@ package util import ( "context" - "encoding/json" "fmt" "time" - "k8s.io/apimachinery/pkg/util/wait" - appsV1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,51 +18,44 @@ import ( const TimoutDuration = 3 * time.Minute const TimeBetweenRetries = 5 * time.Second +// WaitForNewPodCreation takes in a resource either Deployment, DaemonSet, or StatefulSet wait until it is in running stage func WaitForNewPodCreation(clientSet *kubernetes.Clientset, resource interface{}, startTime time.Time) error { - namespace := "" - labelSelector := "" - switch r := resource.(type) { - case *appsV1.Deployment: - namespace = r.Namespace - labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() - case *appsV1.DaemonSet: - namespace = r.Namespace - labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() - case *appsV1.StatefulSet: - namespace = r.Namespace - labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() - default: - return fmt.Errorf("unsupported resource type") - } - return wait.PollUntilContextTimeout(context.TODO(), TimeBetweenRetries, TimoutDuration, true, func(ctx context.Context) (bool, error) { - newPods, err := clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ + start := time.Now() + for { + if time.Since(start) > TimoutDuration { + return fmt.Errorf("timed out waiting for new pod creation") + } + namespace := "" + labelSelector := "" + switch r := resource.(type) { + case *appsV1.Deployment: + namespace = r.Namespace + labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() + case *appsV1.DaemonSet: + namespace = r.Namespace + labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() + case *appsV1.StatefulSet: + namespace = r.Namespace + labelSelector = labels.Set(r.Spec.Selector.MatchLabels).AsSelector().String() + default: + return fmt.Errorf("unsupported resource type") + } + + newPods, _ := clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: labelSelector, }) - if err != nil { - return false, fmt.Errorf("failed to list pods: %v", err) - } + for _, pod := range newPods.Items { - if pod.CreationTimestamp.Time.After(startTime.Add(-time.Second)) { - if pod.Status.Phase == v1.PodRunning { - isReady := isPodReady(&pod) - if isReady { - return true, nil - } - } + if pod.CreationTimestamp.Time.After(startTime) && pod.Status.Phase == v1.PodRunning { + fmt.Printf("Operator pod %s created after start time and is running\n", pod.Name) + return nil + } else if pod.CreationTimestamp.Time.After(startTime) { + fmt.Printf("Operator pod %s created after start time but is not in running stage\n", pod.Name) } } - return false, nil - }) -} -// Helper function to check if pod is ready -func isPodReady(pod *v1.Pod) bool { - for _, condition := range pod.Status.Conditions { - if condition.Type == v1.PodReady { - return condition.Status == v1.ConditionTrue - } + time.Sleep(TimeBetweenRetries) } - return false } func CheckIfPodsAreRunning(pods *v1.PodList) bool { @@ -82,8 +72,3 @@ func CheckIfPodsAreRunning(pods *v1.PodList) bool { fmt.Println("All pods are in the Running phase") return true } - -func PrettyPrint(data interface{}) { - b, _ := json.MarshalIndent(data, "", " ") - fmt.Println(string(b)) -} From 2f987de055789d4104b642c7ad0f993c04762f5c Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 8 May 2025 13:38:33 -0400 Subject: [PATCH 56/63] SupportedTypes() -> SupportedTypes --- main.go | 2 +- pkg/instrumentation/annotationtype.go | 6 +++--- pkg/instrumentation/auto/annotation.go | 2 +- pkg/instrumentation/auto/callback.go | 2 +- pkg/instrumentation/auto/config.go | 4 ++-- pkg/instrumentation/auto/monitor.go | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index f8b185ff0..c788c97be 100644 --- a/main.go +++ b/main.go @@ -352,7 +352,7 @@ func main() { func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, client client.Client, reader client.Reader) (auto.InstrumentationAnnotator, bool) { var autoAnnotationConfig auto.AnnotationConfig - supportedLanguages := instrumentation.SupportedTypes() + supportedLanguages := instrumentation.SupportedTypes if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" { setupLog.Info("Auto-annotation is disabled") diff --git a/pkg/instrumentation/annotationtype.go b/pkg/instrumentation/annotationtype.go index ae90127fb..a6116c178 100644 --- a/pkg/instrumentation/annotationtype.go +++ b/pkg/instrumentation/annotationtype.go @@ -47,9 +47,9 @@ const ( TypeGo Type = "go" ) -func SupportedTypes() TypeSet { - return NewTypeSet(TypeJava, TypeNodeJS, TypePython, TypeDotNet) -} +var ( + SupportedTypes = NewTypeSet(TypeJava, TypeNodeJS, TypePython, TypeDotNet) +) // InjectAnnotationKey maps the instrumentation type to the inject annotation. func InjectAnnotationKey(instType Type) string { diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 2b77c4f6c..90ed4d310 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -148,7 +148,7 @@ func NewAnnotationMutators( } func warnNonNamespacedNames(cfg AnnotationConfig, logger logr.Logger) { - for t := range instrumentation.SupportedTypes() { + for t := range instrumentation.SupportedTypes { resources := cfg.getResources(t) for _, deployment := range resources.Deployments { if !strings.Contains(deployment, "/") { diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index fd5236d0d..444b1d1db 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -133,7 +133,7 @@ func shouldRestartFunc(m InstrumentationAnnotator, namespaceMutatedAnnotations m // shouldRestartResource returns true if a resource requires a restart corresponding to the mutated annotations on its namespace func shouldRestartResource(namespaceMutatedAnnotations map[string]string, obj metav1.Object) bool { var shouldRestart bool - injectAnnotations := buildInjectAnnotations(instrumentation.SupportedTypes()) + injectAnnotations := buildInjectAnnotations(instrumentation.SupportedTypes) if resourceAnnotations := obj.GetAnnotations(); resourceAnnotations != nil { // For each of the namespace mutated annotations, for namespaceMutatedAnnotation, namespaceMutatedAnnotationValue := range namespaceMutatedAnnotations { diff --git a/pkg/instrumentation/auto/config.go b/pkg/instrumentation/auto/config.go index 7a201d78f..e65613df5 100644 --- a/pkg/instrumentation/auto/config.go +++ b/pkg/instrumentation/auto/config.go @@ -42,7 +42,7 @@ func (c AnnotationConfig) LanguagesOf(obj client.Object, checkNamespace bool) in objName := namespacedName(obj) typesSelected := instrumentation.TypeSet{} - types := instrumentation.SupportedTypes() + types := instrumentation.SupportedTypes if checkNamespace { for t := range types { @@ -83,7 +83,7 @@ func (c AnnotationConfig) LanguagesOf(obj client.Object, checkNamespace bool) in } func (c AnnotationConfig) Empty() bool { - for t := range instrumentation.SupportedTypes() { + for t := range instrumentation.SupportedTypes { resources := c.getResources(t) if len(resources.DaemonSets) > 0 { return false diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 33ae019b8..08ebe2d3f 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -66,7 +66,7 @@ func NewMonitor(ctx context.Context, config MonitorConfig, k8sClient kubernetes. // Config default values if len(config.Languages) == 0 { logger.Info("Setting languages to default") - config.Languages = instrumentation.SupportedTypes() + config.Languages = instrumentation.SupportedTypes } logger.Info("AutoMonitor starting...") @@ -294,7 +294,7 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet) ma } allMutatedAnnotations := map[string]string{} - for language := range instrumentation.SupportedTypes() { + for language := range instrumentation.SupportedTypes { insertMutation, removeMutation := buildMutations(language) var mutatedAnnotations map[string]string if _, ok := languagesToMonitor[language]; ok { From ba8063cf594e6e14f0883faddc9d23531e1118ec Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 8 May 2025 13:46:41 -0400 Subject: [PATCH 57/63] update docs --- pkg/instrumentation/auto/callback.go | 5 +++-- pkg/instrumentation/auto/monitor.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index 444b1d1db..b6bb4e0df 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -167,8 +167,7 @@ func RestartNamespace(m InstrumentationAnnotator, ctx context.Context, namespace rangeObjectList(m, ctx, &appsv1.StatefulSetList{}, client.InNamespace(namespace.Name), chainCallbacks(shouldRestartFunc(m, mutatedAnnotations), callbackFunc)) } -// MutateAndPatchAll runs the mutators for each of the supported resources and patches them. - +// MutateAndPatchWorkloads runs the mutators for all workloads and patches them with the updated injection annotations func MutateAndPatchWorkloads(m InstrumentationAnnotator, ctx context.Context) { f := getMutateObjectFunc(m) callbackFunc := patchFunc(m, ctx, f) @@ -177,6 +176,8 @@ func MutateAndPatchWorkloads(m InstrumentationAnnotator, ctx context.Context) { rangeObjectList(m, ctx, &appsv1.StatefulSetList{}, &client.ListOptions{}, callbackFunc) } +// MutateAndPatchNamespaces runs the mutators for all namespaces. +// If restartNamespace is true, RestartNamespace will be called for each affected namespace. func MutateAndPatchNamespaces(m InstrumentationAnnotator, ctx context.Context, restartNamespace bool) { rangeObjectList(m, ctx, &corev1.NamespaceList{}, &client.ListOptions{}, chainCallbacks(patchFunc(m, ctx, getMutateObjectFunc(m)), restartNamespaceFunc(m, ctx, restartNamespace))) } diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 08ebe2d3f..196a741f8 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -313,7 +313,7 @@ func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet) ma // safeToMutate returns whether the customer consents to the operator updating their workload's pods. The user consents if any of the following conditions are true: // // 1. Auto restart enabled. -// 2. MonitorAllServices is enabled AND the workload is already going to restart (aka, the pod template is already modified) +// 2. The user was already modifying the pod template spec (aka a restart would already be triggered) func safeToMutate(oldWorkload client.Object, workload client.Object, restartPods bool) bool { // always ok to mutate namespace if isNamespace(workload) { From 9c6ce5a3c7c9f370d8f2e4b8518f480c40150423 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 8 May 2025 13:47:54 -0400 Subject: [PATCH 58/63] inline function for shouldOverrideMultiInstrumentation --- pkg/instrumentation/podmutator.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/pkg/instrumentation/podmutator.go b/pkg/instrumentation/podmutator.go index bd4bc6224..994434c6d 100644 --- a/pkg/instrumentation/podmutator.go +++ b/pkg/instrumentation/podmutator.go @@ -32,11 +32,11 @@ var ( ) type instPodMutator struct { - Client client.Client - sdkInjector *sdkInjector - Logger logr.Logger - Recorder record.EventRecorder - autoMonitorEnabled bool + Client client.Client + sdkInjector *sdkInjector + Logger logr.Logger + Recorder record.EventRecorder + overrideEnabledMultiInstrumentation bool } type instrumentationWithContainers struct { @@ -193,7 +193,7 @@ func (langInsts *languageInstrumentations) setInstrumentationLanguageContainers( var _ podmutation.PodMutator = (*instPodMutator)(nil) -func NewMutator(logger logr.Logger, client client.Client, recorder record.EventRecorder, autoMonitorEnabled bool) *instPodMutator { +func NewMutator(logger logr.Logger, client client.Client, recorder record.EventRecorder, overrideEnabledMultiInstrumentation bool) *instPodMutator { return &instPodMutator{ Logger: logger, Client: client, @@ -201,8 +201,8 @@ func NewMutator(logger logr.Logger, client client.Client, recorder record.EventR logger: logger, client: client, }, - Recorder: recorder, - autoMonitorEnabled: autoMonitorEnabled, + Recorder: recorder, + overrideEnabledMultiInstrumentation: overrideEnabledMultiInstrumentation, } } @@ -324,7 +324,7 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c } // We retrieve the annotation for podname - if featuregate.EnableMultiInstrumentationSupport.IsEnabled() || pm.shouldOverrideMultiInstrumentation() { + if featuregate.EnableMultiInstrumentationSupport.IsEnabled() || pm.overrideEnabledMultiInstrumentation { // We use annotations specific for instrumentation language insts.Java.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectJavaContainersName) insts.NodeJS.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectNodeJSContainersName) @@ -337,7 +337,7 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c // We check if provided annotations and instrumentations are valid ok, msg := insts.areContainerNamesConfiguredForMultipleInstrumentations() - if !ok && !pm.shouldOverrideMultiInstrumentation() { + if !ok && !pm.overrideEnabledMultiInstrumentation { logger.V(1).Error(msg, "skipping instrumentation injection") return pod, nil } @@ -424,10 +424,6 @@ func (pm *instPodMutator) selectInstrumentationInstanceFromNamespace(ctx context } } -func (pm *instPodMutator) shouldOverrideMultiInstrumentation() bool { - return pm.autoMonitorEnabled -} - func GetAmazonCloudWatchAgentResource(ctx context.Context, c client.Client, name string) v1alpha1.AmazonCloudWatchAgent { cr := &v1alpha1.AmazonCloudWatchAgent{} From b1452442b27d630995b151a91598697625a324a2 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 8 May 2025 13:48:58 -0400 Subject: [PATCH 59/63] name of variable should be same as struct name --- .../webhook/namespacemutation/webhookhandler.go | 12 ++++++------ .../webhook/workloadmutation/webhookhandler.go | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/webhook/namespacemutation/webhookhandler.go b/internal/webhook/namespacemutation/webhookhandler.go index 9e9af4670..ddfc980d3 100644 --- a/internal/webhook/namespacemutation/webhookhandler.go +++ b/internal/webhook/namespacemutation/webhookhandler.go @@ -19,14 +19,14 @@ import ( var _ admission.Handler = (*handler)(nil) type handler struct { - decoder *admission.Decoder - monitor auto.InstrumentationAnnotator + decoder *admission.Decoder + instrumentationAnnotator auto.InstrumentationAnnotator } -func NewWebhookHandler(decoder *admission.Decoder, monitor auto.InstrumentationAnnotator) admission.Handler { +func NewWebhookHandler(decoder *admission.Decoder, instrumentationAnnotator auto.InstrumentationAnnotator) admission.Handler { return &handler{ - decoder: decoder, - monitor: monitor, + decoder: decoder, + instrumentationAnnotator: instrumentationAnnotator, } } @@ -38,7 +38,7 @@ func (h *handler) Handle(_ context.Context, req admission.Request) admission.Res } // do not need to pass in oldObj because it's only used to check for workload pod template diff - h.monitor.MutateObject(nil, namespace) + h.instrumentationAnnotator.MutateObject(nil, namespace) marshaledNamespace, err := json.Marshal(namespace) if err != nil { diff --git a/internal/webhook/workloadmutation/webhookhandler.go b/internal/webhook/workloadmutation/webhookhandler.go index c54272727..f6ce1d720 100644 --- a/internal/webhook/workloadmutation/webhookhandler.go +++ b/internal/webhook/workloadmutation/webhookhandler.go @@ -30,15 +30,15 @@ type WebhookHandler interface { // the implementation. type workloadMutationWebhook struct { - decoder *admission.Decoder - monitor auto.InstrumentationAnnotator + decoder *admission.Decoder + instrumentationAnnotator auto.InstrumentationAnnotator } // NewWebhookHandler creates a new WorkloadWebhookHandler. -func NewWebhookHandler(decoder *admission.Decoder, monitor auto.InstrumentationAnnotator) WebhookHandler { +func NewWebhookHandler(decoder *admission.Decoder, instrumentationAnnotator auto.InstrumentationAnnotator) WebhookHandler { return &workloadMutationWebhook{ - decoder: decoder, - monitor: monitor, + decoder: decoder, + instrumentationAnnotator: instrumentationAnnotator, } } @@ -65,12 +65,12 @@ func (p *workloadMutationWebhook) Handle(_ context.Context, req admission.Reques // populate old object if req.Operation == v1.Update { if err := p.decoder.DecodeRaw(req.OldObject, oldObj); err != nil { - p.monitor.GetLogger().WithName("workload_webhook").Error(err, "failed to unmarshal old object") + p.instrumentationAnnotator.GetLogger().WithName("workload_webhook").Error(err, "failed to unmarshal old object") return admission.Errored(http.StatusBadRequest, err) } } - p.monitor.MutateObject(oldObj, obj) + p.instrumentationAnnotator.MutateObject(oldObj, obj) marshaledObject, err := json.Marshal(obj) if err != nil { From 3ee911cdbf413c531a1572035d30f56b2f9271f8 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 8 May 2025 13:57:40 -0400 Subject: [PATCH 60/63] Remove todo, only check if autoAnnotationConfig is empty to use AutoMonitor --- main.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index c788c97be..bd5845fc2 100644 --- a/main.go +++ b/main.go @@ -360,11 +360,10 @@ func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationC if err := json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { setupLog.Error(err, "Unable to unmarshal auto-annotation config") } else { - // todo: technically a breaking change, because previously an empty autoAnnotationConfig would clear all annotations, but automonitor does not reproduce this behavior by default because it requires restartPods to be enabled. - if autoAnnotationConfig.Empty() && autoMonitorConfigStr != "" { - setupLog.Info("Auto-annotation is disabled because it is empty and the AutoMonitor config is not empty. Trying AutoMonitor...") + if autoAnnotationConfig.Empty() { + setupLog.Info("Auto-annotation is disabled because it is empty. Trying AutoMonitor...") } else { - setupLog.Info("WARNING: Using deprecated autoAnnotateAutoInstrumentation config, Disabling AutoMonitor. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") + setupLog.Info("W! Using deprecated autoAnnotateAutoInstrumentation config, Disabling AutoMonitor. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") return auto.NewAnnotationMutators( client, reader, From ddfd5520145e9d20efdb5579acb8eea4c558df03 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 8 May 2025 13:58:50 -0400 Subject: [PATCH 61/63] move excludedNamespaces to top --- pkg/instrumentation/auto/monitor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go index 196a741f8..9feb76eec 100644 --- a/pkg/instrumentation/auto/monitor.go +++ b/pkg/instrumentation/auto/monitor.go @@ -23,6 +23,8 @@ import ( "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" ) +var excludedNamespaces = []string{"kube-system", "amazon-cloudwatch"} + // InstrumentationAnnotator is the highest level abstraction used to annotate kubernetes resources for instrumentation type InstrumentationAnnotator interface { MutateObject(oldObj client.Object, obj client.Object) any @@ -246,8 +248,6 @@ func (m *Monitor) MutateObject(oldObj client.Object, obj client.Object) any { return mutate(obj, languagesToAnnotate) } -var excludedNamespaces = []string{"kube-system", "amazon-cloudwatch"} - // returns if workload is auto monitored (does not include custom selector) func (m *Monitor) isWorkloadAutoMonitored(obj client.Object) bool { if isNamespace(obj) { From 0e27206956d1093cfbf4a09df35808d91fbb583c Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 8 May 2025 15:40:20 -0400 Subject: [PATCH 62/63] Refactor InstrumentationAnnotator creation out of main.go and into util.go in auto package. Add unit tests --- main.go | 56 +--------- pkg/instrumentation/auto/util.go | 98 ++++++++++++++++ pkg/instrumentation/auto/util_test.go | 154 ++++++++++++++++++++++++++ 3 files changed, 253 insertions(+), 55 deletions(-) create mode 100644 pkg/instrumentation/auto/util.go create mode 100644 pkg/instrumentation/auto/util_test.go diff --git a/main.go b/main.go index bd5845fc2..c3777066c 100644 --- a/main.go +++ b/main.go @@ -19,14 +19,11 @@ import ( colfeaturegate "go.opentelemetry.io/collector/featuregate" k8sruntime "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "k8s.io/client-go/rest" k8sapiflag "k8s.io/component-base/cli/flag" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -290,7 +287,7 @@ func main() { decoder := admission.NewDecoder(mgr.GetScheme()) - instrumentationAnnotator, shouldMonitorAllServices := createInstrumentationAnnotator(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader()) + instrumentationAnnotator, shouldMonitorAllServices := auto.CreateInstrumentationAnnotator(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader(), setupLog) if instrumentationAnnotator != nil { mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ @@ -350,57 +347,6 @@ func main() { } } -func createInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, client client.Client, reader client.Reader) (auto.InstrumentationAnnotator, bool) { - var autoAnnotationConfig auto.AnnotationConfig - supportedLanguages := instrumentation.SupportedTypes - - if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" { - setupLog.Info("Auto-annotation is disabled") - } else if autoAnnotationConfigStr != "" { - if err := json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { - setupLog.Error(err, "Unable to unmarshal auto-annotation config") - } else { - if autoAnnotationConfig.Empty() { - setupLog.Info("Auto-annotation is disabled because it is empty. Trying AutoMonitor...") - } else { - setupLog.Info("W! Using deprecated autoAnnotateAutoInstrumentation config, Disabling AutoMonitor. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") - return auto.NewAnnotationMutators( - client, - reader, - setupLog, - autoAnnotationConfig, - supportedLanguages, - ), false - } - } - } - - if os.Getenv("DISABLE_AUTO_MONITOR") == "true" { - setupLog.Info("Auto-monitor is disabled due to DISABLE_AUTO_MONITOR environment variable") - return nil, false - } - - var autoMonitorConfig *auto.MonitorConfig - if err := json.Unmarshal([]byte(autoMonitorConfigStr), &autoMonitorConfig); err != nil { - setupLog.Error(err, "Unable to unmarshal auto-monitor config, disabling AutoMonitor") - return nil, false - } else { - k8sConfig, err := rest.InClusterConfig() - if err != nil { - setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") - return nil, false - } - - clientSet, err := kubernetes.NewForConfig(k8sConfig) - if err != nil { - setupLog.Error(err, "AutoMonitor: Unable to create in-cluster config, disabling AutoMonitor.") - return nil, false - } - logger := ctrl.Log.WithName("auto_monitor") - return auto.NewMonitor(ctx, *autoMonitorConfig, clientSet, client, reader, logger), autoMonitorConfig.MonitorAllServices - } -} - func waitForWebhookServerStart(ctx context.Context, checker healthz.Checker, callback func(context.Context)) { ticker := time.NewTicker(time.Second) defer ticker.Stop() diff --git a/pkg/instrumentation/auto/util.go b/pkg/instrumentation/auto/util.go new file mode 100644 index 000000000..77401a692 --- /dev/null +++ b/pkg/instrumentation/auto/util.go @@ -0,0 +1,98 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package auto + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/go-logr/logr" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" +) + +// ConfigureAutoAnnotation handles the auto annotation configuration logic +func ConfigureAutoAnnotation(autoAnnotationConfigStr string, client client.Client, reader client.Reader, setupLog logr.Logger) (InstrumentationAnnotator, error) { + // Check environment variables first + if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" { + setupLog.Info("Auto-annotation is disabled") + return nil, fmt.Errorf("detected DISABLE_AUTO_ANNOTATION environment variable") + } + + if autoAnnotationConfigStr == "" { + return nil, fmt.Errorf("auto-annotation configuration not provided, disabling AutoAnnotation") + } + + var autoAnnotationConfig AnnotationConfig + if err := json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { + return nil, fmt.Errorf("unable to unmarshal auto-annotation config, disabling AutoAnnotation: %w", err) + } + + if autoAnnotationConfig.Empty() { + return nil, fmt.Errorf("AutoAnnotation configuration is empty, disabling AutoAnnotation") + } + + setupLog.Info("W! Using deprecated autoAnnotateAutoInstrumentation config, Disabling AutoMonitor. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") + return NewAnnotationMutators( + client, + reader, + setupLog, + autoAnnotationConfig, + instrumentation.SupportedTypes, + ), nil +} + +// ConfigureAutoMonitor handles the auto monitor configuration logic +func ConfigureAutoMonitor(ctx context.Context, autoMonitorConfigStr string, clientSet kubernetes.Interface, client client.Client, reader client.Reader) (*Monitor, error) { + // If auto-annotation is not configured or failed, try auto-monitor + if os.Getenv("DISABLE_AUTO_MONITOR") == "true" { + return nil, fmt.Errorf("auto-monitor is disabled due to DISABLE_AUTO_MONITOR environment variable") + } + + var autoMonitorConfig *MonitorConfig + if err := json.Unmarshal([]byte(autoMonitorConfigStr), &autoMonitorConfig); err != nil { + return nil, fmt.Errorf("unable to unmarshal auto-monitor config: %w", err) + } + + logger := ctrl.Log.WithName("auto_monitor") + return NewMonitor(ctx, *autoMonitorConfig, clientSet, client, reader, logger), nil +} + +// CreateInstrumentationAnnotator creates an instrumentationAnnotator based on config and environment. Returns the InstrumentationAnnotator and whether AutoMonitor is enabled. +func CreateInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, client client.Client, reader client.Reader, setupLog logr.Logger) (InstrumentationAnnotator, bool) { + k8sConfig, err := rest.InClusterConfig() + if err != nil { + setupLog.Error(err, "unable to create in-cluster config") + } + + clientSet, err := kubernetes.NewForConfig(k8sConfig) + if err != nil { + setupLog.Error(err, "unable to create clientset") + } + return createInstrumentationAnnotatorWithClientset(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, clientSet, client, reader, setupLog) +} + +// for testing +func createInstrumentationAnnotatorWithClientset(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, clientSet kubernetes.Interface, client client.Client, reader client.Reader, setupLog logr.Logger) (InstrumentationAnnotator, bool) { + autoAnnotation, err := ConfigureAutoAnnotation(autoAnnotationConfigStr, client, reader, setupLog) + if err != nil { + setupLog.Error(err, "Failed to configure auto-annotation, trying AutoMonitor") + } else if autoAnnotation != nil { + return autoAnnotation, false + } + + monitor, err := ConfigureAutoMonitor(ctx, autoMonitorConfigStr, clientSet, client, reader) + if err != nil { + setupLog.Error(err, "Failed to configure auto-monitor") + return nil, false + } + + return monitor, monitor.config.MonitorAllServices +} diff --git a/pkg/instrumentation/auto/util_test.go b/pkg/instrumentation/auto/util_test.go new file mode 100644 index 000000000..b09e3a04c --- /dev/null +++ b/pkg/instrumentation/auto/util_test.go @@ -0,0 +1,154 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package auto + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes/fake" + + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestCreateInstrumentationAnnotator(t *testing.T) { + // Setup + fakeClient := fakeclient.NewClientBuilder().Build() + ctx := context.Background() + logger := testr.New(t) + + tests := []struct { + name string + envDisableAnnotation bool + envDisableMonitor bool + autoAnnotationConfig string + autoMonitorConfig string + expectNilAnnotator bool + expectedMonitorAll bool + expectedType string + }{ + { + name: "Both annotation and monitor disabled", + envDisableAnnotation: true, + envDisableMonitor: true, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: true, + expectedMonitorAll: false, + expectedType: "", + }, + { + name: "Annotation enabled, valid config", + envDisableAnnotation: false, + envDisableMonitor: false, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: false, + expectedMonitorAll: false, + expectedType: "*auto.AnnotationMutators", + }, + { + name: "Annotation disabled, Monitor enabled with monitorAllServices=true", + envDisableAnnotation: true, + envDisableMonitor: false, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: false, + expectedMonitorAll: true, + expectedType: "*auto.Monitor", + }, + { + name: "Annotation disabled, Monitor enabled with monitorAllServices=false", + envDisableAnnotation: true, + envDisableMonitor: false, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{"monitorAllServices":false}`, + expectNilAnnotator: false, + expectedMonitorAll: false, + expectedType: "*auto.Monitor", + }, + { + name: "Invalid annotation config, valid monitor config", + envDisableAnnotation: false, + envDisableMonitor: false, + autoAnnotationConfig: `{invalid-json}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: false, + expectedMonitorAll: true, + expectedType: "*auto.Monitor", + }, + { + name: "Empty annotation config, valid monitor config", + envDisableAnnotation: false, + envDisableMonitor: false, + autoAnnotationConfig: `{}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: false, + expectedMonitorAll: true, + expectedType: "*auto.Monitor", + }, + { + name: "Valid annotation config, invalid monitor config", + envDisableAnnotation: false, + envDisableMonitor: false, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{invalid-json}`, + expectNilAnnotator: false, + expectedMonitorAll: false, + expectedType: "*auto.AnnotationMutators", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set environment variables + if tt.envDisableAnnotation { + os.Setenv("DISABLE_AUTO_ANNOTATION", "true") + } else { + os.Unsetenv("DISABLE_AUTO_ANNOTATION") + } + + if tt.envDisableMonitor { + os.Setenv("DISABLE_AUTO_MONITOR", "true") + } else { + os.Unsetenv("DISABLE_AUTO_MONITOR") + } + + // Call the function + annotator, monitorAll := createInstrumentationAnnotatorWithClientset(tt.autoMonitorConfig, tt.autoAnnotationConfig, ctx, fake.NewSimpleClientset(), fakeClient, fakeClient, logger) + + // Check results + if tt.expectNilAnnotator { + assert.Nil(t, annotator, "Expected nil annotator") + } else { + assert.NotNil(t, annotator, "Expected non-nil annotator") + + // Check type using type assertion + actualType := fmt.Sprintf("%T", annotator) + assert.Equal(t, tt.expectedType, actualType, "Unexpected annotator type") + + // Specific type assertions + switch tt.expectedType { + case "*auto.AnnotationMutators": + _, ok := annotator.(*AnnotationMutators) + assert.True(t, ok, "Expected annotator to be of type *AnnotationMutators") + case "*auto.Monitor": + monitor, ok := annotator.(*Monitor) + assert.True(t, ok, "Expected annotator to be of type *Monitor") + if tt.expectedMonitorAll { + assert.True(t, monitor.config.MonitorAllServices, "Expected MonitorAllServices to be true") + } else { + assert.False(t, monitor.config.MonitorAllServices, "Expected MonitorAllServices to be false") + } + } + } + + assert.Equal(t, tt.expectedMonitorAll, monitorAll, "Unexpected monitorAll value") + }) + } +} From fe01d21a4b88a2351e9b1cc5c4cfc58fb108fe08 Mon Sep 17 00:00:00 2001 From: Marcus Mann Date: Thu, 8 May 2025 16:12:42 -0400 Subject: [PATCH 63/63] Incorporate auto/util.go feedback --- pkg/instrumentation/auto/util.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/instrumentation/auto/util.go b/pkg/instrumentation/auto/util.go index 77401a692..846b9bf4e 100644 --- a/pkg/instrumentation/auto/util.go +++ b/pkg/instrumentation/auto/util.go @@ -18,12 +18,12 @@ import ( "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" ) -// ConfigureAutoAnnotation handles the auto annotation configuration logic -func ConfigureAutoAnnotation(autoAnnotationConfigStr string, client client.Client, reader client.Reader, setupLog logr.Logger) (InstrumentationAnnotator, error) { +// configureAutoAnnotation handles the auto annotation configuration logic +func configureAutoAnnotation(autoAnnotationConfigStr string, client client.Client, reader client.Reader, setupLog logr.Logger) (InstrumentationAnnotator, error) { // Check environment variables first if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" { - setupLog.Info("Auto-annotation is disabled") - return nil, fmt.Errorf("detected DISABLE_AUTO_ANNOTATION environment variable") + setupLog.Info("detected DISABLE_AUTO_ANNOTATION environment variable, disabling AutoAnnotation") + return nil, nil } if autoAnnotationConfigStr == "" { @@ -49,11 +49,12 @@ func ConfigureAutoAnnotation(autoAnnotationConfigStr string, client client.Clien ), nil } -// ConfigureAutoMonitor handles the auto monitor configuration logic -func ConfigureAutoMonitor(ctx context.Context, autoMonitorConfigStr string, clientSet kubernetes.Interface, client client.Client, reader client.Reader) (*Monitor, error) { +// configureAutoMonitor handles the auto monitor configuration logic +func configureAutoMonitor(ctx context.Context, autoMonitorConfigStr string, clientSet kubernetes.Interface, client client.Client, reader client.Reader, setupLog logr.Logger) (*Monitor, error) { // If auto-annotation is not configured or failed, try auto-monitor if os.Getenv("DISABLE_AUTO_MONITOR") == "true" { - return nil, fmt.Errorf("auto-monitor is disabled due to DISABLE_AUTO_MONITOR environment variable") + setupLog.Info("W! auto-monitor is disabled due to DISABLE_AUTO_MONITOR environment variable") + return nil, nil } var autoMonitorConfig *MonitorConfig @@ -81,18 +82,20 @@ func CreateInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationC // for testing func createInstrumentationAnnotatorWithClientset(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, clientSet kubernetes.Interface, client client.Client, reader client.Reader, setupLog logr.Logger) (InstrumentationAnnotator, bool) { - autoAnnotation, err := ConfigureAutoAnnotation(autoAnnotationConfigStr, client, reader, setupLog) + autoAnnotation, err := configureAutoAnnotation(autoAnnotationConfigStr, client, reader, setupLog) if err != nil { setupLog.Error(err, "Failed to configure auto-annotation, trying AutoMonitor") } else if autoAnnotation != nil { return autoAnnotation, false } - monitor, err := ConfigureAutoMonitor(ctx, autoMonitorConfigStr, clientSet, client, reader) + monitor, err := configureAutoMonitor(ctx, autoMonitorConfigStr, clientSet, client, reader, setupLog) if err != nil { setupLog.Error(err, "Failed to configure auto-monitor") return nil, false + } else if monitor != nil { + return monitor, monitor.config.MonitorAllServices } - return monitor, monitor.config.MonitorAllServices + return nil, false }