From 808d6185434749ef401b6d938194ad967fc0b3b8 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Sat, 3 Aug 2024 15:52:26 +0800 Subject: [PATCH 1/4] chore(tests) move to docker compose v2 --- .github/workflows/tests.yml | 2 +- t/fixtures/docker-compose.yml | 69 +++++++++++++++++++++++++++++++++++ t/fixtures/prepare_env.sh | 25 +++++++------ 3 files changed, 84 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4288dfd..040707a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -171,4 +171,4 @@ jobs: if: failure() run: | pushd t/fixtures - docker-compose logs + docker compose logs diff --git a/t/fixtures/docker-compose.yml b/t/fixtures/docker-compose.yml index e212ef6..6f7b022 100644 --- a/t/fixtures/docker-compose.yml +++ b/t/fixtures/docker-compose.yml @@ -20,6 +20,75 @@ services: acmenet: ipv4_address: 10.30.50.3 + consul: + image: hashicorp/consul + ports: + - "127.0.0.1:8500:8500" + command: agent -server -bootstrap-expect=1 -client=0.0.0.0 + healthcheck: + test: ["CMD", "consul", "members"] + interval: 10s + timeout: 5s + retries: 3 + + vault: + image: hashicorp/vault + user: root + cap_add: + - IPC_LOCK + environment: + - VAULT_DEV_ROOT_TOKEN_ID=root + - VAULT_LOCAL_CONFIG={"listener":{"tcp":{"tls_key_file":"/tmp/key.pem","tls_cert_file":"/tmp/cert.pem","address":"0.0.0.0:8210"}}} + volumes: + - /tmp/key.pem:/tmp/key.pem + - /tmp/cert.pem:/tmp/cert.pem + ports: + - "127.0.0.1:8200:8200" + - "127.0.0.1:8210:8210" + command: server -dev + healthcheck: + test: ["CMD", "vault", "status", "-address", "http://127.0.0.1:8200"] + interval: 10s + timeout: 5s + retries: 3 + + etcd: + image: quay.io/coreos/etcd:v3.4.33 + volumes: + - /usr/share/ca-certificates/:/etc/ssl/certs + ports: + - "4001:4001" + - "2380:2380" + - "2379:2379" + environment: + - HOST_IP=${HOST_IP} + command: > + etcd + -name etcd0 + -advertise-client-urls http://${HOST_IP}:2379,http://${HOST_IP}:4001 + -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 + -initial-advertise-peer-urls http://${HOST_IP}:2380 + -listen-peer-urls http://0.0.0.0:2380 + -initial-cluster-token etcd-cluster-1 + -initial-cluster etcd0=http://${HOST_IP}:2380 + -initial-cluster-state new + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 10s + timeout: 5s + retries: 3 + + dummy: + image: ubuntu + command: tail -f /dev/null + depends_on: + consul: + condition: service_healthy + vault: + condition: service_healthy + etcd: + condition: service_healthy + networks: acmenet: driver: bridge diff --git a/t/fixtures/prepare_env.sh b/t/fixtures/prepare_env.sh index 752ecc6..40085ed 100644 --- a/t/fixtures/prepare_env.sh +++ b/t/fixtures/prepare_env.sh @@ -1,13 +1,21 @@ #!/bin/bash -echo "Prepare containers" -docker run -d -e CONSUL_CLIENT_INTERFACE='eth0' -e CONSUL_BIND_INTERFACE='eth0' -p 127.0.0.1:8500:8500 hashicorp/consul agent -server -bootstrap-expect=1 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export HOST_IP="$(hostname -I | awk '{print $1}')" + openssl req -x509 -newkey rsa:4096 -keyout /tmp/key.pem -out /tmp/cert.pem -days 1 -nodes -subj '/CN=some.vault' chmod 777 /tmp/key.pem /tmp/cert.pem -docker run -d --user root --cap-add=IPC_LOCK -e VAULT_DEV_ROOT_TOKEN_ID=root --name=vault -e 'VAULT_LOCAL_CONFIG={"listener":{"tcp":{"tls_key_file":"/tmp/key.pem","tls_cert_file":"/tmp/cert.pem","address":"0.0.0.0:8210"}}}' -v /tmp/key.pem:/tmp/key.pem -v /tmp/cert.pem:/tmp/cert.pem -p 127.0.0.1:8200:8200 -p 127.0.0.1:8210:8210 hashicorp/vault server -dev -docker logs vault -docker run -d -v /usr/share/ca-certificates/:/etc/ssl/certs -p 4001:4001 -p 2380:2380 -p 2379:2379 --name etcd quay.io/coreos/etcd:v2.3.8 -name etcd0 -advertise-client-urls http://${HostIP}:2379,http://${HostIP}:4001 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls http://${HostIP}:2380 -listen-peer-urls http://0.0.0.0:2380 -initial-cluster-token etcd-cluster-1 -initial-cluster etcd0=http://${HostIP}:2380 -initial-cluster-state new -docker logs etcd + +echo "Prepare containers" +pushd "$SCRIPT_DIR" +docker compose up -d || ( + docker compose logs vault; + docker compose logs etcd; + docker compose logs consul; + exit 1 +) +popd + echo "Prepare vault for JWT auth" curl 'https://localhost:8210/v1/sys/auth/kubernetes.test' -k -X POST -H 'X-Vault-Token: root' -H 'Content-Type: application/json; charset=utf-8' --data-raw '{"path":"kubernetes.test","type":"jwt","config":{}}' @@ -15,17 +23,12 @@ curl 'https://localhost:8210/v1/auth/kubernetes.test/config' -k -X PUT -H 'X-Vau curl 'https://localhost:8210/v1/auth/kubernetes.test/role/root' -k -X POST -H 'X-Vault-Token: root' -H 'content-type: application/json; charset=utf-8' --data-raw '{"token_policies":["acme"],"role_type":"jwt","user_claim":"kubernetes.io/serviceaccount/service-account.uid","bound_subject":"system:serviceaccount:kong:gateway-kong"}' curl 'https://localhost:8210/v1/sys/policies/acl/acme' -k -X PUT -H 'X-Vault-Token: root' -H 'Content-Type: application/json; charset=utf-8' --data-raw '{"name":"acme","policy":"path \"secret/*\" {\n capabilities = [\"create\", \"read\", \"update\", \"delete\"]\n}"}' -echo "Prepare Pebble" -pushd t/fixtures -docker-compose up -d - # on macOS use host.docker.internal if [[ "$OSTYPE" == 'darwin'* ]]; then host_ip=$(docker run -it --rm alpine ping host.docker.internal -c1|grep -oE "\d+\.\d+\.\d+\.\d+"|head -n1) # update the default ip in resolver curl --request POST --data '{"ip":"'$host_ip'"}' http://localhost:8055/set-default-ipv4 fi -popd echo "Generate certs" openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out /tmp/account.key From 754ac4035fd51d8066a656ebb1ef36bcab0d8715 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Sun, 4 Aug 2024 15:35:59 +0800 Subject: [PATCH 2/4] fix(tests) uses v3 protocol for etcd --- t/storage/etcd.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/storage/etcd.t b/t/storage/etcd.t index 418ee06..97732b2 100644 --- a/t/storage/etcd.t +++ b/t/storage/etcd.t @@ -10,7 +10,7 @@ our $HttpConfig = qq{ lua_package_path "$pwd/lib/?.lua;$pwd/lib/?/init.lua;$pwd/../lib/?.lua;$pwd/../lib/?/init.lua;;"; init_by_lua_block { _G.test_lib = require("resty.acme.storage.etcd") - _G.test_cfg = nil + _G.test_cfg = { protocol = "v3" } _G.test_ttl = 1 } }; From 29c3834f97156dd3b87dc7f3968d636871966213 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Sun, 4 Aug 2024 15:44:46 +0800 Subject: [PATCH 3/4] feat(etcd) etcd storage to use v3 protocol --- README.md | 4 +- lib/resty/acme/storage/etcd.lua | 79 +++++++++++++++++++---- t/storage/etcd.t | 109 ++++++++++++++++++++------------ 3 files changed, 135 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 07f07e3..8d193c5 100644 --- a/README.md +++ b/README.md @@ -769,13 +769,13 @@ storage_config = { ### etcd -[etcd](https://etcd.io) based storage. Right now only `v2` protocol is supported. +[etcd](https://etcd.io) based storage. Right now only `v3` protocol is supported, and etcd server +version should be >= v3.4.0. The default config is: ```lua storage_config = { http_host = 'http://127.0.0.1:4001', - protocol = 'v2', key_prefix = '', timeout = 60, ssl_verify = false, diff --git a/lib/resty/acme/storage/etcd.lua b/lib/resty/acme/storage/etcd.lua index 1cd316b..93d6be4 100644 --- a/lib/resty/acme/storage/etcd.lua +++ b/lib/resty/acme/storage/etcd.lua @@ -7,12 +7,16 @@ function _M.new(conf) conf = conf or {} local self = setmetatable({}, mt) + if conf.protocol and conf.protocol ~= "v3" then + return nil, "only v3 protocol is supported" + end + local options = { http_host = conf.http_host or "http://127.0.0.1:4001", - protocol = conf.protocol or "v2", key_prefix = conf.key_prefix or "", timeout = conf.timeout or 60, ssl_verify = conf.ssl_verify, + protocol = "v3", } local client, err = etcd.new(options) @@ -21,30 +25,77 @@ function _M.new(conf) end self.client = client - self.protocol_is_v2 = options.protocol == "v2" return self, nil end +local function grant(self, ttl) + local res, err = self.client:grant(ttl) + if err then + return nil, err + end + return res.body.ID +end + -- set the key regardless of it's existence function _M:set(k, v, ttl) - local _, err = self.client:set(k, v, ttl) + k = "/" .. k + + local lease_id, err + if ttl then + lease_id, err = grant(self, ttl) + if err then + return err + end + end + + local _, err = self.client:set(k, v, { lease = lease_id }) if err then return err end end -- set the key only if the key doesn't exist +-- Note: the key created by etcd:setnx can't be attached to a lease later, it seems to be a bug function _M:add(k, v, ttl) - local res, err = self.client:setnx(k, v, ttl) - if err then - return err + k = "/" .. k + + local lease_id, err + if ttl then + lease_id, err = grant(self, ttl) + if err then + return err + end end - if res and res.body and res.body.errorCode == 105 then + + + local compare = { + { + key = k, + target = "CREATE", + create_revision = 0, + } + } + + local success = { + { + requestPut = { + key = k, + value = v, + lease = lease_id, + } + } + } + + local v, err = self.client:txn(compare, success) + if err then + return nil, err + elseif v and v.body and not v.body.succeeded then return "exists" end end function _M:delete(k) + k = "/" .. k local _, err = self.client:delete(k) if err then return err @@ -52,17 +103,17 @@ function _M:delete(k) end function _M:get(k) + k = "/" .. k local res, err = self.client:get(k) if err then return nil, err - elseif res.status == 404 and res.body and res.body.errorCode == 100 then + elseif res and res.body.kvs == nil then return nil, nil elseif res.status ~= 200 then return nil, "etcd returned status " .. res.status end - local node = res.body.node - -- is it already expired but not evited ? - if node.expiration and not node.ttl and self.protocol_is_v2 then + local node = res.body.kvs[1] + if not node then -- would this ever happen? return nil, nil end return node.value @@ -70,16 +121,16 @@ end local empty_table = {} function _M:list(prefix) - local res, err = self.client:get("/") + local res, err = self.client:readdir("/" .. prefix) if err then return nil, err - elseif not res or not res.body or not res.body.node or not res.body.node.nodes then + elseif not res or not res.body or not res.body.kvs then return empty_table, nil end local ret = {} -- offset 1 to strip leading "/" in original key local prefix_length = #prefix + 1 - for _, node in ipairs(res.body.node.nodes) do + for _, node in ipairs(res.body.kvs) do local key = node.key if key then -- start from 2 to strip leading "/" diff --git a/t/storage/etcd.t b/t/storage/etcd.t index 97732b2..d7de87c 100644 --- a/t/storage/etcd.t +++ b/t/storage/etcd.t @@ -7,10 +7,10 @@ use Cwd qw(cwd); my $pwd = cwd(); our $HttpConfig = qq{ - lua_package_path "$pwd/lib/?.lua;$pwd/lib/?/init.lua;$pwd/../lib/?.lua;$pwd/../lib/?/init.lua;;"; + lua_package_path "/home/wow/.luarocks/share/lua/5.1/?.lua;/home/wow/.luarocks/share/lua/5.1/?/init.lua;$pwd/lib/?.lua;$pwd/lib/?/init.lua;$pwd/../lib/?.lua;$pwd/../lib/?/init.lua;;"; init_by_lua_block { _G.test_lib = require("resty.acme.storage.etcd") - _G.test_cfg = { protocol = "v3" } + _G.test_cfg = nil _G.test_ttl = 1 } }; @@ -24,9 +24,10 @@ __DATA__ location =/t { content_by_lua_block { local st = test_lib.new(test_cfg) - local err = st:set("key1", "2") + local key = "key1_" .. ngx.now() + local err = st:set(key, "2") ngx.say(err) - local err = st:set("key1", "new value") + local err = st:set(key, "new value") ngx.say(err) } } @@ -44,9 +45,10 @@ __DATA__ location =/t { content_by_lua_block { local st = test_lib.new(test_cfg) - local err = st:set("key2", "3") + local key = "key2_" .. ngx.now() + local err = st:set(key, "3") ngx.say(err) - local v, err = st:get("key2") + local v, err = st:get(key) ngx.say(err) ngx.say(v) } @@ -67,21 +69,22 @@ nil location =/t { content_by_lua_block { local st = test_lib.new(test_cfg) - local err = st:set("key3", "3") + local key = "key3_" .. ngx.now() + local err = st:set(key, "3") ngx.say(err) - local v, err = st:get("key3") + local v, err = st:get(key) ngx.say(err) ngx.say(v) - local err = st:delete("key3") + local err = st:delete(key) ngx.say(err) -- now the key should be deleted - local v, err = st:get("key3") + local v, err = st:get(key) ngx.say(err) ngx.say(v) -- delete again with no error - local err = st:delete("key3") + local err = st:delete(key) ngx.say(err) } } @@ -105,14 +108,15 @@ nil location =/t { content_by_lua_block { local st = test_lib.new(test_cfg) - local err = st:set("prefix1", "bb--") + local prefix = "prefix4_" .. ngx.now() + local err = st:set(prefix .. "prefix1", "bb--") ngx.say(err) local err = st:set("pref-x2", "aa--") ngx.say(err) - local err = st:set("prefix3", "aa--") + local err = st:set(prefix .. "prefix3", "aa--") ngx.say(err) - local keys, err = st:list("prefix") + local keys, err = st:list(prefix) ngx.say(err) table.sort(keys) for _, p in ipairs(keys) do ngx.say(p) end @@ -128,8 +132,8 @@ nil nil nil nil -prefix1 -prefix3 +prefix4.+prefix1 +prefix4.+prefix3 0 " --- no_error_log @@ -141,15 +145,24 @@ prefix3 location =/t { content_by_lua_block { local st = test_lib.new(test_cfg) - local err = st:set("setttl", "bb--", test_ttl) + local key = "key5_" .. ngx.now() + local err = st:set(key, "bb--", test_ttl) ngx.say(err) - local v, err = st:get("setttl") - ngx.say(err) - ngx.say(v) - ngx.sleep(test_ttl) - local v, err = st:get("setttl") + local v, err = st:get(key) ngx.say(err) ngx.say(v) + for i=1, 5 do + ngx.sleep(1) + local v, err = st:get(key) + if err then + ngx.say(err) + ngx.exit(0) + elseif not v then + ngx.say(nil) + ngx.exit(0) + end + end + ngx.say("still exists") } } --- request @@ -159,7 +172,6 @@ prefix3 nil bb-- nil -nil " --- no_error_log [error] @@ -170,15 +182,24 @@ nil location =/t { content_by_lua_block { local st = test_lib.new(test_cfg) - local err = st:add("addttl", "bb--", test_ttl) + local key = "key6_" .. ngx.now() + local err = st:add(key, "bb--", test_ttl) ngx.say(err) - local v, err = st:get("addttl") - ngx.say(err) - ngx.say(v) - ngx.sleep(test_ttl) - local v, err = st:get("addttl") + local v, err = st:get(key) ngx.say(err) ngx.say(v) + for i=1, 5 do + ngx.sleep(1) + local v, err = st:get(key) + if err then + ngx.say(err) + ngx.exit(0) + elseif not v then + ngx.say(nil) + ngx.exit(0) + end + end + ngx.say("still exists") } } --- request @@ -188,7 +209,6 @@ nil nil bb-- nil -nil " --- no_error_log [error] @@ -199,24 +219,31 @@ nil location =/t { content_by_lua_block { local st = test_lib.new(test_cfg) - local err = st:set("prefix1", "bb--", test_ttl) + local key = "key7_" .. ngx.now() + local err = st:set(key, "bb--", test_ttl) ngx.say(err) - local err = st:add("prefix1", "aa--") + local err = st:add(key, "aa--") ngx.say(err) - local v, err = st:get("prefix1") + local v, err = st:get(key) ngx.say(err) ngx.say(v) -- note: etcd evit expired node not immediately - ngx.sleep(test_ttl+0.5) - local err = st:add("prefix1", "aa--", test_ttl) + for i=1, 5 do + ngx.sleep(1) + local v, err = st:get(key) + if err then + ngx.say(err) + break + elseif not v then + ngx.say("key evicted") + break + end + end + local err = st:add(key, "aa--", test_ttl) ngx.say(err) - local v, err = st:get("prefix1") + local v, err = st:get(key) ngx.say(err) ngx.say(v) - -- note: etcd evit expired node not immediately - ngx.sleep(test_ttl+0.5) - local err = st:add("prefix1", "aa--", test_ttl) - ngx.say(err) } } --- request @@ -226,10 +253,10 @@ nil exists nil bb-- +key evicted nil nil aa-- -nil " --- no_error_log [error] From 897035208f7a7fd95049b0fe9f8009cae539dfef Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Sun, 4 Aug 2024 17:51:50 +0800 Subject: [PATCH 4/4] fix(tests) use tlsv1.2 in dual cert test --- t/e2e.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/e2e.t b/t/e2e.t index 8ee25e7..b63fb42 100644 --- a/t/e2e.t +++ b/t/e2e.t @@ -204,10 +204,10 @@ __DATA__ } local out for i=0,15,1 do - local proc = ngx_pipe.spawn({'bash', '-c', "echo q |openssl s_client -host 127.0.0.1 -servername ".. ngx.var.domain .. " -port 5001 -cipher ECDHE-RSA-AES128-GCM-SHA256|openssl x509 -noout -text && sleep 0.1"}, opts) + local proc = ngx_pipe.spawn({'bash', '-c', "echo q |openssl s_client -host 127.0.0.1 -servername ".. ngx.var.domain .. " -max_protocol TLSv1.2 -port 5001 -cipher ECDHE-RSA-AES128-GCM-SHA256|openssl x509 -noout -text && sleep 0.1"}, opts) local data, err, partial = proc:stdout_read_all() if ngx.re.match(data, ngx.var.domain) then - local proc2 = ngx_pipe.spawn({'bash', '-c', "echo q |openssl s_client -host 127.0.0.1 -servername ".. ngx.var.domain .. " -port 5001 -cipher ECDHE-ECDSA-AES128-GCM-SHA256|openssl x509 -noout -text && sleep 0.1"}, opts) + local proc2 = ngx_pipe.spawn({'bash', '-c', "echo q |openssl s_client -host 127.0.0.1 -servername ".. ngx.var.domain .. " -port 5001 -max_protocol TLSv1.2 -cipher ECDHE-ECDSA-AES128-GCM-SHA256|openssl x509 -noout -text && sleep 0.1"}, opts) local data2, err, partial = proc2:stdout_read_all() ngx.log(ngx.INFO, data, data2) local f = io.open("/tmp/test2.1", "w")