From 2958d69cded795b62a26d3335d7bc0be4ce0d5f1 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 9 Jul 2024 14:27:13 +0200 Subject: [PATCH 1/3] test: add gateway conformance tests Skips practically all of them to just get the framework in place. I'm not sure all the tests are valid, the spec is missing large chunks of functionality asserted in the tests, but the assertions don't necessarily assert IETF/W3c spec compliant behaviour, for example: https://github.com/ipfs/gateway-conformance/issues/217 --- .env-gwc | 29 +- .github/workflows/gateway-conformance.yml | 3 +- .gitignore | 35 +-- debugging/test-gateways.sh | 46 +-- package-lock.json | 340 ++++++++++++++++++++++ package.json | 9 +- scripts/kubo-init.js | 15 +- src/helia-http-gateway.ts | 81 +++--- src/helia-rpc-api.ts | 40 ++- src/index.ts | 3 +- 10 files changed, 473 insertions(+), 128 deletions(-) diff --git a/.env-gwc b/.env-gwc index f9a2c6b4..b2d9a2d0 100644 --- a/.env-gwc +++ b/.env-gwc @@ -1,12 +1,15 @@ # ENV vars recommended for running gateway-conformance tests -USE_LIBP2P=false -USE_BITSWAP=true -PORT=8090 # helia-http-gateway should be running here -TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 # Kubo should be running here -DELEGATED_ROUTING_V1_HOST=http://127.0.0.1:8080 # Kubo should be running here -DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' -USE_TRUSTLESS_GATEWAYS=true -USE_DELEGATED_ROUTING=true +export USE_LIBP2P=true +export USE_BITSWAP=true +export USE_SUBDOMAINS=false +export PORT="8080" # helia-http-gateway should be running here +export KUBO_PORT="8081" # Kubo should be running here +export TRUSTLESS_GATEWAYS="http://127.0.0.1:8081" # Kubo should be running here +export DELEGATED_ROUTING_V1_HOST="http://127.0.0.1:8081" # Kubo should be running here +# DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' +export DEBUG='helia*,helia*:trace' +export USE_TRUSTLESS_GATEWAYS=true +export USE_DELEGATED_ROUTING=true # Uncomment the two below to save blockstore and datastore to disk # FILE_DATASTORE_PATH=./data/datastore @@ -14,3 +17,13 @@ USE_DELEGATED_ROUTING=true # Uncomment the below to see request & response headers in the logs # ECHO_HEADERS=true + +export GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0 + +# skip most of the tests +export GWC_SKIP="^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$" +export GWC_GATEWAY_URL="http://helia-http-gateway.localhost" +# GWC_SUBDOMAIN_URL="http://helia-http-gateway.localhost" +# GWC_GATEWAY_URL="http://127.0.0.1:8080" +export GWC_GATEWAY_URL="http://host.docker.internal:8080" +export GWC_SUBDOMAIN_URL="http://host.docker.internal:8080" diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 615dfe9b..4c2ba98a 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -101,7 +101,8 @@ jobs: # # only-if-cached: helia-ht does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) # for now disabling these test cases - args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' + args: -skip '^.*(DirectoryListing|TestGatewayCache|proxy|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' + # args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' # 7. Upload the results - name: Upload MD summary diff --git a/.gitignore b/.gitignore index 74633380..104adb87 100644 --- a/.gitignore +++ b/.gitignore @@ -7,35 +7,10 @@ node_modules package-lock.json yarn.lock .vscode - -# env config files -.env* -*.nix -.tool-versions -.direnv - -# helia-http-gateway specific files -/test-results/ -/playwright-report/ -/playwright/.cache/ -screenshots -.envrc -scripts/tmp/fixtures -scripts/tmp/kubo-path.txt -test/fixtures/e2e - -# helia-http-gateway debugging files -debugging/test-gateways.log -permutation-logs -*.csv -*.log -blockstore datastore -*.heapsnapshot - -# grafana/prometheus files -config/grafana/alerting -config/grafana/grafana.db -.DS_Store +debugging/*.log +scripts/tmp +test +test-results +*-report.json *.log -.env* diff --git a/debugging/test-gateways.sh b/debugging/test-gateways.sh index 04fe995e..0fb9f32c 100755 --- a/debugging/test-gateways.sh +++ b/debugging/test-gateways.sh @@ -44,55 +44,55 @@ test_website() { curl -X POST -m $max_timeout -s --no-progress-meter -o /dev/null -w "%{url}: HTTP_%{http_code} in %{time_total} seconds\n" http://localhost:$RPC_PORT/api/v0/repo/gc } -test_website http://localhost:$HTTP_PORT/ipns/blog.ipfs.tech +#test_website http://localhost:$HTTP_PORT/ipns/blog.ipfs.tech -test_website http://localhost:$HTTP_PORT/ipns/blog.libp2p.io +#test_website http://localhost:$HTTP_PORT/ipns/blog.libp2p.io -test_website http://localhost:$HTTP_PORT/ipns/consensuslab.world +#test_website http://localhost:$HTTP_PORT/ipns/consensuslab.world -test_website http://localhost:$HTTP_PORT/ipns/docs.ipfs.tech +#test_website http://localhost:$HTTP_PORT/ipns/docs.ipfs.tech -test_website http://localhost:$HTTP_PORT/ipns/docs.libp2p.io +#test_website http://localhost:$HTTP_PORT/ipns/docs.libp2p.io # test_website http://localhost:$HTTP_PORT/ipns/drand.love #drand.love is not publishing dnslink records -test_website http://localhost:$HTTP_PORT/ipns/fil.org +#test_website http://localhost:$HTTP_PORT/ipns/fil.org -test_website http://localhost:$HTTP_PORT/ipns/filecoin.io +#test_website http://localhost:$HTTP_PORT/ipns/filecoin.io test_website http://localhost:$HTTP_PORT/ipns/green.filecoin.io -test_website http://localhost:$HTTP_PORT/ipns/ipfs.tech +#test_website http://localhost:$HTTP_PORT/ipns/ipfs.tech -test_website http://localhost:$HTTP_PORT/ipns/ipld.io +#test_website http://localhost:$HTTP_PORT/ipns/ipld.io -test_website http://localhost:$HTTP_PORT/ipns/libp2p.io +#test_website http://localhost:$HTTP_PORT/ipns/libp2p.io -test_website http://localhost:$HTTP_PORT/ipns/n0.computer +#test_website http://localhost:$HTTP_PORT/ipns/n0.computer -test_website http://localhost:$HTTP_PORT/ipns/probelab.io +#test_website http://localhost:$HTTP_PORT/ipns/probelab.io -test_website http://localhost:$HTTP_PORT/ipns/protocol.ai +#test_website http://localhost:$HTTP_PORT/ipns/protocol.ai -test_website http://localhost:$HTTP_PORT/ipns/research.protocol.ai +#test_website http://localhost:$HTTP_PORT/ipns/research.protocol.ai -test_website http://localhost:$HTTP_PORT/ipns/singularity.storage +#test_website http://localhost:$HTTP_PORT/ipns/singularity.storage -test_website http://localhost:$HTTP_PORT/ipns/specs.ipfs.tech +#test_website http://localhost:$HTTP_PORT/ipns/specs.ipfs.tech # test_website http://localhost:$HTTP_PORT/ipns/strn.network -test_website http://localhost:$HTTP_PORT/ipns/saturn.tech +#test_website http://localhost:$HTTP_PORT/ipns/saturn.tech test_website http://localhost:$HTTP_PORT/ipns/web3.storage -test_website http://localhost:$HTTP_PORT/ipfs/bafkreiezuss4xkt5gu256vjccx7vocoksxk77vwmdrpwoumfbbxcy2zowq # stock images 3 sec skateboarder video +#test_website http://localhost:$HTTP_PORT/ipfs/bafkreiezuss4xkt5gu256vjccx7vocoksxk77vwmdrpwoumfbbxcy2zowq # stock images 3 sec skateboarder video -test_website http://localhost:$HTTP_PORT/ipfs/bafybeidsp6fva53dexzjycntiucts57ftecajcn5omzfgjx57pqfy3kwbq # big buck bunny +#test_website http://localhost:$HTTP_PORT/ipfs/bafybeidsp6fva53dexzjycntiucts57ftecajcn5omzfgjx57pqfy3kwbq # big buck bunny -test_website http://localhost:$HTTP_PORT/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze # wikipedia +#test_website http://localhost:$HTTP_PORT/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze # wikipedia -test_website http://localhost:$HTTP_PORT/ipfs/bafybeifaiclxh6pc3bdtrrkpbvvqqxq6hz5r6htdzxaga4fikfpu2u56qi # uniswap interface +#test_website http://localhost:$HTTP_PORT/ipfs/bafybeifaiclxh6pc3bdtrrkpbvvqqxq6hz5r6htdzxaga4fikfpu2u56qi # uniswap interface -test_website http://localhost:$HTTP_PORT/ipfs/bafybeiae366charqmeewxags5b2jxtkhfmqyyagvqhrr5l7l7xfpp5ikpa # cid.ipfs.tech +#test_website http://localhost:$HTTP_PORT/ipfs/bafybeiae366charqmeewxags5b2jxtkhfmqyyagvqhrr5l7l7xfpp5ikpa # cid.ipfs.tech -test_website http://localhost:$HTTP_PORT/ipfs/bafybeiedlhslivmuj2iinnpd24ulx3fyd7cjenddbkeoxbf3snjiz3npda # docs.ipfs.tech +#test_website http://localhost:$HTTP_PORT/ipfs/bafybeiedlhslivmuj2iinnpd24ulx3fyd7cjenddbkeoxbf3snjiz3npda # docs.ipfs.tech diff --git a/package-lock.json b/package-lock.json index 2e69c305..b1ff476b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -57,6 +57,7 @@ "dotenv": "^16.4.5", "execa": "^9.3.0", "glob": "^11.0.0", + "kubo": "^0.29.0", "patch-package": "^8.0.0", "typescript": "5.x", "wait-on": "^7.2.0" @@ -9010,6 +9011,19 @@ "integrity": "sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==", "dev": true }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "dev": true, + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -9651,6 +9665,15 @@ "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==", "dev": true }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "dev": true, + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/builtin-modules": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", @@ -9782,6 +9805,15 @@ "integrity": "sha512-WF0LihfemtesFcJgO7xfOoOcnWzY/QHR4qeDqV44jPU3HTI54+LnfXK3SA27AVVGCdZFgjjFFaqUA9Jx7dMJZA==", "dev": true }, + "node_modules/cachedir": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.4.0.tgz", + "integrity": "sha512-9EtFOZR8g22CL7BWjJ9BUx1+A/djkofnyW3aOXZORNW2kxoUpx2h+uN2cOqwPmFhnpVmxg+KW2OjOSgChTEvsQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/caching-transform": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", @@ -10046,6 +10078,27 @@ "node": ">=4" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "dev": true, + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, + "node_modules/chainsaw/node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "dev": true, + "engines": { + "node": "*" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -16089,6 +16142,107 @@ "node": ">=4.x" } }, + "node_modules/gunzip-maybe": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", + "integrity": "sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==", + "dev": true, + "dependencies": { + "browserify-zlib": "^0.1.4", + "is-deflate": "^1.0.0", + "is-gzip": "^1.0.0", + "peek-stream": "^1.1.0", + "pumpify": "^1.3.3", + "through2": "^2.0.3" + }, + "bin": { + "gunzip-maybe": "bin.js" + } + }, + "node_modules/gunzip-maybe/node_modules/browserify-zlib": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.1.4.tgz", + "integrity": "sha512-19OEpq7vWgsH6WkvkBJQDFvJS1uPcbFOQ4v9CU839dO+ZZXUZO6XpE6hNCqvlIIj+4fZvRiJ6DsAQ382GwiyTQ==", + "dev": true, + "dependencies": { + "pako": "~0.2.0" + } + }, + "node_modules/gunzip-maybe/node_modules/duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, + "node_modules/gunzip-maybe/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "node_modules/gunzip-maybe/node_modules/pako": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", + "integrity": "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==", + "dev": true + }, + "node_modules/gunzip-maybe/node_modules/pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/gunzip-maybe/node_modules/pumpify": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "dev": true, + "dependencies": { + "duplexify": "^3.6.0", + "inherits": "^2.0.3", + "pump": "^2.0.0" + } + }, + "node_modules/gunzip-maybe/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/gunzip-maybe/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/gunzip-maybe/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/hamt-sharding": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/hamt-sharding/-/hamt-sharding-3.0.6.tgz", @@ -17546,6 +17700,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-deflate": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-deflate/-/is-deflate-1.0.0.tgz", + "integrity": "sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==", + "dev": true + }, "node_modules/is-directory": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz", @@ -17622,6 +17782,15 @@ "node": ">=0.10.0" } }, + "node_modules/is-gzip": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-gzip/-/is-gzip-1.0.0.tgz", + "integrity": "sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-installed-globally": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.4.0.tgz", @@ -19087,6 +19256,155 @@ "node": ">=6" } }, + "node_modules/kubo": { + "version": "0.29.0", + "resolved": "https://registry.npmjs.org/kubo/-/kubo-0.29.0.tgz", + "integrity": "sha512-y1g+971iy72r0LECZuvvFsmGjWsCAZhT0/U3kBw/71Bm9kRQk+UcpHaSWv2HyXggFP2EDHRufQJeouwu22sliw==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "cachedir": "^2.3.0", + "got": "^11.7.0", + "gunzip-maybe": "^1.4.2", + "hasha": "^5.2.2", + "pkg-conf": "^3.1.0", + "tar-fs": "^2.1.0", + "unzip-stream": "^0.3.0" + }, + "bin": { + "ipfs": "bin/ipfs" + } + }, + "node_modules/kubo/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kubo/node_modules/load-json-file": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-5.3.0.tgz", + "integrity": "sha512-cJGP40Jc/VXUsp8/OrnyKyTZ1y6v/dphm3bioS+RrKXjK2BB6wHUd6JptZEFDGgGahMT+InnZO5i1Ei9mpC8Bw==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.15", + "parse-json": "^4.0.0", + "pify": "^4.0.1", + "strip-bom": "^3.0.0", + "type-fest": "^0.3.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kubo/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kubo/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/kubo/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kubo/node_modules/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==", + "dev": true, + "dependencies": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/kubo/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/kubo/node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/kubo/node_modules/pkg-conf": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pkg-conf/-/pkg-conf-3.1.0.tgz", + "integrity": "sha512-m0OTbR/5VPNPqO1ph6Fqbj7Hv6QU7gR/tQW40ZqrL1rjgCU85W6C1bJn0BItuJqnR98PWzw7Z8hHeChD1WrgdQ==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0", + "load-json-file": "^5.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kubo/node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/kubo/node_modules/type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/ky": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/ky/-/ky-1.4.0.tgz", @@ -34899,6 +35217,28 @@ "node": ">= 0.8" } }, + "node_modules/unzip-stream": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.4.tgz", + "integrity": "sha512-PyofABPVv+d7fL7GOpusx7eRT9YETY2X04PhwbSipdj6bMxVCFJrr+nm0Mxqbf9hUiTin/UsnuFWBXlDZFy0Cw==", + "dev": true, + "dependencies": { + "binary": "^0.3.0", + "mkdirp": "^0.5.1" + } + }, + "node_modules/unzip-stream/node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/update-browserslist-db": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz", diff --git a/package.json b/package.json index 89f1e61d..d68604ec 100644 --- a/package.json +++ b/package.json @@ -63,10 +63,10 @@ "test:e2e-flame": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-flame\" \"wait-on 'tcp:$HTTP_PORT' && npm run test:e2e\"", "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$HTTP_PORT' && npm run test:e2e\"", "test:gwc-kubo": "node ./scripts/kubo-init.js", - "test:gwc-helia": "npm run build && node -r dotenv/config --trace-warnings dist/src/index.js dotenv_config_path=./.env-gwc", - "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\"", - "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url='http://helia-http-gateway.localhost' --subdomain-url='http://helia-http-gateway.localhost' --verbose --json gwc-report.json --specs subdomain-ipns-gateway,subdomain-ipfs-gateway -- -timeout 30m", - "test:gwc": "concurrently -k -s all -n \"kubo&helia,gateway-conformance\" -c \"magenta,blue\" \"npm run test:gwc-setup\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", + "test:gwc-helia": "wait-on \"tcp:$KUBO_PORT\" && npm run start:dev-trace", + "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\"", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url=$GWC_GATEWAY_URL --subdomain-url=$GWC_SUBDOMAIN_URL --verbose --json gwc-report.json $GWC_SPECS -- -test.skip=\"$GWC_SKIP\" -timeout 30m", + "test:gwc": "concurrently -k -s all -n \"kubo,helia,gateway-conformance\" -c \"magenta,blue,green\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\" \"wait-on 'tcp:$PORT' && npm run test:gwc-execute\"", "healthcheck": "node dist/src/healthcheck.js", "debug:until-death": "./debugging/until-death.sh", "debug:test-gateways": "./debugging/test-gateways.sh", @@ -120,6 +120,7 @@ "dotenv": "^16.4.5", "execa": "^9.3.0", "glob": "^11.0.0", + "kubo": "^0.29.0", "patch-package": "^8.0.0", "typescript": "5.x", "wait-on": "^7.2.0" diff --git a/scripts/kubo-init.js b/scripts/kubo-init.js index 8912fa8f..9ac068ed 100644 --- a/scripts/kubo-init.js +++ b/scripts/kubo-init.js @@ -15,6 +15,7 @@ debug.enable('kubo-init*') const kuboFilePath = './scripts/tmp/kubo-path.txt' const GWC_FIXTURES_PATH = `${dirname(kuboFilePath)}/fixtures` +const GWC_DOCKER_IMAGE = process.env.GWC_DOCKER_IMAGE ?? 'ghcr.io/ipfs/gateway-conformance:v0.5.0' async function main () { await $`mkdir -p ${dirname(kuboFilePath)}` @@ -54,7 +55,7 @@ function getExecaOptions ({ cwd, ipfsNsMap, tmpDir }) { async function attemptKuboInit (tmpDir) { const execaOptions = getExecaOptions({ tmpDir }) try { - await $(execaOptions)`npx -y kubo init` + await $(execaOptions)`npx -y kubo init --profile test` log('Kubo initialized at %s', tmpDir) } catch (e) { if (!e.stderr.includes('already exists!')) { @@ -86,8 +87,14 @@ async function writeKuboMetaData () { async function configureKubo (tmpDir) { const execaOptions = getExecaOptions({ tmpDir }) try { - await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/8080` + await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/${process.env.KUBO_PORT ?? 8081}` + await $(execaOptions)`npx -y kubo config --json Bootstrap ${JSON.stringify([])}` + await $(execaOptions)`npx -y kubo config --json Swarm.DisableNatPortMap true` + await $(execaOptions)`npx -y kubo config --json Discovery.MDNS.Enabled false` + await $(execaOptions)`npx -y kubo config --json Gateway.NoFetch true` await $(execaOptions)`npx -y kubo config --json Gateway.ExposeRoutingAPI true` + await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin ${JSON.stringify(['*'])}` + await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Methods ${JSON.stringify(['GET', 'POST', 'PUT', 'OPTIONS'])}` log('Kubo configured') } catch (e) { error('Failed to configure Kubo', e) @@ -95,9 +102,9 @@ async function configureKubo (tmpDir) { } async function downloadFixtures () { - log('Downloading fixtures') + log('Downloading fixtures to %s using %s', GWC_FIXTURES_PATH, GWC_DOCKER_IMAGE) try { - await $`docker run -v ${process.cwd()}:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false` + await $`docker run -v ${process.cwd()}:/workspace -w /workspace ${GWC_DOCKER_IMAGE} extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false` } catch (e) { error('Error downloading fixtures, assuming current or previous success', e) } diff --git a/src/helia-http-gateway.ts b/src/helia-http-gateway.ts index ee98acb5..dc084dd5 100644 --- a/src/helia-http-gateway.ts +++ b/src/helia-http-gateway.ts @@ -31,6 +31,7 @@ export function httpGateway (opts: HeliaHTTPGatewayOptions): RouteOptions[] { const { ns: namespace, '*': relativePath, address } = params as EntryParams log('handling entry: ', { address, namespace, relativePath }) + if (!USE_SUBDOMAINS) { log('subdomains are disabled, fetching without subdomain') return fetch(request, reply) @@ -39,9 +40,11 @@ export function httpGateway (opts: HeliaHTTPGatewayOptions): RouteOptions[] { } const { peerId, cid } = getIpnsAddressDetails(address) + if (peerId != null) { return fetch(request, reply) } + const cidv1Address = cid?.toString() const query = request.query as Record @@ -77,7 +80,8 @@ export function httpGateway (opts: HeliaHTTPGatewayOptions): RouteOptions[] { } /** - * Fetches a content for a subdomain, which basically queries delegated routing API and then fetches the path from helia. + * Fetches a content for a subdomain, which basically queries delegated + * routing API and then fetches the path from helia. */ async function fetch (request: FastifyRequest, reply: FastifyReply): Promise { const url = getFullUrlFromFastifyRequest(request, log) @@ -106,12 +110,18 @@ export function httpGateway (opts: HeliaHTTPGatewayOptions): RouteOptions[] { await reply.code(verifiedFetchResponse.status).send(verifiedFetchResponse.statusText) return } + const contentType = verifiedFetchResponse.headers.get('content-type') + if (contentType == null) { log('verified-fetch response for %s has no content-type', url) await reply.code(200).send(verifiedFetchResponse.body) return } + + console.info('content type', contentType) + console.info('reply', verifiedFetchResponse) + if (verifiedFetchResponse.body == null) { // this should never happen log('verified-fetch response for %s has no body', url) @@ -120,20 +130,26 @@ export function httpGateway (opts: HeliaHTTPGatewayOptions): RouteOptions[] { } const headers: Record = {} + for (const [headerName, headerValue] of verifiedFetchResponse.headers.entries()) { headers[headerName] = headerValue } - // Fastify really does not like streams despite what the documentation and github issues say. + + // Fastify really does not like streams despite what the documentation and + // github issues say const reader = verifiedFetchResponse.body.getReader() reply.raw.writeHead(verifiedFetchResponse.status, headers) + try { let done = false + let value = undefined + while (!done) { - const { done: _done, value } = await raceSignal(reader.read(), options.signal) + ({ done, value } = await raceSignal(reader.read(), options.signal)) + if (value != null) { reply.raw.write(Buffer.from(value)) } - done = _done } } catch (err) { log.error('error reading response for %s', url, err) @@ -144,38 +160,33 @@ export function httpGateway (opts: HeliaHTTPGatewayOptions): RouteOptions[] { } } - return [ - { - // without this non-wildcard postfixed path, the '/*' route will match first. - url: '/:ns(ipfs|ipns)/:address', // ipns/dnsLink or ipfs/cid - method: 'GET', - handler: async (request, reply): Promise => handleEntry(request, reply) - }, - { - url: '/:ns(ipfs|ipns)/:address/*', // ipns/dnsLink/relativePath or ipfs/cid/relativePath - method: 'GET', - handler: async (request, reply): Promise => handleEntry(request, reply) - }, - { - url: '/*', - method: 'GET', - handler: async (request, reply): Promise => { - try { - await fetch(request, reply) - } catch { - await reply.code(200).send('try /ipfs/ or /ipns/') - } - } - }, - { - url: '/', - method: 'GET', - handler: async (request, reply): Promise => { - if (USE_SUBDOMAINS && request.hostname.split('.').length > 1) { - return fetch(request, reply) - } + return [{ + // without this non-wildcard postfixed path, the '/*' route will match first. + url: '/:ns(ipfs|ipns)/:address', // ipns/dnsLink or ipfs/cid + method: 'GET', + handler: async (request, reply): Promise => handleEntry(request, reply) + }, { + url: '/:ns(ipfs|ipns)/:address/*', // ipns/dnsLink/relativePath or ipfs/cid/relativePath + method: 'GET', + handler: async (request, reply): Promise => handleEntry(request, reply) + }, { + url: '/*', + method: 'GET', + handler: async (request, reply): Promise => { + try { + await fetch(request, reply) + } catch { await reply.code(200).send('try /ipfs/ or /ipns/') } } - ] + }, { + url: '/', + method: 'GET', + handler: async (request, reply): Promise => { + if (USE_SUBDOMAINS && request.hostname.split('.').length > 1) { + return fetch(request, reply) + } + await reply.code(200).send('try /ipfs/ or /ipns/') + } + }] } diff --git a/src/helia-rpc-api.ts b/src/helia-rpc-api.ts index aefcf8ba..c13aab1c 100644 --- a/src/helia-rpc-api.ts +++ b/src/helia-rpc-api.ts @@ -80,27 +80,23 @@ export function rpcApi (opts: HeliaRPCAPIOptions): RouteOptions[] { } } - return [ - { - url: '/api/v0/version', - method: ['POST', 'GET'], - handler: async (request, reply): Promise => heliaVersion(request, reply) - }, { - url: '/api/v0/repo/gc', - method: ['POST', 'GET'], - handler: async (request, reply): Promise => gc(request, reply) - }, - { - url: '/api/v0/http-gateway-healthcheck', - method: 'GET', - handler: async (request, reply): Promise => healthCheck(request, reply) - }, - { - url: '/*', - method: 'GET', - handler: async (request, reply): Promise => { - await reply.code(400).send('API + Method not supported') - } + return [{ + url: '/api/v0/version', + method: ['POST', 'GET'], + handler: async (request, reply): Promise => heliaVersion(request, reply) + }, { + url: '/api/v0/repo/gc', + method: ['POST', 'GET'], + handler: async (request, reply): Promise => gc(request, reply) + }, { + url: '/api/v0/http-gateway-healthcheck', + method: 'GET', + handler: async (request, reply): Promise => healthCheck(request, reply) + }, { + url: '/*', + method: 'GET', + handler: async (request, reply): Promise => { + await reply.code(400).send('API + Method not supported') } - ] + }] } diff --git a/src/index.ts b/src/index.ts index e8468a49..2c72389c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -265,7 +265,8 @@ async function createServer (name: string, routes: RouteOptions[], options: Serv ], methods: ['GET', 'HEAD', 'OPTIONS'], strictPreflight: false, - preflightContinue: true + preflightContinue: true, + preflight: true }) // enable compression From 4571fbceb90ce1139db7ae2f7031255a577c1d0c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 9 Jul 2024 14:31:44 +0200 Subject: [PATCH 2/3] chore: update skip list --- .github/workflows/gateway-conformance.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 4c2ba98a..cf6d49d9 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -99,10 +99,10 @@ jobs: # use below to skip specific test if needed # args: -skip 'TestFooBr/GET_response_for_something' # - # only-if-cached: helia-ht does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) + # only-if-cached: helia-ht does not guarantee local cache, we will + # adjust upstream test (which was Kubo-specific) # for now disabling these test cases - args: -skip '^.*(DirectoryListing|TestGatewayCache|proxy|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' - # args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' + args: -skip '^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$' # 7. Upload the results - name: Upload MD summary From 85e5c16c03cbc8716e5752934af7b8bf0711231d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 9 Jul 2024 19:38:26 +0200 Subject: [PATCH 3/3] chore: add dashboard --- .env-gwc | 2 +- .../grafana/dashboards/helia-dashboard.json | 1978 +++++++++++++++++ 2 files changed, 1979 insertions(+), 1 deletion(-) create mode 100644 config/grafana/dashboards/helia-dashboard.json diff --git a/.env-gwc b/.env-gwc index b2d9a2d0..2f3dc298 100644 --- a/.env-gwc +++ b/.env-gwc @@ -21,7 +21,7 @@ export USE_DELEGATED_ROUTING=true export GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0 # skip most of the tests -export GWC_SKIP="^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$" +export GWC_SKIP="^.*(TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$" export GWC_GATEWAY_URL="http://helia-http-gateway.localhost" # GWC_SUBDOMAIN_URL="http://helia-http-gateway.localhost" # GWC_GATEWAY_URL="http://127.0.0.1:8080" diff --git a/config/grafana/dashboards/helia-dashboard.json b/config/grafana/dashboards/helia-dashboard.json new file mode 100644 index 00000000..70dde5aa --- /dev/null +++ b/config/grafana/dashboards/helia-dashboard.json @@ -0,0 +1,1978 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }, + { + "name": "VAR_HEATMAP_BUCKETS", + "type": "constant", + "label": "heatmap_buckets", + "value": "${VAR_HEATMAP_BUCKETS}", + "description": "" + }, + { + "name": "VAR_AXISWIDTH", + "type": "constant", + "label": "axisWidth", + "value": "${VAR_AXISWIDTH}", + "description": "" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.4.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 27, + "panels": [], + "title": "Helia", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "helia_bitswap_peers", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Peers", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Bitswap Peers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "helia_bitswap_message_send_queue", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{helia_bitswap_message_send_queue}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Bitswap Message Send Queue", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 24, + "panels": [], + "title": "js-libp2p", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "libp2p_connection_manager_connections", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{libp2p_connection_manager_connections}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "libp2p_dial_queue", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{libp2p_dial_queue}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Dial Queue", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "libp2p_protocol_streams_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{protocol}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Protocol Streams", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(libp2p_data_transfer_bytes_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{protocol}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Data transfer", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "rate(libp2p_tcp_dialer_events_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{event}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "TCP Dialer Events", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 36, + "panels": [], + "title": "Noise", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(libp2p_noise_decrypt_errors_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Decrypt Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(libp2p_noise_xxhandshake_error_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "XX Handshake Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 32, + "panels": [], + "title": "KAD-DHT", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "libp2p_kad_dht_query_time_seconds", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Query Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "libp2p_kad_dht_ping_queue", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{libp2p_kad_dht_ping_queue}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Ping Queue", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "failure", + "success" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "libp2p_kad_dht_network_message_send_times_milliseconds", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{libp2p_kad_dht_network_message_send_times_milliseconds}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Message Send Times", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 26, + "panels": [], + "title": "NodeJS", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 61 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "nodejs_memory_usage_bytes", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{memory}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 61 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "rate(process_cpu_seconds_total[$__rate_interval]) * 100", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_system_seconds_total[$__rate_interval]) * 100", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "System", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_user_seconds_total[$__rate_interval]) * 100", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "User", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "CPU Seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(nodejs_gc_duration_seconds_count[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "GC Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 69 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "nodejs_active_handles", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Handles", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "nodejs_eventloop_lag_mean_seconds", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "mean", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "nodejs_eventloop_lag_max_seconds", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "max", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "nodejs_eventloop_lag_min_seconds", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "min", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Event Loop Lag", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "hide": 2, + "name": "heatmap_buckets", + "query": "${VAR_HEATMAP_BUCKETS}", + "skipUrlSync": false, + "type": "constant", + "current": { + "value": "${VAR_HEATMAP_BUCKETS}", + "text": "${VAR_HEATMAP_BUCKETS}", + "selected": false + }, + "options": [ + { + "value": "${VAR_HEATMAP_BUCKETS}", + "text": "${VAR_HEATMAP_BUCKETS}", + "selected": false + } + ] + }, + { + "hide": 2, + "name": "axisWidth", + "query": "${VAR_AXISWIDTH}", + "skipUrlSync": false, + "type": "constant", + "current": { + "value": "${VAR_AXISWIDTH}", + "text": "${VAR_AXISWIDTH}", + "selected": false + }, + "options": [ + { + "value": "${VAR_AXISWIDTH}", + "text": "${VAR_AXISWIDTH}", + "selected": false + } + ] + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "helia-http-gateway default dashboard", + "uid": "helia-http-gateway-default", + "version": 28, + "weekStart": "" +} \ No newline at end of file