From ab77b9d45e7aaac76d8fec366689c18bce9109c9 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 10 Jan 2025 20:57:45 +0100 Subject: [PATCH 1/4] fix(cli): support HTTPS in ipfs --api Closes #10539 --- cmd/ipfs/kubo/start.go | 5 +++++ docs/changelogs/v0.33.md | 6 +++++- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cmd/ipfs/kubo/start.go b/cmd/ipfs/kubo/start.go index 474045e7102..efde6e48e9b 100644 --- a/cmd/ipfs/kubo/start.go +++ b/cmd/ipfs/kubo/start.go @@ -330,6 +330,11 @@ func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) { switch network { case "tcp", "tcp4", "tcp6": tpt = http.DefaultTransport + // RPC over HTTPS requires explicit schema in the address passed to cmdhttp.NewClient + httpAddr := apiAddr.String() + if !strings.HasPrefix(host, "http:") && !strings.HasPrefix(host, "https:") && (strings.Contains(httpAddr, "/https") || strings.Contains(httpAddr, "/tls/http")) { + host = "https://" + host + } case "unix": path := host host = "unix" diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index bf012f4e91c..16437194a4e 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -12,6 +12,7 @@ - [Using default `libp2p_rcmgr` metrics](#using-default-libp2p_rcmgr--metrics) - [Flatfs does not `sync` on each write](#flatfs-does-not-sync-on-each-write) - [`ipfs add --to-files` no longer works with `--wrap`](#ipfs-add---to-files-no-longer-works-with---wrap) + - [`ipfs --api` supports HTTPS RPC endpoints](#ipfs---api-supports-https-rpc-endpoints) - [New options for faster writes: `WriteThrough`, `BlockKeyCacheSize`, `BatchMaxNodes`, `BatchMaxSize`](#new-options-for-faster-writes-writethrough-blockkeycachesize-batchmaxnodes-batchmaxsize) - [MFS stability with large number of writes](#mfs-stability-with-large-number-of-writes) - [📦️ Important dependency updates](#-important-dependency-updates) @@ -61,6 +62,10 @@ The old default was overly conservative and caused performance issues in big rep Onboarding files and directories with `ipfs add --to-files` now requires non-empty names. due to this, The `--to-files` and `--wrap` options are now mutually exclusive ([#10612](https://github.com/ipfs/kubo/issues/10612)). +#### `ipfs --api` supports HTTPS RPC endpoints + +CLI and RPC client now supports accessing Kubo RPC over `https://` protocol when multiaddr ending with `/https` or `/tls/http` is passed to `ipfs --api`: + #### New options for faster writes: `WriteThrough`, `BlockKeyCacheSize`, `BatchMaxNodes`, `BatchMaxSize` Now that Kubo supports [`pebble`](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds) as a datastore backend, it becomes very useful to expose some additional configuration options for how the blockservice/blockstore/datastore combo behaves. @@ -90,7 +95,6 @@ We have fixed a number of issues that were triggered by writing or copying many - update `p2p-forge/client` to [v0.2.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.2.0) (incl. [v0.1.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.1.0)) - update `ipfs-webui` to [v4.4.2](https://github.com/ipfs/ipfs-webui/releases/tag/v4.4.2) (incl. [v4.4.1](https://github.com/ipfs/ipfs-webui/releases/tag/v4.4.1)) - #### Escape Redirect URL for Directory When navigating to a subdirectory, served by the Kubo web server, a subdirectory without a trailing slash gets redirected to a URL with a trailing slash. If there are special characters such as "%" in the subdirectory name then these must be escaped in the redirect URL. Previously this was not being done and was preventing navigation to such subdirectories, requiring the user to manually add a trailing slash to the subdirectory URL. This is now fixed to handle the redirect to URLs with characters that must be escaped. diff --git a/go.mod b/go.mod index 1bfc3d4edef..618cda7131e 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/ipfs/go-ds-measure v0.2.0 github.com/ipfs/go-ds-pebble v0.4.0 github.com/ipfs/go-fs-lock v0.0.7 - github.com/ipfs/go-ipfs-cmds v0.14.0 + github.com/ipfs/go-ipfs-cmds v0.14.1-0.20250110194328-5b1af499e0a1 github.com/ipfs/go-ipld-cbor v0.2.0 github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-ipld-git v0.1.1 diff --git a/go.sum b/go.sum index 17b9c4ce28e..64ba5920dd9 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7 github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= -github.com/ipfs/go-ipfs-cmds v0.14.0 h1:sxdurhAHSdQr5VrSNJjc+t92uJObSNq+gRVm/wLZGMM= -github.com/ipfs/go-ipfs-cmds v0.14.0/go.mod h1:zj2jN7bHJ4pDucRmqdq863AQYcsqdxXrfVkr9eqPfvo= +github.com/ipfs/go-ipfs-cmds v0.14.1-0.20250110194328-5b1af499e0a1 h1:IJsO+hla34/mDMrIrErrgt18zIVU4JA0JhU8Zd4T1z4= +github.com/ipfs/go-ipfs-cmds v0.14.1-0.20250110194328-5b1af499e0a1/go.mod h1:SCYxNUVPeVR05cE8DJ6wyH2+aQ8vPgjxxkxQWOXobzo= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From 9b4dce1203a77ef23108c4ecbb43c11e75021056 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 10 Jan 2025 21:15:10 +0100 Subject: [PATCH 2/4] chore: go-ipfs-cmds v0.14.1 https://github.com/ipfs/go-ipfs-cmds/releases/tag/v0.14.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 618cda7131e..671a6599615 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/ipfs/go-ds-measure v0.2.0 github.com/ipfs/go-ds-pebble v0.4.0 github.com/ipfs/go-fs-lock v0.0.7 - github.com/ipfs/go-ipfs-cmds v0.14.1-0.20250110194328-5b1af499e0a1 + github.com/ipfs/go-ipfs-cmds v0.14.1 github.com/ipfs/go-ipld-cbor v0.2.0 github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-ipld-git v0.1.1 diff --git a/go.sum b/go.sum index 64ba5920dd9..c0aa0a8050a 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7 github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= -github.com/ipfs/go-ipfs-cmds v0.14.1-0.20250110194328-5b1af499e0a1 h1:IJsO+hla34/mDMrIrErrgt18zIVU4JA0JhU8Zd4T1z4= -github.com/ipfs/go-ipfs-cmds v0.14.1-0.20250110194328-5b1af499e0a1/go.mod h1:SCYxNUVPeVR05cE8DJ6wyH2+aQ8vPgjxxkxQWOXobzo= +github.com/ipfs/go-ipfs-cmds v0.14.1 h1:TA8vBixPwXL3k7VtcbX3r4FQgw2m+jMOWlslUOlM9Rs= +github.com/ipfs/go-ipfs-cmds v0.14.1/go.mod h1:SCYxNUVPeVR05cE8DJ6wyH2+aQ8vPgjxxkxQWOXobzo= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From e28e3fb1292bb0883f46530d6a3fa525f0af50b1 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 10 Jan 2025 21:27:09 +0100 Subject: [PATCH 3/4] docs: ipfs --api example --- docs/changelogs/v0.33.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index 16437194a4e..ae469cf1fe2 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -66,6 +66,11 @@ Onboarding files and directories with `ipfs add --to-files` now requires non-emp CLI and RPC client now supports accessing Kubo RPC over `https://` protocol when multiaddr ending with `/https` or `/tls/http` is passed to `ipfs --api`: +```console +$ ipfs id --api /dns/kubo-rpc.example.net/tcp/5001/tls/http +# → https://kubo-rpc.example.net:5001 +``` + #### New options for faster writes: `WriteThrough`, `BlockKeyCacheSize`, `BatchMaxNodes`, `BatchMaxSize` Now that Kubo supports [`pebble`](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds) as a datastore backend, it becomes very useful to expose some additional configuration options for how the blockservice/blockstore/datastore combo behaves. From 8aafa37dfc3146dde25d751bc622ac1ddab11e61 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 10 Jan 2025 21:56:48 +0100 Subject: [PATCH 4/4] test(cli): https rpc support makes sure we dont have regression where HTTPS endpoint starts getting cleartext requests --- test/cli/cli_https_test.go | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/cli/cli_https_test.go diff --git a/test/cli/cli_https_test.go b/test/cli/cli_https_test.go new file mode 100644 index 00000000000..e128a191641 --- /dev/null +++ b/test/cli/cli_https_test.go @@ -0,0 +1,46 @@ +package cli + +import ( + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/require" +) + +func TestCLIWithRemoteHTTPS(t *testing.T) { + tests := []struct{ addrSuffix string }{{"https"}, {"tls/http"}} + for _, tt := range tests { + t.Run("with "+tt.addrSuffix+" multiaddr", func(t *testing.T) { + + // Create HTTPS test server + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.TLS == nil { + t.Error("Mocked Kubo RPC received plain HTTP request instead of HTTPS TLS Handshake") + } + _, _ = w.Write([]byte("OK")) + })) + defer server.Close() + + serverURL, _ := url.Parse(server.URL) + _, port, _ := net.SplitHostPort(serverURL.Host) + + // Create Kubo repo + node := harness.NewT(t).NewNode().Init() + + // Attempt to talk to remote Kubo RPC endpoint over HTTPS + resp := node.RunIPFS("id", "--api", fmt.Sprintf("/ip4/127.0.0.1/tcp/%s/%s", port, tt.addrSuffix)) + + // Expect HTTPS error (confirming TLS and https:// were used, and not Cleartext HTTP) + require.Error(t, resp.Err) + require.Contains(t, resp.Stderr.String(), "Error: tls: failed to verify certificate: x509: certificate signed by unknown authority") + + node.StopDaemon() + + }) + } +}