From 789cfac0863dcb462a05b9d1b2e19aee37d77df2 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sun, 9 Jun 2024 23:40:56 +0200 Subject: [PATCH 1/8] test: suffix range request --- .github/workflows/test-kubo-e2e.yml | 8 +- tests/path_gateway_unixfs_test.go | 127 +++++++++++++++++++--------- 2 files changed, 91 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test-kubo-e2e.yml b/.github/workflows/test-kubo-e2e.yml index 1b916bcbb..3a24cc2f1 100644 --- a/.github/workflows/test-kubo-e2e.yml +++ b/.github/workflows/test-kubo-e2e.yml @@ -13,10 +13,12 @@ jobs: strategy: fail-fast: false matrix: - target: ['latest', 'master'] + target: ['latest', 'master', 'fix/range-suffix'] # TODO: remove fix branch before merge defaults: run: shell: bash + permissions: + pull-requests: write # Required for commenting on pull requests steps: - name: Setup Go uses: actions/setup-go@v4 @@ -80,13 +82,13 @@ jobs: - name: Find latest comment id: find-comment if: github.event.pull_request - uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 + uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0 with: issue-number: ${{ github.event.pull_request.number }} body-includes: "Results against Kubo ${{ matrix.target }}" - name: Create comment if: github.event.pull_request - uses: peter-evans/create-or-update-comment@c6c9a1a66007646a28c153e2a8580a5bad27bcfa # v3.0.2 + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 with: issue-number: ${{ github.event.pull_request.number }} comment-id: ${{ steps.find-comment.outputs.comment-id }} diff --git a/tests/path_gateway_unixfs_test.go b/tests/path_gateway_unixfs_test.go index d250d4197..33eb343b7 100644 --- a/tests/path_gateway_unixfs_test.go +++ b/tests/path_gateway_unixfs_test.go @@ -477,7 +477,7 @@ func TestGatewaySymlink(t *testing.T) { func TestGatewayUnixFSFileRanges(t *testing.T) { tooling.LogTestGroup(t, GroupUnixFS) - // Multi-range requests MUST conform to the HTTP semantics. The server does not + // Range requests MUST conform to the HTTP semantics. The server does not // need to be able to support returning multiple ranges. However, it must respond // correctly. fixture := car.MustOpenUnixfsCar("path_gateway_unixfs/dir-with-files.car") @@ -490,6 +490,7 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { RunWithSpecs(t, SugarTests{ { Name: "GET for /ipfs/ file with single range request includes correct bytes", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", Request: Request(). Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). Headers( @@ -504,7 +505,26 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Body(fixture.MustGetRawData("ascii.txt")[6:17]), }, { - Name: "GET for /ipfs/ file with multiple range request includes correct bytes", + Name: "GET for /ipfs/ file with suffix range request includes correct bytes from the end of file", + Hint: "Ensures it is possible to read the tail of a file", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", + Request: Request(). + Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). + Headers( + Header("Range", "bytes=-3"), + ), + Response: Expect(). + Status(206). + Headers( + Header("Content-Type").Contains("text/plain"), + Header("Content-Range").Equals("bytes 28-30/31"), + ). + Body(fixture.MustGetRawData("ascii.txt")[28:31]), + }, + { + Name: "GET for /ipfs/ file with multiple range request returned HTTP 206", + Hint: "This test reads Content-Type and Content-Range of response, which enable later tests to check if response was acceptable (either single range, or multiple ones)", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", Request: Request(). Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). Headers( @@ -515,6 +535,7 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Headers( Header("Content-Type"). Checks(func(v string) bool { + // Not really a test, just inspect value, make sure an explicit value is present contentType = v return v != "" }), @@ -522,6 +543,7 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { ChecksAll(func(v []string) bool { if len(v) == 1 { contentRange = v[0] + } return true }), @@ -531,49 +553,32 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { tests := SugarTests{} - if strings.Contains(contentType, "text/plain") { - // The server is not able to respond to a multi-range request. Therefore, - // there might be only one range or... just the whole file, depending on the headers. + singleRangeSupported := strings.Contains(contentType, "text/plain") && contentRange != "" + multipleRangeSupported := strings.Contains(contentType, "multipart/byteranges") - if contentRange == "" { - // Server does not support range requests and must send back the complete file. - tests = append(tests, SugarTest{ - Name: "GET for /ipfs/ file with multiple range request includes correct bytes", - Request: Request(). - Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). - Headers( - Header("Range", "bytes=6-16,0-4"), - ), - Response: Expect(). - Status(206). - Headers( - Header("Content-Type").Contains("text/plain"), - Header("Content-Range").IsEmpty(), - ). - Body(fixture.MustGetRawData("ascii.txt")), - }) - } else { - // Server supports range requests but only the first range. - tests = append(tests, SugarTest{ - Name: "GET for /ipfs/ file with multiple range request includes correct bytes", - Request: Request(). - Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). - Headers( - Header("Range", "bytes=6-16,0-4"), - ), - Response: Expect(). - Status(206). - Headers( - Header("Content-Type").Contains("text/plain"), - Header("Content-Range", "bytes 6-16/31"), - ). - Body(fixture.MustGetRawData("ascii.txt")[6:17]), - }) - } - } else if strings.Contains(contentType, "multipart/byteranges") { + if singleRangeSupported { + // Server supports range requests but only the first range. + tests = append(tests, SugarTest{ + Name: "GET for /ipfs/ file with multiple range request returns correct bytes for the first range", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", + Request: Request(). + Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). + Headers( + Header("Range", "bytes=6-16,0-4"), + ), + Response: Expect(). + Status(206). + Headers( + Header("Content-Type").Contains("text/plain"), + Header("Content-Range", "bytes 6-16/31"), + ). + Body(fixture.MustGetRawData("ascii.txt")[6:17]), + }) + } else if multipleRangeSupported { // The server supports responding with multi-range requests. tests = append(tests, SugarTest{ Name: "GET for /ipfs/ file with multiple range request includes correct bytes", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", Request: Request(). Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). Headers( @@ -596,6 +601,46 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { t.Error("Content-Type header did not match any of the accepted options") } + // Range request should work when unrelated parts of DAG not available. + missingBlockFixture := car.MustOpenUnixfsCar("trustless_gateway_car/file-3k-and-3-blocks-missing-block.car") + tests = append(tests, SugarTest{ + Name: "GET Range of file succeeds even if the gateway is missing a block AFTER the requested range", + Hint: "This MUST succeed despite the fact that bytes beyond the end of range are not retrievable", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", + Request: Request(). + Path("/ipfs/{{cid}}", missingBlockFixture.MustGetCidWithCodec(0x70)). + Headers( + Header("Range", "bytes=0-1000"), + ), + Response: Expect(). + Status(206). + Headers( + Header("Content-Type").Contains("text/plain"), + Header("Content-Range").Equals("bytes 0-1000/3072"), + ), + // TODO: we are missing helper for returning byte range from a + // CAR. the fixture here is multi-block, and we can use + // missingBlockFixture.MustGetRawData because raw data spans + // across more than one block. + }, SugarTest{ + Name: "GET Range of file succeeds even if the gateway is missing a block BEFORE the requested range", + Hint: "This MUST succeed despite the fact that bytes beyond the end of range are not retrievable", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", + Request: Request(). + Path("/ipfs/{{cid}}", missingBlockFixture.MustGetCidWithCodec(0x70)). + Headers( + Header("Range", "bytes=2200-2201"), + ), + Response: Expect(). + Status(206). + Headers( + Header("Content-Type").Contains("text/plain"), + Header("Content-Range").Equals("bytes 2200-2201/3072"), + ), + }, + // TODO: port test fror range AFTER missing block as well + ) + RunWithSpecs(t, tests, specs.PathGatewayUnixFS) } From 05ebe96c7b334602c8fbf79ccac6d676adf450a6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 28 May 2025 00:57:16 +0200 Subject: [PATCH 2/8] chore: adjust range to not start at 0 makes test more representative of seeking inside of a partially available file --- tests/path_gateway_unixfs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/path_gateway_unixfs_test.go b/tests/path_gateway_unixfs_test.go index 33eb343b7..215994de3 100644 --- a/tests/path_gateway_unixfs_test.go +++ b/tests/path_gateway_unixfs_test.go @@ -610,13 +610,13 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Request: Request(). Path("/ipfs/{{cid}}", missingBlockFixture.MustGetCidWithCodec(0x70)). Headers( - Header("Range", "bytes=0-1000"), + Header("Range", "bytes=997-1000"), ), Response: Expect(). Status(206). Headers( Header("Content-Type").Contains("text/plain"), - Header("Content-Range").Equals("bytes 0-1000/3072"), + Header("Content-Range").Equals("bytes 997-1000/3072"), ), // TODO: we are missing helper for returning byte range from a // CAR. the fixture here is multi-block, and we can use From 0ac0eb5e0f0ca0b02043cd4b1656e6b562db611d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 28 May 2025 01:26:15 +0200 Subject: [PATCH 3/8] chore: focus on Content-Range content-type is not hard requirement, and implementations may have easier or harder time with coming up with the right one. if the content type info is not part of DAG metadata, and needs to me sniffed (majority of content in 2025), then this internal sniffing may produce different results in different file x library permutations. in practice, it does not matter, browsers will stream videos just fine. lets focus on Content-Range and body being correct - if we need to test Content-Type, let's move it to a different test suite --- tests/path_gateway_unixfs_test.go | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/tests/path_gateway_unixfs_test.go b/tests/path_gateway_unixfs_test.go index 215994de3..cafb191d5 100644 --- a/tests/path_gateway_unixfs_test.go +++ b/tests/path_gateway_unixfs_test.go @@ -499,7 +499,6 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Response: Expect(). Status(206). Headers( - Header("Content-Type").Contains("text/plain"), Header("Content-Range").Equals("bytes 6-16/31"), ). Body(fixture.MustGetRawData("ascii.txt")[6:17]), @@ -516,7 +515,6 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Response: Expect(). Status(206). Headers( - Header("Content-Type").Contains("text/plain"), Header("Content-Range").Equals("bytes 28-30/31"), ). Body(fixture.MustGetRawData("ascii.txt")[28:31]), @@ -535,12 +533,13 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Headers( Header("Content-Type"). Checks(func(v string) bool { - // Not really a test, just inspect value, make sure an explicit value is present + // Not really a test, just inspect value contentType = v - return v != "" + return true }), Header("Content-Range"). ChecksAll(func(v []string) bool { + // Not really a test, just inspect value if len(v) == 1 { contentRange = v[0] @@ -553,10 +552,10 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { tests := SugarTests{} - singleRangeSupported := strings.Contains(contentType, "text/plain") && contentRange != "" multipleRangeSupported := strings.Contains(contentType, "multipart/byteranges") + onlySingleRangeSupported := !multipleRangeSupported && contentRange != "" - if singleRangeSupported { + if onlySingleRangeSupported { // Server supports range requests but only the first range. tests = append(tests, SugarTest{ Name: "GET for /ipfs/ file with multiple range request returns correct bytes for the first range", @@ -569,7 +568,6 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Response: Expect(). Status(206). Headers( - Header("Content-Type").Contains("text/plain"), Header("Content-Range", "bytes 6-16/31"), ). Body(fixture.MustGetRawData("ascii.txt")[6:17]), @@ -591,14 +589,13 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { ). Body(And( Contains("Content-Range: bytes 6-16/31"), - Contains("Content-Type: text/plain"), Contains(string(fixture.MustGetRawData("ascii.txt")[6:17])), Contains("Content-Range: bytes 0-4/31"), Contains(string(fixture.MustGetRawData("ascii.txt")[0:5])), )), }) } else { - t.Error("Content-Type header did not match any of the accepted options") + t.Error("Content-Range and Content-Type header did not match any of the accepted options for a Range request (neither single or multiple ranges are supported)") } // Range request should work when unrelated parts of DAG not available. @@ -615,13 +612,8 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Response: Expect(). Status(206). Headers( - Header("Content-Type").Contains("text/plain"), Header("Content-Range").Equals("bytes 997-1000/3072"), ), - // TODO: we are missing helper for returning byte range from a - // CAR. the fixture here is multi-block, and we can use - // missingBlockFixture.MustGetRawData because raw data spans - // across more than one block. }, SugarTest{ Name: "GET Range of file succeeds even if the gateway is missing a block BEFORE the requested range", Hint: "This MUST succeed despite the fact that bytes beyond the end of range are not retrievable", @@ -634,11 +626,9 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Response: Expect(). Status(206). Headers( - Header("Content-Type").Contains("text/plain"), Header("Content-Range").Equals("bytes 2200-2201/3072"), ), }, - // TODO: port test fror range AFTER missing block as well ) RunWithSpecs(t, tests, specs.PathGatewayUnixFS) From ad744bcd756caa1cdbb4eb2be04367b4ae2ebf0c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 28 May 2025 01:56:59 +0200 Subject: [PATCH 4/8] chore: human-readable check results --- tooling/check/check.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tooling/check/check.go b/tooling/check/check.go index 0d70aeb89..29e041409 100644 --- a/tooling/check/check.go +++ b/tooling/check/check.go @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "strings" + "unicode/utf8" "github.com/ipfs/gateway-conformance/tooling/tmpl" ) @@ -157,9 +158,18 @@ func (c CheckIsEqualBytes) Check(v []byte) CheckOutput { } } + var reason string + if utf8.Valid(v) && utf8.Valid(c.Value) { + // Print human-readable plain text, when possible + reason = fmt.Sprintf("expected %q, got %q", c.Value, v) + } else { + // Print byte codes + reason = fmt.Sprintf("expected '%v', got '%v'", c.Value, v) + } + return CheckOutput{ Success: false, - Reason: fmt.Sprintf("expected '%v', got '%v'", c.Value, v), + Reason: reason, } } From 06ee71b201dfa03647b62b452350a2edb94e8e28 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 28 May 2025 17:09:22 +0200 Subject: [PATCH 5/8] test: Accept-Ranges: bytes present ensure behavior described in https://github.com/ipfs/boxo/issues/856#issuecomment-2679171526 is covered by test --- tests/path_gateway_unixfs_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/path_gateway_unixfs_test.go b/tests/path_gateway_unixfs_test.go index cafb191d5..ee3662a58 100644 --- a/tests/path_gateway_unixfs_test.go +++ b/tests/path_gateway_unixfs_test.go @@ -488,6 +488,19 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { ) RunWithSpecs(t, SugarTests{ + { + Name: "GET for /ipfs/ file includes Accept-Ranges header", + Hint: "Gateway returns explicit hint that range requests are supported. This is important for interop with HTTP reverse proxies, CDNs, caches.", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#accept-ranges-response-header", + Request: Request(). + Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()), + Response: Expect(). + Status(200). + Headers( + Header("Accept-Ranges").Equals("bytes"), + ). + Body(fixture.MustGetRawData("ascii.txt")), + }, { Name: "GET for /ipfs/ file with single range request includes correct bytes", Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", From ea7be5ecc49eee913183056f01656e7500fac2de Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 28 May 2025 17:19:07 +0200 Subject: [PATCH 6/8] chore: restore e2e tests --- .github/workflows/test-kubo-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-kubo-e2e.yml b/.github/workflows/test-kubo-e2e.yml index 3a24cc2f1..ad233b383 100644 --- a/.github/workflows/test-kubo-e2e.yml +++ b/.github/workflows/test-kubo-e2e.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - target: ['latest', 'master', 'fix/range-suffix'] # TODO: remove fix branch before merge + target: ['latest', 'master'] defaults: run: shell: bash From 94ded9d5e5caad3708be1453edfdb4358f1e5707 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 28 May 2025 17:37:29 +0200 Subject: [PATCH 7/8] chore: release as 0.8.0 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b34967c..21bc1d52d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.0] - 2025-05-28 +### Changed +- Comprehensive HTTP Range Requests tests were added. At least single range support is now required by `path-gateway` profile [#213](https://github.com/ipfs/gateway-conformance/pull/213) +- Updated dependencies [#236](https://github.com/ipfs/gateway-conformance/pull/236) & [#239](https://github.com/ipfs/gateway-conformance/pull/239) + ## [0.7.1] - 2025-01-03 ### Changed - Expect all URL escapes to use uppercase hex [#232](https://github.com/ipfs/gateway-conformance/pull/232) From 12cb923bf7a91b8e1e2d5118b0f6958324626d4d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 28 May 2025 17:41:19 +0200 Subject: [PATCH 8/8] docs: improve 0.8.0 changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21bc1d52d..997ca1de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.8.0] - 2025-05-28 ### Changed -- Comprehensive HTTP Range Requests tests were added. At least single range support is now required by `path-gateway` profile [#213](https://github.com/ipfs/gateway-conformance/pull/213) +- Comprehensive tests for HTTP Range Requests over deserialized UnixFS files have been added. The `--specs path-gateway` now requires support for at least single-range requests. Deserialized range-requests can be skipped with `--skip 'TestGatewayUnixFSFileRanges'` [#213](https://github.com/ipfs/gateway-conformance/pull/213) - Updated dependencies [#236](https://github.com/ipfs/gateway-conformance/pull/236) & [#239](https://github.com/ipfs/gateway-conformance/pull/239) ## [0.7.1] - 2025-01-03