diff --git a/.github/workflows/test-kubo-e2e.yml b/.github/workflows/test-kubo-e2e.yml index 1b916bcbb..ad233b383 100644 --- a/.github/workflows/test-kubo-e2e.yml +++ b/.github/workflows/test-kubo-e2e.yml @@ -17,6 +17,8 @@ jobs: defaults: run: shell: bash + permissions: + pull-requests: write # Required for commenting on pull requests steps: - name: Setup Go uses: actions/setup-go@v4 @@ -80,13 +82,13 @@ jobs: - name: Find latest comment id: find-comment if: github.event.pull_request - uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 + uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0 with: issue-number: ${{ github.event.pull_request.number }} body-includes: "Results against Kubo ${{ matrix.target }}" - name: Create comment if: github.event.pull_request - uses: peter-evans/create-or-update-comment@c6c9a1a66007646a28c153e2a8580a5bad27bcfa # v3.0.2 + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 with: issue-number: ${{ github.event.pull_request.number }} comment-id: ${{ steps.find-comment.outputs.comment-id }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b34967c..997ca1de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.0] - 2025-05-28 +### Changed +- Comprehensive tests for HTTP Range Requests over deserialized UnixFS files have been added. The `--specs path-gateway` now requires support for at least single-range requests. Deserialized range-requests can be skipped with `--skip 'TestGatewayUnixFSFileRanges'` [#213](https://github.com/ipfs/gateway-conformance/pull/213) +- Updated dependencies [#236](https://github.com/ipfs/gateway-conformance/pull/236) & [#239](https://github.com/ipfs/gateway-conformance/pull/239) + ## [0.7.1] - 2025-01-03 ### Changed - Expect all URL escapes to use uppercase hex [#232](https://github.com/ipfs/gateway-conformance/pull/232) diff --git a/tests/path_gateway_unixfs_test.go b/tests/path_gateway_unixfs_test.go index d250d4197..ee3662a58 100644 --- a/tests/path_gateway_unixfs_test.go +++ b/tests/path_gateway_unixfs_test.go @@ -477,7 +477,7 @@ func TestGatewaySymlink(t *testing.T) { func TestGatewayUnixFSFileRanges(t *testing.T) { tooling.LogTestGroup(t, GroupUnixFS) - // Multi-range requests MUST conform to the HTTP semantics. The server does not + // Range requests MUST conform to the HTTP semantics. The server does not // need to be able to support returning multiple ranges. However, it must respond // correctly. fixture := car.MustOpenUnixfsCar("path_gateway_unixfs/dir-with-files.car") @@ -488,8 +488,22 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { ) RunWithSpecs(t, SugarTests{ + { + Name: "GET for /ipfs/ file includes Accept-Ranges header", + Hint: "Gateway returns explicit hint that range requests are supported. This is important for interop with HTTP reverse proxies, CDNs, caches.", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#accept-ranges-response-header", + Request: Request(). + Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()), + Response: Expect(). + Status(200). + Headers( + Header("Accept-Ranges").Equals("bytes"), + ). + Body(fixture.MustGetRawData("ascii.txt")), + }, { Name: "GET for /ipfs/ file with single range request includes correct bytes", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", Request: Request(). Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). Headers( @@ -498,13 +512,30 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Response: Expect(). Status(206). Headers( - Header("Content-Type").Contains("text/plain"), Header("Content-Range").Equals("bytes 6-16/31"), ). Body(fixture.MustGetRawData("ascii.txt")[6:17]), }, { - Name: "GET for /ipfs/ file with multiple range request includes correct bytes", + Name: "GET for /ipfs/ file with suffix range request includes correct bytes from the end of file", + Hint: "Ensures it is possible to read the tail of a file", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", + Request: Request(). + Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). + Headers( + Header("Range", "bytes=-3"), + ), + Response: Expect(). + Status(206). + Headers( + Header("Content-Range").Equals("bytes 28-30/31"), + ). + Body(fixture.MustGetRawData("ascii.txt")[28:31]), + }, + { + Name: "GET for /ipfs/ file with multiple range request returned HTTP 206", + Hint: "This test reads Content-Type and Content-Range of response, which enable later tests to check if response was acceptable (either single range, or multiple ones)", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", Request: Request(). Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). Headers( @@ -515,13 +546,16 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { Headers( Header("Content-Type"). Checks(func(v string) bool { + // Not really a test, just inspect value contentType = v - return v != "" + return true }), Header("Content-Range"). ChecksAll(func(v []string) bool { + // Not really a test, just inspect value if len(v) == 1 { contentRange = v[0] + } return true }), @@ -531,49 +565,31 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { tests := SugarTests{} - if strings.Contains(contentType, "text/plain") { - // The server is not able to respond to a multi-range request. Therefore, - // there might be only one range or... just the whole file, depending on the headers. + multipleRangeSupported := strings.Contains(contentType, "multipart/byteranges") + onlySingleRangeSupported := !multipleRangeSupported && contentRange != "" - if contentRange == "" { - // Server does not support range requests and must send back the complete file. - tests = append(tests, SugarTest{ - Name: "GET for /ipfs/ file with multiple range request includes correct bytes", - Request: Request(). - Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). - Headers( - Header("Range", "bytes=6-16,0-4"), - ), - Response: Expect(). - Status(206). - Headers( - Header("Content-Type").Contains("text/plain"), - Header("Content-Range").IsEmpty(), - ). - Body(fixture.MustGetRawData("ascii.txt")), - }) - } else { - // Server supports range requests but only the first range. - tests = append(tests, SugarTest{ - Name: "GET for /ipfs/ file with multiple range request includes correct bytes", - Request: Request(). - Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). - Headers( - Header("Range", "bytes=6-16,0-4"), - ), - Response: Expect(). - Status(206). - Headers( - Header("Content-Type").Contains("text/plain"), - Header("Content-Range", "bytes 6-16/31"), - ). - Body(fixture.MustGetRawData("ascii.txt")[6:17]), - }) - } - } else if strings.Contains(contentType, "multipart/byteranges") { + if onlySingleRangeSupported { + // Server supports range requests but only the first range. + tests = append(tests, SugarTest{ + Name: "GET for /ipfs/ file with multiple range request returns correct bytes for the first range", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", + Request: Request(). + Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). + Headers( + Header("Range", "bytes=6-16,0-4"), + ), + Response: Expect(). + Status(206). + Headers( + Header("Content-Range", "bytes 6-16/31"), + ). + Body(fixture.MustGetRawData("ascii.txt")[6:17]), + }) + } else if multipleRangeSupported { // The server supports responding with multi-range requests. tests = append(tests, SugarTest{ Name: "GET for /ipfs/ file with multiple range request includes correct bytes", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", Request: Request(). Path("/ipfs/{{cid}}/ascii.txt", fixture.MustGetCid()). Headers( @@ -586,16 +602,48 @@ func TestGatewayUnixFSFileRanges(t *testing.T) { ). Body(And( Contains("Content-Range: bytes 6-16/31"), - Contains("Content-Type: text/plain"), Contains(string(fixture.MustGetRawData("ascii.txt")[6:17])), Contains("Content-Range: bytes 0-4/31"), Contains(string(fixture.MustGetRawData("ascii.txt")[0:5])), )), }) } else { - t.Error("Content-Type header did not match any of the accepted options") + t.Error("Content-Range and Content-Type header did not match any of the accepted options for a Range request (neither single or multiple ranges are supported)") } + // Range request should work when unrelated parts of DAG not available. + missingBlockFixture := car.MustOpenUnixfsCar("trustless_gateway_car/file-3k-and-3-blocks-missing-block.car") + tests = append(tests, SugarTest{ + Name: "GET Range of file succeeds even if the gateway is missing a block AFTER the requested range", + Hint: "This MUST succeed despite the fact that bytes beyond the end of range are not retrievable", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", + Request: Request(). + Path("/ipfs/{{cid}}", missingBlockFixture.MustGetCidWithCodec(0x70)). + Headers( + Header("Range", "bytes=997-1000"), + ), + Response: Expect(). + Status(206). + Headers( + Header("Content-Range").Equals("bytes 997-1000/3072"), + ), + }, SugarTest{ + Name: "GET Range of file succeeds even if the gateway is missing a block BEFORE the requested range", + Hint: "This MUST succeed despite the fact that bytes beyond the end of range are not retrievable", + Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header", + Request: Request(). + Path("/ipfs/{{cid}}", missingBlockFixture.MustGetCidWithCodec(0x70)). + Headers( + Header("Range", "bytes=2200-2201"), + ), + Response: Expect(). + Status(206). + Headers( + Header("Content-Range").Equals("bytes 2200-2201/3072"), + ), + }, + ) + RunWithSpecs(t, tests, specs.PathGatewayUnixFS) } diff --git a/tooling/check/check.go b/tooling/check/check.go index 0d70aeb89..29e041409 100644 --- a/tooling/check/check.go +++ b/tooling/check/check.go @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "strings" + "unicode/utf8" "github.com/ipfs/gateway-conformance/tooling/tmpl" ) @@ -157,9 +158,18 @@ func (c CheckIsEqualBytes) Check(v []byte) CheckOutput { } } + var reason string + if utf8.Valid(v) && utf8.Valid(c.Value) { + // Print human-readable plain text, when possible + reason = fmt.Sprintf("expected %q, got %q", c.Value, v) + } else { + // Print byte codes + reason = fmt.Sprintf("expected '%v', got '%v'", c.Value, v) + } + return CheckOutput{ Success: false, - Reason: fmt.Sprintf("expected '%v', got '%v'", c.Value, v), + Reason: reason, } }