httpext: replace HTTP digest library

The current HTTP digest library,
`github.com/Soontao/goHttpDigestClient`, has not been updated since
March 2017 and is known to be buggy.

This commit replaces it with `github.com/icholy/digest`, a more modern
library that provides an `http.RoundTripper` implementation, allowing
digest challenges to be reused. This aligns well with our use case.

Closes #800.

Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>

Author: Juneezee

go.mod

@@ -8,7 +8,6 @@ require (
 	buf.build/gen/go/prometheus/prometheus/protocolbuffers/go v1.36.5-20240802094132-5b212ab78fb7.1
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358
 	github.com/PuerkitoBio/goquery v1.9.2
-	github.com/Soontao/goHttpDigestClient v0.0.0-20170320082612-6d28bb1415c5
 	github.com/andybalholm/brotli v1.1.1
 	github.com/chromedp/cdproto v0.0.0-20240919203636-12af5e8a671f
 	github.com/evanw/esbuild v0.25.1
@@ -21,6 +20,7 @@ require (
 	github.com/grafana/xk6-dashboard v0.7.5
 	github.com/grafana/xk6-redis v0.3.3
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
+	github.com/icholy/digest v1.1.0
 	github.com/influxdata/influxdb1-client v0.0.0-20190402204710-8ff2fc3824fc
 	github.com/jhump/protoreflect v1.17.0
 	github.com/klauspost/compress v1.18.0
@@ -31,7 +31,8 @@ require (
 	github.com/mstoykov/atlas v0.0.0-20220811071828-388f114305dd
 	github.com/mstoykov/envconfig v1.5.0
 	github.com/mstoykov/k6-taskqueue-lib v0.1.3
-	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d
+	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_model v0.4.0
 	github.com/serenize/snaker v0.0.0-20201027110005-a7ad2135616e
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/afero v1.1.2
@@ -84,8 +85,6 @@ require (
 	github.com/nxadm/tail v1.4.11 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.16.0
-	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/r3labs/sse/v2 v2.10.0 // indirect

go.sum

@@ -10,8 +10,6 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
 github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
-github.com/Soontao/goHttpDigestClient v0.0.0-20170320082612-6d28bb1415c5 h1:k+1+doEm31k0rRjCjLnGG3YRkuO9ljaEyS2ajZd6GK8=
-github.com/Soontao/goHttpDigestClient v0.0.0-20170320082612-6d28bb1415c5/go.mod h1:5Q4+CyR7+Q3VMG8f78ou+QSX/BNUNUx5W48eFRat8DQ=
 github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
 github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
@@ -91,6 +89,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M=
+github.com/icholy/digest v1.1.0 h1:HfGg9Irj7i+IX1o1QAmPfIBNu/Q5A5Tu3n/MED9k9H4=
+github.com/icholy/digest v1.1.0/go.mod h1:QNrsSGQ5v7v9cReDI0+eyjsXGUoRSUZQHeQ5C4XLa0Y=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20190402204710-8ff2fc3824fc h1:KpMgaYJRieDkHZJWY3LMafvtqS/U8xX6+lUN+OKpl/Y=
@@ -127,8 +127,6 @@ github.com/mstoykov/envconfig v1.5.0 h1:E2FgWf73BQt0ddgn7aoITkQHmgwAcHup1s//MsS5
 github.com/mstoykov/envconfig v1.5.0/go.mod h1:vk/d9jpexY2Z9Bb0uB4Ndesss1Sr0Z9ZiGUrg5o9VGk=
 github.com/mstoykov/k6-taskqueue-lib v0.1.3 h1:sdiSc5NEK/qpQkTQe505vgRYQocZevdO9ON+yMudFqo=
 github.com/mstoykov/k6-taskqueue-lib v0.1.3/go.mod h1:e9R2vtLFHCKT+CMiEjTJVMQiJAi17M1KiXXRs7FYc6w=
-github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ=
-github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U=
 github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
 github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
@@ -345,5 +343,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

lib/netext/httpext/digest_transport.go

@@ -1,10 +1,9 @@
 package httpext
 
 import (
-	"io"
 	"net/http"
 
-	digest "github.com/Soontao/goHttpDigestClient"
+	"github.com/icholy/digest"
 )
 
 type digestTransport struct {
@@ -13,52 +12,20 @@ type digestTransport struct {
 
 // RoundTrip handles digest auth by behaving like an http.RoundTripper
 //
-// TODO: fix - this is a preliminary solution and is somewhat broken! we're
-// always making 2 HTTP requests when digest authentication is enabled... we
-// should cache the nonces and behave more like a browser... or we should
-// ditch the hacky http.RoundTripper approach and write our own client...
-//
-// Github issue: https://github.com/k6io/k6/issues/800
+// GitHub PR: https://github.com/grafana/k6/pull/4599
 func (t digestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	// Make the initial request authentication params to compute the
-	// authorization header
 	username := req.URL.User.Username()
 	password, _ := req.URL.User.Password()
 
 	// Remove the user data from the URL to avoid sending the authorization
 	// header for basic auth
 	req.URL.User = nil
 
-	noAuthResponse, err := t.originalTransport.RoundTrip(req)
-	if err != nil || noAuthResponse.StatusCode != http.StatusUnauthorized {
-		// If there was an error, or if the remote server didn't respond with
-		// status 401, we simply return, so the upstream code can deal with it.
-		return noAuthResponse, err
-	}
-
-	respBody, err := io.ReadAll(noAuthResponse.Body)
-	if err != nil {
-		return nil, err
-	}
-	_ = noAuthResponse.Body.Close()
-
-	// Calculate the Authorization header
-	// TODO: determine if we actually need the body, since I'm not sure that's
-	// what the `entity` means... maybe a moot point if we change the used
-	// digest auth library...
-	challenge := digest.GetChallengeFromHeader(&noAuthResponse.Header)
-	challenge.ComputeResponse(req.Method, req.URL.RequestURI(), string(respBody), username, password)
-	authorization := challenge.ToAuthorizationStr()
-	req.Header.Set(digest.KEY_AUTHORIZATION, authorization)
-
-	if req.GetBody != nil {
-		// Reset the request body if we need to
-		req.Body, err = req.GetBody()
-		if err != nil {
-			return nil, err
-		}
+	digestTr := &digest.Transport{
+		Username:  username,
+		Password:  password,
+		Transport: t.originalTransport,
 	}
 
-	// Actually make the HTTP request with the proper Authorization
-	return t.originalTransport.RoundTrip(req)
+	return digestTr.RoundTrip(req)
 }

lib/netext/httpext/httpdebug_transport.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"net/http/httputil"
 
-	uuid "github.com/nu7hatch/gouuid"
+	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 )
 
@@ -22,10 +22,10 @@ type httpDebugTransport struct {
 //   - https://github.com/k6io/k6/issues/1042
 //   - https://github.com/k6io/k6/issues/774
 func (t httpDebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	id, _ := uuid.NewV4()
-	t.debugRequest(req, id.String())
+	id := uuid.NewString()
+	t.debugRequest(req, id)
 	resp, err := t.originalTransport.RoundTrip(req)
-	t.debugResponse(resp, id.String())
+	t.debugResponse(resp, id)
 	return resp, err
 }
 

lib/netext/httpext/request.go

@@ -205,13 +205,12 @@ func MakeRequest(ctx context.Context, state *lib.State, preq *ParsedHTTPRequest)
 	}
 
 	if preq.Auth == "digest" {
-		// Until digest authentication is refactored, the first response will always
-		// be a 401 error, so we expect that.
+		// The first response will either succeed or return a 401.
 		if tracerTransport.responseCallback != nil {
 			originalResponseCallback := tracerTransport.responseCallback
 			tracerTransport.responseCallback = func(status int) bool {
 				tracerTransport.responseCallback = originalResponseCallback
-				return status == 401
+				return status == 401 || originalResponseCallback(status)
 			}
 		}
 		transport = digestTransport{originalTransport: transport}