From ca9f6404b3d8e6f26931e50d57547b6a6427bf5d Mon Sep 17 00:00:00 2001 From: Jonathan Knight Date: Wed, 7 May 2025 16:32:10 +0300 Subject: [PATCH 1/5] Add auth token for GitHub doc link checks --- utils/linkcheck/main.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/utils/linkcheck/main.go b/utils/linkcheck/main.go index c9737bc0..a18b29dc 100644 --- a/utils/linkcheck/main.go +++ b/utils/linkcheck/main.go @@ -277,7 +277,25 @@ func checkURL(urlToGet *url.URL, fragments []string) int { Timeout: time.Minute * 1, } - if resp, err = netClient.Get(urlToGet.String()); err != nil { + urlStr := urlToGet.String() + + // Create a new request using http + req, err := http.NewRequest("GET", urlStr, nil) + if err != nil { + fmt.Printf(" FAILED error: %v\n", err) + return 1 + } + + if strings.HasPrefix(urlStr, "https://github.com") { + if token, found := os.LookupEnv("GH_TOKEN"); found { + // Create a Bearer string by appending string access token + var bearer = "Bearer " + token + // add authorization header to the req + req.Header.Add("Authorization", bearer) + } + } + + if resp, err = netClient.Do(req); err != nil { fmt.Printf(" FAILED error: %v\n", err) return 1 } From fe089954ff47d4336817d1bf00ae85cef3db4c8f Mon Sep 17 00:00:00 2001 From: Jonathan Knight Date: Wed, 7 May 2025 16:56:32 +0300 Subject: [PATCH 2/5] Add auth token for GitHub doc link checks --- utils/linkcheck/main.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/utils/linkcheck/main.go b/utils/linkcheck/main.go index a18b29dc..0403df41 100644 --- a/utils/linkcheck/main.go +++ b/utils/linkcheck/main.go @@ -287,11 +287,14 @@ func checkURL(urlToGet *url.URL, fragments []string) int { } if strings.HasPrefix(urlStr, "https://github.com") { - if token, found := os.LookupEnv("GH_TOKEN"); found { + if token, found := os.LookupEnv("GH_TOKEN"); found && token != "" { // Create a Bearer string by appending string access token var bearer = "Bearer " + token // add authorization header to the req req.Header.Add("Authorization", bearer) + fmt.Printf("URL is GitHub GH_TOKEN is set") + } else { + fmt.Printf("URL is GitHub but no auth token in GH_TOKEN") } } @@ -303,7 +306,11 @@ func checkURL(urlToGet *url.URL, fragments []string) int { // Check if request was successful if resp.StatusCode != 200 { - fmt.Printf(" FAILED response: %d\n", resp.StatusCode) + body := "" + if content, err = io.ReadAll(resp.Body); err == nil { + body = string(content) + } + fmt.Printf(" FAILED response: %d %s\n", resp.StatusCode, body) return 1 } From e3cd41e0e54db2ac8363b29a0b6458c580e1e9d4 Mon Sep 17 00:00:00 2001 From: Jonathan Knight Date: Wed, 7 May 2025 18:36:51 +0300 Subject: [PATCH 3/5] Add auth token for GitHub doc link checks --- utils/linkcheck/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/linkcheck/main.go b/utils/linkcheck/main.go index 0403df41..d0539914 100644 --- a/utils/linkcheck/main.go +++ b/utils/linkcheck/main.go @@ -304,13 +304,13 @@ func checkURL(urlToGet *url.URL, fragments []string) int { } defer resp.Body.Close() - // Check if request was successful + // Check if the request was successful if resp.StatusCode != 200 { body := "" if content, err = io.ReadAll(resp.Body); err == nil { body = string(content) } - fmt.Printf(" FAILED response: %d %s\n", resp.StatusCode, body) + fmt.Printf(" FAILED response: %d\n%v\n%s\n", resp.StatusCode, body, resp.Header) return 1 } From 2f14d63d05f9b011df15018d76b071b2692e4967 Mon Sep 17 00:00:00 2001 From: Jonathan Knight Date: Thu, 8 May 2025 12:20:32 +0300 Subject: [PATCH 4/5] Add auth token for GitHub doc link checks --- utils/linkcheck/main.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/utils/linkcheck/main.go b/utils/linkcheck/main.go index d0539914..a3e7cbc6 100644 --- a/utils/linkcheck/main.go +++ b/utils/linkcheck/main.go @@ -292,9 +292,9 @@ func checkURL(urlToGet *url.URL, fragments []string) int { var bearer = "Bearer " + token // add authorization header to the req req.Header.Add("Authorization", bearer) - fmt.Printf("URL is GitHub GH_TOKEN is set") + fmt.Print(" (URL is GitHub, GH_TOKEN is set)") } else { - fmt.Printf("URL is GitHub but no auth token in GH_TOKEN") + fmt.Print(" (URL is GitHub, but no auth token in GH_TOKEN)") } } @@ -302,6 +302,24 @@ func checkURL(urlToGet *url.URL, fragments []string) int { fmt.Printf(" FAILED error: %v\n", err) return 1 } + + if resp.StatusCode == 429 { + for i := 0; i < 10; i++ { + _ = resp.Body.Close() + fmt.Println(" received 429 status waiting for one minute") + time.Sleep(1 * time.Minute) + + if resp, err = netClient.Do(req); err != nil { + fmt.Printf(" FAILED error: %v\n", err) + return 1 + } + + if resp.StatusCode != 429 { + break + } + } + } + defer resp.Body.Close() // Check if the request was successful From 8a0b47070bfdaa86ab4fccdb29df10f69975d44e Mon Sep 17 00:00:00 2001 From: Jonathan Knight Date: Thu, 8 May 2025 12:34:32 +0300 Subject: [PATCH 5/5] Add auth token for GitHub doc link checks --- utils/linkcheck/main.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils/linkcheck/main.go b/utils/linkcheck/main.go index a3e7cbc6..c74090a0 100644 --- a/utils/linkcheck/main.go +++ b/utils/linkcheck/main.go @@ -19,6 +19,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sort" + "strconv" "strings" "time" ) @@ -303,8 +304,15 @@ func checkURL(urlToGet *url.URL, fragments []string) int { return 1 } + retryCount := 10 + if retryCountStr, found := os.LookupEnv("LINK_CHECK_RETRY_COUNT"); found { + if c, err := strconv.Atoi(retryCountStr); err == nil { + retryCount = c + } + } + if resp.StatusCode == 429 { - for i := 0; i < 10; i++ { + for i := 0; i < retryCount; i++ { _ = resp.Body.Close() fmt.Println(" received 429 status waiting for one minute") time.Sleep(1 * time.Minute)