From b8c44c48c4a91b4621564dcb57ce7bfd52b8bef2 Mon Sep 17 00:00:00 2001 From: Gio Lodi Date: Tue, 6 Sep 2022 13:08:52 +1000 Subject: [PATCH 1/6] Update `gp_downloadmetadata` to offer to retry when receiving a 429 --- .../helper/metadata_download_helper.rb | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb index 06ab2a571..20efca29b 100644 --- a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb +++ b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb @@ -16,12 +16,7 @@ def initialize(target_folder, target_files) def download(target_locale, glotpress_url, is_source) uri = URI(glotpress_url) response = Net::HTTP.get_response(uri) - response = Net::HTTP.get_response(URI.parse(response.header['location'])) if response.code == '301' - - @alternates.clear - loc_data = JSON.parse(response.body) rescue loc_data = nil - parse_data(target_locale, loc_data, is_source) - reparse_alternates(target_locale, loc_data, is_source) unless @alternates.length == 0 + handle_glotpress_download(response: response, locale: target_locale, is_source: is_source) end # Parse JSON data and update the local files @@ -100,6 +95,31 @@ def delete_existing_metadata(target_locale) def get_target_file_path(locale, file_name) "#{@target_folder}/#{locale}/#{file_name}" end + + private + + def handle_glotpress_download(response:, locale:, is_source:) + case response.code + when '200' + # All good, parse the result + @alternates.clear + loc_data = JSON.parse(response.body) rescue loc_data = nil + parse_data(locale, loc_data, is_source) + reparse_alternates(target_locale, loc_data, is_source) unless @alternates.length == 0 + when '301' + # Follow the redirect + download(locale, response.header['location'], is_source) + when '429' + # We got rate-limited, offer to try again + if UI.confirm("Retry downloading `#{locale}` after receiving 429 from the API?") + download(locale, response.uri, is_source) + else + UI.message("Giving up on attempting to download #{locale}.") + end + else + UI.error("Received unexpected #{response.code} from request to URI #{response.uri}") + end + end end end end From 80af5264b7ea0eb0087fdd611004e32fcbc7943d Mon Sep 17 00:00:00 2001 From: Gio Lodi Date: Tue, 6 Sep 2022 13:21:53 +1000 Subject: [PATCH 2/6] Update a couple of `puts` in `gp_downloadmetadata` to `UI.message` --- .../actions/common/gp_downloadmetadata_action.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb b/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb index 801c2c8a9..2bcf688af 100644 --- a/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb +++ b/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb @@ -19,13 +19,13 @@ def self.run(params) params[:locales].each do |loc| if loc.is_a?(Array) - puts "Downloading language: #{loc[1]}" + UI.message "Downloading language: #{loc[1]}" complete_url = "#{params[:project_url]}#{loc[0]}/default/export-translations?filters[status]=current&format=json" downloader.download(loc[1], complete_url, loc[1] == params[:source_locale]) end if loc.is_a?(String) - puts "Downloading language: #{loc}" + UI.message "Downloading language: #{loc}" complete_url = "#{params[:project_url]}#{loc}/default/export-translations?filters[status]=current&format=json" downloader.download(loc, complete_url, loc == params[:source_locale]) end From 31daea98521d7c7c1d00aceed4f1bdb488d0b3c9 Mon Sep 17 00:00:00 2001 From: Gio Lodi Date: Tue, 6 Sep 2022 13:27:53 +1000 Subject: [PATCH 3/6] Add more UI messages to `gp_downloadmetadata` --- .../wpmreleasetoolkit/helper/metadata_download_helper.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb index 20efca29b..b864211aa 100644 --- a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb +++ b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb @@ -102,22 +102,24 @@ def handle_glotpress_download(response:, locale:, is_source:) case response.code when '200' # All good, parse the result + UI.success("Successfully downloaded `#{locale}`.") @alternates.clear loc_data = JSON.parse(response.body) rescue loc_data = nil parse_data(locale, loc_data, is_source) reparse_alternates(target_locale, loc_data, is_source) unless @alternates.length == 0 when '301' # Follow the redirect + UI.message("Received 301 for `#{locale}`. Following redirect...") download(locale, response.header['location'], is_source) when '429' # We got rate-limited, offer to try again if UI.confirm("Retry downloading `#{locale}` after receiving 429 from the API?") download(locale, response.uri, is_source) else - UI.message("Giving up on attempting to download #{locale}.") + UI.error("Abandoning `#{locale}` download as requested.") end else - UI.error("Received unexpected #{response.code} from request to URI #{response.uri}") + UI.error("Received unexpected #{response.code} from request to URI #{response.uri}.") end end end From 0af9cfae6d815055c2acb5b6207f9bf9908de81e Mon Sep 17 00:00:00 2001 From: Gio Lodi Date: Tue, 6 Sep 2022 13:32:19 +1000 Subject: [PATCH 4/6] Add trailing `/` to URL used by `gp_downloadmetadata` to avoid 301 --- .../actions/common/gp_downloadmetadata_action.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb b/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb index 2bcf688af..81d02a5e7 100644 --- a/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb +++ b/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb @@ -20,13 +20,13 @@ def self.run(params) params[:locales].each do |loc| if loc.is_a?(Array) UI.message "Downloading language: #{loc[1]}" - complete_url = "#{params[:project_url]}#{loc[0]}/default/export-translations?filters[status]=current&format=json" + complete_url = "#{params[:project_url]}#{loc[0]}/default/export-translations/?filters[status]=current&format=json" downloader.download(loc[1], complete_url, loc[1] == params[:source_locale]) end if loc.is_a?(String) UI.message "Downloading language: #{loc}" - complete_url = "#{params[:project_url]}#{loc}/default/export-translations?filters[status]=current&format=json" + complete_url = "#{params[:project_url]}#{loc}/default/export-translations/?filters[status]=current&format=json" downloader.download(loc, complete_url, loc == params[:source_locale]) end end From 60106a515d1732609109d6815971bbbf607a4234 Mon Sep 17 00:00:00 2001 From: Gio Lodi Date: Wed, 7 Sep 2022 09:13:54 +1000 Subject: [PATCH 5/6] Show user unhandled `gp_downloadmetadata` errors and ask to continue Co-authored-by: Olivier Halligon --- .../wpmreleasetoolkit/helper/metadata_download_helper.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb index b864211aa..5e108ecc8 100644 --- a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb +++ b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb @@ -119,7 +119,8 @@ def handle_glotpress_download(response:, locale:, is_source:) UI.error("Abandoning `#{locale}` download as requested.") end else - UI.error("Received unexpected #{response.code} from request to URI #{response.uri}.") + message = "Received unexpected #{response.code} from request to URI #{response.uri}." + UI.abort_with_message!(message) unless UI.confirm("#{message} Continue anyway?") end end end From eafb2ed367a3873eb235ae627d62f0cefa75154d Mon Sep 17 00:00:00 2001 From: Gio Lodi Date: Wed, 7 Sep 2022 09:17:47 +1000 Subject: [PATCH 6/6] Add `CHANGELOG.md` entry for #406 --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e8ea90d5..ad8cb3ee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ### Breaking Changes -_None_ +- Propose to retry when `gp_downloadmetadata` receives a `429 - Too Many Requests` error. [#406] ### New Features @@ -14,7 +14,7 @@ _None_ ### Bug Fixes -_None_ +- Update the URL used by `gp_downloadmetadata` to prevent consistent `301` responses. [#406] ### Internal Changes