From f809cb4230452be415c62f49c0a58409de3db0d2 Mon Sep 17 00:00:00 2001 From: pot <196140351+pot@users.noreply.github.com> Date: Sun, 29 Jun 2025 22:32:29 -0700 Subject: [PATCH 1/5] fix: add temp user agent and temp headers --- .../togetherjava/tjbot/features/rss/RSSHandlerRoutine.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java index 567bbbab95..5529afc82e 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java @@ -101,7 +101,11 @@ public RSSHandlerRoutine(Config config, Database database) { targetChannelPatterns.put(feed, predicate); } }); + this.rssReader = new RssReader(); + this.rssReader.setUserAgent("Mozilla/5.0"); + this.rssReader.addHeader("Accept", "application/xhtml+xml,application/xml;"); + this.rssReader.addHeader("Accept-Language", "en-US,en;"); } @Override From 777a367f114683a1be5e7b88c5adb513a1b44cfb Mon Sep 17 00:00:00 2001 From: Chris Sdogkos Date: Mon, 30 Jun 2025 08:59:34 +0300 Subject: [PATCH 2/5] wip(RSSHandlerRoutine.java): add headers to config Better to have it in the configuration file as default for those who need it than hardcoded in the code. Signed-off-by: Chris Sdogkos --- application/config.json.template | 7 ++++++- .../java/org/togetherjava/tjbot/config/RSSFeedsConfig.java | 4 +++- .../togetherjava/tjbot/features/rss/RSSHandlerRoutine.java | 4 +--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index 3e568329a1..f6837053f5 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -173,7 +173,12 @@ } ], "fallbackChannelPattern": "java-news-and-changes", - "pollIntervalInMinutes": 10 + "pollIntervalInMinutes": 10, + "clientRequestHeaders": { + "User-Agent": "Mozilla/5.0", + "Accept": "application/xhtml+xml,application/xml;", + "Accept-Language": "en-US,en;" + } }, "memberCountCategoryPattern": "Info" } diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java index 1c3371f71a..90bc0594a9 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -16,7 +17,8 @@ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) List feeds, @JsonProperty(value = "fallbackChannelPattern", required = true) String fallbackChannelPattern, - @JsonProperty(value = "pollIntervalInMinutes", required = true) int pollIntervalInMinutes) { + @JsonProperty(value = "pollIntervalInMinutes", required = true) int pollIntervalInMinutes, + @JsonProperty(value = "clientRequestHeaders") Map clientRequestHeaders) { /** * Constructs a new {@link RSSFeedsConfig}. diff --git a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java index 5529afc82e..b736f225d9 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java @@ -103,9 +103,7 @@ public RSSHandlerRoutine(Config config, Database database) { }); this.rssReader = new RssReader(); - this.rssReader.setUserAgent("Mozilla/5.0"); - this.rssReader.addHeader("Accept", "application/xhtml+xml,application/xml;"); - this.rssReader.addHeader("Accept-Language", "en-US,en;"); + this.config.clientRequestHeaders().forEach(this.rssReader::addHeader); } @Override From f3f9c91b3f042ccbc63ddccfa0e1ab26986c7890 Mon Sep 17 00:00:00 2001 From: pot <196140351+pot@users.noreply.github.com> Date: Sun, 29 Jun 2025 23:09:21 -0700 Subject: [PATCH 3/5] docs: add javadocs to rss feed config & update config with more minimal headers --- application/config.json.template | 6 +++--- .../java/org/togetherjava/tjbot/config/RSSFeedsConfig.java | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index f6837053f5..ea34962c1e 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -175,9 +175,9 @@ "fallbackChannelPattern": "java-news-and-changes", "pollIntervalInMinutes": 10, "clientRequestHeaders": { - "User-Agent": "Mozilla/5.0", - "Accept": "application/xhtml+xml,application/xml;", - "Accept-Language": "en-US,en;" + "User-Agent": "Mozilla", + "Accept": "application/xml;", + "Accept-Language": "en-US;" } }, "memberCountCategoryPattern": "Info" diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java index 90bc0594a9..80a1d0ad28 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java @@ -13,6 +13,7 @@ * @param feeds The list of RSS feeds to subscribe to. * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use. * @param pollIntervalInMinutes The interval (in minutes) for polling the RSS feeds for updates. + * @param clientRequestHeaders The headers that will be sent with each request to the RSS feeds. */ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) List feeds, @JsonProperty(value = "fallbackChannelPattern", @@ -26,6 +27,7 @@ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) Lis * @param feeds The list of RSS feeds to subscribe to. * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use. * @param pollIntervalInMinutes The interval (in minutes) for polling the RSS feeds for updates. + * @param clientRequestHeaders The headers that will be sent with each request to the RSS feeds. * @throws NullPointerException if any of the parameters (feeds or fallbackChannelPattern) are * null */ From d373f2177526b9110f959d152a078dcb1533eb7c Mon Sep 17 00:00:00 2001 From: pot <196140351+pot@users.noreply.github.com> Date: Mon, 30 Jun 2025 07:37:25 -0700 Subject: [PATCH 4/5] Update to TJ agent --- application/config.json.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/config.json.template b/application/config.json.template index ea34962c1e..bb286addd0 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -175,7 +175,7 @@ "fallbackChannelPattern": "java-news-and-changes", "pollIntervalInMinutes": 10, "clientRequestHeaders": { - "User-Agent": "Mozilla", + "User-Agent": "TJ-Bot/1.0 (+https://github.com/Together-Java/TJ-Bot)", "Accept": "application/xml;", "Accept-Language": "en-US;" } From 16c84c38f6d73b8dbe6e264b5dbfc824d782ebd5 Mon Sep 17 00:00:00 2001 From: pot <196140351+pot@users.noreply.github.com> Date: Mon, 30 Jun 2025 07:41:19 -0700 Subject: [PATCH 5/5] hardcode user agent --- application/config.json.template | 7 +------ .../java/org/togetherjava/tjbot/config/RSSFeedsConfig.java | 6 +----- .../togetherjava/tjbot/features/rss/RSSHandlerRoutine.java | 4 +++- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index bb286addd0..3e568329a1 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -173,12 +173,7 @@ } ], "fallbackChannelPattern": "java-news-and-changes", - "pollIntervalInMinutes": 10, - "clientRequestHeaders": { - "User-Agent": "TJ-Bot/1.0 (+https://github.com/Together-Java/TJ-Bot)", - "Accept": "application/xml;", - "Accept-Language": "en-US;" - } + "pollIntervalInMinutes": 10 }, "memberCountCategoryPattern": "Info" } diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java index 80a1d0ad28..1c3371f71a 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; -import java.util.Map; import java.util.Objects; /** @@ -13,13 +12,11 @@ * @param feeds The list of RSS feeds to subscribe to. * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use. * @param pollIntervalInMinutes The interval (in minutes) for polling the RSS feeds for updates. - * @param clientRequestHeaders The headers that will be sent with each request to the RSS feeds. */ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) List feeds, @JsonProperty(value = "fallbackChannelPattern", required = true) String fallbackChannelPattern, - @JsonProperty(value = "pollIntervalInMinutes", required = true) int pollIntervalInMinutes, - @JsonProperty(value = "clientRequestHeaders") Map clientRequestHeaders) { + @JsonProperty(value = "pollIntervalInMinutes", required = true) int pollIntervalInMinutes) { /** * Constructs a new {@link RSSFeedsConfig}. @@ -27,7 +24,6 @@ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) Lis * @param feeds The list of RSS feeds to subscribe to. * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use. * @param pollIntervalInMinutes The interval (in minutes) for polling the RSS feeds for updates. - * @param clientRequestHeaders The headers that will be sent with each request to the RSS feeds. * @throws NullPointerException if any of the parameters (feeds or fallbackChannelPattern) are * null */ diff --git a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java index b736f225d9..770915cabe 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java @@ -74,6 +74,8 @@ public final class RSSHandlerRoutine implements Routine { private static final int MAX_CONTENTS = 1000; private static final ZonedDateTime ZONED_TIME_MIN = ZonedDateTime.of(LocalDateTime.MIN, ZoneId.systemDefault()); + private static final String HTTP_USER_AGENT = + "TJ-Bot/1.0 (+https://github.com/Together-Java/TJ-Bot)"; private final RssReader rssReader; private final RSSFeedsConfig config; private final Predicate fallbackChannelPattern; @@ -103,7 +105,7 @@ public RSSHandlerRoutine(Config config, Database database) { }); this.rssReader = new RssReader(); - this.config.clientRequestHeaders().forEach(this.rssReader::addHeader); + this.rssReader.setUserAgent(HTTP_USER_AGENT); } @Override