From 3f2acae9191af577898f73136d798a03c55521a7 Mon Sep 17 00:00:00 2001 From: Mike Jones Date: Wed, 11 Dec 2024 21:53:06 +0000 Subject: [PATCH 1/5] Update filter_by_upstream.py Fixed matching of domains and updated the help. --- proxy/plugin/filter_by_upstream.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/plugin/filter_by_upstream.py b/proxy/plugin/filter_by_upstream.py index 39f70859f4..a5867401ce 100644 --- a/proxy/plugin/filter_by_upstream.py +++ b/proxy/plugin/filter_by_upstream.py @@ -22,7 +22,7 @@ '--filtered-upstream-hosts', type=str, default='facebook.com,www.facebook.com', - help='Default: Blocks Facebook. Comma separated list of IPv4 and IPv6 addresses.', + help='Default: Blocks Facebook. Comma separated list of fully qualified domain names, e.g. "facebook.com,www.facebook.com".', ) @@ -32,7 +32,7 @@ class FilterByUpstreamHostPlugin(HttpProxyBasePlugin): def before_upstream_connection( self, request: HttpParser, ) -> Optional[HttpParser]: - if text_(request.host) in self.flags.filtered_upstream_hosts.split(','): + if request.host.decode() in self.flags.filtered_upstream_hosts.split(','): raise HttpRequestRejected( status_code=httpStatusCodes.I_AM_A_TEAPOT, reason=b'I\'m a tea pot', From 9fab532a7aaaf90cf400c2c02796b048deaa0fcf Mon Sep 17 00:00:00 2001 From: Mike Jones Date: Wed, 11 Dec 2024 23:01:27 +0000 Subject: [PATCH 2/5] whitelist python code --- proxy/plugin/whitelist_upstream_hosts.py | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 proxy/plugin/whitelist_upstream_hosts.py diff --git a/proxy/plugin/whitelist_upstream_hosts.py b/proxy/plugin/whitelist_upstream_hosts.py new file mode 100644 index 0000000000..2d3f69a489 --- /dev/null +++ b/proxy/plugin/whitelist_upstream_hosts.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + Whitelist Plugin modified from (broken) filter_by_upstream plugin by Mike Jones dr.mike.jones@gmail.com + +""" +from typing import Optional + +from ..http import httpStatusCodes +from ..http.proxy import HttpProxyBasePlugin +from ..common.flag import flags +from ..http.parser import HttpParser +from ..common.utils import text_ +from ..http.exception import HttpRequestRejected + +flags.add_argument( + '--whitelist-upstream-hosts', + type=str, + default='datasets.datalad.org,singularity-hub.org', + help='Default: Allows Singularity and Datasets. Comma separated list of domains.', +) + + +class WhitelistUpstreamHostsPlugin(HttpProxyBasePlugin): + """Drop traffic by inspecting upstream host.""" + def before_upstream_connection( + self, request: HttpParser, + ) -> Optional[HttpParser]: + if not request.host.decode() in self.flags.whitelist_upstream_hosts.split(','): + raise HttpRequestRejected( + status_code=httpStatusCodes.I_AM_A_TEAPOT, + reason=b'I\'m a tea pot', + ) + return request From e37e789392c57f9b2a9fbffcd21a44471f27ad13 Mon Sep 17 00:00:00 2001 From: Mike Jones Date: Wed, 11 Dec 2024 23:03:22 +0000 Subject: [PATCH 3/5] Update __init__.py in plugins dir- to activate whitelist plugin --- proxy/plugin/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index 48b7996d56..7ae794d9f2 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -35,6 +35,7 @@ from .modify_request_header import ModifyRequestHeaderPlugin from .redirect_to_custom_server import RedirectToCustomServerPlugin from .tls_intercept_conditionally import TlsInterceptConditionallyPlugin +from .whitelist_upstream_hosts import WhitelistUpstreamHostsPlugin __all__ = [ @@ -57,4 +58,5 @@ 'ProgramNamePlugin', 'ModifyRequestHeaderPlugin', 'TlsInterceptConditionallyPlugin', + 'WhitelistUpstreamHostsPlugin', ] From d85508367f71b8faf68b4cdc0adb02c84d636651 Mon Sep 17 00:00:00 2001 From: Mike Jones Date: Wed, 11 Dec 2024 23:17:52 +0000 Subject: [PATCH 4/5] Update README.md ...for WhitelistUpstreamHostPlugin --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index e176ee51b8..a1ddb9f238 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ - [Mock Api Plugin](#mockrestapiplugin) - [Redirect To Custom Server Plugin](#redirecttocustomserverplugin) - [Filter By Upstream Host Plugin](#filterbyupstreamhostplugin) + - [Whitelist_Upstream_Hosts_Plugin](#whitelistupstreamhostplugin) - [Cache Responses Plugin](#cacheresponsesplugin) - [Cache By Response Type](#cachebyresponsetype) - [Man-In-The-Middle Plugin](#maninthemiddleplugin) @@ -729,6 +730,12 @@ Traceback (most recent call last): ... [redacted] ... ... [redacted] ... - access_log:1157 - ::1:49911 - GET None:None/ - None None - 0 bytes ``` +### WhitelistUpstreamHostPlugin + +Essentially the same as [Filter By Upstream Host Plugin](#filterbyupstreamhostplugin) +except the list of hosts are whitelisted, all others are dropped as being tea-pots that don't serve coffee. + +Can take a comma-separated (no-space) list of domains to construct whitelist. Otherwise The whitelist defaults to datasets.datalad.org,singularity-hub.org (because the author needed singularity to run on workernodes in an HPC setting with no NAT to the outside world). ### CacheResponsesPlugin From c74cc3a989e60cc41d7555eabb046f2082d96dbc Mon Sep 17 00:00:00 2001 From: Mike Jones Date: Mon, 27 Jan 2025 15:58:41 +0000 Subject: [PATCH 5/5] Update whitelist_upstream_hosts.py Added galaxy-dev server as per code on archer2 -18 Dec 24. Also added report of server proxy to route to. --- proxy/plugin/whitelist_upstream_hosts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/plugin/whitelist_upstream_hosts.py b/proxy/plugin/whitelist_upstream_hosts.py index 2d3f69a489..674e10a2be 100644 --- a/proxy/plugin/whitelist_upstream_hosts.py +++ b/proxy/plugin/whitelist_upstream_hosts.py @@ -23,7 +23,7 @@ flags.add_argument( '--whitelist-upstream-hosts', type=str, - default='datasets.datalad.org,singularity-hub.org', + default='datasets.datalad.org,singularity-hub.org,galaxy-dev.mcfe.itservices.manchester.ac.uk', help='Default: Allows Singularity and Datasets. Comma separated list of domains.', ) @@ -36,6 +36,6 @@ def before_upstream_connection( if not request.host.decode() in self.flags.whitelist_upstream_hosts.split(','): raise HttpRequestRejected( status_code=httpStatusCodes.I_AM_A_TEAPOT, - reason=b'I\'m a tea pot', + reason=b'I\'m a tea pot cannot conect to '+request.host, ) return request