Test cert setup

.ci/gen_certs.py

@@ -0,0 +1,59 @@
+import argparse
+import os
+import sys
+import typing as t
+
+import trustme
+
+
+def main(argv: t.Optional[t.List[str]] = None) -> None:
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="gen_certs")
+    parser.add_argument(
+        "-d",
+        "--dir",
+        default=os.getcwd(),
+        help="Directory where certificates and keys are written to. Defaults to cwd.",
+    )
+
+    args = parser.parse_args(argv)
+    cert_dir = args.dir
+
+    if not os.path.isdir(cert_dir):
+        raise ValueError(f"--dir={cert_dir} is not a directory")
+
+    key_type = trustme.KeyType["ECDSA"]
+
+    # Generate the CA certificate
+    ca = trustme.CA(key_type=key_type)
+    # Write the certificate the client should trust
+    ca_cert_path = os.path.join(cert_dir, "ca.pem")
+    ca.cert_pem.write_to_path(path=ca_cert_path)
+
+    # Generate the server certificate
+    server_cert = ca.issue_cert("localhost", "127.0.0.1", "::1", key_type=key_type)
+    # Write the certificate and private key the server should use
+    server_key_path = os.path.join(cert_dir, "server.key")
+    server_cert_path = os.path.join(cert_dir, "server.pem")
+    server_cert.private_key_pem.write_to_path(path=server_key_path)
+    with open(server_cert_path, mode="w") as f:
+        f.truncate()
+    for blob in server_cert.cert_chain_pems:
+        blob.write_to_path(path=server_cert_path, append=True)
+
+    # Generate the client certificate
+    client_cert = ca.issue_cert("admin@example.com", common_name="admin", key_type=key_type)
+    # Write the certificate and private key the client should use
+    client_key_path = os.path.join(cert_dir, "client.key")
+    client_cert_path = os.path.join(cert_dir, "client.pem")
+    client_cert.private_key_pem.write_to_path(path=client_key_path)
+    with open(client_cert_path, mode="w") as f:
+        f.truncate()
+    for blob in client_cert.cert_chain_pems:
+        blob.write_to_path(path=client_cert_path, append=True)
+
+
+if __name__ == "__main__":
+    main()

.ci/nginx.conf.j2

@@ -0,0 +1,147 @@
+# Copy from pulp-oci-images.
+# Ideally we can get it upstream again.
+#
+# TODO: Support IPv6.
+# TODO: Maybe serve multiple `location`s, not just one.
+
+# The "nginx" package on fedora creates this user and group.
+user nginx nginx;
+# Gunicorn docs suggest this value.
+worker_processes 1;
+daemon off;
+events {
+    worker_connections 1024;  # increase if you have lots of clients
+    accept_mutex off;  # set to 'on' if nginx worker_processes > 1
+}
+
+http {
+    include mime.types;
+    # fallback in case we can't determine a type
+    default_type application/octet-stream;
+    sendfile on;
+
+    # If left at the default of 1024, nginx emits a warning about being unable
+    # to build optimal hash types.
+    types_hash_max_size 4096;
+
+    map $ssl_client_s_dn $ssl_client_s_dn_cn {
+        default "";
+        ~CN=(?<CN>[^,]+) $CN;
+    }
+
+    upstream pulp-content {
+         server 127.0.0.1:24816;
+    }
+
+    upstream pulp-api {
+         server 127.0.0.1:24817;
+    }
+
+    server {
+        # Gunicorn docs suggest the use of the "deferred" directive on Linux.
+        {% if https | default(false) -%}
+        listen 443 default_server deferred ssl;
+
+        ssl_certificate /etc/pulp/certs/pulp_webserver.crt;
+        ssl_certificate_key /etc/pulp/certs/pulp_webserver.key;
+        ssl_session_cache shared:SSL:50m;
+        ssl_session_timeout 1d;
+        ssl_session_tickets off;
+
+        # intermediate configuration
+        ssl_protocols TLSv1.2;
+        ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
+        ssl_prefer_server_ciphers on;
+
+        # HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months)
+        add_header Strict-Transport-Security max-age=15768000;
+
+        # Configure client cert authentication
+        ssl_client_certificate /etc/pulp/certs/ca.pem;
+        ssl_verify_client optional;
+        {%- else -%}
+        listen 80 default_server deferred;
+        {%- endif %}
+        server_name $hostname;
+
+        # The default client_max_body_size is 1m. Clients uploading
+        # files larger than this will need to chunk said files.
+        client_max_body_size 10m;
+
+        # Gunicorn docs suggest this value.
+        keepalive_timeout 5;
+
+        location {{ content_path }} {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-content;
+        }
+
+        location {{ api_root }}api/v3/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            proxy_set_header Remoteuser $ssl_client_s_dn_cn;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            client_max_body_size 0;
+        }
+
+        {%- if domain_enabled | default(false) %}
+        location ~ {{ api_root }}.+/api/v3/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            client_max_body_size 0;
+        }
+        {%- endif %}
+
+        location /auth/login/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+        }
+
+        include pulp/*.conf;
+
+        location / {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            # most pulp static files are served through whitenoise
+            # http://whitenoise.evans.io/en/stable/
+        }
+
+        {%- if https | default(false) %}
+        # ACME http-01 tokens, i.e, for Let's Encrypt
+        location /.well-known/ {
+            try_files $uri $uri/ =404;
+        }
+        {%- endif %}
+    }
+    {%- if https | default(false) %}
+    server {
+        listen 80 default_server;
+        server_name _;
+        return 301 https://$host$request_uri;
+    }
+    {%- endif %}
+}

.ci/run_container.sh

@@ -16,11 +16,12 @@ then
 fi
 export CONTAINER_RUNTIME
 
-TMPDIR="$(mktemp -d)"
+PULP_CLI_TEST_TMPDIR="$(mktemp -d)"
+export PULP_CLI_TEST_TMPDIR
 
 cleanup () {
   "${CONTAINER_RUNTIME}" stop pulp-ephemeral && true
-  rm -rf "${TMPDIR}"
+  rm -rf "${PULP_CLI_TEST_TMPDIR}"
 }
 
 trap cleanup EXIT
@@ -48,8 +49,8 @@ else
     SELINUX=""
 fi;
 
-mkdir -p "${TMPDIR}/settings/certs"
-cp "${BASEPATH}/settings/settings.py" "${TMPDIR}/settings"
+mkdir -p "${PULP_CLI_TEST_TMPDIR}/settings/certs"
+cp "${BASEPATH}/settings/settings.py" "${PULP_CLI_TEST_TMPDIR}/settings"
 
 if [ -z "${PULP_HTTPS:+x}" ]
 then
@@ -60,10 +61,16 @@ else
   PROTOCOL="https"
   PORT="443"
   PULP_CONTENT_ORIGIN="https://localhost:8080/"
-  python3 -m trustme -d "${TMPDIR}/settings/certs"
-  export PULP_CA_BUNDLE="${TMPDIR}/settings/certs/client.pem"
-  ln -fs server.pem "${TMPDIR}/settings/certs/pulp_webserver.crt"
-  ln -fs server.key "${TMPDIR}/settings/certs/pulp_webserver.key"
+  python3 "${BASEPATH}/gen_certs.py" -d "${PULP_CLI_TEST_TMPDIR}/settings/certs"
+  export PULP_CA_BUNDLE="${PULP_CLI_TEST_TMPDIR}/settings/certs/ca.pem"
+  ln -fs server.pem "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.crt"
+  ln -fs server.key "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.key"
+  {
+    echo "AUTHENTICATION_BACKENDS = '@merge django.contrib.auth.backends.RemoteUserBackend'"
+    echo "MIDDLEWARE  = '@merge django.contrib.auth.middleware.RemoteUserMiddleware'"
+    echo "REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES = '@merge pulpcore.app.authentication.PulpRemoteUserAuthentication'"
+    echo "REMOTE_USER_ENVIRON_NAME = 'HTTP_REMOTEUSER'"
+  } >> "${PULP_CLI_TEST_TMPDIR}/settings/settings.py"
 fi
 export PULP_CONTENT_ORIGIN
 
@@ -75,7 +82,8 @@ export PULP_CONTENT_ORIGIN
   --env PULP_CONTENT_ORIGIN \
   --detach \
   --name "pulp-ephemeral" \
-  --volume "${TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \
+  --volume "${PULP_CLI_TEST_TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \
+  --volume "${BASEPATH}/nginx.conf.j2:/nginx/nginx.conf.j2${SELINUX:+:Z}" \
   --network bridge \
   --publish "8080:${PORT}" \
   "ghcr.io/pulp/pulp:${IMAGE_TAG}"
@@ -105,7 +113,7 @@ done
 "${CONTAINER_RUNTIME}" exec "pulp-ephemeral" pulpcore-manager reset-admin-password --password password
 
 # Create pulp config
-PULP_CLI_CONFIG="${TMPDIR}/settings/certs/cli.toml"
+PULP_CLI_CONFIG="${PULP_CLI_TEST_TMPDIR}/settings/certs/cli.toml"
 export PULP_CLI_CONFIG
 pulp config create --overwrite --location "${PULP_CLI_CONFIG}" --base-url "${PROTOCOL}://localhost:8080" ${PULP_API_ROOT:+--api-root "${PULP_API_ROOT}"} --username "admin" --password "password"
 # show pulpcore/plugin versions we're using

.ci/settings/settings.py

@@ -1,4 +1,3 @@
 ALLOWED_EXPORT_PATHS = ["/tmp"]
-ORPHAN_PROTECTION_TIME = 0
 ANALYTICS = False
 ALLOWED_CONTENT_CHECKSUMS = ["sha1", "sha256", "sha512"]

.github/workflows/test.yml

@@ -23,19 +23,20 @@ jobs:
             python: "3.11"
           - image_tag: "latest"
             python: "3.11"
+          - image_tag: "3.49"
+            pulp_https: true
+            python: "3.10"
           - image_tag: "3.39"
             python: "3.12"
           - image_tag: "3.28"
-            lower_bounds: true
             python: "3.12"
+            lower_bounds: true
           - image_tag: "3.22"
             python: "3.8"
           - image_tag: "3.21"
             pulp_api_root: "/relocated/djnd/"
             python: "3.9"
-          - image_tag: "3.18"
             lower_bounds: true
-            python: "3.10"
     steps:
       - uses: "actions/checkout@v4"
       - uses: "actions/cache@v4"

pulp-glue/tests/conftest.py

@@ -21,7 +21,13 @@ def pulp_ctx(
     return PulpTestContext(
         api_kwargs={
             "base_url": settings["base_url"],
-            "auth_provider": BasicAuthProvider(settings.get("username"), settings.get("password")),
+            "auth_provider": (
+                BasicAuthProvider(settings.get("username"), settings.get("password"))
+                if "username" in settings
+                else None
+            ),
+            "cert": settings.get("cert"),
+            "key": settings.get("key"),
             "debug_callback": lambda i, s: i <= verbose and print(s),
         },
         api_root=settings.get("api_root", "pulp/"),
@@ -39,7 +45,13 @@ def fake_pulp_ctx(
     return PulpTestContext(
         api_kwargs={
             "base_url": settings["base_url"],
-            "auth_provider": BasicAuthProvider(settings.get("username"), settings.get("password")),
+            "auth_provider": (
+                BasicAuthProvider(settings.get("username"), settings.get("password"))
+                if "username" in settings
+                else None
+            ),
+            "cert": settings.get("cert"),
+            "key": settings.get("key"),
             "debug_callback": lambda i, s: i <= verbose and print(s),
         },
         api_root=settings.get("api_root", "pulp/"),

pytest_pulp_cli/__init__.py

@@ -88,13 +88,25 @@ def pulp_cli_settings() -> t.Dict[str, t.Dict[str, t.Any]]:
     It is most likely not useful to be included standalone.
     The `pulp_cli_env` fixture, however depends on it and sets $XDG_CONFIG_HOME up accordingly.
     """
-    settings = toml.load(os.environ.get("PULP_CLI_CONFIG", "tests/cli.toml"))
+    pulp_cli_test_tmpdir = pathlib.Path(os.environ.get("PULP_CLI_TEST_TMPDIR", "."))
+    settings = {"cli": toml.load(os.environ.get("PULP_CLI_CONFIG", "tests/cli.toml"))["cli"]}
     if os.environ.get("PULP_HTTPS"):
-        for key in settings:
-            settings[key]["base_url"] = settings[key]["base_url"].replace("http://", "https://")
+        settings["cli"]["base_url"] = settings["cli"]["base_url"].replace("http://", "https://")
+        client_cert_path = pulp_cli_test_tmpdir / "settings" / "certs" / "client.pem"
+        client_key_path = pulp_cli_test_tmpdir / "settings" / "certs" / "client.key"
+        if client_cert_path.exists():
+            settings["cli"].pop("username", None)
+            settings["cli"].pop("password", None)
+            settings["cli"]["cert"] = str(client_cert_path)
+            if client_key_path.exists():
+                settings["cli"]["key"] = str(client_key_path)
+
     if os.environ.get("PULP_API_ROOT"):
-        for key in settings:
-            settings[key]["api_root"] = os.environ["PULP_API_ROOT"]
+        settings["cli"]["api_root"] = os.environ["PULP_API_ROOT"]
+
+    settings["cli-noauth"] = {
+        k: v for k, v in settings["cli"].items() if k not in {"username", "password", "cert", "key"}
+    }
     return settings
 
 

tests/scripts/pulp_container/test_role.sh

@@ -18,17 +18,17 @@ expect_succ pulp user create --username "clitest" --password "${USERPASS}"
 expect_succ pulp group create --name "clitest"
 expect_succ pulp group user add --group "clitest" --username "clitest"
 
-expect_succ pulp --username clitest --password "${USERPASS}" container repository list
+expect_succ pulp -p noauth --username clitest --password "${USERPASS}" container repository list
 test "$(echo "${OUTPUT}" | jq -r 'length' )" = "0"
 
-expect_deny pulp --username clitest --password "${USERPASS}" container repository create --name "clitest"
+expect_deny pulp -p noauth --username clitest --password "${USERPASS}" container repository create --name "clitest"
 expect_succ pulp container repository create --name "clitest"
 REPOSITORY_HREF=$(jq -r '.pulp_href' <<<"${OUTPUT}")
-expect_fail pulp --username clitest --password "${USERPASS}" container repository show --repository "clitest"
-expect_fail pulp --username clitest --password "${USERPASS}" container repository show --repository "${REPOSITORY_HREF}"
+expect_fail pulp -p noauth --username clitest --password "${USERPASS}" container repository show --repository "clitest"
+expect_fail pulp -p noauth --username clitest --password "${USERPASS}" container repository show --repository "${REPOSITORY_HREF}"
 
 expect_succ pulp container repository role add --name "clitest" --user "clitest" --role "container.containerrepository_viewer"
-expect_succ pulp --username clitest --password "${USERPASS}" container repository show --repository "clitest"
-expect_succ pulp --username clitest --password "${USERPASS}" container repository show --repository "${REPOSITORY_HREF}"
+expect_succ pulp -p noauth --username clitest --password "${USERPASS}" container repository show --repository "clitest"
+expect_succ pulp -p noauth --username clitest --password "${USERPASS}" container repository show --repository "${REPOSITORY_HREF}"
 
-expect_deny pulp --username clitest --password "${USERPASS}" container repository update --href "${REPOSITORY_HREF}" --retain-repo-versions 1
+expect_deny pulp -p noauth --username clitest --password "${USERPASS}" container repository update --href "${REPOSITORY_HREF}" --retain-repo-versions 1