Back to upstream and upgrade deps, refactor. Reinitialization

Author: kwarunek

.gitignore

@@ -0,0 +1,2 @@
+/dev
+/*.retry

CHANGELOG.md

@@ -0,0 +1,21 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Support for AWS WebIdentity Token
+- Support for AWS EC2 metadata
+- Log of the AWS identity used
+- Support to set custom renewal interval
+- Fallback to AWS_REGION, AWS_DEFAULT_REGION and region in ECR URL
+
+### Changed
+- Upgraded to OpenResty 1.21.4.1
+- Upgraded AWS CLI to 1.34.21
+- Environment variables names simplified
+- Cleaned up the repository structure

Dockerfile

@@ -1,19 +1,19 @@
-FROM openresty/openresty:1.13.6.1-alpine
+FROM openresty/openresty:1.21.4.1-0-alpine
+
 USER root
 
-RUN apk add -v --no-cache bind-tools python py-pip supervisor \
- && mkdir /cache \
- && addgroup -g 101 nginx \
- && adduser -u 100  -D -S -h /cache -s /sbin/nologin -G nginx nginx \
- && pip install --upgrade pip awscli==1.11.183 \
- && apk -v --purge del py-pip
+RUN apk add -v --no-cache bind-tools python3 py-pip py3-urllib3 py3-colorama supervisor
+RUN mkdir /cache
+RUN mkdir /etc/crontab
+RUN addgroup -g 110 nginx && adduser -u 110  -D -S -h /cache -s /sbin/nologin -G nginx nginx
+RUN pip install --upgrade pip awscli==1.34.21
 
-COPY files/startup.sh files/renew_token.sh /
-COPY files/ecr.ini /etc/supervisor.d/ecr.ini
-COPY files/root /etc/crontabs/root
+COPY scripts/startup.sh /
+COPY scripts/renew_token.sh /
+COPY config/supervisord/programs.ini /etc/supervisor.d/programs.ini
 
-COPY files/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf
-COPY files/ssl.conf /usr/local/openresty/nginx/conf/ssl.conf
+COPY config/nginx/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf
+COPY config/nginx/ssl.conf /usr/local/openresty/nginx/conf/ssl.conf
 
 ENV PORT 5000
 RUN chmod a+x /startup.sh /renew_token.sh

LICENSE

@@ -1,38 +1,70 @@
-aws-ecr-http-proxy
-===========
+# aws-ecr-http-proxy
 
-A very simple nginx proxy that forwards requests to AWS ECR and caches the responses locally.
+A very simple nginx push/pull proxy that forwards requests to AWS ECR and caches the responses locally.
 
-Run it like this, replace UPSTREAM with your target address with following required params:
-- `AWS_REGION`
-- `AWS_ACCESS_KEY_ID`
-- `AWS_SECRET_ACCESS_KEY`
+### Differences Between Fork and Upstream Repository
 
-It is also possible to define `CACHE_MAX_SIZE` env to limit maximum cache size on provided volume
+- Added support for AWS WebIdentity Token
+- Added support for AWS EC2 metadata
+- Added log of the AWS identity used
+- Added support to set custom renewal interval
+- Fallback to AWS_REGION, AWS_DEFAULT_REGION and region in ECR URL
+- Upgraded to OpenResty 1.21.4.1
+- Upgraded AWS CLI to 1.34.21
+- Environment variables names simplified
+- Cleaned up the repository structure
 
-For example:
+### Configuration:
+The proxy is packaged in a docker container and can be configured with following environment variables:
+
+| Environment Variable                | Description                                    | Status                            | Default    |
+| :---------------------------------: | :--------------------------------------------: | :-------------------------------: | :--------: |
+| `ECR`                               | URL for AWS ECR                                | Required                          |            |
+| `RESOLVER`                          | DNS server to be used by proxy                 | Required                          |            |
+| `PORT`                              | Port on which proxy listens                    | Required                          |            |
+| `CACHE_MAX_SIZE`                    | Maximum size for cache volume                  | Optional                          |    `75g`   |
+| `CACHE_KEY`                         | Cache key used for the content by nginx        | Optional                          |   `$uri`   |
+| `RENEW_INTERVAL_HOURS`              | Interval for renewing the AWS credentials      | Optional                          |     `6`    |
+| `ENABLE_SSL`                        | Used to enable SSL/TLS for proxy               | Optional                          |  `false`   |
+| `SSL_KEY`                           | Path to TLS key in the container               | Required with SSL                 |            |
+| `SSL_CERTIFICATE`                   | Path to TLS cert in the container              | Required with SSL                 |            |
+
+
+AWS identity can be passed:
+- using environment variables
+- using AWS credentials file (mounted in the container)
+- using WebIdentity Token (mounted in the container)
+- on AWS EC2 via metadata
+
+If `AWS_REGION` is not set, it will be deduced from ECR URL.
+
+| Environment Variable                | Description                                    | Status                            | Default    |
+| :---------------------------------: | :--------------------------------------------: | :-------------------------------: | :--------: |
+| `AWS_REGION`                        | Region                                         | Optional                          |            |
+| `AWS_ACCESS_KEY_ID`                 | Access key                                     | Optional                          |            |
+| `AWS_SECRET_ACCESS_KEY`             | Secret key                                     | Optional                          |            |
+
+
+### Example:
 
 ```sh
-docker run --rm --name docker-registry-proxy --net=host \
-  -v /local-storage/cache:/cache \
+docker run -d --name docker-registry-proxy --net=host \
+  -v /registry/local-storage/cache:/cache \
+  -v /registry/certificate.pem:/opt/ssl/certificate.pem \
+  -v /registry/key.pem:/opt/ssl/key.pem \
   -e PORT=5000 \
   -e RESOLVER=8.8.8.8 \
-  -e UPSTREAM=https://XXXXXXXXXX.dkr.ecr.eu-central-1.amazonaws.com \
-  -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
-  -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
-  -e AWS_REGION=${AWS_DEFAULT_REGION} \
+  -e ECR=https://XXXXXXXXXX.dkr.ecr.eu-central-1.amazonaws.com \
   -e CACHE_MAX_SIZE=100g \
-  esailors/aws-ecr-http-proxy:latest
+  -e ENABLE_SSL=true \
+  -e SSL_KEY=/opt/ssl/key.pem \
+  -e SSL_CERTIFICATE=/opt/ssl/certificate.pem \
+  aws-ecr-http-proxy:latest
 ```
 
 If you ran this command on "registry-proxy.example.com" you can now get your images using `docker pull registry-proxy.example.com:5000/repo/image`.
 
-### Deploying the proxy
-Modify the ansible role variables according to your need and run the playbook as follow:
-```sh
-ansible-playbook -i hosts playbook-docker-registry-proxy.yaml
-```
-The docker registry for project is available [here](https://hub.docker.com/r/esailors/aws-ecr-http-proxy)
-
-### Note
-The proxy has `HTTP` endpoint so in order to avoid docker client complaining about it either mark the registry host as insecure in your [deamon config](https://docs.docker.com/registry/insecure/) or add [SSL/TLS termination](https://docs.docker.com/registry/recipes/nginx)
+### Note on SSL/TLS
+The proxy is using `HTTP` (plain text) as default protocol for now. So in order to avoid docker client complaining either:
+ - (**Recommended**) Enable SSL/TLS using `ENABLE_SSL` configuration. For that you will have to mount your **valid** certificate/key in the container and pass the paths using  `SSL_*` variables.
+ - Mark the registry host as insecure in your client [deamon config](https://docs.docker.com/registry/insecure/).

README.md

@@ -1,25 +1,24 @@
-user  nginx;
-worker_processes  1;
-
 events {
     worker_connections  1024;
 }
 
+user  nginx;
+worker_processes  1;
+
+
 http {
   include       mime.types;
   default_type  application/octet-stream;
 
   keepalive_timeout 65;
   sendfile on;
 
-  proxy_cache_path /cache/cache levels=1:2 keys_zone=cache:16m inactive=1y max_size=CACHE_MAX_SIZE use_temp_path=off;
-  resolver RESOLVER valid=30s;
+  proxy_cache_path /cache/cache levels=1:2 keys_zone=cache:16m inactive=1y max_size=__CACHE_MAX_SIZE__ use_temp_path=off;
+  resolver __RESOLVER__ valid=30s;
 
   # this is necessary for us to be able to disable request buffering in all cases
   proxy_http_version 1.1;
 
-  #SSLCONFIG
-
   # will run before forking out nginx worker processes
   init_by_lua_block { require "cjson" }
 
@@ -29,9 +28,9 @@ http {
   }
 
   server {
-    listen LISTEN default_server;
+    listen __PORT__ __SSL_LISTEN__ default_server;
 
-    #AUTHCONFIG
+    __SSL_INCLUDE__
 
     # Cache
     add_header X-Cache-Status   $upstream_cache_status;
@@ -52,45 +51,37 @@ http {
 
     # disable proxy request buffering
     proxy_request_buffering off;
-    proxy_cache            cache;
-    proxy_cache_key        $scheme$uri$args$request_method;
-    proxy_cache_valid      200  1s;
-    proxy_cache_use_stale  error timeout invalid_header updating
-                           http_500 http_502 http_503 http_504;
-    proxy_cache_lock       on;
 
     add_header 'Docker-Distribution-Api-Version' $docker_distribution_api_version always;
     add_header "Access-Control-Allow-Origin" "*";
 
     location / {
-      set $url        UPSTREAM;
+      set $url        __ECR__;
       proxy_pass      $url;
-      proxy_redirect  $url http://$host:PORT;
+      proxy_redirect  $url __SCHEME__://$host:__PORT__;
 
       # Add AWS ECR authentication headers
       proxy_set_header  X-Real-IP          $remote_addr;
       proxy_set_header  X-Forwarded-For    $remote_addr;
       proxy_set_header  X-Forwarded-User   "Basic $http_authorization";
       proxy_set_header  Authorization      "Basic $http_authorization";
       proxy_set_header  X-Forwarded-Proto  $scheme;
-      proxy_set_header  Authorization      "";
 
     }
 
     # Content addressable files like blobs.
     # https://docs.docker.com/registry/spec/api/#blob
     location ~ ^/v2/.*/blobs/[a-z0-9]+:[a-f0-9]+$ {
-      set $url        UPSTREAM;
+      set $url        __ECR__;
       proxy_pass      $url;
-      proxy_redirect  $url http://$host:PORT;
+      proxy_redirect  $url __SCHEME__://$host:__PORT__;
 
       # Add AWS ECR authentication headers
       proxy_set_header  X-Real-IP          $remote_addr;
       proxy_set_header  X-Forwarded-For    $remote_addr;
       proxy_set_header  X-Forwarded-User   "Basic $http_authorization";
       proxy_set_header  Authorization      "Basic $http_authorization";
       proxy_set_header  X-Forwarded-Proto  $scheme;
-      proxy_set_header  Authorization      "";
 
       # When accessing image blobs using HTTP GET AWS ECR redirects with
       # s3 buckets uri to download the image. This needs to handled by
@@ -106,7 +97,7 @@ http {
       set                    $saved_redirect_location '$upstream_http_location';
       proxy_pass             $saved_redirect_location;
       proxy_cache            cache;
-      proxy_cache_key        $scheme$uri$args$request_method;
+      proxy_cache_key        __CACHE_KEY__;
       proxy_cache_valid      200  1y;
       proxy_cache_use_stale  error timeout invalid_header updating
                              http_500 http_502 http_503 http_504;
@@ -115,7 +106,6 @@ http {
 
     location ~ ^/v2/.*/.*/tags/list+$ {
       # get paginated list of tags
-      proxy_set_header  Authorization      "";
       content_by_lua_block {
         local location, tags, cjson = ngx.var.uri, {}, require "cjson"
         while true do
@@ -146,8 +136,7 @@ http {
     location /get_tags {
       internal;
       set_unescape_uri      $req_uri $arg_req_uri;
-      proxy_pass            UPSTREAM$req_uri;
-      proxy_set_header  Authorization      "";
+      proxy_pass            __ECR__$req_uri;
 
       # Add AWS ECR authentication headers
       proxy_set_header  X-Real-IP          $remote_addr;

files/nginx.conf renamed to config/nginx/nginx.conf

@@ -1,9 +1,8 @@
-ssl_certificate_key REGISTRY_HTTP_TLS_KEY;
-ssl_certificate     REGISTRY_HTTP_TLS_CERTIFICATE;
+ssl_certificate_key __SSL_KEY__;
+ssl_certificate     __SSL_CERTIFICATE__;
 
 ssl_protocols TLSv1.2;
 ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
 ssl_prefer_server_ciphers on;
 
 add_header Strict-Transport-Security max-age=31536000;
-