Merge pull request #200 from rust-lang/modernize

Refactor highfive to be docker-friendly

Author: pietroalbini

.gitignore

@@ -1,3 +1,3 @@
-config
+/.env
 *.pyc
 *~

.travis.yml

@@ -14,3 +14,12 @@ script:
     else
       pytest -m hermetic
     fi
+
+# Publish the Docker image to the infra AWS registry
+before_deploy:
+  - pip install --user awscli; export PATH=$PATH:$HOME/.local/bin
+deploy:
+  provider: script
+  script: sh ci/publish-docker.sh
+  on:
+    branch: master

Dockerfile

@@ -1,29 +1,21 @@
 FROM ubuntu:bionic
 
-RUN apt-get update && apt-get install -y apache2 \
-  ca-certificates \
-  python
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+    ca-certificates \
+    python \
+    python-setuptools \
+    python-wheel \
+    python-pip
 
-# Set up Highfive
 RUN mkdir /highfive
 WORKDIR /highfive
 
 COPY setup.py .
 COPY highfive/*.py highfive/
 COPY highfive/configs/ highfive/configs/
-RUN python setup.py install
-RUN touch highfive/config
-RUN chown -R www-data:www-data .
-
-# Set up Apache
-WORKDIR /etc/apache2
-COPY deployment/highfive.conf conf-available/highfive.conf
-RUN a2enmod cgi
-RUN rm conf-enabled/serve-cgi-bin.conf
-RUN rm sites-enabled/*
-RUN ln -s ../conf-available/highfive.conf conf-enabled
-
-RUN mkdir /var/log/highfive-tracebacks && chown www-data: /var/log/highfive-tracebacks
+RUN pip install .
 
 EXPOSE 80
-CMD apachectl -D FOREGROUND
+ENV HIGHFIVE_PORT 80
+
+CMD highfive

README.md

@@ -142,26 +142,31 @@ Local Development
 You can run Highfive on your machine and configure a repository to use
 your local instance. Here is one approach for running a local server:
 
-- Use [serve.py](/serve.py) to run the Highfive service. From the
-  repository root, do:
+- Create a [virtualenv](https://virtualenv.pypa.io/en/stable/) to isolate the
+  Python environment from the rest of the system, and install highfive in it:
   ```
-  $ PYTHONPATH=$PYTHONPATH:$PWD python serve.py
+  $ virtualenv -p python2 env
+  $ env/bin/pip install -e .
   ```
-  Now you have Highfive listening on port 8000 of your machine.
-- Your Highfive instance will need to be reachable from outside of your machine. One way to do this is to use [ngrok](https://ngrok.com/) to get a temporary domain name that proxies to your Highfive instance. Additionally, you will be able to use ngrok's inspector to easily examine and replay the requests.
+- Run the highfive command to start a development server on port 8000:
+  ```
+  $ env/bin/highfive
+  ```
+- Your Highfive instance will need to be reachable from outside of your
+  machine. One way to do this is to use [ngrok](https://ngrok.com/) to get a
+  temporary domain name that proxies to your Highfive instance. Additionally,
+  you will be able to use ngrok's inspector to easily examine and replay the
+  requests.
 - Set up the webhook by following the instructions in [Enabling a
   Repo](#enabling-a-repo), substituting your local Highfive IP address
   or domain name and port number (if necessary).
 - Obtain an OAuth token. In the account you are creating the token in,
   go to https://github.com/settings/tokens. Grant access to the repository scope.
-- Put the authorization information obtained in the previous step into
-  a file named config in the top of the repository (i.e., the
-  directory containing this file). Here's a template of what it should
-  look like:
+- Put the authorization information obtained in the previous step into a file
+  named `.env` in the top of the repository (i.e., the directory containing
+  this file). Here is a template of what it should look like:
   ```
-  [github]
-  user: OAUTH_TOKEN_USER
-  token: OAUTH_TOKEN
+  HIGHFIVE_GITHUB_TOKEN=your-token
   ```
   _Do not check in this file or commit your OAuth token to a
   repository in any other way. It is a secret._
@@ -181,13 +186,17 @@ Docker
 
 Alternatively, you can build a Docker image that runs Highfive.
 
-    $ docker build -t highfive .
+```
+$ docker build -t highfive .
+```
 
 To run a container, you must mount a config file. Assuming you are
 launching a container from a directory containing a config file, you
 can do the following.
 
-    $ docker run -d --rm --name highfive -p 8080:80 -v $(pwd)/config:/highfive/highfive/config highfive
+```
+$ docker run -d --rm --name highfive -p 8000:80 -e HIGHFIVE_GITHUB_TOKEN=token -e HIGHFIVE_WEBHOOK_SECRET=secret highfive
+```
 
 At this point, Highfive is accessible at http://localhost:8080.
 

ci/publish-docker.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+ECR_IMAGE="890664054962.dkr.ecr.us-west-1.amazonaws.com/rust-highfive:latest"
+
+$(aws ecr get-login --no-include-email --region us-west-1)
+
+docker build -t rust-highfive .
+docker tag rust-highfive:latest "${ECR_IMAGE}"
+docker push "${ECR_IMAGE}"

deployment/highfive.conf

@@ -0,0 +1,74 @@
+import hashlib
+import hmac
+import json
+import sys
+
+from .config import Config, InvalidTokenException
+from .newpr import HighfiveHandler, UnsupportedRepoError
+from .payload import Payload
+
+import click
+import dotenv
+import flask
+import waitress
+
+
+def create_app(config, webhook_secret=None):
+    app = flask.Flask(__name__)
+
+    # The canonical URL is /webhook, but other URLs are accepted for backward
+    # compatibility.
+    @app.route("/webhook", methods=['POST'])
+    @app.route("/newpr.py", methods=['POST'])
+    @app.route("/highfive/newpr.py", methods=['POST'])
+    def new_pr():
+        raw_data = flask.request.get_data()
+
+        # Check the signature only if the secret is configured
+        if 'payload' in flask.request.form and webhook_secret is not None:
+            expected = hmac.new(str(webhook_secret), digestmod=hashlib.sha1)
+            expected.update(raw_data)
+            expected = expected.hexdigest()
+            try:
+                signature = str(flask.request.headers['X-Hub-Signature'])
+            except KeyError:
+                return 'Error: missing signature\n', 400
+            if not hmac.compare_digest('sha1='+expected, signature):
+                return 'Error: invalid signature\n', 403
+
+        try:
+            payload = json.loads(flask.request.form['payload'])
+        except (KeyError, ValueError), _:
+            return 'Error: missing or invalid payload\n', 400
+        try:
+            handler = HighfiveHandler(Payload(payload), config)
+            return handler.run(flask.request.headers['X-GitHub-Event'])
+        except UnsupportedRepoError:
+            return 'Error: this repository is not configured!\n', 400
+
+    @app.route('/')
+    def index():
+        return 'Welcome to highfive!\n'
+
+    return app
+
+
+@click.command()
+@click.option('--port', default=8000)
+@click.option('--github-token', required=True)
+@click.option("--webhook-secret")
+def cli(port, github_token, webhook_secret):
+    try:
+        config = Config(github_token)
+    except InvalidTokenException:
+        print 'error: invalid github token provided!'
+        sys.exit(1)
+    print 'Found a valid GitHub token for user @' + config.github_username
+
+    app = create_app(config, webhook_secret)
+    waitress.serve(app, port=port)
+
+
+def main():
+    dotenv.load_dotenv()
+    cli(auto_envvar_prefix='HIGHFIVE')

highfive/app.py

@@ -0,0 +1,21 @@
+import requests
+
+
+class InvalidTokenException(Exception):
+    pass
+
+
+class Config(object):
+    def __init__(self, github_token):
+        if not github_token:
+            raise InvalidTokenException()
+        self.github_token = github_token
+        self.github_username = self.fetch_github_username()
+
+    def fetch_github_username(self):
+        response = requests.get('https://api.github.com/user', headers={
+            'Authorization': 'token ' + self.github_token
+        })
+        if response.status_code != 200:
+            raise InvalidTokenException()
+        return response.json()['login']

highfive/config.py

@@ -6,7 +6,6 @@
 from copy import deepcopy
 import json
 import random
-import sys
 import ConfigParser
 from StringIO import StringIO
 import gzip
@@ -48,13 +47,11 @@ class UnsupportedRepoError(IOError):
     pass
 
 class HighfiveHandler(object):
-    def __init__(self, payload):
+    def __init__(self, payload, config):
         self.payload = payload
 
-        self.config = ConfigParser.RawConfigParser()
-        self.config.read('./config')
-        self.integration_user = self.config.get('github', 'user')
-        self.integration_token = self.config.get('github', 'token')
+        self.integration_user = config.github_username
+        self.integration_token = config.github_token
 
         self.repo_config = self.load_repo_config()
 
@@ -66,14 +63,17 @@ def load_repo_config(self):
         except IOError:
             raise UnsupportedRepoError
 
-    def run(self):
-        if self.payload["action"] == "opened":
+    def run(self, event):
+        if event == "ping":
+            return "Ping received! The webhook is configured correctly!\n"
+        elif event == "pull_request" and self.payload["action"] == "opened":
             self.new_pr()
-        elif self.payload["action"] == "created":
+            return 'OK\n'
+        elif event == "issue_comment" and self.payload["action"] == "created":
             self.new_comment()
+            return 'OK\n'
         else:
-            print self.payload["action"]
-            sys.exit(0)
+            return 'Unsupported webhook event.\n'
 
     def _load_json_file(self, name):
         configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
@@ -419,18 +419,3 @@ def new_comment(self):
                 reviewer, owner, repo, issue, self.integration_user,
                 author, None
             )
-
-
-if __name__ == "__main__":
-    print "Content-Type: text/html;charset=utf-8"
-    print
-
-    cgitb.enable(display=0, logdir="/var/log/highfive-tracebacks", format="txt")
-
-    post = cgi.FieldStorage()
-    payload_raw = post.getfirst("payload",'')
-    try:
-        handler = HighfiveHandler(payload.Payload(json.loads(payload_raw)))
-        handler.run()
-    except UnsupportedRepoError:
-        print "Unsupported repository"

highfive/newpr.py

@@ -0,0 +1,31 @@
+import pytest
+import responses
+
+from ..config import Config, InvalidTokenException
+
+
+@pytest.mark.unit
+@pytest.mark.hermetic
+class TestConfig(object):
+    def test_empty_token(self):
+        for token in ['', None]:
+            with pytest.raises(InvalidTokenException):
+                Config(token)
+
+    @responses.activate
+    def test_valid_token(self):
+        responses.add(
+            responses.GET, 'https://api.github.com/user',
+            json={'login': 'baz'},
+        )
+
+        config = Config('foobar')
+        assert config.github_token == 'foobar'
+        assert config.github_username == 'baz'
+
+    @responses.activate
+    def test_invalid_token(self):
+        responses.add(responses.GET, 'https://api.github.com/user', status=403)
+
+        with pytest.raises(InvalidTokenException):
+            Config('foobar')