Record all calls to retry to track spurious failures

kennytm · kennytm · commit 9bcd6a457e1e · 2019-05-04T04:13:50.000+08:00
diff --git a/cfg.sample.toml b/cfg.sample.toml
@@ -1,6 +1,10 @@
 # Priority values above max_priority will be refused.
 max_priority = 9001
 
+# How long to keep the retry log
+# Should be a negative interval of time recognized by SQLite3.
+retry_log_expire = '-42 days'
+
 [github]
 
 # Information for securely interacting with GitHub. These are found/generated
diff --git a/homu/html/queue.html b/homu/html/queue.html
@@ -116,6 +116,8 @@ <h1>Homu queue - {% if repo_url %}<a href="{{repo_url}}" target="_blank">{{repo_
             </tbody>
         </table>
 
+        <p><a href="../retry_log/{{repo_label}}">Open retry log</a></p>
+
         <script src="../assets/jquery.min.js"></script>
         <script src="../assets/jquery.dataTables.min.js"></script>
 
diff --git a/homu/html/retry_log.html b/homu/html/retry_log.html
@@ -0,0 +1,42 @@
+<!doctype html>
+<html>
+    <head>
+        <meta charset="utf-8">
+        <title>Homu retry log {{repo_label}}</title>
+        <style>
+            * { font-family: sans-serif; }
+            h1 { font-size: 20px; }
+            h2 { font-size: 16px; }
+            p { font-size: 15px; }
+            pre { margin: 0; }
+
+            table { border-collapse: collapse; }
+            td, th { border: 2px solid white; padding: 5px; font-size: 13px; vertical-align: top; }
+            tr:nth-child(even) { background: #ddd; }
+        </style>
+    </head>
+    <body>
+        <h1>Homu retry log - <a href="{{repo_url}}" target="_blank">{{repo_label}}</a></h1>
+
+        <table id="results">
+            <thead>
+                <tr>
+                    <th>Time (UTC)</th>
+                    <th>PR</th>
+                    <th>Message</th>
+                </tr>
+            </thead>
+
+            <tbody>
+                {% for log in logs %}
+                <tr>
+                    <td>{{log.time}}</td>
+                    <td><a href="{{log.src}}">{{log.num}}</a></td>
+                    <td><pre>{{log.msg}}</pre></td>
+                </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+
+    </body>
+</html>
diff --git a/homu/main.py b/homu/main.py
@@ -382,6 +382,18 @@ def timed_out(self):
         self.add_comment(comments.TimedOut())
         self.change_labels(LabelEvent.TIMED_OUT)
 
+    def record_retry_log(self, src, body):
+        # destroy ancient records
+        db_query(
+            self.db,
+            "DELETE FROM retry_log WHERE repo = ? AND time < date('now', ?)",
+            [self.repo_label, global_cfg.get('retry_log_expire', '-42 days')],
+        )
+        db_query(
+            self.db,
+            'INSERT INTO retry_log (repo, num, src, msg) VALUES (?, ?, ?, ?)',
+            [self.repo_label, self.num, src, body],
+        )
 
 def sha_cmp(short, full):
     return len(short) >= 4 and short == full[:len(short)]
@@ -625,6 +637,7 @@ def parse_commands(body, username, repo_label, repo_cfg, state, my_username,
             state.set_status('')
             if realtime:
                 event = LabelEvent.TRY if state.try_ else LabelEvent.APPROVED
+                state.record_retry_log(command_src, body)
                 state.change_labels(event)
 
         elif word in ['try', 'try-'] and realtime:
@@ -1641,6 +1654,17 @@ def main():
         UNIQUE (repo)
     )''')
 
+    db_query(db, '''CREATE TABLE IF NOT EXISTS retry_log (
+        repo TEXT NOT NULL,
+        num INTEGER NOT NULL,
+        time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        src TEXT NOT NULL,
+        msg TEXT NOT NULL
+    )''')
+    db_query(db, '''
+        CREATE INDEX IF NOT EXISTS retry_log_time_index ON retry_log (repo, time DESC)
+    ''')
+
     # manual DB migration :/
     try:
         db_query(db, 'SELECT treeclosed_src FROM repos LIMIT 0')
diff --git a/homu/server.py b/homu/server.py
@@ -179,6 +179,33 @@ def queue(repo_label):
         multiple=multiple,
     )
 
+@get('/retry_log/<repo_label:path>')
+def retry_log(repo_label):
+    logger = g.logger.getChild('retry_log')
+
+    lazy_debug(logger, lambda: 'repo_label: {}'.format(repo_label))
+
+    repo_url = 'https://github.com/{}/{}'.format(
+        g.cfg['repo'][repo_label]['owner'],
+        g.cfg['repo'][repo_label]['name'],
+    )
+
+    db_query(
+        g.db,
+        'SELECT num, time, src, msg FROM retry_log WHERE repo = ? ORDER BY time DESC',
+        [repo_label],
+    )
+    logs = [
+        {'num': num, 'time': time, 'src': src, 'msg': msg}
+        for num, time, src, msg in g.db.fetchall()
+    ]
+
+    return g.tpls['retry_log'].render(
+        repo_url=repo_url,
+        repo_label=repo_label,
+        logs=logs,
+    )
+
 
 @get('/callback')
 def callback():
@@ -903,6 +930,7 @@ def start(cfg, states, queue_handler, repo_cfgs, repos, logger,
     tpls['index'] = env.get_template('index.html')
     tpls['queue'] = env.get_template('queue.html')
     tpls['build_res'] = env.get_template('build_res.html')
+    tpls['retry_log'] = env.get_template('retry_log.html')
 
     g.cfg = cfg
     g.states = states
