Auto merge of #142 - kennytm:stop-hardcoding-timeout, r=jdm

Allow TEST_TIMEOUT be configured.

Homu currently hard codes the test timeout to 10 hours. This is unnecessarily long for many projects, including Rust. Recently, there is  bug in AppVeyor or GitHub causing the status notification not delivered, and thus the queue can be stuck for 10 hours when unattended.

This PR introduced a fix to allow the timeout be configured for each repository.

```toml
[repo.NAME]
# timeout after 3 hr 20 min
timeout = 12000
```

When the timeout is less than 1 hour, the current `check_timeout` loop will be too coarse. Therefore, I've also refactored and removed the loop in favor of a [`Timer`](https://docs.python.org/3/library/threading.html#timer-objects) object for each pending PR.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/homu/142)
<!-- Reviewable:end -->

Author: bors-servo

cfg.sample.toml

@@ -80,6 +80,10 @@ try_users = []
 # command.
 #status_based_exemption = false
 
+# Maximum test duration allowed for testing a PR in this repository.
+# Default to 10 hours.
+#timeout = 36000
+
 # Branch names. These settings are the defaults; it makes sense to leave these
 # as-is.
 #[repo.NAME.branch]

homu/main.py

@@ -7,7 +7,7 @@
 from . import utils
 from .utils import lazy_debug
 import logging
-from threading import Thread, Lock
+from threading import Thread, Lock, Timer
 import time
 import traceback
 import sqlite3
@@ -34,7 +34,7 @@
 
 INTERRUPTED_BY_HOMU_FMT = 'Interrupted by Homu ({})'
 INTERRUPTED_BY_HOMU_RE = re.compile(r'Interrupted by Homu \((.+?)\)')
-TEST_TIMEOUT = 3600 * 10
+DEFAULT_TEST_TIMEOUT = 3600 * 10
 
 global_cfg = {}
 
@@ -128,7 +128,8 @@ def __init__(self, num, head_sha, status, db, repo_label, mergeable_que,
         self.owner = owner
         self.name = name
         self.repos = repos
-        self.test_started = time.time()  # FIXME: Save in the local database
+        self.timeout_timer = None
+        self.test_started = time.time()
 
     def head_advanced(self, head_sha, *, use_db=True):
         self.head_sha = head_sha
@@ -179,6 +180,9 @@ def add_comment(self, text):
 
     def set_status(self, status):
         self.status = status
+        if self.timeout_timer:
+            self.timeout_timer.cancel()
+            self.timeout_timer = None
 
         db_query(
             self.db,
@@ -320,6 +324,30 @@ def blocked_by_closed_tree(self):
         treeclosed = self.repos[self.repo_label].treeclosed
         return treeclosed if self.priority < treeclosed else None
 
+    def start_testing(self, timeout):
+        self.test_started = time.time()     # FIXME: Save in the local database
+        self.set_status('pending')
+        timer = Timer(timeout, self.timed_out)
+        timer.start()
+        self.timeout_timer = timer
+
+    def timed_out(self):
+        print('* Test timed out: {}'.format(self))
+
+        self.merge_sha = ''
+        self.save()
+        self.set_status('failure')
+
+        desc = 'Test timed out'
+        utils.github_create_status(
+            self.get_repo(),
+            self.head_sha,
+            'failure',
+            '',
+            desc,
+            context='homu')
+        self.add_comment(':boom: {}'.format(desc))
+
 
 def sha_cmp(short, full):
     return len(short) >= 4 and short == full[:len(short)]
@@ -1140,8 +1168,8 @@ def start_build(state, repo_cfgs, buildbot_slots, logger, db, git_cfg):
         branch,
         state.merge_sha))
 
-    state.test_started = time.time()
-    state.set_status('pending')
+    timeout = repo_cfg.get('timeout', DEFAULT_TEST_TIMEOUT)
+    state.start_testing(timeout)
 
     desc = '{} commit {} with merge {}...'.format(
         'Trying' if state.try_ else 'Testing',
@@ -1217,8 +1245,8 @@ def start_rebuild(state, repo_cfgs):
                 state.add_comment(':bomb: Failed to start rebuilding: `{}`'.format(err))  # noqa
                 return False
 
-    state.test_started = time.time()
-    state.set_status('pending')
+    timeout = repo_cfg.get('timeout', DEFAULT_TEST_TIMEOUT)
+    state.start_testing(timeout)
 
     msg_1 = 'Previous build results'
     msg_2 = ' for {}'.format(', '.join('[{}]({})'.format(builder, url) for builder, url in succ_builders))  # noqa
@@ -1326,39 +1354,6 @@ def fetch_mergeability(mergeable_que):
             mergeable_que.task_done()
 
 
-def check_timeout(states, queue_handler):
-    while True:
-        try:
-            for repo_label, repo_states in states.items():
-                for num, state in repo_states.items():
-                    _timout = time.time() - state.test_started >= TEST_TIMEOUT
-                    if state.status == 'pending' and _timout:
-                        print('* Test timed out: {}'.format(state))
-
-                        state.merge_sha = ''
-                        state.save()
-                        state.set_status('failure')
-
-                        desc = 'Test timed out'
-                        utils.github_create_status(
-                            state.get_repo(),
-                            state.head_sha,
-                            'failure',
-                            '',
-                            desc,
-                            context='homu')
-                        state.add_comment(':boom: {}'.format(desc))
-
-                        queue_handler()
-
-        except Exception:
-            print('* Error while checking timeout')
-            traceback.print_exc()
-
-        finally:
-            time.sleep(3600)
-
-
 def synchronize(repo_label, repo_cfg, logger, gh, states, repos, db, mergeable_que, my_username, repo_labels):  # noqa
     logger.info('Synchronizing {}...'.format(repo_label))
 
@@ -1666,7 +1661,6 @@ def queue_handler():
         ]).start()
 
     Thread(target=fetch_mergeability, args=[mergeable_que]).start()
-    Thread(target=check_timeout, args=[states, queue_handler]).start()
 
     queue_handler()