Added github remote repository access.

You turn it on via `--access=github`.

Note A local git checkout is still the default value for `--access`; I
do not want to make github access the default until we make the experience
a bit more user-friendly. Most notable are the following to-do's:

 1. To-do: Fallback to local checkout if the github accessor errors out

 2. To-do: Add a warning to stdout if the user runs github access without
    passing an access token via GITHUB_TOKEN env var (since doing that will
    hit rate limit much faster).

Author: pnkfelix

Cargo.lock

@@ -21,8 +21,10 @@ git2 = "0.11"
 log = "0.4"
 pbr = "1.0.2"
 regex = "1.3.4"
-reqwest = { version = "0.10.2", features = ["blocking"] }
+reqwest = { version = "0.10.2", features = ["blocking", "json"] }
 rustc_version = "0.2"
+serde = { version = "1.0.104", features = ["derive"] }
+serde_json = "1.0"
 structopt = "0.3.9"
 tar = "0.4"
 tee = "0.1"

Cargo.toml

@@ -0,0 +1,167 @@
+use failure::Error;
+use reqwest::{self, blocking::Client, blocking::Response};
+use serde::{Deserialize, Serialize};
+
+use crate::Commit;
+
+#[derive(Serialize, Deserialize, Debug)]
+struct GithubCommitElem { commit: GithubCommit, sha: String }
+#[derive(Serialize, Deserialize, Debug)]
+struct GithubCommit { author: GithubAuthor, committer: GithubAuthor, message: String, }
+#[derive(Serialize, Deserialize, Debug)]
+struct GithubAuthor { date: String, email: String, name: String }
+
+type GitDate = chrono::DateTime<chrono::Utc>;
+
+impl GithubCommitElem {
+    fn date(&self) -> Result<GitDate, Error> {
+        Ok(self.commit.committer.date.parse()?)
+    }
+
+    fn git_commit(self) -> Result<Commit, Error> {
+        let date = self.date()?;
+        Ok(Commit {
+            sha: self.sha,
+            date,
+            summary: self.commit.message,
+        })
+    }
+}
+
+fn headers() -> Result<reqwest::header::HeaderMap, Error> {
+    let mut headers = reqwest::header::HeaderMap::new();
+    let user_agent = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"));
+    let user_agent = reqwest::header::HeaderValue::from_static(user_agent);
+    headers.insert(reqwest::header::USER_AGENT, user_agent);
+    if let Ok(token) = std::env::var("GITHUB_TOKEN") {
+        eprintln!("adding local env GITHUB_TOKEN value to headers in github query");
+        let value = reqwest::header::HeaderValue::from_str(&format!("token {}", token))?;
+        headers.insert(reqwest::header::AUTHORIZATION, value);
+    }
+    Ok(headers)
+}
+
+pub(crate) fn get_commit(sha: &str) -> Result<Commit, Error> {
+    let url = SingleCommitUrl { sha }.url();
+    let client = Client::builder()
+        .default_headers(headers()?)
+        .build()?;
+    let response: Response = client.get(&url).send()?;
+    let elem: GithubCommitElem = response.json()?;
+    elem.git_commit()
+}
+
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct CommitsQuery<'a> {
+    pub since_date: &'a str,
+    pub most_recent_sha: &'a str,
+    pub earliest_sha: &'a str,
+}
+
+/// Returns the bors merge commits between the two specified boundaries
+/// (boundaries inclusive).
+
+impl<'a> CommitsQuery<'a> {
+    pub fn get_commits(&self) -> Result<Vec<Commit>, Error> {
+        get_commits(*self)
+    }
+}
+
+const PER_PAGE: usize = 100;
+const OWNER: &'static str = "rust-lang";
+const REPO: &'static str = "rust";
+
+
+trait ToUrl { fn url(&self) -> String; }
+struct CommitsUrl<'a> { page: usize, author: &'a str, since: &'a str, sha: &'a str }
+struct SingleCommitUrl<'a> { sha: &'a str }
+
+impl<'a> ToUrl for CommitsUrl<'a> {
+    fn url(&self) -> String {
+        format!("https://api.github.com/repos/{OWNER}/{REPO}/commits\
+                 ?page={PAGE}&per_page={PER_PAGE}\
+                 &author={AUTHOR}&since={SINCE}&sha={SHA}",
+                OWNER=OWNER, REPO=REPO,
+                PAGE=self.page, PER_PAGE=PER_PAGE,
+                AUTHOR=self.author, SINCE=self.since, SHA=self.sha)
+    }
+}
+
+impl<'a> ToUrl for SingleCommitUrl<'a> {
+    fn url(&self) -> String {
+        format!("https://api.github.com/repos/{OWNER}/{REPO}/commits/{REF}",
+                OWNER=OWNER, REPO=REPO, REF=self.sha)
+    }
+}
+
+fn get_commits(q: CommitsQuery) -> Result<Vec<Commit>, Error> {
+    // build up commit sequence, by feeding in `sha` as the starting point, and
+    // working way backwards to max(`q.since_date`, `q.earliest_sha`).
+    let mut commits = Vec::new();
+
+    // focus on Pull Request merges, all authored and committed by bors.
+    let author = "bors";
+
+    let client = Client::builder()
+        .default_headers(headers()?)
+        .build()?;
+    for page in 1.. {
+        let url = CommitsUrl { page, author, since: q.since_date, sha: q.most_recent_sha }.url();
+
+        let response: Response = client.get(&url).send()?;
+
+        let action = parse_paged_elems(response, |elem: GithubCommitElem| {
+            let date: chrono::DateTime<chrono::Utc> = match elem.commit.committer.date.parse() {
+                Ok(date) => date,
+                Err(err) => return Loop::Err(err.into()),
+            };
+            let sha = elem.sha.clone();
+            let summary = elem.commit.message;
+            let commit = Commit { sha, date, summary };
+            commits.push(commit);
+
+            if elem.sha == q.earliest_sha {
+                eprintln!("ending github query because we found starting sha: {}", elem.sha);
+                return Loop::Break;
+            }
+
+            Loop::Next
+        })?;
+
+        if let Loop::Break = action { break; }
+    }
+
+    eprintln!("get_commits_between returning commits, len: {}", commits.len());
+
+    // reverse to obtain chronological order
+    commits.reverse();
+    Ok(commits)
+}
+
+enum Loop<E> { Break, Next, Err(E) }
+enum Void { }
+
+fn parse_paged_elems<Elem: for<'a> serde::Deserialize<'a>>(response: Response,
+                                                           mut k: impl FnMut(Elem) -> Loop<Error>)
+                                                           -> Result<Loop<Void>, Error>
+{
+    // parse the JSON into an array of the expected Elem type
+    let elems: Vec<Elem> = response.json()?;
+
+    // if `elems` is empty, then we've run out of useful pages to lookup.
+    if elems.len() == 0 { return Ok(Loop::Break); }
+
+    for elem in elems.into_iter() {
+        let act = k(elem);
+
+        // the callback will tell us if we should terminate loop early (e.g. due to matching `sha`)
+        match act {
+            Loop::Break => return Ok(Loop::Break),
+            Loop::Err(e) => return Err(e),
+            Loop::Next => continue,
+        }
+    }
+
+    // by default, we keep searching on next page from github.
+    Ok(Loop::Next)
+}

src/github.rs

@@ -35,7 +35,7 @@ mod least_satisfying;
 mod repo_access;
 
 use crate::least_satisfying::{least_satisfying, Satisfies};
-use crate::repo_access::{AccessViaLocalGit, RustRepositoryAccessor};
+use crate::repo_access::{AccessViaGithub, AccessViaLocalGit, RustRepositoryAccessor};
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct Commit {
@@ -946,7 +946,7 @@ impl Config {
         let repo_access: Box<dyn RustRepositoryAccessor>;
         repo_access = match args.access.as_ref().map(|x|x.as_str()) {
             None | Some("checkout") => Box::new(AccessViaLocalGit),
-            Some("github") => unimplemented!(),
+            Some("github") => Box::new(AccessViaGithub),
             Some(other) => bail!("unknown access argument: {}", other),
         };
 

src/main.rs

@@ -24,8 +24,13 @@ pub(crate) trait RustRepositoryAccessor {
 #[path="git.rs"]
 mod git;
 
+#[path="github.rs"]
+mod github;
+
 pub(crate) struct AccessViaLocalGit;
 
+pub(crate) struct AccessViaGithub;
+
 impl RustRepositoryAccessor for AccessViaLocalGit {
     fn commit(&self, commit_ref: &str) -> Result<Commit, Error> {
         self::git::get_commit(commit_ref)
@@ -38,3 +43,30 @@ impl RustRepositoryAccessor for AccessViaLocalGit {
             })
     }
 }
+
+impl RustRepositoryAccessor for AccessViaGithub {
+    fn commit(&self, commit_ref: &str) -> Result<Commit, Error> {
+        github::get_commit(commit_ref)
+    }
+
+    fn commits(&self, start_sha: &str, end_sha: &str) -> Result<Vec<Commit>, Error> {
+        // `earliest_date` is an lower bound on what we should search in our
+        // github query. Why is it `start` date minus 1?
+        //
+        // Because: the "since" parameter in the github API is an exclusive
+        // bound. We need an inclusive bound, so we go yet another day prior for
+        // this bound on the github search.
+        let since_date = self.bound_to_date(Bound::Commit(start_sha.to_string()))? - chrono::Duration::days(1);
+
+        eprintln!("fetching (via remote github) commits from max({}, {}) to {}",
+                  start_sha, since_date, end_sha);
+
+        let query = github::CommitsQuery {
+            since_date: &since_date.format(crate::YYYY_MM_DD).to_string(),
+            earliest_sha: start_sha,
+            most_recent_sha: end_sha,
+        };
+
+        query.get_commits()
+    }
+}