Merge branch 'master' of https://github.com/UBC-MDS/RStudio-GitHub-Analysis

bring branch up to date with master

Author: huijuechen

README.md

@@ -31,8 +31,12 @@ From the root directory, run:
 python src/python src/github_analysis/main.py -dp "/home/rayce/Assignments/Capstone/RStudio-Data-Repository/clean_data/commits_by_org.feather".py
 ```
 
-## Installation Instructions
-PLACEHOLDER
+## Installation instructions
+To get credentials file for GitHub Torrent Google Cloud (necessary for re-running the pipeline to generate images):
+
+- Follow the instructions here to create and download a credentials file: https://developers.google.com/adwords/api/docs/guides/authentication#generate_oauth2_credentials
+- Change the name of the file to `credentials_file.json` and put it in the root directory of the project (a sample file with the same name is included as a reference).
+
 
 ## Data Repository
 [RStudio-Data-Repository](https://github.com/UBC-MDS/RStudio-Data-Repository)

Snakefile

@@ -0,0 +1,33 @@
+configfile: "config.json"
+
+rule get_ght_data:
+    params:
+        download_url = config["data_download_url"]
+    output:
+        output_file = "data/commits_by_org.feather"
+    shell: "python src/github_analysis/make_report.py -du {params.download_url} -of {output.output_file}"
+
+rule run_analysis:
+    input:
+        data_path = "data/commits_by_org.feather"
+    output:
+        results_path = directory("results/")
+    params:
+        python_hash_seed = config["python_hash_seed"],
+        n_workers = config["n_workers"],
+        n_projects = config["n_projects"],
+        min_commits = config["min_commits"],
+        min_count = config["min_count"],
+        n_personas = config["n_personas"],
+        n_neurons = config["n_neurons"],
+        n_iter = config["n_iter"],
+        random_state = config["random_state"]
+    shell:
+        "PYTHONHASHSEED={params.python_hash_seed} python src/github_analysis/main.py -dp {input.data_path} -rp {output.results_path} -nw {params.n_workers} -np {params.n_projects} -mc {params.min_commits} -mcount {params.min_count} -nps {params.n_personas} -nn {params.n_neurons} -ni {params.n_iter} -rs {params.random_state}"
+
+rule generate_images:
+    input:
+        data_path="data/commits_by_org.feather",
+        embedding_path="results/embeddings.csv"
+    shell:
+        "python src/github_analysis/make_report.py -dp {input.data_path} -ep {input.embedding_path}"

config.json

@@ -0,0 +1,10 @@
+{"data_download_url": "https://api.figshare.com/v2/file/download/15593951",
+"python_hash_seed": 0,
+"n_workers": 1,
+"n_projects": 1000,
+"min_commits": "None",
+"min_count": 5,
+"n_personas": 5,
+"n_neurons": 128,
+"n_iter": 10,
+"random_state": 1}

credentials_file.json

@@ -0,0 +1,12 @@
+{
+  "type": "INSERT_TYPE",
+  "project_id": "INSERT_PROJECT_ID",
+  "private_key_id": "INSERT_PRIVATE_KEY_ID",
+  "private_key": "INSERT_PRIVATE_KEY",
+  "client_email": "INSERT_EMAIL",
+  "client_id": "INSERT_CLIENT_ID",
+  "auth_uri": "INSERT_AUTH_URI",
+  "token_uri": "INSERT_TOKEN_URI",
+  "auth_provider_x509_cert_url": "INSERT_AUTH_CERT_URL",
+  "client_x509_cert_url": "INSERT_CLIENT_CERT_URL"
+}

src/github_analysis/download_gh_data.py

@@ -0,0 +1,20 @@
+import requests
+import shutil
+
+
+def download_file(download_URL, filename):
+    """Download file from CURL url using request.
+    download_URL: """
+    with requests.get(download_URL, stream=True) as r:
+            with open(filename, 'wb') as f:
+                shutil.copyfileobj(r.raw, f)
+    return filename
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-du",      "--download_URL",   help="The URL to download the file.", default='https://api.figshare.com/v2/file/download/15593951')
+    parser.add_argument("-of",      "--output_file",      help="The number of workers to use when running the analysis.", default='data/commits_by_org.feather')
+    args = parser.parse_args()
+
+    download_file(args.download_URL, args.output_file)

src/github_analysis/main.py

@@ -80,13 +80,19 @@ def main(args):
     print("Frequency Graph Time:\t" +   str(freqGraphTime - motifTime) +                    "\tseconds")
     print("Total Time:\t\t" +           str(freqGraphTime - startTime) +                    "\tseconds")
 
+
+def none_or_int(value):
+    if value == 'None':
+        return None
+    return int(value)
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument("-rp",      "--results_path",   help="The folder to output results of the analysis. e.g. embeddings and plots", default="./results/")
     parser.add_argument("-nw",      "--n_workers",      help="The number of workers to use when running the analysis.", default=8, type=int)
-    parser.add_argument("-dp",      "--data_path",      help="The path to the commits.feather file. e.g. /home/user/RStudio-Data-Repository/clean_data/commits.feather", default="./results/")
+    parser.add_argument("-dp",      "--data_path",      help="The path to the commits.feather file. e.g. /home/user/RStudio-Data-Repository/clean_data/commits_by_org.feather", default="/home/user/RStudio-Data-Repository/clean_data/commits_by_org.feather")
     parser.add_argument("-np",      "--n_projects",     help="The number of projects to sample from the dataset.", default=1000, type=int)
-    parser.add_argument("-mc",      "--min_commits",    help="The minimum number of commits for a project to be included in the sample.", default=None, type=int)
+    parser.add_argument("-mc",      "--min_commits",    help="The minimum number of commits for a project to be included in the sample.", default=None, type=none_or_int)
     parser.add_argument("-mcount",  "--min_count",      help="The min_count parameter for the graph2vec model.", default=5, type=int)
     parser.add_argument("-nps",     "--n_personas",     help="The number of personas to extract from each cluster.", default=5, type=int)
     parser.add_argument("-nn",      "--n_neurons",      help="The number of neurons to use for Graph2Vec (project level)", default=128, type=int)
@@ -134,4 +140,4 @@ def main(args):
     #             left join `ghtorrent-bq.ght.commit_parents` cp on (cp.commit_id = c.id)
     #             where (p.id = """ + str(projectId) + """)
     #         """
-    #
+    #