add snakemake rule for pulling data into repo

Author: rzitomer

Snakefile

@@ -1,8 +1,15 @@
 configfile: "config.json"
 
+rule get_ght_data:
+    params:
+        download_url = config["data_download_url"]
+    output:
+        output_file = "data/commits_by_org.feather"
+    shell: "python src/github_analysis/make_report.py -du {params.download_url} -of {output.output_file}"
+
 rule run_analysis:
     input:
-        data_path = "/Users/richiezitomer/Documents/RStudio-Data-Repository/clean_data/commits_by_org.feather"
+        data_path = "data/commits_by_org.feather"
     output:
         results_path = directory("results/")
     params:
@@ -20,12 +27,7 @@ rule run_analysis:
 
 rule generate_images:
     input:
-        data_path="/Users/richiezitomer/Documents/RStudio-Data-Repository/clean_data/commits_by_org.feather",
+        data_path="data/commits_by_org.feather",
         embedding_path="results/embeddings.csv"
     shell:
         "python src/github_analysis/make_report.py -dp {input.data_path} -ep {input.embedding_path}"
-
-
-# Commented out because repo is currently over bandwidth: https://help.github.com/en/articles/about-storage-and-bandwidth-usage
-#rule clone_data_repo:
-#    shell: "git clone https://github.com/UBC-MDS/RStudio-Data-Repository.git"

config.json

@@ -1,4 +1,5 @@
-{"python_hash_seed": 0,
+{"data_download_url": "https://api.figshare.com/v2/file/download/15593951",
+"python_hash_seed": 0,
 "n_workers": 1,
 "n_projects": 1000,
 "min_commits": "None",

src/github_analysis/download_gh_data.py

@@ -0,0 +1,20 @@
+import requests
+import shutil
+
+
+def download_file(download_URL, filename):
+    """Download file from CURL url using request.
+    download_URL: """
+    with requests.get(download_URL, stream=True) as r:
+            with open(filename, 'wb') as f:
+                shutil.copyfileobj(r.raw, f)
+    return filename
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-du",      "--download_URL",   help="The URL to download the file.", default='https://api.figshare.com/v2/file/download/15593951')
+    parser.add_argument("-of",      "--output_file",      help="The number of workers to use when running the analysis.", default='data/commits_by_org.feather')
+    args = parser.parse_args()
+
+    download_file(args.download_URL, args.output_file)