update to steady main.py

Author: rzitomer

src/github_analysis/main.py

@@ -27,46 +27,31 @@ def main(args):
 
     commits_dl = dl.data_layer(args.data_path, min_number_commits=args.min_commits)
 
-    project_data = commits_dl.getRandomProjects(args.n_projects, 1)
+    project_data = commits_dl.getRandomProjects(args.n_projects, args.random_state)
     getDataTime = time.time()
-
     logging.info("Query Complete: " + str(getDataTime - startTime) + " seconds")
 
+    project_ids = dl.getUniqueProjectIdsFromDf(project_data)
+    project_groups = commits_dl.getGroupedCommitsByProjectIds(project_ids)
+
+    project_graphs = []
+    project_ids_ordered = []
+    for name, group in project_groups:
+        project_graphs.append(nxutils.git_graph(group))
+        project_ids_ordered.append(name)
+
+    generateGraphsTime = time.time()
+    logging.info("NxGraphs Built: " + str(generateGraphsTime - getDataTime) + " seconds")
 
-    for iter in [2, 3, 4, 5, 6, 7, 8, 9, 10]:
-        embeddings_path = None
-        if args.embeddings_file_path is None: # If embeddings not specified, generate the model and set the path to the output embeddings
-            project_ids = dl.getUniqueProjectIdsFromDf(project_data)
-            project_groups = commits_dl.getGroupedCommitsByProjectIds(project_ids)
-
-            project_graphs = []
-            project_ids_ordered = []
-            for name, group in project_groups:
-                project_graphs.append(nxutils.git_graph(group))
-                project_ids_ordered.append(name)
-
-            # with open("project_graphs.pkl", 'w') as f:
-            #     pickle.dump(project_graphs, f)
-            #
-            # with open("project_ids_ordered.pkl", 'w') as f:
-            #     pickle.dump(project_ids_ordered, f)
-
-            generateGraphsTime = time.time()
-            logging.info("NxGraphs Built: " + str(generateGraphsTime - getDataTime) + " seconds")
-
-            embeddings_path = args.results_path + "embeddings.csv"
-            g2vModel = g2v.Graph2Vec(workers=args.n_workers, size=args.n_neurons, min_count=args.min_count, iter=iter, seed=args.random_state)
-            g2vEmbeddings = g2vModel.fit_transform(project_graphs, project_ids_ordered, output_path=embeddings_path)
-            buildModelTime = time.time()
-            logging.info("G2V Model Built: " + str(buildModelTime - generateGraphsTime) + " seconds")
-        else:
-            embeddings_path = args.embeddings_file_path
-            generateGraphsTime = time.time()
-            buildModelTime = time.time()
-
-        red.reduce_dim(workers=args.n_workers, output_path=args.results_path + str(iter) + "/", input_path=embeddings_path, random_state=args.random_state)
-        reduceTime = time.time()
-        logging.info("Dims Reduced: " + str(reduceTime - buildModelTime) + " seconds")
+    embeddings_path = args.results_path + "embeddings.csv"
+    g2vModel = g2v.Graph2Vec(workers=args.n_workers, size=args.n_neurons, min_count=args.min_count, iter=args.n_iter, seed=args.random_state)
+    g2vEmbeddings = g2vModel.fit_transform(project_graphs, project_ids_ordered, output_path=embeddings_path)
+    buildModelTime = time.time()
+    logging.info("G2V Model Built: " + str(buildModelTime - generateGraphsTime) + " seconds")
+
+    red.reduce_dim(workers=args.n_workers, output_path=args.results_path + str(iter) + "/", input_path=embeddings_path, random_state=args.random_state)
+    reduceTime = time.time()
+    logging.info("Dims Reduced: " + str(reduceTime - buildModelTime) + " seconds")
 
     clusters = c.get_embedding_clusters(embedding_input_file=embeddings_path, output_file=args.results_path + "clusters.pickle", random_state=args.random_state)
     projectClusterTime = time.time()
@@ -97,16 +82,16 @@ def main(args):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument("-rp", "--results_path", help="The folder to output results of the analysis. e.g. embeddings and plots", default="./results/")
-    parser.add_argument("-nw", "--n_workers", help="The number of workers to use when running the analysis.", default=8, type=int)
-    parser.add_argument("-dp", "--data_path", help="The path to the commits.feather file. e.g. /home/user/RStudio-Data-Repository/clean_data/commits.feather", default="./results/")
-    parser.add_argument("-np", "--n_projects", help="The number of projects to sample from the dataset.", default=1000, type=int)
-    parser.add_argument("-mc", "--min_commits", help="The minimum number of commits for a project to be included in the sample.", default=None, type=int)
-    parser.add_argument("-mcount", "--min_count", help="The min_count parameter for the graph2vec model.", default=5, type=int)
-    parser.add_argument("-nps", "--n_personas", help="The number of personas to extract from each cluster.", default=5, type=int)
-    parser.add_argument("-nn", "--n_neurons", help="The number of neurons to use for Graph2Vec (project level)", default=128, type=int)
-    parser.add_argument("-emb", "--embeddings_file_path", help="The file to read the embeddings from. Supplying this parameter skips training of the model.", default=None)
-    parser.add_argument("-rs", "--random_state", help="The random state to initalize all random states.", default=1, type=int)
+    parser.add_argument("-rp",      "--results_path",   help="The folder to output results of the analysis. e.g. embeddings and plots", default="./results/")
+    parser.add_argument("-nw",      "--n_workers",      help="The number of workers to use when running the analysis.", default=8, type=int)
+    parser.add_argument("-dp",      "--data_path",      help="The path to the commits.feather file. e.g. /home/user/RStudio-Data-Repository/clean_data/commits.feather", default="./results/")
+    parser.add_argument("-np",      "--n_projects",     help="The number of projects to sample from the dataset.", default=1000, type=int)
+    parser.add_argument("-mc",      "--min_commits",    help="The minimum number of commits for a project to be included in the sample.", default=None, type=int)
+    parser.add_argument("-mcount",  "--min_count",      help="The min_count parameter for the graph2vec model.", default=5, type=int)
+    parser.add_argument("-nps",     "--n_personas",     help="The number of personas to extract from each cluster.", default=5, type=int)
+    parser.add_argument("-nn",      "--n_neurons",      help="The number of neurons to use for Graph2Vec (project level)", default=128, type=int)
+    parser.add_argument("-ni",      "--n_iter",         help="The number of iteration to use to run the WeisfeilerLehmanMachine", default=10, type=int)
+    parser.add_argument("-rs",      "--random_state",   help="The random state to initalize all random states.", default=1, type=int)
 
     args = parser.parse_args()
 
@@ -149,4 +134,4 @@ def main(args):
     #             left join `ghtorrent-bq.ght.commit_parents` cp on (cp.commit_id = c.id)
     #             where (p.id = """ + str(projectId) + """)
     #         """
-    #
+    #