Merge pull request #47 from IntelLabs/nhasabni/cpp_support

Full support for C++ programs

Author: nhasabni

CMakeLists.txt

@@ -55,6 +55,12 @@ if(NOT EXISTS ${PROJECT_SOURCE_DIR}/src/tree-sitter/tree-sitter-php)
     )
 endif()
 
+if(NOT EXISTS ${PROJECT_SOURCE_DIR}/src/tree-sitter/tree-sitter-cpp)
+    execute_process(
+        COMMAND git clone https://github.com/tree-sitter/tree-sitter-cpp.git ${PROJECT_SOURCE_DIR}/src/tree-sitter/tree-sitter-cpp
+    )
+endif()
+
 get_filename_component(TREE_SITTER_INCLUDE src/tree-sitter/tree-sitter/lib/include ABSOLUTE)
 
 enable_testing()

README.md

@@ -39,8 +39,6 @@ More details can be found in our MAPS paper (https://arxiv.org/abs/2011.03616).
 - `scripts`: Scripts for pattern mining and scanning for anomalies
 - `quick_start`: Scripts to run quick start tests
 - `github`: Scripts and data for downloading GitHub repos.
-  It also contains pre-processed training data containing patterns mined from
-6000 GitHub repositories using C as their primary language.
 - `tests`: unit tests
 
 ## Install
@@ -69,14 +67,13 @@ $ cmake .
 $ make -j
 $ make test
 ```
-All tests in `make test` should pass, but currently tests for Verilog are failing because of a version mismatch issue.
-Verilog support is WIP.
+All tests in `make test` should pass.
 
 ## Using ControlFlag
 
 ### Quick start
 
-#### Using patterns obtained from 6000 GitHub repos to scan repository of your choice
+#### Using patterns obtained from several GitHub repos to scan repository of your choice
 
 Download the training data for the language of interest depending on the memory constraints of your device. Note, however, that using smaller datasets may lead to reduced accuracy in the results ControlFlag produces and possibly an increase in the number of false positives it generates.
 
@@ -85,6 +82,9 @@ Language | Dataset name | Size on disk | Memory requirements | Direct link | gdo
 C | Small        | ~100MB       | ~400MB              | [link](https://drive.google.com/file/d/1gvUyRXq1SeZD9g3i__RaamYAMo_QaQIb/view?usp=sharing) | 1gvUyRXq1SeZD9g3i__RaamYAMo_QaQIb | 2825f209aba0430993f7a21e74d99889
 C | Medium       |   ~450MB     | ~1.3GB           | [link](https://drive.google.com/file/d/1zsCFJAKlZlSAWKPfBcVGcQNlFB5Gtwo3/view?usp=sharing) | 1zsCFJAKlZlSAWKPfBcVGcQNlFB5Gtwo3 | aab2427edebe9ed4acab75c3c6227f24
 C | Large        |   ~9GB       | ~13GB           | [link](https://drive.google.com/file/d/1-jzs3zrKU541hwChaciXSk8zrnMN1mYc/view?usp=sharing) | 1-jzs3zrKU541hwChaciXSk8zrnMN1mYc | 1ba954d9716765d44917445d3abf8e85
+C++ | Small | ~200MB | ~500MB | [link](https://drive.google.com/file/d/1ZD9J7vyT61T1D4rsedVXgFi0CrVb5BJl/view?usp=sharing) | 1ZD9J7vyT61T1D4rsedVXgFi0CrVb5BJl | f954486e20961f0838ac08e5d4dbf312
+C++ | Medium | ~500MB | ~1.3GB | [link](https://drive.google.com/file/d/1Pj3bQN3nwy84F5o1w05T1Gz8b4hGuPUr/view?usp=sharing) | 1Pj3bQN3nwy84F5o1w05T1Gz8b4hGuPUr | a5c18ea1cdbe354b93aabf9ecaa5b07a
+C++ | Large | ~1.2GB | ~3GB | [link](https://drive.google.com/file/d/14iNcH3plw3EYnYfX63LntPtyr8Pwo2IP/view?usp=sharing) | 14iNcH3plw3EYnYfX63LntPtyr8Pwo2IP | 4f5ffc1ab942eaba399cafd5be8bb45f
 PHP | Small      | ~120MB       |  ~1GB           | [Link](https://drive.google.com/file/d/1zUnBHMXPIXmlrCfWze8nNoMEQnc0W2K5/view?usp=sharing) | 1zUnBHMXPIXmlrCfWze8nNoMEQnc0W2K5 | 5a1cc4c24a20de7dad1b9f40661d517a
 
 ```
@@ -96,16 +96,22 @@ $ tar -zxf <tgz_file>
 To scan C code of your choice, use below command:
 
 ```
-$ scripts/scan_for_anomalies.sh -d <directory_to_be_scanned_for_anomalies> -t <training_data>.ts -o <output_directory_to_store_log_files>
+$ scripts/scan_for_anomalies.sh -d <directory_to_be_scanned_for_anomalies> -t <training_data>.ts -o <output_directory_to_store_log_files> -l 1
+```
+
+To scan C++ code of your choice, use below command:
+
+```
+$ scripts/scan_for_anomalies.sh -d <directory_to_be_scanned_for_anomalies> -t <training_data>.ts -o <output_directory_to_store_log_files> -l 4
 ```
 
 Once the run is complete (which could take some time depending on your system and the
-number of C programs in your repository,) refer to [the section below to
+number of programs from your repository that can be scanned by ControlFlag,) refer to [the section below to
 understand scan output](#understanding-scan-output).
 
 #### Mining patterns from a small repo and applying them to another small repo
 
-In this test, we will mine patterns from
+In this test for C language programs, we will mine patterns from
 [Glb-director](https://github.com/github/glb-director.git) project of GitHub and
 apply them to flag anomalies in GitHub's [brubeck](https://github.com/github/brubeck.git) project.
 
@@ -143,9 +149,9 @@ statements that appear in C programs.*
 
 If you want to use your own repository for mining patterns, jump to Step 1.2.
 
-1.1 __Downloading Top-100 GitHub repos for C language__
+1.1 __Downloading GitHub repos for C language having more than 100 stars__
 
-Steps below show how to download Top-100 GitHub repos for C language
+Steps below show how to download GitHub repos for C language that have more than 100 stars
 (`c100.txt`) and generate training data. `training_repo_dir` is a directory
 where the command below will clone all the repos.
 
@@ -165,7 +171,8 @@ place of <training_repo_dir>.
 Usage: ./mine_patterns.sh -d <directory_to_mine_patterns_from> -o <output_file_to_store_training_data>
 Optional:
 [-n number_of_processes_to_use_for_mining]  (default: num_cpus_on_system)
-[-l source_language_number] (default: 1 (C), supported: 1 (C), 2 (Verilog), 3 (PHP)
+[-l source_language_number] (default: 1 (C), supported: 1 (C), 2 (Verilog), 3 (PHP), 4 (C++)
+[-g github_repo_id] (default: 0) A unique identifier for GitHub repository, if any
 ```
 
 We use it as:
@@ -178,7 +185,7 @@ found in the specified GitHub repos and their AST (abstract syntax tree) represe
 You can view this file as a text file, if
 you want.
 
-## Evaluation (or scanning for anomalies in C code from test repo)
+## Evaluation (or scanning for anomalies)
 
 We can run `scan_for_anomalies.sh` script to scan target directory of interest.
 Its usage is as below.
@@ -189,14 +196,10 @@ Optional:
  [-n max_number_of_results_for_autocorrect] (default: 5)
  [-j number_of_scanning_threads]            (default: num_cpus_on_systems)
  [-o output_log_dir]                        (default: /tmp)
- [-l source_language_number]                (default: 1 (C), supported: 1 (C), 2 (Verilog), 3 (PHP))
+ [-l source_language_number]                (default: 1 (C), supported: 1 (C), 2 (Verilog), 3 (PHP), 4 (C++))
  [-a anomaly_threshold]                     (default: 3.0)
 ```
 
-```
-scripts/scan_for_anomalies.sh -d <test_directory> -t <training_data_file> -o <output_log_dir>
-```
-
 As a part of scanning for anomalies, ControlFlag also suggests possible
 corrections in case a conditional expression is flagged as an anomaly. `25` is the
 `max_cost` for the correction -- how close should the suggested correction be to
@@ -218,7 +221,7 @@ $ grep "Potential anomaly" <output_log_dir>/thread_*.log
 A sample anomaly report looks like below:
 ```
 Level:<ONE or TWO> Expression: <AST_for_anomalous_expression>
-Source file and line number: <C code with line number having the anomaly>
+Source file and line number: <Source code expression with line number having the anomaly>
 Potential anomaly
 Did you mean ...
 ```

quick_start/test3_cpp.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+QUICK_START_DIR=`dirname $0`
+git clone https://github.com/google/sentencepiece.git
+${QUICK_START_DIR}/../scripts/mine_patterns.sh -d sentencepiece -o sentencepiece_training_data.ts -l 4
+git clone https://github.com/google/xrtl.git
+mkdir test3_scan_output
+${QUICK_START_DIR}/../scripts/scan_for_anomalies.sh -d xrtl/ -t sentencepiece_training_data.ts -o test3_scan_output -l 4

scripts/mine_patterns.sh

@@ -10,8 +10,8 @@ function print_usage() {
   else
     echo "[-n number_of_processes_to_use_for_mining]  (default: num_cpus_on_system)"
   fi
-  echo "[-l source_language_number] (default: 1 (C), supported: 1 (C), 2 (Verilog), 3 (PHP)"
-
+  echo "[-l source_language_number] (default: 1 (C), supported: 1 (C), 2 (Verilog), 3 (PHP), 4 (C++)"
+  echo "[-g github_repo_id] (default: 0) A unique identifier for GitHub repository, if any"
   exit
 }
 
@@ -23,14 +23,16 @@ else
   NUM_MINER_PROCS=`nproc`
 fi
 LANGUAGE=1
+REPO_ID=0
 
-while getopts d:o:n:l: flag
+while getopts d:o:n:l:g: flag
 do
   case "${flag}" in
     d) TRAIN_DIR=${OPTARG};;
     o) OUTPUT_FILE=${OPTARG};;
     n) NUM_MINER_PROCS=${OPTARG};;
     l) LANGUAGE=${OPTARG};;
+    g) REPO_ID=${OPTARG};;
   esac
 done
 
@@ -41,15 +43,15 @@ then
   print_usage $0
 fi
 
-if (( ${LANGUAGE} < 1  || ${LANGUAGE} > 3 ));
+if (( ${LANGUAGE} < 1  || ${LANGUAGE} > 4 ));
 then
-  echo "ERROR: Only 1 (C), 2 (Verilog) or 3 (PHP) are supported languages; received ${LANGUAGE}"
+  echo "ERROR: Only 1 (C), 2 (Verilog), 3 (PHP), and 4 (C++) are supported languages; received ${LANGUAGE}"
   print_usage $0
 fi
 
 if [ -f "${OUTPUT_FILE}" ]
 then
-  echo "ERROR: Output file exists. We don't want to over-write it."
+  echo "ERROR: Output file ${OUTPUT_FILE} exists. We don't want to over-write it."
   print_usage $0
 fi
 
@@ -58,36 +60,41 @@ FILE_LIST=${TMP_DIR}/file_list.txt
 if [ "${LANGUAGE}" = "1" ];
 then
   find "${TRAIN_DIR}" -iname "*.c" -o -iname "*.h" -type f > ${FILE_LIST}
+elif [ "${LANGUAGE}" = "2" ];
+then
+  find "${TRAIN_DIR}" -iname "*.v" -o -iname "*.vh" -type f > ${FILE_LIST}
 elif [ "${LANGUAGE}" = "3" ];
 then
   find "${TRAIN_DIR}" -iname "*.php" -type f | fgrep -v "/vendor/" > ${FILE_LIST}
-else
-  find "${TRAIN_DIR}" -iname "*.v" -o -iname "*.vh" -type f > ${FILE_LIST}
+elif [ "${LANGUAGE}" = "4" ];
+then
+  find "${TRAIN_DIR}" -iname "*.cpp" -o -iname "*.cc" -o -iname "*.cxx" -o -iname "*.h" -o -iname "*.hpp" -o -iname "*.hxx" -type f > ${FILE_LIST}
 fi
 
 SCRIPTS_DIR=`dirname $0`
 function dump_code_blocks() {
-  id=`echo "$1" | cut -d ':' -f 1`
-  f=`echo  "$1" | cut -d ':' -f 2-`
+  id=${REPO_ID}
+  f=$1
   ${SCRIPTS_DIR}/../bin/cf_dump_code_blocks -f "$f" -t 100 -g ${id} -l ${LANGUAGE} >> $2
 }
 export -f dump_code_blocks
 export LANGUAGE
 export SCRIPTS_DIR
+export REPO_ID
 
 if ! command -v parallel &> /dev/null
 then
   echo "GNU Parallel does not exist. Invoking serial dump.."
-  for id_f in `cat $FILE_LIST`;
+  for f in `cat $FILE_LIST`;
   do
-    dump_code_blocks ${id_f} ${OUTPUT_FILE}
+    dump_code_blocks ${f} ${OUTPUT_FILE}
   done
 
 else
 
   echo "GNU Parallel exists. Invoking parallel dump.."
   cat ${FILE_LIST} | parallel --eta --bar --progress \
-    -I% -j0 dump_code_blocks % ${TMP_DIR}/proc_{%}.log
+    -I% -j ${NUM_MINER_PROCS} dump_code_blocks % ${TMP_DIR}/proc_{%}.log
 
   for i in `seq 1 $NUM_MINER_PROCS`;
   do

scripts/scan_for_anomalies.sh

@@ -14,7 +14,7 @@ function print_usage() {
   fi
   echo " [-o output_log_dir]                        (default: /tmp)"
   echo " [-a anomaly_threshold]                     (default: 3.0)"
-  echo " [-l source_language_number]                (default: 1 (C), supported: 1 (C), 2 (Verilog), 3 (PHP)"
+  echo " [-l source_language_number]                (default: 1 (C), supported: 1 (C), 2 (Verilog), 3 (PHP), 4 (C++)"
 
   exit
 }
@@ -58,21 +58,25 @@ then
   print_usage $0
 fi
 
-if (( ${LANGUAGE} < 1  || ${LANGUAGE} > 3 ));
+if (( ${LANGUAGE} < 1  || ${LANGUAGE} > 4 ));
 then
-  echo "ERROR: Only 1 (C), 2 (Verilog) or 3 (PHP) are supported languages; received ${LANGUAGE}"
+  echo "ERROR: Only 1 (C), 2 (Verilog), 3 (PHP), and 4 (C++) are supported languages; received ${LANGUAGE}"
   print_usage $0
 fi 
 
 SCAN_FILE_LIST=`mktemp`
 if [ "${LANGUAGE}" = "1" ];
 then
   find "${SCAN_DIR}" -iname "*.c" -o -iname "*.h" -type f > ${SCAN_FILE_LIST}
+elif [ "${LANGUAGE}" = "2" ];
+then
+  find "${SCAN_DIR}" -iname "*.v" -o -iname "*.vh" -type f > ${SCAN_FILE_LIST}
 elif [ "${LANGUAGE}" = "3" ];
 then
   find "${SCAN_DIR}" -iname "*.php" -type f | fgrep -v "/vendor/" > ${SCAN_FILE_LIST}
-else
-  find "${SCAN_DIR}" -iname "*.v" -o -iname "*.vh" -type f > ${SCAN_FILE_LIST}
+elif [ "${LANGUAGE}" = "4" ];
+then
+  find "${SCAN_DIR}" -iname "*.cpp" -o -iname "*.cc" -o -iname "*.cxx" -o -iname "*.h" -o -iname "*.hpp" -o -iname "*.hxx" -type f > ${SCAN_FILE_LIST}
 fi
 
 SCRIPTS_DIR=`dirname $0`

src/CMakeLists.txt

@@ -27,6 +27,7 @@ set(COMMON_LINK_LIBRARIES
   tree-sitter
   tree-sitter-c
   tree-sitter-php
+  tree-sitter-cpp
   tree-sitter-verilog
   pthread
 )

src/cf_dump_code_blocks.cpp

@@ -93,7 +93,7 @@ int handle_command_args(int argc, char* argv[], CFDumpArgs& command_args) {
                 << std::endl
                 << "  [-l source_language_number]   (default: "
                 << LANGUAGE_C << ")"
-                << ", supported: 1 (C), 2 (Verilog), 3 (PHP)"
+                << ", supported: 1 (C), 2 (Verilog), 3 (PHP), 4 (C++)"
                 << std::endl;
   };
 
@@ -104,9 +104,9 @@ int handle_command_args(int argc, char* argv[], CFDumpArgs& command_args) {
       case 'l':
         command_args.source_language_ = VerifyLanguage(atoi(optarg)); break;
       case 'g':
-        command_args.github_contributor_id_ = std::strtoul(argv[3], NULL, 10);
+        command_args.github_contributor_id_ = std::strtoul(optarg, NULL, 10);
         break;
-      case 't': command_args.level_ = VerifyTreeLevel(atoi(argv[2])); break;
+      case 't': command_args.level_ = VerifyTreeLevel(atoi(optarg)); break;
       default: print_usage(); return EXIT_FAILURE;
     }
   }
@@ -137,6 +137,9 @@ int main(int argc, char* argv[]) {
       case LANGUAGE_PHP:
         DumpCodeBlocksFromSourceFile<LANGUAGE_PHP>(command_args);
         break;
+      case LANGUAGE_CPP:
+        DumpCodeBlocksFromSourceFile<LANGUAGE_CPP>(command_args);
+        break;
       default:
         throw cf_unexpected_situation("Unsupported language:" +
                                       std::to_string(LanguageToInt(

src/cf_file_scanner.cpp

@@ -24,6 +24,7 @@
 #include <string>
 #include <thread>  // NOLINT [build/c++11]
 
+#include "exception.h"
 #include "train_and_scan_util.h"
 #include "trie.h"
 
@@ -54,7 +55,7 @@ static int handle_command_args(int argc, char* argv[], FileScannerArgs& args) {
            << "  [-a anomaly_threshold]                     (default: 3.0)"
            << std::endl
            << "  [-l source_language_number]                (default: 1 (C), "
-           << "supported: 1 (C), 2 (Verilog), 3(PHP))"
+           << "supported: 1 (C), 2 (Verilog), 3 (PHP), 4 (C++))"
            << std::endl
            << "  [-v log_level ]                            (default: 0, "
            << "{ERROR, 0}, {INFO, 1}, {DEBUG, 2})"
@@ -159,7 +160,11 @@ int main(int argc, char* argv[]) {
             break;
           case LANGUAGE_PHP:
             status = train_and_scan_util.ScanFile<LANGUAGE_PHP>(eval_file,
-                                                                    log_file);
+                                                                log_file);
+            break;
+          case LANGUAGE_CPP:
+            status = train_and_scan_util.ScanFile<LANGUAGE_CPP>(eval_file,
+                                                                log_file);
             break;
           default:
             throw cf_unexpected_situation("Unsupported language:" +

src/common_util.cpp

@@ -23,6 +23,7 @@
 #include <sstream>
 
 #include "parser.h"
+#include "exception.h"
 #include "common_util.h"
 
 template <Language L>
@@ -91,8 +92,8 @@ void CollectCodeBlocksOfInterest<LANGUAGE_VERILOG>(const TSNode& node,
   }
 }
 
-// For C and PHP language,
-// we are looking for control structures such as if statements.
+// For C, C++, and PHP language, we are looking for control structures
+// such as if statements.
 template <Language L>
 void CollectCodeBlocksOfInterest(const TSNode& node,
     code_blocks_t& code_blocks) {
@@ -126,17 +127,24 @@ ManagedTSTree GetTSTree<LANGUAGE_VERILOG>(const std::string&, bool);
 template
 ManagedTSTree GetTSTree<LANGUAGE_PHP>(const std::string&, bool);
 template
+ManagedTSTree GetTSTree<LANGUAGE_CPP>(const std::string&, bool);
+template
 ManagedTSTree GetTSTree<LANGUAGE_C>(const std::string&, std::string&);
 template
 ManagedTSTree GetTSTree<LANGUAGE_VERILOG>(const std::string&, std::string&);
 template
 ManagedTSTree GetTSTree<LANGUAGE_PHP>(const std::string&, std::string&);
 template
+ManagedTSTree GetTSTree<LANGUAGE_CPP>(const std::string&, std::string&);
+template
 void CollectCodeBlocksOfInterest<LANGUAGE_C>(const ManagedTSTree&,
                                              code_blocks_t&);
 template
 void CollectCodeBlocksOfInterest<LANGUAGE_VERILOG>(const ManagedTSTree &,
                                                    code_blocks_t&);
 template
 void CollectCodeBlocksOfInterest<LANGUAGE_PHP>(const ManagedTSTree &,
-                                                   code_blocks_t&);
+                                               code_blocks_t&);
+template
+void CollectCodeBlocksOfInterest<LANGUAGE_CPP>(const ManagedTSTree &,
+                                               code_blocks_t&);