Adds Test Coverage (#394)

* Don't run streamer multitab tests on OSX/Travis.

* Disable all socket streamer tests.

* no way to setup proxy yet.

* Temp commit.

* Some more changes but not complete.

* local tests.

* build was successful.

* Added pytest compatible tests. Generates test report at the end.

* Fixed hhfit tests.

* test commit [seg fault] skip ci

* Removed benchamark code.

* Added dependencies to requirements.txt file.  Set verbosity to 0.  Minor
cleanup.  local tests are passing...  Added test: rdesineur loads chan
proto from a given file.

* Fixed hhfit test.  Added scipy to run hhfit test.

* Fixed one more test.

* test it on xcode10.3

* Some fixes to test_hhfit. The module selects a best function
'fuzzingly'.

* - Removed getopt in pymoose code. It was causing segfault on OSX. Not sure why. `getopt` is no longer needed there.
- Use scipy in tests as well. hhfit requires scipy.
- Added target coverage for make.
- Enabled coverage on travis.
- Fixes to unicode/str issues b/w python2 and 3.

* - Fixes to issue #393
- Don't let user use buggy gcc 4.8.x which does not proper regex support. See https://stackoverflow.com/questions/12530406/is-gcc-4-8-or-earlier-buggy-about-regular-expressions

* updated docker file for centos:7 . Install devtoolset-8.

* Some more tweak to test_hhfit.py file. THe hhfit function does not
always return the demanded function but could also return a function
which is a better fit (it is designed this way).  Without this fix, the
test could fail on a separte run or on a separte platform.

* Update writeKkit.py

Fixed email id.

Co-authored-by: Dilawar Singh <dilawars@ncbs.res.in>

Author: dilawar

.ci/coverage.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+python3 -m trace --count \
+    --ignore-dir=/usr/lib64 \
+    --ignore-module=libsbml \
+    --ignore-module=numpy \
+    --ignore-module=neuroml \
+    "$@"
+

.ci/travis_build_linux.sh

@@ -16,14 +16,18 @@ PYTHON2="/usr/bin/python2"
 PYTHON3="/usr/bin/python3"
 
 $PYTHON2 -m pip install pip --upgrade --user
-$PYTHON2 -m pip install libNeuroML pyNeuroML python-libsbml --upgrade --user
+$PYTHON2 -m pip install libNeuroML numpy==1.14 scipy==1.2 \
+    pyNeuroML python-libsbml --upgrade --user
 
 $PYTHON3 -m pip install pip --upgrade --user
 $PYTHON3 -m pip install libNeuroML pyNeuroML python-libsbml --upgrade --user
 
 # sympy is only needed for pretty-priting for one test.
-$PYTHON3 -m pip install sympy --upgrade --user   
+$PYTHON3 -m pip install numpy sympy scipy --upgrade --user   
 
+# pytest requirements.
+$PYTHON3 -m pip install -r ./tests/requirements.txt --user
+ 
 NPROC=$(nproc)
 MAKE="make -j$NPROC"
 
@@ -47,11 +51,23 @@ $PYTHON3 -m compileall -q .
 
     # Run all tests in debug mode.
     MOOSE_NUM_THREADS=$NPROC ctest -j$NPROC --output-on-failure 
+
+
     make install || sudo make install 
     cd /tmp
     $PYTHON3 -c 'import moose;print(moose.__file__);print(moose.version())'
 )
 
+# BOOST and python3
+(
+    mkdir -p $BUILDDIR && cd $BUILDDIR && \
+        cmake -DWITH_BOOST_ODE=ON -DPYTHON_EXECUTABLE="$PYTHON3" \
+        -DCMAKE_INSTALL_PREFIX=/usr ..
+    # Run coverage 
+    export MOOSE_NUM_THREADS=3
+    make coverage 
+)
+
 # GSL and python2, failure is allowed
 set +e
 (
@@ -61,13 +77,4 @@ set +e
 )
 set -e
 
-
-# BOOST and python3
-(
-    mkdir -p $BUILDDIR && cd $BUILDDIR && \
-        cmake -DWITH_BOOST_ODE=ON -DPYTHON_EXECUTABLE="$PYTHON3" \
-        -DCMAKE_INSTALL_PREFIX=/usr ..
-    $MAKE && MOOSE_NUM_THREADS=$NPROC ctest -j$NPROC --output-on-failure 
-)
-
 echo "All done"

.gitlab-ci.yml

@@ -14,7 +14,9 @@ build:
     - apt -y install python-numpy python-matplotlib python-pip python-dev
   script:
     - python3 -m pip install setuptools pip --user --upgrade
+    - python3 -m pip install python-libsbml --user --upgrade
     - python3 -m pip install pyneuroml --user --upgrade
+    - python3 -m pip install sympy scipy --user --upgrade
     - python3 setup.py build test 
     - python3 setup.py install --user
     - python3 -c "import moose; moose.test()"

.travis.yml

@@ -8,7 +8,8 @@ matrix:
     services:
         - xvfb
   - os: osx
-    osx_image: xcode11.3
+    osx_image: 
+     - xcode11.3
 
 notifications:
     email:

CMakeLists.txt

@@ -228,17 +228,18 @@ add_subdirectory(ksolve)
 add_subdirectory(hsolve)
 add_subdirectory(diffusion)
 add_subdirectory(device)
-add_subdirectory(benchmarks)
 add_subdirectory(kinetics)
 add_subdirectory(synapse)
 add_subdirectory(intfire)
 add_subdirectory(external/libsoda)
 
+# development related.
+add_subdirectory(devel)
+
 ###################################### LINKING #################################
 list(APPEND MOOSE_LIBRARIES
     moose_builtins
     msg
-    benchmarks
     shell
     randnum
     scheduling
@@ -364,21 +365,33 @@ if(DEBUG OR "${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
     add_test(NAME moose.bin-raw-run COMMAND moose.bin -u -q)
 endif()
 
-# PyMOOSE tests. These tests complete in reasonable time and should run by
-# default.
+# Core tests.
 set(PYMOOSE_TEST_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/py_moose)
 file(GLOB PY_TEST_SCRIPTS "${PYMOOSE_TEST_DIRECTORY}/test_*.py" )
 foreach(_test_script ${PY_TEST_SCRIPTS} )
     get_filename_component(_name_we ${_test_script} NAME_WE)
-    set(_test_name "pymoose_${_name_we}")
+    set(_test_name "core_${_name_we}")
     add_test(NAME ${_test_name}
         COMMAND ${PYTHON_EXECUTABLE} ${_test_script}
-        WORKING_DIRECTORY ${PYMOOSE_TEST_DIRECTORY}
+        WORKING_DIRECTORY ${PYMOOSE_TEST_DIRECTORY})
+     set_tests_properties(${_test_name}
+         PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/python"
         )
+endforeach()
+
+# Tests for supported formats such as neuroml, sbml etc.
+set(SUPPORT_TEST_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/support)
+file(GLOB PY_TEST_SCRIPTS "${SUPPORT_TEST_DIRECTORY}/test_*.py" )
+foreach(_test_script ${PY_TEST_SCRIPTS} )
+    get_filename_component(_name_we ${_test_script} NAME_WE)
+    set(_test_name "support_${_name_we}")
+    add_test(NAME ${_test_name}
+        COMMAND ${PYTHON_EXECUTABLE} ${_test_script}
+        WORKING_DIRECTORY ${SUPPORT_TEST_DIRECTORY})
      set_tests_properties(${_test_name}
          PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/python"
         )
-endforeach( )
+endforeach()
 
 # rdesigneur tests. These tests require matplotlib.
 set(RDES_TEST_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/py_rdesigneur) 
@@ -396,7 +409,7 @@ foreach(_test_script ${RDES_TEST_SCRIPTS})
 endforeach()
 
 # FIXME TESTS. These should not run by default. We need to fix them.
-set(PYMOOSE_FIXME_TEST_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/fixme)
+set(PYMOOSE_FIXME_TEST_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/devel/fixme)
 file(GLOB PY_FIXME_TEST_SCRIPTS "${PYMOOSE_FIXME_TEST_DIRECTORY}/*.py" )
 foreach( _test_script ${PY_FIXME_TEST_SCRIPTS} )
     get_filename_component( _name_we ${_test_script} NAME_WE)
@@ -426,9 +439,36 @@ foreach(_test_script ${PY_ISSUES_SCRIPTS})
         )
 endforeach()
 
-############################ CPACK ######################################
-# This is not maintained anymore since packaging is done using OpenBuildService 
-include(${CMAKE_CURRENT_SOURCE_DIR}/cmake_moose_cpack.cmake)
+################################# COVERAGE ###################################
+
+add_custom_target(coverage)
+set(PYTEST_TEST_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests)
+file(GLOB_RECURSE PYTEST_TEST_SCRIPTS ${PYTEST_TEST_DIRECTORY}/test_*.py)
+
+foreach(_test_script ${PYTEST_TEST_SCRIPTS})
+    get_filename_component(_name_we ${_test_script} NAME_WE)
+    set(_test_name "pytest_${_name_we}")
+    # message(STATUS "Adding test ${_test_name}")
+    add_custom_target(${_test_name}
+        # We collect coverage data but do not show report during each test.
+        # We show report at the end of all tests. Set `--cov-report=` 
+        COMMAND ${PYTHON_EXECUTABLE} -m pytest 
+            --cov=moose --cov=rdesigneur --cov-append --cov-report= 
+            ${_test_script}
+        DEPENDS _moose
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+        COMMENT "Executing ${_test_script} using pytest"
+        VERBATIM)
+    add_dependencies(coverage  ${_test_name})
+endforeach()
+
+# Generate report at the end.
+add_custom_command(TARGET coverage POST_BUILD
+    COMMAND ${PYTHON_EXECUTABLE} -m coverage report 
+    COMMAND ${PYTHON_EXECUTABLE} -m coverage html 
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Genearting coverage report"
+    VERBATIM)
 
 
 ########################### RELEASE #########################################
@@ -479,10 +519,3 @@ foreach(_py_script ${PY_SCRIPTS})
     add_dependencies(pylint ${TGT_NAME} )
 endforeach( )
 
-# Replicate Travis-CI building using docker
-add_custom_target(docker
-    COMMAND docker build -t bhallalab/travis:latest
-        -f ${CMAKE_CURRENT_SOURCE_DIR}/devel/docker/travis/Dockerfile .
-    COMMENT "Replicating Travis-CI building using Docker."
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    VERBATIM)

CheckCXXCompiler.cmake

@@ -18,16 +18,18 @@ add_definitions(-Wall
     -Wno-unused-function
     #-Wno-unused-private-field
     )
+
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+    # gcc-4.9.0 has regex supports though moose will compile with 4.8.x;
+    # <regex> won't work.
+    if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9")
+        message(FATAL_ERROR "Insufficient gcc version. Minimum requried 4.9")
+    endif()
     add_definitions( -Wno-unused-local-typedefs )
 elseif(("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") OR ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang"))
     add_definitions( -Wno-unused-local-typedef )
 endif()
 
-if( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" )
-    add_definitions( -Wno-unused-local-typedefs )
-endif()
-
 add_definitions(-fPIC)
 if(COMPILER_WARNS_STRICT_ALIASING)
     add_definitions( -Wno-strict-aliasing )