From 1f91b9a9f36a7eca58b8786147d0da922044a9a7 Mon Sep 17 00:00:00 2001 From: Vid Date: Mon, 10 Oct 2022 18:28:31 -0700 Subject: [PATCH 1/6] #404 updating docker-compose for macos compatibility --- docker/Arkime/Arkime.dockerfile | 2 +- docker/docker-compose.yml | 36 ++++++++++++++++----------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docker/Arkime/Arkime.dockerfile b/docker/Arkime/Arkime.dockerfile index b2e6df7c7..51bf3908f 100644 --- a/docker/Arkime/Arkime.dockerfile +++ b/docker/Arkime/Arkime.dockerfile @@ -24,7 +24,7 @@ RUN apt-get update && \ mkdir -p /suricata-logs && \ cd /data && \ wget -q "https://s3.amazonaws.com/files.molo.ch/builds/ubuntu-"$UBUNTU_VERSION"/"$ARKIME_DEB_PACKAGE && \ - apt-get install -y ./$ARKIME_DEB_PACKAGE && \ + apt-get install -y ${PWD}/$ARKIME_DEB_PACKAGE && \ mv $ARKIMEDIR/etc /data/config && \ ln -s /data/config $ARKIMEDIR/etc && \ ln -s /data/logs $ARKIMEDIR/logs && \ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 124bccafb..031eefe9e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -32,7 +32,7 @@ volumes: driver_opts: type: none o: bind - device: ./containers-data/suricata/logrotate + device: ${PWD}/containers-data/suricata/logrotate logstash-sincedb: #where logstash stores it's state so it doesn't re-ingest arkime-logs: arkime-pcap: @@ -95,9 +95,9 @@ services: mem_limit: ${LOGSTASH_MEMORY:-2G} volumes: - logstash-sincedb:/since.db - - ./containers-data/suricata/logs:/var/log/suricata:ro - - ./containers-data/logstash/conf.d/logstash.conf:/usr/share/logstash/pipeline/logstash.conf - - ./containers-data/logstash/templates/elasticsearch7-template.json:/usr/share/logstash/config/elasticsearch7-template.json + - ${PWD}/containers-data/suricata/logs:/var/log/suricata:ro + - ${PWD}/containers-data/logstash/conf.d/logstash.conf:/usr/share/logstash/pipeline/logstash.conf + - ${PWD}/containers-data/logstash/templates/elasticsearch7-template.json:/usr/share/logstash/config/elasticsearch7-template.json networks: network: @@ -116,11 +116,11 @@ services: - SYS_NICE network_mode: host volumes: - - ./containers-data/suricata/logs:/var/log/suricata + - ${PWD}/containers-data/suricata/logs:/var/log/suricata - suricata-rules:/etc/suricata/rules - suricata-run:/var/run/suricata/ - - ./containers-data/suricata/etc:/etc/suricata - - suricata-logrotate:/etc/logrotate.d/ + - ${PWD}/containers-data/suricata/etc:/etc/suricata + - ${PWD}/containers-data/suricata/logrotate:/etc/logrotate.d/ scirius: container_name: scirius @@ -147,7 +147,7 @@ services: volumes: - scirius-static:/static/ - scirius-data:/data/ - - ./containers-data/scirius/logs/:/logs/ + - ${PWD}/containers-data/scirius/logs/:/logs/ - suricata-rules:/rules - suricata-run:/var/run/suricata @@ -169,9 +169,9 @@ services: restart: ${RESTART_MODE:-unless-stopped} volumes: - scirius-static:/static/:ro - - ./containers-data/nginx/conf.d/:/etc/nginx/conf.d/:ro - - ./containers-data/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./containers-data/nginx/ssl:/etc/nginx/ssl:ro + - ${PWD}/containers-data/nginx/conf.d/selks6.conf:/etc/nginx/conf.d/selks6.conf:ro + - ${PWD}/containers-data/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ${PWD}/containers-data/nginx/ssl:/etc/nginx/ssl:ro ports: - 443:443 networks: @@ -188,12 +188,12 @@ services: - /var/run/docker.sock:/var/run/docker.sock # This bind-mout allows using the hosts docker deamon instead of created one inside the container # Those volumes will contain the cron jobs - - ./containers-data/cron-jobs/1min:/etc/periodic/1min/:ro - - ./containers-data/cron-jobs/15min:/etc/periodic/15min/:ro - - ./containers-data/cron-jobs/daily:/etc/periodic/daily/:ro - - ./containers-data/cron-jobs/hourly:/etc/periodic/hourly/:ro - - ./containers-data/cron-jobs/monthly:/etc/periodic/monthly/:ro - - ./containers-data/cron-jobs/weekly:/etc/periodic/weekly/:ro + - ${PWD}/containers-data/cron-jobs/1min:/etc/periodic/1min/:ro + - ${PWD}/containers-data/cron-jobs/15min:/etc/periodic/15min/:ro + - ${PWD}/containers-data/cron-jobs/daily:/etc/periodic/daily/:ro + - ${PWD}/containers-data/cron-jobs/hourly:/etc/periodic/hourly/:ro + - ${PWD}/containers-data/cron-jobs/monthly:/etc/periodic/monthly/:ro + - ${PWD}/containers-data/cron-jobs/weekly:/etc/periodic/weekly/:ro arkime: @@ -201,7 +201,7 @@ services: image: ghcr.io/stamusnetworks/arkimeviewer:${ARKIMEVIEWER_VERSION:-master} ## Repo will need to be changed to stamusnetwork once image built restart: ${RESTART_MODE:-no} volumes: - - ./containers-data/suricata/logs:/suricata-logs:ro + - ${PWD}/containers-data/suricata/logs:/suricata-logs:ro - arkime-config:/data/config - arkime-pcap:/data/pcap - arkime-logs:/data/logs From aa9a84f3321c1a8a223d9f591ed6cbc74020af87 Mon Sep 17 00:00:00 2001 From: Vid Date: Mon, 10 Oct 2022 18:29:22 -0700 Subject: [PATCH 2/6] #404 tracking required dirs --- docker/containers-data/nginx/static/.gitkeep | 0 docker/containers-data/suricata/logrotate/.gitkeep | 0 docker/containers-data/suricata/logs/.gitkeep | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 docker/containers-data/nginx/static/.gitkeep create mode 100644 docker/containers-data/suricata/logrotate/.gitkeep create mode 100644 docker/containers-data/suricata/logs/.gitkeep diff --git a/docker/containers-data/nginx/static/.gitkeep b/docker/containers-data/nginx/static/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker/containers-data/suricata/logrotate/.gitkeep b/docker/containers-data/suricata/logrotate/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker/containers-data/suricata/logs/.gitkeep b/docker/containers-data/suricata/logs/.gitkeep new file mode 100644 index 000000000..e69de29bb From 30b0cde076ec31945323f70f163111e1c0390052 Mon Sep 17 00:00:00 2001 From: Vid Date: Mon, 10 Oct 2022 18:30:33 -0700 Subject: [PATCH 3/6] #404 accomodate modern syntax: Docker Compose in easy-setup --- docker/easy-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/easy-setup.sh b/docker/easy-setup.sh index 6137a936f..44d57c642 100755 --- a/docker/easy-setup.sh +++ b/docker/easy-setup.sh @@ -393,7 +393,7 @@ function is_docker_installed(){ } function is_compose_installed(){ composeV=$(docker-compose --version 2>/dev/null) - if [[ $composeV == *"docker-compose version"* ]]; then + if [[ $composeV == *"ompose version"* ]]; then echo "yes" else echo "no" From bbe3aa89a309e4b2f3224e4a2b037839b6a73b76 Mon Sep 17 00:00:00 2001 From: Vid Date: Mon, 10 Oct 2022 18:31:11 -0700 Subject: [PATCH 4/6] #404 improving error feedback for verbosity and MacOS --- docker/easy-setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/easy-setup.sh b/docker/easy-setup.sh index 44d57c642..12498e4fb 100755 --- a/docker/easy-setup.sh +++ b/docker/easy-setup.sh @@ -470,17 +470,17 @@ function check_docker_version(){ dockerV=$(docker version --format '{{.Server.Version}}') if Version $dockerV '<' "${MINIMAL_DOCKER_VERSION}"; then - echo -e "${red}-${reset} Docker version is too old, please upgrade it to ${MINIMAL_DOCKER_VERSION} minimum" + echo -e "${red}-${reset} Docker version ($dockerV) is too old, please upgrade it to ${MINIMAL_DOCKER_VERSION} minimum" exit fi } function check_compose_version(){ composeV=$(docker-compose --version) composeV=( $composeV ) - composeV=$( echo ${composeV[2]} |tr ',' ' ') + composeV=$( echo ${composeV[ $((${#composeV[@]} - 1)) ]} |tr ',' ' ' ) if Version $composeV '<' "${MINIMAL_COMPOSE_VERSION}"; then - echo -e "${red}-${reset} Docker version is too old, please upgrade it to ${MINIMAL_COMPOSE_VERSION} minimum" + echo -e "${red}-${reset} Docker-compose version ($composeV) is too old, please upgrade it to ${MINIMAL_COMPOSE_VERSION} minimum" exit fi } From 80933c370d2461e8e0401e67c48ef20f94f585fe Mon Sep 17 00:00:00 2001 From: Vid Date: Mon, 10 Oct 2022 18:32:02 -0700 Subject: [PATCH 5/6] #404 adding MacOS accomodations to .sh --- docker/easy-setup.sh | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/docker/easy-setup.sh b/docker/easy-setup.sh index 12498e4fb..9f358b7d1 100755 --- a/docker/easy-setup.sh +++ b/docker/easy-setup.sh @@ -743,8 +743,19 @@ echo "COMPOSE_PROJECT_NAME=SELKS" > ${BASEDIR}/.env function getInterfaces { echo -e " Network interfaces detected:" intfnum=0 - for interface in $(ls /sys/class/net); do echo "${intfnum}: ${interface}"; ((intfnum++)) ; done + isMacOS=false + if [[ $OSTYPE == darwin* ]]; then # OSTYPE is a Bash Built-in OS detector. darwin=OSX/macOS + isMacOS=true; + fi + if $isMacOS; then + ifaceSource=$(networksetup -listallhardwareports | grep Device | awk '{ print $2}') + else + ifaceSource=$(ls /sys/class/net) + fi + + for interface in $ifaceSource; do echo "${intfnum}: ${interface}"; ((intfnum++)) ; done + echo -e "Please type in interface or space delimited interfaces below and hit \"Enter\"." echo -e "Choose the interface(s) that is (are) one the network(s) you want to monitor" echo -e "Example: eth1" @@ -768,10 +779,15 @@ function getInterfaces { INTERFACE_EXISTS="NO" exit 1 fi - for interface in ${interfaces} do - if ! cat /sys/class/net/${interface}/operstate > /dev/null 2>&1 ; then + interfaceCheck=$(cat /sys/class/net/${interface}/operstate > /dev/null 2>&1) + + if $isMacOS; then + interfaceCheck=$(ipconfig getifaddr ${interface} > /dev/null 2>&1) + fi + + if ! $interfaceCheck ; then echo -e "\nUSAGE: `basename $0` -> the script requires at least 1 argument - a network interface!" echo -e "#######################################" echo -e "Interface: ${interface} is NOT existing." @@ -779,7 +795,6 @@ function getInterfaces { echo -e "Please supply a correct/existing network interface or check your spelling.\n" INTERFACE_EXISTS="NO" fi - done } @@ -854,10 +869,15 @@ echo ###################### docker_root_dir=$(docker system info |grep "Docker Root Dir") -docker_root_dir=${docker_root_dir/'Docker Root Dir: '/''} +docker_root_dir=$(echo $docker_root_dir | awk -F': ' '{print $2}') +if $isMacOS; then + docker_vol_df=$(docker run -it --rm --privileged --pid=host debian nsenter -t 1 -m -u -n -i bash -c "df -h /var/lib/docker/" | sed '1d' | awk '{print $(NF-2)}') +else + docker_vol_df=$(df --output=avail -h ${docker_root_dir} | tail -n 1 ) +fi echo "" -echo -e "By default, elasticsearch database is stored in a docker volume in ${docker_root_dir} (free space: $(df --output=avail -h ${docker_root_dir} | tail -n 1 )" +echo -e "By default, elasticsearch database is stored in a docker volume in ${docker_root_dir} (free space: $docker_vol_df" echo -e "With SELKS running, database can take up a lot of disk space" echo -e "You might want to save them on an other disk/partition" echo -e "Alternatively, You can specify a path where you want the data to be saved, or hit enter for default." @@ -976,7 +996,11 @@ fi ###################### # Starting # ###################### -echo -e "\n\n${green}To start SELKS, run 'sudo docker-compose up -d'${reset}\n" +if $isMacOS; then + echo -e "\n\n${green}To start SELKS, run 'docker-compose up -d'${reset}\n" +else + echo -e "\n\n${green}To start SELKS, run 'sudo docker-compose up -d'${reset}\n" +fi if [[ "$PORTAINER_INSTALLED" == "true" ]]; then echo -e "${red}IMPORTANT:${reset} You chose to install Portainer, visit https://localhost:9443 to set your portainer admin password" From e0970731ed36d4106a04439ae02cd441391aa34b Mon Sep 17 00:00:00 2001 From: Vid Date: Mon, 10 Oct 2022 18:46:52 -0700 Subject: [PATCH 6/6] #404 removed trailing whitespace --- docker/easy-setup.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docker/easy-setup.sh b/docker/easy-setup.sh index 9f358b7d1..07e34fa7b 100755 --- a/docker/easy-setup.sh +++ b/docker/easy-setup.sh @@ -410,7 +410,7 @@ function is_docker_availabale_for_user(){ function test_docker(){ hello=$(docker run --rm hello-world) || \ echo "${red}-${reset} Docker test failed" - + if [[ $hello == *"Hello from Docker"* ]]; then echo -e "${green}+${reset} Docker seems to be installed properly" else @@ -591,10 +591,10 @@ echo -e "\n" load_docker_images_from_tar ${BASEDIR}/tar_images if [[ "${_arg_skip_checks}" == "off" ]] ; then - + ############################# # DOCKER # - ############################# + ############################# if [[ $(is_docker_installed) == "yes" ]]; then echo -e "${green}+${reset} Docker installation found: $(docker -v)" @@ -654,7 +654,7 @@ if [[ "${_arg_skip_checks}" == "off" ]] ; then ############################# # PORTAINER # ############################# - + if $(docker ps | grep -q 'portainer'); then echo -e " Found existing portainer installation, skipping...\n" else @@ -679,7 +679,7 @@ if [[ "${_arg_skip_checks}" == "off" ]] ; then esac done fi - + fi ############################# @@ -691,12 +691,12 @@ function check_scirius_key_cert(){ # usage : check_scirius_key_cert [path_to_files] [filename_without_extension] # example : check_scirius_key_cert [path_to_files] [filename_without_extension] output=$(docker run --rm -it -v ${1}:/etc/nginx/ssl nginx /bin/bash -c "openssl x509 -in /etc/nginx/ssl/scirius.crt -pubkey -noout -outform pem | sha256sum; openssl pkey -in /etc/nginx/ssl/scirius.key -pubout -outform pem | sha256sum" || echo -e "${red}-${reset} Error while checking certificate against key") - + SAVEIFS=$IFS # Save current IFS IFS=$'\n' # Change IFS to new line output=($output) # split to array $names IFS=$SAVEIFS # Restore IFS - + if [[ ${output[0]}==${output[1]} ]]; then echo -e "${green}+${reset} Certificate match private key" return 0 @@ -747,7 +747,7 @@ function getInterfaces { if [[ $OSTYPE == darwin* ]]; then # OSTYPE is a Bash Built-in OS detector. darwin=OSX/macOS isMacOS=true; fi - + if $isMacOS; then ifaceSource=$(networksetup -listallhardwareports | grep Device | awk '{ print $2}') else @@ -762,14 +762,14 @@ function getInterfaces { echo -e "OR" echo -e "Example: eth1 eth2 eth3" echo -e "\nConfigure threat detection for INTERFACE(S): " - + if [[ "${INTERFACES}" == "" ]] && [[ "${INTERACTIVE}" == "true" ]]; then read interfaces else echo "${INTERFACES}" interfaces=${INTERFACES} fi - + echo -e "\nThe supplied network interface(s): ${interfaces}" echo ""; INTERFACE_EXISTS="YES" @@ -891,7 +891,7 @@ fi if ! [ -z "${elastic_data_path}" ]; then - while ! [ -w "${elastic_data_path}" ]; do + while ! [ -w "${elastic_data_path}" ]; do echo -e "\nYou don't seem to own write access to this directory\n" echo -e "You can specify a path where you want the data to be saved, or hit ENTER to use a [docker volume]." if [[ "${INTERACTIVE}" == "true" ]]; then