Merge pull request #651 from dtag-dev-sec/dev

merge dev to master

Author: t3chn0m4g3

docker/elk/elasticsearch/Dockerfile

@@ -1,7 +1,7 @@
 FROM alpine
 #
 # VARS
-ENV ES_VER=7.6.2 \
+ENV ES_VER=7.7.1 \
     JAVA_HOME=/usr/lib/jvm/java-11-openjdk
 # Include dist
 ADD dist/ /root/dist/

docker/elk/kibana/Dockerfile

@@ -1,7 +1,7 @@
 FROM node:10.19.0-alpine
 #
 # VARS
-ENV KB_VER=7.6.2
+ENV KB_VER=7.7.1
 # 
 # Include dist
 ADD dist/ /root/dist/
@@ -47,6 +47,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
     echo "xpack.security.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "xpack.siem.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    echo "xpack.ml.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "elasticsearch.requestTimeout: 60000" >> /usr/share/kibana/config/kibana.yml && \
     echo "elasticsearch.shardTimeout: 60000" >> /usr/share/kibana/config/kibana.yml && \
     rm -rf /usr/share/kibana/optimize/bundles/* && \

docker/elk/logstash/Dockerfile

@@ -1,7 +1,7 @@
 FROM alpine
 #
 # VARS
-ENV LS_VER=7.6.2
+ENV LS_VER=7.7.1
 # Include dist
 ADD dist/ /root/dist/
 #
@@ -36,7 +36,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
     chmod u+x /usr/bin/update.sh && \
     mkdir -p /etc/logstash/conf.d && \
     cp logstash.conf /etc/logstash/conf.d/ && \
-    cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.3.3-java/lib/logstash/outputs/elasticsearch/ && \
+    cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.4.2-java/lib/logstash/outputs/elasticsearch/ && \
 #
 # Setup user, groups and configs
     addgroup -g 2000 logstash && \

iso/installer/install.sh

@@ -12,9 +12,9 @@ mySITES="https://hub.docker.com https://github.com https://pypi.python.org https
 myTPOTCOMPOSE="/opt/tpot/etc/tpot.yml"
 myLSB_STABLE_SUPPORTED="stretch buster"
 myLSB_TESTING_SUPPORTED="stable"
-myREMOTESITES="https://hub.docker.com https://github.com https://pypi.python.org https://debian.org"
+myREMOTESITES="https://hub.docker.com https://github.com https://pypi.python.org https://debian.org https://listbot.sicherheitstacho.eu"
 myPREINSTALLPACKAGES="aria2 apache2-utils cracklib-runtime curl dialog figlet fuse grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet"
-myINSTALLPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit console-setup console-setup-linux cracklib-runtime curl debconf-utils dialog dnsutils docker.io docker-compose elasticsearch-curator ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 libpam-google-authenticator man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant"
+myINSTALLPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit console-setup console-setup-linux cracklib-runtime curl debconf-utils dialog dnsutils docker.io docker-compose ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 libpam-google-authenticator man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant"
 myINFO="\
 ###########################################
 ### T-Pot Installer for Debian (Stable) ###
@@ -310,9 +310,9 @@ EOF
   apt-fast -y install $myINSTALLPACKAGES
   # Remove exim4
   echo "### Removing and holding back problematic packages ..."
-  apt-fast -y purge exim4-base mailutils pcp cockpit-pcp
+  apt-fast -y purge exim4-base mailutils pcp cockpit-pcp elasticsearch-curator
   apt-fast -y autoremove
-  apt-mark hold exim4-base mailutils pcp cockpit-pcp
+  apt-mark hold exim4-base mailutils pcp cockpit-pcp elasticsearch-curator
 }
 
 # Check for other services
@@ -681,10 +681,10 @@ echo "$myNETWORK_WLANEXAMPLE" | tee -a /etc/network/interfaces
 fuBANNER "SSH roaming off"
 echo "UseRoaming no" | tee -a /etc/ssh/ssh_config
 
-# Installing elasticdump, yq
+# Installing elasticdump, elasticsearch-curator, yq
 fuBANNER "Installing pkgs"
 npm install elasticdump -g
-pip3 install yq
+pip3 install elasticsearch-curator yq
 hash -r
 
 # Cloning T-Pot from GitHub
@@ -775,29 +775,30 @@ echo "$myCRONJOBS" | tee -a /etc/crontab
 
 # Let's create some files and folders
 fuBANNER "Files & folders"
-mkdir -p /data/adbhoney/downloads /data/adbhoney/log \
+mkdir -vp /data/adbhoney/{downloads,log} \
          /data/ciscoasa/log \
-	 /data/citrixhoneypot/logs \
-      	 /data/conpot/log \
-         /data/cowrie/log/tty/ /data/cowrie/downloads/ /data/cowrie/keys/ /data/cowrie/misc/ \
-         /data/dionaea/log /data/dionaea/bistreams /data/dionaea/binaries /data/dionaea/rtp /data/dionaea/roots/ftp /data/dionaea/roots/tftp /data/dionaea/roots/www /data/dionaea/roots/upnp \
+         /data/conpot/log \
+         /data/citrixhoneypot/logs \
+         /data/cowrie/{downloads,keys,misc,log,log/tty} \
+         /data/dionaea/{log,bistreams,binaries,rtp,roots,roots/ftp,roots/tftp,roots/www,roots/upnp} \
          /data/elasticpot/log \
-         /data/elk/data /data/elk/log \
-	 /data/fatt/log \
-         /data/honeytrap/log/ /data/honeytrap/attacks/ /data/honeytrap/downloads/ \
+         /data/elk/{data,log} \
+         /data/fatt/log \
+         /data/honeytrap/{log,attacks,downloads} \
          /data/glutton/log \
          /data/heralding/log \
          /data/honeypy/log \
          /data/mailoney/log \
          /data/medpot/log \
-         /data/nginx/log /data/nginx/heimdall \
+         /data/nginx/{log,heimdall} \
          /data/emobility/log \
          /data/ews/conf \
          /data/rdpy/log \
          /data/spiderfoot \
-         /data/suricata/log /home/tsec/.ssh/ \
-      	 /data/tanner/log /data/tanner/files \
-         /data/p0f/log
+         /data/suricata/log \
+         /data/tanner/{log,files} \
+         /data/p0f/log \
+         /home/tsec/.ssh/
 touch /data/spiderfoot/spiderfoot.db
 touch /data/nginx/log/error.log
 

update.sh

@@ -183,9 +183,10 @@ function fuUPDATER () {
 export DEBIAN_FRONTEND=noninteractive
 echo "### Installing apt-fast"
 /bin/bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/quick-install.sh)"
-local myPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit console-setup console-setup-linux cracklib-runtime curl debconf-utils dialog dnsutils docker.io docker-compose elasticsearch-curator ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 libpam-google-authenticator man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-elasticsearch-curator python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant"
-echo "### Removing pip based install of elasticsearch-curator"
-pip3 uninstall elasticsearch-curator -y
+local myPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit console-setup console-setup-linux cracklib-runtime curl debconf-utils dialog dnsutils docker.io docker-compose ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 libpam-google-authenticator man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-elasticsearch-curator python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant"
+# Remove purge in the future
+echo "### Removing repository based install of elasticsearch-curator"
+apt-get purge elasticsearch-curator -y
 hash -r
 echo "### Now upgrading packages ..."
 dpkg --configure -a
@@ -201,10 +202,12 @@ apt-fast -y dist-upgrade -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::
 dpkg --configure -a
 npm install elasticdump -g
 pip3 install --upgrade yq
+# Remove --force switch in the future ...
+pip3 install elasticsearch-curator --force -y
 hash -r
 echo "### Removing and holding back problematic packages ..."
-apt-fast -y purge exim4-base mailutils pcp cockpit-pcp
-apt-mark hold exim4-base mailutils pcp cockpit-pcp
+apt-fast -y purge exim4-base mailutils pcp cockpit-pcp elasticsearch-curator
+apt-mark hold exim4-base mailutils pcp cockpit-pcp elasticsearch-curator
 echo
 
 echo "### Now replacing T-Pot related config files on host"
@@ -219,29 +222,30 @@ echo "Port 64295" >> /etc/ssh/sshd_config
 echo
 
 ### Ensure creation of T-Pot related folders, just in case
-mkdir -p /data/adbhoney/downloads /data/adbhoney/log \
+mkdir -vp /data/adbhoney/{downloads,log} \
          /data/ciscoasa/log \
          /data/conpot/log \
 	 /data/citrixhoneypot/logs \
-         /data/cowrie/log/tty/ /data/cowrie/downloads/ /data/cowrie/keys/ /data/cowrie/misc/ \
-         /data/dionaea/log /data/dionaea/bistreams /data/dionaea/binaries /data/dionaea/rtp /data/dionaea/roots/ftp /data/dionaea/roots/tftp /data/dionaea/roots/www /data/dionaea/roots/upnp \
+         /data/cowrie/{downloads,keys,misc,log,log/tty} \
+         /data/dionaea/{log,bistreams,binaries,rtp,roots,roots/ftp,roots/tftp,roots/www,roots/upnp} \
          /data/elasticpot/log \
-         /data/elk/data /data/elk/log \
+         /data/elk/{data,log} \
 	 /data/fatt/log \
-         /data/honeytrap/log/ /data/honeytrap/attacks/ /data/honeytrap/downloads/ \
+         /data/honeytrap/{log,attacks,downloads} \
          /data/glutton/log \
          /data/heralding/log \
          /data/honeypy/log \
          /data/mailoney/log \
          /data/medpot/log \
-         /data/nginx/log /data/nginx/heimdall \
+         /data/nginx/{log,heimdall} \
          /data/emobility/log \
          /data/ews/conf \
          /data/rdpy/log \
          /data/spiderfoot \
-         /data/suricata/log /home/tsec/.ssh/ \
-         /data/tanner/log /data/tanner/files \
-         /data/p0f/log
+         /data/suricata/log \
+         /data/tanner/{log,files} \
+         /data/p0f/log \
+	 /home/tsec/.ssh/
 
 ### Let's take care of some files and permissions
 chmod 770 -R /data