Docker: fix full-image build and compose warnings\n\n- RetireJS/Node/…

Dockerfile.full

@@ -6,7 +6,7 @@
 # Written by: TheTechromancer
 #
 
-FROM python:3
+FROM python:3.12
 
 # Install tools/dependencies from apt
 RUN apt-get -y update && apt-get -y install nbtscan onesixtyone nmap
@@ -33,35 +33,41 @@ RUN groupadd spiderfoot \
     -c "SpiderFoot User" spiderfoot
 
 # Install RetireJS
-RUN apt remove -y cmdtest \
-    && apt remove -y yarn \
-    && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
-    && echo 'deb https://dl.yarnpkg.com/debian/ stable main' |tee /etc/apt/sources.list.d/yarn.list \
-    && apt-get update \
-    && apt-get install yarn -y \
-    && yarn install \
-    && curl -fsSL https://deb.nodesource.com/setup_17.x | bash - \
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y nodejs \
+    && npm install -g yarn \
     && npm install -g retire
 
 # Install Google Chrome the New Way (Not via apt-key)
 RUN wget -qO - https://dl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/googlechrome-linux-keyring.gpg \
     && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/googlechrome-linux-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main" | tee /etc/apt/sources.list.d/google-chrome.list \
     && apt -y update && apt install --allow-unauthenticated -y google-chrome-stable
 
-# Install Wappalyzer
-RUN git clone https://github.com/AliasIO/wappalyzer.git \
-    && cd wappalyzer \
-    && yarn install && yarn run link
+# Install Wappalyzer CLI via npm (avoids git clone issues)
+RUN npm install -g wappalyzer
 
 # Install Nuclei
 RUN wget https://github.com/projectdiscovery/nuclei/releases/download/v2.6.5/nuclei_2.6.5_linux_amd64.zip \
-    && unzip nuclei_2.6.5_linux_amd64.zip \
-    && git clone https://github.com/projectdiscovery/nuclei-templates.git
+    && unzip nuclei_2.6.5_linux_amd64.zip
+RUN sh -c 'set -e; \
+      for i in 1 2 3; do \
+        git -c http.lowSpeedLimit=1000 -c http.lowSpeedTime=30 clone --depth 1 https://github.com/projectdiscovery/nuclei-templates.git && exit 0 || { \
+          echo "git clone nuclei-templates failed, retry $i..."; \
+          sleep $((i*5)); \
+        }; \
+      done; \
+      echo "git clone nuclei-templates failed after retries"; exit 1'
 
 # Install testssl.sh
 RUN apt-get install -y bsdmainutils dnsutils coreutils
-RUN git clone https://github.com/drwetter/testssl.sh.git
+RUN sh -c 'set -e; \
+      for i in 1 2 3; do \
+        git -c http.lowSpeedLimit=1000 -c http.lowSpeedTime=30 clone --depth 1 https://github.com/drwetter/testssl.sh.git && exit 0 || { \
+          echo "git clone testssl.sh failed, retry $i..."; \
+          sleep $((i*5)); \
+        }; \
+      done; \
+      echo "git clone testssl.sh failed after retries"; exit 1'
 
 # Install Snallygaster and TruffleHog
 RUN pip3 install snallygaster trufflehog
@@ -128,7 +134,7 @@ db.configSet({ \
     "sfp_tool_trufflehog:trufflehog_path": "/usr/local/bin/trufflehog", \
     "sfp_tool_nuclei:nuclei_path": "/tools/nuclei", \
     "sfp_tool_nuclei:template_path": "/tools/nuclei-templates", \
-    "sfp_tool_wappalyzer:wappalyzer_path": "/tools/wappalyzer/src/drivers/npm/cli.js", \
+    "sfp_tool_wappalyzer:wappalyzer_path": "/usr/local/bin/wappalyzer", \
     "sfp_tool_nbtscan:nbtscan_path": "/usr/bin/nbtscan", \
     "sfp_tool_nmap:nmappath": "DISABLED_BECAUSE_NMAP_REQUIRES_ROOT_TO_WORK" \
 })' || true && ./sf.py -l 0.0.0.0:5001

README.md

@@ -95,22 +95,72 @@ To install and run SpiderFoot, you need at least Python 3.7 and a number of Pyth
 #### Stable build (packaged release):
 
 ```
- wget https://github.com/smicallef/spiderfoot/archive/v4.0.tar.gz
- tar zxvf v4.0.tar.gz
- cd spiderfoot-4.0
- pip3 install -r requirements.txt
- python3 ./sf.py -l 127.0.0.1:5001
+wget https://github.com/smicallef/spiderfoot/archive/v4.0.tar.gz
+tar zxvf v4.0.tar.gz
+cd spiderfoot-4.0
+pip3 install -r requirements.txt
+python3 ./sf.py -l 127.0.0.1:5001
 ```
 
 #### Development build (cloning git master branch):
 
 ```
- git clone https://github.com/smicallef/spiderfoot.git
- cd spiderfoot
- pip3 install -r requirements.txt
- python3 ./sf.py -l 127.0.0.1:5001
+git clone https://github.com/smicallef/spiderfoot.git
+cd spiderfoot
+pip3 install -r requirements.txt
+python3 ./sf.py -l 127.0.0.1:5001
 ```
 
+#### Using Docker Compose
+
+SpiderFoot provides multiple Docker Compose configurations for different use cases:
+
+**Basic Setup (recommended for most users):**
+
+```bash
+docker-compose up -d
+```
+
+Access the web interface at `http://localhost:5001`
+
+**Development Setup (with mounted source code for development):**
+
+```bash
+docker-compose -f docker-compose-dev.yml up -d
+```
+
+Access the web interface at `http://localhost:5001`
+
+**Full Setup (includes all dependencies and tools):**
+
+```bash
+docker-compose -f docker-compose-full.yml up -d
+```
+
+Access the web interface at `http://localhost:5001`
+
+**Stopping Containers:**
+
+To stop and remove containers:
+
+```bash
+docker-compose down
+# or for dev/full setups:
+docker-compose -f docker-compose-dev.yml down
+docker-compose -f docker-compose-full.yml down
+```
+
+**Persistent Data:**
+
+- All configurations and scan data are stored in the `spiderfoot-db` volume
+- To completely remove all data, add the `-v` flag when running `docker-compose down`
+
+**Environment Variables:**
+
+- `TZ`: Set the timezone (e.g., `TZ=UTC`)
+- `SF_SERVER_HOST`: Web server host (default: `0.0.0.0`)
+- `SF_SERVER_PORT`: Web server port (default: `5001`)
+
 Check out the [documentation](https://www.spiderfoot.net/documentation) and our [asciinema videos](https://asciinema.org/~spiderfoot) for more tutorials.
 
 ### COMMUNITY

docker-compose-full.yml

@@ -1,5 +1,5 @@
-version: "3"
-
+# Docker Compose v2+: explicit 'version' key is obsolete; compose schema is detected automatically
+ 
 services:
   spiderfoot:
     build:

docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3"
+# Docker Compose v2+: explicit 'version' key is obsolete; compose schema is detected automatically
 
 # Basic usage:
 #     $ docker-compose up