Merge branch 'master' into feature/related-files

Author: psrok1

deploy/docker/Dockerfile

@@ -1,21 +1,32 @@
+FROM python:3.8-alpine AS build
+
+WORKDIR /app
+RUN python3 -m venv /app/venv
+RUN /app/venv/bin/pip --no-cache-dir install wheel
+
+RUN apk add --no-cache libffi libffi-dev py3-cffi build-base python3-dev automake m4 autoconf libtool gcc g++ musl-dev openssl-dev cargo postgresql-dev
+
+COPY requirements.txt /app
+RUN /app/venv/bin/pip --no-cache-dir install -r /app/requirements.txt
+
+COPY docker/plugins /app/plugins
+ARG plugins
+RUN for plugin in $plugins $(find /app/plugins -name 'setup.py' -exec dirname {} \; | sort -u);  \
+    do /app/venv/bin/pip --no-cache-dir install $plugin; done
+
 FROM python:3.8-alpine
 
 LABEL maintainer="info@cert.pl"
 
 RUN apk add --no-cache postgresql-client postgresql-dev libmagic
 
-COPY requirements.txt docker/plugins/requirements-*.txt /tmp/
-RUN apk add --no-cache -t build libffi libffi-dev py3-cffi build-base python3-dev automake m4 autoconf libtool gcc g++ musl-dev openssl-dev cargo \
-    && pip --no-cache-dir install -r /tmp/requirements.txt \
-    && ls /tmp/requirements-*.txt | xargs -i,, pip --no-cache-dir install -r ,, \
-    && apk del build
-
 # Copy backend files
+COPY --from=build /app/venv /app/venv
 COPY docker/ setup.py MANIFEST.in requirements.txt /app/
 COPY mwdb /app/mwdb/
 
 # Install mwdb-core package
-RUN pip install /app
+RUN /app/venv/bin/pip install /app
 
 # Create a /app/uploads directory
 # Give +r to everything in /app and +x for directories

deploy/docker/Dockerfile-web

@@ -1,20 +1,25 @@
-FROM node:14 AS build
+FROM node:16-alpine AS build
+
+LABEL maintainer="info@cert.pl"
 
 COPY ./mwdb/web /app
-COPY ./docker/plugins /plugins
+COPY ./docker/plugins /app/plugins
 
+ARG web_plugins
 RUN cd /app \
-    && npm install --unsafe-perm . $(find /plugins -name 'package.json' -printf "%h\n" | sort -u) \
+    && npm install --unsafe-perm . $web_plugins $(find /app/plugins -name 'package.json' -exec dirname {} \; | sort -u) \
     && CI=true npm run build \
     && npm cache clean --force
 
 FROM nginx:stable
 
+LABEL maintainer="info@cert.pl"
+
 ENV PROXY_BACKEND_URL http://mwdb.:8080
 
 COPY docker/nginx.conf.template /etc/nginx/conf.d/default.conf.template
 COPY docker/start-web.sh /start-web.sh
-COPY --from=build /app/build /usr/share/nginx/html
+COPY --from=build /app/dist /usr/share/nginx/html
 
 # Give +r to everything in /usr/share/nginx/html and +x for directories
 RUN chmod u=rX,go= -R /usr/share/nginx/html

deploy/docker/Dockerfile-web-dev

@@ -1,14 +1,17 @@
-FROM node:14 AS build
+FROM node:16-alpine AS build
+
+LABEL maintainer="info@cert.pl"
 
 COPY ./mwdb/web /app
-COPY ./docker/plugins /plugins
+COPY ./docker/plugins /app/plugins
 
+ARG web_plugins
 RUN cd /app \
-    && npm install --unsafe-perm . $(find /plugins -name 'package.json' -printf "%h\n" | sort -u) \
+    && npm install --unsafe-perm . $web_plugins $(find /app/plugins -name 'package.json' -exec dirname {} \; | sort -u) \
     && CI=true npm run build \
     && npm cache clean --force
 
 ENV PROXY_BACKEND_URL http://mwdb.:8080
 
 WORKDIR /app
-CMD ["npm", "run", "start"]
+CMD ["npm", "run", "dev"]

docker-compose-dev-karton.yml

@@ -26,8 +26,7 @@ services:
       # NOTE: use gen_vars.sh in order to generate this file
       - mwdb-vars.env
     environment:
-      UWSGI_PY_AUTORELOAD: 1
-      UWSGI_ENABLE_THREADS: 1
+      HOT_RELOAD: 1
       MWDB_MAIL_SMTP: "mailhog:1025"
       MWDB_MAIL_FROM: "noreply@mwdb.dev"
       MWDB_RECAPTCHA_SITE_KEY: "6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI"

docker-compose-dev-remote.yml

@@ -25,8 +25,7 @@ services:
       # NOTE: use gen_vars.sh in order to generate this file
       - mwdb-vars.env
     environment:
-      UWSGI_PY_AUTORELOAD: 1
-      UWSGI_ENABLE_THREADS: 1
+      HOT_RELOAD: 1
       MWDB_MAIL_SMTP: "mailhog:1025"
       MWDB_MAIL_FROM: "noreply@mwdb.dev"
       MWDB_RECAPTCHA_SITE_KEY: "6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI"

docker-compose-dev.yml

@@ -25,8 +25,7 @@ services:
       # NOTE: use gen_vars.sh in order to generate this file
       - mwdb-vars.env
     environment:
-      UWSGI_PY_AUTORELOAD: 1
-      UWSGI_ENABLE_THREADS: 1
+      HOT_RELOAD: 1
       MWDB_MAIL_SMTP: "mailhog:1025"
       MWDB_MAIL_FROM: "noreply@mwdb.dev"
       MWDB_RECAPTCHA_SITE_KEY: "6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI"
@@ -58,6 +57,7 @@ services:
     volumes:
       - "./mwdb/web/public:/app/public"
       - "./mwdb/web/src:/app/src"
+      - "./docker/plugins:/app/plugins"
     restart: on-failure
   postgres:
     image: postgres

docker-compose-e2e.yml

@@ -23,7 +23,6 @@ services:
       MWDB_BASE_URL: http://127.0.0.1
       MWDB_ENABLE_RATE_LIMIT: 0
       MWDB_ENABLE_REGISTRATION: 1
-      UWSGI_PROCESSES: 4
       MWDB_MAIL_SMTP: "mailhog:1025"
       MWDB_MAIL_FROM: "noreply@mwdb.dev"
     volumes:

docker-compose-oidc-dev.yml

@@ -25,8 +25,7 @@ services:
       # NOTE: use gen_vars.sh in order to generate this file
       - mwdb-vars.env
     environment:
-      UWSGI_PY_AUTORELOAD: 1
-      UWSGI_ENABLE_THREADS: 1
+      HOT_RELOAD: 1
       MWDB_MAIL_SMTP: "mailhog:1025"
       MWDB_MAIL_FROM: "noreply@mwdb.dev"
       MWDB_RECAPTCHA_SITE_KEY: "6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI"

docker/gunicorn.conf.py

@@ -0,0 +1,7 @@
+import os
+
+wsgi_app = "mwdb.app:app"
+bind = "0.0.0.0:8080"
+user = "nobody"
+reload = bool(int(os.getenv("HOT_RELOAD", "0")))
+workers = int(os.getenv("GUNICORN_WORKERS", "4"))

docker/start.sh

@@ -7,4 +7,4 @@ until psql "$MWDB_POSTGRES_URI" -c "\q" ; do
 done
 
 echo "Configuring mwdb-core instance"
-mwdb-core configure --quiet basic && exec uwsgi --ini /app/uwsgi.ini
+/app/venv/bin/mwdb-core configure --quiet basic && exec /app/venv/bin/gunicorn

docker/uwsgi.ini

@@ -1,16 +1,48 @@
 Setup and configuration
 =======================
 
-Prerequisites
--------------
+Installation and configuration with Docker Compose
+--------------------------------------------------
+
+The quickest way setup MWDB is to just clone the repository and use Docker-Compose with all batteries included.
+
+.. code-block:: console
+
+    $ git clone https://github.com/CERT-Polska/mwdb-core.git
+
+After cloning repository, the first step is to go to the ``mwdb-core`` directory and generate configuration using ``./gen_vars.sh`` script.
+Generated variables can be found in mwdb-vars.env.
+
+.. code-block:: console
+
+   $ ./gen_vars.sh 
+   Credentials for initial mwdb account:
+
+   -----------------------------------------
+   Admin login: admin
+   Admin password: la/Z7MsmKA3UxW8Psrk1Opap
+   -----------------------------------------
+
+   Please be aware that initial account will be only set up on the first run. If you already have a database with at least one user, then this setting will be ignored for security reasons. You can always create an admin account manually by executing a command. See "flask create_admin --help" for reference.
+
+Then build images via ``docker-compose build`` and run MWDB via ``docker-compose up -d``.
+
+Your MWDB instance will be available on default HTTP port (80): http://127.0.0.1/
+
+If you want to use Docker Compose for MWDB development, check out :ref:`Developer guide`.
+
+Standalone installation
+-----------------------
+
+Step 1.: Prerequisites
+~~~~~~~~~~~~~~~~~~~~~~
 
 MWDB was tested on Debian-based systems, but should work as well on other Linux distributions.
 
 For production environments, you need to install:
 
 
 * **PostgreSQL database** (minimum supported version: 12, https://www.postgresql.org/download/linux/debian/)
-* **python-ssdeep library dependencies for Python 3** (https://python-ssdeep.readthedocs.io/en/latest/installation.html#id9) 
 
 Optionally you can install:
 
@@ -41,8 +73,8 @@ It's highly recommended to create a fresh `virtualenv <https://docs.python.org/3
 
    The connection string is: ``postgresql://mwdb:mwdb@127.0.0.1:54322/mwdb``
 
-Installation & Configuration
-----------------------------
+Step 2.: Installation and configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The recommended installation method is pip:
 
@@ -87,14 +119,14 @@ Then, use ``mwdb-core configure`` to provide first configuration for your MWDB s
    3) Current directory
    : 3
 
-For first installation we recommend to install everything in current folder via ``3`` option. If you want to install MWDB system-wide or locally for user: choose ``1`` or ``2``. 
+For first installation we recommend to install everything in current folder via ``3`` option. If you want to install MWDB system-wide or locally for user: choose ``1`` or ``2``.
 
 Then, input the connection string for PostgreSQL database. The database must be online and reachable at the time of configuration. After that, you will be asked for path for uploads and instance base URL. If the default value is ok, press Enter:
 
 .. code-block::
 
    PostgreSQL database connection string [postgresql://localhost/mwdb]:
-   Uploads storage path [./uploads]: 
+   Uploads storage path [./uploads]:
    Base public URL of Malwarecage service [http://127.0.0.1]:
 
 Depending on the installation type, your configuration will be stored in ``mwdb.ini`` file and can be changed any time you want:
@@ -136,42 +168,30 @@ And you are done! ``run`` command will start the Flask server:
 
 Your MWDB instance will be available on port 5000 (use ``--port`` to change that): http://127.0.0.1:5000/
 
-.. warning::
-
-   Remember to run ``mwdb-core configure`` after each version upgrade to apply database migrations
-
-
-Alternative setup with Docker Compose
---------------------------------------
+Keep in mind that Flask server is meant to be used as development server and **is not suitable for production**.
+See also: https://flask.palletsprojects.com/en/2.2.x/server/
 
-The quickest way setup MWDB is to just clone the repository and use Docker-Compose. We recommend this method **only for testing** because it can be a bit more difficult to install extensions and integrate with other services.
-
-.. code-block:: console
-
-    $ git clone https://github.com/CERT-Polska/mwdb-core.git
-
-After cloning repository, the first step is to go to the ``mwdb-core`` directory and generate configuration using ``./gen_vars.sh`` script.
+.. warning::
 
-.. code-block:: console
+   In standalone setup, remember to run ``mwdb-core configure`` after each version upgrade to apply database migrations.
 
-   $ ./gen_vars.sh 
-   Credentials for initial mwdb account:
+Step 3.: Setting up gunicorn and nginx
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-   -----------------------------------------
-   Admin login: admin
-   Admin password: la/Z7MsmKA3UxW8Psrk1Opap
-   -----------------------------------------
+It's recommended to deploy Flask applications using dedicated WSGI server. We highly recommend Gunicorn as it's used
+in our Docker images and combine it with Nginx serving as proxy server for best security and performance
 
-   Please be aware that initial account will be only set up on the first run. If you already have a database with at least one user, then this setting will be ignored for security reasons. You can always create an admin account manually by executing a command. See "flask create_admin --help" for reference.
+.. seealso::
 
-Then build images via ``docker-compose build`` and run MWDB via ``docker-compose up -d``.
+    https://flask.palletsprojects.com/en/2.2.x/deploying/gunicorn/
 
-Your MWDB instance will be available on default HTTP port (80): http://127.0.0.1/
+    https://docs.gunicorn.org/en/latest/deploy.html#deploying-gunicorn
 
-If you want to use Docker Compose for MWDB development, check out :ref:`Developer guide`.
+Proper configuration files and templates used in our Docker images can be found in `docker directory on our Github repository
+<https://github.com/CERT-Polska/mwdb-core/tree/master/docker>`_
 
-Upgrade mwdb-core to latest version
------------------------------------
+Upgrading mwdb-core to latest version
+-------------------------------------
 
 For standalone installation (pip-based), upgrade mwdb-core package to the latest version.
 

docs/setup-and-configuration.rst

@@ -173,7 +173,12 @@ Each capability has its own name and scope:
 *
   **sharing_with_all - Can share objects with all groups in system**
 
-  Implies the access to the list of all group names, but without access to the membership information and management features. Allows to share object with arbitrary group in MWDB.
+  Implies the access to the list of all group names, but without access to the membership information and management features. Allows to share object with arbitrary group in MWDB. It also allows the user to view full history of sharing an object (if the user has access to the object).
+
+*
+  **access_uploader_info - Can view who uploaded object and filter by uploader**
+
+  Can view who uploaded object and filter by uploader. Without this capability users can filter by / see only users in their workspaces.
 
 *
   **adding_tags - Can add tags**

docs/user-guide/9-Sharing-objects.rst

@@ -7,6 +7,8 @@ class Capabilities(object):
     access_all_objects = "access_all_objects"
     # Can share objects with all groups, have access to complete list of groups
     sharing_with_all = "sharing_with_all"
+    # Can view who uploaded object and filter by uploader
+    access_uploader_info = "access_uploader_info"
     # Can add tags
     adding_tags = "adding_tags"
     # Can remove tags

mwdb/core/capabilities.py

@@ -389,6 +389,17 @@ def _get_condition(
                 .join(Member.group)
                 .filter(Group.name == value)
             ).all()
+        elif g.auth_user.has_rights(Capabilities.access_uploader_info):
+            uploaders = (
+                db.session.query(User)
+                .join(User.memberships)
+                .join(Member.group)
+                .filter(Group.name == value)
+            ).all()
+            # Regular users can see only uploads to its own groups
+            condition = and_(
+                condition, g.auth_user.is_member(ObjectPermission.group_id)
+            )
         else:
             uploaders = (
                 db.session.query(User)

mwdb/core/search/fields.py

@@ -5,6 +5,7 @@
 import tempfile
 
 import pyzipper
+from Cryptodome.Util.strxor import strxor_c
 from flask import g
 from sqlalchemy import not_, or_
 from sqlalchemy.dialects.postgresql.array import ARRAY
@@ -231,6 +232,32 @@ def iterate(self, chunk_size=1024 * 256):
         finally:
             File.close(fh)
 
+    def iterate_obfuscated(self, chunk_size=1024 * 256):
+        r"""
+        Iterates over bytes in the file contents with xor applied
+        The idea behind xoring before send is to prevent browsers
+        from caching original samples (malware). Unxoring is provided
+        in mwdb\web\src\components\ShowSample.js in SamplePreview
+        """
+
+        def negate_bits(chunk):
+            return strxor_c(chunk, 255)
+
+        fh = self.open()
+        try:
+            if hasattr(fh, "stream"):
+                yield from map(negate_bits, fh.stream(chunk_size))
+            else:
+                while True:
+                    chunk = fh.read(chunk_size)
+                    chunk = negate_bits(chunk)
+                    if chunk:
+                        yield chunk
+                    else:
+                        return
+        finally:
+            File.close(fh)
+
     @staticmethod
     def close(fh):
         """