added Malcolm

2021-08-06 10:35:01 +02:00
parent f043730066
commit 70f1922e80
751 changed files with 195277 additions and 0 deletions
--- a/Vagrant/resources/malcolm/file-monitor/capa-update.sh
+++ b/Vagrant/resources/malcolm/file-monitor/capa-update.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -e
+set -u
+set -o pipefail
+
+ENCODING="utf-8"
+
+[[ "${EXTRACTED_FILE_UPDATE_RULES:-"false"}" == "true" ]] || exit 0
+
+CAPA=${CAPA_BIN:-"/usr/local/bin/capa"}
+
+BASE_DIR="$(mktemp -d -t capa-XXXXXX)"
+
+function git_latest_release () {
+  if [ "$1" ]; then
+    (set -o pipefail && curl -sL -f "https://api.github.com/repos/$1/releases/latest" | jq '.tag_name' | sed -e 's/^"//' -e 's/"$//' ) || \
+      (set -o pipefail && curl -sL -f "https://api.github.com/repos/$1/releases" | jq '.[0].tag_name' | sed -e 's/^"//' -e 's/"$//' ) || \
+      echo unknown
+  else
+    echo "unknown">&2
+  fi
+}
+
+function finish {
+  rm -rf "$BASE_DIR"
+}
+
+if type git >/dev/null 2>&1; then
+  trap finish EXIT
+  if [[ -f "$CAPA" ]]; then
+    CAPA_PREVIOUS_VERSION="$("$CAPA" --version 2>&1 | awk '{print $2}' | cut -d'-' -f1)"
+  else
+    CAPA_PREVIOUS_VERSION="none"
+  fi
+  CAPA_LATEST_VERSION="$(git_latest_release fireeye/capa)"
+  if [[ -n $CAPA_LATEST_VERSION ]] && [[ "$CAPA_PREVIOUS_VERSION" != "$CAPA_LATEST_VERSION" ]]; then
+    pushd "$BASE_DIR" >/dev/null 2>&1
+    curl -fsSL -o ./capa.zip "https://github.com/fireeye/capa/releases/download/${CAPA_LATEST_VERSION}/capa-${CAPA_LATEST_VERSION}-linux.zip"
+    unzip ./capa.zip
+    chmod 755 ./capa
+    mkdir -p "$(dirname "$CAPA")"
+    rm -f "$CAPA".old 2>/dev/null || true
+    mv -f "$CAPA" "$CAPA".old 2>/dev/null || true
+    mv -f ./capa "$CAPA"
+    popd >/dev/null 2>&1
+  fi
+else
+  exit 1
+fi
--- a/Vagrant/resources/malcolm/file-monitor/docker-entrypoint.sh
+++ b/Vagrant/resources/malcolm/file-monitor/docker-entrypoint.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+if [[ -z $EXTRACTED_FILE_ENABLE_CLAMAV ]]; then
+  EXTRACTED_FILE_ENABLE_CLAMAV=false
+fi
+
+if [[ -z $EXTRACTED_FILE_ENABLE_YARA ]]; then
+  EXTRACTED_FILE_ENABLE_YARA=false
+fi
+
+if [[ -z $EXTRACTED_FILE_ENABLE_CAPA ]]; then
+  EXTRACTED_FILE_ENABLE_CAPA=false
+fi
+
+if [[ -z $EXTRACTED_FILE_ENABLE_MALASS ]]; then
+  [[ ${#MALASS_HOST} -gt 1 ]] && EXTRACTED_FILE_ENABLE_MALASS=true || EXTRACTED_FILE_ENABLE_MALASS=false
+fi
+
+if [[ -z $EXTRACTED_FILE_ENABLE_VTOT ]]; then
+  [[ ${#VTOT_API2_KEY} -gt 1 ]] && EXTRACTED_FILE_ENABLE_VTOT=true || EXTRACTED_FILE_ENABLE_VTOT=false
+fi
+
+export EXTRACTED_FILE_ENABLE_CLAMAV
+export EXTRACTED_FILE_ENABLE_YARA
+export EXTRACTED_FILE_ENABLE_CAPA
+export EXTRACTED_FILE_ENABLE_MALASS
+export EXTRACTED_FILE_ENABLE_VTOT
+
+exec "$@"
--- a/Vagrant/resources/malcolm/file-monitor/supervisord.conf
+++ b/Vagrant/resources/malcolm/file-monitor/supervisord.conf
@@ -0,0 +1,197 @@
+; Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+[unix_http_server]
+file=/tmp/supervisor.sock   ; (the path to the socket file)
+chmod=0700
+
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+pidfile=/tmp/supervisord.pid
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///tmp/supervisor.sock
+
+[program:watcher]
+command=/usr/local/bin/zeek_carve_watcher.py
+  --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s
+  --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s
+  --start-sleep %(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s
+  --ignore-existing %(ENV_EXTRACTED_FILE_IGNORE_EXISTING)s
+  --min-bytes %(ENV_EXTRACTED_FILE_MIN_BYTES)s
+  --max-bytes %(ENV_EXTRACTED_FILE_MAX_BYTES)s
+  --directory "%(ENV_ZEEK_EXTRACTOR_PATH)s"
+autostart=true
+startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[group:scanners]
+programs=virustotal,clamav,yara,capa,malass
+
+[program:virustotal]
+command=/usr/local/bin/vtot_scan.py
+  --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s
+  --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s
+  --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s
+  --vtot-api %(ENV_VTOT_API2_KEY)s
+  --req-limit %(ENV_VTOT_REQUESTS_PER_MINUTE)s
+autostart=%(ENV_EXTRACTED_FILE_ENABLE_VTOT)s
+startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:clamav]
+command=/usr/local/bin/clam_scan.py
+  --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s
+  --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s
+  --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s
+  --clamav %(ENV_EXTRACTED_FILE_ENABLE_CLAMAV)s
+  --clamav-socket "%(ENV_CLAMD_SOCKET_FILE)s"
+  --req-limit %(ENV_CLAMD_MAX_REQUESTS)s
+autostart=%(ENV_EXTRACTED_FILE_ENABLE_CLAMAV)s
+startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:yara]
+command=/usr/local/bin/yara_scan.py
+  --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s
+  --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s
+  --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s
+  --yara %(ENV_EXTRACTED_FILE_ENABLE_YARA)s
+  --yara-custom-only %(ENV_EXTRACTED_FILE_YARA_CUSTOM_ONLY)s
+  --req-limit %(ENV_YARA_MAX_REQUESTS)s
+autostart=%(ENV_EXTRACTED_FILE_ENABLE_YARA)s
+startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:capa]
+command=/usr/local/bin/capa_scan.py
+  --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s
+  --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s
+  --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s
+  --capa %(ENV_EXTRACTED_FILE_ENABLE_CAPA)s
+  --capa-verbose %(ENV_EXTRACTED_FILE_CAPA_VERBOSE)s
+  --req-limit %(ENV_CAPA_MAX_REQUESTS)s
+autostart=%(ENV_EXTRACTED_FILE_ENABLE_CAPA)s
+startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:malass]
+command=/usr/local/bin/malass_scan.py
+  --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s
+  --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s
+  --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s
+  --malass-host "%(ENV_MALASS_HOST)s"
+  --malass-port %(ENV_MALASS_PORT)s
+  --req-limit %(ENV_MALASS_MAX_REQUESTS)s
+autostart=%(ENV_EXTRACTED_FILE_ENABLE_MALASS)s
+startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:logger]
+command=/usr/local/bin/zeek_carve_logger.py
+  --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s
+  --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s
+  --start-sleep %(ENV_EXTRACTED_FILE_LOGGER_START_SLEEP)s
+  --preserve %(ENV_EXTRACTED_FILE_PRESERVATION)s
+  --directory "%(ENV_ZEEK_EXTRACTOR_PATH)s"
+  --zeek-log "%(ENV_ZEEK_LOG_DIRECTORY)s"
+autostart=true
+startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:freshclam]
+command=/usr/bin/freshclam freshclam --config-file=/etc/clamav/freshclam.conf --daemon
+autostart=%(ENV_EXTRACTED_FILE_UPDATE_RULES)s
+autorestart=true
+startsecs=0
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:clamd]
+command=/usr/sbin/clamd -c /etc/clamav/clamd.conf
+autostart=%(ENV_EXTRACTED_FILE_ENABLE_CLAMAV)s
+autorestart=true
+startsecs=0
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:fileserve]
+command=/usr/local/bin/zeek_carved_http_server.py
+  --port %(ENV_EXTRACTED_FILE_HTTP_SERVER_PORT)s
+  --encrypt %(ENV_EXTRACTED_FILE_HTTP_SERVER_ENCRYPT)s
+  --directory /data/zeek/extract_files
+autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s
+autorestart=true
+startsecs=0
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:cron]
+autorestart=true
+command=/usr/local/bin/supercronic -json "%(ENV_SUPERCRONIC_CRONTAB)s"
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
--- a/Vagrant/resources/malcolm/file-monitor/yara-rules-update.sh
+++ b/Vagrant/resources/malcolm/file-monitor/yara-rules-update.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -e
+set -u
+set -o pipefail
+
+ENCODING="utf-8"
+
+[[ "${EXTRACTED_FILE_UPDATE_RULES:-"false"}" == "true" ]] || exit 0
+
+SRC_DIR=${YARA_RULES_SRC_DIR:-"/usr/local/src/signature-base"}
+LINK_DIR=${YARA_RULES_DIR:-"/yara-rules"}
+
+if type git >/dev/null 2>&1 && [[ -d "$SRC_DIR" ]] && [[ -d "$LINK_DIR" ]]; then
+  pushd "$SRC_DIR" >/dev/null 2>&1
+  git remote update
+  git fetch --depth 1 origin
+  git reset --hard origin
+  popd >/dev/null 2>&1
+  rm -f "$LINK_DIR"/*.yar
+  ln -v -f -s -r "$YARA_RULES_SRC_DIR"/yara/*.yar "$YARA_RULES_SRC_DIR"/vendor/yara/*.yar "$LINK_DIR"/
+else
+  exit 1
+fi