added Malcolm

Vagrant/resources/malcolm/pcap-monitor/scripts/watch-pcap-uploads-folder.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+PCAP_BASE_PATH=${PCAP_PATH:-"/pcap"}
+ZEEK_BASE_PATH=${ZEEK_PATH:-"/zeek"}
+
+PROCESS_DIR="$PCAP_BASE_PATH/processed"
+PCAP_UPLOAD_DIR="$PCAP_BASE_PATH/upload"
+ZEEK_UPLOAD_DIR="$ZEEK_BASE_PATH/upload"
+mkdir -p "$PCAP_UPLOAD_DIR"
+
+# as new pcaps are closed for writing in /pcap/upload, move them to /pcap/processed for processing
+inotifywait -m -e close_write --format '%w%f' "${PCAP_UPLOAD_DIR}" | while read NEWFILE
+do
+  FILEMAGIC=$(file -b "$NEWFILE")
+  FILEMIME=$(file -b --mime-type "$NEWFILE")
+  if [[ "$FILEMIME" == 'application/vnd.tcpdump.pcap' ]] || [[ "$FILEMIME" == 'application/x-pcapng' ]] || [[ "$FILEMAGIC" == *"pcap-ng"* ]]; then
+    # a pcap file to be processed by dropping it into $PROCESS_DIR
+    sleep 0.1 && chown ${PUID:-${DEFAULT_UID}}:${PGID:-${DEFAULT_GID}} "$NEWFILE" && (>&2 mv -v "$NEWFILE" "$PROCESS_DIR/")
+
+  elif [[ -d "$ZEEK_UPLOAD_DIR" ]] && ( echo "$FILEMIME" | grep --quiet -P "(application/gzip|application/x-gzip|application/x-7z-compressed|application/x-bzip2|application/x-cpio|application/x-lzip|application/x-lzma|application/x-rar-compressed|application/x-tar|application/x-xz|application/zip)" ); then
+    # looks like this is a compressed file, we're assuming it's a zeek log archive to be processed by filebeat
+    sleep 0.1 && chown ${PUID:-${DEFAULT_UID}}:${PGID:-${DEFAULT_GID}} "$NEWFILE" && (>&2 mv -v "$NEWFILE" "$ZEEK_UPLOAD_DIR/")
+
+  else
+    # unhandled file type uploaded, delete it
+    sleep 0.1 && (>&2 rm "$NEWFILE")
+    echo "Removed \"$NEWFILE\", unhandled file type \"$FILEMIME\""
+
+  fi
+done

Vagrant/resources/malcolm/pcap-monitor/supervisord.conf

@@ -0,0 +1,46 @@
+; Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+[unix_http_server]
+file=/tmp/supervisor.sock   ; (the path to the socket file)
+chmod=0700
+
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+pidfile=/tmp/supervisord.pid
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///tmp/supervisor.sock
+
+[program:watch-upload]
+command=/bin/bash -c "sleep 30 && /usr/local/bin/watch-pcap-uploads-folder.sh"
+startsecs=35
+startretries=1
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:pcap-publisher]
+command=python3 /usr/local/bin/pcap_watcher.py
+  --verbose "%(ENV_PCAP_PIPELINE_DEBUG)s"
+  --extra-verbose "%(ENV_PCAP_PIPELINE_DEBUG_EXTRA)s"
+  --elasticsearch "%(ENV_ELASTICSEARCH_URL)s"
+  --elasticsearch-wait
+  --moloch-node arkime
+  --ignore-existing "%(ENV_PCAP_PIPELINE_IGNORE_PREEXISTING)s"
+  --start-sleep 60
+  --directory "%(ENV_PCAP_PATH)s"/processed
+user=%(ENV_PUSER)s
+startsecs=65
+startretries=1
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true