trinitor/DetectionLab
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added Malcolm

Browse Source
This commit is contained in:
Trinitor
2021-08-06 10:35:01 +02:00
parent f043730066
commit 70f1922e80
751 changed files with 195277 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

237
Vagrant/resources/malcolm/Dockerfiles/arkime.Dockerfile Normal file
View File

@@ -0,0 +1,237 @@
FROM debian:buster-slim AS build
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
ENV DEBIAN_FRONTEND noninteractive
ENV ARKIME_VERSION "2.7.1"
ENV ARKIMEDIR "/data/moloch"
ENV ARKIME_URL "https://codeload.github.com/arkime/arkime/tar.gz/v${ARKIME_VERSION}"
ENV ARKIME_LOCALELASTICSEARCH no
ENV ARKIME_INET yes
ADD moloch/scripts/bs4_remove_div.py /data/
ADD moloch/patch/* /data/patches/
ADD README.md $ARKIMEDIR/doc/
ADD doc.css $ARKIMEDIR/doc/
ADD docs/images $ARKIMEDIR/doc/images/
RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
apt-get -q update && \
apt-get install -q -y --no-install-recommends \
binutils \
bison \
cmake \
curl \
file \
flex \
g++ \
gcc \
gettext \
git-core \
groff \
groff-base \
imagemagick \
libcap-dev \
libjson-perl \
libkrb5-dev \
libmaxminddb-dev \
libpcap0.8-dev \
libssl-dev \
libtool \
libwww-perl \
libyaml-dev \
make \
meson \
ninja-build \
pandoc \
patch \
python3-dev \
python3-pip \
python3-setuptools \
python3-wheel \
rename \
sudo \
swig \
wget \
zlib1g-dev && \
pip3 install --no-cache-dir beautifulsoup4 && \
cd $ARKIMEDIR/doc/images && \
find . -name "*.png" -exec bash -c 'convert "{}" -fuzz 2% -transparent white -background white -alpha remove -strip -interlace Plane -quality 85% "{}.jpg" && rename "s/\.png//" "{}.jpg"' \; && \
cd $ARKIMEDIR/doc && \
sed -i "s/^# Malcolm$//" README.md && \
sed -i '/./,$!d' README.md && \
sed -i "s/.png/.jpg/g" README.md && \
sed -i "s@docs/images@images@g" README.md && \
pandoc -s --self-contained --metadata title="Malcolm README" --css $ARKIMEDIR/doc/doc.css -o $ARKIMEDIR/doc/README.html $ARKIMEDIR/doc/README.md && \
cd /data && \
mkdir -p "./moloch-"$ARKIME_VERSION && \
curl -sSL "$ARKIME_URL" | tar xzvf - -C "./moloch-"$ARKIME_VERSION --strip-components 1 && \
cd "./moloch-"$ARKIME_VERSION && \
bash -c 'for i in /data/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \
find $ARKIMEDIR/doc/images/screenshots -name "*.png" -delete && \
export PATH="$ARKIMEDIR/bin:${PATH}" && \
ln -sfr $ARKIMEDIR/bin/npm /usr/local/bin/npm && \
ln -sfr $ARKIMEDIR/bin/node /usr/local/bin/node && \
ln -sfr $ARKIMEDIR/bin/npx /usr/local/bin/npx && \
python3 /data/bs4_remove_div.py -i ./viewer/vueapp/src/components/users/Users.vue -o ./viewer/vueapp/src/components/users/Users.new -c "new-user-form" && \
mv -vf ./viewer/vueapp/src/components/users/Users.new ./viewer/vueapp/src/components/users/Users.vue && \
sed -i 's/v-if.*password.*"/v-if="false"/g' ./viewer/vueapp/src/components/settings/Settings.vue && \
rm -rf ./viewer/vueapp/src/components/upload && \
sed -i "s/^\(ARKIME_LOCALELASTICSEARCH=\).*/\1"$ARKIME_LOCALELASTICSEARCH"/" ./release/Configure && \
sed -i "s/^\(ARKIME_INET=\).*/\1"$ARKIME_INET"/" ./release/Configure && \
./easybutton-build.sh --install && \
npm cache clean --force && \
bash -c "file ${ARKIMEDIR}/bin/* ${ARKIMEDIR}/node-v*/bin/* | grep 'ELF 64-bit' | sed 's/:.*//' | xargs -l -r strip -v --strip-unneeded"
FROM debian:buster-slim
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/arkime'
LABEL org.opencontainers.image.description='Malcolm container providing Arkime'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "arkime"
ENV PGROUP "arkime"
ENV PUSER_PRIV_DROP true
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
ARG ES_HOST=elasticsearch
ARG ES_PORT=9200
ARG MALCOLM_USERNAME=admin
ARG ARKIME_INTERFACE=eth0
ARG ARKIME_ANALYZE_PCAP_THREADS=1
ARG WISE=off
ARG VIEWER=on
#Whether or not Arkime is in charge of deleting old PCAP files to reclaim space
ARG MANAGE_PCAP_FILES=false
#Whether or not to auto-tag logs based on filename
ARG AUTO_TAG=true
ARG PCAP_PIPELINE_DEBUG=false
ARG PCAP_PIPELINE_DEBUG_EXTRA=false
ARG PCAP_MONITOR_HOST=pcap-monitor
ARG MAXMIND_GEOIP_DB_LICENSE_KEY=""
# Declare envs vars for each arg
ENV ES_HOST $ES_HOST
ENV ES_PORT $ES_PORT
ENV ARKIME_ELASTICSEARCH "http://"$ES_HOST":"$ES_PORT
ENV ARKIME_INTERFACE $ARKIME_INTERFACE
ENV MALCOLM_USERNAME $MALCOLM_USERNAME
# this needs to be present, but is unused as nginx is going to handle auth for us
ENV ARKIME_PASSWORD "ignored"
ENV ARKIMEDIR "/data/moloch"
ENV ARKIME_ANALYZE_PCAP_THREADS $ARKIME_ANALYZE_PCAP_THREADS
ENV WISE $WISE
ENV VIEWER $VIEWER
ENV MANAGE_PCAP_FILES $MANAGE_PCAP_FILES
ENV AUTO_TAG $AUTO_TAG
ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG
ENV PCAP_PIPELINE_DEBUG_EXTRA $PCAP_PIPELINE_DEBUG_EXTRA
ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST
COPY --from=build $ARKIMEDIR $ARKIMEDIR
RUN sed -i "s/buster main/buster main contrib non-free/" /etc/apt/sources.list && \
apt-get -q update && \
apt-get install -q -y --no-install-recommends \
curl \
file \
geoip-bin \
gettext \
libcap2-bin \
libjson-perl \
libkrb5-3 \
libmaxminddb0 \
libpcap0.8 \
libssl1.0 \
libtool \
libwww-perl \
libyaml-0-2 \
libzmq5 \
procps \
psmisc \
python \
python3 \
python3-pip \
python3-setuptools \
python3-wheel \
rename \
sudo \
supervisor \
vim-tiny \
wget \
tar gzip unzip cpio bzip2 lzma xz-utils p7zip-full unrar zlib1g && \
pip3 install --no-cache-dir beautifulsoup4 pyzmq && \
ln -sfr $ARKIMEDIR/bin/npm /usr/local/bin/npm && \
ln -sfr $ARKIMEDIR/bin/node /usr/local/bin/node && \
ln -sfr $ARKIMEDIR/bin/npx /usr/local/bin/npx && \
apt-get -q -y --purge remove gcc gcc-8 cpp cpp-8 libssl-dev && \
apt-get -q -y autoremove && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# add configuration and scripts
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD moloch/scripts /data/
ADD shared/bin/pcap_moloch_and_zeek_processor.py /data/
ADD shared/bin/pcap_utils.py /data/
ADD shared/bin/elastic_search_status.sh /data/
ADD moloch/etc $ARKIMEDIR/etc/
ADD moloch/wise/source.*.js $ARKIMEDIR/wiseService/
ADD moloch/supervisord.conf /etc/supervisord.conf
# MaxMind now requires a (free) license key to download the free versions of
# their GeoIP databases. This should be provided as a build argument.
# see https://dev.maxmind.com/geoip/geoipupdate/#Direct_Downloads
# see https://github.com/arkime/arkime/issues/1350
# see https://github.com/arkime/arkime/issues/1352
RUN [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do \
cd /tmp && \
curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" && \
tar xf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 && \
mkdir -p $ARKIMEDIR/etc/ $ARKIMEDIR/logs/ && \
mv -v "GeoLite2-$DB.mmdb" $ARKIMEDIR/etc/; \
rm -f "GeoLite2-$DB*"; \
done; \
curl -s -S -L -o $ARKIMEDIR/etc/ipv4-address-space.csv "https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv" && \
curl -s -S -L -o $ARKIMEDIR/etc/oui.txt "https://raw.githubusercontent.com/wireshark/wireshark/master/manuf"
RUN groupadd --gid $DEFAULT_GID $PGROUP && \
useradd -M --uid $DEFAULT_UID --gid $DEFAULT_GID --home $ARKIMEDIR $PUSER && \
usermod -a -G tty $PUSER && \
chmod 755 /data/*.sh && \
ln -sfr /data/pcap_moloch_and_zeek_processor.py /data/pcap_moloch_processor.py && \
cp -f /data/moloch_update_geo.sh $ARKIMEDIR/bin/moloch_update_geo.sh && \
chmod u+s $ARKIMEDIR/bin/moloch-capture && \
mkdir -p /var/run/moloch && \
chown -R $PUSER:$PGROUP $ARKIMEDIR/etc $ARKIMEDIR/logs /var/run/moloch
#Update Path
ENV PATH="/data:$ARKIMEDIR/bin:${PATH}"
EXPOSE 8000 8005 8081
WORKDIR $ARKIMEDIR
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

57
Vagrant/resources/malcolm/Dockerfiles/elasticsearch.Dockerfile Normal file
View File

@@ -0,0 +1,57 @@
FROM amazon/opendistro-for-elasticsearch:1.13.2
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/elasticsearch-od'
LABEL org.opencontainers.image.description='Malcolm container providing Elasticsearch (the Apache-licensed Open Distro variant)'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUID $DEFAULT_UID
ENV PUSER "elasticsearch"
ENV PGROUP "elasticsearch"
ENV PUSER_PRIV_DROP true
ENV TERM xterm
ARG GITHUB_OAUTH_TOKEN=""
ARG DISABLE_INSTALL_DEMO_CONFIG=true
ENV DISABLE_INSTALL_DEMO_CONFIG $DISABLE_INSTALL_DEMO_CONFIG
ENV JAVA_HOME=/usr/share/elasticsearch/jdk
# Malcolm manages authentication and encryption via NGINX reverse proxy
# https://opendistro.github.io/for-elasticsearch-docs/docs/security/configuration/disable/
# https://opendistro.github.io/for-elasticsearch-docs/docs/install/docker/#customize-the-docker-image
# https://github.com/opendistro-for-elasticsearch/opendistro-build/issues/613
RUN yum install -y openssl && \
/usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security && \
echo -e 'cluster.name: "docker-cluster"\nnetwork.host: 0.0.0.0' > /usr/share/elasticsearch/config/elasticsearch.yml && \
chown -R $PUSER:$PGROUP /usr/share/elasticsearch/config/elasticsearch.yml && \
sed -i "s/user=1000\b/user=%(ENV_PUID)s/g" /usr/share/elasticsearch/plugins/opendistro-performance-analyzer/pa_config/supervisord.conf && \
sed -i "s/user=1000\b/user=%(ENV_PUID)s/g" /usr/share/elasticsearch/performance-analyzer-rca/pa_config/supervisord.conf && \
sed -i '/[^#].*\/usr\/share\/elasticsearch\/bin\/elasticsearch.*/i /usr/local/bin/jdk-cacerts-auto-import.sh || true' /usr/local/bin/docker-entrypoint.sh
# just used for initial keystore creation
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD shared/bin/jdk-cacerts-auto-import.sh /usr/local/bin/
USER root
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/local/bin/docker-entrypoint.sh"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

248
Vagrant/resources/malcolm/Dockerfiles/file-monitor.Dockerfile Normal file
View File

@@ -0,0 +1,248 @@
FROM debian:buster-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/file-monitor'
LABEL org.opencontainers.image.description='Malcolm container for scanning files extracted by Zeek'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "monitor"
ENV PGROUP "monitor"
ENV PUSER_PRIV_DROP true
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
ARG ZEEK_EXTRACTOR_PATH=/data/zeek/extract_files
ARG ZEEK_LOG_DIRECTORY=/data/zeek/logs
ARG EXTRACTED_FILE_IGNORE_EXISTING=false
ARG EXTRACTED_FILE_PRESERVATION=quarantined
ARG EXTRACTED_FILE_WATCHER_START_SLEEP=30
ARG EXTRACTED_FILE_SCANNER_START_SLEEP=10
ARG EXTRACTED_FILE_LOGGER_START_SLEEP=5
ARG EXTRACTED_FILE_MIN_BYTES=64
ARG EXTRACTED_FILE_MAX_BYTES=134217728
ARG VTOT_API2_KEY=0
ARG VTOT_REQUESTS_PER_MINUTE=4
ARG MALASS_HOST=0
ARG MALASS_PORT=80
ARG MALASS_MAX_REQUESTS=20
ARG EXTRACTED_FILE_ENABLE_CLAMAV=false
ARG EXTRACTED_FILE_UPDATE_RULES=false
ARG EXTRACTED_FILE_PIPELINE_DEBUG=false
ARG EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA=false
ARG CLAMD_SOCKET_FILE=/tmp/clamd.ctl
ARG CLAMD_MAX_REQUESTS=8
ARG YARA_MAX_REQUESTS=8
ARG CAPA_MAX_REQUESTS=4
ARG EXTRACTED_FILE_ENABLE_YARA=false
ARG EXTRACTED_FILE_YARA_CUSTOM_ONLY=false
ARG EXTRACTED_FILE_ENABLE_CAPA=false
ARG EXTRACTED_FILE_CAPA_VERBOSE=false
ARG EXTRACTED_FILE_HTTP_SERVER_DEBUG=false
ARG EXTRACTED_FILE_HTTP_SERVER_ENABLE=false
ARG EXTRACTED_FILE_HTTP_SERVER_ENCRYPT=false
ARG EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined
ARG EXTRACTED_FILE_HTTP_SERVER_PORT=8440
ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
ENV ZEEK_LOG_DIRECTORY $ZEEK_LOG_DIRECTORY
ENV EXTRACTED_FILE_IGNORE_EXISTING $EXTRACTED_FILE_IGNORE_EXISTING
ENV EXTRACTED_FILE_PRESERVATION $EXTRACTED_FILE_PRESERVATION
ENV EXTRACTED_FILE_WATCHER_START_SLEEP $EXTRACTED_FILE_WATCHER_START_SLEEP
ENV EXTRACTED_FILE_SCANNER_START_SLEEP $EXTRACTED_FILE_SCANNER_START_SLEEP
ENV EXTRACTED_FILE_LOGGER_START_SLEEP $EXTRACTED_FILE_LOGGER_START_SLEEP
ENV EXTRACTED_FILE_MIN_BYTES $EXTRACTED_FILE_MIN_BYTES
ENV EXTRACTED_FILE_MAX_BYTES $EXTRACTED_FILE_MAX_BYTES
ENV VTOT_API2_KEY $VTOT_API2_KEY
ENV VTOT_REQUESTS_PER_MINUTE $VTOT_REQUESTS_PER_MINUTE
ENV MALASS_HOST $MALASS_HOST
ENV MALASS_PORT $MALASS_PORT
ENV MALASS_MAX_REQUESTS $MALASS_MAX_REQUESTS
ENV EXTRACTED_FILE_ENABLE_CLAMAV $EXTRACTED_FILE_ENABLE_CLAMAV
ENV EXTRACTED_FILE_UPDATE_RULES $EXTRACTED_FILE_UPDATE_RULES
ENV EXTRACTED_FILE_PIPELINE_DEBUG $EXTRACTED_FILE_PIPELINE_DEBUG
ENV EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA $EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA
ENV CLAMD_SOCKET_FILE $CLAMD_SOCKET_FILE
ENV CLAMD_MAX_REQUESTS $CLAMD_MAX_REQUESTS
ENV YARA_MAX_REQUESTS $YARA_MAX_REQUESTS
ENV CAPA_MAX_REQUESTS $CAPA_MAX_REQUESTS
ENV EXTRACTED_FILE_ENABLE_YARA $EXTRACTED_FILE_ENABLE_YARA
ENV EXTRACTED_FILE_YARA_CUSTOM_ONLY $EXTRACTED_FILE_YARA_CUSTOM_ONLY
ENV EXTRACTED_FILE_ENABLE_CAPA $EXTRACTED_FILE_ENABLE_CAPA
ENV EXTRACTED_FILE_CAPA_VERBOSE $EXTRACTED_FILE_CAPA_VERBOSE
ENV SRC_BASE_DIR "/usr/local/src"
ENV CLAMAV_RULES_DIR "/var/lib/clamav"
ENV YARA_VERSION "4.1.1"
ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
ENV YARA_RULES_URL "https://github.com/Neo23x0/signature-base"
ENV YARA_RULES_DIR "/yara-rules"
ENV YARA_RULES_SRC_DIR "$SRC_BASE_DIR/signature-base"
ENV CAPA_VERSION "1.6.3"
ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip"
ENV CAPA_DIR "/opt/capa"
ENV CAPA_BIN "${CAPA_DIR}/capa"
ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG
ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE
ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT
ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY
ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT
ENV SUPERCRONIC_VERSION "0.1.12"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
ENV SUPERCRONIC "supercronic-linux-amd64"
ENV SUPERCRONIC_SHA1SUM "048b95b48b708983effb2e5c935a1ef8483d9e3e"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
apt-get update && \
apt-get install --no-install-recommends -y -q \
automake \
bc \
clamav \
clamav-daemon \
clamav-freshclam \
curl \
gcc \
git \
jq \
libclamunrar9 \
libjansson-dev \
libjansson4 \
libmagic-dev \
libmagic1 \
libssl-dev \
libssl1.1 \
libtool \
make \
pkg-config \
unzip && \
apt-get -y -q install \
inotify-tools \
libzmq5 \
psmisc \
python3 \
python3-bs4 \
python3-dev \
python3-pip \
python3-pyinotify \
python3-requests \
python3-zmq && \
pip3 install clamd supervisor yara-python python-magic psutil pycryptodome && \
curl -fsSLO "$SUPERCRONIC_URL" && \
echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \
chmod +x "$SUPERCRONIC" && \
mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
mkdir -p "${SRC_BASE_DIR}" && \
cd "${SRC_BASE_DIR}" && \
curl -sSL "${YARA_URL}" | tar xzf - -C "${SRC_BASE_DIR}" && \
cd "./yara-${YARA_VERSION}" && \
./bootstrap.sh && \
./configure --prefix=/usr \
--with-crypto \
--enable-magic \
--enable-cuckoo \
--enable-dotnet && \
make && \
make install && \
rm -rf "${SRC_BASE_DIR}"/yara* && \
cd /tmp && \
git clone --depth 1 --single-branch "${YARA_RULES_URL}" "${YARA_RULES_SRC_DIR}" && \
mkdir -p "${YARA_RULES_DIR}" && \
ln -f -s -r "${YARA_RULES_SRC_DIR}"/yara/* "${YARA_RULES_SRC_DIR}"/vendor/yara/* "${YARA_RULES_DIR}"/ && \
cd /tmp && \
curl -fsSL -o ./capa.zip "${CAPA_URL}" && \
unzip ./capa.zip && \
chmod 755 ./capa && \
mkdir -p "${CAPA_DIR}" && \
mv ./capa "${CAPA_BIN}" && \
rm -f ./capa.zip && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove \
automake \
build-essential \
gcc \
gcc-8 \
libc6-dev \
libgcc-8-dev \
libjansson-dev \
libmagic-dev \
libssl-dev \
libtool \
make \
python3-dev && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* && \
mkdir -p /var/log/clamav "${CLAMAV_RULES_DIR}" && \
curl -s -S -L -o "${CLAMAV_RULES_DIR}"/main.cvd http://database.clamav.net/main.cvd && \
curl -s -S -L -o "${CLAMAV_RULES_DIR}"/daily.cvd http://database.clamav.net/daily.cvd && \
curl -s -S -L -o "${CLAMAV_RULES_DIR}"/bytecode.cvd http://database.clamav.net/bytecode.cvd && \
groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
useradd -m --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
usermod -a -G tty ${PUSER} && \
chown -R ${PUSER}:${PGROUP} /var/log/clamav "${CLAMAV_RULES_DIR}" "${CAPA_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" && \
find /var/log/clamav "${CLAMAV_RULES_DIR}" "${CAPA_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" -type d -exec chmod 750 "{}" \; && \
sed -i 's/^Foreground .*$/Foreground true/g' /etc/clamav/clamd.conf && \
sed -i "s/^User .*$/User ${PUSER}/g" /etc/clamav/clamd.conf && \
sed -i "s|^LocalSocket .*$|LocalSocket $CLAMD_SOCKET_FILE|g" /etc/clamav/clamd.conf && \
sed -i "s/^LocalSocketGroup .*$/LocalSocketGroup ${PGROUP}/g" /etc/clamav/clamd.conf && \
sed -i "s/^MaxFileSize .*$/MaxFileSize $EXTRACTED_FILE_MAX_BYTES/g" /etc/clamav/clamd.conf && \
sed -i "s/^MaxScanSize .*$/MaxScanSize $(echo "$EXTRACTED_FILE_MAX_BYTES * 4" | bc)/g" /etc/clamav/clamd.conf && \
echo "TCPSocket 3310" >> /etc/clamav/clamd.conf && \
if ! [ -z $HTTPProxyServer ]; then echo "HTTPProxyServer $HTTPProxyServer" >> /etc/clamav/freshclam.conf; fi && \
if ! [ -z $HTTPProxyPort ]; then echo "HTTPProxyPort $HTTPProxyPort" >> /etc/clamav/freshclam.conf; fi && \
sed -i 's/^Foreground .*$/Foreground true/g' /etc/clamav/freshclam.conf && \
sed -i "s/^DatabaseOwner .*$/DatabaseOwner ${PUSER}/g" /etc/clamav/freshclam.conf && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/vtot_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/clam_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/yara_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/capa_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/malass_scan.py && \
echo "0 */6 * * * /bin/bash /usr/local/bin/capa-update.sh\n0 */6 * * * /bin/bash /usr/local/bin/yara-rules-update.sh" > ${SUPERCRONIC_CRONTAB}
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD shared/bin/zeek_carve*.py /usr/local/bin/
ADD shared/bin/malass_client.py /usr/local/bin/
ADD file-monitor/supervisord.conf /etc/supervisord.conf
ADD file-monitor/docker-entrypoint.sh /docker-entrypoint.sh
ADD file-monitor/*update.sh /usr/local/bin/
USER ${PUSER}
RUN /usr/bin/freshclam freshclam --config-file=/etc/clamav/freshclam.conf
USER root
WORKDIR /data/zeek/extract_files
ENV PATH "${CAPA_DIR}:${PATH}"
VOLUME ["$CAPA_DIR"]
VOLUME ["$CLAMAV_RULES_DIR"]
VOLUME ["$YARA_RULES_DIR"]
VOLUME ["$YARA_RULES_SRC_DIR"]
EXPOSE 3310
EXPOSE $EXTRACTED_FILE_HTTP_SERVER_PORT
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/docker-entrypoint.sh"]
CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

109
Vagrant/resources/malcolm/Dockerfiles/file-upload.Dockerfile Normal file
View File

@@ -0,0 +1,109 @@
FROM debian:buster-slim AS build
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
ENV DEBIAN_FRONTEND noninteractive
ARG SITE_NAME="Capture File and Log Archive Upload"
ENV SITE_NAME $SITE_NAME
ENV JQUERY_FILE_UPLOAD_VERSION v9.19.1
ENV JQUERY_FILE_UPLOAD_URL "https://github.com/blueimp/jQuery-File-Upload/archive/${JQUERY_FILE_UPLOAD_VERSION}.tar.gz"
RUN apt-get update && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages install --no-install-recommends npm node-encoding git ca-certificates curl wget && \
npm install -g bower && \
mkdir -p /jQuery-File-Upload && \
curl -sSL "$JQUERY_FILE_UPLOAD_URL" | tar xzvf - -C /jQuery-File-Upload --strip-components 1 && \
cd /jQuery-File-Upload && \
bower --allow-root install bootstrap && \
bower --allow-root install jquery && \
bower --allow-root install blueimp-gallery && \
bower --allow-root install bootstrap-tagsinput && \
rm -rf /jQuery-File-Upload/*.html /jQuery-File-Upload/test/ /jQuery-File-Upload/server/gae-go/ \
/jQuery-File-Upload/server/gae-python/
FROM debian:buster-slim AS runtime
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/file-upload'
LABEL org.opencontainers.image.description='Malcolm container providing an interface for uploading PCAP files and Zeek logs for processing'
ARG DEFAULT_UID=33
ARG DEFAULT_GID=33
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "www-data"
ENV PGROUP "www-data"
# not dropping privileges globally in this container as required to run SFTP server. this can
# be handled by supervisord instead on an as-needed basis, and/or php-fpm/nginx itself
# will drop privileges to www-data as well.
ENV PUSER_PRIV_DROP false
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
COPY --from=build /jQuery-File-Upload/ /var/www/upload/
RUN apt-get update && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages install --no-install-recommends \
wget \
ca-certificates \
openssh-server \
supervisor \
vim-tiny \
less \
php7.3-gd \
php7.3-fpm \
php7.3-apcu \
nginx-light && \
apt-get clean -y -q && \
rm -rf /var/lib/apt/lists/*
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD docs/images/logo/Malcolm_banner.png /var/www/upload/Malcolm_banner.png
ADD file-upload/docker-entrypoint.sh /docker-entrypoint.sh
ADD file-upload/jquery-file-upload/bootstrap.min.css /var/www/upload/bower_components/bootstrap/dist/css/bootstrap.min.css
ADD file-upload/jquery-file-upload/index.html /var/www/upload/index.html
ADD file-upload/jquery-file-upload/index.php /var/www/upload/server/php/index.php
ADD file-upload/nginx/sites-available/default /etc/nginx/sites-available/default
ADD file-upload/php/php.ini /etc/php/7.3/fpm/php.ini
ADD file-upload/sshd_config /tmp/sshd_config
ADD file-upload/supervisord.conf /supervisord.conf
RUN mkdir -p /var/run/sshd /var/www/upload/server/php/chroot /run/php && \
mv /var/www/upload/server/php/files /var/www/upload/server/php/chroot && \
ln -s /var/www/upload/server/php/chroot/files /var/www/upload/server/php/files && \
ln -sr /var/www/upload /var/www/upload/upload && \
perl -i -pl -e 's/^#?(\s*PermitRootLogin\s+)[\w\-]+$/$1no/i;' \
-e 's/^#?(\s*PasswordAuthentication\s+)\w+$/$1no/i' /etc/ssh/sshd_config && \
chmod a+x /docker-entrypoint.sh && \
cat /tmp/sshd_config >>/etc/ssh/sshd_config && \
chmod 775 /var/www/upload/server/php/chroot/files && \
chmod 755 /var /var/www /var/www/upload /var/www/upload/server /var/www/upload/server/php \
/var/www/upload/server/php/chroot && \
echo "Put your files into /files. Don't use subdirectories." \
>/var/www/upload/server/php/chroot/README.txt && \
rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/* /var/tmp/* /var/www/upload/server/php/chroot/files/.gitignore /tmp/sshd_config
VOLUME [ "/var/www/upload/server/php/chroot/files" ]
EXPOSE 22 80
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/docker-entrypoint.sh"]
CMD ["/usr/bin/supervisord", "-c", "/supervisord.conf", "-u", "root", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

108
Vagrant/resources/malcolm/Dockerfiles/filebeat.Dockerfile Normal file
View File

@@ -0,0 +1,108 @@
FROM docker.elastic.co/beats/filebeat-oss:7.10.2
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/filebeat-oss'
LABEL org.opencontainers.image.description='Malcolm container providing Filebeat (the Apache-licensed variant)'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "filebeat"
ENV PGROUP "filebeat"
# not dropping privileges globally: supervisord will take care of it
# on a case-by-case basis so that one script (filebeat-watch-zeeklogs-uploads-folder.sh)
# can chown uploaded files
ENV PUSER_PRIV_DROP false
ENV TERM xterm
ARG FILEBEAT_LOG_CLEANUP_MINUTES=0
ARG FILEBEAT_ZIP_CLEANUP_MINUTES=0
ARG FILEBEAT_SCAN_FREQUENCY=10s
ARG FILEBEAT_CLEAN_INACTIVE=45m
ARG FILEBEAT_IGNORE_OLDER=30m
ARG FILEBEAT_CLOSE_INACTIVE=30s
ARG FILEBEAT_CLOSE_RENAMED=true
ARG FILEBEAT_CLOSE_REMOVED=true
ARG FILEBEAT_CLOSE_EOF=true
ARG FILEBEAT_CLEAN_REMOVED=true
ARG FILEBEAT_LOG_PATH="/data/zeek/current"
ARG FILEBEAT_NGINX_LOG_PATH="/data/nginx"
ARG NGINX_LOG_ACCESS_AND_ERRORS=false
ARG AUTO_TAG=true
ENV SUPERCRONIC_VERSION "0.1.12"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
ENV SUPERCRONIC "supercronic-linux-amd64"
ENV SUPERCRONIC_SHA1SUM "048b95b48b708983effb2e5c935a1ef8483d9e3e"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
USER root
RUN yum install -y epel-release && \
yum update -y && \
yum install -y curl inotify-tools file psmisc tar gzip unzip cpio bzip2 lzma xz p7zip p7zip-plugins unar python3-setuptools python3-pip && \
yum clean all && \
ln -sr /usr/sbin/fuser /bin/fuser && \
python3 -m pip install patool entrypoint2 pyunpack python-magic ordered-set supervisor && \
curl -fsSLO "$SUPERCRONIC_URL" && \
echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \
chmod +x "$SUPERCRONIC" && \
mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD filebeat/filebeat.yml /usr/share/filebeat/filebeat.yml
ADD filebeat/filebeat-nginx.yml /usr/share/filebeat-nginx/filebeat-nginx.yml
ADD filebeat/scripts /data/
ADD shared/bin/elastic_search_status.sh /data/
ADD filebeat/supervisord.conf /etc/supervisord.conf
RUN mkdir -p /usr/share/filebeat-nginx/data && \
chown -R root:${PGROUP} /usr/share/filebeat-nginx && \
cp -a /usr/share/filebeat/module /usr/share/filebeat-nginx/module && \
chmod 750 /usr/share/filebeat-nginx && \
chmod 770 /usr/share/filebeat-nginx/data && \
chmod 755 /data/*.sh /data/*.py && \
(echo -e "* * * * * /data/filebeat-process-zeek-folder.sh\n*/5 * * * * /data/filebeat-clean-zeeklogs-processed-folder.py" > ${SUPERCRONIC_CRONTAB})
ENV FILEBEAT_LOG_CLEANUP_MINUTES $FILEBEAT_LOG_CLEANUP_MINUTES
ENV FILEBEAT_ZIP_CLEANUP_MINUTES $FILEBEAT_ZIP_CLEANUP_MINUTES
ENV FILEBEAT_SCAN_FREQUENCY $FILEBEAT_SCAN_FREQUENCY
ENV FILEBEAT_CLEAN_INACTIVE $FILEBEAT_CLEAN_INACTIVE
ENV FILEBEAT_IGNORE_OLDER $FILEBEAT_IGNORE_OLDER
ENV FILEBEAT_CLOSE_INACTIVE $FILEBEAT_CLOSE_INACTIVE
ENV FILEBEAT_CLOSE_RENAMED $FILEBEAT_CLOSE_RENAMED
ENV FILEBEAT_CLOSE_REMOVED $FILEBEAT_CLOSE_REMOVED
ENV FILEBEAT_CLOSE_EOF $FILEBEAT_CLOSE_EOF
ENV FILEBEAT_CLEAN_REMOVED $FILEBEAT_CLEAN_REMOVED
ENV FILEBEAT_LOG_PATH $FILEBEAT_LOG_PATH
ENV FILEBEAT_NGINX_LOG_PATH $FILEBEAT_NGINX_LOG_PATH
ENV NGINX_LOG_ACCESS_AND_ERRORS $NGINX_LOG_ACCESS_AND_ERRORS
ENV AUTO_TAG $AUTO_TAG
ENV FILEBEAT_REGISTRY_FILE "/usr/share/filebeat/data/registry/filebeat/data.json"
ENV FILEBEAT_ZEEK_DIR "/data/zeek/"
ENV PATH="/data:${PATH}"
VOLUME ["/usr/share/filebeat/data", "/usr/share/filebeat-nginx/data"]
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf", "-u", "root", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

75
Vagrant/resources/malcolm/Dockerfiles/freq.Dockerfile Normal file
View File

@@ -0,0 +1,75 @@
FROM debian:buster-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/freq'
LABEL org.opencontainers.image.description='Malcolm container providing an interface to Mark Baggett''s freq_server.py'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "freq"
ENV PGROUP "freq"
ENV PUSER_PRIV_DROP true
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
ARG FREQ_PORT=10004
ARG FREQ_LOOKUP=true
ENV FREQ_PORT $FREQ_PORT
ENV FREQ_LOOKUP $FREQ_LOOKUP
ENV FREQ_URL "https://codeload.github.com/markbaggett/freq/tar.gz/master"
RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
apt-get update && \
apt-get -y -q install \
curl \
procps \
psmisc \
python3 \
python3-dev \
python3-pip && \
pip3 install supervisor && \
cd /opt && \
mkdir -p ./freq_server && \
curl -sSL "$FREQ_URL" | tar xzvf - -C ./freq_server --strip-components 1 && \
rm -rf /opt/freq_server/systemd /opt/freq_server/upstart /opt/freq_server/*.md /opt/freq_server/*.exe && \
mv -v "$(ls /opt/freq_server/*.freq | tail -n 1)" /opt/freq_server/freq_table.freq && \
groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \
chown -R ${PUSER}:${PGROUP} /opt/freq_server && \
usermod -a -G tty ${PUSER} && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove git python3-dev build-essential && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD freq-server/supervisord.conf /etc/supervisord.conf
WORKDIR /opt/freq_server
EXPOSE $FREQ_PORT
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

99
Vagrant/resources/malcolm/Dockerfiles/htadmin.Dockerfile Normal file
View File

@@ -0,0 +1,99 @@
FROM debian:buster-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/htadmin'
LABEL org.opencontainers.image.description='Malcolm container providing htadmin for managing login accounts in an htpasswd file'
ARG DEFAULT_UID=33
ARG DEFAULT_GID=33
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "www-data"
ENV PGROUP "www-data"
# not dropping privileges globally so nginx can bind privileged ports internally.
# nginx and php-fpm will drop privileges to "www-data" user for worker processes
ENV PUSER_PRIV_DROP false
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
ARG PHP_VERSION=7.3
ARG MCRYPT_VERSION=1.0.2
ARG BOOTSTRAP_VERSION=3.3.6
ENV PHP_VERSION $PHP_VERSION
ENV MCRYPT_VERSION $MCRYPT_VERSION
ENV BOOTSTRAP_VERSION $BOOTSTRAP_VERSION
ENV HTADMIN_URL "https://codeload.github.com/mmguero-dev/htadmin/tar.gz/master"
RUN apt-get update && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --no-install-recommends install \
bcrypt \
ca-certificates \
curl \
libmcrypt-dev \
libmcrypt4 \
make \
mcrypt \
nginx-light \
php-dev \
php-pear \
php$PHP_VERSION-apcu \
php$PHP_VERSION-cli \
php$PHP_VERSION-curl \
php$PHP_VERSION-fpm \
php$PHP_VERSION-gd \
procps \
supervisor && \
( yes '' | pecl channel-update pecl.php.net ) && \
( yes '' | pecl install mcrypt-$MCRYPT_VERSION ) && \
ln -s -r /usr/lib/php/20??????/*.so /usr/lib/php/$PHP_VERSION/ && \
mkdir -p /run/php && \
cd /tmp && \
mkdir -p ./htadmin && \
curl -sSL "$HTADMIN_URL" | tar xzvf - -C ./htadmin --strip-components 1 && \
mv /tmp/htadmin/sites/html/htadmin /var/www/htadmin && \
cd /var/www/htadmin && \
( grep -rhoPi "(src|href)=['\"]https?://.+?['\"]" ./includes/* | sed "s/^[a-zA-Z]*=['\"]*//" | sed "s/['\"]$//" | xargs -r -l curl -s -S -L -J -O ) && \
sed -i "s@http[^'\"]*/@@gI" ./includes/* && \
mkdir fonts && cd fonts && \
curl -s -S -L -J -O "https://maxcdn.bootstrapcdn.com/bootstrap/$BOOTSTRAP_VERSION/fonts/glyphicons-halflings-regular.ttf" && \
curl -s -S -L -J -O "https://maxcdn.bootstrapcdn.com/bootstrap/$BOOTSTRAP_VERSION/fonts/glyphicons-halflings-regular.woff" && \
curl -s -S -L -J -O "https://maxcdn.bootstrapcdn.com/bootstrap/$BOOTSTRAP_VERSION/fonts/glyphicons-halflings-regular.woff2" && \
chown -R ${PUSER}:${PGROUP} /var/www && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove \
make libmcrypt-dev php-pear php-dev && \
apt-get autoremove -y -q && \
apt-get clean -y -q && \
rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/* /var/tmp/* /var/www/html
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD docs/images/favicon/favicon.ico /var/www/htadmin/
ADD htadmin/supervisord.conf /supervisord.conf
ADD htadmin/htadmin.sh /usr/local/bin/
ADD htadmin/src /var/www/htadmin/
ADD htadmin/php/php.ini /etc/php/$PHP_VERSION/fpm/php.ini
ADD htadmin/nginx/sites-available/default /etc/nginx/sites-available/default
EXPOSE 80
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/bin/supervisord", "-c", "/supervisord.conf", "-u", "root", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

92
Vagrant/resources/malcolm/Dockerfiles/kibana-helper.Dockerfile Normal file
View File

@@ -0,0 +1,92 @@
FROM alpine:3.14
# Copyright (c) 2020 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/kibana-helper'
LABEL org.opencontainers.image.description='Malcolm container providing Kibana support functions'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "helper"
ENV PGROUP "helper"
ENV PUSER_PRIV_DROP true
ENV TERM xterm
ARG ARKIME_INDEX_PATTERN="sessions2-*"
ARG ARKIME_INDEX_PATTERN_ID="sessions2-*"
ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
ARG CREATE_ES_ARKIME_SESSION_INDEX="true"
ARG ELASTICSEARCH_URL="http://elasticsearch:9200"
ARG ISM_SNAPSHOT_COMPRESSED=false
ARG ISM_SNAPSHOT_REPO=logs
ARG KIBANA_OFFLINE_REGION_MAPS_PORT="28991"
ARG KIBANA_URL="http://kibana:5601/kibana"
ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
ENV ARKIME_INDEX_PATTERN_ID $ARKIME_INDEX_PATTERN_ID
ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
ENV CREATE_ES_ARKIME_SESSION_INDEX $CREATE_ES_ARKIME_SESSION_INDEX
ENV ELASTICSEARCH_URL $ELASTICSEARCH_URL
ENV ISM_SNAPSHOT_COMPRESSED $ISM_SNAPSHOT_COMPRESSED
ENV ISM_SNAPSHOT_REPO $ISM_SNAPSHOT_REPO
ENV KIBANA_OFFLINE_REGION_MAPS_PORT $KIBANA_OFFLINE_REGION_MAPS_PORT
ENV KIBANA_URL $KIBANA_URL
ENV PATH="/data:${PATH}"
ENV SUPERCRONIC_VERSION "0.1.12"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
ENV SUPERCRONIC "supercronic-linux-amd64"
ENV SUPERCRONIC_SHA1SUM "048b95b48b708983effb2e5c935a1ef8483d9e3e"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
ADD kibana/dashboards /opt/kibana/dashboards
ADD kibana/maps /opt/maps
ADD kibana/scripts /data/
ADD kibana/supervisord.conf /etc/supervisord.conf
ADD kibana/zeek_template.json /data/zeek_template.json
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD shared/bin/elastic_search_status.sh /data/
ADD shared/bin/elastic_index_size_prune.py /data/
RUN apk --no-cache add bash python3 py3-pip curl procps psmisc npm shadow jq && \
npm install -g http-server && \
pip3 install supervisor humanfriendly && \
curl -fsSLO "$SUPERCRONIC_URL" && \
echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \
chmod +x "$SUPERCRONIC" && \
mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
addgroup -g ${DEFAULT_GID} ${PGROUP} ; \
adduser -D -H -u ${DEFAULT_UID} -h /nonexistant -s /sbin/nologin -G ${PGROUP} -g ${PUSER} ${PUSER} ; \
addgroup ${PUSER} tty ; \
addgroup ${PUSER} shadow ; \
mkdir -p /data/init && \
chown -R ${PUSER}:${PGROUP} /opt/kibana/dashboards /opt/maps /data/init && \
chmod 755 /data/*.sh /data/*.py /data/init && \
chmod 400 /opt/maps/* && \
(echo -e "*/2 * * * * /data/kibana-create-moloch-sessions-index.sh\n0 10 * * * /data/kibana_index_refresh.py --template zeek_template\n*/20 * * * * /data/elastic_index_size_prune.py" > ${SUPERCRONIC_CRONTAB})
EXPOSE $KIBANA_OFFLINE_REGION_MAPS_PORT
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
VOLUME ["/data/init"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

157
Vagrant/resources/malcolm/Dockerfiles/kibana.Dockerfile Normal file
View File

@@ -0,0 +1,157 @@
FROM amazonlinux:2 AS build
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
# set up build environment for kibana plugins built from source
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "kibana"
ENV PGROUP "kibana"
ENV TERM xterm
ARG ELASTIC_VERSION="7.10.2"
ENV ELASTIC_VERSION $ELASTIC_VERSION
# base system dependencies for checking out and building elastic/kibana-based plugins
USER root
RUN amazon-linux-extras install -y epel && \
yum install -y curl patch procps psmisc tar zip unzip gcc-c++ make moreutils jq git && \
groupadd -g ${DEFAULT_GID} ${PGROUP} && \
adduser -u ${DEFAULT_UID} -d /home/kibana -s /bin/bash -G ${PGROUP} -g ${PUSER} ${PUSER} && \
mkdir -p /usr/share && \
git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch --branch "v${ELASTIC_VERSION}" https://github.com/elastic/elasticsearch /usr/share/elastic && \
git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch --branch "v${ELASTIC_VERSION}" https://github.com/elastic/kibana /usr/share/kibana && \
chown -R ${DEFAULT_UID}:${DEFAULT_GID} /usr/share/kibana /usr/share/elastic
# build plugins as non-root
USER ${PUSER}
# use nodenv (https://github.com/nodenv/nodenv) to manage nodejs/yarn
ENV PATH "/home/kibana/.nodenv/bin:${PATH}"
RUN git clone --single-branch --depth=1 --recurse-submodules --shallow-submodules https://github.com/nodenv/nodenv.git /home/kibana/.nodenv && \
cd /home/kibana/.nodenv && \
./src/configure && \
make -C src && \
cd /tmp && \
eval "$(nodenv init -)" && \
mkdir -p "$(nodenv root)"/plugins && \
git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch https://github.com/nodenv/node-build.git "$(nodenv root)"/plugins/node-build && \
git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch https://github.com/nodenv/nodenv-update.git "$(nodenv root)"/plugins/nodenv-update && \
git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch https://github.com/pine/nodenv-yarn-install.git "$(nodenv root)"/plugins/nodenv-yarn-install && \
nodenv install "$(cat /usr/share/kibana/.node-version)" && \
nodenv global "$(cat /usr/share/kibana/.node-version)"
# check out and build plugins
RUN eval "$(nodenv init -)" && \
mkdir -p /usr/share/kibana/plugins && \
git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch --branch feature/update_7.10.1 https://github.com/mmguero-dev/kbn_sankey_vis.git /usr/share/kibana/plugins/sankey_vis && \
cd /usr/share/kibana/plugins/sankey_vis && \
yarn kbn bootstrap && \
yarn install && \
yarn build --kibana-version "${ELASTIC_VERSION}" && \
mv ./build/kbnSankeyVis-7.10.2.zip ./build/kbnSankeyVis.zip
FROM amazon/opendistro-for-elasticsearch-kibana:1.13.2
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/kibana-od'
LABEL org.opencontainers.image.description='Malcolm container providing Kibana (the Apache-licensed Open Distro variant)'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "kibana"
ENV PGROUP "kibana"
ENV PUSER_PRIV_DROP true
ENV TERM xterm
ARG ELASTICSEARCH_URL="http://elasticsearch:9200"
ARG CREATE_ES_ARKIME_SESSION_INDEX="true"
ARG ARKIME_INDEX_PATTERN="sessions2-*"
ARG ARKIME_INDEX_PATTERN_ID="sessions2-*"
ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
ARG KIBANA_DEFAULT_DASHBOARD="0ad3d7c2-3441-485e-9dfe-dbb22e84e576"
ENV CREATE_ES_ARKIME_SESSION_INDEX $CREATE_ES_ARKIME_SESSION_INDEX
ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
ENV ARKIME_INDEX_PATTERN_ID $ARKIME_INDEX_PATTERN_ID
ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
ENV KIBANA_DEFAULT_DASHBOARD $KIBANA_DEFAULT_DASHBOARD
ENV KIBANA_OFFLINE_REGION_MAPS $KIBANA_OFFLINE_REGION_MAPS
ENV KIBANA_OFFLINE_REGION_MAPS_PORT $KIBANA_OFFLINE_REGION_MAPS_PORT
ENV PATH="/data:${PATH}"
ENV ELASTICSEARCH_URL $ELASTICSEARCH_URL
ENV KIBANA_DEFAULT_DASHBOARD $KIBANA_DEFAULT_DASHBOARD
USER root
# curl -sSL -o /tmp/kibana-drilldown.zip "https://codeload.github.com/mmguero-dev/kibana-plugin-drilldownmenu/zip/master" && \
# cd /tmp && \
# echo "Installing Drilldown menu plugin..." && \
# unzip /tmp/kibana-drilldown.zip && \
# mkdir ./kibana &&\
# mv ./kibana-plugin-drilldownmenu-* ./kibana/kibana-plugin-drilldownmenu && \
# cd ./kibana/kibana-plugin-drilldownmenu && \
# sed -i "s/7\.6\.2/7\.10\.0/g" ./package.json && \
# npm install && \
# cd /tmp && \
# zip -r drilldown.zip kibana --exclude ./kibana/kibana-plugin-drilldownmenu/.git\* && \
# cd /usr/share/kibana/plugins && \
# /usr/share/kibana/bin/kibana-plugin install file:///tmp/drilldown.zip --allow-root && \
# rm -rf /tmp/kibana /tmp/*drilldown* && \
# cd /tmp && \
# rm -rf /tmp/npm-*
COPY --from=build /usr/share/kibana/plugins/sankey_vis/build/kbnSankeyVis.zip /tmp/kbnSankeyVis.zip
ADD "https://github.com/dlumbrer/kbn_network/releases/download/7.10.0-1/kbn_network-7.10.0.zip" /tmp/kibana-network.zip
RUN yum install -y curl psmisc zip unzip && \
yum clean all && \
usermod -a -G tty ${PUSER} && \
# Malcolm manages authentication and encryption via NGINX reverse proxy
/usr/share/kibana/bin/kibana-plugin remove opendistroSecurityKibana --allow-root && \
cd /usr/share/kibana/plugins && \
/usr/share/kibana/bin/kibana-plugin install file:///tmp/kbnSankeyVis.zip --allow-root && \
cd /tmp && \
unzip kibana-network.zip kibana/kbn_network/kibana.json kibana/kbn_network/package.json && \
sed -i "s/7\.10\.0/7\.10\.2/g" kibana/kbn_network/kibana.json && \
sed -i "s/7\.10\.0/7\.10\.2/g" kibana/kbn_network/package.json && \
zip kibana-network.zip kibana/kbn_network/kibana.json kibana/kbn_network/package.json && \
cd /usr/share/kibana/plugins && \
/usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-network.zip --allow-root && \
rm -rf /tmp/kibana-comments.zip /tmp/kibana
ADD kibana/kibana.yml /usr/share/kibana/config/kibana.yml
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/local/bin/kibana-docker"]
EXPOSE 5601
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

138
Vagrant/resources/malcolm/Dockerfiles/logstash.Dockerfile Normal file
View File

@@ -0,0 +1,138 @@
FROM amazonlinux:2 AS build
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
RUN amazon-linux-extras install -y epel && \
yum install -y \
autoconf \
automake \
bison \
bzip2 \
curl \
gcc-c++ \
glibc-devel \
glibc-headers \
java-latest-openjdk-devel \
libffi-devel \
libtool \
libyaml-devel \
make \
openssl-devel \
patch \
procps \
readline-devel \
tar \
wget \
which \
zlib-devel
RUN /bin/bash -lc "command curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -" && \
/bin/bash -lc "command curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import -" && \
/bin/bash -lc "curl -L get.rvm.io | bash -s stable" && \
/bin/bash -lc "rvm autolibs fail" && \
/bin/bash -lc "rvm install jruby-9.2.17.0" && \
/bin/bash -lc "rvm use jruby-9.2.17.0 --default" && \
/bin/bash -lc "gem install bundler --no-document"
ENV OUIFILTER_URL "https://codeload.github.com/mmguero-dev/logstash-filter-ieee_oui/tar.gz/master"
RUN cd /opt && \
mkdir -p ./logstash-filter-ieee_oui && \
curl -sSL "$OUIFILTER_URL" | tar xzvf - -C ./logstash-filter-ieee_oui --strip-components 1 && \
/bin/bash -lc "export JAVA_HOME=$(realpath $(dirname $(find /usr/lib/jvm -name javac -type f))/../) && cd /opt/logstash-filter-ieee_oui && ( bundle install || bundle install ) && gem build logstash-filter-ieee_oui.gemspec && bundle info logstash-filter-ieee_oui"
FROM docker.elastic.co/logstash/logstash-oss:7.10.2
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/logstash-oss'
LABEL org.opencontainers.image.description='Malcolm container providing Logstash (the Apache-licensed variant)'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "logstash"
ENV PGROUP "logstash"
ENV PUSER_PRIV_DROP true
ENV TERM xterm
ARG LOGSTASH_ENRICHMENT_PIPELINE=enrichment
ARG LOGSTASH_PARSE_PIPELINE_ADDRESSES=zeek-parse
ARG LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_INTERNAL=internal-es
ARG LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_EXTERNAL=external-es
ARG LOGSTASH_ELASTICSEARCH_OUTPUT_PIPELINE_ADDRESSES=internal-es,external-es
ENV LOGSTASH_ENRICHMENT_PIPELINE $LOGSTASH_ENRICHMENT_PIPELINE
ENV LOGSTASH_PARSE_PIPELINE_ADDRESSES $LOGSTASH_PARSE_PIPELINE_ADDRESSES
ENV LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_INTERNAL $LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_INTERNAL
ENV LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_EXTERNAL $LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_EXTERNAL
ENV LOGSTASH_ELASTICSEARCH_OUTPUT_PIPELINE_ADDRESSES $LOGSTASH_ELASTICSEARCH_OUTPUT_PIPELINE_ADDRESSES
ENV JAVA_HOME=/usr/share/logstash/jdk
USER root
COPY --from=build /opt/logstash-filter-ieee_oui /opt/logstash-filter-ieee_oui
RUN yum install -y epel-release && \
yum update -y && \
yum install -y curl gettext python-setuptools python-pip python-requests python-yaml openssl && \
yum clean all && \
pip install py2-ipaddress supervisor && \
logstash-plugin install logstash-filter-translate logstash-filter-cidr logstash-filter-dns \
logstash-filter-json logstash-filter-prune logstash-filter-http \
logstash-filter-grok logstash-filter-geoip logstash-filter-uuid \
logstash-filter-kv logstash-filter-mutate logstash-filter-dissect \
logstash-input-beats logstash-output-elasticsearch && \
logstash-plugin install /opt/logstash-filter-ieee_oui/logstash-filter-ieee_oui-1.0.6.gem && \
rm -rf /opt/logstash-filter-ieee_oui /root/.cache /root/.gem /root/.bundle
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD shared/bin/jdk-cacerts-auto-import.sh /usr/local/bin/
ADD logstash/maps/*.yaml /etc/
ADD logstash/config/log4j2.properties /usr/share/logstash/config/
ADD logstash/config/logstash.yml /usr/share/logstash/config/
ADD logstash/pipelines/ /usr/share/logstash/malcolm-pipelines/
ADD logstash/scripts /usr/local/bin/
ADD logstash/supervisord.conf /etc/supervisord.conf
RUN bash -c "chmod --silent 755 /usr/local/bin/*.sh /usr/local/bin/*.py || true" && \
usermod -a -G tty ${PUSER} && \
rm -f /usr/share/logstash/pipeline/logstash.conf && \
rmdir /usr/share/logstash/pipeline && \
mkdir /logstash-persistent-queue && \
chown --silent -R ${PUSER}:root /usr/share/logstash/malcolm-pipelines /logstash-persistent-queue && \
curl -sSL -o /usr/share/logstash/config/oui.txt "https://raw.githubusercontent.com/wireshark/wireshark/master/manuf" && \
( awk -F '\t' '{gsub(":", "", $1); if (length($1) == 6) {if ($3) {print $1"\t"$3} else if ($2) {print $1"\t"$2}}}' /usr/share/logstash/config/oui.txt > /usr/share/logstash/config/oui-logstash.txt) && \
python /usr/local/bin/ja3_build_list.py -o /etc/ja3.yaml
# As the keystore is encapsulated in logstash, this isn't really necessary. It's included
# here just to suppress the prompt when creating the keystore. If you're concerned about it
# you could change or remove this from the Dockerfile, and/or override it with your
# own envrionment variable at runtime.
ENV LOGSTASH_KEYSTORE_PASS "a410a267b1404c949284dee25518a917"
VOLUME ["/logstash-persistent-queue"]
EXPOSE 5044
EXPOSE 9001
EXPOSE 9600
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

80
Vagrant/resources/malcolm/Dockerfiles/name-map-ui.Dockerfile Normal file
View File

@@ -0,0 +1,80 @@
FROM alpine:3.14
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/name-map-ui'
LABEL org.opencontainers.image.description='Malcolm container providing a user interface for mapping names to network hosts and subnets'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "nginxsrv"
ENV PGROUP "nginxsrv"
ENV PUSER_PRIV_DROP true
ENV PUSER_CHOWN "/var/www/html;/var/lib/nginx;/var/log/nginx"
ENV TERM xterm
ENV JQUERY_VERSION 1.6.4
ENV LISTJS_VERSION v1.5.0
RUN apk --no-cache add bash php7 php7-fpm php7-mysqli php7-json php7-openssl php7-curl php7-fileinfo \
php7-zlib php7-xml php7-phar php7-intl php7-dom php7-xmlreader php7-ctype php7-session \
php7-mbstring php7-gd nginx supervisor curl inotify-tools file psmisc shadow
COPY name-map-ui/config/nginx.conf /etc/nginx/nginx.conf
COPY name-map-ui/config/fpm-pool.conf /etc/php7/php-fpm.d/www.conf
COPY name-map-ui/config/php.ini /etc/php7/conf.d/custom.ini
COPY name-map-ui/config/supervisord.conf /etc/supervisord.conf
COPY name-map-ui/config/supervisor_logstash_ctl.conf /etc/supervisor/logstash/supervisord.conf
COPY name-map-ui/scripts/*.sh /usr/local/bin/
RUN curl -sSL -o /tmp/jquery.min.js "https://code.jquery.com/jquery-${JQUERY_VERSION}.min.js" && \
curl -sSL -o /tmp/list.min.js "https://raw.githubusercontent.com/javve/list.js/${LISTJS_VERSION}/dist/list.min.js" && \
rm -rf /etc/nginx/conf.d/default.conf /var/www/html/* && \
mkdir -p /var/www/html/upload /var/www/html/maps && \
cd /var/www/html && \
mv /tmp/jquery.min.js /tmp/list.min.js ./ && \
chmod 644 ./jquery.min.js ./list.min.js && \
ln -s . name-map-ui && \
addgroup -g ${DEFAULT_GID} ${PGROUP} ; \
adduser -D -H -u ${DEFAULT_UID} -h /var/www/html -s /sbin/nologin -G ${PGROUP} -g ${PUSER} ${PUSER} ; \
addgroup ${PUSER} nginx ; \
addgroup ${PUSER} shadow ; \
addgroup ${PUSER} tty ; \
addgroup nginx tty ; \
chown -R ${PUSER}:${PGROUP} /var/www/html && \
chown -R ${PUSER}:${PGROUP} /var/lib/nginx && \
chown -R ${PUSER}:${PGROUP} /var/log/nginx && \
chmod 755 /usr/local/bin/*.sh
VOLUME /var/www/html
WORKDIR /var/www/html
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
COPY name-map-ui/site/ /var/www/html/
COPY docs/images/logo/Malcolm_banner.png /var/www/html/
COPY docs/images/favicon/favicon.ico /var/www/html/
EXPOSE 8080
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

252
Vagrant/resources/malcolm/Dockerfiles/nginx.Dockerfile Normal file
View File

@@ -0,0 +1,252 @@
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
####################################################################################
# thanks to: nginx - https://github.com/nginxinc/docker-nginx/blob/master/mainline/alpine/Dockerfile
# kvspb/nginx-auth-ldap - https://github.com/kvspb/nginx-auth-ldap
# tiredofit/docker-nginx-ldap - https://github.com/tiredofit/docker-nginx-ldap/blob/master/Dockerfile
# jwilder/nginx-proxy - https://github.com/jwilder/nginx-proxy/blob/master/Dockerfile.alpine
####################################################################################
FROM alpine:3.13 as stunnel_build
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=300
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "builder"
ENV PGROUP "abuild"
ADD https://codeload.github.com/alpinelinux/aports/tar.gz/master /aports-master.tar.gz
USER root
RUN set -x ; \
apk add --no-cache alpine-sdk sudo openssl-dev linux-headers; \
sed -i 's/^#\s*\(%wheel\s\+ALL=(ALL)\s\+NOPASSWD:\s\+ALL\)/\1/' /etc/sudoers ; \
adduser -D -u ${DEFAULT_UID} -h /apkbuild -G ${PGROUP} ${PUSER} ; \
addgroup ${PUSER} wheel ; \
chmod 644 /aports-master.tar.gz
USER ${PUSER}
RUN set -x ; \
cd /apkbuild ; \
tar xvf /aports-master.tar.gz aports-master/community/stunnel ; \
cd /apkbuild/aports-master/community/stunnel ; \
abuild-keygen -a -i -n ; \
abuild checksum ; \
abuild -R
####################################################################################
FROM alpine:3.13
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/nginx-proxy'
LABEL org.opencontainers.image.description='Malcolm container providing an NGINX reverse proxy for the other services'
ARG DEFAULT_UID=101
ARG DEFAULT_GID=101
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "nginx"
ENV PGROUP "nginx"
# not dropping privileges globally so nginx and stunnel can bind privileged ports internally.
# nginx itself will drop privileges to "nginx" user for worker processes
ENV PUSER_PRIV_DROP false
ENV TERM xterm
USER root
# authentication method: encrypted HTTP basic authentication ('true') vs nginx-auth-ldap ('false')
ARG NGINX_BASIC_AUTH=true
# NGINX LDAP (NGINX_BASIC_AUTH=false) can support LDAP, LDAPS, or LDAP+StartTLS.
# For StartTLS, set NGINX_LDAP_TLS_STUNNEL=true to issue the StartTLS command
# and use stunnel to tunnel the connection.
ARG NGINX_LDAP_TLS_STUNNEL=false
# stunnel will require and verify certificates for StartTLS when one or more
# trusted CA certificate files are placed in the ./nginx/ca-trust directory.
# For additional security, hostname or IP address checking of the associated
# CA certificate(s) can be enabled by providing these values.
# see https://www.stunnel.org/howto.html
# https://www.openssl.org/docs/man1.1.1/man3/X509_check_host.html
ARG NGINX_LDAP_TLS_STUNNEL_CHECK_HOST=
ARG NGINX_LDAP_TLS_STUNNEL_CHECK_IP=
ARG NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL=2
ENV NGINX_BASIC_AUTH $NGINX_BASIC_AUTH
ENV NGINX_LDAP_TLS_STUNNEL $NGINX_LDAP_TLS_STUNNEL
ENV NGINX_LDAP_TLS_STUNNEL_CHECK_HOST $NGINX_LDAP_TLS_STUNNEL_CHECK_HOST
ENV NGINX_LDAP_TLS_STUNNEL_CHECK_IP $NGINX_LDAP_TLS_STUNNEL_CHECK_IP
ENV NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL $NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL
# build latest nginx with nginx-auth-ldap
ENV NGINX_VERSION=1.20.1
ENV NGINX_AUTH_LDAP_BRANCH=master
ADD https://codeload.github.com/mmguero-dev/nginx-auth-ldap/tar.gz/$NGINX_AUTH_LDAP_BRANCH /nginx-auth-ldap.tar.gz
ADD http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz /nginx.tar.gz
COPY --from=stunnel_build /apkbuild/packages/community/x86_64/stunnel-*.apk /tmp/
RUN set -x ; \
CONFIG="\
--prefix=/etc/nginx \
--sbin-path=/usr/sbin/nginx \
--modules-path=/usr/lib/nginx/modules \
--conf-path=/etc/nginx/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/run/nginx.pid \
--lock-path=/var/run/nginx.lock \
--http-client-body-temp-path=/var/cache/nginx/client_temp \
--http-proxy-temp-path=/var/cache/nginx/proxy_temp \
--http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \
--http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \
--http-scgi-temp-path=/var/cache/nginx/scgi_temp \
--user=${PUSER} \
--group=${PGROUP} \
--with-http_ssl_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_sub_module \
--with-http_dav_module \
--with-http_flv_module \
--with-http_mp4_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_random_index_module \
--with-http_secure_link_module \
--with-http_stub_status_module \
--with-http_auth_request_module \
--with-http_xslt_module=dynamic \
--with-http_image_filter_module=dynamic \
--with-http_geoip_module=dynamic \
--with-http_perl_module=dynamic \
--with-threads \
--with-stream \
--with-stream_ssl_module \
--with-stream_ssl_preread_module \
--with-stream_realip_module \
--with-stream_geoip_module=dynamic \
--with-http_slice_module \
--with-mail \
--with-mail_ssl_module \
--with-compat \
--with-file-aio \
--with-http_v2_module \
--add-module=/usr/src/nginx-auth-ldap \
" ; \
apk add --no-cache curl shadow; \
addgroup -g ${DEFAULT_GID} -S ${PGROUP} ; \
adduser -S -D -H -u ${DEFAULT_UID} -h /var/cache/nginx -s /sbin/nologin -G ${PGROUP} -g ${PUSER} ${PUSER} ; \
addgroup ${PUSER} shadow ; \
mkdir -p /var/cache/nginx ; \
chown ${PUSER}:${PGROUP} /var/cache/nginx ; \
apk add --no-cache --virtual .nginx-build-deps \
gcc \
gd-dev \
geoip-dev \
gnupg \
libc-dev \
libressl-dev \
libxslt-dev \
linux-headers \
make \
openldap-dev \
pcre-dev \
perl-dev \
tar \
zlib-dev \
; \
\
mkdir -p /usr/src/nginx-auth-ldap /www /www/logs/nginx ; \
tar -zxC /usr/src -f /nginx.tar.gz ; \
tar -zxC /usr/src/nginx-auth-ldap --strip=1 -f /nginx-auth-ldap.tar.gz ; \
cd /usr/src/nginx-$NGINX_VERSION ; \
./configure $CONFIG --with-debug ; \
make -j$(getconf _NPROCESSORS_ONLN) ; \
mv objs/nginx objs/nginx-debug ; \
mv objs/ngx_http_xslt_filter_module.so objs/ngx_http_xslt_filter_module-debug.so ; \
mv objs/ngx_http_image_filter_module.so objs/ngx_http_image_filter_module-debug.so ; \
mv objs/ngx_http_geoip_module.so objs/ngx_http_geoip_module-debug.so ; \
mv objs/ngx_http_perl_module.so objs/ngx_http_perl_module-debug.so ; \
mv objs/ngx_stream_geoip_module.so objs/ngx_stream_geoip_module-debug.so ; \
./configure $CONFIG ; \
make -j$(getconf _NPROCESSORS_ONLN) ; \
make install ; \
rm -rf /etc/nginx/html/ ; \
mkdir -p /etc/nginx/conf.d/ ; \
mkdir -p /usr/share/nginx/html/ ; \
install -m644 html/index.html /usr/share/nginx/html/ ; \
install -m644 html/50x.html /usr/share/nginx/html/ ; \
install -m755 objs/nginx-debug /usr/sbin/nginx-debug ; \
install -m755 objs/ngx_http_xslt_filter_module-debug.so /usr/lib/nginx/modules/ngx_http_xslt_filter_module-debug.so ; \
install -m755 objs/ngx_http_image_filter_module-debug.so /usr/lib/nginx/modules/ngx_http_image_filter_module-debug.so ; \
install -m755 objs/ngx_http_geoip_module-debug.so /usr/lib/nginx/modules/ngx_http_geoip_module-debug.so ; \
install -m755 objs/ngx_http_perl_module-debug.so /usr/lib/nginx/modules/ngx_http_perl_module-debug.so ; \
install -m755 objs/ngx_stream_geoip_module-debug.so /usr/lib/nginx/modules/ngx_stream_geoip_module-debug.so ; \
ln -s ../../usr/lib/nginx/modules /etc/nginx/modules ; \
strip /usr/sbin/nginx* ; \
strip /usr/lib/nginx/modules/*.so ; \
rm -rf /usr/src/nginx-$NGINX_VERSION ; \
\
# Bring in gettext so we can get `envsubst`, then throw
# the rest away. To do this, we need to install `gettext`
# then move `envsubst` out of the way so `gettext` can
# be deleted completely, then move `envsubst` back.
apk add --no-cache --virtual .gettext gettext ; \
mv /usr/bin/envsubst /tmp/ ; \
\
runDeps="$( \
scanelf --needed --nobanner /usr/sbin/nginx /usr/lib/nginx/modules/*.so /tmp/envsubst \
| awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \
| sort -u \
| xargs -r apk info --installed \
| sort -u \
)" ; \
apk add --no-cache --virtual .nginx-rundeps $runDeps ca-certificates bash wget openssl apache2-utils openldap supervisor tzdata; \
update-ca-certificates; \
apk add --no-cache --allow-untrusted /tmp/stunnel-*.apk; \
apk del .nginx-build-deps ; \
apk del .gettext ; \
mv /tmp/envsubst /usr/local/bin/ ; \
rm -rf /usr/src/* /var/tmp/* /var/cache/apk/* /tmp/stunnel-*.apk /nginx.tar.gz /nginx-auth-ldap.tar.gz; \
touch /etc/nginx/nginx_ldap.conf /etc/nginx/nginx_blank.conf;
COPY --from=jwilder/nginx-proxy:alpine /app/nginx.tmpl /etc/nginx/
COPY --from=jwilder/nginx-proxy:alpine /etc/nginx/network_internal.conf /etc/nginx/
COPY --from=jwilder/nginx-proxy:alpine /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD nginx/scripts /usr/local/bin/
ADD nginx/*.conf /etc/nginx/
ADD nginx/supervisord.conf /etc/
ADD docs/images/icon/favicon.ico /etc/nginx/favicon.ico
EXPOSE 80
VOLUME ["/etc/nginx/certs", "/etc/nginx/dhparam"]
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/usr/local/bin/docker_entrypoint.sh"]
CMD ["supervisord", "-c", "/etc/supervisord.conf", "-u", "root", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

97
Vagrant/resources/malcolm/Dockerfiles/pcap-capture.Dockerfile Normal file
View File

@@ -0,0 +1,97 @@
FROM debian:buster-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/pcap-capture'
LABEL org.opencontainers.image.description='Malcolm container providing network traffic capture capabilities via netsniff-ng and tcpdump'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "pcap"
ENV PGROUP "pcap"
# not dropping privileges globally: supervisord will take care of it
# for all processes, but first we need root to sure capabilities for
# traffic capturing tools are in-place before they are started.
# despite doing setcap here in the Dockerfile, the chown in
# docker-uid-gid-setup.sh will cause them to be lost, so we need
# a final check in supervisor.sh before startup
ENV PUSER_PRIV_DROP false
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
ARG PCAP_ENABLE_TCPDUMP=false
ARG PCAP_ENABLE_NETSNIFF=false
# PCAP_IFACE=comma-separated list of capture interfaces
ARG PCAP_IFACE=eth0
ARG PCAP_NETSNIFF_MAGIC=0xa1b2c3d4
ARG PCAP_TCPDUMP_FILENAME_PATTERN=%Y%m%d%H%M%S.pcap
ARG PCAP_ROTATE_MINUTES=30
ARG PCAP_ROTATE_MEGABYTES=500
ARG PCAP_PATH=/pcap
ARG PCAP_FILTER=
ARG PCAP_SNAPLEN=0
ENV PCAP_ENABLE_TCPDUMP $PCAP_ENABLE_TCPDUMP
ENV PCAP_ENABLE_NETSNIFF $PCAP_ENABLE_NETSNIFF
ENV PCAP_IFACE $PCAP_IFACE
ENV PCAP_NETSNIFF_MAGIC $PCAP_NETSNIFF_MAGIC
ENV PCAP_TCPDUMP_FILENAME_PATTERN $PCAP_TCPDUMP_FILENAME_PATTERN
ENV PCAP_ROTATE_MINUTES $PCAP_ROTATE_MINUTES
ENV PCAP_ROTATE_MEGABYTES $PCAP_ROTATE_MEGABYTES
ENV PCAP_PATH $PCAP_PATH
ENV PCAP_FILTER $PCAP_FILTER
ENV PCAP_SNAPLEN $PCAP_SNAPLEN
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD pcap-capture/supervisord.conf /etc/supervisord.conf
ADD pcap-capture/scripts/*.sh /usr/local/bin/
ADD pcap-capture/templates/*.template /etc/supervisor.d/
RUN apt-get update && \
apt-get install --no-install-recommends -y -q \
bc \
ethtool \
libcap2-bin \
netsniff-ng \
procps \
psmisc \
supervisor \
tcpdump && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
mkdir -p /etc/supervisor.d && \
chown -R ${PUSER}:${PGROUP} /etc/supervisor.d && \
chmod -R 750 /etc/supervisor.d && \
chown root:${PGROUP} /sbin/ethtool && \
setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \
chown root:${PGROUP} /usr/sbin/tcpdump && \
setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/sbin/tcpdump && \
chown root:${PGROUP} /usr/sbin/netsniff-ng && \
setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng && \
chmod 755 /usr/local/bin/*.sh
WORKDIR "$PCAP_PATH"
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/local/bin/supervisor.sh"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

80
Vagrant/resources/malcolm/Dockerfiles/pcap-monitor.Dockerfile Normal file
View File

@@ -0,0 +1,80 @@
FROM debian:buster-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/pcap-monitor'
LABEL org.opencontainers.image.description='Malcolm container watching for captured or uploaded artifacts to be processed'
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "watcher"
ENV PGROUP "watcher"
# not dropping privileges globally: supervisord will take care of it
# on a case-by-case basis so that one script (watch-pcap-uploads-folder.sh)
# can chown uploaded files
ENV PUSER_PRIV_DROP false
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
ARG ELASTICSEARCH_URL="http://elasticsearch:9200"
ARG PCAP_PATH=/pcap
ARG PCAP_PIPELINE_DEBUG=false
ARG PCAP_PIPELINE_DEBUG_EXTRA=false
ARG PCAP_PIPELINE_IGNORE_PREEXISTING=false
ARG ZEEK_PATH=/zeek
ENV ELASTICSEARCH_URL $ELASTICSEARCH_URL
ENV PCAP_PATH $PCAP_PATH
ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG
ENV PCAP_PIPELINE_DEBUG_EXTRA $PCAP_PIPELINE_DEBUG_EXTRA
ENV PCAP_PIPELINE_IGNORE_PREEXISTING $PCAP_PIPELINE_IGNORE_PREEXISTING
ENV ZEEK_PATH $ZEEK_PATH
RUN apt-get update && \
apt-get install --no-install-recommends -y -q \
file \
inotify-tools \
libzmq5 \
procps \
psmisc \
python \
python3-pip \
python3-setuptools \
python3-wheel \
supervisor \
vim-tiny && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
pip3 install --no-cache-dir elasticsearch elasticsearch_dsl pyzmq pyinotify python-magic && \
groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER}
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD pcap-monitor/supervisord.conf /etc/supervisord.conf
ADD pcap-monitor/scripts/ /usr/local/bin/
ADD shared/bin/pcap_watcher.py /usr/local/bin/
ADD shared/bin/pcap_utils.py /usr/local/bin/
EXPOSE 30441
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-u", "root", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION

234
Vagrant/resources/malcolm/Dockerfiles/zeek.Dockerfile Normal file
View File

@@ -0,0 +1,234 @@
FROM debian:buster-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm.netsec@gmail.com"
LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
LABEL org.opencontainers.image.title='malcolmnetsec/zeek'
LABEL org.opencontainers.image.description='Malcolm container providing Zeek'
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
# configure unprivileged user and runtime parameters
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "zeeker"
ENV PGROUP "zeeker"
ENV PUSER_PRIV_DROP true
# for download and install
ARG ZEEK_LTS=1
ARG ZEEK_VERSION=4.0.3-0
ARG SPICY_VERSION=1.1.0
ENV ZEEK_LTS $ZEEK_LTS
ENV ZEEK_VERSION $ZEEK_VERSION
ENV SPICY_VERSION $SPICY_VERSION
# for build
ENV LLVM_VERSION "11"
ENV CC "clang-${LLVM_VERSION}"
ENV CXX "clang++-${LLVM_VERSION}"
ENV ASM "clang-${LLVM_VERSION}"
ENV CCACHE_DIR "/var/spool/ccache"
ENV CCACHE_COMPRESS 1
# put Zeek and Spicy in PATH
ENV ZEEK_DIR "/opt/zeek"
ENV SPICY_DIR "/opt/spicy"
ENV PATH "${ZEEK_DIR}/bin:${SPICY_DIR}/bin:${ZEEK_DIR}/lib/zeek/plugins/packages/spicy-plugin/bin:${PATH}"
# add script for building 3rd-party plugins
ADD shared/bin/zeek_install_plugins.sh /usr/local/bin/
# build and install system packages, zeek, spicy and plugins
RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \
apt-get -q update && \
apt-get install -q -y --no-install-recommends \
ca-certificates \
curl \
file \
git \
gnupg2 \
jq \
less \
libcap2-bin \
moreutils \
procps \
psmisc \
vim-tiny && \
( curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - ) && \
echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-${LLVM_VERSION} main" >> /etc/apt/sources.list && \
echo "deb-src http://apt.llvm.org/buster/ llvm-toolchain-buster-${LLVM_VERSION} main" >> /etc/apt/sources.list && \
apt-get -q update && \
apt-get install -q -y -t buster-backports --no-install-recommends \
bison \
ccache \
clang-${LLVM_VERSION} \
cmake \
flex \
libatomic1 \
libclang-${LLVM_VERSION}-dev \
libfl-dev \
libgoogle-perftools4 \
libkrb5-3 \
libmaxminddb-dev \
libmaxminddb0 \
libpcap-dev \
libpcap0.8 \
libssl-dev \
libtcmalloc-minimal4 \
libunwind8 \
libzmq5 \
llvm-${LLVM_VERSION}-dev \
locales-all \
make \
ninja-build \
python3 \
python3-git \
python3-pip \
python3-semantic-version \
python3-setuptools \
python3-wheel \
supervisor \
zlib1g-dev && \
python3 -m pip install --no-cache-dir pyzmq && \
mkdir -p /tmp/zeek-packages && \
cd /tmp/zeek-packages && \
if [ -n "${ZEEK_LTS}" ]; then ZEEK_LTS="-lts"; fi && export ZEEK_LTS && \
curl -sSL --remote-name-all \
"https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/libbroker${ZEEK_LTS}-dev_${ZEEK_VERSION}_amd64.deb" \
"https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-core-dev_${ZEEK_VERSION}_amd64.deb" \
"https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-core_${ZEEK_VERSION}_amd64.deb" \
"https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-libcaf-dev_${ZEEK_VERSION}_amd64.deb" \
"https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}_${ZEEK_VERSION}_amd64.deb" \
"https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-btest_${ZEEK_VERSION}_amd64.deb" \
"https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-zkg_${ZEEK_VERSION}_amd64.deb" \
"https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeekctl${ZEEK_LTS}_${ZEEK_VERSION}_amd64.deb" && \
dpkg -i ./*.deb && \
mkdir -p /tmp/spicy-packages && \
cd /tmp/spicy-packages && \
curl -sSL --remote-name-all \
"https://github.com/zeek/spicy/releases/download/v${SPICY_VERSION}/spicy_linux_debian10.deb" && \
dpkg -i ./*.deb && \
cd /tmp && \
mkdir -p "${CCACHE_DIR}" && \
zkg autoconfig --force && \
zkg install --force --skiptests zeek/spicy-plugin && \
bash /usr/local/bin/zeek_install_plugins.sh && \
( find "${ZEEK_DIR}"/lib -type d -name CMakeFiles -exec rm -rf "{}" \; 2>/dev/null || true ) && \
( find "${ZEEK_DIR}"/var/lib/zkg -type d -name build -exec rm -rf "{}" \; 2>/dev/null || true ) && \
( find "${ZEEK_DIR}"/var/lib/zkg/clones -type d -name .git -execdir bash -c "pwd; du -sh; git pull --depth=1 --ff-only; git reflog expire --expire=all --all; git tag -l | xargs -r git tag -d; git gc --prune=all; du -sh" \; ) && \
rm -rf "${ZEEK_DIR}"/var/lib/zkg/scratch && \
( find "${ZEEK_DIR}/" "${SPICY_DIR}/" -type f -exec file "{}" \; | grep -Pi "ELF 64-bit.*not stripped" | sed 's/:.*//' | xargs -l -r strip --strip-unneeded ) && \
mkdir -p "${ZEEK_DIR}"/var/lib/zkg/clones/package/spicy-plugin/build/plugin/bin/ && \
ln -s -r "${ZEEK_DIR}"/lib/zeek/plugins/packages/spicy-plugin/bin/spicyz \
"${ZEEK_DIR}"/var/lib/zkg/clones/package/spicy-plugin/build/plugin/bin/spicyz && \
mkdir -p "${ZEEK_DIR}"/var/lib/zkg/clones/package/spicy-plugin/plugin/lib/ && \
ln -s -r "${ZEEK_DIR}"/lib/zeek/plugins/packages/spicy-plugin/lib/bif \
"${ZEEK_DIR}"/var/lib/zkg/clones/package/spicy-plugin/plugin/lib/bif && \
cd /usr/lib/locale && \
( ls | grep -Piv "^(en|en_US|en_US\.utf-?8|C\.utf-?8)$" | xargs -l -r rm -rf ) && \
cd /tmp && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/cache/*/*
# add configuration and scripts
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD shared/bin/pcap_moloch_and_zeek_processor.py /usr/local/bin/
ADD shared/bin/pcap_utils.py /usr/local/bin/
ADD shared/pcaps /tmp/pcaps
ADD zeek/supervisord.conf /etc/supervisord.conf
ADD zeek/config/*.zeek ${ZEEK_DIR}/share/zeek/site/
ADD zeek/config/*.txt ${ZEEK_DIR}/share/zeek/site/
# sanity checks to make sure the plugins installed and copied over correctly
# these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh
# todo: Bro::LDAP is broken right now, disabled
ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 19
ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(_Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OPENVPN|ANALYZER_SPICY_IPSEC|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_LDAP_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP_IP|ICSNPP::BSAP_SERIAL|ICSNPP::ENIP|Salesforce::GQUIC|Zeek::PROFINET|Zeek::S7comm|Zeek::TDS)"
ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 15
ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-31166/detect|hassh/hassh|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)"
RUN mkdir -p /tmp/logs && \
cd /tmp/logs && \
"$ZEEK_DIR"/bin/zeek -NN local >zeeknn.log 2>/dev/null && \
bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_PLUGINS_GREP" zeeknn.log) >= $ZEEK_THIRD_PARTY_PLUGINS_COUNT)) && echo 'Zeek plugins loaded correctly' || (echo 'One or more Zeek plugins did not load correctly' && cat zeeknn.log && exit 1)" && \
"$ZEEK_DIR"/bin/zeek -C -r /tmp/pcaps/udp.pcap local policy/misc/loaded-scripts 2>/dev/null && \
bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_SCRIPTS_GREP" loaded_scripts.log) == $ZEEK_THIRD_PARTY_SCRIPTS_COUNT)) && echo 'Zeek scripts loaded correctly' || (echo 'One or more Zeek scripts did not load correctly' && cat loaded_scripts.log && exit 1)" && \
cd /tmp && \
rm -rf /tmp/logs /tmp/pcaps
RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \
useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \
usermod -a -G tty ${PUSER} && \
ln -sfr /usr/local/bin/pcap_moloch_and_zeek_processor.py /usr/local/bin/pcap_zeek_processor.py
#Whether or not to auto-tag logs based on filename
ARG AUTO_TAG=true
#Whether or not to run "zeek -r XXXXX.pcap local" on each pcap file
ARG ZEEK_AUTO_ANALYZE_PCAP_FILES=false
ARG ZEEK_AUTO_ANALYZE_PCAP_THREADS=1
ARG ZEEK_EXTRACTOR_MODE=none
ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files
ARG PCAP_PIPELINE_DEBUG=false
ARG PCAP_PIPELINE_DEBUG_EXTRA=false
ARG PCAP_MONITOR_HOST=pcap-monitor
ENV AUTO_TAG $AUTO_TAG
ENV ZEEK_AUTO_ANALYZE_PCAP_FILES $ZEEK_AUTO_ANALYZE_PCAP_FILES
ENV ZEEK_AUTO_ANALYZE_PCAP_THREADS $ZEEK_AUTO_ANALYZE_PCAP_THREADS
ENV ZEEK_EXTRACTOR_MODE $ZEEK_EXTRACTOR_MODE
ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG
ENV PCAP_PIPELINE_DEBUG_EXTRA $PCAP_PIPELINE_DEBUG_EXTRA
ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST
# environment variables for zeek runtime tweaks (used in local.zeek)
ARG ZEEK_DISABLE_HASH_ALL_FILES=
ARG ZEEK_DISABLE_LOG_PASSWORDS=
ARG ZEEK_DISABLE_SSL_VALIDATE_CERTS=
ARG ZEEK_DISABLE_TRACK_ALL_ASSETS=
ARG ZEEK_DISABLE_BEST_GUESS_ICS=true
# TODO: assess spicy-analyzer that replace built-in Zeek parsers
# for now, disable them by default when a Zeek parser exists
ARG ZEEK_DISABLE_SPICY_DHCP=true
ARG ZEEK_DISABLE_SPICY_DNS=true
ARG ZEEK_DISABLE_SPICY_HTTP=true
ARG ZEEK_DISABLE_SPICY_IPSEC=
ARG ZEEK_DISABLE_SPICY_OPENVPN=
ARG ZEEK_DISABLE_SPICY_TFTP=
ARG ZEEK_DISABLE_SPICY_WIREGUARD=
ENV ZEEK_DISABLE_HASH_ALL_FILES $ZEEK_DISABLE_HASH_ALL_FILES
ENV ZEEK_DISABLE_LOG_PASSWORDS $ZEEK_DISABLE_LOG_PASSWORDS
ENV ZEEK_DISABLE_SSL_VALIDATE_CERTS $ZEEK_DISABLE_SSL_VALIDATE_CERTS
ENV ZEEK_DISABLE_TRACK_ALL_ASSETS $ZEEK_DISABLE_TRACK_ALL_ASSETS
ENV ZEEK_DISABLE_BEST_GUESS_ICS $ZEEK_DISABLE_BEST_GUESS_ICS
ENV ZEEK_DISABLE_SPICY_DHCP $ZEEK_DISABLE_SPICY_DHCP
ENV ZEEK_DISABLE_SPICY_DNS $ZEEK_DISABLE_SPICY_DNS
ENV ZEEK_DISABLE_SPICY_HTTP $ZEEK_DISABLE_SPICY_HTTP
ENV ZEEK_DISABLE_SPICY_OPENVPN $ZEEK_DISABLE_SPICY_OPENVPN
ENV ZEEK_DISABLE_SPICY_TFTP $ZEEK_DISABLE_SPICY_TFTP
ENV ZEEK_DISABLE_SPICY_WIREGUARD $ZEEK_DISABLE_SPICY_WIREGUARD
ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
# to be populated at build-time:
ARG BUILD_DATE
ARG MALCOLM_VERSION
ARG VCS_REVISION
LABEL org.opencontainers.image.created=$BUILD_DATE
LABEL org.opencontainers.image.version=$MALCOLM_VERSION
LABEL org.opencontainers.image.revision=$VCS_REVISION