added Malcolm

This commit is contained in:
2021-08-06 10:35:01 +02:00
parent f043730066
commit 70f1922e80
751 changed files with 195277 additions and 0 deletions

View File

@@ -0,0 +1,3 @@
*
!.gitignore

View File

@@ -0,0 +1,3 @@
*
!.gitignore

View File

@@ -0,0 +1 @@
vagrant:$2y$05$EKbzasbwZS/S9cM71hCvsulXyvGObG6fTS3h04MwmwOe4C7P1Zk3C

View File

@@ -0,0 +1,308 @@
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
daemon off;
worker_processes 1;
events { worker_connections 1024; }
http {
include /etc/nginx/mime.types;
sendfile on;
client_max_body_size 20m;
fastcgi_buffers 16 64k;
fastcgi_buffer_size 256k;
fastcgi_read_timeout 300s;
fastcgi_busy_buffers_size 384k;
fastcgi_request_buffering off;
proxy_connect_timeout 180s;
proxy_read_timeout 300s;
proxy_send_timeout 300s;
proxy_buffer_size 512k;
proxy_buffers 16 4m;
proxy_busy_buffers_size 16m;
# if LDAP authentication is enabled, this will configure the ldap_server section
include /etc/nginx/nginx_ldap_rt.conf;
upstream docker-arkime {
server arkime:8005;
}
upstream docker-malcolm-readme {
server arkime:8000;
}
upstream docker-upload {
server upload:80;
}
upstream docker-htadmin {
server htadmin:80;
}
upstream docker-kibana {
server kibana:5601;
}
upstream docker-kibana-maps {
server kibana-helper:28991;
}
upstream docker-elasticsearch {
server elasticsearch:9200;
}
upstream docker-logstash-stats {
server logstash:9600;
}
upstream docker-name-map-ui {
server name-map-ui:8080;
}
upstream docker-extracted-file-http-server {
server file-monitor:8440;
}
# htadmin (htpasswd/user management)
server {
listen 488 ssl;
ssl_certificate /etc/nginx/certs/cert.pem;
ssl_certificate_key /etc/nginx/certs/key.pem;
location / {
proxy_pass http://docker-htadmin;
proxy_redirect off;
proxy_set_header Host htadmin.malcolm.local;
}
}
# Arkime interface
server {
listen 443 ssl;
ssl_certificate /etc/nginx/certs/cert.pem;
ssl_certificate_key /etc/nginx/certs/key.pem;
# use either auth_basic or auth_ldap
include /etc/nginx/nginx_auth_rt.conf;
# Malcolm readme
location /readme {
proxy_pass http://docker-malcolm-readme/README.html;
proxy_redirect off;
proxy_set_header Host arkime.malcolm.local;
}
# Malcolm file upload
location /upload {
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_pass http://docker-upload/;
proxy_redirect off;
proxy_set_header Host upload.malcolm.local;
proxy_request_buffering off;
proxy_buffering off;
client_max_body_size 20G;
}
location /server/php {
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_pass http://docker-upload/server/php/;
proxy_redirect off;
proxy_set_header Host upload.malcolm.local;
proxy_request_buffering off;
proxy_buffering off;
client_max_body_size 20G;
}
# Logstash statistics
location ~* ^/logstash\b(.*) {
proxy_pass http://docker-logstash-stats/_node/stats$1;
proxy_redirect off;
proxy_set_header Host arkime.malcolm.local;
}
# Arkime -> Kibana shortcut
location ~* ^/idmol2kib(.*) {
set $filter_start_time now-1d;
if ($arg_start != '') {
set $filter_start_time \'$arg_start\';
}
set $filter_stop_time now;
if ($arg_stop != '') {
set $filter_stop_time \'$arg_stop\';
}
set $filter_field undefined;
if ($arg_field != '') {
set $filter_field $arg_field;
}
set $filter_value undefined;
if ($arg_value != '') {
set $filter_value $arg_value;
}
rewrite ^/idmol2kib/(.*) /kibana/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'sessions2-*',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'sessions2-*',interval:auto,query:(language:lucene,query:''),sort:!(firstPacket,desc)) redirect;
proxy_pass http://docker-kibana;
proxy_redirect off;
proxy_set_header Host kibana.malcolm.local;
}
# Kibana -> Arkime shortcut
location ~* /idkib2mol/(.*) {
rewrite ^.*/idkib2mol/(.*) /sessions?expression=($1) redirect;
proxy_pass http://docker-arkime;
proxy_redirect off;
proxy_set_header Host arkime.malcolm.local;
proxy_set_header http_auth_http_user $remote_user;
proxy_set_header Authorization "";
}
# Kibana/Arkime -> extracted file download
location ~* /dl-extracted-files/(.*) {
rewrite ^.*/dl-extracted-files/(.*) /extracted-files/$1 redirect;
proxy_pass http://docker-extracted-file-http-server;
proxy_redirect off;
proxy_set_header Host file-monitor.malcolm.local;
}
# already prepended /kibana to match the server.basePath in kibana's YML config file
location /kibana {
proxy_pass http://docker-kibana;
proxy_redirect off;
proxy_set_header Host kibana.malcolm.local;
}
# otherwise (from old kibana bookmarks), prepend /kibana to match the server.basePath in kibana's YML config file
location /app {
proxy_pass http://docker-kibana/kibana/app;
proxy_redirect off;
proxy_set_header Host kibana.malcolm.local;
}
# offline region maps for kibana
location /world.geojson {
proxy_pass http://docker-kibana-maps;
proxy_redirect off;
proxy_set_header Host kibana-helper.malcolm.local;
}
# name-map-ui (UI for mapping names to network hosts and subnets)
location /name-map-ui {
proxy_pass http://docker-name-map-ui/;
proxy_redirect off;
proxy_set_header Host name-map-ui.malcolm.local;
proxy_cache off;
}
location ~* ^/extracted-files\b(.*) {
proxy_pass http://docker-extracted-file-http-server$1;
proxy_redirect off;
proxy_set_header Host file-monitor.malcolm.local;
}
location = /favicon.ico {
alias /etc/nginx/favicon.ico;
}
# Fix cyberchef JS module(s)
# https://localhost/moloch/session/190924-KgO9H30qhdREw7ltsDXn1Rgp/modules/Regex.js
location ~* ^/moloch/session/.*/(modules/.*\.js) {
proxy_hide_header Content-Type;
proxy_set_header Content-Type "application/javascript";
add_header Content-Type "application/javascript";
default_type application/javascript;
add_header X-Content-Type-Options 'nosniff';
proxy_pass http://docker-arkime/cyberchef/$1;
proxy_redirect off;
proxy_set_header Host arkime.malcolm.local;
proxy_set_header http_auth_http_user $remote_user;
proxy_set_header Authorization "";
}
location / {
proxy_pass http://docker-arkime;
proxy_redirect off;
proxy_set_header Host arkime.malcolm.local;
proxy_set_header http_auth_http_user $remote_user;
proxy_set_header Authorization "";
}
}
# Kibana interface
server {
listen 5601 ssl;
ssl_certificate /etc/nginx/certs/cert.pem;
ssl_certificate_key /etc/nginx/certs/key.pem;
# use either auth_basic or auth_ldap
include /etc/nginx/nginx_auth_rt.conf;
location = /favicon.ico {
alias /etc/nginx/favicon.ico;
}
# Kibana -> Arkime shortcut
location ~* /idkib2mol/(.*) {
rewrite ^.*/idkib2mol/(.*) /sessions?expression=($1) redirect;
proxy_pass http://docker-arkime;
proxy_redirect off;
proxy_set_header Host arkime.malcolm.local;
proxy_set_header http_auth_http_user $remote_user;
proxy_set_header Authorization "";
}
# Kibana -> extracted file download
location ~* /dl-extracted-files/(.*) {
rewrite ^.*/dl-extracted-files/(.*) /extracted-files/$1 redirect;
proxy_pass http://docker-extracted-file-http-server;
proxy_redirect off;
proxy_set_header Host file-monitor.malcolm.local;
}
# already prepended /kibana to match the server.basePath in kibana's YML config file
location /kibana {
proxy_pass http://docker-kibana;
proxy_redirect off;
proxy_set_header Host kibana.malcolm.local;
}
# otherwise prepend /kibana to match the server.basePath in kibana's YML config file
location / {
rewrite ^/(.*) /kibana/$1;
proxy_pass http://docker-kibana;
proxy_redirect off;
proxy_set_header Host kibana.malcolm.local;
}
}
# Elasticsearch API
server {
listen 9200 ssl;
ssl_certificate /etc/nginx/certs/cert.pem;
ssl_certificate_key /etc/nginx/certs/key.pem;
# use either auth_basic or auth_ldap
include /etc/nginx/nginx_auth_rt.conf;
location = /favicon.ico {
alias /etc/nginx/favicon.ico;
}
location / {
proxy_pass http://docker-elasticsearch;
proxy_redirect off;
proxy_set_header Host es.malcolm.local;
client_max_body_size 50m;
}
}
}

View File

@@ -0,0 +1,2 @@
auth_basic "Authentication Required";
auth_basic_user_file /etc/nginx/.htpasswd;

View File

@@ -0,0 +1,2 @@
auth_ldap "LDAP Authentication Required";
auth_ldap_servers ad_server;

View File

@@ -0,0 +1,20 @@
# This is a sample configuration for the ldap_server section of nginx.conf.
# Yours will vary depending on how your Active Directory/LDAP server is configured.
# See https://github.com/kvspb/nginx-auth-ldap#available-config-parameters for options.
ldap_server ad_server {
url "ldap://ds.example.com:3268/DC=example,DC=com?sAMAccountName?sub?(objectClass=person)";
binddn "bind_dn";
binddn_passwd "bind_dn_password";
group_attribute member;
group_attribute_is_dn on;
require group "CN=malcolm,OU=groups,DC=example,DC=com";
require valid_user;
satisfy all;
}
auth_ldap_cache_enabled on;
auth_ldap_cache_expiration_time 10000;
auth_ldap_cache_size 1000;

View File

@@ -0,0 +1,212 @@
#!/bin/bash
set -e
# Warn if the DOCKER_HOST socket does not exist
if [[ $DOCKER_HOST = unix://* ]]; then
socket_file=${DOCKER_HOST#unix://}
if ! [ -S $socket_file ]; then
cat >&2 <<-EOT
ERROR: you need to share your Docker host socket with a volume at $socket_file
Typically you should run your container with: \`-v /var/run/docker.sock:$socket_file:ro\`
See the jwilder/nginx-proxy documentation at http://git.io/vZaGJ
EOT
socketMissing=1
fi
fi
# Compute the DNS resolvers for use in the templates - if the IP contains ":", it's IPv6 and must be enclosed in []
export RESOLVERS=$(awk '$1 == "nameserver" {print ($2 ~ ":")? "["$2"]": $2}' ORS=' ' /etc/resolv.conf | sed 's/ *$//g')
if [ "x$RESOLVERS" = "x" ]; then
echo "Warning: unable to determine DNS resolvers for nginx" >&2
unset RESOLVERS
fi
# If the user has run the default command and the socket doesn't exist, fail
if [ "$socketMissing" = 1 -a "$1" = 'supervisord' -a "$2" = '-c' -a "$3" = '/etc/supervisord.conf' ]; then
exit 1
fi
# set up for NGINX HTTP basic vs. LDAP/LDAPS/LDAP+StartTLS auth
# a blank file just to use as an "include" placeholder for the nginx's LDAP config when LDAP is not used
NGINX_BLANK_CONF=/etc/nginx/nginx_blank.conf
# "include" file for auth_basic, prompt, and .htpasswd location
NGINX_BASIC_AUTH_CONF=/etc/nginx/nginx_auth_basic.conf
# "include" file for auth_ldap, prompt, and "auth_ldap_servers" name
NGINX_LDAP_AUTH_CONF=/etc/nginx/nginx_auth_ldap.conf
# volume-mounted user configuration containing "ldap_server ad_server" section with URL, binddn, etc.
NGINX_LDAP_USER_CONF=/etc/nginx/nginx_ldap.conf
# runtime "include" file for auth method (link to either NGINX_BASIC_AUTH_CONF or NGINX_LDAP_AUTH_CONF)
NGINX_RUNTIME_AUTH_CONF=/etc/nginx/nginx_auth_rt.conf
# runtime "include" file for ldap config (link to either NGINX_BLANK_CONF or (possibly modified) NGINX_LDAP_USER_CONF)
NGINX_RUNTIME_LDAP_CONF=/etc/nginx/nginx_ldap_rt.conf
# config file for stunnel if using stunnel to issue LDAP StartTLS function
STUNNEL_CONF=/etc/stunnel/stunnel.conf
CA_TRUST_HOST_DIR=/etc/nginx/ca-trust
CA_TRUST_RUN_DIR=/var/run/ca-trust
# copy trusted CA certs to runtime directory and c_rehash them to create symlinks
STUNNEL_CA_PATH_LINE=""
STUNNEL_VERIFY_LINE=""
STUNNEL_CHECK_HOST_LINE=""
STUNNEL_CHECK_IP_LINE=""
NGINX_LDAP_CA_PATH_LINE=""
NGINX_LDAP_CHECK_REMOTE_CERT_LINE=""
mkdir -p "$CA_TRUST_RUN_DIR"
# attempt to make sure trusted CA certs dir is readable by unprivileged nginx worker
chmod 755 "$CA_TRUST_RUN_DIR" || true
CA_FILES=$(shopt -s nullglob dotglob; echo "$CA_TRUST_HOST_DIR"/*)
if (( ${#CA_FILES} )) ; then
rm -f "$CA_TRUST_RUN_DIR"/*
pushd "$CA_TRUST_RUN_DIR" >/dev/null 2>&1
if cp "$CA_TRUST_HOST_DIR"/* ./ ; then
# attempt to make sure trusted CA certs are readable by unprivileged nginx worker
chmod 644 * || true
# create hash symlinks
c_rehash -compat .
# variables for stunnel config
STUNNEL_CA_PATH_LINE="CApath = $CA_TRUST_RUN_DIR"
[[ -n $NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL ]] && STUNNEL_VERIFY_LINE="verify = $NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL" || STUNNEL_VERIFY_LINE="verify = 2"
[[ -n $NGINX_LDAP_TLS_STUNNEL_CHECK_HOST ]] && STUNNEL_CHECK_HOST_LINE="checkHost = $NGINX_LDAP_TLS_STUNNEL_CHECK_HOST"
[[ -n $NGINX_LDAP_TLS_STUNNEL_CHECK_IP ]] && STUNNEL_CHECK_IP_LINE="checkIP = $NGINX_LDAP_TLS_STUNNEL_CHECK_IP"
# variables for nginx config
NGINX_LDAP_CA_PATH_LINE=" ssl_ca_dir $CA_TRUST_RUN_DIR;"
( [[ -n $NGINX_LDAP_TLS_STUNNEL_CHECK_HOST ]] || [[ -n $NGINX_LDAP_TLS_STUNNEL_CHECK_IP ]] ) && NGINX_LDAP_CHECK_REMOTE_CERT_LINE=" ssl_check_cert on;" || NGINX_LDAP_CHECK_REMOTE_CERT_LINE=" ssl_check_cert chain;"
fi
popd >/dev/null 2>&1
fi
if [[ -z $NGINX_BASIC_AUTH ]] || [[ "$NGINX_BASIC_AUTH" == "true" ]]; then
# doing HTTP basic auth instead of ldap
# point nginx_auth_rt.conf to nginx_auth_basic.conf
ln -sf "$NGINX_BASIC_AUTH_CONF" "$NGINX_RUNTIME_AUTH_CONF"
# ldap configuration is empty
ln -sf "$NGINX_BLANK_CONF" "$NGINX_RUNTIME_LDAP_CONF"
else
# point nginx_auth_rt.conf to nginx_auth_ldap.conf
ln -sf "$NGINX_LDAP_AUTH_CONF" "$NGINX_RUNTIME_AUTH_CONF"
# parse URL information out of user ldap configuration
# example:
# url "ldap://localhost:3268/DC=ds,DC=example,DC=com?sAMAccountName?sub?(objectClass=person)";
# "url" quote protocol h/p uri
# ↓ ↓ ↓ ↓ ↓
PATTERN='^(\s*url\s+)([''"]?)(\w+)://([^/]+)(/.*)$'
unset HEADER
unset OPEN_QUOTE
unset PROTOCOL
unset REMOTE_HOST
unset REMOTE_PORT
unset URI_TO_END
URL_LINE_NUM=0
READ_LINE_NUM=0
while IFS= read -r LINE; do
READ_LINE_NUM=$((READ_LINE_NUM+1))
if [[ $LINE =~ $PATTERN ]]; then
URL_LINE_NUM=$READ_LINE_NUM
HEADER=${BASH_REMATCH[1]}
OPEN_QUOTE=${BASH_REMATCH[2]}
PROTOCOL=${BASH_REMATCH[3]}
REMOTE=${BASH_REMATCH[4]}
REMOTE_ARR=(${REMOTE//:/ })
[[ -n ${REMOTE_ARR[0]} ]] && REMOTE_HOST=${REMOTE_ARR[0]}
[[ -n ${REMOTE_ARR[1]} ]] && REMOTE_PORT=${REMOTE_ARR[1]} || REMOTE_PORT=3268
URI_TO_END=${BASH_REMATCH[5]}
break
fi
done < "$NGINX_LDAP_USER_CONF"
if [[ "$NGINX_LDAP_TLS_STUNNEL" == "true" ]]; then
# user provided LDAP configuration, but we need to tweak it and set up stunnel to issue StartTLS
if [[ -z $REMOTE_HOST ]]; then
# missing LDAP info needed to configure tunnel, abort
exit 1
fi
# pick a random local port to listen on for the client side of the tunnel
read PORT_LOWER POWER_UPPER < /proc/sys/net/ipv4/ip_local_port_range
LOCAL_PORT=$(shuf -i $PORT_LOWER-$POWER_UPPER -n 1)
# create PEM key for stunnel (this key doesn't matter as we're only using stunnel in client mode)
pushd /tmp >/dev/null 2>&1
openssl genrsa -out key.pem 2048
openssl req -new -x509 -key key.pem -out cert.pem -days 3650 -subj "/CN=$(hostname)/O=Malcolm/C=US"
cat key.pem cert.pem > /etc/stunnel/stunnel.pem
chmod 600 /etc/stunnel/stunnel.pem
rm -f key.pem cert.pem
popd >/dev/null 2>&1
# configure stunnel
cat <<EOF > "$STUNNEL_CONF"
setuid = nginx
setgid = nginx
pid = /tmp/stunnel.pid
socket = l:TCP_NODELAY=1
socket = r:TCP_NODELAY=1
client = yes
foreground = yes
cert = /etc/stunnel/stunnel.pem
$STUNNEL_CA_PATH_LINE
$STUNNEL_VERIFY_LINE
$STUNNEL_CHECK_HOST_LINE
$STUNNEL_CHECK_IP_LINE
[stunnel.ldap_start_tls]
accept = localhost:$LOCAL_PORT
connect = $REMOTE_HOST:$REMOTE_PORT
protocol = ldap
EOF
# rewrite modified copy of user ldap configuration to point to local end of tunnel instead of remote
rm -f "$NGINX_RUNTIME_LDAP_CONF"
touch "$NGINX_RUNTIME_LDAP_CONF"
chmod 600 "$NGINX_RUNTIME_LDAP_CONF"
READ_LINE_NUM=0
while IFS= read -r LINE; do
READ_LINE_NUM=$((READ_LINE_NUM+1))
if (( $URL_LINE_NUM == $READ_LINE_NUM )); then
echo "${HEADER}${OPEN_QUOTE}ldap://localhost:${LOCAL_PORT}${URI_TO_END}" >> "$NGINX_RUNTIME_LDAP_CONF"
else
echo "$LINE" >> "$NGINX_RUNTIME_LDAP_CONF"
fi
done < "$NGINX_LDAP_USER_CONF"
else
# we're doing either LDAP or LDAPS, but not StartTLS, so we don't need to use stunnel.
# however, we do want to set SSL CA trust stuff if specified, so do that
rm -f "$NGINX_RUNTIME_LDAP_CONF"
touch "$NGINX_RUNTIME_LDAP_CONF"
chmod 600 "$NGINX_RUNTIME_LDAP_CONF"
READ_LINE_NUM=0
while IFS= read -r LINE; do
READ_LINE_NUM=$((READ_LINE_NUM+1))
echo "$LINE" >> "$NGINX_RUNTIME_LDAP_CONF"
if (( $URL_LINE_NUM == $READ_LINE_NUM )); then
echo "$NGINX_LDAP_CHECK_REMOTE_CERT_LINE" >> "$NGINX_RUNTIME_LDAP_CONF"
echo "$NGINX_LDAP_CA_PATH_LINE" >> "$NGINX_RUNTIME_LDAP_CONF"
fi
done < "$NGINX_LDAP_USER_CONF"
fi # stunnel/starttls vs. ldap/ldaps
fi # basic vs. ldap
# start supervisor (which will spawn nginx, stunnel, etc.) or whatever the default command is
exec "$@"

View File

@@ -0,0 +1,64 @@
; Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
[unix_http_server]
file=/tmp/supervisor.sock ; (the path to the socket file)
chmod=0700
[supervisord]
nodaemon=true
logfile=/dev/null
logfile_maxbytes=0
pidfile=/tmp/supervisord.pid
[rpcinterface:supervisor]
supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///tmp/supervisor.sock
[program:nginx]
command=/usr/sbin/nginx
startsecs=10
startretries=2000000000
stopasgroup=true
killasgroup=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
redirect_stderr=true
[program:logaccess]
command=/usr/bin/tail -F /var/log/nginx/access.log
user=%(ENV_PUSER)s
startsecs=10
startretries=2000000000
stopasgroup=true
killasgroup=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
stderr_logfile=/dev/null
stdout_logfile_maxbytes=0
redirect_stderr=false
[program:logerrors]
command=/usr/bin/tail -F /var/log/nginx/error.log
user=%(ENV_PUSER)s
startsecs=10
startretries=2000000000
stopasgroup=true
killasgroup=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
stderr_logfile=/dev/null
stdout_logfile_maxbytes=0
redirect_stderr=false
[program:stunnel]
command=/usr/bin/stunnel /etc/stunnel/stunnel.conf
startsecs=10
startretries=2000000000
stopasgroup=true
killasgroup=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
redirect_stderr=true
autostart=%(ENV_NGINX_LDAP_TLS_STUNNEL)s