541 lines
16 KiB
YAML
541 lines
16 KiB
YAML
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
|
|
|
|
version: '3.7'
|
|
|
|
################################################################################
|
|
# Commonly tweaked configuration options
|
|
#-------------------------------------------------------------------------------
|
|
x-process-variables: &process-variables
|
|
# docker containers will run processes as unprivileged user with UID:GID
|
|
PUID : 1000
|
|
PGID : 1000
|
|
|
|
x-auth-variables: &auth-variables
|
|
# authentication method: encrypted HTTP basic authentication ('true') vs LDAP ('false')
|
|
NGINX_BASIC_AUTH : 'true'
|
|
# NGINX LDAP (NGINX_BASIC_AUTH=false) can support LDAP, LDAPS, or LDAP+StartTLS.
|
|
# For StartTLS, set NGINX_LDAP_TLS_STUNNEL=true to issue the StartTLS command
|
|
# and use stunnel to tunnel the connection.
|
|
NGINX_LDAP_TLS_STUNNEL : 'false'
|
|
# stunnel will require and verify certificates for StartTLS when one or more
|
|
# trusted CA certificate files are placed in the ./nginx/ca-trust directory.
|
|
# For additional security, hostname or IP address checking of the associated
|
|
# CA certificate(s) can be enabled by providing these values.
|
|
NGINX_LDAP_TLS_STUNNEL_CHECK_HOST : ''
|
|
NGINX_LDAP_TLS_STUNNEL_CHECK_IP : ''
|
|
NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL : 2
|
|
|
|
x-nginx-variables: &nginx-variables
|
|
# Whether or not to write nginx's access.log and error.log to Elasticsearch
|
|
NGINX_LOG_ACCESS_AND_ERRORS : 'false'
|
|
|
|
x-arkime-variables: &arkime-variables
|
|
MANAGE_PCAP_FILES : 'false'
|
|
ARKIME_ANALYZE_PCAP_THREADS : 1
|
|
MAXMIND_GEOIP_DB_LICENSE_KEY : '0'
|
|
|
|
x-zeek-variables: &zeek-variables
|
|
ZEEK_AUTO_ANALYZE_PCAP_FILES : 'true'
|
|
ZEEK_AUTO_ANALYZE_PCAP_THREADS : 1
|
|
ZEEK_EXTRACTOR_MODE : 'none'
|
|
EXTRACTED_FILE_IGNORE_EXISTING : 'false'
|
|
EXTRACTED_FILE_PRESERVATION : 'quarantined'
|
|
EXTRACTED_FILE_MIN_BYTES : 64
|
|
EXTRACTED_FILE_MAX_BYTES : 134217728
|
|
VTOT_API2_KEY : '0'
|
|
VTOT_REQUESTS_PER_MINUTE : 4
|
|
CLAMD_MAX_REQUESTS : 8
|
|
YARA_MAX_REQUESTS : 8
|
|
CAPA_MAX_REQUESTS : 4
|
|
EXTRACTED_FILE_ENABLE_YARA : 'false'
|
|
EXTRACTED_FILE_YARA_CUSTOM_ONLY : 'false'
|
|
EXTRACTED_FILE_ENABLE_CAPA : 'false'
|
|
EXTRACTED_FILE_CAPA_VERBOSE : 'false'
|
|
EXTRACTED_FILE_ENABLE_CLAMAV : 'false'
|
|
EXTRACTED_FILE_UPDATE_RULES : 'false'
|
|
EXTRACTED_FILE_PIPELINE_DEBUG : 'false'
|
|
EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA : 'false'
|
|
EXTRACTED_FILE_HTTP_SERVER_ENABLE : 'false'
|
|
EXTRACTED_FILE_HTTP_SERVER_ENCRYPT : 'true'
|
|
EXTRACTED_FILE_HTTP_SERVER_KEY : 'quarantined'
|
|
# environment variables for tweaking Zeek at runtime (see local.zeek)
|
|
# set to a non-blank value to disable the corresponding feature
|
|
ZEEK_DISABLE_HASH_ALL_FILES : ''
|
|
ZEEK_DISABLE_LOG_PASSWORDS : ''
|
|
ZEEK_DISABLE_SSL_VALIDATE_CERTS : ''
|
|
ZEEK_DISABLE_TRACK_ALL_ASSETS : ''
|
|
ZEEK_DISABLE_BEST_GUESS_ICS : 'true'
|
|
ZEEK_DISABLE_SPICY_DHCP : 'true'
|
|
ZEEK_DISABLE_SPICY_DNS : 'true'
|
|
ZEEK_DISABLE_SPICY_HTTP : 'true'
|
|
ZEEK_DISABLE_SPICY_IPSEC : ''
|
|
ZEEK_DISABLE_SPICY_OPENVPN : ''
|
|
ZEEK_DISABLE_SPICY_TFTP : ''
|
|
ZEEK_DISABLE_SPICY_WIREGUARD : ''
|
|
|
|
x-kibana-helper-variables: &kibana-helper-variables
|
|
ELASTICSEARCH_INDEX_SIZE_PRUNE_LIMIT : '0'
|
|
ELASTICSEARCH_INDEX_SIZE_PRUNE_NAME_SORT : 'false'
|
|
ISM_CLOSE_AGE : '60d'
|
|
ISM_COLD_AGE : '30d'
|
|
ISM_DELETE_AGE : '365d'
|
|
ISM_POLICY_NAME : 'session_index_policy'
|
|
ISM_SNAPSHOT_AGE : '1d'
|
|
ISM_SNAPSHOT_COMPRESSED : 'false'
|
|
ISM_SNAPSHOT_REPO : 'logs'
|
|
|
|
x-logstash-variables: &logstash-variables
|
|
LOGSTASH_OUI_LOOKUP : 'true'
|
|
LOGSTASH_REVERSE_DNS : 'false'
|
|
# ES_EXTERNAL_HOSTS : '10.0.0.123:9200'
|
|
# ES_EXTERNAL_SSL : 'true'
|
|
# ES_EXTERNAL_SSL_CERTIFICATE_VERIFICATION : 'false'
|
|
# For security, ES_EXTERNAL_USER and ES_EXTERNAL_PASSWORD should be stored in Logstash keystore using ./scripts/auth_setup
|
|
# See also volume mount for logstash.keystore below.
|
|
# ES_EXTERNAL_USER : 'janedoe'
|
|
# ES_EXTERNAL_PASSWORD : 'secret'
|
|
|
|
x-common-upload-variables: &common-upload-variables
|
|
AUTO_TAG : 'true'
|
|
PCAP_PIPELINE_DEBUG : 'false'
|
|
PCAP_PIPELINE_DEBUG_EXTRA : 'false'
|
|
PCAP_PIPELINE_IGNORE_PREEXISTING : 'false'
|
|
PCAP_MONITOR_HOST : 'pcap-monitor'
|
|
|
|
x-common-lookup-variables: &common-lookup-variables
|
|
FREQ_LOOKUP : 'false'
|
|
|
|
x-common-beats-variables: &common-beats-variables
|
|
BEATS_SSL : 'false'
|
|
|
|
x-pcap-capture-variables: &pcap-capture-variables
|
|
PCAP_ENABLE_NETSNIFF : 'false'
|
|
PCAP_ENABLE_TCPDUMP : 'false'
|
|
PCAP_IFACE : 'eth0'
|
|
PCAP_ROTATE_MEGABYTES : 1024
|
|
PCAP_ROTATE_MINUTES : 10
|
|
PCAP_FILTER : ''
|
|
################################################################################
|
|
|
|
services:
|
|
elasticsearch:
|
|
image: malcolmnetsec/elasticsearch-od:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: elasticsearch
|
|
environment:
|
|
<< : *process-variables
|
|
logger.level : 'WARN'
|
|
bootstrap.memory_lock : 'true'
|
|
ES_JAVA_OPTS : '-Xms4g -Xmx4g -Xss256k -Djava.security.egd=file:/dev/./urandom'
|
|
VIRTUAL_HOST : 'es.malcolm.local'
|
|
discovery.type : 'single-node'
|
|
discovery.zen.minimum_master_nodes : 1
|
|
cluster.routing.allocation.disk.threshold_enabled : 'false'
|
|
cluster.routing.allocation.node_initial_primaries_recoveries : 8
|
|
indices.query.bool.max_clause_count : 2048
|
|
path.repo : '/opt/elasticsearch/backup'
|
|
expose:
|
|
- 9200
|
|
ulimits:
|
|
memlock:
|
|
soft: -1
|
|
hard: -1
|
|
cap_add:
|
|
- IPC_LOCK
|
|
volumes:
|
|
- ./elasticsearch/elasticsearch.keystore:/usr/share/elasticsearch/config/elasticsearch.keystore:rw
|
|
- ./nginx/ca-trust:/usr/share/elasticsearch/ca-trust:ro
|
|
- ./elasticsearch:/usr/share/elasticsearch/data:delegated
|
|
- ./elasticsearch-backup:/opt/elasticsearch/backup:delegated
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9200"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 180s
|
|
kibana-helper:
|
|
image: malcolmnetsec/kibana-helper:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: kibana-helper
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *kibana-helper-variables
|
|
ELASTICSEARCH_URL : 'http://elasticsearch:9200'
|
|
KIBANA_URL : 'http://kibana:5601/kibana'
|
|
VIRTUAL_HOST : 'kibana-helper.malcolm.local'
|
|
ARKIME_INDEX_PATTERN : 'sessions2-*'
|
|
ARKIME_INDEX_PATTERN_ID : 'sessions2-*'
|
|
ARKIME_INDEX_TIME_FIELD : 'firstPacket'
|
|
CREATE_ES_ARKIME_SESSION_INDEX : 'true'
|
|
depends_on:
|
|
- elasticsearch
|
|
expose:
|
|
- 28991
|
|
volumes:
|
|
- ./index-management-policy.json:/data/index-management-policy.json:ro
|
|
healthcheck:
|
|
test: ["CMD", "supervisorctl", "status", "cron", "maps"]
|
|
interval: 60s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 30s
|
|
kibana:
|
|
image: malcolmnetsec/kibana-od:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: kibana
|
|
environment:
|
|
<< : *process-variables
|
|
ELASTICSEARCH_URL : 'http://elasticsearch:9200'
|
|
VIRTUAL_HOST : 'kibana.malcolm.local'
|
|
depends_on:
|
|
- elasticsearch
|
|
- kibana-helper
|
|
expose:
|
|
- 5601
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--silent", "--fail", "http://localhost:5601/kibana/api/status"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 210s
|
|
logstash:
|
|
image: malcolmnetsec/logstash-oss:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: logstash
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *logstash-variables
|
|
<< : *common-beats-variables
|
|
<< : *common-lookup-variables
|
|
ES_HOSTS : 'elasticsearch:9200'
|
|
LS_JAVA_OPTS : '-Xms2g -Xmx2g -Xss2m -Djava.security.egd=file:/dev/./urandom'
|
|
depends_on:
|
|
- elasticsearch
|
|
expose:
|
|
- 5044
|
|
- 9001
|
|
- 9600
|
|
volumes:
|
|
- ./logstash/certs/logstash.keystore:/usr/share/logstash/config/logstash.keystore:rw
|
|
- ./nginx/ca-trust:/usr/share/logstash/ca-trust:ro
|
|
- ./logstash/certs/ca.crt:/certs/ca.crt:ro
|
|
- ./logstash/certs/server.crt:/certs/server.crt:ro
|
|
- ./logstash/certs/server.key:/certs/server.key:ro
|
|
- ./cidr-map.txt:/usr/share/logstash/config/cidr-map.txt:ro
|
|
- ./host-map.txt:/usr/share/logstash/config/host-map.txt:ro
|
|
- ./net-map.json:/usr/share/logstash/config/net-map.json:ro
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 600s
|
|
filebeat:
|
|
image: malcolmnetsec/filebeat-oss:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: filebeat
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *nginx-variables
|
|
<< : *common-upload-variables
|
|
<< : *common-beats-variables
|
|
FILEBEAT_LOG_PATH : '/data/zeek/current'
|
|
FILEBEAT_NGINX_LOG_PATH : '/data/nginx'
|
|
FILEBEAT_LOG_CLEANUP_MINUTES : 180
|
|
FILEBEAT_ZIP_CLEANUP_MINUTES : 360
|
|
FILEBEAT_SCAN_FREQUENCY : '10s'
|
|
FILEBEAT_CLEAN_INACTIVE : '45m'
|
|
FILEBEAT_IGNORE_OLDER : '30m'
|
|
FILEBEAT_CLOSE_INACTIVE : '30s'
|
|
FILEBEAT_CLOSE_RENAMED : 'true'
|
|
FILEBEAT_CLOSE_REMOVED : 'true'
|
|
FILEBEAT_CLOSE_EOF : 'true'
|
|
FILEBEAT_CLEAN_REMOVED : 'true'
|
|
depends_on:
|
|
- logstash
|
|
volumes:
|
|
- nginx-log-path:/data/nginx:ro
|
|
- ./zeek-logs:/data/zeek
|
|
- ./filebeat/certs/ca.crt:/certs/ca.crt:ro
|
|
- ./filebeat/certs/client.crt:/certs/client.crt:ro
|
|
- ./filebeat/certs/client.key:/certs/client.key:ro
|
|
healthcheck:
|
|
test: ["CMD", "supervisorctl", "status", "filebeat"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 60s
|
|
arkime:
|
|
image: malcolmnetsec/arkime:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: arkime
|
|
env_file:
|
|
- ./auth.env
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *common-upload-variables
|
|
<< : *arkime-variables
|
|
ARKIME_VERSION : '2.7.1'
|
|
VIRTUAL_HOST : 'arkime.malcolm.local'
|
|
ES_HOST : 'elasticsearch'
|
|
ES_PORT : 9200
|
|
ES_MAX_SHARDS_PER_NODE : 2500
|
|
VIEWER : 'on'
|
|
WISE : 'on'
|
|
ulimits:
|
|
memlock:
|
|
soft: -1
|
|
hard: -1
|
|
depends_on:
|
|
- elasticsearch
|
|
expose:
|
|
- 8000
|
|
- 8005
|
|
- 8081
|
|
volumes:
|
|
- ./pcap:/data/pcap
|
|
- ./moloch-logs:/data/moloch/logs
|
|
- ./moloch-raw:/data/moloch/raw
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--silent", "--fail", "http://localhost:8005/_ns_/nstest.html"]
|
|
interval: 90s
|
|
timeout: 30s
|
|
retries: 3
|
|
start_period: 210s
|
|
zeek:
|
|
image: malcolmnetsec/zeek:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: zeek
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *common-upload-variables
|
|
<< : *zeek-variables
|
|
ulimits:
|
|
memlock:
|
|
soft: -1
|
|
hard: -1
|
|
depends_on:
|
|
- elasticsearch
|
|
volumes:
|
|
- ./pcap:/pcap
|
|
- ./zeek-logs/upload:/zeek/upload
|
|
- ./zeek-logs/extract_files:/zeek/extract_files
|
|
healthcheck:
|
|
test: ["CMD", "supervisorctl", "status", "pcap-zeek"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 60s
|
|
file-monitor:
|
|
image: malcolmnetsec/file-monitor:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: file-monitor
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *zeek-variables
|
|
VIRTUAL_HOST : 'file-monitor.malcolm.local'
|
|
expose:
|
|
- 3310
|
|
- 8440
|
|
volumes:
|
|
- ./zeek-logs/extract_files:/data/zeek/extract_files
|
|
- ./zeek-logs/current:/data/zeek/logs
|
|
- ./yara/rules:/yara-rules/custom:ro
|
|
healthcheck:
|
|
test: ["CMD", "supervisorctl", "status", "watcher", "logger"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 60s
|
|
pcap-capture:
|
|
image: malcolmnetsec/pcap-capture:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
network_mode: host
|
|
ulimits:
|
|
memlock:
|
|
soft: -1
|
|
hard: -1
|
|
cap_add:
|
|
- IPC_LOCK
|
|
- NET_ADMIN
|
|
- NET_RAW
|
|
- SYS_ADMIN
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *pcap-capture-variables
|
|
volumes:
|
|
- ./pcap/upload:/pcap
|
|
healthcheck:
|
|
test: ["CMD", "supervisorctl", "status"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 60s
|
|
pcap-monitor:
|
|
image: malcolmnetsec/pcap-monitor:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: pcapmon
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *common-upload-variables
|
|
ELASTICSEARCH_URL : 'http://elasticsearch:9200'
|
|
depends_on:
|
|
- elasticsearch
|
|
expose:
|
|
- 30441
|
|
volumes:
|
|
- ./zeek-logs:/zeek
|
|
- ./pcap:/pcap
|
|
healthcheck:
|
|
test: ["CMD", "supervisorctl", "status", "watch-upload", "pcap-publisher"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 90s
|
|
upload:
|
|
image: malcolmnetsec/file-upload:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: upload
|
|
env_file:
|
|
- ./auth.env
|
|
environment:
|
|
<< : *process-variables
|
|
SITE_NAME : 'Capture File and Log Archive Upload'
|
|
VIRTUAL_HOST : 'upload.malcolm.local'
|
|
depends_on:
|
|
- arkime
|
|
expose:
|
|
- 80
|
|
ports:
|
|
- "127.0.0.1:8022:22"
|
|
volumes:
|
|
- ./pcap/upload:/var/www/upload/server/php/chroot/files
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-qO-", "http://localhost"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 60s
|
|
htadmin:
|
|
image: malcolmnetsec/htadmin:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: htadmin
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *auth-variables
|
|
VIRTUAL_HOST : 'htadmin.malcolm.local'
|
|
expose:
|
|
- 80
|
|
volumes:
|
|
- ./htadmin/config.ini:/var/www/htadmin/config/config.ini:rw
|
|
- ./htadmin/metadata:/var/www/htadmin/config/metadata:rw
|
|
- ./nginx/htpasswd:/var/www/htadmin/config/htpasswd:rw
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--silent", "--fail", "http://localhost"]
|
|
interval: 60s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 60s
|
|
freq:
|
|
image: malcolmnetsec/freq:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: freq
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *common-lookup-variables
|
|
VIRTUAL_HOST : 'freq.malcolm.local'
|
|
expose:
|
|
- 10004
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--silent", "--fail", "http://localhost:10004"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 60s
|
|
name-map-ui:
|
|
image: malcolmnetsec/name-map-ui:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: name-map-ui
|
|
environment:
|
|
<< : *process-variables
|
|
VIRTUAL_HOST : 'name-map-ui.malcolm.local'
|
|
expose:
|
|
- 8080
|
|
volumes:
|
|
- ./cidr-map.txt:/var/www/html/maps/cidr-map.txt:ro
|
|
- ./host-map.txt:/var/www/html/maps/host-map.txt:ro
|
|
- ./net-map.json:/var/www/html/maps/net-map.json:rw
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--silent", "--fail", "http://localhost:8080/fpm-ping"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 60s
|
|
nginx-proxy:
|
|
image: malcolmnetsec/nginx-proxy:3.2.1
|
|
restart: "no"
|
|
stdin_open: false
|
|
tty: true
|
|
hostname: nginx-proxy
|
|
environment:
|
|
<< : *process-variables
|
|
<< : *auth-variables
|
|
<< : *nginx-variables
|
|
depends_on:
|
|
- arkime
|
|
- kibana
|
|
- upload
|
|
- htadmin
|
|
- name-map-ui
|
|
- file-monitor
|
|
ports:
|
|
- "0.0.0.0:443:443"
|
|
- "0.0.0.0:488:488"
|
|
- "0.0.0.0:5601:5601"
|
|
# - "0.0.0.0:9200:9200"
|
|
volumes:
|
|
- nginx-log-path:/var/log/nginx:rw
|
|
- ./nginx/nginx_ldap.conf:/etc/nginx/nginx_ldap.conf:ro
|
|
- ./nginx/htpasswd:/etc/nginx/.htpasswd:ro
|
|
- ./nginx/ca-trust:/etc/nginx/ca-trust:ro
|
|
- ./nginx/certs:/etc/nginx/certs:ro
|
|
- ./nginx/certs/dhparam.pem:/etc/nginx/dhparam/dhparam.pem:ro
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--insecure", "--silent", "https://localhost:443"]
|
|
interval: 30s
|
|
timeout: 15s
|
|
retries: 3
|
|
start_period: 120s
|
|
|
|
# shared named volume so filebeat can access nginx access logs
|
|
volumes:
|
|
nginx-log-path:
|