120 lines
4.9 KiB
Bash
Executable File
120 lines
4.9 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
|
|
|
|
if [ -z "$BASH_VERSION" ]; then
|
|
echo "Wrong interpreter, please run \"$0\" with bash"
|
|
exit 1
|
|
fi
|
|
|
|
[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath
|
|
[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname
|
|
[[ "$(uname -s)" = 'Darwin' ]] && GREP=ggrep || GREP=grep
|
|
if ! (type "$REALPATH" && type "$DIRNAME" && type "$GREP") > /dev/null; then
|
|
echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME and $GREP"
|
|
exit 1
|
|
fi
|
|
|
|
if docker-compose version >/dev/null 2>&1; then
|
|
DOCKER_COMPOSE_BIN=docker-compose
|
|
DOCKER_BIN=docker
|
|
elif $GREP -q Microsoft /proc/version && docker-compose.exe version >/dev/null 2>&1; then
|
|
DOCKER_COMPOSE_BIN=docker-compose.exe
|
|
DOCKER_BIN=docker.exe
|
|
fi
|
|
|
|
if [ "$1" ]; then
|
|
CONFIG_FILE="$1"
|
|
DOCKER_COMPOSE_COMMAND="$DOCKER_COMPOSE_BIN -f "$CONFIG_FILE""
|
|
shift # use remainder of arguments for services
|
|
else
|
|
CONFIG_FILE="docker-compose.yml"
|
|
DOCKER_COMPOSE_COMMAND="$DOCKER_COMPOSE_BIN"
|
|
fi
|
|
|
|
function filesize_in_image() {
|
|
FILESPEC="$2"
|
|
IMAGE="$($GREP -P "^\s+image:.*$1" docker-compose-standalone.yml | awk '{print $2}')"
|
|
$DOCKER_BIN run --rm --entrypoint /bin/sh "$IMAGE" -c "stat --printf='%s' \"$FILESPEC\" 2>/dev/null || stat -c '%s' \"$FILESPEC\" 2>/dev/null"
|
|
}
|
|
|
|
# force-navigate to Malcolm base directory (parent of scripts/ directory)
|
|
SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))"
|
|
pushd "$SCRIPT_PATH/.." >/dev/null 2>&1
|
|
|
|
# make sure docker is installed, at this point it's required
|
|
if ! $DOCKER_BIN info >/dev/null 2>&1 ; then
|
|
echo "Docker is not installed, or not runable as this user."
|
|
echo "Install docker (install.py may help with that) and try again later."
|
|
exit 1
|
|
fi
|
|
|
|
# make sure docker-compose is installed, at this point it's required
|
|
if ! $DOCKER_COMPOSE_BIN version >/dev/null 2>&1 ; then
|
|
echo "Docker Compose is not installed, or not runable as this user."
|
|
echo "Install docker-compose (install.py may help with that) and try again later."
|
|
exit 1
|
|
fi
|
|
|
|
unset CONFIRMATION
|
|
read -p "Malcolm Docker images will now be built and/or pulled, force full clean (non-cached) rebuild [y/N]? " CONFIRMATION
|
|
CONFIRMATION=${CONFIRMATION:-N}
|
|
|
|
BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
|
MALCOLM_VERSION="$($GREP -P "^\s+image:\s*malcolm" "$CONFIG_FILE" | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)"
|
|
VCS_REVISION="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)"
|
|
|
|
[[ ! -f ./auth.env ]] && touch ./auth.env
|
|
|
|
# MaxMind now requires a (free) license key to download the free versions of their GeoIP databases.
|
|
if [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]; then
|
|
# prefer a local environment variable
|
|
MAXMIND_API_KEY="$MAXMIND_GEOIP_DB_LICENSE_KEY"
|
|
else
|
|
# but default to what they have saved in the docker-compose YML file
|
|
MAXMIND_API_KEY="$($GREP -P "^\s*MAXMIND_GEOIP_DB_LICENSE_KEY\s*:\s" "$CONFIG_FILE" | cut -d: -f2 | tr -d '[:space:]'\'\" | head -n 1)"
|
|
fi
|
|
|
|
# for some debug branches this may be used to download artifacts from github
|
|
if [ ${#GITHUB_OAUTH_TOKEN} -gt 1 ]; then
|
|
# prefer a local environment variable
|
|
GITHUB_TOKEN="$GITHUB_OAUTH_TOKEN"
|
|
else
|
|
# nope
|
|
GITHUB_TOKEN="0"
|
|
fi
|
|
|
|
if [[ $CONFIRMATION =~ ^[Yy] ]]; then
|
|
$DOCKER_COMPOSE_COMMAND build --force-rm --no-cache --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg GITHUB_OAUTH_TOKEN="$GITHUB_TOKEN" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
|
|
else
|
|
$DOCKER_COMPOSE_COMMAND build --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg GITHUB_OAUTH_TOKEN="$GITHUB_TOKEN" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
|
|
fi
|
|
|
|
# we're going to do some validation that some things got pulled/built correctly
|
|
FILES_IN_IMAGES=(
|
|
"/usr/share/filebeat/filebeat.yml;filebeat-oss"
|
|
"/var/lib/clamav/main.cvd;file-monitor"
|
|
"/var/lib/clamav/daily.cvd;file-monitor"
|
|
"/var/lib/clamav/bytecode.cvd;file-monitor"
|
|
"/var/www/upload/js/jquery.fileupload.js;file-upload"
|
|
"/opt/freq_server/freq_server.py;freq"
|
|
"/var/www/htadmin/index.php;htadmin"
|
|
"/usr/share/logstash/config/oui-logstash.txt;logstash"
|
|
"/etc/ip_protocol_numbers.yaml;logstash"
|
|
"/etc/ja3.yaml;logstash"
|
|
"/data/moloch/etc/GeoLite2-ASN.mmdb;arkime"
|
|
"/data/moloch/etc/GeoLite2-Country.mmdb;arkime"
|
|
"/data/moloch/etc/ipv4-address-space.csv;arkime"
|
|
"/data/moloch/etc/oui.txt;arkime"
|
|
"/data/moloch/bin/moloch-capture;arkime"
|
|
"/var/www/html/list.min.js;name-map-ui"
|
|
"/var/www/html/jquery.min.js;name-map-ui"
|
|
"/opt/zeek/bin/zeek;zeek"
|
|
"/opt/spicy/lib/libspicy.so;zeek"
|
|
)
|
|
for i in ${FILES_IN_IMAGES[@]}; do
|
|
FILE="$(echo "$i" | cut -d';' -f1)"
|
|
IMAGE="$(echo "$i" | cut -d';' -f2)"
|
|
(( "$(filesize_in_image $IMAGE "$FILE")" > 0 )) || { echo "Failed to create \"$FILE\" in \"$IMAGE\""; exit 1; }
|
|
done
|