added Malcolm
This commit is contained in:
119
Vagrant/resources/malcolm/scripts/build.sh
Executable file
119
Vagrant/resources/malcolm/scripts/build.sh
Executable file
@@ -0,0 +1,119 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
|
||||
|
||||
if [ -z "$BASH_VERSION" ]; then
|
||||
echo "Wrong interpreter, please run \"$0\" with bash"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath
|
||||
[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname
|
||||
[[ "$(uname -s)" = 'Darwin' ]] && GREP=ggrep || GREP=grep
|
||||
if ! (type "$REALPATH" && type "$DIRNAME" && type "$GREP") > /dev/null; then
|
||||
echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME and $GREP"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if docker-compose version >/dev/null 2>&1; then
|
||||
DOCKER_COMPOSE_BIN=docker-compose
|
||||
DOCKER_BIN=docker
|
||||
elif $GREP -q Microsoft /proc/version && docker-compose.exe version >/dev/null 2>&1; then
|
||||
DOCKER_COMPOSE_BIN=docker-compose.exe
|
||||
DOCKER_BIN=docker.exe
|
||||
fi
|
||||
|
||||
if [ "$1" ]; then
|
||||
CONFIG_FILE="$1"
|
||||
DOCKER_COMPOSE_COMMAND="$DOCKER_COMPOSE_BIN -f "$CONFIG_FILE""
|
||||
shift # use remainder of arguments for services
|
||||
else
|
||||
CONFIG_FILE="docker-compose.yml"
|
||||
DOCKER_COMPOSE_COMMAND="$DOCKER_COMPOSE_BIN"
|
||||
fi
|
||||
|
||||
function filesize_in_image() {
|
||||
FILESPEC="$2"
|
||||
IMAGE="$($GREP -P "^\s+image:.*$1" docker-compose-standalone.yml | awk '{print $2}')"
|
||||
$DOCKER_BIN run --rm --entrypoint /bin/sh "$IMAGE" -c "stat --printf='%s' \"$FILESPEC\" 2>/dev/null || stat -c '%s' \"$FILESPEC\" 2>/dev/null"
|
||||
}
|
||||
|
||||
# force-navigate to Malcolm base directory (parent of scripts/ directory)
|
||||
SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))"
|
||||
pushd "$SCRIPT_PATH/.." >/dev/null 2>&1
|
||||
|
||||
# make sure docker is installed, at this point it's required
|
||||
if ! $DOCKER_BIN info >/dev/null 2>&1 ; then
|
||||
echo "Docker is not installed, or not runable as this user."
|
||||
echo "Install docker (install.py may help with that) and try again later."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# make sure docker-compose is installed, at this point it's required
|
||||
if ! $DOCKER_COMPOSE_BIN version >/dev/null 2>&1 ; then
|
||||
echo "Docker Compose is not installed, or not runable as this user."
|
||||
echo "Install docker-compose (install.py may help with that) and try again later."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
unset CONFIRMATION
|
||||
read -p "Malcolm Docker images will now be built and/or pulled, force full clean (non-cached) rebuild [y/N]? " CONFIRMATION
|
||||
CONFIRMATION=${CONFIRMATION:-N}
|
||||
|
||||
BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||
MALCOLM_VERSION="$($GREP -P "^\s+image:\s*malcolm" "$CONFIG_FILE" | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)"
|
||||
VCS_REVISION="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)"
|
||||
|
||||
[[ ! -f ./auth.env ]] && touch ./auth.env
|
||||
|
||||
# MaxMind now requires a (free) license key to download the free versions of their GeoIP databases.
|
||||
if [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]; then
|
||||
# prefer a local environment variable
|
||||
MAXMIND_API_KEY="$MAXMIND_GEOIP_DB_LICENSE_KEY"
|
||||
else
|
||||
# but default to what they have saved in the docker-compose YML file
|
||||
MAXMIND_API_KEY="$($GREP -P "^\s*MAXMIND_GEOIP_DB_LICENSE_KEY\s*:\s" "$CONFIG_FILE" | cut -d: -f2 | tr -d '[:space:]'\'\" | head -n 1)"
|
||||
fi
|
||||
|
||||
# for some debug branches this may be used to download artifacts from github
|
||||
if [ ${#GITHUB_OAUTH_TOKEN} -gt 1 ]; then
|
||||
# prefer a local environment variable
|
||||
GITHUB_TOKEN="$GITHUB_OAUTH_TOKEN"
|
||||
else
|
||||
# nope
|
||||
GITHUB_TOKEN="0"
|
||||
fi
|
||||
|
||||
if [[ $CONFIRMATION =~ ^[Yy] ]]; then
|
||||
$DOCKER_COMPOSE_COMMAND build --force-rm --no-cache --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg GITHUB_OAUTH_TOKEN="$GITHUB_TOKEN" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
|
||||
else
|
||||
$DOCKER_COMPOSE_COMMAND build --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg GITHUB_OAUTH_TOKEN="$GITHUB_TOKEN" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
|
||||
fi
|
||||
|
||||
# we're going to do some validation that some things got pulled/built correctly
|
||||
FILES_IN_IMAGES=(
|
||||
"/usr/share/filebeat/filebeat.yml;filebeat-oss"
|
||||
"/var/lib/clamav/main.cvd;file-monitor"
|
||||
"/var/lib/clamav/daily.cvd;file-monitor"
|
||||
"/var/lib/clamav/bytecode.cvd;file-monitor"
|
||||
"/var/www/upload/js/jquery.fileupload.js;file-upload"
|
||||
"/opt/freq_server/freq_server.py;freq"
|
||||
"/var/www/htadmin/index.php;htadmin"
|
||||
"/usr/share/logstash/config/oui-logstash.txt;logstash"
|
||||
"/etc/ip_protocol_numbers.yaml;logstash"
|
||||
"/etc/ja3.yaml;logstash"
|
||||
"/data/moloch/etc/GeoLite2-ASN.mmdb;arkime"
|
||||
"/data/moloch/etc/GeoLite2-Country.mmdb;arkime"
|
||||
"/data/moloch/etc/ipv4-address-space.csv;arkime"
|
||||
"/data/moloch/etc/oui.txt;arkime"
|
||||
"/data/moloch/bin/moloch-capture;arkime"
|
||||
"/var/www/html/list.min.js;name-map-ui"
|
||||
"/var/www/html/jquery.min.js;name-map-ui"
|
||||
"/opt/zeek/bin/zeek;zeek"
|
||||
"/opt/spicy/lib/libspicy.so;zeek"
|
||||
)
|
||||
for i in ${FILES_IN_IMAGES[@]}; do
|
||||
FILE="$(echo "$i" | cut -d';' -f1)"
|
||||
IMAGE="$(echo "$i" | cut -d';' -f2)"
|
||||
(( "$(filesize_in_image $IMAGE "$FILE")" > 0 )) || { echo "Failed to create \"$FILE\" in \"$IMAGE\""; exit 1; }
|
||||
done
|
||||
Reference in New Issue
Block a user