131 lines
5.9 KiB
Bash
Executable File
131 lines
5.9 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
|
|
|
|
set -euo pipefail
|
|
shopt -s nocasematch
|
|
|
|
if [[ -n $ELASTICSEARCH_URL ]]; then
|
|
ES_URL="$ELASTICSEARCH_URL"
|
|
elif [[ -n $ES_HOST ]] && [[ -n $ES_PORT ]]; then
|
|
ES_URL="http://$ES_HOST:$ES_PORT"
|
|
else
|
|
ES_URL="http://elasticsearch:9200"
|
|
fi
|
|
|
|
if [[ -n $KIBANA_URL ]]; then
|
|
KIB_URL="$KIBANA_URL"
|
|
elif [[ -n $KIBANA_HOST ]] && [[ -n $KIBANA_PORT ]]; then
|
|
KIB_URL="http://$KIBANA_HOST:$KIBANA_PORT"
|
|
else
|
|
KIB_URL="http://kibana:5601/kibana"
|
|
fi
|
|
|
|
INDEX_PATTERN=${ARKIME_INDEX_PATTERN:-"sessions2-*"}
|
|
INDEX_PATTERN_ID=${ARKIME_INDEX_PATTERN_ID:-"sessions2-*"}
|
|
INDEX_TIME_FIELD=${ARKIME_INDEX_TIME_FIELD:-"firstPacket"}
|
|
|
|
INDEX_POLICY_FILE="/data/init/index-management-policy.json"
|
|
INDEX_POLICY_FILE_HOST="/data/index-management-policy.json"
|
|
ZEEK_TEMPLATE_FILE="/data/init/zeek_template.json"
|
|
ZEEK_TEMPLATE_FILE_ORIG="/data/zeek_template.json"
|
|
INDEX_POLICY_NAME=${ISM_POLICY_NAME:-"session_index_policy"}
|
|
|
|
# is the argument to automatically create this index enabled?
|
|
if [[ "$CREATE_ES_ARKIME_SESSION_INDEX" = "true" ]] ; then
|
|
|
|
# give Elasticsearch time to start before configuring Kibana
|
|
/data/elastic_search_status.sh >/dev/null 2>&1
|
|
|
|
# is the kibana process server up and responding to requests?
|
|
if curl -L --silent --output /dev/null --fail -XGET "$KIB_URL/api/status" ; then
|
|
|
|
# have we not not already created the index pattern?
|
|
if ! curl -L --silent --output /dev/null --fail -XGET "$KIB_URL/api/saved_objects/index-pattern/$INDEX_PATTERN_ID" ; then
|
|
|
|
echo "Elasticsearch is running! Setting up index management policies..."
|
|
|
|
# register the repo location for elasticsearch snapshots
|
|
/data/register-elasticsearch-snapshot-repo.sh
|
|
|
|
# tweak the sessions template (sessions2-* zeek template file) to use the index management policy
|
|
if [[ -f "$INDEX_POLICY_FILE_HOST" ]] && (( $(jq length "$INDEX_POLICY_FILE_HOST") > 0 )); then
|
|
# user has provided a file for index management, use it
|
|
cp "$INDEX_POLICY_FILE_HOST" "$INDEX_POLICY_FILE"
|
|
INDEX_POLICY_NAME="$(cat "$INDEX_POLICY_FILE" | jq '..|objects|.policy_id//empty' | tr -d '"')"
|
|
|
|
else
|
|
# need to generate index management file based on environment variables
|
|
/data/elastic_index_policy_create.py \
|
|
--policy "$INDEX_POLICY_NAME" \
|
|
--index-pattern "$INDEX_PATTERN" \
|
|
--priority 100 \
|
|
--snapshot ${ISM_SNAPSHOT_AGE:-"0"} \
|
|
--cold ${ISM_COLD_AGE:-"0"} \
|
|
--close ${ISM_CLOSE_AGE:-"0"} \
|
|
--delete ${ISM_DELETE_AGE:-"0"} \
|
|
> "$INDEX_POLICY_FILE"
|
|
fi
|
|
|
|
if [[ -f "$INDEX_POLICY_FILE" ]]; then
|
|
# make API call to define index management policy
|
|
# https://opendistro.github.io/for-elasticsearch-docs/docs/ism/api/#create-policy
|
|
curl -w "\n" -L --silent --output /dev/null --show-error -XPUT -H "Content-Type: application/json" "$ES_URL/_opendistro/_ism/policies/$INDEX_POLICY_NAME" -d "@$INDEX_POLICY_FILE"
|
|
|
|
if [[ -f "$ZEEK_TEMPLATE_FILE_ORIG" ]]; then
|
|
# insert opendistro.index_state_management.policy_id into index template settings: will be
|
|
# imported by kibana-create-moloch-sessions-index.sh
|
|
cat "$ZEEK_TEMPLATE_FILE_ORIG" | jq ".settings += {\"opendistro.index_state_management.policy_id\": \"$INDEX_POLICY_NAME\"}" > "$ZEEK_TEMPLATE_FILE"
|
|
fi
|
|
fi
|
|
|
|
echo "Importing zeek_template..."
|
|
|
|
if [[ -f "$ZEEK_TEMPLATE_FILE_ORIG" ]] && [[ ! -f "$ZEEK_TEMPLATE_FILE" ]]; then
|
|
cp "$ZEEK_TEMPLATE_FILE_ORIG" "$ZEEK_TEMPLATE_FILE"
|
|
fi
|
|
|
|
# load zeek_template containing zeek field type mappings (merged from /data/zeek_template.json to /data/init/zeek_template.json in kibana_helpers.sh on startup)
|
|
curl -w "\n" -sSL --fail -XPOST -H "Content-Type: application/json" \
|
|
"$ES_URL/_template/zeek_template?include_type_name=true" -d "@$ZEEK_TEMPLATE_FILE" 2>&1
|
|
|
|
echo "Importing index pattern..."
|
|
|
|
# From https://github.com/elastic/kibana/issues/3709
|
|
# Create index pattern
|
|
curl -w "\n" -sSL --fail -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \
|
|
"$KIB_URL/api/saved_objects/index-pattern/$INDEX_PATTERN_ID" \
|
|
-d"{\"attributes\":{\"title\":\"$INDEX_PATTERN\",\"timeFieldName\":\"$INDEX_TIME_FIELD\"}}" 2>&1
|
|
|
|
echo "Setting default index pattern..."
|
|
|
|
# Make it the default index
|
|
curl -w "\n" -sSL -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \
|
|
"$KIB_URL/api/kibana/settings/defaultIndex" \
|
|
-d"{\"value\":\"$INDEX_PATTERN_ID\"}"
|
|
|
|
echo "Importing Kibana saved objects..."
|
|
|
|
# install default dashboards, index patterns, etc.
|
|
for i in /opt/kibana/dashboards/*.json; do
|
|
curl -L --silent --output /dev/null --show-error -XPOST "$KIB_URL/api/kibana/dashboards/import?force=true" -H 'kbn-xsrf:true' -H 'Content-type:application/json' -d "@$i"
|
|
done
|
|
|
|
# set dark theme
|
|
curl -L --silent --output /dev/null --show-error -XPOST "$KIB_URL/api/kibana/settings/theme:darkMode" -H 'kbn-xsrf:true' -H 'Content-type:application/json' -d '{"value":true}'
|
|
|
|
# set default query time range
|
|
curl -L --silent --output /dev/null --show-error -XPOST "$KIB_URL/api/kibana/settings" -H 'kbn-xsrf:true' -H 'Content-type:application/json' -d \
|
|
'{"changes":{"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\",\n \"mode\": \"quick\"}"}}'
|
|
|
|
# turn off telemetry
|
|
curl -L --silent --output /dev/null --show-error -XPOST "$KIB_URL/api/telemetry/v2/optIn" -H 'kbn-xsrf:true' -H 'Content-type:application/json' -d '{"enabled":false}'
|
|
|
|
# pin filters by default
|
|
curl -L --silent --output /dev/null --show-error -XPOST "$KIB_URL/api/kibana/settings/filters:pinnedByDefault" -H 'kbn-xsrf:true' -H 'Content-type:application/json' -d '{"value":true}'
|
|
|
|
echo "Kibana saved objects import complete!"
|
|
fi
|
|
fi
|
|
fi
|