added Malcolm

Vagrant/resources/malcolm/moloch/scripts/bs4_remove_div.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+from bs4 import BeautifulSoup
+import argparse
+import os
+import sys
+
+###################################################################################################
+debug = False
+PY3 = (sys.version_info.major >= 3)
+scriptName = os.path.basename(__file__)
+scriptPath = os.path.dirname(os.path.realpath(__file__))
+origPath = os.getcwd()
+
+###################################################################################################
+if not PY3:
+  if hasattr(__builtins__, 'raw_input'): input = raw_input
+
+try:
+  FileNotFoundError
+except NameError:
+  FileNotFoundError = IOError
+
+###################################################################################################
+# print to stderr
+def eprint(*args, **kwargs):
+  print(*args, file=sys.stderr, **kwargs)
+
+###################################################################################################
+# convenient boolean argument parsing
+def str2bool(v):
+  if v.lower() in ('yes', 'true', 't', 'y', '1'):
+    return True
+  elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+    return False
+  else:
+    raise argparse.ArgumentTypeError('Boolean value expected.')
+
+###################################################################################################
+# main
+def main():
+  global debug
+
+  parser = argparse.ArgumentParser(description=scriptName, add_help=False, usage='{} <arguments>'.format(scriptName))
+  parser.add_argument('-v', '--verbose', dest='debug', type=str2bool, nargs='?', const=True, default=False, help="Verbose output")
+  parser.add_argument('-i', '--input', required=True, metavar='<STR>', type=str, help='Input file')
+  parser.add_argument('-o', '--output', required=True, metavar='<STR>', type=str, help='Output file')
+  parser.add_argument('-c', '--div-class', required=True, dest='divClass', metavar='<STR>', type=str, default='', help='div class to remove')
+  parser.add_argument('-p', '--parser', required=False, dest='parser', metavar='<STR>', type=str, default='html.parser', help='BeautifulSoup parser')
+  parser.add_argument('-e', '--encoding', required=False, dest='encoding', metavar='<STR>', type=str, default='utf-8', help='Encoding for output file')
+  try:
+    parser.error = parser.exit
+    args = parser.parse_args()
+  except SystemExit:
+    parser.print_help()
+    exit(2)
+
+  debug = args.debug
+  if debug:
+    eprint(os.path.join(scriptPath, scriptName))
+    eprint("Arguments: {}".format(sys.argv[1:]))
+    eprint("Arguments: {}".format(args))
+  else:
+    sys.tracebacklimit = 0
+
+  soup = BeautifulSoup(open(args.input), args.parser)
+  for div in soup.find_all("div", { 'class' : args.divClass }):
+    div.decompose()
+
+  with open(args.output, 'wb') as f:
+    f.write(soup.prettify(args.encoding))
+
+if __name__ == '__main__':
+  main()

Vagrant/resources/malcolm/moloch/scripts/initmoloch.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+rm -f /var/run/moloch/initialized /var/run/moloch/runwise
+
+echo "Giving Elasticsearch time to start..."
+/data/elastic_search_status.sh 2>&1 && echo "Elasticsearch is running!"
+
+# download and/or update geo updates
+$ARKIMEDIR/bin/moloch_update_geo.sh
+
+# start and wait patiently for WISE
+if [[ "$WISE" = "on" ]] ; then
+  touch /var/run/moloch/runwise
+  echo "Giving WISE time to start..."
+  sleep 5
+  until curl -sSf --output /dev/null "http://127.0.0.1:8081/fields?ver=1"
+  do
+      echo "Waiting for WISE to start"
+      sleep 1
+  done
+  echo "WISE is running!"
+  echo
+fi
+
+# initialize the contents of the Elasticearch database if it has never been initialized (ie., the users_v# table hasn't been created)
+if [[ $(curl -fs -XGET -H'Content-Type: application/json' "http://$ES_HOST:$ES_PORT/_cat/indices/users_v*" | wc -l) < 1 ]]; then
+
+  echo "Initializing Elasticsearch database..."
+
+	$ARKIMEDIR/db/db.pl http://$ES_HOST:$ES_PORT initnoprompt
+
+	# this password isn't going to be used by Arkime, nginx will do the auth instead
+	$ARKIMEDIR/bin/moloch_add_user.sh "${MALCOLM_USERNAME}" "${MALCOLM_USERNAME}" "ignored" --admin --webauthonly --webauth
+
+  # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S.
+  touch /tmp/not_a_packet.pcap
+  $ARKIMEDIR/bin/moloch-capture --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1
+  rm -f /tmp/not_a_packet.pcap
+
+  #set some default settings I want for moloch
+  curl -sS -H'Content-Type: application/json' -XPOST http://$ES_HOST:$ES_PORT/users_v7/user/$MALCOLM_USERNAME/_update -d "@$ARKIMEDIR/etc/user_settings.json"
+
+  echo -e "\nElasticsearch database initialized!\n"
+
+else
+  echo "Elasticsearch database previously initialized!"
+  echo
+
+  if /data/moloch-needs-upgrade.sh 2>&1; then
+    echo "Elasticsearch database needs to be upgraded for $ARKIME_VERSION!"
+    $ARKIMEDIR/db/db.pl http://$ES_HOST:$ES_PORT upgradenoprompt
+    echo "Elasticsearch database upgrade complete!"
+    echo
+
+  else
+    echo "Elasticsearch database is up-to-date for Arkime version $ARKIME_VERSION!"
+    echo
+
+  fi # if /data/moloch-needs-upgrade.sh
+fi # if/else Elasticsearch database initialized
+
+# increase Elasticsearch max shards per node from default if desired
+if [[ -n $ES_MAX_SHARDS_PER_NODE ]]; then
+  # see https://github.com/elastic/elasticsearch/issues/40803
+  curl -sS -H'Content-Type: application/json' -XPUT http://$ES_HOST:$ES_PORT/_cluster/settings -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$ES_MAX_SHARDS_PER_NODE\" } }"
+fi
+
+# before running viewer, call _refresh to make sure everything is available for search first
+curl -sS -XPOST http://$ES_HOST:$ES_PORT/_refresh
+
+touch /var/run/moloch/initialized
+
+# the (viewer|wise)_service.sh scripts will start/restart those processes

Vagrant/resources/malcolm/moloch/scripts/moloch-needs-upgrade.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+# this script returns:
+#   0 - an UPGRADE IS NEEDED for Arkime indices
+#   1 - an UPGRADE IS NOT NEEDED for Arkime indices
+RETURN_CODE=1
+
+set -e
+
+# see Arkime's db.pl (https://github.com/arkime/arkime/blob/master/db/db.pl) near the bottom for this list
+ declare -A ARKIME_INDEX_CURRENT_VERSIONS=(
+  [dstats_v]=dstats_v4
+  [fields_v]=fields_v3
+  [files_v]=files_v6
+  [hunts_v]=hunts_v2
+  [lookups_v]=lookups_v1
+  [queries_v]=queries_v3
+  [sequence_v]=sequence_v3
+  [stats_v]=stats_v4
+  [users_v]=users_v7
+)
+
+# get a list of all current indices and loop over them
+while read INDEX_NAME; do
+
+  # for each current index, check to see if it's one of the Arkime indices (prefixed by
+  # the key of ARKIME_INDEX_CURRENT_VERSIONS)
+  for INDEX_PREFIX in "${!ARKIME_INDEX_CURRENT_VERSIONS[@]}"; do
+    if [[ ${INDEX_NAME} = ${INDEX_PREFIX}* ]]; then
+
+      # if this is a Arkime index, make sure the version matches what we think it should
+      if [[ ${INDEX_NAME} != ${ARKIME_INDEX_CURRENT_VERSIONS[$INDEX_PREFIX]} ]]; then
+        RETURN_CODE=0
+        echo "${INDEX_NAME}:${ARKIME_INDEX_CURRENT_VERSIONS[$INDEX_PREFIX]}" 1>&2
+        break
+      fi # compare INDEX_NAME vs. full Arkime index name with version
+
+    fi # compare INDEX_NAME vs. INDEX_PREFIX
+  done # loop over ARKIME_INDEX_CURRENT_VERSIONS
+
+done <<<$(curl -fsS -H"Content-Type: application/json" -XGET "http://$ES_HOST:$ES_PORT/_cat/indices?v" | tail -n +2 | awk '{print $3}')
+
+exit $RETURN_CODE

Vagrant/resources/malcolm/moloch/scripts/moloch_update_geo.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+cd "${ARKIMEDIR:-/data/moloch}"/etc
+
+wget -nv --no-check-certificate -O ipv4-address-space.csv_new https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv && \
+  mv -f ipv4-address-space.csv_new ipv4-address-space.csv || \
+  rm -f ipv4-address-space.csv_new
+
+wget -nv -O oui.txt_new https://raw.githubusercontent.com/wireshark/wireshark/master/manuf && \
+  mv -f oui.txt_new oui.txt || \
+  rm -f oui.txt_new
+
+# MaxMind now requires a (free) license key to download the free versions of
+# their GeoIP databases. This should be provided as an environment variable.
+#   see https://dev.maxmind.com/geoip/geoipupdate/#Direct_Downloads
+#   see https://github.com/arkime/arkime/issues/1350
+#   see https://github.com/arkime/arkime/issues/1352
+if [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]; then
+  for DB in ASN Country City; do
+    curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" && \
+      tar xf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 && \
+      chmod 644 "GeoLite2-$DB.mmdb" && \
+      rm -f "GeoLite2-$DB.mmdb.tar.gz"
+  done
+fi

Vagrant/resources/malcolm/moloch/scripts/viewer_service.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+
+while true; do
+  if [[ -f /var/run/moloch/initialized && "$VIEWER" == "on" ]]; then
+    echo "Launch viewer..."
+    cd $ARKIMEDIR/viewer
+    $ARKIMEDIR/bin/node viewer.js -c $ARKIMEDIR/etc/config.ini | tee -a $ARKIMEDIR/logs/viewer.log 2>&1
+  fi
+  sleep 5
+done

Vagrant/resources/malcolm/moloch/scripts/wipemoloch.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+
+echo "Checking Elasticsearch..."
+/data/elastic_search_status.sh 2>&1 && echo "Elasticsearch is running!"
+
+#Wipe is the same initalize except it keeps users intact
+echo WIPE | /data/moloch/db/db.pl http://$ES_HOST:$ES_PORT wipe
+
+#this is a hacky way to get all of the parseable field definitions put into E.S.
+touch /tmp/not_a_packet.pcap
+$ARKIMEDIR/bin/moloch-capture --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1
+rm -f /tmp/not_a_packet.pcap

Vagrant/resources/malcolm/moloch/scripts/wise_service.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+
+while true; do
+  if [[ ("$WISE" == "on") && (-f /var/run/moloch/runwise) && (-f $ARKIMEDIR/etc/wise.ini) ]]; then
+    echo "Launch wise..."
+    pushd $ARKIMEDIR/wiseService >/dev/null 2>&1
+    $ARKIMEDIR/bin/node wiseService.js -c $ARKIMEDIR/etc/wise.ini
+    popd >/dev/null 2>&1
+  fi
+  sleep 5
+done