diff --git a/Vagrant/bootstrap.sh b/Vagrant/bootstrap.sh index 697a83d..53deca6 100644 --- a/Vagrant/bootstrap.sh +++ b/Vagrant/bootstrap.sh @@ -1,8 +1,8 @@ #! /bin/bash export DEBIAN_FRONTEND=noninteractive -echo "apt-fast apt-fast/maxdownloads string 10" | debconf-set-selections; -echo "apt-fast apt-fast/dlflag boolean true" | debconf-set-selections; +echo "apt-fast apt-fast/maxdownloads string 10" | debconf-set-selections +echo "apt-fast apt-fast/dlflag boolean true" | debconf-set-selections sed -i "2ideb mirror://mirrors.ubuntu.com/mirrors.txt bionic main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt bionic-updates main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt bionic-backports main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt bionic-security main restricted universe multiverse" /etc/apt/sources.list @@ -12,7 +12,7 @@ apt_install_prerequisites() { add-apt-repository -y ppa:apt-fast/stable # Add repository for yq add-apt-repository -y ppa:rmescandon/yq -　# Add repository for suricata + 　# Add repository for suricata add-apt-repository -y ppa:oisf/suricata-stable # Install prerequisites and useful tools echo "[$(date +%H:%M:%S)]: Running apt-get clean..." @@ -37,15 +37,14 @@ modify_motd() { } test_prerequisites() { - for package in jq whois build-essential git docker docker-compose unzip yq - do + for package in jq whois build-essential git docker docker-compose unzip yq; do echo "[$(date +%H:%M:%S)]: [TEST] Validating that $package is correctly installed..." # Loop through each package using dpkg - if ! dpkg -S $package > /dev/null; then + if ! dpkg -S $package >/dev/null; then # If which returns a non-zero return code, try to re-install the package echo "[-] $package was not found. Attempting to reinstall." apt-get -qq update && apt-get install -y $package - if ! which $package > /dev/null; then + if ! which $package >/dev/null; then # If the reinstall fails, give up echo "[X] Unable to install $package even after a retry. Exiting." exit 1 @@ -64,8 +63,8 @@ fix_eth1_static_ip() { fi if [ -f /sys/class/net/eth2/address ]; then if [ "$(cat /sys/class/net/eth2/address)" == "00:50:56:a3:b1:c4" ]; then - echo "[*] Using ESXi, no need to change anything" - return 0 + echo "[*] Using ESXi, no need to change anything" + return 0 fi fi # There's a fun issue where dhclient keeps messing with eth1 despite the fact @@ -73,7 +72,7 @@ fix_eth1_static_ip() { echo -e 'interface "eth1" { send host-name = gethostname(); send dhcp-requested-address 192.168.38.105; - }' >> /etc/dhcp/dhclient.conf + }' >>/etc/dhcp/dhclient.conf netplan apply # Fix eth1 if the IP isn't set correctly ETH1_IP=$(ip -4 addr show eth1 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') @@ -90,6 +89,12 @@ fix_eth1_static_ip() { exit 1 fi fi + + # Make sure we do have a DNS resolution + while true; do + if [ "$(dig +short @8.8.8.8 github.com)" ]; then break; fi + sleep 1 + done } install_splunk() { @@ -99,9 +104,9 @@ install_splunk() { else echo "[$(date +%H:%M:%S)]: Installing Splunk..." # Get download.splunk.com into the DNS cache. Sometimes resolution randomly fails during wget below - dig @8.8.8.8 download.splunk.com > /dev/null - dig @8.8.8.8 splunk.com > /dev/null - dig @8.8.8.8 www.splunk.com > /dev/null + dig @8.8.8.8 download.splunk.com >/dev/null + dig @8.8.8.8 splunk.com >/dev/null + dig @8.8.8.8 www.splunk.com >/dev/null # Try to resolve the latest version of Splunk by parsing the HTML on the downloads page echo "[$(date +%H:%M:%S)]: Attempting to autoresolve the latest version of Splunk..." @@ -131,15 +136,15 @@ install_splunk() { /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/asn-lookup-generator_101.tgz -auth 'admin:changeme' /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/lookup-file-editor_331.tgz -auth 'admin:changeme' /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/splunk-add-on-for-zeek-aka-bro_400.tgz -auth 'admin:changeme' - /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/force-directed-app-for-splunk_200.tgz -auth 'admin:changeme' - /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/punchcard-custom-visualization_130.tgz -auth 'admin:changeme' - /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/sankey-diagram-custom-visualization_130.tgz -auth 'admin:changeme' + /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/force-directed-app-for-splunk_200.tgz -auth 'admin:changeme' + /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/punchcard-custom-visualization_130.tgz -auth 'admin:changeme' + /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/sankey-diagram-custom-visualization_130.tgz -auth 'admin:changeme' /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/link-analysis-app-for-splunk_161.tgz -auth 'admin:changeme' - /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/threathunting_141.tgz -auth 'admin:changeme' + /opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/threathunting_141.tgz -auth 'admin:changeme' - # Uncomment the following block to install BOTSv2 + # Uncomment the following block to install BOTSv2 # Thanks to @MHaggis for this addition! - # It is recommended to only uncomment the attack-only dataset comment block. + # It is recommended to only uncomment the attack-only dataset comment block. # You may also link to the full dataset which is ~12GB if you prefer. # More information on BOTSv2 can be found at https://github.com/splunk/botsv2 @@ -179,7 +184,7 @@ install_splunk() { # echo "[$(date +%H:%M:%S)]: Extracting to Splunk Apps directory" # tar zxvf botsv2_data_set.tgz /opt/splunk/etc/apps ### FULL DATASET COMMENT BLOCK ENDS ### - + ### BOTSv2 COMMENT BLOCK ENDS ### # Add custom Macro definitions for ThreatHunting App @@ -192,7 +197,7 @@ install_splunk() { rm /opt/splunk/etc/apps/force_directed_viz/default/savedsearches.conf # Add a Splunk TCP input on port 9997 - echo -e "[splunktcp://9997]\nconnection_host = ip" > /opt/splunk/etc/apps/search/local/inputs.conf + echo -e "[splunktcp://9997]\nconnection_host = ip" >/opt/splunk/etc/apps/search/local/inputs.conf # Add props.conf and transforms.conf cp /vagrant/resources/splunk_server/props.conf /opt/splunk/etc/apps/search/local/ cp /vagrant/resources/splunk_server/transforms.conf /opt/splunk/etc/apps/search/local/ @@ -204,7 +209,7 @@ install_splunk() { echo "[$(date +%H:%M:%S)]: Disabling the Splunk tour prompt..." touch /opt/splunk/etc/.ui_login mkdir -p /opt/splunk/etc/users/admin/search/local - echo -e "[search-tour]\nviewed = 1" > /opt/splunk/etc/system/local/ui-tour.conf + echo -e "[search-tour]\nviewed = 1" >/opt/splunk/etc/system/local/ui-tour.conf # Source: https://answers.splunk.com/answers/660728/how-to-disable-the-modal-pop-up-help-us-to-improve.html echo '[general] render_version_messages = 0 @@ -213,7 +218,7 @@ dismissedInstrumentationOptInVersion = 1 [general_default] hideInstrumentationOptInModal = 1 showWhatsNew = 0 -notification_python_3_impact = false' > /opt/splunk/etc/system/local/user-prefs.conf +notification_python_3_impact = false' >/opt/splunk/etc/system/local/user-prefs.conf echo '[general] render_version_messages = 0 hideInstrumentationOptInModal = 1 @@ -221,12 +226,12 @@ dismissedInstrumentationOptInVersion = 1 [general_default] hideInstrumentationOptInModal = 1 showWhatsNew = 0 -notification_python_3_impact = false' > /opt/splunk/etc/apps/user-prefs/local/user-prefs.conf - # Disable the instrumentation popup - echo -e "showOptInModal = 0\noptInVersionAcknowledged = 4" >> /opt/splunk/etc/apps/splunk_instrumentation/local/telemetry.conf +notification_python_3_impact = false' >/opt/splunk/etc/apps/user-prefs/local/user-prefs.conf + # Disable the instrumentation popup + echo -e "showOptInModal = 0\noptInVersionAcknowledged = 4" >>/opt/splunk/etc/apps/splunk_instrumentation/local/telemetry.conf # Enable SSL Login for Splunk - echo -e "[settings]\nenableSplunkWebSSL = true" > /opt/splunk/etc/system/local/web.conf + echo -e "[settings]\nenableSplunkWebSSL = true" >/opt/splunk/etc/system/local/web.conf # Reboot Splunk to make changes take effect /opt/splunk/bin/splunk restart /opt/splunk/bin/splunk enable boot-start @@ -241,8 +246,8 @@ install_fleet() { echo "[$(date +%H:%M:%S)]: Fleet is already installed" else echo "[$(date +%H:%M:%S)]: Installing Fleet..." - echo -e "\n127.0.0.1 kolide" >> /etc/hosts - echo -e "\n127.0.0.1 logger" >> /etc/hosts + echo -e "\n127.0.0.1 kolide" >>/etc/hosts + echo -e "\n127.0.0.1 logger" >>/etc/hosts cd /opt && git clone https://github.com/kolide/kolide-quickstart.git cd /opt/kolide-quickstart || echo "Something went wrong while trying to clone the kolide-quickstart repository" cp /vagrant/resources/fleet/server.* . @@ -294,8 +299,8 @@ import_osquery_config_into_fleet() { # Use fleetctl to import YAML files fleetctl apply -f osquery-configuration/Fleet/Endpoints/MacOS/osquery.yaml fleetctl apply -f osquery-configuration/Fleet/Endpoints/Windows/osquery.yaml - for pack in osquery-configuration/Fleet/Endpoints/packs/*.yaml - do fleetctl apply -f "$pack" + for pack in osquery-configuration/Fleet/Endpoints/packs/*.yaml; do + fleetctl apply -f "$pack" done # Add Splunk monitors for Fleet @@ -313,7 +318,7 @@ install_zeek() { SPLUNK_SURICATA_SOURCETYPE='json_suricata' sh -c "echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_18.04/ /' > /etc/apt/sources.list.d/security:zeek.list" wget -nv https://download.opensuse.org/repositories/security:zeek/xUbuntu_18.04/Release.key -O /tmp/Release.key - apt-key add - < /tmp/Release.key + apt-key add - > /opt/zeek/share/zeek/site/local.zeek + ' >>/opt/zeek/share/zeek/site/local.zeek # Configure Zeek crudini --del $NODECFG zeek @@ -367,23 +372,23 @@ install_zeek() { mkdir -p $SPLUNK_ZEEK_JSON/local cp $SPLUNK_ZEEK_JSON/default/inputs.conf $SPLUNK_ZEEK_JSON/local/inputs.conf - crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR index zeek - crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR sourcetype bro:json - crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR whitelist '.*\.log$' - crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR blacklist '.*(communication|stderr)\.log$' - crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR disabled 0 - crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR index suricata - crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR sourcetype suricata:json - crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR whitelist 'eve.json' - crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR disabled 0 - crudini --set $SPLUNK_ZEEK_JSON/local/props.conf $SPLUNK_SURICATA_SOURCETYPE TRUNCATE 0 + crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR index zeek + crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR sourcetype bro:json + crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR whitelist '.*\.log$' + crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR blacklist '.*(communication|stderr)\.log$' + crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR disabled 0 + crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR index suricata + crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR sourcetype suricata:json + crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR whitelist 'eve.json' + crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR disabled 0 + crudini --set $SPLUNK_ZEEK_JSON/local/props.conf $SPLUNK_SURICATA_SOURCETYPE TRUNCATE 0 # Ensure permissions are correct and restart splunk chown -R splunk $SPLUNK_ZEEK_JSON /opt/splunk/bin/splunk restart # Verify that Zeek is running - if ! pgrep -f zeek > /dev/null; then + if ! pgrep -f zeek >/dev/null; then echo "Zeek attempted to start but is not running. Exiting" exit 1 fi @@ -407,7 +412,7 @@ install_suricata() { # update suricata signature sources suricata-update update-sources # disable protocol decode as it is duplicative of Zeek - echo re:protocol-command-decode >> /etc/suricata/disable.conf + echo re:protocol-command-decode >>/etc/suricata/disable.conf # enable et-open and attackdetection sources suricata-update enable-source et/open suricata-update enable-source ptresearch/attackdetection @@ -419,22 +424,21 @@ install_suricata() { sleep 3 # Verify that Suricata is running - if ! pgrep -f suricata > /dev/null; then + if ! pgrep -f suricata >/dev/null; then echo "Suricata attempted to start but is not running. Exiting" exit 1 fi } test_suricata_prerequisites() { - for package in suricata crudini - do + for package in suricata crudini; do echo "[$(date +%H:%M:%S)]: [TEST] Validating that $package is correctly installed..." # Loop through each package using dpkg - if ! dpkg -S $package > /dev/null; then + if ! dpkg -S $package >/dev/null; then # If which returns a non-zero return code, try to re-install the package echo "[-] $package was not found. Attempting to reinstall." apt-get clean && apt-get -qq update && apt-get install -y $package - if ! which $package > /dev/null; then + if ! which $package >/dev/null; then # If the reinstall fails, give up echo "[X] Unable to install $package even after a retry. Exiting." exit 1 @@ -451,7 +455,7 @@ install_guacamole() { apt-get -qq install -y libcairo2-dev libjpeg62-dev libpng-dev libossp-uuid-dev libfreerdp-dev libpango1.0-dev libssh2-1-dev libssh-dev tomcat8 tomcat8-admin tomcat8-user wget --progress=bar:force "http://apache.org/dyn/closer.cgi?action=download&filename=guacamole/1.0.0/source/guacamole-server-1.0.0.tar.gz" -O guacamole-server-1.0.0.tar.gz tar -xvf guacamole-server-1.0.0.tar.gz && cd guacamole-server-1.0.0 - ./configure &> /dev/null && make --quiet &> /dev/null && make --quiet install &> /dev/null || echo "[-] An error occurred while installing Guacamole." + ./configure &>/dev/null && make --quiet &>/dev/null && make --quiet install &>/dev/null || echo "[-] An error occurred while installing Guacamole." ldconfig cd /var/lib/tomcat8/webapps wget --progress=bar:force "http://apache.org/dyn/closer.cgi?action=download&filename=guacamole/1.0.0/binary/guacamole-1.0.0.war" -O guacamole.war @@ -470,7 +474,7 @@ install_guacamole() { postinstall_tasks() { # Include Splunk and Zeek in the PATH - echo export PATH="$PATH:/opt/splunk/bin:/opt/zeek/bin" >> ~/.bashrc + echo export PATH="$PATH:/opt/splunk/bin:/opt/zeek/bin" >>~/.bashrc # Ping DetectionLab server for usage statistics curl -A "DetectionLab-logger" "https://detectionlab.network/logger" }