trinitor/DetectionLab
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Idempodency fixes, bug fixes, make shellcheck happy

Browse Source
This commit is contained in:
Chris Long
2021-07-11 19:55:54 -07:00
parent 21301e7240
commit 41fb6c2158
3 changed files with 97 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
ESXi/ansible/roles/logger/tasks/main.yml
View File

@@ -125,11 +125,13 @@
fi
# There's a fun issue where dhclient keeps messing with eth1 despite the fact
# that eth1 has a static IP set. We workaround this by setting a static DHCP lease.
if ! grep 'interface "eth1"' /etc/dhcp/dhclient.conf; then
echo -e 'interface "eth1" {
send host-name = gethostname();
send dhcp-requested-address 192.168.38.105;
}' >>/etc/dhcp/dhclient.conf
netplan apply
fi
# Fix eth1 if the IP isn't set correctly
ETH1_IP=$(ip -4 addr show eth1 | grep -oP '(?<=inet\s)\d+(\.\d+){3}')
if [ "$ETH1_IP" != "192.168.38.105" ]; then
@@ -307,8 +309,12 @@
cd /opt || exit 1
echo "[$(date +%H:%M:%S)]: Installing Fleet..."
echo -e "\n127.0.0.1 fleet" >>/etc/hosts
echo -e "\n127.0.0.1 logger" >>/etc/hosts
if ! grep 'fleet' /etc/hosts; then
echo -e "\n127.0.0.1 fleet" >>/etc/hosts
fi
if ! grep 'logger' /etc/hosts; then
echo -e "\n127.0.0.1 logger" >>/etc/hosts
fi
# Set MySQL username and password, create fleet database
mysql -uroot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'fleet';"
@@ -327,11 +333,14 @@
cp /vagrant/resources/fleet/server.* /opt/fleet/
cp /vagrant/resources/fleet/fleet.service /etc/systemd/system/fleet.service
# Create directory for logs
mkdir /var/log/fleet
# Install the service file
/bin/systemctl enable fleet.service
/bin/systemctl start fleet.service
# Start Fleet
echo "[$(date +%H:%M:%S)]: Waiting for fleet service to start..."
while true; do
result=$(curl --silent -k https://127.0.0.1:8412)
@@ -494,6 +503,22 @@
echo "Suricata attempted to start but is not running. Exiting"
exit 1
fi
# Configure a logrotate policy for Suricata
cat >/etc/logrotate.d/suricata <<EOF
/var/log/suricata/*.log /var/log/suricata/*.json
{
hourly
rotate 0
missingok
nocompress
size=500M
sharedscripts
postrotate
/bin/kill -HUP \`cat /var/run/suricata.pid 2>/dev/null\` 2>/dev/null || true
endscript
}
EOF
register: install_suricata
failed_when: "'error' in install_suricata.stderr"
@@ -504,7 +529,9 @@
become: yes
shell: |
echo "[$(date +%H:%M:%S)]: Installing Zeek..."
sh -c "echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_18.04/ /' > /etc/apt/sources.list.d/security:zeek.list"
if ! grep 'zeek' /etc/apt/sources.list.d/security:zeek.list; then
sh -c "echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_18.04/ /' > /etc/apt/sources.list.d/security:zeek.list"
fi
wget -nv https://download.opensuse.org/repositories/security:zeek/xUbuntu_18.04/Release.key -O /tmp/Release.key
apt-key add - </tmp/Release.key &>/dev/null

44
Vagrant/logger_bootstrap.sh
View File

@@ -7,7 +7,10 @@ if ! curl -s 169.254.169.254 --connect-timeout 2 >/dev/null; then
echo -e " eth1:\n dhcp4: true\n nameservers:\n addresses: [8.8.8.8,8.8.4.4]" >>/etc/netplan/01-netcfg.yaml
netplan apply
fi
sed -i 's/nameserver 127.0.0.53/nameserver 8.8.8.8/g' /etc/resolv.conf && chattr +i /etc/resolv.conf
if grep '127.0.0.53' /etc/resolv.conf; then
sed -i 's/nameserver 127.0.0.53/nameserver 8.8.8.8/g' /etc/resolv.conf && chattr +i /etc/resolv.conf
fi
# Source variables from logger_variables.sh
# shellcheck disable=SC1091
@@ -24,7 +27,9 @@ export DEBIAN_FRONTEND=noninteractive
echo "apt-fast apt-fast/maxdownloads string 10" | debconf-set-selections
echo "apt-fast apt-fast/dlflag boolean true" | debconf-set-selections
sed -i "2ideb mirror://mirrors.ubuntu.com/mirrors.txt bionic main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt bionic-updates main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt bionic-backports main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt bionic-security main restricted universe multiverse" /etc/apt/sources.list
if ! grep 'mirrors.ubuntu.com/mirrors.txt' /etc/apt/sources.list; then
sed -i "2ideb mirror://mirrors.ubuntu.com/mirrors.txt bionic main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt bionic-updates main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt bionic-backports main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt bionic-security main restricted universe multiverse" /etc/apt/sources.list
fi
apt_install_prerequisites() {
echo "[$(date +%H:%M:%S)]: Adding apt repositories..."
@@ -89,11 +94,14 @@ fix_eth1_static_ip() {
fi
# There's a fun issue where dhclient keeps messing with eth1 despite the fact
# that eth1 has a static IP set. We workaround this by setting a static DHCP lease.
echo -e 'interface "eth1" {
send host-name = gethostname();
send dhcp-requested-address 192.168.38.105;
}' >>/etc/dhcp/dhclient.conf
netplan apply
if ! grep 'interface "eth1"' /etc/dhcp/dhclient.conf; then
echo -e 'interface "eth1" {
send host-name = gethostname();
send dhcp-requested-address 192.168.38.105;
}' >>/etc/dhcp/dhclient.conf
netplan apply
fi
# Fix eth1 if the IP isn't set correctly
ETH1_IP=$(ip -4 addr show eth1 | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -1)
if [ "$ETH1_IP" != "192.168.38.105" ]; then
@@ -264,8 +272,12 @@ install_fleet_import_osquery_config() {
cd /opt || exit 1
echo "[$(date +%H:%M:%S)]: Installing Fleet..."
echo -e "\n127.0.0.1 fleet" >>/etc/hosts
echo -e "\n127.0.0.1 logger" >>/etc/hosts
if ! grep 'fleet' /etc/hosts; then
echo -e "\n127.0.0.1 fleet" >>/etc/hosts
fi
if ! grep 'logger' /etc/hosts; then
echo -e "\n127.0.0.1 logger" >>/etc/hosts
fi
# Set MySQL username and password, create fleet database
mysql -uroot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'fleet';"
@@ -284,11 +296,14 @@ install_fleet_import_osquery_config() {
cp /vagrant/resources/fleet/server.* /opt/fleet/
cp /vagrant/resources/fleet/fleet.service /etc/systemd/system/fleet.service
# Create directory for logs
mkdir /var/log/fleet
# Install the service file
/bin/systemctl enable fleet.service
/bin/systemctl start fleet.service
# Start Fleet
echo "[$(date +%H:%M:%S)]: Waiting for fleet service to start..."
while true; do
result=$(curl --silent -k https://127.0.0.1:8412)
@@ -340,7 +355,9 @@ install_zeek() {
echo "[$(date +%H:%M:%S)]: Installing Zeek..."
# Environment variables
NODECFG=/opt/zeek/etc/node.cfg
sh -c "echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_18.04/ /' > /etc/apt/sources.list.d/security:zeek.list"
if ! grep 'zeek' /etc/apt/sources.list.d/security:zeek.list; then
sh -c "echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_18.04/ /' > /etc/apt/sources.list.d/security:zeek.list"
fi
wget -nv https://download.opensuse.org/repositories/security:zeek/xUbuntu_18.04/Release.key -O /tmp/Release.key
apt-key add - </tmp/Release.key &>/dev/null
# Update APT repositories
@@ -501,6 +518,7 @@ install_suricata() {
exit 1
fi
# Configure a logrotate policy for Suricata
cat >/etc/logrotate.d/suricata <<EOF
/var/log/suricata/*.log /var/log/suricata/*.json
{
@@ -591,8 +609,6 @@ configure_splunk_inputs() {
# Ensure permissions are correct and restart splunk
chown -R splunk:splunk /opt/splunk/etc/apps/Splunk_TA_bro
/opt/splunk/bin/splunk restart
}
main() {
@@ -616,8 +632,8 @@ splunk_only() {
}
# Allow custom modes via CLI args
if [ ! -z $1 ]; then
eval $1
if [ -n "$1" ]; then
eval "$1"
else
main
fi

82
ci/build_machine_bootstrap.sh
View File

@@ -116,64 +116,56 @@ fi
# Recreate a barebones version of the build script so we have some sense of return codes
cat << 'EOF' > /opt/DetectionLab/build.sh
#! /usr/bin/env bash
build_vagrant_hosts() {
# Kick off builds for logger and dc
cd "$DL_DIR"/Vagrant || exit 1
for HOST in logger dc; do
(vagrant up $HOST &> "$DL_DIR/Vagrant/vagrant_up_$HOST.log" || vagrant reload $HOST --provision &> "$DL_DIR/Vagrant/vagrant_up_$HOST.log") &
declare ${HOST}_PID=$!
done
# We only have to wait for DC to create the domain before kicking off wef and win10 builds
DC_CREATION_TIMEOUT=30
MINUTES_PASSED=0
while ! grep 'I am domain joined!' "$DL_DIR/Vagrant/vagrant_up_dc.log" > /dev/null; do
# Kick off builds for logger and dc
cd "$DL_DIR"/Vagrant || exit 1
for HOST in logger dc; do
(vagrant up $HOST &>"$DL_DIR/Vagrant/vagrant_up_$HOST.log" || vagrant reload $HOST --provision &>"$DL_DIR/Vagrant/vagrant_up_$HOST.log") &
declare ${HOST}_PID=$!
done
# We only have to wait for DC to create the domain before kicking off wef and win10 builds
DC_CREATION_TIMEOUT=30
MINUTES_PASSED=0
while ! grep 'I am domain joined!' "$DL_DIR/Vagrant/vagrant_up_dc.log" >/dev/null; do
(echo >&2 "[$(date +%H:%M:%S)]: Waiting for DC to complete creation of the domain...")
sleep 60
((MINUTES_PAST += 1))
if [ $MINUTES_PAST -gt $DC_CREATION_TIMEOUT ]; then
((MINUTES_PASSED += 1))
if [ "$MINUTES_PASSED" -gt "$DC_CREATION_TIMEOUT" ]; then
(echo >&2 "Timed out waiting for DC to create the domain controller. Exiting.")
exit 1
fi
done;
# Kick off builds for wef and win10
cd "$DL_DIR"/Vagrant || exit 1
for HOST in wef win10; do
(vagrant up $HOST &> "$DL_DIR/Vagrant/vagrant_up_$HOST.log" || vagrant reload $HOST --provision &> "$DL_DIR/Vagrant/vagrant_up_$HOST.log") &
declare ${HOST}_PID=$!
done
# Wait for all the builds to finish
while ps -p $logger_PID > /dev/null || ps -p $dc_PID > /dev/null || ps -p $wef_PID > /dev/null || ps -p $win10_PID > /dev/null; do
(echo >&2 "[$(date +%H:%M:%S)]: Waiting for all of the builds to complete...")
sleep 60
done
for HOST in logger dc wef win10; do
if wait $HOST_PID; then # After this command, the return code gets set to what the return code of the PID was
(echo >&2 "$HOST was built successfully!")
else
(echo >&2 "Failed to bring up $HOST after a reload. Exiting")
exit 1
fi
done
fi
done
# Kick off builds for wef and win10
cd "$DL_DIR"/Vagrant || exit 1
for HOST in wef win10; do
(vagrant up $HOST &>>"$DL_DIR/Vagrant/vagrant_up_$HOST.log" || vagrant reload $HOST --provision &>>"$DL_DIR/Vagrant/vagrant_up_$HOST.log") &
declare ${HOST}_PID=$!
done
# Wait for all the builds to finish
# shellcheck disable=SC2154
while ps -p "$logger_PID" >/dev/null || ps -p "$dc_PID" >/dev/null || ps -p "$wef_PID" >/dev/null || ps -p "$win10_PID" >/dev/null; do
(echo >&2 "[$(date +%H:%M:%S)]: Waiting for all of the hosts to finish provisioning...")
sleep 60
done
for HOST in logger dc wef win10; do
if wait "$HOST_PID"; then # After this command, the return code gets set to what the return code of the PID was
(echo >&2 "[$(date +%H:%M:%S)]: $HOST was built successfully!")
else
(echo >&2 "Failed to bring up $HOST after a reload. Exiting")
exit 1
fi
done
}
main() {
# Get location of build.sh
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Build and Test Vagrant hosts
cd Vagrant
# Build and Test Vagrant hosts
cd Vagrant || exit 1
build_vagrant_hosts
/bin/bash $DL_DIR/Vagrant/post_build_checks.sh
/bin/bash "$DL_DIR/Vagrant/post_build_checks.sh"
}
main
exit 0
EOF