From fce294c7b06d5d0797848c0bfda883681217b110 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Thu, 29 Oct 2020 23:28:51 -0700 Subject: [PATCH 1/5] Fixing CircleCI Build Steps --- Vagrant/Vagrantfile | 3 +++ Vagrant/bootstrap.sh | 1 + ci/build_machine_bootstrap.sh | 23 ++++++++++++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Vagrant/Vagrantfile b/Vagrant/Vagrantfile index 74c12e2..056eeb8 100644 --- a/Vagrant/Vagrantfile +++ b/Vagrant/Vagrantfile @@ -79,6 +79,7 @@ Vagrant.configure("2") do |config| cfg.vm.provision "shell", inline: 'wevtutil el | Select-String -notmatch "Microsoft-Windows-LiveId" | Foreach-Object {wevtutil cl "$_"}', privileged: false cfg.vm.provision "shell", inline: "Set-SmbServerConfiguration -AuditSmb1Access $true -Force", privileged: false cfg.vm.provision "shell", inline: 'cscript c:\windows\system32\slmgr.vbs /dlv', privileged: false + cfg.vm.provision "shell", inline: 'Write-Host "DC Provisioning Complete!"', privileged: false cfg.vm.provider "vmware_desktop" do |v, override| v.vmx["displayname"] = "dc.windomain.local" @@ -146,6 +147,7 @@ Vagrant.configure("2") do |config| cfg.vm.provision "shell", inline: "Set-SmbServerConfiguration -AuditSmb1Access $true -Force", privileged: false cfg.vm.provision "shell", path: "scripts/install-microsoft-ata.ps1", privileged: false cfg.vm.provision "shell", inline: 'cscript c:\windows\system32\slmgr.vbs /dlv', privileged: false + cfg.vm.provision "shell", inline: 'Write-Host "WEF Provisioning Complete!"', privileged: false cfg.vm.provider "vmware_desktop" do |v, override| v.vmx["displayname"] = "wef.windomain.local" @@ -204,6 +206,7 @@ Vagrant.configure("2") do |config| cfg.vm.provision "shell", path: "scripts/install-velociraptor.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/install-autorunstowineventlog.ps1", privileged: false cfg.vm.provision "shell", inline: 'cscript c:\windows\system32\slmgr.vbs /dlv', privileged: false + cfg.vm.provision "shell", inline: 'Write-Host "WIN10 Provisioning Complete!"', privileged: false cfg.vm.provider "vmware_desktop" do |v, override| v.vmx["displayname"] = "win10.windomain.local" diff --git a/Vagrant/bootstrap.sh b/Vagrant/bootstrap.sh index 24bf7f5..6b3b32d 100644 --- a/Vagrant/bootstrap.sh +++ b/Vagrant/bootstrap.sh @@ -550,6 +550,7 @@ main() { install_zeek install_guacamole postinstall_tasks + echo "Logger Provisioning Complete!" } main diff --git a/ci/build_machine_bootstrap.sh b/ci/build_machine_bootstrap.sh index 3369c4e..bf786f3 100755 --- a/ci/build_machine_bootstrap.sh +++ b/ci/build_machine_bootstrap.sh @@ -113,4 +113,25 @@ fi # Start the build in a tmux session sn=tmuxsession tmux new-session -s "$sn" -d -tmux send-keys -t "$sn:0" 'cd /opt/DetectionLab/Vagrant && vagrant up | tee -a vagrant_up_all.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html; umount /mnt && /usr/local/bin/packet-block-storage-detach' Enter +tmux send-keys -t "$sn:0" 'for host in logger dc wef win10; do vagrant up $host | tee -a vagrant_up_$host.log; COMPLETE="$(grep -ci "$host Provisioning Complete" vagrant_up_$host.log)"; echo "Complete is $COMPLETE for $host" ; if [ "$COMPLETE" -lt 1 ]; then echo "failed" > /var/www/html/index.html; break; fi; done; ALL_SUCCESS="$(grep -ci "Provisioning Complete" vagrant_up_*.log | cut -d : -f 2 | awk '"'"'{ sum += $1 } END { print sum }'"'"')"; echo "All success is $ALL_SUCCESS"; if [ "$ALL_SUCCESS" -eq 4 ]; then echo "success" > /var/www/html/index.html; else echo "failed" > /var/www/html/index.html; fi' Enter + + +# Expanding the send-keys command +# Vagrant isn't great about return codes, so we'll hack this up for now +# for host in logger dc wef win10; +# do +# vagrant up $host | tee -a vagrant_up_$host.log +# COMPLETE="$(grep -ci "$host Provisioning Complete" vagrant_up_$host.log)" +# echo "Complete is $COMPLETE for $host" +# if [ "$COMPLETE" -lt 1 ]; then +# echo "failed" > /var/www/html/index.html +# break +# fi +# done +# ALL_SUCCESS="$(grep -ci "Provisioning Complete" vagrant_up_*.log | cut -d : -f 2 | awk '"'"'{ sum += $1 } END { print sum }'"'"')" +# echo "All success is $ALL_SUCCESS" +# if [ "$ALL_SUCCESS" -eq 4 ]; then +# echo "success" > /var/www/html/index.html +# else +# echo "failed" > /var/www/html/index.html +# fi \ No newline at end of file From 3df9a91becbc227cd627efb603c87eb8a657c616 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Sat, 31 Oct 2020 14:58:41 -0700 Subject: [PATCH 2/5] Simplify this --- Vagrant/Vagrantfile | 3 - Vagrant/bootstrap.sh | 1 - ci/build_machine_bootstrap.sh | 115 +++++++++++++++++++++++++++------- 3 files changed, 93 insertions(+), 26 deletions(-) diff --git a/Vagrant/Vagrantfile b/Vagrant/Vagrantfile index 056eeb8..74c12e2 100644 --- a/Vagrant/Vagrantfile +++ b/Vagrant/Vagrantfile @@ -79,7 +79,6 @@ Vagrant.configure("2") do |config| cfg.vm.provision "shell", inline: 'wevtutil el | Select-String -notmatch "Microsoft-Windows-LiveId" | Foreach-Object {wevtutil cl "$_"}', privileged: false cfg.vm.provision "shell", inline: "Set-SmbServerConfiguration -AuditSmb1Access $true -Force", privileged: false cfg.vm.provision "shell", inline: 'cscript c:\windows\system32\slmgr.vbs /dlv', privileged: false - cfg.vm.provision "shell", inline: 'Write-Host "DC Provisioning Complete!"', privileged: false cfg.vm.provider "vmware_desktop" do |v, override| v.vmx["displayname"] = "dc.windomain.local" @@ -147,7 +146,6 @@ Vagrant.configure("2") do |config| cfg.vm.provision "shell", inline: "Set-SmbServerConfiguration -AuditSmb1Access $true -Force", privileged: false cfg.vm.provision "shell", path: "scripts/install-microsoft-ata.ps1", privileged: false cfg.vm.provision "shell", inline: 'cscript c:\windows\system32\slmgr.vbs /dlv', privileged: false - cfg.vm.provision "shell", inline: 'Write-Host "WEF Provisioning Complete!"', privileged: false cfg.vm.provider "vmware_desktop" do |v, override| v.vmx["displayname"] = "wef.windomain.local" @@ -206,7 +204,6 @@ Vagrant.configure("2") do |config| cfg.vm.provision "shell", path: "scripts/install-velociraptor.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/install-autorunstowineventlog.ps1", privileged: false cfg.vm.provision "shell", inline: 'cscript c:\windows\system32\slmgr.vbs /dlv', privileged: false - cfg.vm.provision "shell", inline: 'Write-Host "WIN10 Provisioning Complete!"', privileged: false cfg.vm.provider "vmware_desktop" do |v, override| v.vmx["displayname"] = "win10.windomain.local" diff --git a/Vagrant/bootstrap.sh b/Vagrant/bootstrap.sh index 6b3b32d..24bf7f5 100644 --- a/Vagrant/bootstrap.sh +++ b/Vagrant/bootstrap.sh @@ -550,7 +550,6 @@ main() { install_zeek install_guacamole postinstall_tasks - echo "Logger Provisioning Complete!" } main diff --git a/ci/build_machine_bootstrap.sh b/ci/build_machine_bootstrap.sh index bf786f3..723fe44 100755 --- a/ci/build_machine_bootstrap.sh +++ b/ci/build_machine_bootstrap.sh @@ -110,28 +110,99 @@ if [ $BOXES_PRESENT -eq 1 ]; then sed -i 's#"detectionlab/win10"#"/mnt/windows_10_virtualbox.box"#g' /opt/DetectionLab/Vagrant/Vagrantfile fi +# Recreate a barebones version of the build script so we have some sense of return codes +cat << EOF > /opt/DetectionLab/Vagrant/build.sh +#! /bin/bash + +# Brings up a single host using Vagrant +vagrant_up_host() { + HOST="$1" + (echo >&2 "Attempting to bring up the $HOST host using Vagrant") + cd "$DL_DIR"/Vagrant || exit 1 + $(which vagrant) up "$HOST" --provider="$PROVIDER" &> "$DL_DIR/Vagrant/vagrant_up_$HOST.log" + echo "$?" +} + +# Attempts to reload and re-provision a host if the intial "vagrant up" fails +vagrant_reload_host() { + HOST="$1" + cd "$DL_DIR"/Vagrant || exit 1 + # Attempt to reload the host if the vagrant up command didn't exit cleanly + $(which vagrant) reload "$HOST" --provision >>"$DL_DIR/Vagrant/vagrant_up_$HOST.log" 2>&1 + echo "$?" +} + +# A series of checks to ensure important services are responsive after the build completes. +post_build_checks() { + # If the curl operation fails, we'll just leave the variable equal to 0 + # This is needed to prevent the script from exiting if the curl operation fails + SPLUNK_CHECK=$(curl -ks -m 2 https://192.168.38.105:8000/en-US/account/login?return_to=%2Fen-US%2F | grep -c 'This browser is not supported by Splunk' || echo "") + FLEET_CHECK=$(curl -ks -m 2 https://192.168.38.105:8412 | grep -c 'Kolide Fleet' || echo "") + ATA_CHECK=$(curl --fail --write-out "%{http_code}" -ks https://192.168.38.103 -m 2) + [[ $ATA_CHECK == 401 ]] && ATA_CHECK=1 + + BASH_MAJOR_VERSION=$(/bin/bash --version | grep 'GNU bash' | grep -oi version\.\.. | cut -d ' ' -f 2 | cut -d '.' -f 1) + # Associative arrays are only supported in bash 4 and up + if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then + declare -A SERVICES + SERVICES=(["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK" ["ms_ata"]="$ATA_CHECK") + for SERVICE in "${!SERVICES[@]}"; do + if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then + (echo >&2 "Warning: $SERVICE failed post-build tests and may not be functioning correctly.") + fi + done + else + if [ "$SPLUNK_CHECK" -lt 1 ]; then + (echo >&2 "Warning: Splunk failed post-build tests and may not be functioning correctly.") + fi + if [ "$FLEET_CHECK" -lt 1 ]; then + (echo >&2 "Warning: Fleet failed post-build tests and may not be functioning correctly.") + fi + if [ "$ATA_CHECK" -lt 1 ]; then + (echo >&2 "Warning: MS ATA failed post-build tests and may not be functioning correctly.") + fi + fi +} + +build_vagrant_hosts() { + LAB_HOSTS=("logger" "dc" "wef" "win10") + + # Vagrant up each box and attempt to reload one time if it fails + for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do + RET=$(vagrant_up_host "$VAGRANT_HOST") + if [ "$RET" -eq 0 ]; then + (echo >&2 "Good news! $VAGRANT_HOST was built successfully!") + fi + # Attempt to recover if the intial "vagrant up" fails + if [ "$RET" -ne 0 ]; then + (echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.") + (echo >&2 "Attempting to reload and reprovision the host...") + RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST") + if [ "$RETRY_STATUS" -eq 0 ]; then + (echo >&2 "Good news! $VAGRANT_HOST was built successfully after a reload!") + else + (echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.") + exit 1 + fi + fi + done +} + +main() { + # Get location of build.sh + # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within + DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + + # Build and Test Vagrant hosts + build_vagrant_hosts + post_build_checks +} + +main +exit 0 +EOF + # Start the build in a tmux session sn=tmuxsession tmux new-session -s "$sn" -d -tmux send-keys -t "$sn:0" 'for host in logger dc wef win10; do vagrant up $host | tee -a vagrant_up_$host.log; COMPLETE="$(grep -ci "$host Provisioning Complete" vagrant_up_$host.log)"; echo "Complete is $COMPLETE for $host" ; if [ "$COMPLETE" -lt 1 ]; then echo "failed" > /var/www/html/index.html; break; fi; done; ALL_SUCCESS="$(grep -ci "Provisioning Complete" vagrant_up_*.log | cut -d : -f 2 | awk '"'"'{ sum += $1 } END { print sum }'"'"')"; echo "All success is $ALL_SUCCESS"; if [ "$ALL_SUCCESS" -eq 4 ]; then echo "success" > /var/www/html/index.html; else echo "failed" > /var/www/html/index.html; fi' Enter - - -# Expanding the send-keys command -# Vagrant isn't great about return codes, so we'll hack this up for now -# for host in logger dc wef win10; -# do -# vagrant up $host | tee -a vagrant_up_$host.log -# COMPLETE="$(grep -ci "$host Provisioning Complete" vagrant_up_$host.log)" -# echo "Complete is $COMPLETE for $host" -# if [ "$COMPLETE" -lt 1 ]; then -# echo "failed" > /var/www/html/index.html -# break -# fi -# done -# ALL_SUCCESS="$(grep -ci "Provisioning Complete" vagrant_up_*.log | cut -d : -f 2 | awk '"'"'{ sum += $1 } END { print sum }'"'"')" -# echo "All success is $ALL_SUCCESS" -# if [ "$ALL_SUCCESS" -eq 4 ]; then -# echo "success" > /var/www/html/index.html -# else -# echo "failed" > /var/www/html/index.html -# fi \ No newline at end of file +tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html; umount /mnt && /usr/local/bin/packet-block-storage-detach' Enter From 9315f80370d8740a479fa9574ace5eb03820403d Mon Sep 17 00:00:00 2001 From: Chris Long Date: Sat, 31 Oct 2020 15:14:54 -0700 Subject: [PATCH 3/5] Fix path --- ci/build_machine_bootstrap.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/build_machine_bootstrap.sh b/ci/build_machine_bootstrap.sh index 723fe44..c596423 100755 --- a/ci/build_machine_bootstrap.sh +++ b/ci/build_machine_bootstrap.sh @@ -111,7 +111,7 @@ if [ $BOXES_PRESENT -eq 1 ]; then fi # Recreate a barebones version of the build script so we have some sense of return codes -cat << EOF > /opt/DetectionLab/Vagrant/build.sh +cat << EOF > /opt/DetectionLab/build.sh #! /bin/bash # Brings up a single host using Vagrant @@ -201,6 +201,7 @@ main() { main exit 0 EOF +chmod +x /opt/DetectionLab/build.sh # Start the build in a tmux session sn=tmuxsession From 1a84e03d476ef7e1efea99050a0808ddb14328b9 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Sat, 31 Oct 2020 16:30:00 -0700 Subject: [PATCH 4/5] Remove args --- ci/build_machine_bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_machine_bootstrap.sh b/ci/build_machine_bootstrap.sh index c596423..9a6f82b 100755 --- a/ci/build_machine_bootstrap.sh +++ b/ci/build_machine_bootstrap.sh @@ -206,4 +206,4 @@ chmod +x /opt/DetectionLab/build.sh # Start the build in a tmux session sn=tmuxsession tmux new-session -s "$sn" -d -tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html; umount /mnt && /usr/local/bin/packet-block-storage-detach' Enter +tmux send-keys -t "$sn:0" './build.sh && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html; umount /mnt && /usr/local/bin/packet-block-storage-detach' Enter From fa0bc1cbbd3f22582cda5de298e4618c3eded8a8 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Sat, 31 Oct 2020 17:32:51 -0700 Subject: [PATCH 5/5] EOF needs to be in quotes --- ci/build_machine_bootstrap.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/build_machine_bootstrap.sh b/ci/build_machine_bootstrap.sh index 9a6f82b..71408d5 100755 --- a/ci/build_machine_bootstrap.sh +++ b/ci/build_machine_bootstrap.sh @@ -111,7 +111,7 @@ if [ $BOXES_PRESENT -eq 1 ]; then fi # Recreate a barebones version of the build script so we have some sense of return codes -cat << EOF > /opt/DetectionLab/build.sh +cat << 'EOF' > /opt/DetectionLab/build.sh #! /bin/bash # Brings up a single host using Vagrant @@ -194,6 +194,7 @@ main() { DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Build and Test Vagrant hosts + cd Vagrant build_vagrant_hosts post_build_checks } @@ -206,4 +207,4 @@ chmod +x /opt/DetectionLab/build.sh # Start the build in a tmux session sn=tmuxsession tmux new-session -s "$sn" -d -tmux send-keys -t "$sn:0" './build.sh && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html; umount /mnt && /usr/local/bin/packet-block-storage-detach' Enter +tmux send-keys -t "$sn:0" 'cd /opt/DetectionLab && ./build.sh && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html; umount /mnt && /usr/local/bin/packet-block-storage-detach' Enter