diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..f0bfbea --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,70 @@ +version: 2 + +jobs: + build: + machine: true + working_directory: ~/repo + + steps: + - checkout + + - run: + name: Delete stale Packet servers + command: | + export DELETE_DEVICE_ID=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"') + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID" + - run: + name: Provisioning a baremetal Packet.net server + command: | + curl -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' > /tmp/device + - run: cat /tmp/device | jq ."id" | tr -d '"' > /tmp/device_id + - run: sleep 300 + - run: echo "Sleeping 5 more minutes while Packet server is provisioned" + - run: sleep 300 + - run: + name: Recording the IP address of the Packet server + command: | + export DEVICE_ID=$(cat /tmp/device_id); + curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" > /tmp/ip_info; + - run: cat /tmp/ip_info | jq ."ip_addresses[0].address" | tr -d '"' > /tmp/ip_address + - run: cd ~/repo + - run: + name: Copying install script to Packet server + command: | + export IP_ADDRESS=$(cat /tmp/ip_address); + scp -i ~/.ssh/id_rsa ./ci/automated_install_vagrant_only.sh root@"$IP_ADDRESS":/root + - run: + name: Running install script on Packet server + command: | + export IP_ADDRESS=$(cat /tmp/ip_address) + ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'chmod +x /root/automated_install_vagrant_only.sh; /bin/bash -c /root/automated_install_vagrant_only.sh' + + - run: + name: Waiting for Packet server to post build results + shell: /bin/bash + command: | + export IP_ADDRESS=$(cat /tmp/ip_address) + while true + do + export STATUS=$(curl $IP_ADDRESS) + if [ "$STATUS" == "building" ]; then + echo "$STATUS" + sleep 300 + else + echo "$STATUS" > /tmp/status + break + fi + done + - run: + name: Recording build results + shell: /bin/bash + command: | + export DEVICE_ID=$(cat /tmp/device_id) + export STATUS=$(cat /tmp/status) + echo $STATUS + if [ "$STATUS" != "success" ]; then + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + exit 1 + fi + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + exit 0 diff --git a/Packer/windows_10.json b/Packer/windows_10.json index bb691d9..40ed3ca 100644 --- a/Packer/windows_10.json +++ b/Packer/windows_10.json @@ -44,7 +44,7 @@ "boot_command": "", "winrm_username": "vagrant", "winrm_password": "vagrant", - "winrm_timeout": "2h", + "winrm_timeout": "4h", "shutdown_command": "shutdown /s /t 10 /f /d p:4:1 /c \"Packer Shutdown\"", "guest_os_type": "windows9-64", "disk_size": "{{user `disk_size`}}", @@ -85,7 +85,7 @@ "boot_command": "", "winrm_username": "vagrant", "winrm_password": "vagrant", - "winrm_timeout": "2h", + "winrm_timeout": "4h", "shutdown_command": "shutdown /s /t 10 /f /d p:4:1 /c \"Packer Shutdown\"", "guest_os_type": "Windows81_64", "guest_additions_mode": "disable", diff --git a/Packer/windows_2016.json b/Packer/windows_2016.json index 2785ceb..f7ec057 100644 --- a/Packer/windows_2016.json +++ b/Packer/windows_2016.json @@ -41,7 +41,7 @@ "boot_wait": "2m", "winrm_username": "vagrant", "winrm_password": "vagrant", - "winrm_timeout": "2h", + "winrm_timeout": "4h", "shutdown_timeout": "2h", "shutdown_command": "a:/sysprep.bat", "guest_os_type": "windows8srv-64", @@ -79,7 +79,7 @@ "boot_wait": "2m", "winrm_username": "vagrant", "winrm_password": "vagrant", - "winrm_timeout": "2h", + "winrm_timeout": "4h", "shutdown_command": "a:/sysprep.bat", "guest_os_type": "Windows2012_64", "guest_additions_mode": "disable", diff --git a/README.md b/README.md index a2a938a..f22473f 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ OSX 10.12.4 | 1.9.2 | 1.0.0 | VMWare Fusion (8.5.6) OSX 10.12.5 | 1.9.3 | 1.0.0 | VMWare Fusion (8.5.8) OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.9) OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.10) +Ubuntu 16.04 | 2.0.1 | 1.1.3 | Virtualbox (5.1) **Known Bad Versions:** * Packer 1.1.2 will fail to build VMWare-ISOs correctly due to [this issue](https://github.com/hashicorp/packer/issues/5622). @@ -42,8 +43,30 @@ OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.10) --- ## Quickstart +DetectionLab now contains build scripts for \*NIX and MacOS users! + +There are two build scripts: +- `build.sh` - Builds the entire lab from scratch. Takes 3-5 hours depending on hardware resources and bandwidth +- `build_vagrant_only.sh` - Downloads pre-built Packer boxes from S3 and builds the lab from those boxes. This option is recommended if you have more bandwidth than time. + +--- + +## Building from Scratch 1. Determine which Vagrant provider you want to use. * Note: Virtualbox is free, the [VMWare vagrant plugin](https://www.vagrantup.com/vmware/#buy-now) is $80. + + **NOTE:** If you have more bandwidth than time, you can skip the building of the Packer boxes and download the boxes directly from S3 and put them into the `Boxes` directory: + +Provider | Box | URL | MD5 | Size +------------|-----|-----|----|---- +Virtualbox |Windows 2016 | https://s3-us-west-1.amazonaws.com/detectionlab/windows_2016_virtualbox.box | 614f984c82b51471b5bb753940b59d38 | 6.4GB +Virtualbox | Windows 10 | https://s3-us-west-1.amazonaws.com/detectionlab/windows_10_virtualbox.box | 30b06e30b36b02ccf1dc5c04017654aa | 5.8GB +VMware | Windows 2016 | https://s3-us-west-1.amazonaws.com/detectionlab/windows_2016_vmware.box | 1511b9dc942c69c2cc5a8dc471fa8865 | 6.7GB +VMware | Windows 10 | https://s3-us-west-1.amazonaws.com/detectionlab/windows_10_vmware.box | 174ad0f0fd2089ff74a880c6dadac74c | 6.0GB + +If you choose to download the boxes, you may skip steps 2 and 3. If you don't trust pre-built boxes, I recommend following steps 2 and 3 to build them on your machine. + + 2. `cd` to the Packer directory and build the Windows 10 and Windows Server 2016 boxes using the commands below. Each build will take about 1 hour. As far as I know, you can only build one box at a time. ``` @@ -51,6 +74,7 @@ $ cd detectionlab/Packer $ packer build --only=[vmware|virtualbox]-iso windows_10.json $ packer build --only=[vmware|virtualbox]-iso windows_2016.json ``` + 3. Once both boxes have built successfully, move the resulting boxes (.box files) in the Packer folder to the Boxes folder: `mv *.box ../Boxes` diff --git a/Vagrant/scripts/install-utilities.ps1 b/Vagrant/scripts/install-utilities.ps1 index 47018b1..6db1045 100755 --- a/Vagrant/scripts/install-utilities.ps1 +++ b/Vagrant/scripts/install-utilities.ps1 @@ -20,10 +20,12 @@ apm install language-batch apm install language-docker # Disable Windows Defender realtime scanning before downloading Mimikatz -set-MpPreference -DisableRealtimeMonitoring $true +If ($hostname -eq "win10") { + set-MpPreference -DisableRealtimeMonitoring $true +} # Purpose: Downloads and unzips a copy of the latest Mimikatz trunk -Write-Host Determining latest release of Mimikatz... +Write-Host "Determining latest release of Mimikatz..." $tag = (Invoke-WebRequest "https://api.github.com/repos/gentilkiwi/mimikatz/releases" -UseBasicParsing | ConvertFrom-Json)[0].tag_name $mimikatzDownloadUrl = "https://github.com/gentilkiwi/mimikatz/releases/download/$tag/mimikatz_trunk.zip" $mimikatzRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\mimikatz_trunk.zip' diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..20fde4e --- /dev/null +++ b/build.sh @@ -0,0 +1,304 @@ +#! /bin/bash + +# This script is meant to be used with a fresh clone of DetectionLab and +# will fail to run if boxes have already been created or any of the steps +# from the README have already been run followed. +# Only MacOS and Linux are supported. +# If you encounter issues, feel free to open an issue at +# https://github.com/clong/DetectionLab/issues + +set -e + +print_usage() { + echo "Usage: ./build.sh " + exit 0 +} + +check_packer_and_vagrant() { + # Check for existence of Vagrant and Packer in PATH + which vagrant > /dev/null + if [ "$?" -ne 0 ]; then + echo "Vagrant was not found in your PATH." + echo "Please correct this before continuing. Quitting." + exit 1 + fi + which packer > /dev/null + if [ "$?" -ne 0 ]; then + echo "Packer was not found in your PATH." + echo "Please correct this before continuing. Quitting." + echo "Hint: sudo cp ./packer /usr/local/bin/packer; sudo chmod +x /usr/local/bin/packer" + exit 1 + fi +} + +# Returns 0 if not installed or 1 if installed +check_virtualbox_installed() { + which VBoxManage > /dev/null + if [ "$?" -eq 0 ]; then + echo "1" + else + echo "0" + fi +} + +# Returns 0 if not installed or 1 if installed +check_vmware_fusion_installed() { + echo "$(ls /Applications | grep -ci 'VMware Fusion.app')" +} + +# Returns 0 if not installed or 1 if installed +check_vmware_vagrant_plugin_installed() { + VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion') + if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then + (>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.") + (>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it") + (>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.") + echo "0" + else + echo $VAGRANT_VMWARE_PLUGIN_PRESENT + fi +} + +# List the available Vagrant providers present on the system +list_providers() { + VBOX_PRESENT=0 + VMWARE_FUSION_PRESENT=0 + + if [ $(uname) == "Darwin" ]; then + # Detect Providers on OSX + VBOX_PRESENT=$(check_virtualbox_installed) + VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed) + VAGRANT_VMWARE_PLUGIN_PRESENT=$(check_vmware_vagrant_plugin_installed) + else + # Assume the only other available provider is VirtualBox + VBOX_PRESENT=$(check_virtualbox_installed) + fi + + (>&2 echo "Available Providers:") + if [ "$VBOX_PRESENT" == "1" ]; then + (>&2 echo "virtualbox";) + fi + if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]] + then + (>&2 echo "vmware_fusion";) + fi + if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]] + then + (>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.") + exit 1 + fi + (>&2 echo -e "\nWhich provider would you like to use?") + read PROVIDER + # Sanity check + if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]] + then + (>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option") + exit 1 + fi + echo $PROVIDER +} + +# A series of checks to identify potential issues before starting the build +preflight_checks() { + DL_DIR="$1" + + # Check to see if curl is in PATH + which curl > /dev/null + if [ "$?" -ne 0 ]; then + (>&2 echo "Please install curl and make sure it is in your PATH.") + exit 1 + fi + # Check to see if boxes exist already + BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l) + if [ $BOXES_BUILT -gt 0 ]; then + (>&2 echo "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.") + exit 1 + fi + # Check to see if any Vagrant instances exist already + cd "$DL_DIR"/Vagrant/ + VAGRANT_BUILT=$(vagrant status | grep -c 'not created') + if [ $VAGRANT_BUILT -ne 4 ]; then + (>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.") + exit 1 + fi + # Check available disk space. Recommend 80GB free, warn if less. + FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4) + if [ $FREE_DISK_SPACE -lt 80000 ]; then + (>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n") + (>&2 df -m $HOME) + (>&2 echo "") + fi + # Check Packer version against known bad + if [ $(packer --version) == '1.1.2' ]; then + (>&2 echo "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.") + exit 1 + fi + # Ensure the vagrant-reload plugin is installed + VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload') + if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then + (>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.") + $(which vagrant) plugin install "vagrant-reload" + if [ "$?" -ne 0 ]; then + (>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.") + exit 1 + fi + fi +} + +# Builds a box using Packer +packer_build_box() { + PROVIDER="$1" + BOX="$2" + DL_DIR="$3" + if [ "$PROVIDER" == "vmware_fusion" ]; then + PROVIDER="vmware" + fi + cd "$DL_DIR/Packer" + (>&2 echo "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.") + $(which packer) build --only="$PROVIDER-iso" $BOX.json + if [ "$?" -ne 0 ]; then + (>&2 echo "Something went wrong while attempting to build the $BOX box.") + (>&2 echo "To file an issue, please visit https://github.com/clong/DetectionLab/issues/") + fi +} + +# Moves the boxes from the Packer directory to the Boxes directory +move_boxes() { + PROVIDER="$1" + DL_DIR="$2" + # Hacky workaround for VMware + if [ "$PROVIDER" == "vmware_fusion" ]; then + PROVIDER="vmware" + fi + mv "$DL_DIR"/Packer/*.box "$DL_DIR"/Boxes + # Ensure Windows 10 box exists + if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then + (>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.") + exit 1 + fi + # Ensure Windows 2016 box exists + if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then + (>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.") + exit 1 + fi +} + +# Brings up a single host using Vagrant +vagrant_up_host() { + PROVIDER="$1" + HOST="$2" + DL_DIR="$3" + (>&2 echo "Attempting to bring up the $HOST host using Vagrant") + cd "$DL_DIR"/Vagrant + $(which vagrant) up $HOST --provider="$PROVIDER" 1>&2 + echo "$?" +} + +# Attempts to reload and re-provision a host if the intial "vagrant up" fails +vagrant_reload_host() { + HOST="$1" + DL_DIR="$2" + cd "$DL_DIR"/Vagrant + # Attempt to reload the host if the vagrant up command didn't exit cleanly + $(which vagrant) reload $HOST --provision 1>&2 + echo "$?" +} + +# A series of checks to ensure important services are responsive after the build completes. +post_build_checks() { + # If the curl operation fails, we'll just leave the variable equal to 0 + # This is needed to prevent the script from exiting if the curl operation fails + CALDERA_CHECK=$(curl -ks -m 2 https://192.168.38.5:8888 | grep -c '302: Found' || echo "") + SPLUNK_CHECK=$(curl -ks -m 2 https://192.168.38.5:8000/en-US/account/login?return_to=%2Fen-US%2F | grep -c 'This browser is not supported by Splunk' || echo "") + FLEET_CHECK=$(curl -ks -m 2 https://192.168.38.5:8412 | grep -c 'Kolide Fleet' || echo "") + + BASH_MAJOR_VERSION=$(/bin/bash --version | grep 'GNU bash' | grep -o version\.\.. | cut -d ' ' -f 2 | cut -d '.' -f 1) + # Associative arrays are only supported in bash 4 and up + if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then + declare -A SERVICES + SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK") + for SERVICE in "${!SERVICES[@]}" + do + if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then + (>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.") + fi + done + else + if [ "$CALDERA_CHECK" -lt 1 ]; then + (>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.") + fi + if [ "$SPLUNK_CHECK" -lt 1 ]; then + (>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.") + fi + if [ "$FLEET_CHECK" -lt 1 ]; then + (>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.") + fi + fi +} + +main() { + # Get location of build.sh + # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within + DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + PROVIDER="" + LAB_HOSTS=("logger" "dc" "wef" "win10") + # If no argument was supplied, list available providers + if [ $# -eq 0 ]; then + PROVIDER=$(list_providers) + fi + # If more than one argument was supplied, print usage message + if [ $# -gt 1 ]; then + print_usage + exit 1 + fi + if [ $# -eq 1 ]; then + # If the user specifies the provider as an agument, set the variable + # TODO: Check to make sure they actually have their provider installed + case "$1" in + virtualbox) + PROVIDER="$1" + ;; + vmware_fusion) + PROVIDER="$1" + ;; + *) + echo "\"$1\" is not a valid provider. Listing available providers:" + PROVIDER=$(list_providers) + ;; + esac + fi + + preflight_checks $DL_DIR + packer_build_box $PROVIDER "windows_2016" $DL_DIR + packer_build_box $PROVIDER "windows_10" $DL_DIR + move_boxes $PROVIDER $DL_DIR + + # Change provider back to original selection if using vmware_fusion + if [ "$PROVIDER" == "vmware" ]; then + PROVIDER="vmware_fusion" + fi + + # Vagrant up each box and attempt to reload one time if it fails + for VAGRANT_HOST in "${LAB_HOSTS[@]}" + do + RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR) + if [ "$RET" -eq 0 ]; then + (>&2 echo "Good news! $VAGRANT_HOST was built successfully!") + fi + # Attempt to recover if the intial "vagrant up" fails + if [ "$RET" -ne 0 ]; then + (>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.") + (>&2 echo "Attempting to reload and reprovision the host...") + RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR) + if [ "$RETRY_STATUS" -ne 0 ]; then + (>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.") + exit 1 + fi + fi + done + + post_build_checks +} + +main $@ +exit 0 diff --git a/build_vagrant_only.sh b/build_vagrant_only.sh new file mode 100755 index 0000000..d4bcae4 --- /dev/null +++ b/build_vagrant_only.sh @@ -0,0 +1,305 @@ +#! /bin/bash + +# This script skips the entire Packer box building process. +# This script downloads pre-built VirtualBox Packer images from S3 and puts +# them into the "Boxes" directory. +# Only MacOS and Linux are supported. +# If you encounter issues, feel free to open an issue at +# https://github.com/clong/DetectionLab/issues + +set -e + +print_usage() { + echo "Usage: ./build.sh " + exit 0 +} + +check_vagrant() { + # Check for existence of Vagrant and Packer in PATH + which vagrant > /dev/null + if [ "$?" -ne 0 ]; then + echo "Vagrant was not found in your PATH." + echo "Please correct this before continuing. Quitting." + exit 1 + fi +} + +# Returns 0 if not installed or 1 if installed +check_virtualbox_installed() { + which VBoxManage > /dev/null + if [ "$?" -eq 0 ]; then + echo "1" + else + echo "0" + fi +} + +# Returns 0 if not installed or 1 if installed +check_vmware_fusion_installed() { + echo "$(ls /Applications | grep -ci 'VMware Fusion.app')" +} + +# Returns 0 if not installed or 1 if installed +check_vmware_vagrant_plugin_installed() { + VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion') + if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then + (>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.") + (>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it") + (>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.") + echo "0" + else + echo $VAGRANT_VMWARE_PLUGIN_PRESENT + fi +} + +# List the available Vagrant providers present on the system +list_providers() { + VBOX_PRESENT=0 + VMWARE_FUSION_PRESENT=0 + + if [ $(uname) == "Darwin" ]; then + # Detect Providers on OSX + VBOX_PRESENT=$(check_virtualbox_installed) + VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed) + VAGRANT_VMWARE_PLUGIN_PRESENT=$(check_vmware_vagrant_plugin_installed) + else + # Assume the only other available provider is VirtualBox + VBOX_PRESENT=$(check_virtualbox_installed) + fi + + (>&2 echo "Available Providers:") + if [ "$VBOX_PRESENT" == "1" ]; then + (>&2 echo "virtualbox";) + fi + if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]] + then + (>&2 echo "vmware_fusion";) + fi + if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]] + then + (>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.") + exit 1 + fi + (>&2 echo -e "\nWhich provider would you like to use?") + read PROVIDER + # Sanity check + if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]] + then + (>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option") + exit 1 + fi + echo $PROVIDER +} + +# A series of checks to identify potential issues before starting the build +preflight_checks() { + DL_DIR="$1" + + # Check to see if curl is in PATH + which curl > /dev/null + if [ "$?" -ne 0 ]; then + (>&2 echo "Please install curl and make sure it is in your PATH.") + exit 1 + fi + # Check to see if wget is in PATH + which wget > /dev/null + if [ "$?" -ne 0 ]; then + (>&2 echo "Please install curl and make sure it is in your PATH.") + exit 1 + fi + # Check to see if boxes exist already + BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l) + if [ $BOXES_BUILT -gt 0 ]; then + (>&2 echo "You appear to have already built at least one box using Packer. This script automatically downloads them for you. Please either delete the existing boxes or follow the build steps in the README to continue.") + exit 1 + fi + # Check to see if any Vagrant instances exist already + cd "$DL_DIR"/Vagrant/ + VAGRANT_BUILT=$(vagrant status | grep -c 'not created') + if [ $VAGRANT_BUILT -ne 4 ]; then + (>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.") + exit 1 + fi + # Check available disk space. Recommend 80GB free, warn if less. + FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4) + if [ $FREE_DISK_SPACE -lt 80000 ]; then + (>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n") + (>&2 df -m $HOME) + (>&2 echo "") + fi + # Ensure the vagrant-reload plugin is installed + VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload') + if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then + (>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.") + $(which vagrant) plugin install "vagrant-reload" + if [ "$?" -ne 0 ]; then + (>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.") + exit 1 + fi + fi +} + +# Downloads pre-build Packer boxes from S3 to save time during CI builds +download_boxes() { + DL_DIR="$1" + PROVIDER="$2" + + if [ "$PROVIDER" == "virtualbox" ]; then + wget "https://s3-us-west-1.amazonaws.com/detectionlab/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box + wget "https://s3-us-west-1.amazonaws.com/detectionlab/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box + elif [ "$PROVIDER" == "vmware_fusion" ]; then + wget "https://s3-us-west-1.amazonaws.com/detectionlab/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box + wget "https://s3-us-west-1.amazonaws.com/detectionlab/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box + fi + + # Hacky workaround + if [ "$PROVIDER" == "vmware_fusion" ]; then + PROVIDER="vmware" + fi + + # Ensure Windows 10 box exists + if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then + (>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.") + exit 1 + fi + # Ensure Windows 2016 box exists + if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then + (>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.") + exit 1 + fi + # Verify hashes of VirtualBox boxes + if [ "$PROVIDER" == "virtualbox" ]; then + if [ "$(md5sum windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then + (>&2 echo "Hash mismatch on windows_10_virtualbox.box") + fi + if [ "$(md5sum windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "614f984c82b51471b5bb753940b59d38" ]; then + (>&2 echo "Hash mismatch on windows_2016_virtualbox.box") + fi + # Verify hashes of VMware boxes + elif [ "$PROVIDER" == "vmware" ]; then + if [ "$(md5 windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then + (>&2 echo "Hash mismatch on windows_10_vmware.box") + exit 1 + fi + if [ "$(md5 windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then + (>&2 echo "Hash mismatch on windows_2016_vmware.box") + exit 1 + fi + # Reset PROVIDER variable + PROVIDER="vmware_fusion" + fi +} + +# Brings up a single host using Vagrant +vagrant_up_host() { + PROVIDER="$1" + HOST="$2" + DL_DIR="$3" + (>&2 echo "Attempting to bring up the $HOST host using Vagrant") + cd "$DL_DIR"/Vagrant + $(which vagrant) up $HOST --provider="$PROVIDER" 1>&2 + echo "$?" +} + +# Attempts to reload and re-provision a host if the intial "vagrant up" fails +vagrant_reload_host() { + HOST="$1" + DL_DIR="$2" + cd "$DL_DIR"/Vagrant + # Attempt to reload the host if the vagrant up command didn't exit cleanly + $(which vagrant) reload $HOST --provision 1>&2 + echo "$?" +} + +# A series of checks to ensure important services are responsive after the build completes. +post_build_checks() { + # If the curl operation fails, we'll just leave the variable equal to 0 + # This is needed to prevent the script from exiting if the curl operation fails + CALDERA_CHECK=$(curl -ks -m 2 https://192.168.38.5:8888 | grep -c '302: Found' || echo "") + SPLUNK_CHECK=$(curl -ks -m 2 https://192.168.38.5:8000/en-US/account/login?return_to=%2Fen-US%2F | grep -c 'This browser is not supported by Splunk' || echo "") + FLEET_CHECK=$(curl -ks -m 2 https://192.168.38.5:8412 | grep -c 'Kolide Fleet' || echo "") + + BASH_MAJOR_VERSION=$(/bin/bash --version | grep 'GNU bash' | grep -o version\.\.. | cut -d ' ' -f 2 | cut -d '.' -f 1) + # Associative arrays are only supported in bash 4 and up + if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then + declare -A SERVICES + SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK") + for SERVICE in "${!SERVICES[@]}" + do + if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then + (>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.") + fi + done + else + if [ "$CALDERA_CHECK" -lt 1 ]; then + (>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.") + fi + if [ "$SPLUNK_CHECK" -lt 1 ]; then + (>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.") + fi + if [ "$FLEET_CHECK" -lt 1 ]; then + (>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.") + fi + fi +} + +main() { + # Get location of build_vagrant_only.sh + # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within + DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + PROVIDER="" + LAB_HOSTS=("logger" "dc" "wef" "win10") + # If no argument was supplied, list available providers + if [ $# -eq 0 ]; then + PROVIDER=$(list_providers) + fi + # If more than one argument was supplied, print usage message + if [ $# -gt 1 ]; then + print_usage + exit 1 + fi + if [ $# -eq 1 ]; then + # If the user specifies the provider as an agument, set the variable + # TODO: Check to make sure they actually have their provider installed + case "$1" in + virtualbox) + PROVIDER="$1" + ;; + vmware_fusion) + PROVIDER="$1" + ;; + *) + echo "\"$1\" is not a valid provider. Listing available providers:" + PROVIDER=$(list_providers) + ;; + esac + fi + + check_vagrant + preflight_checks $DL_DIR + download_boxes $DL_DIR $PROVIDER + + # Vagrant up each box and attempt to reload one time if it fails + for VAGRANT_HOST in "${LAB_HOSTS[@]}" + do + RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR) + if [ "$RET" -eq 0 ]; then + (>&2 echo "Good news! $VAGRANT_HOST was built successfully!") + fi + # Attempt to recover if the intial "vagrant up" fails + if [ "$RET" -ne 0 ]; then + (>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.") + (>&2 echo "Attempting to reload and reprovision the host...") + RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR) + if [ "$RETRY_STATUS" -ne 0 ]; then + (>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.") + exit 1 + fi + fi + done + + post_build_checks +} + +main $@ +exit 0 diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..8174854 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,6 @@ +# ci + +The files in this directory are used to bootstrap an Ubuntu 16.04 baremetal server +for continuous integration testing by installing the prerequisites needed for +Detection Lab. After the prerequisites are installed, the build script is called +and the build will begin in a tmux session. diff --git a/ci/automated_install.sh b/ci/automated_install.sh new file mode 100644 index 0000000..c39d59d --- /dev/null +++ b/ci/automated_install.sh @@ -0,0 +1,60 @@ +#! /bin/bash + +# This script is run on the Packet.net baremetal server for CI tests. +# This script will build the entire lab from scratch and takes 3-4 hours +# on a Packet.net host +# While building, the server will start a webserver on Port 80 that contains +# the text "building". Once the test is completed, the text will be replaced +# with "success" or "failed". + +# Install Virtualbox 5.1 +echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list +wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - +apt-get update +apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2 + +echo "building" > /var/www/html/index.html + +# Set up firewall +ufw allow ssh +ufw allow http +ufw default allow outgoing +ufw --force enable + +# Install Vagrant +mkdir /opt/vagrant +cd /opt/vagrant +wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb +dpkg -i vagrant_2.0.1_x86_64.deb +vagrant plugin install vagrant-reload + +# Install Packer +mkdir /opt/packer +cd /opt/packer +wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip +unzip packer_1.1.3_linux_amd64.zip +cp packer /usr/local/bin/packer + +# Clone DetectionLab +cd /opt +git clone https://github.com/clong/DetectionLab.git +cd /opt/DetectionLab/Packer + +# Make the packer images headless +for file in $(ls *.json); do + sed -i 's/"headless": false,/"headless": true,/g' "$file"; +done + +# Make the Vagrant instances headless +cd /opt/DetectionLab/Vagrant +sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile + +# Download the build script from Github +wget https://gist.githubusercontent.com/clong/4d0865cf022f0fbc63caf900d91852fa/raw/479d6dbb18fdfc2e5d610613683752da3d4140fa/build.sh -O /opt/DetectionLab/build.sh ## Change this once we move to the actual DL repo. +chmod +x /opt/DetectionLab/build.sh +cd /opt/DetectionLab + +# Start the build in a tmux session +sn=tmuxsession +tmux new-session -s "$sn" -d +tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter diff --git a/ci/automated_install_vagrant_only.sh b/ci/automated_install_vagrant_only.sh new file mode 100644 index 0000000..99f9ebf --- /dev/null +++ b/ci/automated_install_vagrant_only.sh @@ -0,0 +1,47 @@ +#! /bin/bash + +# This script is run on the Packet.net baremetal server for CI tests. +# This script will bootstrap the build by downloading pre-build Packer boxes +# and should take no more than 90 minutes on a Packet.net host. +# While building, the server will start a webserver on Port 80 that contains +# the text "building". Once the test is completed, the text will be replaced +# with "success" or "failed". + +# Install Virtualbox 5.1 +echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list +wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - +apt-get update +apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2 + +echo "building" > /var/www/html/index.html + +# Set up firewall +ufw allow ssh +ufw allow http +ufw default allow outgoing +ufw --force enable + +# Install Vagrant +mkdir /opt/vagrant +cd /opt/vagrant +wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb +dpkg -i vagrant_2.0.1_x86_64.deb +vagrant plugin install vagrant-reload + +# Clone DetectionLab +cd /opt +git clone https://github.com/clong/DetectionLab.git + +# Make the Vagrant instances headless +cd /opt/DetectionLab/Vagrant +sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile + +# Download the build script from Github +wget https://gist.githubusercontent.com/clong/5ee211b8533f7d33eaa31d6b83231d5e/raw/08fc4a3e8cc806c5f6278226607e4a36aa7e03fc/build_vagrant_only.sh -O /opt/DetectionLab/build_vagrant_only.sh # Change to repo path once we go to prod +chmod +x /opt/DetectionLab/build_vagrant_only.sh +cd /opt/DetectionLab + +# Start the build in a tmux session +sn=tmuxsession +tmux new-session -s "$sn" -d +tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter