Merge pull request #77 from clong/update_packer_scripts

Add multiple continuous integration build suites and update Packer build

.circleci/config.yml

@@ -9,79 +9,67 @@ jobs:
       - checkout
 
       - run:
-          name: Delete stale Packet servers
+          name: Choose which test suite to run based on which files were modified
           command: |
-            export DELETE_DEVICE_ID=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
+            ## Checking commits for changes to Packer files
-            curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
+            ## This handles the cases where there are multiple commits
-      - run:
+            if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then
-          name: Provisioning a baremetal Packet.net server
+              if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Packer/)" -gt 0 ]; then
-          command: |
+                export PACKER_MODIFIED=1
-           curl -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' > /tmp/device
+              else
-      - run: cat /tmp/device | jq ."id" | tr -d '"' > /tmp/device_id
+                export PACKER_MODIFIED=0
-      - run: sleep 300
+              fi
-      - run: echo "Sleeping 5 more minutes while Packet server is provisioned"
-      - run: sleep 300
-      - run:
-          name: Recording the IP address of the Packet server
-          command: |
-           export DEVICE_ID=$(cat /tmp/device_id);
-           curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" > /tmp/ip_info;
-      - run: cat /tmp/ip_info | jq ."ip_addresses[0].address" | tr -d '"' > /tmp/ip_address
-      - run: cd ~/repo
-      - run:
-         name: Copying DetectionLab repo to Packet server
-         command: |
-           export IP_ADDRESS=$(cat /tmp/ip_address);
-           scp -i ~/.ssh/id_rsa -r ~/repo root@"$IP_ADDRESS":/opt/DetectionLab
-      - run:
-          name: Copying install script to Packet server
-          command: |
-            export IP_ADDRESS=$(cat /tmp/ip_address);
-            scp -i ~/.ssh/id_rsa ./ci/automated_install_vagrant_only.sh root@"$IP_ADDRESS":/root
-      - run:
-          name: Running install script on Packet server
-          command: |
-            export IP_ADDRESS=$(cat /tmp/ip_address)
-            ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'chmod +x /root/automated_install_vagrant_only.sh; /bin/bash -c /root/automated_install_vagrant_only.sh'
-
-      - run:
-          name: Waiting for Packet server to post build results
-          shell: /bin/bash
-          command: |
-            export IP_ADDRESS=$(cat /tmp/ip_address)
-            MINUTES_PAST=0
-            while [ "$MINUTES_PAST" -lt 120 ]
-            do
-            export STATUS=$(curl $IP_ADDRESS)
-            if [ "$STATUS" == "building" ]; then
-            echo "$STATUS"
-            sleep 300
-            ((MINUTES_PAST+=5))
             else
-            echo "$STATUS" > /tmp/status
+              ## This handles the cases where there is only a single commit
-            break
+              export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)"
+              if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c ^Packer/)" -gt 0 ]; then
+                export PACKER_MODIFIED=1
+              else
+                export PACKER_MODIFIED=0
+              fi
             fi
-            done
+            ## Checking commits for changes to Vagrant files
-            if [ "$MINUTES_PAST" -gt 120 ]; then
+            if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then
-            export IP_ADDRESS=$(cat /tmp/ip_address);
+              if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Vagrant/)" -gt 0 ]; then
-            scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log .
+                export VAGRANT_MODIFIED=1
-            cat vagrant.log
+              else
-            curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
+                export VAGRANT_MODIFIED=0
-            exit 1
+              fi
+            else
+              export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)"
+              if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Vagrant/\")" -gt 0 ]; then
+                export VAGRANT_MODIFIED=1
+              else
+                export VAGRANT_MODIFIED=0
+              fi
             fi
-      - run:
+            echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED"
-          name: Recording build results
+            echo "PACKER_MODIFIED=$PACKER_MODIFIED"
-          shell: /bin/bash
+            ## Choosing which test suite to run based on the files that were changed
-          command: |
+            if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then
-            export DEVICE_ID=$(cat /tmp/device_id)
+              echo "Running the test suite for Packer and Vagrant changes"
-            export STATUS=$(cat /tmp/status)
+              chmod +x ci/circle_workflows/packer_and_vagrant_changes.sh
-            echo $STATUS
+              ci/circle_workflows/packer_and_vagrant_changes.sh
-            if [ "$STATUS" != "success" ]; then
+              exit 0
-            export IP_ADDRESS=$(cat /tmp/ip_address);
-            scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log .
-            cat vagrant.log
-            curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
-            exit 1
             fi
-            curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
+            if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then
-            exit 0
+              echo "Running the default test suite (Vagrant-only)"
+              chmod +x ci/circle_workflows/vagrant_changes.sh
+              ci/circle_workflows/vagrant_changes.sh
+              exit 0
+            fi
+            if [ "$PACKER_MODIFIED" -eq 1 ]; then
+              echo "Running the test suite for Packer-only changes"
+              chmod +x ci/circle_workflows/packer_changes.sh
+              ci/circle_workflows/packer_changes.sh
+              exit 0
+            fi
+            if [ "$VAGRANT_MODIFIED" -eq 1 ]; then
+              echo "Running the test suite for Vagrant-only changes"
+              chmod +x ci/circle_workflows/vagrant_changes.sh
+              ci/circle_workflows/vagrant_changes.sh
+              exit 0
+            fi
+
+      - store_artifacts:
+          path: /tmp/artifacts

.gitignore

@@ -1,4 +1,6 @@
 Vagrant/.vagrant/*
+Vagrant/vagrant_build.log
 Packer/packer_cache/*
+Packer/packer_build.log
 Boxes/*
 .DS_Store

Packer/scripts/compile-dotnet-assemblies.bat

@@ -3,13 +3,15 @@
 
 if "%PROCESSOR_ARCHITECTURE%"=="AMD64" goto 64BIT
 
-%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue
+%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue > NUL
-%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems
+%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems > NUL
 
-exit /b
+exit 0
 
 :64BIT
-%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue
+%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue > NUL
-%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe update /force /queue
+%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe update /force /queue > NUL
-%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems
+%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems > NUL
-%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe executequeueditems
+%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe executequeueditems > NUL
+
+exit 0

Packer/scripts/set-powerplan.ps1

@@ -1,7 +1,20 @@
 Try {
+  Write-Output "Set power plan to high performance"
+
   $HighPerf = powercfg -l | %{if($_.contains("High performance")) {$_.split()[3]}}
+
+  # $HighPerf cannot be $null, we try activate this power profile with powercfg
+  # 
+  if ($HighPerf -eq $null)
+  {
+    throw "Error: HighPerf is null"
+  }
+
   $CurrPlan = $(powercfg -getactivescheme).split()[3]
+
   if ($CurrPlan -ne $HighPerf) {powercfg -setactive $HighPerf}
+
 } Catch {
   Write-Warning -Message "Unable to set power plan to high performance"
+  Write-Warning $Error[0]
 }

Packer/scripts/unattend.xml

@@ -34,7 +34,7 @@
     </settings>
     <settings pass="specialize">
         <component name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-            <ComputerName>vagrant-2012</ComputerName>
+            <ComputerName>vagrant-2016</ComputerName>
             <CopyProfile>false</CopyProfile>
         </component>
     </settings>

Packer/scripts/vm-guest-tools.bat

@@ -15,7 +15,7 @@ if exist "C:\Users\vagrant\windows.iso" (
 )
 
 if not exist "C:\Windows\Temp\windows.iso" (
-    powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://softwareupdate.vmware.com/cds/vmw-desktop/ws/12.5.7/5813279/windows/packages/tools-windows.tar', 'C:\Windows\Temp\vmware-tools.tar')" <NUL
+    powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://softwareupdate.vmware.com/cds/vmw-desktop/ws/14.1.1/7528167/windows/packages/tools-windows.tar', 'C:\Windows\Temp\vmware-tools.tar')" <NUL
     cmd /c ""C:\Program Files\7-Zip\7z.exe" x C:\Windows\Temp\vmware-tools.tar -oC:\Windows\Temp"
     FOR /r "C:\Windows\Temp" %%a in (VMware-tools-windows-*.iso) DO REN "%%~a" "windows.iso"
     rd /S /Q "C:\Program Files (x86)\VMWare"
@@ -31,16 +31,12 @@ goto :done
 
 :virtualbox
 
-:: There needs to be Oracle CA (Certificate Authority) certificates installed in order
-:: to prevent user intervention popups which will undermine a silent installation.
-cmd /c certutil -addstore -f "TrustedPublisher" A:\oracle-cert.cer
-
 if exist "C:\Users\vagrant\VBoxGuestAdditions.iso" (
     move /Y C:\Users\vagrant\VBoxGuestAdditions.iso C:\Windows\Temp
 )
 
 if not exist "C:\Windows\Temp\VBoxGuestAdditions.iso" (
-    powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://download.virtualbox.org/virtualbox/5.1.28/VBoxGuestAdditions_5.1.28.iso', 'C:\Windows\Temp\VBoxGuestAdditions.iso')" <NUL
+    powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://download.virtualbox.org/virtualbox/5.2.4/VBoxGuestAdditions_5.2.4.iso', 'C:\Windows\Temp\VBoxGuestAdditions.iso')" <NUL
 )
 
 cmd /c ""C:\Program Files\7-Zip\7z.exe" x C:\Windows\Temp\VBoxGuestAdditions.iso -oC:\Windows\Temp\virtualbox"

Packer/windows_2016.json

@@ -81,7 +81,7 @@
       "winrm_password": "vagrant",
       "winrm_timeout": "4h",
       "shutdown_command": "a:/sysprep.bat",
-      "guest_os_type": "Windows2012_64",
+      "guest_os_type": "Windows2016_64",
       "guest_additions_mode": "disable",
       "disk_size": 61440,
       "floppy_files": [

Vagrant/bootstrap.sh

@@ -90,7 +90,7 @@ go build
 cd /home/vagrant
 
 # Modify the config to work with config importer
-cat /home/vagrant/osquery-configuration/Endpoints/Windows/osquery.conf  | sed 's#packs/#../packs/#g' | grep -v unwanted-chrome-extensions | grep -v security-tooling-checks | grep -v performance-metrics > /home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf
+cat /home/vagrant/osquery-configuration/Endpoints/Windows/osquery.conf  | sed 's#packs/#../packs/#g' | grep -v unwanted-chrome-extensions | grep -v security-tooling-checks | grep -v performance-metrics | grep -v logger_snapshot_event_type > /home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf
 # Install configimporter
 echo "Installing configimporter"
 echo "Sleeping for 5"

Vagrant/scripts/download_palantir_wef.ps1

@@ -1,8 +1,5 @@
 # Purpose: Downloads and unzips a copy of the Palantir WEF Github Repo. This includes WEF subscriptions and custom WEF channels.
 
-# GitHub requires TLS 1.2
-[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
-
 Write-Host "Downloading and unzipping the Palantir Windows Event Forwarding Repo from Github..."
 
 $wefRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\wef-Master.zip'

build.sh

@@ -7,25 +7,13 @@
 # If you encounter issues, feel free to open an issue at
 # https://github.com/clong/DetectionLab/issues
 
-set -e
-
 print_usage() {
   echo "Usage: ./build.sh <virtualbox|vmware_fusion>"
   exit 0
 }
 
-check_packer_and_vagrant() {
+check_packer_path() {
-  # Check for existence of Vagrant in PATH
+# Check for existence of Packer in PATH
-  if ! which vagrant >/dev/null; then
-    (echo >&2 "Vagrant was not found in your PATH.")
-    (echo >&2 "Please correct this before continuing. Quitting.")
-    exit 1
-  fi
-  # Ensure Vagrant >= 2.0.0
-  if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
-    (echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
-  fi
-  # Check for existence of Packer in PATH
   if ! which packer >/dev/null; then
     (echo >&2 "Packer was not found in your PATH.")
     (echo >&2 "Please correct this before continuing. Quitting.")
@@ -34,6 +22,19 @@ check_packer_and_vagrant() {
   fi
 }
 
+check_vagrant_path() {
+# Check for existence of Vagrant in PATH
+if ! which vagrant >/dev/null; then
+  (echo >&2 "Vagrant was not found in your PATH.")
+  (echo >&2 "Please correct this before continuing. Quitting.")
+  exit 1
+fi
+# Ensure Vagrant >= 2.0.0
+if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
+  (echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
+fi
+}
+
 # Returns 0 if not installed or 1 if installed
 check_virtualbox_installed() {
   if which VBoxManage >/dev/null; then
@@ -101,41 +102,40 @@ list_providers() {
   echo "$PROVIDER"
 }
 
-# A series of checks to identify potential issues before starting the build
+# Check to see if boxes exist in the "Boxes" directory already
-preflight_checks() {
+check_boxes_built() {
-  DL_DIR="$1"
-
-  # Check to see if curl is in PATH
-  if ! which curl >/dev/null; then
-    (echo >&2 "Please install curl and make sure it is in your PATH.")
-    exit 1
-  fi
-  # Check to see if boxes exist already
   BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l)
   if [ "$BOXES_BUILT" -gt 0 ]; then
-    (echo >&2 "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.")
+    if [ "$VAGRANT_ONLY" -eq 1 ]; then
+      (echo >&2 "WARNING: You seem to have at least one .box file present in $DL_DIR/Boxes already. If you would like fresh boxes downloaded, please remove all files from the Boxes directory and re-run this script.")
+      DOWNLOAD_BOXES=0
+    else
+      (echo >&2 "You seem to have at least one .box file in $DL_DIR/Boxes. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.")
+      exit 1
+    fi
+  fi
+}
+
+# Check to ensure either "md5" or "md5sum" is installed for verifying integrity of downloaded boxes
+check_md5_tool_exists() {
+  if ! which md5 > /dev/null && ! which md5sum > /dev/null; then
+    (echo >&2 "md5 or md5sum not found in PATH. Please install at least one of these utilities to continue.")
     exit 1
   fi
-  # Check to see if any Vagrant instances exist already
+}
-  cd "$DL_DIR"/Vagrant/
+
+# Check to see if any Vagrant instances exist already
+check_vagrant_instances_exist() {
+  cd "$DL_DIR"/Vagrant/ || exit 1
   # Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true"
   VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true
   if [ "$VAGRANT_BUILT" -ne 4 ]; then
     (echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
     exit 1
   fi
-  # Check available disk space. Recommend 80GB free, warn if less.
+}
-  FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
+
-  if [ "$FREE_DISK_SPACE" -lt 80000 ]; then
+check_vagrant_reload_plugin() {
-    (echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n")
-    (df >&2 -m "$HOME")
-    (echo >&2 "")
-  fi
-  # Check Packer version against known bad
-  if [ "$(packer --version)" == '1.1.2' ]; then
-    (echo >&2 "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.")
-    exit 1
-  fi
   # Ensure the vagrant-reload plugin is installed
   VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
   if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
@@ -147,26 +147,70 @@ preflight_checks() {
   fi
 }
 
+# Check available disk space. Recommend 80GB free, warn if less.
+check_disk_free_space() {
+  FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
+  if [ "$FREE_DISK_SPACE" -lt 80000 ]; then
+    (echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n")
+    (df >&2 -m "$HOME")
+    (echo >&2 "")
+  fi
+}
+
+# Check to see if curl is in PATH - needed for post-install checks
+check_curl(){
+  if ! which curl >/dev/null; then
+    (echo >&2 "Please install curl and make sure it is in your PATH.")
+    exit 1
+  fi
+}
+
+# Check Packer version against known "bad" versions
+check_packer_known_bad() {
+  if [ "$(packer --version)" == '1.1.2' ]; then
+    (echo >&2 "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.")
+    exit 1
+  fi
+}
+
+# A series of checks to identify potential issues before starting the build
+preflight_checks() {
+  # If it's not a Vagrant-only build, then run Packer-related checks
+  if [ "$VAGRANT_ONLY" -eq 0 ]; then
+    check_packer_path
+    check_packer_known_bad
+  else
+    # If it is a Vagrant-only build, set appropriate checks
+    DOWNLOAD_BOXES=1
+    check_md5_tool_exists
+  fi
+
+  # If it's not a Packer-only build, then run Vagrant-related checks
+  if [ "$PACKER_ONLY" -eq 0 ]; then
+    check_vagrant_path
+    check_vagrant_instances_exist
+    check_vagrant_reload_plugin
+  fi
+
+  check_boxes_built
+  check_disk_free_space
+  check_curl
+}
+
 # Builds a box using Packer
 packer_build_box() {
-  PROVIDER="$1"
+  BOX="$1"
-  BOX="$2"
-  DL_DIR="$3"
   if [ "$PROVIDER" == "vmware_fusion" ]; then
     PROVIDER="vmware"
   fi
-  cd "$DL_DIR/Packer"
+  cd "$DL_DIR/Packer" || exit 1
   (echo >&2 "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.")
-  if ! $(which packer) build --only="$PROVIDER-iso" "$BOX".json; then
+  PACKER_LOG=1 PACKER_LOG_PATH="$DL_DIR/Packer/packer_build.log" $(which packer) build --only="$PROVIDER-iso" "$BOX".json >&2
-    (echo >&2 "Something went wrong while attempting to build the $BOX box.")
+  echo "$?"
-    (echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
-  fi
 }
 
 # Moves the boxes from the Packer directory to the Boxes directory
 move_boxes() {
-  PROVIDER="$1"
-  DL_DIR="$2"
   # Hacky workaround for VMware
   if [ "$PROVIDER" == "vmware_fusion" ]; then
     PROVIDER="vmware"
@@ -186,20 +230,17 @@ move_boxes() {
 
 # Brings up a single host using Vagrant
 vagrant_up_host() {
-  PROVIDER="$1"
+  HOST="$1"
-  HOST="$2"
-  DL_DIR="$3"
   (echo >&2 "Attempting to bring up the $HOST host using Vagrant")
-  cd "$DL_DIR"/Vagrant
+  cd "$DL_DIR"/Vagrant || exit 1
-  $(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2
+  VAGRANT_LOG=info $(which vagrant) up "$HOST" --provider="$PROVIDER" 3>&1 1>&2 2>&3 | tee -a "$DL_DIR/Vagrant/vagrant_build.log"
   echo "$?"
 }
 
 # Attempts to reload and re-provision a host if the intial "vagrant up" fails
 vagrant_reload_host() {
   HOST="$1"
-  DL_DIR="$2"
+  cd "$DL_DIR"/Vagrant || exit 1
-  cd "$DL_DIR"/Vagrant
   # Attempt to reload the host if the vagrant up command didn't exit cleanly
   $(which vagrant) reload "$HOST" --provision 1>&2
   echo "$?"
@@ -236,22 +277,17 @@ post_build_checks() {
   fi
 }
 
-main() {
+parse_cli_arguments() {
-  # Get location of build.sh
-  # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
-  DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-  PROVIDER=""
-  LAB_HOSTS=("logger" "dc" "wef" "win10")
   # If no argument was supplied, list available providers
-  if [ $# -eq 0 ]; then
+  if [ "$#" -eq 0 ]; then
     PROVIDER=$(list_providers)
   fi
-  # If more than one argument was supplied, print usage message
+  # If more than two arguments were supplied, print usage message
-  if [ $# -gt 1 ]; then
+  if [ "$#" -gt 2 ]; then
     print_usage
     exit 1
   fi
-  if [ $# -eq 1 ]; then
+  if [ "$#" -ge 1 ]; then
     # If the user specifies the provider as an agument, set the variable
     # TODO: Check to make sure they actually have their provider installed
     case "$1" in
@@ -267,12 +303,121 @@ main() {
         ;;
     esac
   fi
+  if [ $# -eq 2 ]; then
+    case "$2" in
+      --packer-only)
+        PACKER_ONLY=1
+        ;;
+      --vagrant-only)
+        VAGRANT_ONLY=1
+        ;;
+      *)
+      echo -e "\"$2\" is not recognized as an option. Available options are:\\n--packer-only\\n--vagrant-only"
+      exit 1
+      ;;
+    esac
+  fi
+}
 
-  preflight_checks "$DL_DIR"
+build_packer_boxes() {
-  packer_build_box "$PROVIDER" "windows_2016" "$DL_DIR"
+  PACKER_BOXES=("windows_2016" "windows_10")
-  packer_build_box "$PROVIDER" "windows_10" "$DL_DIR"
-  move_boxes "$PROVIDER" "$DL_DIR"
 
+  if [ "$(hostname)" == "packerwindows10" ]; then   # Workaround for CI environment
+    (echo >&2 "CI Environment detected. If you are a user and are seeing this, please file an issue on GitHub.")
+    RET=$(packer_build_box "windows_10")
+    if [ "$RET" -eq 0 ]; then
+      (echo >&2 "Good news! The windows_10 box was built with Packer successfully!")
+    else
+      (echo >&2 "Something went wrong while attempting to build the windows_10 box.")
+      (echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
+      exit 1
+    fi
+  elif [ "$(hostname)" == "packerwindows2016" ]; then  # Workaround for CI environment
+    (echo >&2 "CI Environment detected. If you are a user and are seeing this, please file an issue on GitHub.")
+    RET=$(packer_build_box "windows_2016")
+    if [ "$RET" -eq 0 ]; then
+      (echo >&2 "Good news! The windows_2016 box was built with Packer successfully!")
+    else
+      (echo >&2 "Something went wrong while attempting to build the windows_2016 box.")
+      (echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
+      exit 1
+    fi
+  else
+    for PACKER_BOX in "${PACKER_BOXES[@]}"; do  # Normal user workflow
+      RET=$(packer_build_box "$PACKER_BOX")
+      if [ "$RET" -eq 0 ]; then
+        (echo >&2 "Good news! $PACKER_BOX was built successfully!")
+      else
+        (echo >&2 "Something went wrong while attempting to build the $PACKER_BOX box.")
+        (echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
+        exit 1
+      fi
+    done
+  fi
+}
+
+choose_md5_tool() {
+  if which md5; then
+    MD5TOOL="$(which md5)"
+    CUT_INDEX=4
+  else
+    MD5TOOL="$(which md5sum)"
+    CUT_INDEX=1
+  fi
+}
+
+# Downloads pre-built Packer boxes from detectionlab.network to save time during CI builds
+download_boxes() {
+  if [ "$PROVIDER" == "virtualbox" ]; then
+    wget "https://www.detectionlab.network/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box
+    wget "https://www.detectionlab.network/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box
+  elif [ "$PROVIDER" == "vmware_fusion" ]; then
+    wget "https://www.detectionlab.network/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box
+    wget "https://www.detectionlab.network/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box
+  fi
+
+  # Hacky workaround
+  if [ "$PROVIDER" == "vmware_fusion" ]; then
+    PROVIDER="vmware"
+  fi
+
+  # Ensure Windows 10 box exists
+  if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
+    (echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.")
+    exit 1
+  fi
+  # Ensure Windows 2016 box exists
+  if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
+    (echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.")
+    exit 1
+  fi
+  # Verify hashes of VirtualBox boxes
+  if [ "$PROVIDER" == "virtualbox" ]; then
+    if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then
+      (echo >&2 "Hash mismatch on windows_10_virtualbox.box")
+    fi
+    if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "614f984c82b51471b5bb753940b59d38" ]; then
+      (echo >&2 "Hash mismatch on windows_2016_virtualbox.box")
+    fi
+    # Verify hashes of VMware boxes
+  elif [ "$PROVIDER" == "vmware" ]; then
+    if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then
+      (echo >&2 "Hash mismatch on windows_10_vmware.box")
+      exit 1
+    fi
+    if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then
+      (echo >&2 "Hash mismatch on windows_2016_vmware.box")
+      exit 1
+    fi
+    # Reset PROVIDER variable if using VMware
+    if [ "$PROVIDER" == "vmware" ]; then
+      PROVIDER="vmware_fusion"
+    fi
+  fi
+}
+
+build_vagrant_hosts() {
+  LAB_HOSTS=("logger" "dc" "wef" "win10")
   # Change provider back to original selection if using vmware_fusion
   if [ "$PROVIDER" == "vmware" ]; then
     PROVIDER="vmware_fusion"
@@ -280,7 +425,7 @@ main() {
 
   # Vagrant up each box and attempt to reload one time if it fails
   for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do
-    RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR")
+    RET=$(vagrant_up_host "$VAGRANT_HOST")
     if [ "$RET" -eq 0 ]; then
       (echo >&2 "Good news! $VAGRANT_HOST was built successfully!")
     fi
@@ -288,16 +433,43 @@ main() {
     if [ "$RET" -ne 0 ]; then
       (echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.")
       (echo >&2 "Attempting to reload and reprovision the host...")
-      RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR")
+      RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST")
       if [ "$RETRY_STATUS" -ne 0 ]; then
         (echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
         exit 1
       fi
     fi
   done
-
-  post_build_checks
 }
 
+main() {
+  # Get location of build.sh
+  # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
+  DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+  PACKER_ONLY=0
+  VAGRANT_ONLY=0
+
+  parse_cli_arguments "$@"
+  preflight_checks
+  if [[ "$DOWNLOAD_BOXES" -eq 1 ]] && [[ "$VAGRANT_ONLY" -eq 1 ]]; then
+    download_boxes
+  fi
+
+  # Build Packer boxes if this isn't a Vagrant-only build
+  if [ "$VAGRANT_ONLY" -eq 0 ]; then
+    build_packer_boxes
+    # The only time we will need to move boxes is if we're doing a full build
+    if [ "$PACKER_ONLY" -eq 0 ]; then
+      move_boxes
+    fi
+  fi
+
+  # Build and Test Vagrant hosts if this isn't a Packer-only build
+  if [ "$PACKER_ONLY" -eq 0 ]; then
+    build_vagrant_hosts
+    post_build_checks
+  fi
+ }
+
 main "$@"
 exit 0

build_vagrant_only.sh

@@ -1,309 +0,0 @@
-#! /bin/bash
-
-# This script skips the entire Packer box building process.
-# This script downloads pre-built VirtualBox Packer images from S3 and puts
-# them into the "Boxes" directory.
-# Only MacOS and Linux are supported.
-# If you encounter issues, feel free to open an issue at
-# https://github.com/clong/DetectionLab/issues
-
-set -e
-
-print_usage() {
-  echo "Usage: ./build.sh <virtualbox|vmware_fusion>"
-  exit 0
-}
-
-check_vagrant() {
-  # Check for existence of Vagrant in PATH
-  if ! which vagrant >/dev/null; then
-    (echo >&2 "Vagrant was not found in your PATH.")
-    (echo >&2 "Please correct this before continuing. Quitting.")
-    exit 1
-  fi
-  # Ensure Vagrant >= 2.0.0
-  if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
-    (echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
-  fi
-}
-
-# Returns 0 if not installed or 1 if installed
-check_virtualbox_installed() {
-  if which VBoxManage >/dev/null; then
-    echo "1"
-  else
-    echo "0"
-  fi
-}
-
-# Returns 0 if not installed or 1 if installed
-check_vmware_fusion_installed() {
-  if [ -e "/Applications/VMware Fusion.app" ]; then
-    echo "1"
-  else
-    echo "0"
-  fi
-}
-
-# Returns 0 if not installed or 1 if installed
-check_vmware_vagrant_plugin_installed() {
-  VAGRANT_VMWARE_PLUGIN_PRESENT="$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')"
-  if [ "$VAGRANT_VMWARE_PLUGIN_PRESENT" -eq 0 ]; then
-    (echo >&2 "VMWare Fusion is installed, but the Vagrant plugin is not.")
-    (echo >&2 "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
-    (echo >&2 "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
-    echo "0"
-  else
-    echo "$VAGRANT_VMWARE_PLUGIN_PRESENT"
-  fi
-}
-
-# List the available Vagrant providers present on the system
-list_providers() {
-  VBOX_PRESENT=0
-  VMWARE_FUSION_PRESENT=0
-
-  if [ "$(uname)" == "Darwin" ]; then
-    # Detect Providers on OSX
-    VBOX_PRESENT=$(check_virtualbox_installed)
-    VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed)
-    VAGRANT_VMWARE_PLUGIN_PRESENT=$(check_vmware_vagrant_plugin_installed)
-  else
-    # Assume the only other available provider is VirtualBox
-    VBOX_PRESENT=$(check_virtualbox_installed)
-  fi
-
-  (echo >&2 "Available Providers:")
-  if [ "$VBOX_PRESENT" == "1" ]; then
-    (echo >&2 "virtualbox")
-  fi
-  if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]; then
-    (echo >&2 "vmware_fusion")
-  fi
-  if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]; then
-    (echo >&2 "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
-    exit 1
-  fi
-  (echo >&2 -e "\\nWhich provider would you like to use?")
-  read -r PROVIDER
-  # Sanity check
-  if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]; then
-    (echo >&2 "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
-    exit 1
-  fi
-  echo "$PROVIDER"
-}
-
-# A series of checks to identify potential issues before starting the build
-preflight_checks() {
-  DL_DIR="$1"
-  DOWNLOAD_BOXES=1
-
-  # Check to see if curl is in PATH
-  if ! which curl >/dev/null; then
-    (echo >&2 "Please install curl and make sure it is in your PATH.")
-    exit 1
-  fi
-  # Check to see if wget is in PATH
-  if ! which wget >/dev/null; then
-    (echo >&2 "Please install curl and make sure it is in your PATH.")
-    exit 1
-  fi
-  # Check to see if boxes exist already
-  BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l)
-  if [ "$BOXES_BUILT" -gt 0 ]; then
-    (echo >&2 "WARNING: You seem to have boxes present in the Boxes directory already. If you would like fresh boxes downloaded, please remove all files from the Boxes directory and re-run this script.")
-    DOWNLOAD_BOXES=0
-  fi
-  # Check to see if any Vagrant instances exist already
-  cd "$DL_DIR"/Vagrant/
-  # Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true"
-  VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true
-  if [ "$VAGRANT_BUILT" -ne 4 ]; then
-    (echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
-    exit 1
-  fi
-  # Check available disk space. Recommend 80GB free, warn if less.
-  FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
-  if [ "$FREE_DISK_SPACE" -lt 80000 ]; then
-    (echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n")
-    (df >&2 -m "$HOME")
-    (echo >&2 "")
-  fi
-  # Ensure the vagrant-reload plugin is installed
-  VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
-  if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
-    (echo >&2 "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
-    if ! $(which vagrant) plugin install "vagrant-reload"; then
-      (echo >&2 "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
-      exit 1
-    fi
-  fi
-}
-
-# Downloads pre-built Packer boxes from detectionlab.network to save time during CI builds
-download_boxes() {
-  DL_DIR="$1"
-  PROVIDER="$2"
-
-  if [ "$PROVIDER" == "virtualbox" ]; then
-    wget "https://www.detectionlab.network/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box
-    wget "https://www.detectionlab.network/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box
-  elif [ "$PROVIDER" == "vmware_fusion" ]; then
-    wget "https://www.detectionlab.network/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box
-    wget "https://www.detectionlab.network/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box
-  fi
-
-  # Hacky workaround
-  if [ "$PROVIDER" == "vmware_fusion" ]; then
-    PROVIDER="vmware"
-  fi
-
-  # Ensure Windows 10 box exists
-  if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
-    (echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.")
-    exit 1
-  fi
-  # Ensure Windows 2016 box exists
-  if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
-    (echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.")
-    exit 1
-  fi
-  # Verify hashes of VirtualBox boxes
-  if [ "$PROVIDER" == "virtualbox" ]; then
-    if [ "$(md5sum windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then
-      (echo >&2 "Hash mismatch on windows_10_virtualbox.box")
-    fi
-    if [ "$(md5sum windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "614f984c82b51471b5bb753940b59d38" ]; then
-      (echo >&2 "Hash mismatch on windows_2016_virtualbox.box")
-    fi
-    # Verify hashes of VMware boxes
-  elif [ "$PROVIDER" == "vmware" ]; then
-    if [ "$(md5 windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then
-      (echo >&2 "Hash mismatch on windows_10_vmware.box")
-      exit 1
-    fi
-    if [ "$(md5 windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then
-      (echo >&2 "Hash mismatch on windows_2016_vmware.box")
-      exit 1
-    fi
-    # Reset PROVIDER variable
-    PROVIDER="vmware_fusion"
-  fi
-}
-
-# Brings up a single host using Vagrant
-vagrant_up_host() {
-  PROVIDER="$1"
-  HOST="$2"
-  DL_DIR="$3"
-  (echo >&2 "Attempting to bring up the $HOST host using Vagrant")
-  cd "$DL_DIR"/Vagrant
-  $(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2
-  echo "$?"
-}
-
-# Attempts to reload and re-provision a host if the intial "vagrant up" fails
-vagrant_reload_host() {
-  HOST="$1"
-  DL_DIR="$2"
-  cd "$DL_DIR"/Vagrant
-  # Attempt to reload the host if the vagrant up command didn't exit cleanly
-  $(which vagrant) reload "$HOST" --provision 1>&2
-  echo "$?"
-}
-
-# A series of checks to ensure important services are responsive after the build completes.
-post_build_checks() {
-  # If the curl operation fails, we'll just leave the variable equal to 0
-  # This is needed to prevent the script from exiting if the curl operation fails
-  CALDERA_CHECK=$(curl -ks -m 2 https://192.168.38.5:8888 | grep -c '302: Found' || echo "")
-  SPLUNK_CHECK=$(curl -ks -m 2 https://192.168.38.5:8000/en-US/account/login?return_to=%2Fen-US%2F | grep -c 'This browser is not supported by Splunk' || echo "")
-  FLEET_CHECK=$(curl -ks -m 2 https://192.168.38.5:8412 | grep -c 'Kolide Fleet' || echo "")
-
-  BASH_MAJOR_VERSION=$(/bin/bash --version | grep 'GNU bash' | grep -o version\.\.. | cut -d ' ' -f 2 | cut -d '.' -f 1)
-  # Associative arrays are only supported in bash 4 and up
-  if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then
-    declare -A SERVICES
-    SERVICES=(["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
-    for SERVICE in "${!SERVICES[@]}"; do
-      if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then
-        (echo >&2 "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
-      fi
-    done
-  else
-    if [ "$CALDERA_CHECK" -lt 1 ]; then
-      (echo >&2 "Warning: Caldera failed post-build tests and may not be functioning correctly.")
-    fi
-    if [ "$SPLUNK_CHECK" -lt 1 ]; then
-      (echo >&2 "Warning: Splunk failed post-build tests and may not be functioning correctly.")
-    fi
-    if [ "$FLEET_CHECK" -lt 1 ]; then
-      (echo >&2 "Warning: Fleet failed post-build tests and may not be functioning correctly.")
-    fi
-  fi
-}
-
-main() {
-  # Get location of build_vagrant_only.sh
-  # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
-  DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-  PROVIDER=""
-  LAB_HOSTS=("logger" "dc" "wef" "win10")
-  # If no argument was supplied, list available providers
-  if [ $# -eq 0 ]; then
-    PROVIDER=$(list_providers)
-  fi
-  # If more than one argument was supplied, print usage message
-  if [ $# -gt 1 ]; then
-    print_usage
-    exit 1
-  fi
-  if [ $# -eq 1 ]; then
-    # If the user specifies the provider as an agument, set the variable
-    # TODO: Check to make sure they actually have their provider installed
-    case "$1" in
-      virtualbox)
-        PROVIDER="$1"
-        ;;
-      vmware_fusion)
-        PROVIDER="$1"
-        ;;
-      *)
-        echo "\"$1\" is not a valid provider. Listing available providers:"
-        PROVIDER=$(list_providers)
-        ;;
-    esac
-  fi
-
-  check_vagrant
-  preflight_checks "$DL_DIR"
-  if [ "$DOWNLOAD_BOXES" -eq 0 ]; then
-    (echo >&2 "Skipping box downloads since .box files are already present in the Boxes/ directory.")
-  else
-    download_boxes "$DL_DIR" "$PROVIDER"
-  fi
-
-  # Vagrant up each box and attempt to reload one time if it fails
-  for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do
-    RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR")
-    if [ "$RET" -eq 0 ]; then
-      (echo >&2 "Good news! $VAGRANT_HOST was built successfully!")
-    fi
-    # Attempt to recover if the intial "vagrant up" fails
-    if [ "$RET" -ne 0 ]; then
-      (echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.")
-      (echo >&2 "Attempting to reload and reprovision the host...")
-      RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR")
-      if [ "$RETRY_STATUS" -ne 0 ]; then
-        (echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
-        exit 1
-      fi
-    fi
-  done
-
-  post_build_checks
-}
-
-main "$@"
-exit 0

ci/README.md

@@ -1,6 +1,104 @@
-# ci
+# Continuous Integration
 
 The files in this directory are used to bootstrap an Ubuntu 16.04 baremetal server
 for continuous integration testing by installing the prerequisites needed for
 Detection Lab. After the prerequisites are installed, the build script is called
 and the build will begin in a tmux session.
+
+## Understanding the build process
+
+Once a PR is created, the contents of that PR will be copied to a CircleCI worker to be tested.
+The CircleCI worker will evaluate which files have been modified and set environment variables accordingly. There are 4 possible options and 3 different tests:
+
+1. Code in both the Packer and Vagrant directories was modified
+  * In this case, the CircleCI worker will execute `ci/circle_workflows/packer_and_vagrant_changes.sh`
+2. Code in neither the Packer and Vagrant directories was modified
+  * In this case, the CircleCI worker will execute the default test `ci/circle_worker/vagrant_changes.sh`
+3. Code in only the Packer directory was modified
+  * In this case, the CircleCI worker will execute `ci/circle_worker/packer_changes.sh`
+4. Code in only the Vagrant directory was modified
+  * In this case, the CircleCI worker will execute `ci/circle_worker/vagrant_changes.sh`
+
+## Test Case Walkthroughs
+
+### packer_and_vagrant_changes.sh
+1. Spins up a single Packet server
+2. Bootstraps the Packet server by calling `ci/build_machine_bootstrap.sh` with no arguments
+3. Builds the Windows10 and Windows2016 images one at a time
+4. Moves the resulting boxes to the Boxes directory
+5. Brings each Vagrant host online one-by-one
+6. CircleCI records the build results from the Packet server
+
+### vagrant_changes.sh
+1. Spins up a single Packet server
+2. Bootstraps the Packet server by calling `ci/build_machine_bootstrap.sh` with the `--vagrant-only` argument
+3. Downloads the pre-build Windows10 and Windows2016 boxes from https://detectionlab.network directly to the Boxes directory
+4. Brings each Vagrant host online one-by-one
+5. CircleCI records the build results from the Packet server
+
+
+### packer_changes.sh
+1. Spins up two separate Packet servers to allow the Packer boxes to be built in parallel
+2. Bootstraps each packet Server by calling `ci/build_machine_bootstrap.sh` with the `--packer-only` argument
+3. Starts the Packer build process on each server
+4. CircleCI records the build result from each Packet server
+
+```
+                                             +------------+
+                                             |            |
+                                             |            |
+                                             |            |
+                                             |   Github   |
+                                             |            |
+                                             |            |
+                                             +------+-----+
+                                                    |
+                                                    |
+                                                    | Pull Request
+                                                    |
+                                                    v
+                                             +------+-----+
+                                             |            |
+                                             |            |
+                                             |   Circle   |
+              +----------------------------->|   Worker   |
+              |                              |            |
+              |                              |            |
+              |                              |            |
+              |                              +------+-----+
+              |                                     |
+              |                                     | Code changes are evaluated
+              |                                     | to determine which test suite
+              |                                     | to run
+              |                                     |
+              |                                     v
+              |                    +----------------+--------------+
+Circle Worker |                    | packer_and_vagrant_changes.sh |
+quries for    |                    | vagrant_changes.sh            |
+build results |                    | packer_changes.sh             |        
+              |                    +----------------+--------------+             
+              |                                     |
+              |                                     |
+              |          |                          |
+              |                                     |
+              |                                     |
+              |                                     |
+              |                                     |
+              |                                     |  1. Provision Packet server(s)
+              |                                     |  2. Copy repo to server
+              |                                     |  3. Run server bootstrap
+              |                                     |  4. Bootstrap calls build.sh with
+              |                                     |  the appropriate arguments
+              |                                     |
+              |                                     |
+              |                           +---------v---------+
+              |                           |                   |
+              |                           |                   |
+              |                           |                   |
+              +-------------------------->|   Packet Server   |
+                                          |                   |
+                                          |                   |
+                                          |                   |
+                                          +-------------------+
+
+```

ci/automated_install.sh

@@ -1,54 +0,0 @@
-#! /bin/bash
-
-# This script is run on the Packet.net baremetal server for CI tests.
-# This script will build the entire lab from scratch and takes 3-4 hours
-# on a Packet.net host
-# While building, the server will start a webserver on Port 80 that contains
-# the text "building". Once the test is completed, the text will be replaced
-# with "success" or "failed".
-
-# Install Virtualbox 5.2
-echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
-wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
-apt-get update
-apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2
-
-echo "building" > /var/www/html/index.html
-
-# Set up firewall
-ufw allow ssh
-ufw allow http
-ufw default allow outgoing
-ufw --force enable
-
-# Install Vagrant
-mkdir /opt/vagrant
-cd /opt/vagrant || exit 1
-wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb
-dpkg -i vagrant_2.0.2_x86_64.deb
-vagrant plugin install vagrant-reload
-
-# Install Packer
-mkdir /opt/packer
-cd /opt/packer || exit 1
-wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip
-unzip packer_1.1.3_linux_amd64.zip
-cp packer /usr/local/bin/packer
-
-# Make the packer images headless
-for file in $(ls *.json); do
-  sed -i 's/"headless": false,/"headless": true,/g' "$file";
-done
-
-# Make the Vagrant instances headless
-cd /opt/DetectionLab/Vagrant || exit 1
-sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
-
-# Ensure the script is executable
-chmod +x /opt/DetectionLab/build.sh
-cd /opt/DetectionLab || exit 1
-
-# Start the build in a tmux session
-sn=tmuxsession
-tmux new-session -s "$sn" -d
-tmux send-keys -t "$sn:0" './build.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter

ci/automated_install_vagrant_only.sh

@@ -1,42 +0,0 @@
-#! /bin/bash
-
-# This script is run on the Packet.net baremetal server for CI tests.
-# This script will bootstrap the build by downloading pre-build Packer boxes
-# and should take no more than 90 minutes on a Packet.net host.
-# While building, the server will start a webserver on Port 80 that contains
-# the text "building". Once the test is completed, the text will be replaced
-# with "success" or "failed".
-
-# Install Virtualbox 5.2
-echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
-wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
-apt-get update
-apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2
-
-echo "building" > /var/www/html/index.html
-
-# Set up firewall
-ufw allow ssh
-ufw allow http
-ufw default allow outgoing
-ufw --force enable
-
-# Install Vagrant
-mkdir /opt/vagrant
-cd /opt/vagrant || exit 1
-wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb
-dpkg -i vagrant_2.0.2_x86_64.deb
-vagrant plugin install vagrant-reload
-
-# Make the Vagrant instances headless
-cd /opt/DetectionLab/Vagrant || exit 1
-sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
-
-# Ensure the script is executable
-chmod +x /opt/DetectionLab/build_vagrant_only.sh
-cd /opt/DetectionLab || exit 1
-
-# Start the build in a tmux session
-sn=tmuxsession
-tmux new-session -s "$sn" -d
-tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter

ci/build_machine_bootstrap.sh

@@ -0,0 +1,93 @@
+#! /bin/bash
+
+# This script is run on the Packet.net baremetal server for CI tests.
+# This script will build the entire lab from scratch and takes 3-4 hours
+# on a Packet.net host
+# While building, the server will start a webserver on Port 80 that contains
+# the text "building". Once the test is completed, the text will be replaced
+# with "success" or "failed".
+
+ARGS="$1"
+PACKER_ONLY=0
+VAGRANT_ONLY=0
+
+if [ ! -z "$1" ]; then
+  case "$1" in
+    --packer-only)
+    PACKER_ONLY=1
+    ;;
+    --vagrant-only)
+    VAGRANT_ONLY=1
+    ;;
+    *)
+    echo "\"$ARGS\" is not a supported argument to this script. Quitting"
+    exit 1
+    ;;
+  esac
+fi
+
+echo "Args: $ARGS"
+
+if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then
+  echo "Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong."
+  exit 1
+fi
+
+# Install Virtualbox 5.2
+echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
+wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
+apt-get update
+apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2
+
+echo "building" > /var/www/html/index.html
+
+# Set up firewall
+ufw allow ssh
+ufw allow http
+ufw default allow outgoing
+ufw --force enable
+
+if [ "$PACKER_ONLY" -eq 0 ]; then
+  # Install Vagrant
+  mkdir /opt/vagrant
+  cd /opt/vagrant || exit 1
+  wget https://releases.hashicorp.com/vagrant/2.0.3/vagrant_2.0.3_x86_64.deb
+  dpkg -i vagrant_2.0.3_x86_64.deb
+  vagrant plugin install vagrant-reload
+
+  # Make the Vagrant instances headless
+  cd /opt/DetectionLab/Vagrant || exit 1
+  sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
+fi
+
+if [ "$VAGRANT_ONLY" -eq 0 ]; then
+  # Install Packer
+  mkdir /opt/packer
+  cd /opt/packer || exit 1
+  wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip
+  unzip packer_1.1.3_linux_amd64.zip
+  cp packer /usr/local/bin/packer
+
+  # Make the Packer images headless
+  cd /opt/DetectionLab/Packer || exit 1
+  for file in *.json; do
+    sed -i 's/"headless": false,/"headless": true,/g' "$file";
+  done
+fi
+
+# Ensure the script is executable
+chmod +x /opt/DetectionLab/build.sh
+cd /opt/DetectionLab || exit 1
+
+# Start the build in a tmux session
+sn=tmuxsession
+tmux new-session -s "$sn" -d
+if [ "$PACKER_ONLY" -eq 1 ]; then
+  tmux send-keys -t "$sn:0" './build.sh virtualbox --packer-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
+fi
+if [ "$VAGRANT_ONLY" -eq 1 ]; then
+  tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
+fi
+if [[ "$PACKER_ONLY" -eq 0 ]] && [[ "$VAGRANT_ONLY" -eq 0 ]]; then
+  tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
+fi

ci/circle_workflows/packer_and_vagrant_changes.sh

@@ -0,0 +1,80 @@
+#! /bin/bash
+
+set -e
+
+# Create artifacts directory
+if [ ! -d "/tmp/artifacts" ]; then
+  mkdir /tmp/artifacts
+fi
+
+## Delete stale servers if they exist
+DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token:  '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
+if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then
+  echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID"
+  curl -X DELETE -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
+fi
+
+## Provision a Type1 baremetal Packet.net server
+echo "Provisioning a server on Packet.net"
+DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
+# Make sure the device ID is sane.
+# TODO: maybe make this a regex
+if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
+  echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
+  exit 1
+fi
+echo "Server successfully provisioned with ID: $DEVICE_ID"
+
+echo "Sleeping 10 minutes to wait for Packet server to be provisioned"
+sleep 300
+echo "Sleeping 5 more minutes (CircleCI Keepalive)"
+sleep 300
+
+## Recording the IP address of the newly provisioned Packet server
+IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
+
+# Copy repo to Packet server
+# TODO: Tar up the repo and expand it remotely
+cd ~/repo
+rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
+
+## Running install script on Packet server
+ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh
+
+echo "Sleeping 5 minutes to allow the build process to start"
+sleep 300
+
+## Waiting for Packet server to post build results
+MINUTES_PAST=0
+while [ "$MINUTES_PAST" -lt 400 ]; do
+  STATUS=$(curl $IP_ADDRESS)
+  if [ "$STATUS" == "building" ]; then
+    echo "$STATUS"
+    scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present"
+    scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
+    sleep 300
+    ((MINUTES_PAST += 5))
+  else
+    break
+  fi
+done
+if [ "$MINUTES_PAST" -gt 400 ]; then
+  echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
+  scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present"
+  scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
+  curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
+  exit 1
+fi
+
+## Recording the build results
+echo $STATUS
+if [ "$STATUS" != "success" ]; then
+  echo "Build failed. Cleaning up server with ID $DEVICE_ID"
+  scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present"
+  scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
+  curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
+  exit 1
+fi
+echo "Build was successful. Cleaning up server with ID $DEVICE_ID"
+curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
+exit 0

ci/circle_workflows/packer_changes.sh

@@ -0,0 +1,99 @@
+#! /bin/bash
+
+set -e
+
+# Create artifacts directory
+if [ ! -d "/tmp/artifacts" ]; then
+  mkdir /tmp/artifacts
+fi
+
+## Delete stale servers if they exist
+echo "Deleting stale Packet.net servers"
+DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
+if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then
+  echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID"
+  curl -X DELETE -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
+fi
+
+## Provision two Type1 baremetal Packet.net servers
+echo "Provisioning packerwindows2016 on Packet.net"
+SERVER1_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows2016", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
+if [ "$(echo -n $SERVER1_ID | wc -c)" -ne 36 ]; then
+  echo "Server may have failed provisionining. Device ID is set to: $SERVER1_ID"
+  exit 1
+fi
+echo "packerwindows2016 successfully provisioned with ID: $SERVER1_ID"
+
+sleep 5 # Wait a bit before issuing another provision command
+
+echo "Provisioning packerwindows10 on Packet.net"
+SERVER2_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows10", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
+if [ "$(echo -n $SERVER2_ID | wc -c)" -ne 36 ]; then
+  echo "Server may have failed provisionining. Device ID is set to: $SERVER2_ID"
+  exit 1
+fi
+echo "packerwindows10 successfully provisioned with ID: $SERVER2_ID"
+
+echo "Sleeping 10 minutes to wait for Packet servers to finish provisiong"
+sleep 300
+echo "Sleeping 5 more minutes (CircleCI Keepalive)"
+sleep 300
+
+## Recording the IP address of the newly provisioned Packet servers
+SERVER1_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER1_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
+SERVER2_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER2_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
+
+# Copy repo to Packet servers
+# TODO: Tar up the repo and expand it remotely
+cd ~/repo
+rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab
+rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab
+
+## Running install script on Packet server
+ssh -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only
+ssh -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only
+
+sleep 30
+
+## Waiting for Packet server to post build results
+MINUTES_PAST=0
+while [ "$MINUTES_PAST" -lt 150 ]; do
+  SERVER1_STATUS=$(curl $SERVER1_IP_ADDRESS)
+  SERVER2_STATUS=$(curl $SERVER2_IP_ADDRESS)
+  if [[ "$SERVER1_STATUS" == "building" ]] || [[ "$SERVER2_STATUS" == "building" ]]; then
+    echo "$SERVER1_STATUS" :: "$SERVER2_STATUS"
+    scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log
+    scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log
+    sleep 300
+    ((MINUTES_PAST += 5))
+  fi
+  if [[ "$SERVER1_STATUS" != "building" ]] && [[ "$SERVER2_STATUS" != "building" ]]; then
+    break
+  fi
+  if [ "$MINUTES_PAST" -gt 150 ]; then
+    echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
+    curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
+    curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
+    exit 1
+  fi
+done
+
+## Recording the build results
+echo "Server1 Status: $SERVER1_STATUS"
+echo "Server2 Status: $SERVER2_STATUS"
+if [ "$SERVER1_STATUS" != "success" ]; then
+  echo "Build failed. Cleaning up server with ID $SERVER1_ID"
+  scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log || echo "Serveer1 packer_build.log not available yet"
+  curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
+  exit 1
+fi
+if [ "$SERVER2_STATUS" != "success" ]; then
+  echo "Build failed. Cleaning up server with ID $SERVER2_ID"
+  scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log || echo "Server2 packer_build.log not available yet"
+  curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
+  exit 1
+fi
+echo "Builds were successful. Cleaning up servers with IDs $SERVER1_ID and $SERVER2_ID"
+curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
+curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
+exit 0

ci/circle_workflows/vagrant_changes.sh

@@ -0,0 +1,74 @@
+#! /bin/bash
+
+set -e
+
+# Create artifacts directory
+if [ ! -d "/tmp/artifacts" ]; then
+  mkdir /tmp/artifacts
+fi
+
+## Delete stale servers if they exist
+DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token:  '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
+if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then
+  echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID"
+  curl -X -s DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
+fi
+
+## Provision a Type1 baremetal Packet.net server
+echo "Provisioning a server on Packet.net"
+DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
+# Make sure the device ID is sane.
+# TODO: maybe make this a regex
+if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
+  echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
+  exit 1
+fi
+echo "Server successfully provisioned with ID: $DEVICE_ID"
+
+echo "Sleeping 10 minutes to wait for Packet server to be provisioned"
+sleep 300
+echo "Sleeping 5 more minutes (CircleCI Keepalive)"
+sleep 300
+
+## Recording the IP address of the newly provisioned Packet server
+IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
+
+# Copy repo to Packet server
+# TODO: Tar up the repo and expand it remotely
+cd ~/repo
+rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
+
+## Running install script on Packet server
+ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --vagrant-only
+
+## Waiting for Packet server to post build results
+MINUTES_PAST=0
+while [ "$MINUTES_PAST" -lt 120 ]; do
+  STATUS=$(curl $IP_ADDRESS)
+  if [ "$STATUS" == "building" ]; then
+    echo "$STATUS"
+    scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "vagrant_build.log not available yet"
+    sleep 300
+    ((MINUTES_PAST += 5))
+  else
+    break
+  fi
+  if [ "$MINUTES_PAST" -gt 120 ]; then
+    echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
+    scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log
+    curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
+    exit 1
+  fi
+done
+
+## Recording the build results
+echo $STATUS
+if [ "$STATUS" != "success" ]; then
+  scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log
+  echo "Build failed. Cleaning up server with ID $DEVICE_ID"
+  curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
+  exit 1
+fi
+echo "Build was successful. Cleaning up server with ID $DEVICE_ID"
+curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
+exit 0