From c0929c8cd31b67ced119e9e85577453d1c66aeb3 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Wed, 28 Feb 2018 01:02:20 -0800 Subject: [PATCH 01/16] Fixing formatting and some shell anti-patterns --- build.sh | 176 ++++++++++++++++++++---------------------- build_vagrant_only.sh | 173 ++++++++++++++++++++--------------------- 2 files changed, 172 insertions(+), 177 deletions(-) diff --git a/build.sh b/build.sh index 279a8cc..c43613f 100755 --- a/build.sh +++ b/build.sh @@ -16,30 +16,27 @@ print_usage() { check_packer_and_vagrant() { # Check for existence of Vagrant in PATH - which vagrant > /dev/null - if [ "$?" -ne 0 ]; then - (>&2 echo "Vagrant was not found in your PATH.") - (>&2 echo "Please correct this before continuing. Quitting.") + if ! which vagrant >/dev/null; then + (echo >&2 "Vagrant was not found in your PATH.") + (echo >&2 "Please correct this before continuing. Quitting.") exit 1 fi # Ensure Vagrant >= 2.0.0 if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then - (>&2 echo "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing") + (echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing") fi # Check for existence of Packer in PATH - which packer > /dev/null - if [ "$?" -ne 0 ]; then - (>&2 echo "Packer was not found in your PATH.") - (>&2 echo "Please correct this before continuing. Quitting.") - (>&2 echo "Hint: sudo cp ./packer /usr/local/bin/packer; sudo chmod +x /usr/local/bin/packer") + if ! which packer >/dev/null; then + (echo >&2 "Packer was not found in your PATH.") + (echo >&2 "Please correct this before continuing. Quitting.") + (echo >&2 "Hint: sudo cp ./packer /usr/local/bin/packer; sudo chmod +x /usr/local/bin/packer") exit 1 fi } # Returns 0 if not installed or 1 if installed check_virtualbox_installed() { - which VBoxManage > /dev/null - if [ "$?" -eq 0 ]; then + if which VBoxManage >/dev/null; then echo "1" else echo "0" @@ -48,19 +45,23 @@ check_virtualbox_installed() { # Returns 0 if not installed or 1 if installed check_vmware_fusion_installed() { - echo "$(ls /Applications | grep -ci 'VMware Fusion.app')" + if [ -e "/Applications/VMware Fusion.app" ]; then + echo "1" + else + echo "0" + fi } # Returns 0 if not installed or 1 if installed check_vmware_vagrant_plugin_installed() { - VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion') - if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then - (>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.") - (>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it") - (>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.") + VAGRANT_VMWARE_PLUGIN_PRESENT="$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')" + if [ "$VAGRANT_VMWARE_PLUGIN_PRESENT" -eq 0 ]; then + (echo >&2 "VMWare Fusion is installed, but the Vagrant plugin is not.") + (echo >&2 "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it") + (echo >&2 "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.") echo "0" else - echo $VAGRANT_VMWARE_PLUGIN_PRESENT + echo "$VAGRANT_VMWARE_PLUGIN_PRESENT" fi } @@ -69,7 +70,7 @@ list_providers() { VBOX_PRESENT=0 VMWARE_FUSION_PRESENT=0 - if [ $(uname) == "Darwin" ]; then + if [ "$(uname)" == "Darwin" ]; then # Detect Providers on OSX VBOX_PRESENT=$(check_virtualbox_installed) VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed) @@ -79,28 +80,25 @@ list_providers() { VBOX_PRESENT=$(check_virtualbox_installed) fi - (>&2 echo "Available Providers:") + (echo >&2 "Available Providers:") if [ "$VBOX_PRESENT" == "1" ]; then - (>&2 echo "virtualbox";) + (echo >&2 "virtualbox") fi - if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]] - then - (>&2 echo "vmware_fusion";) + if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]; then + (echo >&2 "vmware_fusion") fi - if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]] - then - (>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.") + if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]; then + (echo >&2 "You need to install a provider such as VirtualBox or VMware Fusion to continue.") exit 1 fi - (>&2 echo -e "\nWhich provider would you like to use?") - read PROVIDER + (echo >&2 -e "\\nWhich provider would you like to use?") + read -r PROVIDER # Sanity check - if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]] - then - (>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option") + if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]; then + (echo >&2 "Please choose a valid provider. \"$PROVIDER\" is not a valid option") exit 1 fi - echo $PROVIDER + echo "$PROVIDER" } # A series of checks to identify potential issues before starting the build @@ -108,43 +106,42 @@ preflight_checks() { DL_DIR="$1" # Check to see if curl is in PATH - which curl > /dev/null - if [ "$?" -ne 0 ]; then - (>&2 echo "Please install curl and make sure it is in your PATH.") + if ! which curl >/dev/null; then + (echo >&2 "Please install curl and make sure it is in your PATH.") exit 1 fi # Check to see if boxes exist already - BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l) - if [ $BOXES_BUILT -gt 0 ]; then - (>&2 echo "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.") + BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l) + if [ "$BOXES_BUILT" -gt 0 ]; then + (echo >&2 "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.") exit 1 fi # Check to see if any Vagrant instances exist already cd "$DL_DIR"/Vagrant/ - VAGRANT_BUILT=$(vagrant status | grep -c 'not created') - if [ $VAGRANT_BUILT -ne 4 ]; then - (>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.") + # Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true" + VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true + if [ "$VAGRANT_BUILT" -ne 4 ]; then + (echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.") exit 1 fi # Check available disk space. Recommend 80GB free, warn if less. - FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4) - if [ $FREE_DISK_SPACE -lt 80000 ]; then - (>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n") - (>&2 df -m $HOME) - (>&2 echo "") + FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4) + if [ "$FREE_DISK_SPACE" -lt 80000 ]; then + (echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n") + (df >&2 -m "$HOME") + (echo >&2 "") fi # Check Packer version against known bad if [ "$(packer --version)" == '1.1.2' ]; then - (>&2 echo "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.") + (echo >&2 "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.") exit 1 fi # Ensure the vagrant-reload plugin is installed VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload') if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then - (>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.") - $(which vagrant) plugin install "vagrant-reload" - if [ "$?" -ne 0 ]; then - (>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.") + (echo >&2 "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.") + if ! $(which vagrant) plugin install "vagrant-reload"; then + (echo >&2 "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.") exit 1 fi fi @@ -159,11 +156,10 @@ packer_build_box() { PROVIDER="vmware" fi cd "$DL_DIR/Packer" - (>&2 echo "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.") - $(which packer) build --only="$PROVIDER-iso" $BOX.json - if [ "$?" -ne 0 ]; then - (>&2 echo "Something went wrong while attempting to build the $BOX box.") - (>&2 echo "To file an issue, please visit https://github.com/clong/DetectionLab/issues/") + (echo >&2 "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.") + if ! $(which packer) build --only="$PROVIDER-iso" "$BOX".json; then + (echo >&2 "Something went wrong while attempting to build the $BOX box.") + (echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/") fi } @@ -178,12 +174,12 @@ move_boxes() { mv "$DL_DIR"/Packer/*.box "$DL_DIR"/Boxes # Ensure Windows 10 box exists if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then - (>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.") + (echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.") exit 1 fi # Ensure Windows 2016 box exists if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then - (>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.") + (echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.") exit 1 fi } @@ -193,9 +189,9 @@ vagrant_up_host() { PROVIDER="$1" HOST="$2" DL_DIR="$3" - (>&2 echo "Attempting to bring up the $HOST host using Vagrant") + (echo >&2 "Attempting to bring up the $HOST host using Vagrant") cd "$DL_DIR"/Vagrant - $(which vagrant) up $HOST --provider="$PROVIDER" 1>&2 + $(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2 echo "$?" } @@ -205,7 +201,7 @@ vagrant_reload_host() { DL_DIR="$2" cd "$DL_DIR"/Vagrant # Attempt to reload the host if the vagrant up command didn't exit cleanly - $(which vagrant) reload $HOST --provision 1>&2 + $(which vagrant) reload "$HOST" --provision 1>&2 echo "$?" } @@ -221,22 +217,21 @@ post_build_checks() { # Associative arrays are only supported in bash 4 and up if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then declare -A SERVICES - SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK") - for SERVICE in "${!SERVICES[@]}" - do + SERVICES=(["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK") + for SERVICE in "${!SERVICES[@]}"; do if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then - (>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.") + (echo >&2 "Warning: $SERVICE failed post-build tests and may not be functioning correctly.") fi done else if [ "$CALDERA_CHECK" -lt 1 ]; then - (>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.") + (echo >&2 "Warning: Caldera failed post-build tests and may not be functioning correctly.") fi if [ "$SPLUNK_CHECK" -lt 1 ]; then - (>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.") + (echo >&2 "Warning: Splunk failed post-build tests and may not be functioning correctly.") fi if [ "$FLEET_CHECK" -lt 1 ]; then - (>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.") + (echo >&2 "Warning: Fleet failed post-build tests and may not be functioning correctly.") fi fi } @@ -244,7 +239,7 @@ post_build_checks() { main() { # Get location of build.sh # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within - DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROVIDER="" LAB_HOSTS=("logger" "dc" "wef" "win10") # If no argument was supplied, list available providers @@ -261,22 +256,22 @@ main() { # TODO: Check to make sure they actually have their provider installed case "$1" in virtualbox) - PROVIDER="$1" - ;; + PROVIDER="$1" + ;; vmware_fusion) - PROVIDER="$1" - ;; + PROVIDER="$1" + ;; *) - echo "\"$1\" is not a valid provider. Listing available providers:" - PROVIDER=$(list_providers) - ;; + echo "\"$1\" is not a valid provider. Listing available providers:" + PROVIDER=$(list_providers) + ;; esac fi - preflight_checks $DL_DIR - packer_build_box $PROVIDER "windows_2016" $DL_DIR - packer_build_box $PROVIDER "windows_10" $DL_DIR - move_boxes $PROVIDER $DL_DIR + preflight_checks "$DL_DIR" + packer_build_box "$PROVIDER" "windows_2016" "$DL_DIR" + packer_build_box "$PROVIDER" "windows_10" "$DL_DIR" + move_boxes "$PROVIDER" "$DL_DIR" # Change provider back to original selection if using vmware_fusion if [ "$PROVIDER" == "vmware" ]; then @@ -284,19 +279,18 @@ main() { fi # Vagrant up each box and attempt to reload one time if it fails - for VAGRANT_HOST in "${LAB_HOSTS[@]}" - do - RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR) + for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do + RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR") if [ "$RET" -eq 0 ]; then - (>&2 echo "Good news! $VAGRANT_HOST was built successfully!") + (echo >&2 "Good news! $VAGRANT_HOST was built successfully!") fi # Attempt to recover if the intial "vagrant up" fails if [ "$RET" -ne 0 ]; then - (>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.") - (>&2 echo "Attempting to reload and reprovision the host...") - RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR) + (echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.") + (echo >&2 "Attempting to reload and reprovision the host...") + RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR") if [ "$RETRY_STATUS" -ne 0 ]; then - (>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.") + (echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.") exit 1 fi fi @@ -305,5 +299,5 @@ main() { post_build_checks } -main $@ +main "$@" exit 0 diff --git a/build_vagrant_only.sh b/build_vagrant_only.sh index 673c34e..5e17595 100755 --- a/build_vagrant_only.sh +++ b/build_vagrant_only.sh @@ -16,22 +16,20 @@ print_usage() { check_vagrant() { # Check for existence of Vagrant in PATH - which vagrant > /dev/null - if [ "$?" -ne 0 ]; then - (>&2 echo "Vagrant was not found in your PATH.") - (>&2 echo "Please correct this before continuing. Quitting.") + if ! which vagrant >/dev/null; then + (echo >&2 "Vagrant was not found in your PATH.") + (echo >&2 "Please correct this before continuing. Quitting.") exit 1 fi # Ensure Vagrant >= 2.0.0 if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then - (>&2 echo "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing") + (echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing") fi } # Returns 0 if not installed or 1 if installed check_virtualbox_installed() { - which VBoxManage > /dev/null - if [ "$?" -eq 0 ]; then + if ! which VBoxManage >/dev/null; then echo "1" else echo "0" @@ -40,19 +38,23 @@ check_virtualbox_installed() { # Returns 0 if not installed or 1 if installed check_vmware_fusion_installed() { - echo "$(ls /Applications | grep -ci 'VMware Fusion.app')" + if [ -e "/Applications/VMware Fusion.app" ]; then + echo "1" + else + echo "0" + fi } # Returns 0 if not installed or 1 if installed check_vmware_vagrant_plugin_installed() { - VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion') - if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then - (>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.") - (>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it") - (>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.") + VAGRANT_VMWARE_PLUGIN_PRESENT="$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')" + if [ "$VAGRANT_VMWARE_PLUGIN_PRESENT" -eq 0 ]; then + (echo >&2 "VMWare Fusion is installed, but the Vagrant plugin is not.") + (echo >&2 "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it") + (echo >&2 "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.") echo "0" else - echo $VAGRANT_VMWARE_PLUGIN_PRESENT + echo "$VAGRANT_VMWARE_PLUGIN_PRESENT" fi } @@ -61,7 +63,7 @@ list_providers() { VBOX_PRESENT=0 VMWARE_FUSION_PRESENT=0 - if [ $(uname) == "Darwin" ]; then + if [ "$(uname)" == "Darwin" ]; then # Detect Providers on OSX VBOX_PRESENT=$(check_virtualbox_installed) VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed) @@ -71,72 +73,69 @@ list_providers() { VBOX_PRESENT=$(check_virtualbox_installed) fi - (>&2 echo "Available Providers:") + (echo >&2 "Available Providers:") if [ "$VBOX_PRESENT" == "1" ]; then - (>&2 echo "virtualbox";) + (echo >&2 "virtualbox") fi - if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]] - then - (>&2 echo "vmware_fusion";) + if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]; then + (echo >&2 "vmware_fusion") fi - if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]] - then - (>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.") + if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]; then + (echo >&2 "You need to install a provider such as VirtualBox or VMware Fusion to continue.") exit 1 fi - (>&2 echo -e "\nWhich provider would you like to use?") - read PROVIDER + (echo >&2 -e "\\nWhich provider would you like to use?") + read -r PROVIDER # Sanity check - if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]] - then - (>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option") + if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]; then + (echo >&2 "Please choose a valid provider. \"$PROVIDER\" is not a valid option") exit 1 fi - echo $PROVIDER + echo "$PROVIDER" } # A series of checks to identify potential issues before starting the build preflight_checks() { DL_DIR="$1" + DOWNLOAD_BOXES=1 # Check to see if curl is in PATH - which curl > /dev/null - if [ "$?" -ne 0 ]; then - (>&2 echo "Please install curl and make sure it is in your PATH.") + if ! which curl >/dev/null; then + (echo >&2 "Please install curl and make sure it is in your PATH.") exit 1 fi # Check to see if wget is in PATH - which wget > /dev/null - if [ "$?" -ne 0 ]; then - (>&2 echo "Please install curl and make sure it is in your PATH.") + if ! which wget >/dev/null; then + (echo >&2 "Please install curl and make sure it is in your PATH.") exit 1 fi # Check to see if boxes exist already - BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l) - if [ $BOXES_BUILT -gt 0 ]; then - (>&2 echo "WARNING: You seem to have boxes present in the Boxes/ directory already. If you would like fresh boxes downloaded, please remove all files from the Boxes/ directory and re-run this script.") + BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l) + if [ "$BOXES_BUILT" -gt 0 ]; then + (echo >&2 "WARNING: You seem to have boxes present in the Boxes directory already. If you would like fresh boxes downloaded, please remove all files from the Boxes directory and re-run this script.") + DOWNLOAD_BOXES=0 fi # Check to see if any Vagrant instances exist already cd "$DL_DIR"/Vagrant/ - VAGRANT_BUILT=$(vagrant status | grep -c 'not created') - if [ $VAGRANT_BUILT -ne 4 ]; then - (>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.") + # Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true" + VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true + if [ "$VAGRANT_BUILT" -ne 4 ]; then + (echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.") exit 1 fi # Check available disk space. Recommend 80GB free, warn if less. - FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4) - if [ $FREE_DISK_SPACE -lt 80000 ]; then - (>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n") - (>&2 df -m $HOME) - (>&2 echo "") + FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4) + if [ "$FREE_DISK_SPACE" -lt 80000 ]; then + (echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n") + (df >&2 -m "$HOME") + (echo >&2 "") fi # Ensure the vagrant-reload plugin is installed VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload') if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then - (>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.") - $(which vagrant) plugin install "vagrant-reload" - if [ "$?" -ne 0 ]; then - (>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.") + (echo >&2 "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.") + if ! $(which vagrant) plugin install "vagrant-reload"; then + (echo >&2 "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.") exit 1 fi fi @@ -162,30 +161,30 @@ download_boxes() { # Ensure Windows 10 box exists if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then - (>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.") + (echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.") exit 1 fi # Ensure Windows 2016 box exists if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then - (>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.") + (echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.") exit 1 fi # Verify hashes of VirtualBox boxes if [ "$PROVIDER" == "virtualbox" ]; then if [ "$(md5sum windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then - (>&2 echo "Hash mismatch on windows_10_virtualbox.box") + (echo >&2 "Hash mismatch on windows_10_virtualbox.box") fi if [ "$(md5sum windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "614f984c82b51471b5bb753940b59d38" ]; then - (>&2 echo "Hash mismatch on windows_2016_virtualbox.box") + (echo >&2 "Hash mismatch on windows_2016_virtualbox.box") fi - # Verify hashes of VMware boxes + # Verify hashes of VMware boxes elif [ "$PROVIDER" == "vmware" ]; then if [ "$(md5 windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then - (>&2 echo "Hash mismatch on windows_10_vmware.box") + (echo >&2 "Hash mismatch on windows_10_vmware.box") exit 1 fi if [ "$(md5 windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then - (>&2 echo "Hash mismatch on windows_2016_vmware.box") + (echo >&2 "Hash mismatch on windows_2016_vmware.box") exit 1 fi # Reset PROVIDER variable @@ -198,9 +197,9 @@ vagrant_up_host() { PROVIDER="$1" HOST="$2" DL_DIR="$3" - (>&2 echo "Attempting to bring up the $HOST host using Vagrant") + (echo >&2 "Attempting to bring up the $HOST host using Vagrant") cd "$DL_DIR"/Vagrant - $(which vagrant) up $HOST --provider="$PROVIDER" 1>&2 + $(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2 echo "$?" } @@ -210,7 +209,7 @@ vagrant_reload_host() { DL_DIR="$2" cd "$DL_DIR"/Vagrant # Attempt to reload the host if the vagrant up command didn't exit cleanly - $(which vagrant) reload $HOST --provision 1>&2 + $(which vagrant) reload "$HOST" --provision 1>&2 echo "$?" } @@ -226,22 +225,21 @@ post_build_checks() { # Associative arrays are only supported in bash 4 and up if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then declare -A SERVICES - SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK") - for SERVICE in "${!SERVICES[@]}" - do + SERVICES=(["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK") + for SERVICE in "${!SERVICES[@]}"; do if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then - (>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.") + (echo >&2 "Warning: $SERVICE failed post-build tests and may not be functioning correctly.") fi done else if [ "$CALDERA_CHECK" -lt 1 ]; then - (>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.") + (echo >&2 "Warning: Caldera failed post-build tests and may not be functioning correctly.") fi if [ "$SPLUNK_CHECK" -lt 1 ]; then - (>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.") + (echo >&2 "Warning: Splunk failed post-build tests and may not be functioning correctly.") fi if [ "$FLEET_CHECK" -lt 1 ]; then - (>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.") + (echo >&2 "Warning: Fleet failed post-build tests and may not be functioning correctly.") fi fi } @@ -249,7 +247,7 @@ post_build_checks() { main() { # Get location of build_vagrant_only.sh # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within - DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROVIDER="" LAB_HOSTS=("logger" "dc" "wef" "win10") # If no argument was supplied, list available providers @@ -266,36 +264,39 @@ main() { # TODO: Check to make sure they actually have their provider installed case "$1" in virtualbox) - PROVIDER="$1" - ;; + PROVIDER="$1" + ;; vmware_fusion) - PROVIDER="$1" - ;; + PROVIDER="$1" + ;; *) - echo "\"$1\" is not a valid provider. Listing available providers:" - PROVIDER=$(list_providers) - ;; + echo "\"$1\" is not a valid provider. Listing available providers:" + PROVIDER=$(list_providers) + ;; esac fi check_vagrant - preflight_checks $DL_DIR - download_boxes $DL_DIR $PROVIDER + preflight_checks "$DL_DIR" + if [ "$DOWNLOAD_BOXES" -eq 0 ]; then + (echo >&2 "Skipping box downloads since .box files are already present in the Boxes/ directory.") + else + download_boxes "$DL_DIR" "$PROVIDER" + fi # Vagrant up each box and attempt to reload one time if it fails - for VAGRANT_HOST in "${LAB_HOSTS[@]}" - do - RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR) + for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do + RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR") if [ "$RET" -eq 0 ]; then - (>&2 echo "Good news! $VAGRANT_HOST was built successfully!") + (echo >&2 "Good news! $VAGRANT_HOST was built successfully!") fi # Attempt to recover if the intial "vagrant up" fails if [ "$RET" -ne 0 ]; then - (>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.") - (>&2 echo "Attempting to reload and reprovision the host...") - RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR) + (echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.") + (echo >&2 "Attempting to reload and reprovision the host...") + RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR") if [ "$RETRY_STATUS" -ne 0 ]; then - (>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.") + (echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.") exit 1 fi fi @@ -304,5 +305,5 @@ main() { post_build_checks } -main $@ +main "$@" exit 0 From fbb5a0deceb3a32be8e16cf13c8b1ed6adc665d2 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Wed, 28 Feb 2018 01:22:32 -0800 Subject: [PATCH 02/16] Upgrading to Virtualbox 5.2 in the CI build --- ci/automated_install.sh | 4 ++-- ci/automated_install_vagrant_only.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/automated_install.sh b/ci/automated_install.sh index 30a64a6..ba7ab4a 100644 --- a/ci/automated_install.sh +++ b/ci/automated_install.sh @@ -7,11 +7,11 @@ # the text "building". Once the test is completed, the text will be replaced # with "success" or "failed". -# Install Virtualbox 5.1 +# Install Virtualbox 5.2 echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - apt-get update -apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2 +apt-get install -y virtualbox-5.2 build-essential unzip git ufw apache2 echo "building" > /var/www/html/index.html diff --git a/ci/automated_install_vagrant_only.sh b/ci/automated_install_vagrant_only.sh index e317927..db7eff3 100644 --- a/ci/automated_install_vagrant_only.sh +++ b/ci/automated_install_vagrant_only.sh @@ -7,11 +7,11 @@ # the text "building". Once the test is completed, the text will be replaced # with "success" or "failed". -# Install Virtualbox 5.1 +# Install Virtualbox 5.2 echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - apt-get update -apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2 +apt-get install -y virtualbox-5.2 build-essential unzip git ufw apache2 echo "building" > /var/www/html/index.html From 3d1ba91ad903ad3001b2d766111785c3989bfb50 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Wed, 28 Feb 2018 23:04:16 -0800 Subject: [PATCH 03/16] Updating Powershell web calls to use TLS1.2 --- Packer/scripts/debloat-windows.ps1 | 2 ++ Vagrant/scripts/download_palantir_osquery.ps1 | 2 ++ Vagrant/scripts/download_palantir_wef.ps1 | 6 ++++++ Vagrant/scripts/install-caldera-agent.ps1 | 2 ++ Vagrant/scripts/install-utilities.ps1 | 2 ++ ci/automated_install.sh | 10 +++++----- ci/automated_install_vagrant_only.sh | 8 ++++---- 7 files changed, 23 insertions(+), 9 deletions(-) diff --git a/Packer/scripts/debloat-windows.ps1 b/Packer/scripts/debloat-windows.ps1 index 530ee25..686943c 100755 --- a/Packer/scripts/debloat-windows.ps1 +++ b/Packer/scripts/debloat-windows.ps1 @@ -2,6 +2,8 @@ if ($env:PACKER_BUILDER_TYPE -And $($env:PACKER_BUILDER_TYPE).startsWith("hyperv Write-Host Skip debloat steps in Hyper-V build. } else { Write-Host Downloading debloat zip + # GitHub requires TLS 1.2 as of 2/1/2018 + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $url="https://github.com/StefanScherer/Debloat-Windows-10/archive/master.zip" (New-Object System.Net.WebClient).DownloadFile($url, "$env:TEMP\debloat.zip") Expand-Archive -Path $env:TEMP\debloat.zip -DestinationPath $env:TEMP -Force diff --git a/Vagrant/scripts/download_palantir_osquery.ps1 b/Vagrant/scripts/download_palantir_osquery.ps1 index 4daaac3..1d03e1f 100644 --- a/Vagrant/scripts/download_palantir_osquery.ps1 +++ b/Vagrant/scripts/download_palantir_osquery.ps1 @@ -3,5 +3,7 @@ Write-Host "Downloading and unzipping the Palantir osquery Repo from Github..." $osqueryRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\osquery-Master.zip' +# GitHub requires TLS 1.2 as of 2/1/2018 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 Invoke-WebRequest -Uri "https://github.com/palantir/osquery-configuration/archive/master.zip" -OutFile $osqueryRepoPath Expand-Archive -path "$osqueryRepoPath" -destinationpath 'c:\Users\vagrant\AppData\Local\Temp' -Force diff --git a/Vagrant/scripts/download_palantir_wef.ps1 b/Vagrant/scripts/download_palantir_wef.ps1 index 263fe4c..2fa766e 100644 --- a/Vagrant/scripts/download_palantir_wef.ps1 +++ b/Vagrant/scripts/download_palantir_wef.ps1 @@ -1,7 +1,13 @@ # Purpose: Downloads and unzips a copy of the Palantir WEF Github Repo. This includes WEF subscriptions and custom WEF channels. + +# GitHub requires TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Write-Host "Downloading and unzipping the Palantir Windows Event Forwarding Repo from Github..." $wefRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\wef-Master.zip' +# GitHub requires TLS 1.2 as of 2/1/2018 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 Invoke-WebRequest -Uri "https://github.com/palantir/windows-event-forwarding/archive/master.zip" -OutFile $wefRepoPath Expand-Archive -path "$wefRepoPath" -destinationpath 'c:\Users\vagrant\AppData\Local\Temp' -Force diff --git a/Vagrant/scripts/install-caldera-agent.ps1 b/Vagrant/scripts/install-caldera-agent.ps1 index 9ab10f2..f59133b 100644 --- a/Vagrant/scripts/install-caldera-agent.ps1 +++ b/Vagrant/scripts/install-caldera-agent.ps1 @@ -14,6 +14,8 @@ If (-not (Test-Path "C:\Program Files\cagent\cagent.exe")) { Write-Host "Downloading Caldera Agent (cagent.exe)" $cagentPath = "C:\Program Files\cagent\cagent.exe" $cagentConfPath = "C:\Program Files\cagent\conf.yml" + # GitHub requires TLS 1.2 as of 2/1/2018 + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 (New-Object System.Net.WebClient).DownloadFile('https://github.com/mitre/caldera-agent/releases/download/v0.1.0/cagent.exe', $cagentPath) # Ignore SSL warning for conf file download # https://stackoverflow.com/questions/34331206/ignore-ssl-warning-with-powershell-downloadstring diff --git a/Vagrant/scripts/install-utilities.ps1 b/Vagrant/scripts/install-utilities.ps1 index 6272a19..e5d8414 100755 --- a/Vagrant/scripts/install-utilities.ps1 +++ b/Vagrant/scripts/install-utilities.ps1 @@ -27,6 +27,8 @@ If ($env:computername -eq "WIN10") { # Purpose: Downloads and unzips a copy of the latest Mimikatz trunk Write-Host "Determining latest release of Mimikatz..." +# GitHub requires TLS 1.2 as of 2/27 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $tag = (Invoke-WebRequest "https://api.github.com/repos/gentilkiwi/mimikatz/releases" -UseBasicParsing | ConvertFrom-Json)[0].tag_name $mimikatzDownloadUrl = "https://github.com/gentilkiwi/mimikatz/releases/download/$tag/mimikatz_trunk.zip" $mimikatzRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\mimikatz_trunk.zip' diff --git a/ci/automated_install.sh b/ci/automated_install.sh index ba7ab4a..3a3f447 100644 --- a/ci/automated_install.sh +++ b/ci/automated_install.sh @@ -11,7 +11,7 @@ echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - apt-get update -apt-get install -y virtualbox-5.2 build-essential unzip git ufw apache2 +apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2 echo "building" > /var/www/html/index.html @@ -23,14 +23,14 @@ ufw --force enable # Install Vagrant mkdir /opt/vagrant -cd /opt/vagrant +cd /opt/vagrant || exit 1 wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb dpkg -i vagrant_2.0.1_x86_64.deb vagrant plugin install vagrant-reload # Install Packer mkdir /opt/packer -cd /opt/packer +cd /opt/packer || exit 1 wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip unzip packer_1.1.3_linux_amd64.zip cp packer /usr/local/bin/packer @@ -41,12 +41,12 @@ for file in $(ls *.json); do done # Make the Vagrant instances headless -cd /opt/DetectionLab/Vagrant +cd /opt/DetectionLab/Vagrant || exit 1 sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile # Ensure the script is executable chmod +x /opt/DetectionLab/build.sh -cd /opt/DetectionLab +cd /opt/DetectionLab || exit 1 # Start the build in a tmux session sn=tmuxsession diff --git a/ci/automated_install_vagrant_only.sh b/ci/automated_install_vagrant_only.sh index db7eff3..8569b79 100644 --- a/ci/automated_install_vagrant_only.sh +++ b/ci/automated_install_vagrant_only.sh @@ -11,7 +11,7 @@ echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - apt-get update -apt-get install -y virtualbox-5.2 build-essential unzip git ufw apache2 +apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2 echo "building" > /var/www/html/index.html @@ -23,18 +23,18 @@ ufw --force enable # Install Vagrant mkdir /opt/vagrant -cd /opt/vagrant +cd /opt/vagrant || exit 1 wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb dpkg -i vagrant_2.0.1_x86_64.deb vagrant plugin install vagrant-reload # Make the Vagrant instances headless -cd /opt/DetectionLab/Vagrant +cd /opt/DetectionLab/Vagrant || exit 1 sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile # Ensure the script is executable chmod +x /opt/DetectionLab/build_vagrant_only.sh -cd /opt/DetectionLab +cd /opt/DetectionLab || exit 1 # Start the build in a tmux session sn=tmuxsession From 33e9d3ea6487dc92c9a7d01bf01258fe5e6e4b2a Mon Sep 17 00:00:00 2001 From: Chris Long Date: Thu, 1 Mar 2018 00:36:54 -0800 Subject: [PATCH 04/16] Updating Splunk --- Vagrant/bootstrap.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Vagrant/bootstrap.sh b/Vagrant/bootstrap.sh index 6fad236..a30bc52 100644 --- a/Vagrant/bootstrap.sh +++ b/Vagrant/bootstrap.sh @@ -41,8 +41,8 @@ else # Get Splunk.com into the DNS cache. Sometimes resolution randomly fails during wget below dig @8.8.8.8 splunk.com # Download Splunk - wget --progress=bar:force -O splunk-7.0.1-2b5b15c4ee89-linux-2.6-amd64.deb 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=7.0.1&product=splunk&filename=splunk-7.0.1-2b5b15c4ee89-linux-2.6-amd64.deb&wget=true' - dpkg -i splunk-7.0.1-2b5b15c4ee89-linux-2.6-amd64.deb + wget --progress=bar:force -O splunk-7.0.2-03bbabbd5c0f-linux-2.6-amd64.deb 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=7.0.2&product=splunk&filename=splunk-7.0.2-03bbabbd5c0f-linux-2.6-amd64.deb&wget=true' + dpkg -i splunk-7.0.2-03bbabbd5c0f-linux-2.6-amd64.deb /opt/splunk/bin/splunk start --accept-license /opt/splunk/bin/splunk add index wineventlog -auth 'admin:changeme' /opt/splunk/bin/splunk add index osquery -auth 'admin:changeme' From 79ce9bfcddf3ff3b93d9d50fd1702650dd70bdc5 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Thu, 1 Mar 2018 09:43:05 -0800 Subject: [PATCH 05/16] Bump Vagrant version and copy debug log back to circle --- .circleci/config.yml | 6 ++++++ ci/automated_install.sh | 6 +++--- ci/automated_install_vagrant_only.sh | 6 +++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5b94bb..569165e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,9 @@ jobs: fi done if [ "$MINUTES_PAST" -gt 120 ]; then + export IP_ADDRESS=$(cat /tmp/ip_address); + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log . + cat vagrant.log curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" exit 1 fi @@ -74,6 +77,9 @@ jobs: export STATUS=$(cat /tmp/status) echo $STATUS if [ "$STATUS" != "success" ]; then + export IP_ADDRESS=$(cat /tmp/ip_address); + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log . + cat vagrant.log curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" exit 1 fi diff --git a/ci/automated_install.sh b/ci/automated_install.sh index 3a3f447..dfd2111 100644 --- a/ci/automated_install.sh +++ b/ci/automated_install.sh @@ -24,8 +24,8 @@ ufw --force enable # Install Vagrant mkdir /opt/vagrant cd /opt/vagrant || exit 1 -wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb -dpkg -i vagrant_2.0.1_x86_64.deb +wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb +dpkg -i vagrant_2.0.2_x86_64.deb vagrant plugin install vagrant-reload # Install Packer @@ -51,4 +51,4 @@ cd /opt/DetectionLab || exit 1 # Start the build in a tmux session sn=tmuxsession tmux new-session -s "$sn" -d -tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter +tmux send-keys -t "$sn:0" './build.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter diff --git a/ci/automated_install_vagrant_only.sh b/ci/automated_install_vagrant_only.sh index 8569b79..463d559 100644 --- a/ci/automated_install_vagrant_only.sh +++ b/ci/automated_install_vagrant_only.sh @@ -24,8 +24,8 @@ ufw --force enable # Install Vagrant mkdir /opt/vagrant cd /opt/vagrant || exit 1 -wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb -dpkg -i vagrant_2.0.1_x86_64.deb +wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb +dpkg -i vagrant_2.0.2_x86_64.deb vagrant plugin install vagrant-reload # Make the Vagrant instances headless @@ -39,4 +39,4 @@ cd /opt/DetectionLab || exit 1 # Start the build in a tmux session sn=tmuxsession tmux new-session -s "$sn" -d -tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter +tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter From e604d4300ab4bf35249b60980b1a90e0fb0943c5 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Sat, 3 Mar 2018 23:32:50 -0800 Subject: [PATCH 06/16] Update README.md Fixing urls, wording --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 75bc502..c76fec7 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ OSX 10.12.5 | 1.9.3 | 1.0.0 | VMWare Fusion (8.5.8) OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.9) OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.10) Ubuntu 16.04 | 2.0.1 | 1.1.3 | Virtualbox (5.1) +Ubuntu 16.04 | 2.0.2 | N/A | Virtualbox (5.2) **Known Bad Versions:** * Packer 1.1.2 will fail to build VMWare-ISOs correctly due to [this issue](https://github.com/hashicorp/packer/issues/5622). @@ -48,15 +49,15 @@ DetectionLab now contains build scripts for \*NIX and MacOS users! There are two build scripts: - `build.sh` - Builds the entire lab from scratch. Takes 3-5 hours depending on hardware resources and bandwidth -- `build_vagrant_only.sh` - Downloads pre-built Packer boxes from S3 and builds the lab from those boxes. This option is recommended if you have more bandwidth than time. +- `build_vagrant_only.sh` - Downloads pre-built Packer boxes from https://detectionlab.network and builds the lab from those boxes. This option is recommended if you have more bandwidth than time or are having trouble building boxes. --- -## Building from Scratch +## Building DetectionLab from Scratch 1. Determine which Vagrant provider you want to use. * Note: Virtualbox is free, the [VMWare vagrant plugin](https://www.vagrantup.com/vmware/#buy-now) is $80. - **NOTE:** If you have more bandwidth than time, you can skip the building of the Packer boxes and download the boxes directly from S3 and put them into the `Boxes` directory: + **NOTE:** If you'd like to save time, you can skip the building of the Packer boxes and download the boxes directly from https://detectionlab.network and put them into the `Boxes` directory: Provider | Box | URL | MD5 | Size ------------|-----|-----|----|---- From e9d24d11a34c6a456fd8dcda268a31ea7cef29d3 Mon Sep 17 00:00:00 2001 From: Filippo Anarratone Date: Thu, 15 Mar 2018 12:42:59 +0000 Subject: [PATCH 07/16] Fix virtualbox presence check --- build_vagrant_only.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_vagrant_only.sh b/build_vagrant_only.sh index 5e17595..3226be3 100755 --- a/build_vagrant_only.sh +++ b/build_vagrant_only.sh @@ -30,9 +30,9 @@ check_vagrant() { # Returns 0 if not installed or 1 if installed check_virtualbox_installed() { if ! which VBoxManage >/dev/null; then - echo "1" - else echo "0" + else + echo "1" fi } From 1bcb1af9e2eb6300b597a539e87eb327f766e684 Mon Sep 17 00:00:00 2001 From: Filippo Anarratone Date: Thu, 15 Mar 2018 13:58:52 +0000 Subject: [PATCH 08/16] Fix virtualbox presence check --- build_vagrant_only.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_vagrant_only.sh b/build_vagrant_only.sh index 3226be3..0e96a43 100755 --- a/build_vagrant_only.sh +++ b/build_vagrant_only.sh @@ -29,10 +29,10 @@ check_vagrant() { # Returns 0 if not installed or 1 if installed check_virtualbox_installed() { - if ! which VBoxManage >/dev/null; then - echo "0" - else + if which VBoxManage >/dev/null; then echo "1" + else + echo "0" fi } From 0fd7d0647b14250d21fc5884150150c076f4cf00 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Wed, 21 Mar 2018 18:02:54 -0700 Subject: [PATCH 09/16] Fix compile script, update VM tools, Vagrant -> 2.0.3 --- .gitignore | 2 ++ Packer/scripts/compile-dotnet-assemblies.bat | 16 +++++++++------- Packer/scripts/set-powerplan.ps1 | 13 +++++++++++++ Packer/scripts/unattend.xml | 2 +- Packer/scripts/vm-guest-tools.bat | 8 ++------ Packer/windows_2016.json | 2 +- Vagrant/bootstrap.sh | 2 +- Vagrant/scripts/download_palantir_wef.ps1 | 3 --- ci/automated_install.sh | 4 ++-- ci/automated_install_vagrant_only.sh | 4 ++-- 10 files changed, 33 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index cefdb27..a5307ee 100755 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ Vagrant/.vagrant/* +Vagrant/vagrant_build.log Packer/packer_cache/* +Packer/packer_build.log Boxes/* .DS_Store diff --git a/Packer/scripts/compile-dotnet-assemblies.bat b/Packer/scripts/compile-dotnet-assemblies.bat index 14f81aa..2c0fe0d 100755 --- a/Packer/scripts/compile-dotnet-assemblies.bat +++ b/Packer/scripts/compile-dotnet-assemblies.bat @@ -3,13 +3,15 @@ if "%PROCESSOR_ARCHITECTURE%"=="AMD64" goto 64BIT -%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue -%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems +%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue > NUL +%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems > NUL -exit /b +exit 0 :64BIT -%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue -%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe update /force /queue -%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems -%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe executequeueditems \ No newline at end of file +%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue > NUL +%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe update /force /queue > NUL +%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems > NUL +%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe executequeueditems > NUL + +exit 0 diff --git a/Packer/scripts/set-powerplan.ps1 b/Packer/scripts/set-powerplan.ps1 index 65ee32c..646eba7 100755 --- a/Packer/scripts/set-powerplan.ps1 +++ b/Packer/scripts/set-powerplan.ps1 @@ -1,7 +1,20 @@ Try { + Write-Output "Set power plan to high performance" + $HighPerf = powercfg -l | %{if($_.contains("High performance")) {$_.split()[3]}} + + # $HighPerf cannot be $null, we try activate this power profile with powercfg + # + if ($HighPerf -eq $null) + { + throw "Error: HighPerf is null" + } + $CurrPlan = $(powercfg -getactivescheme).split()[3] + if ($CurrPlan -ne $HighPerf) {powercfg -setactive $HighPerf} + } Catch { Write-Warning -Message "Unable to set power plan to high performance" + Write-Warning $Error[0] } diff --git a/Packer/scripts/unattend.xml b/Packer/scripts/unattend.xml index f3a6c2d..ea8e759 100755 --- a/Packer/scripts/unattend.xml +++ b/Packer/scripts/unattend.xml @@ -34,7 +34,7 @@ - vagrant-2012 + vagrant-2016 false diff --git a/Packer/scripts/vm-guest-tools.bat b/Packer/scripts/vm-guest-tools.bat index 18891bc..cd33b74 100755 --- a/Packer/scripts/vm-guest-tools.bat +++ b/Packer/scripts/vm-guest-tools.bat @@ -15,7 +15,7 @@ if exist "C:\Users\vagrant\windows.iso" ( ) if not exist "C:\Windows\Temp\windows.iso" ( - powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://softwareupdate.vmware.com/cds/vmw-desktop/ws/12.5.7/5813279/windows/packages/tools-windows.tar', 'C:\Windows\Temp\vmware-tools.tar')" /home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf +cat /home/vagrant/osquery-configuration/Endpoints/Windows/osquery.conf | sed 's#packs/#../packs/#g' | grep -v unwanted-chrome-extensions | grep -v security-tooling-checks | grep -v performance-metrics | grep -v logger_snapshot_event_type > /home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf # Install configimporter echo "Installing configimporter" echo "Sleeping for 5" diff --git a/Vagrant/scripts/download_palantir_wef.ps1 b/Vagrant/scripts/download_palantir_wef.ps1 index 2fa766e..dc71750 100644 --- a/Vagrant/scripts/download_palantir_wef.ps1 +++ b/Vagrant/scripts/download_palantir_wef.ps1 @@ -1,8 +1,5 @@ # Purpose: Downloads and unzips a copy of the Palantir WEF Github Repo. This includes WEF subscriptions and custom WEF channels. -# GitHub requires TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - Write-Host "Downloading and unzipping the Palantir Windows Event Forwarding Repo from Github..." $wefRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\wef-Master.zip' diff --git a/ci/automated_install.sh b/ci/automated_install.sh index dfd2111..8b20cda 100644 --- a/ci/automated_install.sh +++ b/ci/automated_install.sh @@ -24,8 +24,8 @@ ufw --force enable # Install Vagrant mkdir /opt/vagrant cd /opt/vagrant || exit 1 -wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb -dpkg -i vagrant_2.0.2_x86_64.deb +wget https://releases.hashicorp.com/vagrant/2.0.3/vagrant_2.0.3_x86_64.deb +dpkg -i vagrant_2.0.3_x86_64.deb vagrant plugin install vagrant-reload # Install Packer diff --git a/ci/automated_install_vagrant_only.sh b/ci/automated_install_vagrant_only.sh index 463d559..ce4f418 100644 --- a/ci/automated_install_vagrant_only.sh +++ b/ci/automated_install_vagrant_only.sh @@ -24,8 +24,8 @@ ufw --force enable # Install Vagrant mkdir /opt/vagrant cd /opt/vagrant || exit 1 -wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb -dpkg -i vagrant_2.0.2_x86_64.deb +wget https://releases.hashicorp.com/vagrant/2.0.3/vagrant_2.0.3_x86_64.deb +dpkg -i vagrant_2.0.3_x86_64.deb vagrant plugin install vagrant-reload # Make the Vagrant instances headless From d75579f46e01cbecdab4e004e464abbb76c6c371 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Wed, 21 Mar 2018 23:04:38 -0700 Subject: [PATCH 10/16] Adding updated build scripts --- .circleci/config.yml | 102 ++---- build.sh | 320 ++++++++++++++---- build_vagrant_only.sh | 309 ----------------- ci/README.md | 100 +++++- ci/automated_install.sh | 54 --- ci/automated_install_vagrant_only.sh | 42 --- ci/build_machine_bootstrap.sh | 93 +++++ .../packer_and_vagrant_changes.sh | 80 +++++ ci/circle_workflows/packer_changes.sh | 99 ++++++ ci/circle_workflows/vagrant_changes.sh | 74 ++++ 10 files changed, 726 insertions(+), 547 deletions(-) delete mode 100755 build_vagrant_only.sh delete mode 100644 ci/automated_install.sh delete mode 100644 ci/automated_install_vagrant_only.sh create mode 100755 ci/build_machine_bootstrap.sh create mode 100644 ci/circle_workflows/packer_and_vagrant_changes.sh create mode 100644 ci/circle_workflows/packer_changes.sh create mode 100644 ci/circle_workflows/vagrant_changes.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 569165e..0e2a398 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,79 +9,47 @@ jobs: - checkout - run: - name: Delete stale Packet servers + name: Check if Packer files were modified command: | - export DELETE_DEVICE_ID=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"') - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID" - - run: - name: Provisioning a baremetal Packet.net server - command: | - curl -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' > /tmp/device - - run: cat /tmp/device | jq ."id" | tr -d '"' > /tmp/device_id - - run: sleep 300 - - run: echo "Sleeping 5 more minutes while Packet server is provisioned" - - run: sleep 300 - - run: - name: Recording the IP address of the Packet server - command: | - export DEVICE_ID=$(cat /tmp/device_id); - curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" > /tmp/ip_info; - - run: cat /tmp/ip_info | jq ."ip_addresses[0].address" | tr -d '"' > /tmp/ip_address - - run: cd ~/repo - - run: - name: Copying DetectionLab repo to Packet server - command: | - export IP_ADDRESS=$(cat /tmp/ip_address); - scp -i ~/.ssh/id_rsa -r ~/repo root@"$IP_ADDRESS":/opt/DetectionLab - - run: - name: Copying install script to Packet server - command: | - export IP_ADDRESS=$(cat /tmp/ip_address); - scp -i ~/.ssh/id_rsa ./ci/automated_install_vagrant_only.sh root@"$IP_ADDRESS":/root - - run: - name: Running install script on Packet server - command: | - export IP_ADDRESS=$(cat /tmp/ip_address) - ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'chmod +x /root/automated_install_vagrant_only.sh; /bin/bash -c /root/automated_install_vagrant_only.sh' + if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Packer/\")" -gt 0 ]; then + export PACKER_MODIFIED=1 + else + export PACKER_MODIFIED=0 + fi - run: - name: Waiting for Packet server to post build results - shell: /bin/bash + name: Check if Vagrant files were modified command: | - export IP_ADDRESS=$(cat /tmp/ip_address) - MINUTES_PAST=0 - while [ "$MINUTES_PAST" -lt 120 ] - do - export STATUS=$(curl $IP_ADDRESS) - if [ "$STATUS" == "building" ]; then - echo "$STATUS" - sleep 300 - ((MINUTES_PAST+=5)) + if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Vagrant/\")" -gt 0 ]; then + export VAGRANT_MODIFIED=1 else - echo "$STATUS" > /tmp/status - break - fi - done - if [ "$MINUTES_PAST" -gt 120 ]; then - export IP_ADDRESS=$(cat /tmp/ip_address); - scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log . - cat vagrant.log - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" - exit 1 + export VAGRANT_MODIFIED=0 fi + + - run: - name: Recording build results - shell: /bin/bash + name: Choose which build gets run command: | - export DEVICE_ID=$(cat /tmp/device_id) - export STATUS=$(cat /tmp/status) - echo $STATUS - if [ "$STATUS" != "success" ]; then - export IP_ADDRESS=$(cat /tmp/ip_address); - scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log . - cat vagrant.log - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" - exit 1 + if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then + chmod +x ci/circle_workflows/packer_and_vagrant_changes.sh + ci/circle_workflows/packer_and_vagrant_changes.sh + exit 0 fi - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" - exit 0 + if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then + chmod +x ci/circle_workflows/vagrant_changes.sh + ci/circle_workflows/vagrant_changes.sh + exit 0 + fi + if [ "$PACKER_MODIFIED" -eq 1 ]; then + chmod +x ci/circle_workflows/packer_changes.sh + ci/circle_workflows/packer_changes.sh + exit 0 + fi + if [ "$VAGRANT_MODIFIED" -eq 1 ]; then + chmod +x ci/circle_workflows/vagrant_changes.sh + ci/circle_workflows/vagrant_changes.sh + exit 0 + fi + + - store_artifacts: + path: /tmp/artifacts diff --git a/build.sh b/build.sh index c43613f..4b30f4e 100755 --- a/build.sh +++ b/build.sh @@ -7,25 +7,13 @@ # If you encounter issues, feel free to open an issue at # https://github.com/clong/DetectionLab/issues -set -e - print_usage() { echo "Usage: ./build.sh " exit 0 } -check_packer_and_vagrant() { - # Check for existence of Vagrant in PATH - if ! which vagrant >/dev/null; then - (echo >&2 "Vagrant was not found in your PATH.") - (echo >&2 "Please correct this before continuing. Quitting.") - exit 1 - fi - # Ensure Vagrant >= 2.0.0 - if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then - (echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing") - fi - # Check for existence of Packer in PATH +check_packer_path() { +# Check for existence of Packer in PATH if ! which packer >/dev/null; then (echo >&2 "Packer was not found in your PATH.") (echo >&2 "Please correct this before continuing. Quitting.") @@ -34,6 +22,19 @@ check_packer_and_vagrant() { fi } +check_vagrant_path() { +# Check for existence of Vagrant in PATH +if ! which vagrant >/dev/null; then + (echo >&2 "Vagrant was not found in your PATH.") + (echo >&2 "Please correct this before continuing. Quitting.") + exit 1 +fi +# Ensure Vagrant >= 2.0.0 +if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then + (echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing") +fi +} + # Returns 0 if not installed or 1 if installed check_virtualbox_installed() { if which VBoxManage >/dev/null; then @@ -101,41 +102,40 @@ list_providers() { echo "$PROVIDER" } -# A series of checks to identify potential issues before starting the build -preflight_checks() { - DL_DIR="$1" - - # Check to see if curl is in PATH - if ! which curl >/dev/null; then - (echo >&2 "Please install curl and make sure it is in your PATH.") - exit 1 - fi - # Check to see if boxes exist already +# Check to see if boxes exist in the "Boxes" directory already +check_boxes_built() { BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l) if [ "$BOXES_BUILT" -gt 0 ]; then - (echo >&2 "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.") + if [ "$VAGRANT_ONLY" -eq 1 ]; then + (echo >&2 "WARNING: You seem to have at least one .box file present in $DL_DIR/Boxes already. If you would like fresh boxes downloaded, please remove all files from the Boxes directory and re-run this script.") + DOWNLOAD_BOXES=0 + else + (echo >&2 "You seem to have at least one .box file in $DL_DIR/Boxes. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.") + exit 1 + fi + fi +} + +# Check to ensure either "md5" or "md5sum" is installed for verifying integrity of downloaded boxes +check_md5_tool_exists() { + if ! which md5 > /dev/null && ! which md5sum > /dev/null; then + (echo >&2 "md5 or md5sum not found in PATH. Please install at least one of these utilities to continue.") exit 1 fi - # Check to see if any Vagrant instances exist already - cd "$DL_DIR"/Vagrant/ +} + +# Check to see if any Vagrant instances exist already +check_vagrant_instances_exist() { + cd "$DL_DIR"/Vagrant/ || exit 1 # Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true" VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true if [ "$VAGRANT_BUILT" -ne 4 ]; then (echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.") exit 1 fi - # Check available disk space. Recommend 80GB free, warn if less. - FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4) - if [ "$FREE_DISK_SPACE" -lt 80000 ]; then - (echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n") - (df >&2 -m "$HOME") - (echo >&2 "") - fi - # Check Packer version against known bad - if [ "$(packer --version)" == '1.1.2' ]; then - (echo >&2 "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.") - exit 1 - fi +} + +check_vagrant_reload_plugin() { # Ensure the vagrant-reload plugin is installed VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload') if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then @@ -147,26 +147,70 @@ preflight_checks() { fi } +# Check available disk space. Recommend 80GB free, warn if less. +check_disk_free_space() { + FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4) + if [ "$FREE_DISK_SPACE" -lt 80000 ]; then + (echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n") + (df >&2 -m "$HOME") + (echo >&2 "") + fi +} + +# Check to see if curl is in PATH - needed for post-install checks +check_curl(){ + if ! which curl >/dev/null; then + (echo >&2 "Please install curl and make sure it is in your PATH.") + exit 1 + fi +} + +# Check Packer version against known "bad" versions +check_packer_known_bad() { + if [ "$(packer --version)" == '1.1.2' ]; then + (echo >&2 "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.") + exit 1 + fi +} + +# A series of checks to identify potential issues before starting the build +preflight_checks() { + # If it's not a Vagrant-only build, then run Packer-related checks + if [ "$VAGRANT_ONLY" -eq 0 ]; then + check_packer_path + check_packer_known_bad + else + # If it is a Vagrant-only build, set appropriate checks + DOWNLOAD_BOXES=1 + check_md5_tool_exists + fi + + # If it's not a Packer-only build, then run Vagrant-related checks + if [ "$PACKER_ONLY" -eq 0 ]; then + check_vagrant_path + check_vagrant_instances_exist + check_vagrant_reload_plugin + fi + + check_boxes_built + check_disk_free_space + check_curl +} + # Builds a box using Packer packer_build_box() { - PROVIDER="$1" - BOX="$2" - DL_DIR="$3" + BOX="$1" if [ "$PROVIDER" == "vmware_fusion" ]; then PROVIDER="vmware" fi - cd "$DL_DIR/Packer" + cd "$DL_DIR/Packer" || exit 1 (echo >&2 "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.") - if ! $(which packer) build --only="$PROVIDER-iso" "$BOX".json; then - (echo >&2 "Something went wrong while attempting to build the $BOX box.") - (echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/") - fi + PACKER_LOG=1 PACKER_LOG_PATH="$DL_DIR/Packer/packer_build.log" $(which packer) build --only="$PROVIDER-iso" "$BOX".json >&2 + echo "$?" } # Moves the boxes from the Packer directory to the Boxes directory move_boxes() { - PROVIDER="$1" - DL_DIR="$2" # Hacky workaround for VMware if [ "$PROVIDER" == "vmware_fusion" ]; then PROVIDER="vmware" @@ -186,20 +230,17 @@ move_boxes() { # Brings up a single host using Vagrant vagrant_up_host() { - PROVIDER="$1" - HOST="$2" - DL_DIR="$3" + HOST="$1" (echo >&2 "Attempting to bring up the $HOST host using Vagrant") - cd "$DL_DIR"/Vagrant - $(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2 + cd "$DL_DIR"/Vagrant || exit 1 + VAGRANT_LOG=info $(which vagrant) up "$HOST" --provider="$PROVIDER" 3>&1 1>&2 2>&3 | tee -a "$DL_DIR/Vagrant/vagrant_build.log" echo "$?" } # Attempts to reload and re-provision a host if the intial "vagrant up" fails vagrant_reload_host() { HOST="$1" - DL_DIR="$2" - cd "$DL_DIR"/Vagrant + cd "$DL_DIR"/Vagrant || exit 1 # Attempt to reload the host if the vagrant up command didn't exit cleanly $(which vagrant) reload "$HOST" --provision 1>&2 echo "$?" @@ -236,22 +277,17 @@ post_build_checks() { fi } -main() { - # Get location of build.sh - # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within - DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - PROVIDER="" - LAB_HOSTS=("logger" "dc" "wef" "win10") +parse_cli_arguments() { # If no argument was supplied, list available providers - if [ $# -eq 0 ]; then + if [ "$#" -eq 0 ]; then PROVIDER=$(list_providers) fi - # If more than one argument was supplied, print usage message - if [ $# -gt 1 ]; then + # If more than two arguments were supplied, print usage message + if [ "$#" -gt 2 ]; then print_usage exit 1 fi - if [ $# -eq 1 ]; then + if [ "$#" -ge 1 ]; then # If the user specifies the provider as an agument, set the variable # TODO: Check to make sure they actually have their provider installed case "$1" in @@ -267,12 +303,121 @@ main() { ;; esac fi + if [ $# -eq 2 ]; then + case "$2" in + --packer-only) + PACKER_ONLY=1 + ;; + --vagrant-only) + VAGRANT_ONLY=1 + ;; + *) + echo -e "\"$2\" is not recognized as an option. Available options are:\\n--packer-only\\n--vagrant-only" + exit 1 + ;; + esac + fi +} - preflight_checks "$DL_DIR" - packer_build_box "$PROVIDER" "windows_2016" "$DL_DIR" - packer_build_box "$PROVIDER" "windows_10" "$DL_DIR" - move_boxes "$PROVIDER" "$DL_DIR" +build_packer_boxes() { + PACKER_BOXES=("windows_2016" "windows_10") + if [ "$(hostname)" == "packerwindows10" ]; then # Workaround for CI environment + (echo >&2 "CI Environment detected. If you are a user and are seeing this, please file an issue on GitHub.") + RET=$(packer_build_box "windows_10") + if [ "$RET" -eq 0 ]; then + (echo >&2 "Good news! The windows_10 box was built with Packer successfully!") + else + (echo >&2 "Something went wrong while attempting to build the windows_10 box.") + (echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/") + exit 1 + fi + elif [ "$(hostname)" == "packerwindows2016" ]; then # Workaround for CI environment + (echo >&2 "CI Environment detected. If you are a user and are seeing this, please file an issue on GitHub.") + RET=$(packer_build_box "windows_2016") + if [ "$RET" -eq 0 ]; then + (echo >&2 "Good news! The windows_2016 box was built with Packer successfully!") + else + (echo >&2 "Something went wrong while attempting to build the windows_2016 box.") + (echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/") + exit 1 + fi + else + for PACKER_BOX in "${PACKER_BOXES[@]}"; do # Normal user workflow + RET=$(packer_build_box "$PACKER_BOX") + if [ "$RET" -eq 0 ]; then + (echo >&2 "Good news! $PACKER_BOX was built successfully!") + else + (echo >&2 "Something went wrong while attempting to build the $PACKER_BOX box.") + (echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/") + exit 1 + fi + done + fi +} + +choose_md5_tool() { + if which md5; then + MD5TOOL="$(which md5)" + CUT_INDEX=4 + else + MD5TOOL="$(which md5sum)" + CUT_INDEX=1 + fi +} + +# Downloads pre-built Packer boxes from detectionlab.network to save time during CI builds +download_boxes() { + if [ "$PROVIDER" == "virtualbox" ]; then + wget "https://www.detectionlab.network/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box + wget "https://www.detectionlab.network/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box + elif [ "$PROVIDER" == "vmware_fusion" ]; then + wget "https://www.detectionlab.network/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box + wget "https://www.detectionlab.network/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box + fi + + # Hacky workaround + if [ "$PROVIDER" == "vmware_fusion" ]; then + PROVIDER="vmware" + fi + + # Ensure Windows 10 box exists + if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then + (echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.") + exit 1 + fi + # Ensure Windows 2016 box exists + if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then + (echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.") + exit 1 + fi + # Verify hashes of VirtualBox boxes + if [ "$PROVIDER" == "virtualbox" ]; then + if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then + (echo >&2 "Hash mismatch on windows_10_virtualbox.box") + fi + if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "614f984c82b51471b5bb753940b59d38" ]; then + (echo >&2 "Hash mismatch on windows_2016_virtualbox.box") + fi + # Verify hashes of VMware boxes + elif [ "$PROVIDER" == "vmware" ]; then + if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then + (echo >&2 "Hash mismatch on windows_10_vmware.box") + exit 1 + fi + if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then + (echo >&2 "Hash mismatch on windows_2016_vmware.box") + exit 1 + fi + # Reset PROVIDER variable if using VMware + if [ "$PROVIDER" == "vmware" ]; then + PROVIDER="vmware_fusion" + fi + fi +} + +build_vagrant_hosts() { + LAB_HOSTS=("logger" "dc" "wef" "win10") # Change provider back to original selection if using vmware_fusion if [ "$PROVIDER" == "vmware" ]; then PROVIDER="vmware_fusion" @@ -280,7 +425,7 @@ main() { # Vagrant up each box and attempt to reload one time if it fails for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do - RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR") + RET=$(vagrant_up_host "$VAGRANT_HOST") if [ "$RET" -eq 0 ]; then (echo >&2 "Good news! $VAGRANT_HOST was built successfully!") fi @@ -288,16 +433,43 @@ main() { if [ "$RET" -ne 0 ]; then (echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.") (echo >&2 "Attempting to reload and reprovision the host...") - RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR") + RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST") if [ "$RETRY_STATUS" -ne 0 ]; then (echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.") exit 1 fi fi done - - post_build_checks } +main() { + # Get location of build.sh + # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within + DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + PACKER_ONLY=0 + VAGRANT_ONLY=0 + + parse_cli_arguments "$@" + preflight_checks + if [[ "$DOWNLOAD_BOXES" -eq 1 ]] && [[ "$VAGRANT_ONLY" -eq 1 ]]; then + download_boxes + fi + + # Build Packer boxes if this isn't a Vagrant-only build + if [ "$VAGRANT_ONLY" -eq 0 ]; then + build_packer_boxes + # The only time we will need to move boxes is if we're doing a full build + if [ "$PACKER_ONLY" -eq 0 ]; then + move_boxes + fi + fi + + # Build and Test Vagrant hosts if this isn't a Packer-only build + if [ "$PACKER_ONLY" -eq 0 ]; then + build_vagrant_hosts + post_build_checks + fi + } + main "$@" exit 0 diff --git a/build_vagrant_only.sh b/build_vagrant_only.sh deleted file mode 100755 index 0e96a43..0000000 --- a/build_vagrant_only.sh +++ /dev/null @@ -1,309 +0,0 @@ -#! /bin/bash - -# This script skips the entire Packer box building process. -# This script downloads pre-built VirtualBox Packer images from S3 and puts -# them into the "Boxes" directory. -# Only MacOS and Linux are supported. -# If you encounter issues, feel free to open an issue at -# https://github.com/clong/DetectionLab/issues - -set -e - -print_usage() { - echo "Usage: ./build.sh " - exit 0 -} - -check_vagrant() { - # Check for existence of Vagrant in PATH - if ! which vagrant >/dev/null; then - (echo >&2 "Vagrant was not found in your PATH.") - (echo >&2 "Please correct this before continuing. Quitting.") - exit 1 - fi - # Ensure Vagrant >= 2.0.0 - if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then - (echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing") - fi -} - -# Returns 0 if not installed or 1 if installed -check_virtualbox_installed() { - if which VBoxManage >/dev/null; then - echo "1" - else - echo "0" - fi -} - -# Returns 0 if not installed or 1 if installed -check_vmware_fusion_installed() { - if [ -e "/Applications/VMware Fusion.app" ]; then - echo "1" - else - echo "0" - fi -} - -# Returns 0 if not installed or 1 if installed -check_vmware_vagrant_plugin_installed() { - VAGRANT_VMWARE_PLUGIN_PRESENT="$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')" - if [ "$VAGRANT_VMWARE_PLUGIN_PRESENT" -eq 0 ]; then - (echo >&2 "VMWare Fusion is installed, but the Vagrant plugin is not.") - (echo >&2 "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it") - (echo >&2 "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.") - echo "0" - else - echo "$VAGRANT_VMWARE_PLUGIN_PRESENT" - fi -} - -# List the available Vagrant providers present on the system -list_providers() { - VBOX_PRESENT=0 - VMWARE_FUSION_PRESENT=0 - - if [ "$(uname)" == "Darwin" ]; then - # Detect Providers on OSX - VBOX_PRESENT=$(check_virtualbox_installed) - VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed) - VAGRANT_VMWARE_PLUGIN_PRESENT=$(check_vmware_vagrant_plugin_installed) - else - # Assume the only other available provider is VirtualBox - VBOX_PRESENT=$(check_virtualbox_installed) - fi - - (echo >&2 "Available Providers:") - if [ "$VBOX_PRESENT" == "1" ]; then - (echo >&2 "virtualbox") - fi - if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]; then - (echo >&2 "vmware_fusion") - fi - if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]; then - (echo >&2 "You need to install a provider such as VirtualBox or VMware Fusion to continue.") - exit 1 - fi - (echo >&2 -e "\\nWhich provider would you like to use?") - read -r PROVIDER - # Sanity check - if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]; then - (echo >&2 "Please choose a valid provider. \"$PROVIDER\" is not a valid option") - exit 1 - fi - echo "$PROVIDER" -} - -# A series of checks to identify potential issues before starting the build -preflight_checks() { - DL_DIR="$1" - DOWNLOAD_BOXES=1 - - # Check to see if curl is in PATH - if ! which curl >/dev/null; then - (echo >&2 "Please install curl and make sure it is in your PATH.") - exit 1 - fi - # Check to see if wget is in PATH - if ! which wget >/dev/null; then - (echo >&2 "Please install curl and make sure it is in your PATH.") - exit 1 - fi - # Check to see if boxes exist already - BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l) - if [ "$BOXES_BUILT" -gt 0 ]; then - (echo >&2 "WARNING: You seem to have boxes present in the Boxes directory already. If you would like fresh boxes downloaded, please remove all files from the Boxes directory and re-run this script.") - DOWNLOAD_BOXES=0 - fi - # Check to see if any Vagrant instances exist already - cd "$DL_DIR"/Vagrant/ - # Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true" - VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true - if [ "$VAGRANT_BUILT" -ne 4 ]; then - (echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.") - exit 1 - fi - # Check available disk space. Recommend 80GB free, warn if less. - FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4) - if [ "$FREE_DISK_SPACE" -lt 80000 ]; then - (echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n") - (df >&2 -m "$HOME") - (echo >&2 "") - fi - # Ensure the vagrant-reload plugin is installed - VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload') - if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then - (echo >&2 "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.") - if ! $(which vagrant) plugin install "vagrant-reload"; then - (echo >&2 "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.") - exit 1 - fi - fi -} - -# Downloads pre-built Packer boxes from detectionlab.network to save time during CI builds -download_boxes() { - DL_DIR="$1" - PROVIDER="$2" - - if [ "$PROVIDER" == "virtualbox" ]; then - wget "https://www.detectionlab.network/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box - wget "https://www.detectionlab.network/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box - elif [ "$PROVIDER" == "vmware_fusion" ]; then - wget "https://www.detectionlab.network/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box - wget "https://www.detectionlab.network/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box - fi - - # Hacky workaround - if [ "$PROVIDER" == "vmware_fusion" ]; then - PROVIDER="vmware" - fi - - # Ensure Windows 10 box exists - if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then - (echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.") - exit 1 - fi - # Ensure Windows 2016 box exists - if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then - (echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.") - exit 1 - fi - # Verify hashes of VirtualBox boxes - if [ "$PROVIDER" == "virtualbox" ]; then - if [ "$(md5sum windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then - (echo >&2 "Hash mismatch on windows_10_virtualbox.box") - fi - if [ "$(md5sum windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "614f984c82b51471b5bb753940b59d38" ]; then - (echo >&2 "Hash mismatch on windows_2016_virtualbox.box") - fi - # Verify hashes of VMware boxes - elif [ "$PROVIDER" == "vmware" ]; then - if [ "$(md5 windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then - (echo >&2 "Hash mismatch on windows_10_vmware.box") - exit 1 - fi - if [ "$(md5 windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then - (echo >&2 "Hash mismatch on windows_2016_vmware.box") - exit 1 - fi - # Reset PROVIDER variable - PROVIDER="vmware_fusion" - fi -} - -# Brings up a single host using Vagrant -vagrant_up_host() { - PROVIDER="$1" - HOST="$2" - DL_DIR="$3" - (echo >&2 "Attempting to bring up the $HOST host using Vagrant") - cd "$DL_DIR"/Vagrant - $(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2 - echo "$?" -} - -# Attempts to reload and re-provision a host if the intial "vagrant up" fails -vagrant_reload_host() { - HOST="$1" - DL_DIR="$2" - cd "$DL_DIR"/Vagrant - # Attempt to reload the host if the vagrant up command didn't exit cleanly - $(which vagrant) reload "$HOST" --provision 1>&2 - echo "$?" -} - -# A series of checks to ensure important services are responsive after the build completes. -post_build_checks() { - # If the curl operation fails, we'll just leave the variable equal to 0 - # This is needed to prevent the script from exiting if the curl operation fails - CALDERA_CHECK=$(curl -ks -m 2 https://192.168.38.5:8888 | grep -c '302: Found' || echo "") - SPLUNK_CHECK=$(curl -ks -m 2 https://192.168.38.5:8000/en-US/account/login?return_to=%2Fen-US%2F | grep -c 'This browser is not supported by Splunk' || echo "") - FLEET_CHECK=$(curl -ks -m 2 https://192.168.38.5:8412 | grep -c 'Kolide Fleet' || echo "") - - BASH_MAJOR_VERSION=$(/bin/bash --version | grep 'GNU bash' | grep -o version\.\.. | cut -d ' ' -f 2 | cut -d '.' -f 1) - # Associative arrays are only supported in bash 4 and up - if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then - declare -A SERVICES - SERVICES=(["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK") - for SERVICE in "${!SERVICES[@]}"; do - if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then - (echo >&2 "Warning: $SERVICE failed post-build tests and may not be functioning correctly.") - fi - done - else - if [ "$CALDERA_CHECK" -lt 1 ]; then - (echo >&2 "Warning: Caldera failed post-build tests and may not be functioning correctly.") - fi - if [ "$SPLUNK_CHECK" -lt 1 ]; then - (echo >&2 "Warning: Splunk failed post-build tests and may not be functioning correctly.") - fi - if [ "$FLEET_CHECK" -lt 1 ]; then - (echo >&2 "Warning: Fleet failed post-build tests and may not be functioning correctly.") - fi - fi -} - -main() { - # Get location of build_vagrant_only.sh - # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within - DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - PROVIDER="" - LAB_HOSTS=("logger" "dc" "wef" "win10") - # If no argument was supplied, list available providers - if [ $# -eq 0 ]; then - PROVIDER=$(list_providers) - fi - # If more than one argument was supplied, print usage message - if [ $# -gt 1 ]; then - print_usage - exit 1 - fi - if [ $# -eq 1 ]; then - # If the user specifies the provider as an agument, set the variable - # TODO: Check to make sure they actually have their provider installed - case "$1" in - virtualbox) - PROVIDER="$1" - ;; - vmware_fusion) - PROVIDER="$1" - ;; - *) - echo "\"$1\" is not a valid provider. Listing available providers:" - PROVIDER=$(list_providers) - ;; - esac - fi - - check_vagrant - preflight_checks "$DL_DIR" - if [ "$DOWNLOAD_BOXES" -eq 0 ]; then - (echo >&2 "Skipping box downloads since .box files are already present in the Boxes/ directory.") - else - download_boxes "$DL_DIR" "$PROVIDER" - fi - - # Vagrant up each box and attempt to reload one time if it fails - for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do - RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR") - if [ "$RET" -eq 0 ]; then - (echo >&2 "Good news! $VAGRANT_HOST was built successfully!") - fi - # Attempt to recover if the intial "vagrant up" fails - if [ "$RET" -ne 0 ]; then - (echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.") - (echo >&2 "Attempting to reload and reprovision the host...") - RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR") - if [ "$RETRY_STATUS" -ne 0 ]; then - (echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.") - exit 1 - fi - fi - done - - post_build_checks -} - -main "$@" -exit 0 diff --git a/ci/README.md b/ci/README.md index 8174854..8a6dd62 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,6 +1,104 @@ -# ci +# Continuous Integration The files in this directory are used to bootstrap an Ubuntu 16.04 baremetal server for continuous integration testing by installing the prerequisites needed for Detection Lab. After the prerequisites are installed, the build script is called and the build will begin in a tmux session. + +## Understanding the build process + +Once a PR is created, the contents of that PR will be copied to a CircleCI worker to be tested. +The CircleCI worker will evaluate which files have been modified and set environment variables accordingly. There are 4 possible options and 3 different tests: + +1. Code in both the Packer and Vagrant directories was modified + * In this case, the CircleCI worker will execute `ci/circle_workflows/packer_and_vagrant_changes.sh` +2. Code in neither the Packer and Vagrant directories was modified + * In this case, the CircleCI worker will execute the default test `ci/circle_worker/vagrant_changes.sh` +3. Code in only the Packer directory was modified + * In this case, the CircleCI worker will execute `ci/circle_worker/packer_changes.sh` +4. Code in only the Vagrant directory was modified + * In this case, the CircleCI worker will execute `ci/circle_worker/vagrant_changes.sh` + +## Test Case Walkthroughs + +### packer_and_vagrant_changes.sh +1. Spins up a single Packet server +2. Bootstraps the Packet server by calling `ci/build_machine_bootstrap.sh` with no arguments +3. Builds the Windows10 and Windows2016 images one at a time +4. Moves the resulting boxes to the Boxes directory +5. Brings each Vagrant host online one-by-one +6. CircleCI records the build results from the Packet server + +### vagrant_changes.sh +1. Spins up a single Packet server +2. Bootstraps the Packet server by calling `ci/build_machine_bootstrap.sh` with the `--vagrant-only` argument +3. Downloads the pre-build Windows10 and Windows2016 boxes from https://detectionlab.network directly to the Boxes directory +4. Brings each Vagrant host online one-by-one +5. CircleCI records the build results from the Packet server + + +### packer_changes.sh +1. Spins up two separate Packet servers to allow the Packer boxes to be built in parallel +2. Bootstraps each packet Server by calling `ci/build_machine_bootstrap.sh` with the `--packer-only` argument +3. Starts the Packer build process on each server +4. CircleCI records the build result from each Packet server + +``` + +------------+ + | | + | | + | | + | Github | + | | + | | + +------+-----+ + | + | + | Pull Request + | + v + +------+-----+ + | | + | | + | Circle | + +----------------------------->| Worker | + | | | + | | | + | | | + | +------+-----+ + | | + | | Code changes are evaluated + | | to determine which test suite + | | to run + | | + | v + | +----------------+--------------+ +Circle Worker | | packer_and_vagrant_changes.sh | +quries for | | vagrant_changes.sh | +build results | | packer_changes.sh | + | +----------------+--------------+ + | | + | | + | | | + | | + | | + | | + | | + | | 1. Provision Packet server(s) + | | 2. Copy repo to server + | | 3. Run server bootstrap + | | 4. Bootstrap calls build.sh with + | | the appropriate arguments + | | + | | + | +---------v---------+ + | | | + | | | + | | | + +-------------------------->| Packet Server | + | | + | | + | | + +-------------------+ + +``` diff --git a/ci/automated_install.sh b/ci/automated_install.sh deleted file mode 100644 index 8b20cda..0000000 --- a/ci/automated_install.sh +++ /dev/null @@ -1,54 +0,0 @@ -#! /bin/bash - -# This script is run on the Packet.net baremetal server for CI tests. -# This script will build the entire lab from scratch and takes 3-4 hours -# on a Packet.net host -# While building, the server will start a webserver on Port 80 that contains -# the text "building". Once the test is completed, the text will be replaced -# with "success" or "failed". - -# Install Virtualbox 5.2 -echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list -wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - -apt-get update -apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2 - -echo "building" > /var/www/html/index.html - -# Set up firewall -ufw allow ssh -ufw allow http -ufw default allow outgoing -ufw --force enable - -# Install Vagrant -mkdir /opt/vagrant -cd /opt/vagrant || exit 1 -wget https://releases.hashicorp.com/vagrant/2.0.3/vagrant_2.0.3_x86_64.deb -dpkg -i vagrant_2.0.3_x86_64.deb -vagrant plugin install vagrant-reload - -# Install Packer -mkdir /opt/packer -cd /opt/packer || exit 1 -wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip -unzip packer_1.1.3_linux_amd64.zip -cp packer /usr/local/bin/packer - -# Make the packer images headless -for file in $(ls *.json); do - sed -i 's/"headless": false,/"headless": true,/g' "$file"; -done - -# Make the Vagrant instances headless -cd /opt/DetectionLab/Vagrant || exit 1 -sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile - -# Ensure the script is executable -chmod +x /opt/DetectionLab/build.sh -cd /opt/DetectionLab || exit 1 - -# Start the build in a tmux session -sn=tmuxsession -tmux new-session -s "$sn" -d -tmux send-keys -t "$sn:0" './build.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter diff --git a/ci/automated_install_vagrant_only.sh b/ci/automated_install_vagrant_only.sh deleted file mode 100644 index ce4f418..0000000 --- a/ci/automated_install_vagrant_only.sh +++ /dev/null @@ -1,42 +0,0 @@ -#! /bin/bash - -# This script is run on the Packet.net baremetal server for CI tests. -# This script will bootstrap the build by downloading pre-build Packer boxes -# and should take no more than 90 minutes on a Packet.net host. -# While building, the server will start a webserver on Port 80 that contains -# the text "building". Once the test is completed, the text will be replaced -# with "success" or "failed". - -# Install Virtualbox 5.2 -echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list -wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - -apt-get update -apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2 - -echo "building" > /var/www/html/index.html - -# Set up firewall -ufw allow ssh -ufw allow http -ufw default allow outgoing -ufw --force enable - -# Install Vagrant -mkdir /opt/vagrant -cd /opt/vagrant || exit 1 -wget https://releases.hashicorp.com/vagrant/2.0.3/vagrant_2.0.3_x86_64.deb -dpkg -i vagrant_2.0.3_x86_64.deb -vagrant plugin install vagrant-reload - -# Make the Vagrant instances headless -cd /opt/DetectionLab/Vagrant || exit 1 -sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile - -# Ensure the script is executable -chmod +x /opt/DetectionLab/build_vagrant_only.sh -cd /opt/DetectionLab || exit 1 - -# Start the build in a tmux session -sn=tmuxsession -tmux new-session -s "$sn" -d -tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter diff --git a/ci/build_machine_bootstrap.sh b/ci/build_machine_bootstrap.sh new file mode 100755 index 0000000..a6089d1 --- /dev/null +++ b/ci/build_machine_bootstrap.sh @@ -0,0 +1,93 @@ +#! /bin/bash + +# This script is run on the Packet.net baremetal server for CI tests. +# This script will build the entire lab from scratch and takes 3-4 hours +# on a Packet.net host +# While building, the server will start a webserver on Port 80 that contains +# the text "building". Once the test is completed, the text will be replaced +# with "success" or "failed". + +ARGS="$1" +PACKER_ONLY=0 +VAGRANT_ONLY=0 + +if [ ! -z "$1" ]; then + case "$1" in + --packer-only) + PACKER_ONLY=1 + ;; + --vagrant-only) + VAGRANT_ONLY=1 + ;; + *) + echo "\"$ARGS\" is not a supported argument to this script. Quitting" + exit 1 + ;; + esac +fi + +echo "Args: $ARGS" + +if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then + echo "Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong." + exit 1 +fi + +# Install Virtualbox 5.2 +echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list +wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - +apt-get update +apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2 + +echo "building" > /var/www/html/index.html + +# Set up firewall +ufw allow ssh +ufw allow http +ufw default allow outgoing +ufw --force enable + +if [ "$PACKER_ONLY" -eq 0 ]; then + # Install Vagrant + mkdir /opt/vagrant + cd /opt/vagrant || exit 1 + wget https://releases.hashicorp.com/vagrant/2.0.3/vagrant_2.0.3_x86_64.deb + dpkg -i vagrant_2.0.3_x86_64.deb + vagrant plugin install vagrant-reload + + # Make the Vagrant instances headless + cd /opt/DetectionLab/Vagrant || exit 1 + sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile +fi + +if [ "$VAGRANT_ONLY" -eq 0 ]; then + # Install Packer + mkdir /opt/packer + cd /opt/packer || exit 1 + wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip + unzip packer_1.1.3_linux_amd64.zip + cp packer /usr/local/bin/packer + + # Make the Packer images headless + cd /opt/DetectionLab/Packer || exit 1 + for file in *.json; do + sed -i 's/"headless": false,/"headless": true,/g' "$file"; + done +fi + +# Ensure the script is executable +chmod +x /opt/DetectionLab/build.sh +cd /opt/DetectionLab || exit 1 + +# Start the build in a tmux session +sn=tmuxsession +tmux new-session -s "$sn" -d +if [ "$PACKER_ONLY" -eq 1 ]; then + tmux send-keys -t "$sn:0" './build.sh virtualbox --packer-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter +fi +if [ "$VAGRANT_ONLY" -eq 1 ]; then + tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter +fi +if [[ "$PACKER_ONLY" -eq 0 ]] && [[ "$VAGRANT_ONLY" -eq 0 ]]; then + tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter +fi diff --git a/ci/circle_workflows/packer_and_vagrant_changes.sh b/ci/circle_workflows/packer_and_vagrant_changes.sh new file mode 100644 index 0000000..d20b7b7 --- /dev/null +++ b/ci/circle_workflows/packer_and_vagrant_changes.sh @@ -0,0 +1,80 @@ +#! /bin/bash + +set -e + +# Create artifacts directory +if [ ! -d "/tmp/artifacts" ]; then + mkdir /tmp/artifacts +fi + +## Delete stale servers if they exist +DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"') +if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then + echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID" + curl -X DELETE -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID" +fi + +## Provision a Type1 baremetal Packet.net server +echo "Provisioning a server on Packet.net" +DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') +# Make sure the device ID is sane. +# TODO: maybe make this a regex +if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then + echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID" + exit 1 +fi +echo "Server successfully provisioned with ID: $DEVICE_ID" + +echo "Sleeping 10 minutes to wait for Packet server to be provisioned" +sleep 300 +echo "Sleeping 5 more minutes (CircleCI Keepalive)" +sleep 300 + +## Recording the IP address of the newly provisioned Packet server +IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') + +# Copy repo to Packet server +# TODO: Tar up the repo and expand it remotely +cd ~/repo +rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab + +## Running install script on Packet server +ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh + +echo "Sleeping 5 minutes to allow the build process to start" +sleep 300 + +## Waiting for Packet server to post build results +MINUTES_PAST=0 +while [ "$MINUTES_PAST" -lt 400 ]; do + STATUS=$(curl $IP_ADDRESS) + if [ "$STATUS" == "building" ]; then + echo "$STATUS" + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present" + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" + sleep 300 + ((MINUTES_PAST += 5)) + else + break + fi +done +if [ "$MINUTES_PAST" -gt 400 ]; then + echo "Serer timed out. Uptime: $MINUTES_PAST minutes." + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present" + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + exit 1 +fi + +## Recording the build results +echo $STATUS +if [ "$STATUS" != "success" ]; then + echo "Build failed. Cleaning up server with ID $DEVICE_ID" + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present" + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + exit 1 +fi +echo "Build was successful. Cleaning up server with ID $DEVICE_ID" +curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" +exit 0 diff --git a/ci/circle_workflows/packer_changes.sh b/ci/circle_workflows/packer_changes.sh new file mode 100644 index 0000000..b13dac4 --- /dev/null +++ b/ci/circle_workflows/packer_changes.sh @@ -0,0 +1,99 @@ +#! /bin/bash + +set -e + +# Create artifacts directory +if [ ! -d "/tmp/artifacts" ]; then + mkdir /tmp/artifacts +fi + +## Delete stale servers if they exist +echo "Deleting stale Packet.net servers" +DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"') +if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then + echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID" + curl -X DELETE -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID" +fi + +## Provision two Type1 baremetal Packet.net servers +echo "Provisioning packerwindows2016 on Packet.net" +SERVER1_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows2016", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') +if [ "$(echo -n $SERVER1_ID | wc -c)" -ne 36 ]; then + echo "Server may have failed provisionining. Device ID is set to: $SERVER1_ID" + exit 1 +fi +echo "packerwindows2016 successfully provisioned with ID: $SERVER1_ID" + +sleep 5 # Wait a bit before issuing another provision command + +echo "Provisioning packerwindows10 on Packet.net" +SERVER2_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows10", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') +if [ "$(echo -n $SERVER2_ID | wc -c)" -ne 36 ]; then + echo "Server may have failed provisionining. Device ID is set to: $SERVER2_ID" + exit 1 +fi +echo "packerwindows10 successfully provisioned with ID: $SERVER2_ID" + +echo "Sleeping 10 minutes to wait for Packet servers to finish provisiong" +sleep 300 +echo "Sleeping 5 more minutes (CircleCI Keepalive)" +sleep 300 + +## Recording the IP address of the newly provisioned Packet servers +SERVER1_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER1_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') +SERVER2_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER2_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') + +# Copy repo to Packet servers +# TODO: Tar up the repo and expand it remotely +cd ~/repo +rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab +rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab + +## Running install script on Packet server +ssh -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only +ssh -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only + +sleep 30 + +## Waiting for Packet server to post build results +MINUTES_PAST=0 +while [ "$MINUTES_PAST" -lt 150 ]; do + SERVER1_STATUS=$(curl $SERVER1_IP_ADDRESS) + SERVER2_STATUS=$(curl $SERVER2_IP_ADDRESS) + if [[ "$SERVER1_STATUS" == "building" ]] || [[ "$SERVER2_STATUS" == "building" ]]; then + echo "$SERVER1_STATUS" :: "$SERVER2_STATUS" + scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log + scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log + sleep 300 + ((MINUTES_PAST += 5)) + fi + if [[ "$SERVER1_STATUS" != "building" ]] && [[ "$SERVER2_STATUS" != "building" ]]; then + break + fi + if [ "$MINUTES_PAST" -gt 150 ]; then + echo "Serer timed out. Uptime: $MINUTES_PAST minutes." + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" + exit 1 + fi +done + +## Recording the build results +echo "Server1 Status: $SERVER1_STATUS" +echo "Server2 Status: $SERVER2_STATUS" +if [ "$SERVER1_STATUS" != "success" ]; then + echo "Build failed. Cleaning up server with ID $SERVER1_ID" + scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log || echo "Serveer1 packer_build.log not available yet" + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" + exit 1 +fi +if [ "$SERVER2_STATUS" != "success" ]; then + echo "Build failed. Cleaning up server with ID $SERVER2_ID" + scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log || echo "Server2 packer_build.log not available yet" + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" + exit 1 +fi +echo "Builds were successful. Cleaning up servers with IDs $SERVER1_ID and $SERVER2_ID" +curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" +curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" +exit 0 diff --git a/ci/circle_workflows/vagrant_changes.sh b/ci/circle_workflows/vagrant_changes.sh new file mode 100644 index 0000000..6fbd206 --- /dev/null +++ b/ci/circle_workflows/vagrant_changes.sh @@ -0,0 +1,74 @@ +#! /bin/bash + +set -e + +# Create artifacts directory +if [ ! -d "/tmp/artifacts" ]; then + mkdir /tmp/artifacts +fi + +## Delete stale servers if they exist +DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"') +if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then + echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID" + curl -X -s DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID" +fi + +## Provision a Type1 baremetal Packet.net server +echo "Provisioning a server on Packet.net" +DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') +# Make sure the device ID is sane. +# TODO: maybe make this a regex +if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then + echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID" + exit 1 +fi +echo "Server successfully provisioned with ID: $DEVICE_ID" + +echo "Sleeping 10 minutes to wait for Packet server to be provisioned" +sleep 300 +echo "Sleeping 5 more minutes (CircleCI Keepalive)" +sleep 300 + +## Recording the IP address of the newly provisioned Packet server +IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') + +# Copy repo to Packet server +# TODO: Tar up the repo and expand it remotely +cd ~/repo +rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab + +## Running install script on Packet server +ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --vagrant-only + +## Waiting for Packet server to post build results +MINUTES_PAST=0 +while [ "$MINUTES_PAST" -lt 120 ]; do + STATUS=$(curl $IP_ADDRESS) + if [ "$STATUS" == "building" ]; then + echo "$STATUS" + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "vagrant_build.log not available yet" + sleep 300 + ((MINUTES_PAST += 5)) + else + break + fi + if [ "$MINUTES_PAST" -gt 120 ]; then + echo "Serer timed out. Uptime: $MINUTES_PAST minutes." + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + exit 1 + fi +done + +## Recording the build results +echo $STATUS +if [ "$STATUS" != "success" ]; then + scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log + echo "Build failed. Cleaning up server with ID $DEVICE_ID" + curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + exit 1 +fi +echo "Build was successful. Cleaning up server with ID $DEVICE_ID" +curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" +exit 0 From 11b0da94bac6877813607fa2d18961353dda0155 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Wed, 21 Mar 2018 23:21:16 -0700 Subject: [PATCH 11/16] Add commit_id parsing --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e2a398..37d2556 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,7 @@ jobs: - run: name: Check if Packer files were modified command: | + export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Packer/\")" -gt 0 ]; then export PACKER_MODIFIED=1 else @@ -20,6 +21,7 @@ jobs: - run: name: Check if Vagrant files were modified command: | + export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Vagrant/\")" -gt 0 ]; then export VAGRANT_MODIFIED=1 else From 8239eac52f4efcca3ae2cf03b2e4f7960aa7e151 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Thu, 22 Mar 2018 00:04:20 -0700 Subject: [PATCH 12/16] Updating the Circle build to support branches with multiple commits --- .circleci/config.yml | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 37d2556..ca009ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,43 +11,67 @@ jobs: - run: name: Check if Packer files were modified command: | - export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" - if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Packer/\")" -gt 0 ]; then - export PACKER_MODIFIED=1 + ## This handles the cases where there are multiple commits + if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then + if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Packer/)" -gt 0 ]; then + export PACKER_MODIFIED=1 + else + export PACKER_MODIFIED=0 + fi else - export PACKER_MODIFIED=0 + ## This handles the cases where there is only a single commit + export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" + if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c ^Packer/)" -gt 0 ]; then + export PACKER_MODIFIED=1 + else + export PACKER_MODIFIED=0 + fi fi + echo "PACKER_MODIFIED=$PACKER_MODIFIED" - run: name: Check if Vagrant files were modified command: | - export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" - if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Vagrant/\")" -gt 0 ]; then - export VAGRANT_MODIFIED=1 + if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then + if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Vagrant/)" -gt 0 ]; then + export VAGRANT_MODIFIED=1 + else + export VAGRANT_MODIFIED=0 + fi else - export VAGRANT_MODIFIED=0 + export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" + if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Vagrant/\")" -gt 0 ]; then + export VAGRANT_MODIFIED=1 + else + export VAGRANT_MODIFIED=0 + fi fi + echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED" - run: name: Choose which build gets run command: | if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then + echo "Running the test suite for Packer and Vagrant changes" chmod +x ci/circle_workflows/packer_and_vagrant_changes.sh ci/circle_workflows/packer_and_vagrant_changes.sh exit 0 fi if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then + echo "Running the default test suite (Vagrant-only)" chmod +x ci/circle_workflows/vagrant_changes.sh ci/circle_workflows/vagrant_changes.sh exit 0 fi if [ "$PACKER_MODIFIED" -eq 1 ]; then + echo "Running the test suite for Packer-only changes" chmod +x ci/circle_workflows/packer_changes.sh ci/circle_workflows/packer_changes.sh exit 0 fi if [ "$VAGRANT_MODIFIED" -eq 1 ]; then + echo "Running the test suite for Vagrant-only changes" chmod +x ci/circle_workflows/vagrant_changes.sh ci/circle_workflows/vagrant_changes.sh exit 0 From d13f0ea89743b201101450a677c893a07802de93 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Thu, 22 Mar 2018 00:08:37 -0700 Subject: [PATCH 13/16] Testing to see if environment variables are not persisting across commands --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ca009ec..46814d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,6 +52,8 @@ jobs: - run: name: Choose which build gets run command: | + echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED" + echo "PACKER_MODIFIED=$PACKER_MODIFIED" if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then echo "Running the test suite for Packer and Vagrant changes" chmod +x ci/circle_workflows/packer_and_vagrant_changes.sh From 29ee83d6d6671b5fe1e3c2cd42aed3eb17488dae Mon Sep 17 00:00:00 2001 From: Chris Long Date: Thu, 22 Mar 2018 00:13:18 -0700 Subject: [PATCH 14/16] Merge circle config into a single big command --- .circleci/config.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 46814d7..3bd8c39 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,8 +9,9 @@ jobs: - checkout - run: - name: Check if Packer files were modified + name: Choose which test suite to run based on which files were modified command: | + ## Checking commits for changes to Packer files ## This handles the cases where there are multiple commits if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Packer/)" -gt 0 ]; then @@ -27,11 +28,7 @@ jobs: export PACKER_MODIFIED=0 fi fi - echo "PACKER_MODIFIED=$PACKER_MODIFIED" - - - run: - name: Check if Vagrant files were modified - command: | + ## Checking commits for changes to Vagrant files if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Vagrant/)" -gt 0 ]; then export VAGRANT_MODIFIED=1 @@ -47,13 +44,8 @@ jobs: fi fi echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED" - - - - run: - name: Choose which build gets run - command: | - echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED" echo "PACKER_MODIFIED=$PACKER_MODIFIED" + ## Choosing which test suite to run based on the files that were changed if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then echo "Running the test suite for Packer and Vagrant changes" chmod +x ci/circle_workflows/packer_and_vagrant_changes.sh From 8d4bae508815c842b5bbff633aedc6c7ec074af8 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Thu, 22 Mar 2018 09:07:46 -0700 Subject: [PATCH 15/16] Fixing a typo in the CircleCI config --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3bd8c39..8e038f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: ci/circle_workflows/packer_and_vagrant_changes.sh exit 0 fi - if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then + if [[ "$PACKER_MODIFIED" -eq 0 ]] && [[ "$VAGRANT_MODIFIED" -eq 0 ]]; then echo "Running the default test suite (Vagrant-only)" chmod +x ci/circle_workflows/vagrant_changes.sh ci/circle_workflows/vagrant_changes.sh From 2469d3d85db56d53dd92588b87931f0af3d3ef54 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Fri, 23 Mar 2018 18:11:28 -0700 Subject: [PATCH 16/16] Updating the CircleCI logic --- .circleci/config.yml | 56 ++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e038f9..60ffbea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,38 +11,38 @@ jobs: - run: name: Choose which test suite to run based on which files were modified command: | + ## As far as I'm aware, there are three possibile cases to check for here. The CIRCLE_COMPARE_URL environment variable will either contain: + ## - A SHA1 hash for the commit (If there is only a single commit on the branch) + ## - Two SHA1 hashes in a "xxxxxxxx..yyyyyyyy" (comparing the two most recent commits + ## - Nothing + ## There is certainly a better way to accomplish all of this, but my limited knowledge of git has lead me to write this awful spaghetti code :-/ + ## We'll handle all 3 of these cases below: ## Checking commits for changes to Packer files - ## This handles the cases where there are multiple commits - if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then - if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Packer/)" -gt 0 ]; then - export PACKER_MODIFIED=1 - else - export PACKER_MODIFIED=0 - fi + COMMIT_SHA1="" + PACKER_MODIFIED=0 + VAGRANT_MODIFIED=0 + ## Check for empty CIRCLE_COMPARE_URL. If it is, set the SHA1 hash to the CIRCLE_SHA1 environment variable + if [ "$(echo -n $CIRCLE_COMPARE_URL | wc -c)" -eq 0 ]; then + export COMMIT_SHA1=$CIRCLE_SHA1 + ## Check for two short-hashes in the CIRCLE_COMPARE_URL by searching for '..'. If it exists, use the second short-hash + elif echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then + export COMMIT_SHA1="$(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5)" + ## Check for a single short hash in the CIRCLE_COMPARE_URL and use it if it exists + ## TODO: This check may not be needed. else - ## This handles the cases where there is only a single commit - export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" - if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c ^Packer/)" -gt 0 ]; then - export PACKER_MODIFIED=1 - else - export PACKER_MODIFIED=0 - fi + export COMMIT_SHA1="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" fi - ## Checking commits for changes to Vagrant files - if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then - if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Vagrant/)" -gt 0 ]; then - export VAGRANT_MODIFIED=1 - else - export VAGRANT_MODIFIED=0 - fi - else - export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" - if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Vagrant/\")" -gt 0 ]; then - export VAGRANT_MODIFIED=1 - else - export VAGRANT_MODIFIED=0 - fi + ## Display the files that were modified in this branch + echo "Files modified since origin/Master:" + git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" + ## Check to see if Packer files were modified + if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" | grep -c ^Packer/)" -gt 0 ]; then + export PACKER_MODIFIED=1 fi + if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" | grep -c ^Vagrant/)" -gt 0 ]; then + export VAGRANT_MODIFIED=1 + fi + echo "Displaying the values of the modifier environment variables:" echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED" echo "PACKER_MODIFIED=$PACKER_MODIFIED" ## Choosing which test suite to run based on the files that were changed