trinitor/DetectionLab
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #67 from clong/build_scripts_format

Browse Source 
Fixing formatting and some shell anti-patterns
This commit is contained in:
Chris Long
2018-03-01 11:22:16 -08:00
committed by GitHub
parent db1ed146f8 79ce9bfcdd
commit e7d937f8d0
11 changed files with 211 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.circleci/config.yml
View File

@@ -63,6 +63,9 @@ jobs:
fi
done
if [ "$MINUTES_PAST" -gt 120 ]; then
export IP_ADDRESS=$(cat /tmp/ip_address);
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log .
cat vagrant.log
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
@@ -74,6 +77,9 @@ jobs:
export STATUS=$(cat /tmp/status)
echo $STATUS
if [ "$STATUS" != "success" ]; then
export IP_ADDRESS=$(cat /tmp/ip_address);
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log .
cat vagrant.log
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi

2
Packer/scripts/debloat-windows.ps1
View File

@@ -2,6 +2,8 @@ if ($env:PACKER_BUILDER_TYPE -And $($env:PACKER_BUILDER_TYPE).startsWith("hyperv
Write-Host Skip debloat steps in Hyper-V build.
} else {
Write-Host Downloading debloat zip
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$url="https://github.com/StefanScherer/Debloat-Windows-10/archive/master.zip"
(New-Object System.Net.WebClient).DownloadFile($url, "$env:TEMP\debloat.zip")
Expand-Archive -Path $env:TEMP\debloat.zip -DestinationPath $env:TEMP -Force

4
Vagrant/bootstrap.sh
View File

@@ -41,8 +41,8 @@ else
# Get Splunk.com into the DNS cache. Sometimes resolution randomly fails during wget below
dig @8.8.8.8 splunk.com
# Download Splunk
wget --progress=bar:force -O splunk-7.0.1-2b5b15c4ee89-linux-2.6-amd64.deb 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=7.0.1&product=splunk&filename=splunk-7.0.1-2b5b15c4ee89-linux-2.6-amd64.deb&wget=true'
dpkg -i splunk-7.0.1-2b5b15c4ee89-linux-2.6-amd64.deb
wget --progress=bar:force -O splunk-7.0.2-03bbabbd5c0f-linux-2.6-amd64.deb 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=7.0.2&product=splunk&filename=splunk-7.0.2-03bbabbd5c0f-linux-2.6-amd64.deb&wget=true'
dpkg -i splunk-7.0.2-03bbabbd5c0f-linux-2.6-amd64.deb
/opt/splunk/bin/splunk start --accept-license
/opt/splunk/bin/splunk add index wineventlog -auth 'admin:changeme'
/opt/splunk/bin/splunk add index osquery -auth 'admin:changeme'

2
Vagrant/scripts/download_palantir_osquery.ps1
View File

@@ -3,5 +3,7 @@ Write-Host "Downloading and unzipping the Palantir osquery Repo from Github..."
$osqueryRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\osquery-Master.zip'
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Invoke-WebRequest -Uri "https://github.com/palantir/osquery-configuration/archive/master.zip" -OutFile $osqueryRepoPath
Expand-Archive -path "$osqueryRepoPath" -destinationpath 'c:\Users\vagrant\AppData\Local\Temp' -Force

6
Vagrant/scripts/download_palantir_wef.ps1
View File

@@ -1,7 +1,13 @@
# Purpose: Downloads and unzips a copy of the Palantir WEF Github Repo. This includes WEF subscriptions and custom WEF channels.
# GitHub requires TLS 1.2
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Write-Host "Downloading and unzipping the Palantir Windows Event Forwarding Repo from Github..."
$wefRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\wef-Master.zip'
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Invoke-WebRequest -Uri "https://github.com/palantir/windows-event-forwarding/archive/master.zip" -OutFile $wefRepoPath
Expand-Archive -path "$wefRepoPath" -destinationpath 'c:\Users\vagrant\AppData\Local\Temp' -Force

2
Vagrant/scripts/install-caldera-agent.ps1
View File

@@ -14,6 +14,8 @@ If (-not (Test-Path "C:\Program Files\cagent\cagent.exe")) {
Write-Host "Downloading Caldera Agent (cagent.exe)"
$cagentPath = "C:\Program Files\cagent\cagent.exe"
$cagentConfPath = "C:\Program Files\cagent\conf.yml"
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
(New-Object System.Net.WebClient).DownloadFile('https://github.com/mitre/caldera-agent/releases/download/v0.1.0/cagent.exe', $cagentPath)
# Ignore SSL warning for conf file download
# https://stackoverflow.com/questions/34331206/ignore-ssl-warning-with-powershell-downloadstring

2
Vagrant/scripts/install-utilities.ps1
View File

@@ -27,6 +27,8 @@ If ($env:computername -eq "WIN10") {
# Purpose: Downloads and unzips a copy of the latest Mimikatz trunk
Write-Host "Determining latest release of Mimikatz..."
# GitHub requires TLS 1.2 as of 2/27
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$tag = (Invoke-WebRequest "https://api.github.com/repos/gentilkiwi/mimikatz/releases" -UseBasicParsing | ConvertFrom-Json)[0].tag_name
$mimikatzDownloadUrl = "https://github.com/gentilkiwi/mimikatz/releases/download/$tag/mimikatz_trunk.zip"
$mimikatzRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\mimikatz_trunk.zip'

176
build.sh
View File

@@ -16,30 +16,27 @@ print_usage() {
check_packer_and_vagrant() {
# Check for existence of Vagrant in PATH
which vagrant > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Vagrant was not found in your PATH.")
(>&2 echo "Please correct this before continuing. Quitting.")
if ! which vagrant >/dev/null; then
(echo >&2 "Vagrant was not found in your PATH.")
(echo >&2 "Please correct this before continuing. Quitting.")
exit 1
fi
# Ensure Vagrant >= 2.0.0
if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
(>&2 echo "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
(echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
fi
# Check for existence of Packer in PATH
which packer > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Packer was not found in your PATH.")
(>&2 echo "Please correct this before continuing. Quitting.")
(>&2 echo "Hint: sudo cp ./packer /usr/local/bin/packer; sudo chmod +x /usr/local/bin/packer")
if ! which packer >/dev/null; then
(echo >&2 "Packer was not found in your PATH.")
(echo >&2 "Please correct this before continuing. Quitting.")
(echo >&2 "Hint: sudo cp ./packer /usr/local/bin/packer; sudo chmod +x /usr/local/bin/packer")
exit 1
fi
}
# Returns 0 if not installed or 1 if installed
check_virtualbox_installed() {
which VBoxManage > /dev/null
if [ "$?" -eq 0 ]; then
if which VBoxManage >/dev/null; then
echo "1"
else
echo "0"
@@ -48,19 +45,23 @@ check_virtualbox_installed() {
# Returns 0 if not installed or 1 if installed
check_vmware_fusion_installed() {
echo "$(ls /Applications | grep -ci 'VMware Fusion.app')"
if [ -e "/Applications/VMware Fusion.app" ]; then
echo "1"
else
echo "0"
fi
}
# Returns 0 if not installed or 1 if installed
check_vmware_vagrant_plugin_installed() {
VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')
if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then
(>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.")
(>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
(>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
VAGRANT_VMWARE_PLUGIN_PRESENT="$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')"
if [ "$VAGRANT_VMWARE_PLUGIN_PRESENT" -eq 0 ]; then
(echo >&2 "VMWare Fusion is installed, but the Vagrant plugin is not.")
(echo >&2 "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
(echo >&2 "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
echo "0"
else
echo $VAGRANT_VMWARE_PLUGIN_PRESENT
echo "$VAGRANT_VMWARE_PLUGIN_PRESENT"
fi
}
@@ -69,7 +70,7 @@ list_providers() {
VBOX_PRESENT=0
VMWARE_FUSION_PRESENT=0
if [ $(uname) == "Darwin" ]; then
if [ "$(uname)" == "Darwin" ]; then
# Detect Providers on OSX
VBOX_PRESENT=$(check_virtualbox_installed)
VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed)
@@ -79,28 +80,25 @@ list_providers() {
VBOX_PRESENT=$(check_virtualbox_installed)
fi
(>&2 echo "Available Providers:")
(echo >&2 "Available Providers:")
if [ "$VBOX_PRESENT" == "1" ]; then
(>&2 echo "virtualbox";)
(echo >&2 "virtualbox")
fi
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]
then
(>&2 echo "vmware_fusion";)
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]; then
(echo >&2 "vmware_fusion")
fi
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]
then
(>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]; then
(echo >&2 "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
exit 1
fi
(>&2 echo -e "\nWhich provider would you like to use?")
read PROVIDER
(echo >&2 -e "\\nWhich provider would you like to use?")
read -r PROVIDER
# Sanity check
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]
then
(>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]; then
(echo >&2 "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
exit 1
fi
echo $PROVIDER
echo "$PROVIDER"
}
# A series of checks to identify potential issues before starting the build
@@ -108,43 +106,42 @@ preflight_checks() {
DL_DIR="$1"
# Check to see if curl is in PATH
which curl > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Please install curl and make sure it is in your PATH.")
if ! which curl >/dev/null; then
(echo >&2 "Please install curl and make sure it is in your PATH.")
exit 1
fi
# Check to see if boxes exist already
BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l)
if [ $BOXES_BUILT -gt 0 ]; then
(>&2 echo "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.")
BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l)
if [ "$BOXES_BUILT" -gt 0 ]; then
(echo >&2 "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.")
exit 1
fi
# Check to see if any Vagrant instances exist already
cd "$DL_DIR"/Vagrant/
VAGRANT_BUILT=$(vagrant status | grep -c 'not created')
if [ $VAGRANT_BUILT -ne 4 ]; then
(>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
# Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true"
VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true
if [ "$VAGRANT_BUILT" -ne 4 ]; then
(echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
exit 1
fi
# Check available disk space. Recommend 80GB free, warn if less.
FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
if [ $FREE_DISK_SPACE -lt 80000 ]; then
(>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n")
(>&2 df -m $HOME)
(>&2 echo "")
FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
if [ "$FREE_DISK_SPACE" -lt 80000 ]; then
(echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n")
(df >&2 -m "$HOME")
(echo >&2 "")
fi
# Check Packer version against known bad
if [ "$(packer --version)" == '1.1.2' ]; then
(>&2 echo "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.")
(echo >&2 "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.")
exit 1
fi
# Ensure the vagrant-reload plugin is installed
VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
(>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
$(which vagrant) plugin install "vagrant-reload"
if [ "$?" -ne 0 ]; then
(>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
(echo >&2 "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
if ! $(which vagrant) plugin install "vagrant-reload"; then
(echo >&2 "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
exit 1
fi
fi
@@ -159,11 +156,10 @@ packer_build_box() {
PROVIDER="vmware"
fi
cd "$DL_DIR/Packer"
(>&2 echo "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.")
$(which packer) build --only="$PROVIDER-iso" $BOX.json
if [ "$?" -ne 0 ]; then
(>&2 echo "Something went wrong while attempting to build the $BOX box.")
(>&2 echo "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
(echo >&2 "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.")
if ! $(which packer) build --only="$PROVIDER-iso" "$BOX".json; then
(echo >&2 "Something went wrong while attempting to build the $BOX box.")
(echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
fi
}
@@ -178,12 +174,12 @@ move_boxes() {
mv "$DL_DIR"/Packer/*.box "$DL_DIR"/Boxes
# Ensure Windows 10 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
(>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.")
(echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Ensure Windows 2016 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
(>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.")
(echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.")
exit 1
fi
}
@@ -193,9 +189,9 @@ vagrant_up_host() {
PROVIDER="$1"
HOST="$2"
DL_DIR="$3"
(>&2 echo "Attempting to bring up the $HOST host using Vagrant")
(echo >&2 "Attempting to bring up the $HOST host using Vagrant")
cd "$DL_DIR"/Vagrant
$(which vagrant) up $HOST --provider="$PROVIDER" 1>&2
$(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2
echo "$?"
}
@@ -205,7 +201,7 @@ vagrant_reload_host() {
DL_DIR="$2"
cd "$DL_DIR"/Vagrant
# Attempt to reload the host if the vagrant up command didn't exit cleanly
$(which vagrant) reload $HOST --provision 1>&2
$(which vagrant) reload "$HOST" --provision 1>&2
echo "$?"
}
@@ -221,22 +217,21 @@ post_build_checks() {
# Associative arrays are only supported in bash 4 and up
if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then
declare -A SERVICES
SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
for SERVICE in "${!SERVICES[@]}"
do
SERVICES=(["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
for SERVICE in "${!SERVICES[@]}"; do
if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then
(>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
fi
done
else
if [ "$CALDERA_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: Caldera failed post-build tests and may not be functioning correctly.")
fi
if [ "$SPLUNK_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: Splunk failed post-build tests and may not be functioning correctly.")
fi
if [ "$FLEET_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: Fleet failed post-build tests and may not be functioning correctly.")
fi
fi
}
@@ -244,7 +239,7 @@ post_build_checks() {
main() {
# Get location of build.sh
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROVIDER=""
LAB_HOSTS=("logger" "dc" "wef" "win10")
# If no argument was supplied, list available providers
@@ -261,22 +256,22 @@ main() {
# TODO: Check to make sure they actually have their provider installed
case "$1" in
virtualbox)
PROVIDER="$1"
;;
PROVIDER="$1"
;;
vmware_fusion)
PROVIDER="$1"
;;
PROVIDER="$1"
;;
*)
echo "\"$1\" is not a valid provider. Listing available providers:"
PROVIDER=$(list_providers)
;;
echo "\"$1\" is not a valid provider. Listing available providers:"
PROVIDER=$(list_providers)
;;
esac
fi
preflight_checks $DL_DIR
packer_build_box $PROVIDER "windows_2016" $DL_DIR
packer_build_box $PROVIDER "windows_10" $DL_DIR
move_boxes $PROVIDER $DL_DIR
preflight_checks "$DL_DIR"
packer_build_box "$PROVIDER" "windows_2016" "$DL_DIR"
packer_build_box "$PROVIDER" "windows_10" "$DL_DIR"
move_boxes "$PROVIDER" "$DL_DIR"
# Change provider back to original selection if using vmware_fusion
if [ "$PROVIDER" == "vmware" ]; then
@@ -284,19 +279,18 @@ main() {
fi
# Vagrant up each box and attempt to reload one time if it fails
for VAGRANT_HOST in "${LAB_HOSTS[@]}"
do
RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR)
for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do
RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR")
if [ "$RET" -eq 0 ]; then
(>&2 echo "Good news! $VAGRANT_HOST was built successfully!")
(echo >&2 "Good news! $VAGRANT_HOST was built successfully!")
fi
# Attempt to recover if the intial "vagrant up" fails
if [ "$RET" -ne 0 ]; then
(>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.")
(>&2 echo "Attempting to reload and reprovision the host...")
RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR)
(echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.")
(echo >&2 "Attempting to reload and reprovision the host...")
RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR")
if [ "$RETRY_STATUS" -ne 0 ]; then
(>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
(echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
exit 1
fi
fi
@@ -305,5 +299,5 @@ main() {
post_build_checks
}
main $@
main "$@"
exit 0

173
build_vagrant_only.sh
View File

@@ -16,22 +16,20 @@ print_usage() {
check_vagrant() {
# Check for existence of Vagrant in PATH
which vagrant > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Vagrant was not found in your PATH.")
(>&2 echo "Please correct this before continuing. Quitting.")
if ! which vagrant >/dev/null; then
(echo >&2 "Vagrant was not found in your PATH.")
(echo >&2 "Please correct this before continuing. Quitting.")
exit 1
fi
# Ensure Vagrant >= 2.0.0
if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
(>&2 echo "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
(echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
fi
}
# Returns 0 if not installed or 1 if installed
check_virtualbox_installed() {
which VBoxManage > /dev/null
if [ "$?" -eq 0 ]; then
if ! which VBoxManage >/dev/null; then
echo "1"
else
echo "0"
@@ -40,19 +38,23 @@ check_virtualbox_installed() {
# Returns 0 if not installed or 1 if installed
check_vmware_fusion_installed() {
echo "$(ls /Applications | grep -ci 'VMware Fusion.app')"
if [ -e "/Applications/VMware Fusion.app" ]; then
echo "1"
else
echo "0"
fi
}
# Returns 0 if not installed or 1 if installed
check_vmware_vagrant_plugin_installed() {
VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')
if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then
(>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.")
(>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
(>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
VAGRANT_VMWARE_PLUGIN_PRESENT="$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')"
if [ "$VAGRANT_VMWARE_PLUGIN_PRESENT" -eq 0 ]; then
(echo >&2 "VMWare Fusion is installed, but the Vagrant plugin is not.")
(echo >&2 "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
(echo >&2 "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
echo "0"
else
echo $VAGRANT_VMWARE_PLUGIN_PRESENT
echo "$VAGRANT_VMWARE_PLUGIN_PRESENT"
fi
}
@@ -61,7 +63,7 @@ list_providers() {
VBOX_PRESENT=0
VMWARE_FUSION_PRESENT=0
if [ $(uname) == "Darwin" ]; then
if [ "$(uname)" == "Darwin" ]; then
# Detect Providers on OSX
VBOX_PRESENT=$(check_virtualbox_installed)
VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed)
@@ -71,72 +73,69 @@ list_providers() {
VBOX_PRESENT=$(check_virtualbox_installed)
fi
(>&2 echo "Available Providers:")
(echo >&2 "Available Providers:")
if [ "$VBOX_PRESENT" == "1" ]; then
(>&2 echo "virtualbox";)
(echo >&2 "virtualbox")
fi
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]
then
(>&2 echo "vmware_fusion";)
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]; then
(echo >&2 "vmware_fusion")
fi
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]
then
(>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]; then
(echo >&2 "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
exit 1
fi
(>&2 echo -e "\nWhich provider would you like to use?")
read PROVIDER
(echo >&2 -e "\\nWhich provider would you like to use?")
read -r PROVIDER
# Sanity check
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]
then
(>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]; then
(echo >&2 "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
exit 1
fi
echo $PROVIDER
echo "$PROVIDER"
}
# A series of checks to identify potential issues before starting the build
preflight_checks() {
DL_DIR="$1"
DOWNLOAD_BOXES=1
# Check to see if curl is in PATH
which curl > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Please install curl and make sure it is in your PATH.")
if ! which curl >/dev/null; then
(echo >&2 "Please install curl and make sure it is in your PATH.")
exit 1
fi
# Check to see if wget is in PATH
which wget > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Please install curl and make sure it is in your PATH.")
if ! which wget >/dev/null; then
(echo >&2 "Please install curl and make sure it is in your PATH.")
exit 1
fi
# Check to see if boxes exist already
BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l)
if [ $BOXES_BUILT -gt 0 ]; then
(>&2 echo "WARNING: You seem to have boxes present in the Boxes/ directory already. If you would like fresh boxes downloaded, please remove all files from the Boxes/ directory and re-run this script.")
BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l)
if [ "$BOXES_BUILT" -gt 0 ]; then
(echo >&2 "WARNING: You seem to have boxes present in the Boxes directory already. If you would like fresh boxes downloaded, please remove all files from the Boxes directory and re-run this script.")
DOWNLOAD_BOXES=0
fi
# Check to see if any Vagrant instances exist already
cd "$DL_DIR"/Vagrant/
VAGRANT_BUILT=$(vagrant status | grep -c 'not created')
if [ $VAGRANT_BUILT -ne 4 ]; then
(>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
# Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true"
VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true
if [ "$VAGRANT_BUILT" -ne 4 ]; then
(echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
exit 1
fi
# Check available disk space. Recommend 80GB free, warn if less.
FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
if [ $FREE_DISK_SPACE -lt 80000 ]; then
(>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n")
(>&2 df -m $HOME)
(>&2 echo "")
FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
if [ "$FREE_DISK_SPACE" -lt 80000 ]; then
(echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n")
(df >&2 -m "$HOME")
(echo >&2 "")
fi
# Ensure the vagrant-reload plugin is installed
VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
(>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
$(which vagrant) plugin install "vagrant-reload"
if [ "$?" -ne 0 ]; then
(>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
(echo >&2 "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
if ! $(which vagrant) plugin install "vagrant-reload"; then
(echo >&2 "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
exit 1
fi
fi
@@ -162,30 +161,30 @@ download_boxes() {
# Ensure Windows 10 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
(>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.")
(echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Ensure Windows 2016 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
(>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.")
(echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Verify hashes of VirtualBox boxes
if [ "$PROVIDER" == "virtualbox" ]; then
if [ "$(md5sum windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then
(>&2 echo "Hash mismatch on windows_10_virtualbox.box")
(echo >&2 "Hash mismatch on windows_10_virtualbox.box")
fi
if [ "$(md5sum windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "614f984c82b51471b5bb753940b59d38" ]; then
(>&2 echo "Hash mismatch on windows_2016_virtualbox.box")
(echo >&2 "Hash mismatch on windows_2016_virtualbox.box")
fi
# Verify hashes of VMware boxes
# Verify hashes of VMware boxes
elif [ "$PROVIDER" == "vmware" ]; then
if [ "$(md5 windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then
(>&2 echo "Hash mismatch on windows_10_vmware.box")
(echo >&2 "Hash mismatch on windows_10_vmware.box")
exit 1
fi
if [ "$(md5 windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then
(>&2 echo "Hash mismatch on windows_2016_vmware.box")
(echo >&2 "Hash mismatch on windows_2016_vmware.box")
exit 1
fi
# Reset PROVIDER variable
@@ -198,9 +197,9 @@ vagrant_up_host() {
PROVIDER="$1"
HOST="$2"
DL_DIR="$3"
(>&2 echo "Attempting to bring up the $HOST host using Vagrant")
(echo >&2 "Attempting to bring up the $HOST host using Vagrant")
cd "$DL_DIR"/Vagrant
$(which vagrant) up $HOST --provider="$PROVIDER" 1>&2
$(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2
echo "$?"
}
@@ -210,7 +209,7 @@ vagrant_reload_host() {
DL_DIR="$2"
cd "$DL_DIR"/Vagrant
# Attempt to reload the host if the vagrant up command didn't exit cleanly
$(which vagrant) reload $HOST --provision 1>&2
$(which vagrant) reload "$HOST" --provision 1>&2
echo "$?"
}
@@ -226,22 +225,21 @@ post_build_checks() {
# Associative arrays are only supported in bash 4 and up
if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then
declare -A SERVICES
SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
for SERVICE in "${!SERVICES[@]}"
do
SERVICES=(["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
for SERVICE in "${!SERVICES[@]}"; do
if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then
(>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
fi
done
else
if [ "$CALDERA_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: Caldera failed post-build tests and may not be functioning correctly.")
fi
if [ "$SPLUNK_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: Splunk failed post-build tests and may not be functioning correctly.")
fi
if [ "$FLEET_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: Fleet failed post-build tests and may not be functioning correctly.")
fi
fi
}
@@ -249,7 +247,7 @@ post_build_checks() {
main() {
# Get location of build_vagrant_only.sh
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROVIDER=""
LAB_HOSTS=("logger" "dc" "wef" "win10")
# If no argument was supplied, list available providers
@@ -266,36 +264,39 @@ main() {
# TODO: Check to make sure they actually have their provider installed
case "$1" in
virtualbox)
PROVIDER="$1"
;;
PROVIDER="$1"
;;
vmware_fusion)
PROVIDER="$1"
;;
PROVIDER="$1"
;;
*)
echo "\"$1\" is not a valid provider. Listing available providers:"
PROVIDER=$(list_providers)
;;
echo "\"$1\" is not a valid provider. Listing available providers:"
PROVIDER=$(list_providers)
;;
esac
fi
check_vagrant
preflight_checks $DL_DIR
download_boxes $DL_DIR $PROVIDER
preflight_checks "$DL_DIR"
if [ "$DOWNLOAD_BOXES" -eq 0 ]; then
(echo >&2 "Skipping box downloads since .box files are already present in the Boxes/ directory.")
else
download_boxes "$DL_DIR" "$PROVIDER"
fi
# Vagrant up each box and attempt to reload one time if it fails
for VAGRANT_HOST in "${LAB_HOSTS[@]}"
do
RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR)
for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do
RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR")
if [ "$RET" -eq 0 ]; then
(>&2 echo "Good news! $VAGRANT_HOST was built successfully!")
(echo >&2 "Good news! $VAGRANT_HOST was built successfully!")
fi
# Attempt to recover if the intial "vagrant up" fails
if [ "$RET" -ne 0 ]; then
(>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.")
(>&2 echo "Attempting to reload and reprovision the host...")
RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR)
(echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.")
(echo >&2 "Attempting to reload and reprovision the host...")
RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR")
if [ "$RETRY_STATUS" -ne 0 ]; then
(>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
(echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
exit 1
fi
fi
@@ -304,5 +305,5 @@ main() {
post_build_checks
}
main $@
main "$@"
exit 0

18
ci/automated_install.sh
View File

@@ -7,11 +7,11 @@
# the text "building". Once the test is completed, the text will be replaced
# with "success" or "failed".
# Install Virtualbox 5.1
# Install Virtualbox 5.2
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
apt-get update
apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2
apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2
echo "building" > /var/www/html/index.html
@@ -23,14 +23,14 @@ ufw --force enable
# Install Vagrant
mkdir /opt/vagrant
cd /opt/vagrant
wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb
dpkg -i vagrant_2.0.1_x86_64.deb
cd /opt/vagrant || exit 1
wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb
dpkg -i vagrant_2.0.2_x86_64.deb
vagrant plugin install vagrant-reload
# Install Packer
mkdir /opt/packer
cd /opt/packer
cd /opt/packer || exit 1
wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip
unzip packer_1.1.3_linux_amd64.zip
cp packer /usr/local/bin/packer
@@ -41,14 +41,14 @@ for file in $(ls *.json); do
done
# Make the Vagrant instances headless
cd /opt/DetectionLab/Vagrant
cd /opt/DetectionLab/Vagrant || exit 1
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
# Ensure the script is executable
chmod +x /opt/DetectionLab/build.sh
cd /opt/DetectionLab
cd /opt/DetectionLab || exit 1
# Start the build in a tmux session
sn=tmuxsession
tmux new-session -s "$sn" -d
tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
tmux send-keys -t "$sn:0" './build.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter

16
ci/automated_install_vagrant_only.sh
View File

@@ -7,11 +7,11 @@
# the text "building". Once the test is completed, the text will be replaced
# with "success" or "failed".
# Install Virtualbox 5.1
# Install Virtualbox 5.2
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
apt-get update
apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2
apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2
echo "building" > /var/www/html/index.html
@@ -23,20 +23,20 @@ ufw --force enable
# Install Vagrant
mkdir /opt/vagrant
cd /opt/vagrant
wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb
dpkg -i vagrant_2.0.1_x86_64.deb
cd /opt/vagrant || exit 1
wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb
dpkg -i vagrant_2.0.2_x86_64.deb
vagrant plugin install vagrant-reload
# Make the Vagrant instances headless
cd /opt/DetectionLab/Vagrant
cd /opt/DetectionLab/Vagrant || exit 1
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
# Ensure the script is executable
chmod +x /opt/DetectionLab/build_vagrant_only.sh
cd /opt/DetectionLab
cd /opt/DetectionLab || exit 1
# Start the build in a tmux session
sn=tmuxsession
tmux new-session -s "$sn" -d
tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter