Adding build scripts, continuous integration, upping timeout

This commit is contained in:
Chris Long
2018-02-01 22:10:22 -08:00
parent 4871cb8ecf
commit f10f4a2f67
10 changed files with 824 additions and 6 deletions

70
.circleci/config.yml Normal file
View File

@@ -0,0 +1,70 @@
version: 2
jobs:
build:
machine: true
working_directory: ~/repo
steps:
- checkout
- run:
name: Delete stale Packet servers
command: |
export DELETE_DEVICE_ID=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
- run:
name: Provisioning a baremetal Packet.net server
command: |
curl -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' > /tmp/device
- run: cat /tmp/device | jq ."id" | tr -d '"' > /tmp/device_id
- run: sleep 300
- run: echo "Sleeping 5 more minutes while Packet server is provisioned"
- run: sleep 300
- run:
name: Recording the IP address of the Packet server
command: |
export DEVICE_ID=$(cat /tmp/device_id);
curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" > /tmp/ip_info;
- run: cat /tmp/ip_info | jq ."ip_addresses[0].address" | tr -d '"' > /tmp/ip_address
- run: cd ~/repo
- run:
name: Copying install script to Packet server
command: |
export IP_ADDRESS=$(cat /tmp/ip_address);
scp -i ~/.ssh/id_rsa ./ci/automated_install_vagrant_only.sh root@"$IP_ADDRESS":/root
- run:
name: Running install script on Packet server
command: |
export IP_ADDRESS=$(cat /tmp/ip_address)
ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'chmod +x /root/automated_install_vagrant_only.sh; /bin/bash -c /root/automated_install_vagrant_only.sh'
- run:
name: Waiting for Packet server to post build results
shell: /bin/bash
command: |
export IP_ADDRESS=$(cat /tmp/ip_address)
while true
do
export STATUS=$(curl $IP_ADDRESS)
if [ "$STATUS" == "building" ]; then
echo "$STATUS"
sleep 300
else
echo "$STATUS" > /tmp/status
break
fi
done
- run:
name: Recording build results
shell: /bin/bash
command: |
export DEVICE_ID=$(cat /tmp/device_id)
export STATUS=$(cat /tmp/status)
echo $STATUS
if [ "$STATUS" != "success" ]; then
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 0

View File

@@ -44,7 +44,7 @@
"boot_command": "",
"winrm_username": "vagrant",
"winrm_password": "vagrant",
"winrm_timeout": "2h",
"winrm_timeout": "4h",
"shutdown_command": "shutdown /s /t 10 /f /d p:4:1 /c \"Packer Shutdown\"",
"guest_os_type": "windows9-64",
"disk_size": "{{user `disk_size`}}",
@@ -85,7 +85,7 @@
"boot_command": "",
"winrm_username": "vagrant",
"winrm_password": "vagrant",
"winrm_timeout": "2h",
"winrm_timeout": "4h",
"shutdown_command": "shutdown /s /t 10 /f /d p:4:1 /c \"Packer Shutdown\"",
"guest_os_type": "Windows81_64",
"guest_additions_mode": "disable",

View File

@@ -41,7 +41,7 @@
"boot_wait": "2m",
"winrm_username": "vagrant",
"winrm_password": "vagrant",
"winrm_timeout": "2h",
"winrm_timeout": "4h",
"shutdown_timeout": "2h",
"shutdown_command": "a:/sysprep.bat",
"guest_os_type": "windows8srv-64",
@@ -79,7 +79,7 @@
"boot_wait": "2m",
"winrm_username": "vagrant",
"winrm_password": "vagrant",
"winrm_timeout": "2h",
"winrm_timeout": "4h",
"shutdown_command": "a:/sysprep.bat",
"guest_os_type": "Windows2012_64",
"guest_additions_mode": "disable",

View File

@@ -35,6 +35,7 @@ OSX 10.12.4 | 1.9.2 | 1.0.0 | VMWare Fusion (8.5.6)
OSX 10.12.5 | 1.9.3 | 1.0.0 | VMWare Fusion (8.5.8)
OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.9)
OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.10)
Ubuntu 16.04 | 2.0.1 | 1.1.3 | Virtualbox (5.1)
**Known Bad Versions:**
* Packer 1.1.2 will fail to build VMWare-ISOs correctly due to [this issue](https://github.com/hashicorp/packer/issues/5622).
@@ -42,8 +43,30 @@ OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.10)
---
## Quickstart
DetectionLab now contains build scripts for \*NIX and MacOS users!
There are two build scripts:
- `build.sh` - Builds the entire lab from scratch. Takes 3-5 hours depending on hardware resources and bandwidth
- `build_vagrant_only.sh` - Downloads pre-built Packer boxes from S3 and builds the lab from those boxes. This option is recommended if you have more bandwidth than time.
---
## Building from Scratch
1. Determine which Vagrant provider you want to use.
* Note: Virtualbox is free, the [VMWare vagrant plugin](https://www.vagrantup.com/vmware/#buy-now) is $80.
**NOTE:** If you have more bandwidth than time, you can skip the building of the Packer boxes and download the boxes directly from S3 and put them into the `Boxes` directory:
Provider | Box | URL | MD5 | Size
------------|-----|-----|----|----
Virtualbox |Windows 2016 | https://s3-us-west-1.amazonaws.com/detectionlab/windows_2016_virtualbox.box | 614f984c82b51471b5bb753940b59d38 | 6.4GB
Virtualbox | Windows 10 | https://s3-us-west-1.amazonaws.com/detectionlab/windows_10_virtualbox.box | 30b06e30b36b02ccf1dc5c04017654aa | 5.8GB
VMware | Windows 2016 | https://s3-us-west-1.amazonaws.com/detectionlab/windows_2016_vmware.box | 1511b9dc942c69c2cc5a8dc471fa8865 | 6.7GB
VMware | Windows 10 | https://s3-us-west-1.amazonaws.com/detectionlab/windows_10_vmware.box | 174ad0f0fd2089ff74a880c6dadac74c | 6.0GB
If you choose to download the boxes, you may skip steps 2 and 3. If you don't trust pre-built boxes, I recommend following steps 2 and 3 to build them on your machine.
2. `cd` to the Packer directory and build the Windows 10 and Windows Server 2016 boxes using the commands below. Each build will take about 1 hour. As far as I know, you can only build one box at a time.
```
@@ -51,6 +74,7 @@ $ cd detectionlab/Packer
$ packer build --only=[vmware|virtualbox]-iso windows_10.json
$ packer build --only=[vmware|virtualbox]-iso windows_2016.json
```
3. Once both boxes have built successfully, move the resulting boxes (.box files) in the Packer folder to the Boxes folder:
`mv *.box ../Boxes`

View File

@@ -20,10 +20,12 @@ apm install language-batch
apm install language-docker
# Disable Windows Defender realtime scanning before downloading Mimikatz
set-MpPreference -DisableRealtimeMonitoring $true
If ($hostname -eq "win10") {
set-MpPreference -DisableRealtimeMonitoring $true
}
# Purpose: Downloads and unzips a copy of the latest Mimikatz trunk
Write-Host Determining latest release of Mimikatz...
Write-Host "Determining latest release of Mimikatz..."
$tag = (Invoke-WebRequest "https://api.github.com/repos/gentilkiwi/mimikatz/releases" -UseBasicParsing | ConvertFrom-Json)[0].tag_name
$mimikatzDownloadUrl = "https://github.com/gentilkiwi/mimikatz/releases/download/$tag/mimikatz_trunk.zip"
$mimikatzRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\mimikatz_trunk.zip'

304
build.sh Executable file
View File

@@ -0,0 +1,304 @@
#! /bin/bash
# This script is meant to be used with a fresh clone of DetectionLab and
# will fail to run if boxes have already been created or any of the steps
# from the README have already been run followed.
# Only MacOS and Linux are supported.
# If you encounter issues, feel free to open an issue at
# https://github.com/clong/DetectionLab/issues
set -e
print_usage() {
echo "Usage: ./build.sh <virtualbox|vmware_fusion>"
exit 0
}
check_packer_and_vagrant() {
# Check for existence of Vagrant and Packer in PATH
which vagrant > /dev/null
if [ "$?" -ne 0 ]; then
echo "Vagrant was not found in your PATH."
echo "Please correct this before continuing. Quitting."
exit 1
fi
which packer > /dev/null
if [ "$?" -ne 0 ]; then
echo "Packer was not found in your PATH."
echo "Please correct this before continuing. Quitting."
echo "Hint: sudo cp ./packer /usr/local/bin/packer; sudo chmod +x /usr/local/bin/packer"
exit 1
fi
}
# Returns 0 if not installed or 1 if installed
check_virtualbox_installed() {
which VBoxManage > /dev/null
if [ "$?" -eq 0 ]; then
echo "1"
else
echo "0"
fi
}
# Returns 0 if not installed or 1 if installed
check_vmware_fusion_installed() {
echo "$(ls /Applications | grep -ci 'VMware Fusion.app')"
}
# Returns 0 if not installed or 1 if installed
check_vmware_vagrant_plugin_installed() {
VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')
if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then
(>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.")
(>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
(>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
echo "0"
else
echo $VAGRANT_VMWARE_PLUGIN_PRESENT
fi
}
# List the available Vagrant providers present on the system
list_providers() {
VBOX_PRESENT=0
VMWARE_FUSION_PRESENT=0
if [ $(uname) == "Darwin" ]; then
# Detect Providers on OSX
VBOX_PRESENT=$(check_virtualbox_installed)
VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed)
VAGRANT_VMWARE_PLUGIN_PRESENT=$(check_vmware_vagrant_plugin_installed)
else
# Assume the only other available provider is VirtualBox
VBOX_PRESENT=$(check_virtualbox_installed)
fi
(>&2 echo "Available Providers:")
if [ "$VBOX_PRESENT" == "1" ]; then
(>&2 echo "virtualbox";)
fi
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]
then
(>&2 echo "vmware_fusion";)
fi
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]
then
(>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
exit 1
fi
(>&2 echo -e "\nWhich provider would you like to use?")
read PROVIDER
# Sanity check
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]
then
(>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
exit 1
fi
echo $PROVIDER
}
# A series of checks to identify potential issues before starting the build
preflight_checks() {
DL_DIR="$1"
# Check to see if curl is in PATH
which curl > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Please install curl and make sure it is in your PATH.")
exit 1
fi
# Check to see if boxes exist already
BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l)
if [ $BOXES_BUILT -gt 0 ]; then
(>&2 echo "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.")
exit 1
fi
# Check to see if any Vagrant instances exist already
cd "$DL_DIR"/Vagrant/
VAGRANT_BUILT=$(vagrant status | grep -c 'not created')
if [ $VAGRANT_BUILT -ne 4 ]; then
(>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
exit 1
fi
# Check available disk space. Recommend 80GB free, warn if less.
FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
if [ $FREE_DISK_SPACE -lt 80000 ]; then
(>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n")
(>&2 df -m $HOME)
(>&2 echo "")
fi
# Check Packer version against known bad
if [ $(packer --version) == '1.1.2' ]; then
(>&2 echo "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.")
exit 1
fi
# Ensure the vagrant-reload plugin is installed
VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
(>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
$(which vagrant) plugin install "vagrant-reload"
if [ "$?" -ne 0 ]; then
(>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
exit 1
fi
fi
}
# Builds a box using Packer
packer_build_box() {
PROVIDER="$1"
BOX="$2"
DL_DIR="$3"
if [ "$PROVIDER" == "vmware_fusion" ]; then
PROVIDER="vmware"
fi
cd "$DL_DIR/Packer"
(>&2 echo "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.")
$(which packer) build --only="$PROVIDER-iso" $BOX.json
if [ "$?" -ne 0 ]; then
(>&2 echo "Something went wrong while attempting to build the $BOX box.")
(>&2 echo "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
fi
}
# Moves the boxes from the Packer directory to the Boxes directory
move_boxes() {
PROVIDER="$1"
DL_DIR="$2"
# Hacky workaround for VMware
if [ "$PROVIDER" == "vmware_fusion" ]; then
PROVIDER="vmware"
fi
mv "$DL_DIR"/Packer/*.box "$DL_DIR"/Boxes
# Ensure Windows 10 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
(>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Ensure Windows 2016 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
(>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.")
exit 1
fi
}
# Brings up a single host using Vagrant
vagrant_up_host() {
PROVIDER="$1"
HOST="$2"
DL_DIR="$3"
(>&2 echo "Attempting to bring up the $HOST host using Vagrant")
cd "$DL_DIR"/Vagrant
$(which vagrant) up $HOST --provider="$PROVIDER" 1>&2
echo "$?"
}
# Attempts to reload and re-provision a host if the intial "vagrant up" fails
vagrant_reload_host() {
HOST="$1"
DL_DIR="$2"
cd "$DL_DIR"/Vagrant
# Attempt to reload the host if the vagrant up command didn't exit cleanly
$(which vagrant) reload $HOST --provision 1>&2
echo "$?"
}
# A series of checks to ensure important services are responsive after the build completes.
post_build_checks() {
# If the curl operation fails, we'll just leave the variable equal to 0
# This is needed to prevent the script from exiting if the curl operation fails
CALDERA_CHECK=$(curl -ks -m 2 https://192.168.38.5:8888 | grep -c '302: Found' || echo "")
SPLUNK_CHECK=$(curl -ks -m 2 https://192.168.38.5:8000/en-US/account/login?return_to=%2Fen-US%2F | grep -c 'This browser is not supported by Splunk' || echo "")
FLEET_CHECK=$(curl -ks -m 2 https://192.168.38.5:8412 | grep -c 'Kolide Fleet' || echo "")
BASH_MAJOR_VERSION=$(/bin/bash --version | grep 'GNU bash' | grep -o version\.\.. | cut -d ' ' -f 2 | cut -d '.' -f 1)
# Associative arrays are only supported in bash 4 and up
if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then
declare -A SERVICES
SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
for SERVICE in "${!SERVICES[@]}"
do
if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then
(>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
fi
done
else
if [ "$CALDERA_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.")
fi
if [ "$SPLUNK_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.")
fi
if [ "$FLEET_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.")
fi
fi
}
main() {
# Get location of build.sh
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROVIDER=""
LAB_HOSTS=("logger" "dc" "wef" "win10")
# If no argument was supplied, list available providers
if [ $# -eq 0 ]; then
PROVIDER=$(list_providers)
fi
# If more than one argument was supplied, print usage message
if [ $# -gt 1 ]; then
print_usage
exit 1
fi
if [ $# -eq 1 ]; then
# If the user specifies the provider as an agument, set the variable
# TODO: Check to make sure they actually have their provider installed
case "$1" in
virtualbox)
PROVIDER="$1"
;;
vmware_fusion)
PROVIDER="$1"
;;
*)
echo "\"$1\" is not a valid provider. Listing available providers:"
PROVIDER=$(list_providers)
;;
esac
fi
preflight_checks $DL_DIR
packer_build_box $PROVIDER "windows_2016" $DL_DIR
packer_build_box $PROVIDER "windows_10" $DL_DIR
move_boxes $PROVIDER $DL_DIR
# Change provider back to original selection if using vmware_fusion
if [ "$PROVIDER" == "vmware" ]; then
PROVIDER="vmware_fusion"
fi
# Vagrant up each box and attempt to reload one time if it fails
for VAGRANT_HOST in "${LAB_HOSTS[@]}"
do
RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR)
if [ "$RET" -eq 0 ]; then
(>&2 echo "Good news! $VAGRANT_HOST was built successfully!")
fi
# Attempt to recover if the intial "vagrant up" fails
if [ "$RET" -ne 0 ]; then
(>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.")
(>&2 echo "Attempting to reload and reprovision the host...")
RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR)
if [ "$RETRY_STATUS" -ne 0 ]; then
(>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
exit 1
fi
fi
done
post_build_checks
}
main $@
exit 0

305
build_vagrant_only.sh Executable file
View File

@@ -0,0 +1,305 @@
#! /bin/bash
# This script skips the entire Packer box building process.
# This script downloads pre-built VirtualBox Packer images from S3 and puts
# them into the "Boxes" directory.
# Only MacOS and Linux are supported.
# If you encounter issues, feel free to open an issue at
# https://github.com/clong/DetectionLab/issues
set -e
print_usage() {
echo "Usage: ./build.sh <virtualbox|vmware_fusion>"
exit 0
}
check_vagrant() {
# Check for existence of Vagrant and Packer in PATH
which vagrant > /dev/null
if [ "$?" -ne 0 ]; then
echo "Vagrant was not found in your PATH."
echo "Please correct this before continuing. Quitting."
exit 1
fi
}
# Returns 0 if not installed or 1 if installed
check_virtualbox_installed() {
which VBoxManage > /dev/null
if [ "$?" -eq 0 ]; then
echo "1"
else
echo "0"
fi
}
# Returns 0 if not installed or 1 if installed
check_vmware_fusion_installed() {
echo "$(ls /Applications | grep -ci 'VMware Fusion.app')"
}
# Returns 0 if not installed or 1 if installed
check_vmware_vagrant_plugin_installed() {
VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')
if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then
(>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.")
(>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
(>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
echo "0"
else
echo $VAGRANT_VMWARE_PLUGIN_PRESENT
fi
}
# List the available Vagrant providers present on the system
list_providers() {
VBOX_PRESENT=0
VMWARE_FUSION_PRESENT=0
if [ $(uname) == "Darwin" ]; then
# Detect Providers on OSX
VBOX_PRESENT=$(check_virtualbox_installed)
VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed)
VAGRANT_VMWARE_PLUGIN_PRESENT=$(check_vmware_vagrant_plugin_installed)
else
# Assume the only other available provider is VirtualBox
VBOX_PRESENT=$(check_virtualbox_installed)
fi
(>&2 echo "Available Providers:")
if [ "$VBOX_PRESENT" == "1" ]; then
(>&2 echo "virtualbox";)
fi
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]
then
(>&2 echo "vmware_fusion";)
fi
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]
then
(>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
exit 1
fi
(>&2 echo -e "\nWhich provider would you like to use?")
read PROVIDER
# Sanity check
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]
then
(>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
exit 1
fi
echo $PROVIDER
}
# A series of checks to identify potential issues before starting the build
preflight_checks() {
DL_DIR="$1"
# Check to see if curl is in PATH
which curl > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Please install curl and make sure it is in your PATH.")
exit 1
fi
# Check to see if wget is in PATH
which wget > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Please install curl and make sure it is in your PATH.")
exit 1
fi
# Check to see if boxes exist already
BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l)
if [ $BOXES_BUILT -gt 0 ]; then
(>&2 echo "You appear to have already built at least one box using Packer. This script automatically downloads them for you. Please either delete the existing boxes or follow the build steps in the README to continue.")
exit 1
fi
# Check to see if any Vagrant instances exist already
cd "$DL_DIR"/Vagrant/
VAGRANT_BUILT=$(vagrant status | grep -c 'not created')
if [ $VAGRANT_BUILT -ne 4 ]; then
(>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
exit 1
fi
# Check available disk space. Recommend 80GB free, warn if less.
FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
if [ $FREE_DISK_SPACE -lt 80000 ]; then
(>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n")
(>&2 df -m $HOME)
(>&2 echo "")
fi
# Ensure the vagrant-reload plugin is installed
VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
(>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
$(which vagrant) plugin install "vagrant-reload"
if [ "$?" -ne 0 ]; then
(>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
exit 1
fi
fi
}
# Downloads pre-build Packer boxes from S3 to save time during CI builds
download_boxes() {
DL_DIR="$1"
PROVIDER="$2"
if [ "$PROVIDER" == "virtualbox" ]; then
wget "https://s3-us-west-1.amazonaws.com/detectionlab/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box
wget "https://s3-us-west-1.amazonaws.com/detectionlab/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box
elif [ "$PROVIDER" == "vmware_fusion" ]; then
wget "https://s3-us-west-1.amazonaws.com/detectionlab/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box
wget "https://s3-us-west-1.amazonaws.com/detectionlab/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box
fi
# Hacky workaround
if [ "$PROVIDER" == "vmware_fusion" ]; then
PROVIDER="vmware"
fi
# Ensure Windows 10 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
(>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Ensure Windows 2016 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
(>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Verify hashes of VirtualBox boxes
if [ "$PROVIDER" == "virtualbox" ]; then
if [ "$(md5sum windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then
(>&2 echo "Hash mismatch on windows_10_virtualbox.box")
fi
if [ "$(md5sum windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "614f984c82b51471b5bb753940b59d38" ]; then
(>&2 echo "Hash mismatch on windows_2016_virtualbox.box")
fi
# Verify hashes of VMware boxes
elif [ "$PROVIDER" == "vmware" ]; then
if [ "$(md5 windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then
(>&2 echo "Hash mismatch on windows_10_vmware.box")
exit 1
fi
if [ "$(md5 windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then
(>&2 echo "Hash mismatch on windows_2016_vmware.box")
exit 1
fi
# Reset PROVIDER variable
PROVIDER="vmware_fusion"
fi
}
# Brings up a single host using Vagrant
vagrant_up_host() {
PROVIDER="$1"
HOST="$2"
DL_DIR="$3"
(>&2 echo "Attempting to bring up the $HOST host using Vagrant")
cd "$DL_DIR"/Vagrant
$(which vagrant) up $HOST --provider="$PROVIDER" 1>&2
echo "$?"
}
# Attempts to reload and re-provision a host if the intial "vagrant up" fails
vagrant_reload_host() {
HOST="$1"
DL_DIR="$2"
cd "$DL_DIR"/Vagrant
# Attempt to reload the host if the vagrant up command didn't exit cleanly
$(which vagrant) reload $HOST --provision 1>&2
echo "$?"
}
# A series of checks to ensure important services are responsive after the build completes.
post_build_checks() {
# If the curl operation fails, we'll just leave the variable equal to 0
# This is needed to prevent the script from exiting if the curl operation fails
CALDERA_CHECK=$(curl -ks -m 2 https://192.168.38.5:8888 | grep -c '302: Found' || echo "")
SPLUNK_CHECK=$(curl -ks -m 2 https://192.168.38.5:8000/en-US/account/login?return_to=%2Fen-US%2F | grep -c 'This browser is not supported by Splunk' || echo "")
FLEET_CHECK=$(curl -ks -m 2 https://192.168.38.5:8412 | grep -c 'Kolide Fleet' || echo "")
BASH_MAJOR_VERSION=$(/bin/bash --version | grep 'GNU bash' | grep -o version\.\.. | cut -d ' ' -f 2 | cut -d '.' -f 1)
# Associative arrays are only supported in bash 4 and up
if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then
declare -A SERVICES
SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
for SERVICE in "${!SERVICES[@]}"
do
if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then
(>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
fi
done
else
if [ "$CALDERA_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.")
fi
if [ "$SPLUNK_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.")
fi
if [ "$FLEET_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.")
fi
fi
}
main() {
# Get location of build_vagrant_only.sh
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROVIDER=""
LAB_HOSTS=("logger" "dc" "wef" "win10")
# If no argument was supplied, list available providers
if [ $# -eq 0 ]; then
PROVIDER=$(list_providers)
fi
# If more than one argument was supplied, print usage message
if [ $# -gt 1 ]; then
print_usage
exit 1
fi
if [ $# -eq 1 ]; then
# If the user specifies the provider as an agument, set the variable
# TODO: Check to make sure they actually have their provider installed
case "$1" in
virtualbox)
PROVIDER="$1"
;;
vmware_fusion)
PROVIDER="$1"
;;
*)
echo "\"$1\" is not a valid provider. Listing available providers:"
PROVIDER=$(list_providers)
;;
esac
fi
check_vagrant
preflight_checks $DL_DIR
download_boxes $DL_DIR $PROVIDER
# Vagrant up each box and attempt to reload one time if it fails
for VAGRANT_HOST in "${LAB_HOSTS[@]}"
do
RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR)
if [ "$RET" -eq 0 ]; then
(>&2 echo "Good news! $VAGRANT_HOST was built successfully!")
fi
# Attempt to recover if the intial "vagrant up" fails
if [ "$RET" -ne 0 ]; then
(>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.")
(>&2 echo "Attempting to reload and reprovision the host...")
RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR)
if [ "$RETRY_STATUS" -ne 0 ]; then
(>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
exit 1
fi
fi
done
post_build_checks
}
main $@
exit 0

6
ci/README.md Normal file
View File

@@ -0,0 +1,6 @@
# ci
The files in this directory are used to bootstrap an Ubuntu 16.04 baremetal server
for continuous integration testing by installing the prerequisites needed for
Detection Lab. After the prerequisites are installed, the build script is called
and the build will begin in a tmux session.

60
ci/automated_install.sh Normal file
View File

@@ -0,0 +1,60 @@
#! /bin/bash
# This script is run on the Packet.net baremetal server for CI tests.
# This script will build the entire lab from scratch and takes 3-4 hours
# on a Packet.net host
# While building, the server will start a webserver on Port 80 that contains
# the text "building". Once the test is completed, the text will be replaced
# with "success" or "failed".
# Install Virtualbox 5.1
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
apt-get update
apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2
echo "building" > /var/www/html/index.html
# Set up firewall
ufw allow ssh
ufw allow http
ufw default allow outgoing
ufw --force enable
# Install Vagrant
mkdir /opt/vagrant
cd /opt/vagrant
wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb
dpkg -i vagrant_2.0.1_x86_64.deb
vagrant plugin install vagrant-reload
# Install Packer
mkdir /opt/packer
cd /opt/packer
wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip
unzip packer_1.1.3_linux_amd64.zip
cp packer /usr/local/bin/packer
# Clone DetectionLab
cd /opt
git clone https://github.com/clong/DetectionLab.git
cd /opt/DetectionLab/Packer
# Make the packer images headless
for file in $(ls *.json); do
sed -i 's/"headless": false,/"headless": true,/g' "$file";
done
# Make the Vagrant instances headless
cd /opt/DetectionLab/Vagrant
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
# Download the build script from Github
wget https://gist.githubusercontent.com/clong/4d0865cf022f0fbc63caf900d91852fa/raw/479d6dbb18fdfc2e5d610613683752da3d4140fa/build.sh -O /opt/DetectionLab/build.sh ## Change this once we move to the actual DL repo.
chmod +x /opt/DetectionLab/build.sh
cd /opt/DetectionLab
# Start the build in a tmux session
sn=tmuxsession
tmux new-session -s "$sn" -d
tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter

View File

@@ -0,0 +1,47 @@
#! /bin/bash
# This script is run on the Packet.net baremetal server for CI tests.
# This script will bootstrap the build by downloading pre-build Packer boxes
# and should take no more than 90 minutes on a Packet.net host.
# While building, the server will start a webserver on Port 80 that contains
# the text "building". Once the test is completed, the text will be replaced
# with "success" or "failed".
# Install Virtualbox 5.1
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
apt-get update
apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2
echo "building" > /var/www/html/index.html
# Set up firewall
ufw allow ssh
ufw allow http
ufw default allow outgoing
ufw --force enable
# Install Vagrant
mkdir /opt/vagrant
cd /opt/vagrant
wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb
dpkg -i vagrant_2.0.1_x86_64.deb
vagrant plugin install vagrant-reload
# Clone DetectionLab
cd /opt
git clone https://github.com/clong/DetectionLab.git
# Make the Vagrant instances headless
cd /opt/DetectionLab/Vagrant
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
# Download the build script from Github
wget https://gist.githubusercontent.com/clong/5ee211b8533f7d33eaa31d6b83231d5e/raw/08fc4a3e8cc806c5f6278226607e4a36aa7e03fc/build_vagrant_only.sh -O /opt/DetectionLab/build_vagrant_only.sh # Change to repo path once we go to prod
chmod +x /opt/DetectionLab/build_vagrant_only.sh
cd /opt/DetectionLab
# Start the build in a tmux session
sn=tmuxsession
tmux new-session -s "$sn" -d
tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter