Merge branch 'windows-build' of github.com:jaredhaight/DetectionLab into windows-build

fixing problems identified by @clong
This commit is contained in:
Jared Haight
2018-03-31 13:19:43 -07:00
23 changed files with 863 additions and 638 deletions

View File

@@ -9,73 +9,67 @@ jobs:
- checkout
- run:
name: Delete stale Packet servers
name: Choose which test suite to run based on which files were modified
command: |
export DELETE_DEVICE_ID=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
- run:
name: Provisioning a baremetal Packet.net server
command: |
curl -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' > /tmp/device
- run: cat /tmp/device | jq ."id" | tr -d '"' > /tmp/device_id
- run: sleep 300
- run: echo "Sleeping 5 more minutes while Packet server is provisioned"
- run: sleep 300
- run:
name: Recording the IP address of the Packet server
command: |
export DEVICE_ID=$(cat /tmp/device_id);
curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" > /tmp/ip_info;
- run: cat /tmp/ip_info | jq ."ip_addresses[0].address" | tr -d '"' > /tmp/ip_address
- run: cd ~/repo
- run:
name: Copying DetectionLab repo to Packet server
command: |
export IP_ADDRESS=$(cat /tmp/ip_address);
scp -i ~/.ssh/id_rsa -r ~/repo root@"$IP_ADDRESS":/opt/DetectionLab
- run:
name: Copying install script to Packet server
command: |
export IP_ADDRESS=$(cat /tmp/ip_address);
scp -i ~/.ssh/id_rsa ./ci/automated_install_vagrant_only.sh root@"$IP_ADDRESS":/root
- run:
name: Running install script on Packet server
command: |
export IP_ADDRESS=$(cat /tmp/ip_address)
ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'chmod +x /root/automated_install_vagrant_only.sh; /bin/bash -c /root/automated_install_vagrant_only.sh'
- run:
name: Waiting for Packet server to post build results
shell: /bin/bash
command: |
export IP_ADDRESS=$(cat /tmp/ip_address)
MINUTES_PAST=0
while [ "$MINUTES_PAST" -lt 120 ]
do
export STATUS=$(curl $IP_ADDRESS)
if [ "$STATUS" == "building" ]; then
echo "$STATUS"
sleep 300
((MINUTES_PAST+=5))
## As far as I'm aware, there are three possibile cases to check for here. The CIRCLE_COMPARE_URL environment variable will either contain:
## - A SHA1 hash for the commit (If there is only a single commit on the branch)
## - Two SHA1 hashes in a "xxxxxxxx..yyyyyyyy" (comparing the two most recent commits
## - Nothing
## There is certainly a better way to accomplish all of this, but my limited knowledge of git has lead me to write this awful spaghetti code :-/
## We'll handle all 3 of these cases below:
## Checking commits for changes to Packer files
COMMIT_SHA1=""
PACKER_MODIFIED=0
VAGRANT_MODIFIED=0
## Check for empty CIRCLE_COMPARE_URL. If it is, set the SHA1 hash to the CIRCLE_SHA1 environment variable
if [ "$(echo -n $CIRCLE_COMPARE_URL | wc -c)" -eq 0 ]; then
export COMMIT_SHA1=$CIRCLE_SHA1
## Check for two short-hashes in the CIRCLE_COMPARE_URL by searching for '..'. If it exists, use the second short-hash
elif echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then
export COMMIT_SHA1="$(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5)"
## Check for a single short hash in the CIRCLE_COMPARE_URL and use it if it exists
## TODO: This check may not be needed.
else
echo "$STATUS" > /tmp/status
break
export COMMIT_SHA1="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)"
fi
done
if [ "$MINUTES_PAST" -gt 120 ]; then
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
## Display the files that were modified in this branch
echo "Files modified since origin/Master:"
git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1"
## Check to see if Packer files were modified
if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" | grep -c ^Packer/)" -gt 0 ]; then
export PACKER_MODIFIED=1
fi
- run:
name: Recording build results
shell: /bin/bash
command: |
export DEVICE_ID=$(cat /tmp/device_id)
export STATUS=$(cat /tmp/status)
echo $STATUS
if [ "$STATUS" != "success" ]; then
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" | grep -c ^Vagrant/)" -gt 0 ]; then
export VAGRANT_MODIFIED=1
fi
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 0
echo "Displaying the values of the modifier environment variables:"
echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED"
echo "PACKER_MODIFIED=$PACKER_MODIFIED"
## Choosing which test suite to run based on the files that were changed
if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then
echo "Running the test suite for Packer and Vagrant changes"
chmod +x ci/circle_workflows/packer_and_vagrant_changes.sh
ci/circle_workflows/packer_and_vagrant_changes.sh
exit 0
fi
if [[ "$PACKER_MODIFIED" -eq 0 ]] && [[ "$VAGRANT_MODIFIED" -eq 0 ]]; then
echo "Running the default test suite (Vagrant-only)"
chmod +x ci/circle_workflows/vagrant_changes.sh
ci/circle_workflows/vagrant_changes.sh
exit 0
fi
if [ "$PACKER_MODIFIED" -eq 1 ]; then
echo "Running the test suite for Packer-only changes"
chmod +x ci/circle_workflows/packer_changes.sh
ci/circle_workflows/packer_changes.sh
exit 0
fi
if [ "$VAGRANT_MODIFIED" -eq 1 ]; then
echo "Running the test suite for Vagrant-only changes"
chmod +x ci/circle_workflows/vagrant_changes.sh
ci/circle_workflows/vagrant_changes.sh
exit 0
fi
- store_artifacts:
path: /tmp/artifacts

2
.gitignore vendored
View File

@@ -1,4 +1,6 @@
Vagrant/.vagrant/*
Vagrant/vagrant_build.log
Packer/packer_cache/*
Packer/packer_build.log
Boxes/*
.DS_Store

View File

@@ -3,13 +3,15 @@
if "%PROCESSOR_ARCHITECTURE%"=="AMD64" goto 64BIT
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue > NUL
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems > NUL
exit /b
exit 0
:64BIT
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue
%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe update /force /queue
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems
%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe executequeueditems
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue > NUL
%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe update /force /queue > NUL
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems > NUL
%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe executequeueditems > NUL
exit 0

View File

@@ -2,6 +2,8 @@ if ($env:PACKER_BUILDER_TYPE -And $($env:PACKER_BUILDER_TYPE).startsWith("hyperv
Write-Host Skip debloat steps in Hyper-V build.
} else {
Write-Host Downloading debloat zip
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$url="https://github.com/StefanScherer/Debloat-Windows-10/archive/master.zip"
(New-Object System.Net.WebClient).DownloadFile($url, "$env:TEMP\debloat.zip")
Expand-Archive -Path $env:TEMP\debloat.zip -DestinationPath $env:TEMP -Force

View File

@@ -1,7 +1,20 @@
Try {
Write-Output "Set power plan to high performance"
$HighPerf = powercfg -l | %{if($_.contains("High performance")) {$_.split()[3]}}
# $HighPerf cannot be $null, we try activate this power profile with powercfg
#
if ($HighPerf -eq $null)
{
throw "Error: HighPerf is null"
}
$CurrPlan = $(powercfg -getactivescheme).split()[3]
if ($CurrPlan -ne $HighPerf) {powercfg -setactive $HighPerf}
} Catch {
Write-Warning -Message "Unable to set power plan to high performance"
Write-Warning $Error[0]
}

View File

@@ -34,7 +34,7 @@
</settings>
<settings pass="specialize">
<component name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<ComputerName>vagrant-2012</ComputerName>
<ComputerName>vagrant-2016</ComputerName>
<CopyProfile>false</CopyProfile>
</component>
</settings>

View File

@@ -15,7 +15,7 @@ if exist "C:\Users\vagrant\windows.iso" (
)
if not exist "C:\Windows\Temp\windows.iso" (
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://softwareupdate.vmware.com/cds/vmw-desktop/ws/12.5.7/5813279/windows/packages/tools-windows.tar', 'C:\Windows\Temp\vmware-tools.tar')" <NUL
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://softwareupdate.vmware.com/cds/vmw-desktop/ws/14.1.1/7528167/windows/packages/tools-windows.tar', 'C:\Windows\Temp\vmware-tools.tar')" <NUL
cmd /c ""C:\Program Files\7-Zip\7z.exe" x C:\Windows\Temp\vmware-tools.tar -oC:\Windows\Temp"
FOR /r "C:\Windows\Temp" %%a in (VMware-tools-windows-*.iso) DO REN "%%~a" "windows.iso"
rd /S /Q "C:\Program Files (x86)\VMWare"
@@ -31,16 +31,12 @@ goto :done
:virtualbox
:: There needs to be Oracle CA (Certificate Authority) certificates installed in order
:: to prevent user intervention popups which will undermine a silent installation.
cmd /c certutil -addstore -f "TrustedPublisher" A:\oracle-cert.cer
if exist "C:\Users\vagrant\VBoxGuestAdditions.iso" (
move /Y C:\Users\vagrant\VBoxGuestAdditions.iso C:\Windows\Temp
)
if not exist "C:\Windows\Temp\VBoxGuestAdditions.iso" (
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://download.virtualbox.org/virtualbox/5.1.28/VBoxGuestAdditions_5.1.28.iso', 'C:\Windows\Temp\VBoxGuestAdditions.iso')" <NUL
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://download.virtualbox.org/virtualbox/5.2.4/VBoxGuestAdditions_5.2.4.iso', 'C:\Windows\Temp\VBoxGuestAdditions.iso')" <NUL
)
cmd /c ""C:\Program Files\7-Zip\7z.exe" x C:\Windows\Temp\VBoxGuestAdditions.iso -oC:\Windows\Temp\virtualbox"

View File

@@ -81,7 +81,7 @@
"winrm_password": "vagrant",
"winrm_timeout": "4h",
"shutdown_command": "a:/sysprep.bat",
"guest_os_type": "Windows2012_64",
"guest_os_type": "Windows2016_64",
"guest_additions_mode": "disable",
"disk_size": 61440,
"floppy_files": [

View File

@@ -37,6 +37,7 @@ OSX 10.12.5 | 1.9.3 | 1.0.0 | VMWare Fusion (8.5.8)
OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.9)
OSX 10.12.6 | 2.0.1 | 1.1.3 | VMWare Fusion (8.5.10)
Ubuntu 16.04 | 2.0.1 | 1.1.3 | Virtualbox (5.1)
Ubuntu 16.04 | 2.0.2 | N/A | Virtualbox (5.2)
**Known Bad Versions:**
* Packer 1.1.2 will fail to build VMWare-ISOs correctly due to [this issue](https://github.com/hashicorp/packer/issues/5622).
@@ -48,15 +49,15 @@ DetectionLab now contains build scripts for \*NIX and MacOS users!
There are two build scripts:
- `build.sh` - Builds the entire lab from scratch. Takes 3-5 hours depending on hardware resources and bandwidth
- `build_vagrant_only.sh` - Downloads pre-built Packer boxes from S3 and builds the lab from those boxes. This option is recommended if you have more bandwidth than time.
- `build_vagrant_only.sh` - Downloads pre-built Packer boxes from https://detectionlab.network and builds the lab from those boxes. This option is recommended if you have more bandwidth than time or are having trouble building boxes.
---
## Building from Scratch
## Building DetectionLab from Scratch
1. Determine which Vagrant provider you want to use.
* Note: Virtualbox is free, the [VMWare vagrant plugin](https://www.vagrantup.com/vmware/#buy-now) is $80.
**NOTE:** If you have more bandwidth than time, you can skip the building of the Packer boxes and download the boxes directly from S3 and put them into the `Boxes` directory:
**NOTE:** If you'd like to save time, you can skip the building of the Packer boxes and download the boxes directly from https://detectionlab.network and put them into the `Boxes` directory:
Provider | Box | URL | MD5 | Size
------------|-----|-----|----|----

View File

@@ -41,8 +41,8 @@ else
# Get Splunk.com into the DNS cache. Sometimes resolution randomly fails during wget below
dig @8.8.8.8 splunk.com
# Download Splunk
wget --progress=bar:force -O splunk-7.0.1-2b5b15c4ee89-linux-2.6-amd64.deb 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=7.0.1&product=splunk&filename=splunk-7.0.1-2b5b15c4ee89-linux-2.6-amd64.deb&wget=true'
dpkg -i splunk-7.0.1-2b5b15c4ee89-linux-2.6-amd64.deb
wget --progress=bar:force -O splunk-7.0.2-03bbabbd5c0f-linux-2.6-amd64.deb 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=7.0.2&product=splunk&filename=splunk-7.0.2-03bbabbd5c0f-linux-2.6-amd64.deb&wget=true'
dpkg -i splunk-7.0.2-03bbabbd5c0f-linux-2.6-amd64.deb
/opt/splunk/bin/splunk start --accept-license
/opt/splunk/bin/splunk add index wineventlog -auth 'admin:changeme'
/opt/splunk/bin/splunk add index osquery -auth 'admin:changeme'
@@ -90,7 +90,7 @@ go build
cd /home/vagrant
# Modify the config to work with config importer
cat /home/vagrant/osquery-configuration/Endpoints/Windows/osquery.conf | sed 's#packs/#../packs/#g' | grep -v unwanted-chrome-extensions | grep -v security-tooling-checks | grep -v performance-metrics > /home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf
cat /home/vagrant/osquery-configuration/Endpoints/Windows/osquery.conf | sed 's#packs/#../packs/#g' | grep -v unwanted-chrome-extensions | grep -v security-tooling-checks | grep -v performance-metrics | grep -v logger_snapshot_event_type > /home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf
# Install configimporter
echo "Installing configimporter"
echo "Sleeping for 5"

View File

@@ -3,5 +3,7 @@ Write-Host "Downloading and unzipping the Palantir osquery Repo from Github..."
$osqueryRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\osquery-Master.zip'
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Invoke-WebRequest -Uri "https://github.com/palantir/osquery-configuration/archive/master.zip" -OutFile $osqueryRepoPath
Expand-Archive -path "$osqueryRepoPath" -destinationpath 'c:\Users\vagrant\AppData\Local\Temp' -Force

View File

@@ -1,7 +1,10 @@
# Purpose: Downloads and unzips a copy of the Palantir WEF Github Repo. This includes WEF subscriptions and custom WEF channels.
Write-Host "Downloading and unzipping the Palantir Windows Event Forwarding Repo from Github..."
$wefRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\wef-Master.zip'
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Invoke-WebRequest -Uri "https://github.com/palantir/windows-event-forwarding/archive/master.zip" -OutFile $wefRepoPath
Expand-Archive -path "$wefRepoPath" -destinationpath 'c:\Users\vagrant\AppData\Local\Temp' -Force

View File

@@ -14,6 +14,8 @@ If (-not (Test-Path "C:\Program Files\cagent\cagent.exe")) {
Write-Host "Downloading Caldera Agent (cagent.exe)"
$cagentPath = "C:\Program Files\cagent\cagent.exe"
$cagentConfPath = "C:\Program Files\cagent\conf.yml"
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
(New-Object System.Net.WebClient).DownloadFile('https://github.com/mitre/caldera-agent/releases/download/v0.1.0/cagent.exe', $cagentPath)
# Ignore SSL warning for conf file download
# https://stackoverflow.com/questions/34331206/ignore-ssl-warning-with-powershell-downloadstring

View File

@@ -27,6 +27,8 @@ If ($env:computername -eq "WIN10") {
# Purpose: Downloads and unzips a copy of the latest Mimikatz trunk
Write-Host "Determining latest release of Mimikatz..."
# GitHub requires TLS 1.2 as of 2/27
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$tag = (Invoke-WebRequest "https://api.github.com/repos/gentilkiwi/mimikatz/releases" -UseBasicParsing | ConvertFrom-Json)[0].tag_name
$mimikatzDownloadUrl = "https://github.com/gentilkiwi/mimikatz/releases/download/$tag/mimikatz_trunk.zip"
$mimikatzRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\mimikatz_trunk.zip'

460
build.sh
View File

@@ -7,39 +7,37 @@
# If you encounter issues, feel free to open an issue at
# https://github.com/clong/DetectionLab/issues
set -e
print_usage() {
echo "Usage: ./build.sh <virtualbox|vmware_fusion>"
exit 0
}
check_packer_and_vagrant() {
# Check for existence of Vagrant in PATH
which vagrant > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Vagrant was not found in your PATH.")
(>&2 echo "Please correct this before continuing. Quitting.")
exit 1
fi
# Ensure Vagrant >= 2.0.0
if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
(>&2 echo "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
fi
# Check for existence of Packer in PATH
which packer > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Packer was not found in your PATH.")
(>&2 echo "Please correct this before continuing. Quitting.")
(>&2 echo "Hint: sudo cp ./packer /usr/local/bin/packer; sudo chmod +x /usr/local/bin/packer")
check_packer_path() {
# Check for existence of Packer in PATH
if ! which packer >/dev/null; then
(echo >&2 "Packer was not found in your PATH.")
(echo >&2 "Please correct this before continuing. Quitting.")
(echo >&2 "Hint: sudo cp ./packer /usr/local/bin/packer; sudo chmod +x /usr/local/bin/packer")
exit 1
fi
}
check_vagrant_path() {
# Check for existence of Vagrant in PATH
if ! which vagrant >/dev/null; then
(echo >&2 "Vagrant was not found in your PATH.")
(echo >&2 "Please correct this before continuing. Quitting.")
exit 1
fi
# Ensure Vagrant >= 2.0.0
if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
(echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
fi
}
# Returns 0 if not installed or 1 if installed
check_virtualbox_installed() {
which VBoxManage > /dev/null
if [ "$?" -eq 0 ]; then
if which VBoxManage >/dev/null; then
echo "1"
else
echo "0"
@@ -48,19 +46,23 @@ check_virtualbox_installed() {
# Returns 0 if not installed or 1 if installed
check_vmware_fusion_installed() {
echo "$(ls /Applications | grep -ci 'VMware Fusion.app')"
if [ -e "/Applications/VMware Fusion.app" ]; then
echo "1"
else
echo "0"
fi
}
# Returns 0 if not installed or 1 if installed
check_vmware_vagrant_plugin_installed() {
VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')
if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then
(>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.")
(>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
(>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
VAGRANT_VMWARE_PLUGIN_PRESENT="$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')"
if [ "$VAGRANT_VMWARE_PLUGIN_PRESENT" -eq 0 ]; then
(echo >&2 "VMWare Fusion is installed, but the Vagrant plugin is not.")
(echo >&2 "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
(echo >&2 "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
echo "0"
else
echo $VAGRANT_VMWARE_PLUGIN_PRESENT
echo "$VAGRANT_VMWARE_PLUGIN_PRESENT"
fi
}
@@ -69,7 +71,7 @@ list_providers() {
VBOX_PRESENT=0
VMWARE_FUSION_PRESENT=0
if [ $(uname) == "Darwin" ]; then
if [ "$(uname)" == "Darwin" ]; then
# Detect Providers on OSX
VBOX_PRESENT=$(check_virtualbox_installed)
VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed)
@@ -79,98 +81,136 @@ list_providers() {
VBOX_PRESENT=$(check_virtualbox_installed)
fi
(>&2 echo "Available Providers:")
(echo >&2 "Available Providers:")
if [ "$VBOX_PRESENT" == "1" ]; then
(>&2 echo "virtualbox";)
(echo >&2 "virtualbox")
fi
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]
then
(>&2 echo "vmware_fusion";)
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]; then
(echo >&2 "vmware_fusion")
fi
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]
then
(>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]; then
(echo >&2 "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
exit 1
fi
(>&2 echo -e "\nWhich provider would you like to use?")
read PROVIDER
(echo >&2 -e "\\nWhich provider would you like to use?")
read -r PROVIDER
# Sanity check
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]
then
(>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]; then
(echo >&2 "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
exit 1
fi
echo $PROVIDER
echo "$PROVIDER"
}
# A series of checks to identify potential issues before starting the build
preflight_checks() {
DL_DIR="$1"
# Check to see if curl is in PATH
which curl > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Please install curl and make sure it is in your PATH.")
exit 1
fi
# Check to see if boxes exist already
BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l)
if [ $BOXES_BUILT -gt 0 ]; then
(>&2 echo "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.")
exit 1
fi
# Check to see if any Vagrant instances exist already
cd "$DL_DIR"/Vagrant/
VAGRANT_BUILT=$(vagrant status | grep -c 'not created')
if [ $VAGRANT_BUILT -ne 4 ]; then
(>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
exit 1
fi
# Check available disk space. Recommend 80GB free, warn if less.
FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
if [ $FREE_DISK_SPACE -lt 80000 ]; then
(>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n")
(>&2 df -m $HOME)
(>&2 echo "")
fi
# Check Packer version against known bad
if [ "$(packer --version)" == '1.1.2' ]; then
(>&2 echo "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.")
exit 1
fi
# Ensure the vagrant-reload plugin is installed
VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
(>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
$(which vagrant) plugin install "vagrant-reload"
if [ "$?" -ne 0 ]; then
(>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
# Check to see if boxes exist in the "Boxes" directory already
check_boxes_built() {
BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l)
if [ "$BOXES_BUILT" -gt 0 ]; then
if [ "$VAGRANT_ONLY" -eq 1 ]; then
(echo >&2 "WARNING: You seem to have at least one .box file present in $DL_DIR/Boxes already. If you would like fresh boxes downloaded, please remove all files from the Boxes directory and re-run this script.")
DOWNLOAD_BOXES=0
else
(echo >&2 "You seem to have at least one .box file in $DL_DIR/Boxes. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.")
exit 1
fi
fi
}
# Check to ensure either "md5" or "md5sum" is installed for verifying integrity of downloaded boxes
check_md5_tool_exists() {
if ! which md5 > /dev/null && ! which md5sum > /dev/null; then
(echo >&2 "md5 or md5sum not found in PATH. Please install at least one of these utilities to continue.")
exit 1
fi
}
# Check to see if any Vagrant instances exist already
check_vagrant_instances_exist() {
cd "$DL_DIR"/Vagrant/ || exit 1
# Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true"
VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true
if [ "$VAGRANT_BUILT" -ne 4 ]; then
(echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
exit 1
fi
}
check_vagrant_reload_plugin() {
# Ensure the vagrant-reload plugin is installed
VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
(echo >&2 "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
if ! $(which vagrant) plugin install "vagrant-reload"; then
(echo >&2 "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
exit 1
fi
fi
}
# Check available disk space. Recommend 80GB free, warn if less.
check_disk_free_space() {
FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
if [ "$FREE_DISK_SPACE" -lt 80000 ]; then
(echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n")
(df >&2 -m "$HOME")
(echo >&2 "")
fi
}
# Check to see if curl is in PATH - needed for post-install checks
check_curl(){
if ! which curl >/dev/null; then
(echo >&2 "Please install curl and make sure it is in your PATH.")
exit 1
fi
}
# Check Packer version against known "bad" versions
check_packer_known_bad() {
if [ "$(packer --version)" == '1.1.2' ]; then
(echo >&2 "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.")
exit 1
fi
}
# A series of checks to identify potential issues before starting the build
preflight_checks() {
# If it's not a Vagrant-only build, then run Packer-related checks
if [ "$VAGRANT_ONLY" -eq 0 ]; then
check_packer_path
check_packer_known_bad
else
# If it is a Vagrant-only build, set appropriate checks
DOWNLOAD_BOXES=1
check_md5_tool_exists
fi
# If it's not a Packer-only build, then run Vagrant-related checks
if [ "$PACKER_ONLY" -eq 0 ]; then
check_vagrant_path
check_vagrant_instances_exist
check_vagrant_reload_plugin
fi
check_boxes_built
check_disk_free_space
check_curl
}
# Builds a box using Packer
packer_build_box() {
PROVIDER="$1"
BOX="$2"
DL_DIR="$3"
BOX="$1"
if [ "$PROVIDER" == "vmware_fusion" ]; then
PROVIDER="vmware"
fi
cd "$DL_DIR/Packer"
(>&2 echo "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.")
$(which packer) build --only="$PROVIDER-iso" $BOX.json
if [ "$?" -ne 0 ]; then
(>&2 echo "Something went wrong while attempting to build the $BOX box.")
(>&2 echo "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
fi
cd "$DL_DIR/Packer" || exit 1
(echo >&2 "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.")
PACKER_LOG=1 PACKER_LOG_PATH="$DL_DIR/Packer/packer_build.log" $(which packer) build --only="$PROVIDER-iso" "$BOX".json >&2
echo "$?"
}
# Moves the boxes from the Packer directory to the Boxes directory
move_boxes() {
PROVIDER="$1"
DL_DIR="$2"
# Hacky workaround for VMware
if [ "$PROVIDER" == "vmware_fusion" ]; then
PROVIDER="vmware"
@@ -178,34 +218,31 @@ move_boxes() {
mv "$DL_DIR"/Packer/*.box "$DL_DIR"/Boxes
# Ensure Windows 10 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
(>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.")
(echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Ensure Windows 2016 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
(>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.")
(echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.")
exit 1
fi
}
# Brings up a single host using Vagrant
vagrant_up_host() {
PROVIDER="$1"
HOST="$2"
DL_DIR="$3"
(>&2 echo "Attempting to bring up the $HOST host using Vagrant")
cd "$DL_DIR"/Vagrant
$(which vagrant) up $HOST --provider="$PROVIDER" 1>&2
HOST="$1"
(echo >&2 "Attempting to bring up the $HOST host using Vagrant")
cd "$DL_DIR"/Vagrant || exit 1
VAGRANT_LOG=info $(which vagrant) up "$HOST" --provider="$PROVIDER" 3>&1 1>&2 2>&3 | tee -a "$DL_DIR/Vagrant/vagrant_build.log"
echo "$?"
}
# Attempts to reload and re-provision a host if the intial "vagrant up" fails
vagrant_reload_host() {
HOST="$1"
DL_DIR="$2"
cd "$DL_DIR"/Vagrant
cd "$DL_DIR"/Vagrant || exit 1
# Attempt to reload the host if the vagrant up command didn't exit cleanly
$(which vagrant) reload $HOST --provision 1>&2
$(which vagrant) reload "$HOST" --provision 1>&2
echo "$?"
}
@@ -221,89 +258,218 @@ post_build_checks() {
# Associative arrays are only supported in bash 4 and up
if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then
declare -A SERVICES
SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
for SERVICE in "${!SERVICES[@]}"
do
SERVICES=(["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
for SERVICE in "${!SERVICES[@]}"; do
if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then
(>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
fi
done
else
if [ "$CALDERA_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: Caldera failed post-build tests and may not be functioning correctly.")
fi
if [ "$SPLUNK_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: Splunk failed post-build tests and may not be functioning correctly.")
fi
if [ "$FLEET_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.")
(echo >&2 "Warning: Fleet failed post-build tests and may not be functioning correctly.")
fi
fi
}
main() {
# Get location of build.sh
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROVIDER=""
LAB_HOSTS=("logger" "dc" "wef" "win10")
parse_cli_arguments() {
# If no argument was supplied, list available providers
if [ $# -eq 0 ]; then
if [ "$#" -eq 0 ]; then
PROVIDER=$(list_providers)
fi
# If more than one argument was supplied, print usage message
if [ $# -gt 1 ]; then
# If more than two arguments were supplied, print usage message
if [ "$#" -gt 2 ]; then
print_usage
exit 1
fi
if [ $# -eq 1 ]; then
if [ "$#" -ge 1 ]; then
# If the user specifies the provider as an agument, set the variable
# TODO: Check to make sure they actually have their provider installed
case "$1" in
virtualbox)
PROVIDER="$1"
;;
PROVIDER="$1"
;;
vmware_fusion)
PROVIDER="$1"
;;
PROVIDER="$1"
;;
*)
echo "\"$1\" is not a valid provider. Listing available providers:"
PROVIDER=$(list_providers)
echo "\"$1\" is not a valid provider. Listing available providers:"
PROVIDER=$(list_providers)
;;
esac
fi
if [ $# -eq 2 ]; then
case "$2" in
--packer-only)
PACKER_ONLY=1
;;
--vagrant-only)
VAGRANT_ONLY=1
;;
*)
echo -e "\"$2\" is not recognized as an option. Available options are:\\n--packer-only\\n--vagrant-only"
exit 1
;;
esac
fi
}
preflight_checks $DL_DIR
packer_build_box $PROVIDER "windows_2016" $DL_DIR
packer_build_box $PROVIDER "windows_10" $DL_DIR
move_boxes $PROVIDER $DL_DIR
build_packer_boxes() {
PACKER_BOXES=("windows_2016" "windows_10")
if [ "$(hostname)" == "packerwindows10" ]; then # Workaround for CI environment
(echo >&2 "CI Environment detected. If you are a user and are seeing this, please file an issue on GitHub.")
RET=$(packer_build_box "windows_10")
if [ "$RET" -eq 0 ]; then
(echo >&2 "Good news! The windows_10 box was built with Packer successfully!")
else
(echo >&2 "Something went wrong while attempting to build the windows_10 box.")
(echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
exit 1
fi
elif [ "$(hostname)" == "packerwindows2016" ]; then # Workaround for CI environment
(echo >&2 "CI Environment detected. If you are a user and are seeing this, please file an issue on GitHub.")
RET=$(packer_build_box "windows_2016")
if [ "$RET" -eq 0 ]; then
(echo >&2 "Good news! The windows_2016 box was built with Packer successfully!")
else
(echo >&2 "Something went wrong while attempting to build the windows_2016 box.")
(echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
exit 1
fi
else
for PACKER_BOX in "${PACKER_BOXES[@]}"; do # Normal user workflow
RET=$(packer_build_box "$PACKER_BOX")
if [ "$RET" -eq 0 ]; then
(echo >&2 "Good news! $PACKER_BOX was built successfully!")
else
(echo >&2 "Something went wrong while attempting to build the $PACKER_BOX box.")
(echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
exit 1
fi
done
fi
}
choose_md5_tool() {
if which md5; then
MD5TOOL="$(which md5)"
CUT_INDEX=4
else
MD5TOOL="$(which md5sum)"
CUT_INDEX=1
fi
}
# Downloads pre-built Packer boxes from detectionlab.network to save time during CI builds
download_boxes() {
if [ "$PROVIDER" == "virtualbox" ]; then
wget "https://www.detectionlab.network/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box
wget "https://www.detectionlab.network/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box
elif [ "$PROVIDER" == "vmware_fusion" ]; then
wget "https://www.detectionlab.network/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box
wget "https://www.detectionlab.network/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box
fi
# Hacky workaround
if [ "$PROVIDER" == "vmware_fusion" ]; then
PROVIDER="vmware"
fi
# Ensure Windows 10 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
(echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Ensure Windows 2016 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
(echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Verify hashes of VirtualBox boxes
if [ "$PROVIDER" == "virtualbox" ]; then
if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then
(echo >&2 "Hash mismatch on windows_10_virtualbox.box")
fi
if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "614f984c82b51471b5bb753940b59d38" ]; then
(echo >&2 "Hash mismatch on windows_2016_virtualbox.box")
fi
# Verify hashes of VMware boxes
elif [ "$PROVIDER" == "vmware" ]; then
if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then
(echo >&2 "Hash mismatch on windows_10_vmware.box")
exit 1
fi
if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then
(echo >&2 "Hash mismatch on windows_2016_vmware.box")
exit 1
fi
# Reset PROVIDER variable if using VMware
if [ "$PROVIDER" == "vmware" ]; then
PROVIDER="vmware_fusion"
fi
fi
}
build_vagrant_hosts() {
LAB_HOSTS=("logger" "dc" "wef" "win10")
# Change provider back to original selection if using vmware_fusion
if [ "$PROVIDER" == "vmware" ]; then
PROVIDER="vmware_fusion"
fi
# Vagrant up each box and attempt to reload one time if it fails
for VAGRANT_HOST in "${LAB_HOSTS[@]}"
do
RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR)
for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do
RET=$(vagrant_up_host "$VAGRANT_HOST")
if [ "$RET" -eq 0 ]; then
(>&2 echo "Good news! $VAGRANT_HOST was built successfully!")
(echo >&2 "Good news! $VAGRANT_HOST was built successfully!")
fi
# Attempt to recover if the intial "vagrant up" fails
if [ "$RET" -ne 0 ]; then
(>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.")
(>&2 echo "Attempting to reload and reprovision the host...")
RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR)
(echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.")
(echo >&2 "Attempting to reload and reprovision the host...")
RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST")
if [ "$RETRY_STATUS" -ne 0 ]; then
(>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
(echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
exit 1
fi
fi
done
post_build_checks
}
main $@
main() {
# Get location of build.sh
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PACKER_ONLY=0
VAGRANT_ONLY=0
parse_cli_arguments "$@"
preflight_checks
if [[ "$DOWNLOAD_BOXES" -eq 1 ]] && [[ "$VAGRANT_ONLY" -eq 1 ]]; then
download_boxes
fi
# Build Packer boxes if this isn't a Vagrant-only build
if [ "$VAGRANT_ONLY" -eq 0 ]; then
build_packer_boxes
# The only time we will need to move boxes is if we're doing a full build
if [ "$PACKER_ONLY" -eq 0 ]; then
move_boxes
fi
fi
# Build and Test Vagrant hosts if this isn't a Packer-only build
if [ "$PACKER_ONLY" -eq 0 ]; then
build_vagrant_hosts
post_build_checks
fi
}
main "$@"
exit 0

View File

@@ -1,308 +0,0 @@
#! /bin/bash
# This script skips the entire Packer box building process.
# This script downloads pre-built VirtualBox Packer images from S3 and puts
# them into the "Boxes" directory.
# Only MacOS and Linux are supported.
# If you encounter issues, feel free to open an issue at
# https://github.com/clong/DetectionLab/issues
set -e
print_usage() {
echo "Usage: ./build.sh <virtualbox|vmware_fusion>"
exit 0
}
check_vagrant() {
# Check for existence of Vagrant in PATH
which vagrant > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Vagrant was not found in your PATH.")
(>&2 echo "Please correct this before continuing. Quitting.")
exit 1
fi
# Ensure Vagrant >= 2.0.0
if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
(>&2 echo "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
fi
}
# Returns 0 if not installed or 1 if installed
check_virtualbox_installed() {
which VBoxManage > /dev/null
if [ "$?" -eq 0 ]; then
echo "1"
else
echo "0"
fi
}
# Returns 0 if not installed or 1 if installed
check_vmware_fusion_installed() {
echo "$(ls /Applications | grep -ci 'VMware Fusion.app')"
}
# Returns 0 if not installed or 1 if installed
check_vmware_vagrant_plugin_installed() {
VAGRANT_VMWARE_PLUGIN_PRESENT=$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')
if [ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 0 ]; then
(>&2 echo "VMWare Fusion is installed, but the Vagrant plugin is not.")
(>&2 echo "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
(>&2 echo "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
echo "0"
else
echo $VAGRANT_VMWARE_PLUGIN_PRESENT
fi
}
# List the available Vagrant providers present on the system
list_providers() {
VBOX_PRESENT=0
VMWARE_FUSION_PRESENT=0
if [ $(uname) == "Darwin" ]; then
# Detect Providers on OSX
VBOX_PRESENT=$(check_virtualbox_installed)
VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed)
VAGRANT_VMWARE_PLUGIN_PRESENT=$(check_vmware_vagrant_plugin_installed)
else
# Assume the only other available provider is VirtualBox
VBOX_PRESENT=$(check_virtualbox_installed)
fi
(>&2 echo "Available Providers:")
if [ "$VBOX_PRESENT" == "1" ]; then
(>&2 echo "virtualbox";)
fi
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]
then
(>&2 echo "vmware_fusion";)
fi
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]
then
(>&2 echo "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
exit 1
fi
(>&2 echo -e "\nWhich provider would you like to use?")
read PROVIDER
# Sanity check
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]
then
(>&2 echo "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
exit 1
fi
echo $PROVIDER
}
# A series of checks to identify potential issues before starting the build
preflight_checks() {
DL_DIR="$1"
# Check to see if curl is in PATH
which curl > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Please install curl and make sure it is in your PATH.")
exit 1
fi
# Check to see if wget is in PATH
which wget > /dev/null
if [ "$?" -ne 0 ]; then
(>&2 echo "Please install curl and make sure it is in your PATH.")
exit 1
fi
# Check to see if boxes exist already
BOXES_BUILT=$(ls -al "$DL_DIR"/Boxes/*.box 2> /dev/null | wc -l)
if [ $BOXES_BUILT -gt 0 ]; then
(>&2 echo "WARNING: You seem to have boxes present in the Boxes/ directory already. If you would like fresh boxes downloaded, please remove all files from the Boxes/ directory and re-run this script.")
fi
# Check to see if any Vagrant instances exist already
cd "$DL_DIR"/Vagrant/
VAGRANT_BUILT=$(vagrant status | grep -c 'not created')
if [ $VAGRANT_BUILT -ne 4 ]; then
(>&2 echo "You appear to have already created at least one Vagrant instance. This script does not support already created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
exit 1
fi
# Check available disk space. Recommend 80GB free, warn if less.
FREE_DISK_SPACE=$(df -m $HOME | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
if [ $FREE_DISK_SPACE -lt 80000 ]; then
(>&2 echo -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\n")
(>&2 df -m $HOME)
(>&2 echo "")
fi
# Ensure the vagrant-reload plugin is installed
VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
(>&2 echo "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
$(which vagrant) plugin install "vagrant-reload"
if [ "$?" -ne 0 ]; then
(>&2 echo "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
exit 1
fi
fi
}
# Downloads pre-built Packer boxes from detectionlab.network to save time during CI builds
download_boxes() {
DL_DIR="$1"
PROVIDER="$2"
if [ "$PROVIDER" == "virtualbox" ]; then
wget "https://www.detectionlab.network/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box
wget "https://www.detectionlab.network/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box
elif [ "$PROVIDER" == "vmware_fusion" ]; then
wget "https://www.detectionlab.network/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box
wget "https://www.detectionlab.network/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box
fi
# Hacky workaround
if [ "$PROVIDER" == "vmware_fusion" ]; then
PROVIDER="vmware"
fi
# Ensure Windows 10 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
(>&2 echo "Windows 10 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Ensure Windows 2016 box exists
if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
(>&2 echo "Windows 2016 box is missing from the Boxes directory. Qutting.")
exit 1
fi
# Verify hashes of VirtualBox boxes
if [ "$PROVIDER" == "virtualbox" ]; then
if [ "$(md5sum windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then
(>&2 echo "Hash mismatch on windows_10_virtualbox.box")
fi
if [ "$(md5sum windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "614f984c82b51471b5bb753940b59d38" ]; then
(>&2 echo "Hash mismatch on windows_2016_virtualbox.box")
fi
# Verify hashes of VMware boxes
elif [ "$PROVIDER" == "vmware" ]; then
if [ "$(md5 windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then
(>&2 echo "Hash mismatch on windows_10_vmware.box")
exit 1
fi
if [ "$(md5 windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then
(>&2 echo "Hash mismatch on windows_2016_vmware.box")
exit 1
fi
# Reset PROVIDER variable
PROVIDER="vmware_fusion"
fi
}
# Brings up a single host using Vagrant
vagrant_up_host() {
PROVIDER="$1"
HOST="$2"
DL_DIR="$3"
(>&2 echo "Attempting to bring up the $HOST host using Vagrant")
cd "$DL_DIR"/Vagrant
$(which vagrant) up $HOST --provider="$PROVIDER" 1>&2
echo "$?"
}
# Attempts to reload and re-provision a host if the intial "vagrant up" fails
vagrant_reload_host() {
HOST="$1"
DL_DIR="$2"
cd "$DL_DIR"/Vagrant
# Attempt to reload the host if the vagrant up command didn't exit cleanly
$(which vagrant) reload $HOST --provision 1>&2
echo "$?"
}
# A series of checks to ensure important services are responsive after the build completes.
post_build_checks() {
# If the curl operation fails, we'll just leave the variable equal to 0
# This is needed to prevent the script from exiting if the curl operation fails
CALDERA_CHECK=$(curl -ks -m 2 https://192.168.38.5:8888 | grep -c '302: Found' || echo "")
SPLUNK_CHECK=$(curl -ks -m 2 https://192.168.38.5:8000/en-US/account/login?return_to=%2Fen-US%2F | grep -c 'This browser is not supported by Splunk' || echo "")
FLEET_CHECK=$(curl -ks -m 2 https://192.168.38.5:8412 | grep -c 'Kolide Fleet' || echo "")
BASH_MAJOR_VERSION=$(/bin/bash --version | grep 'GNU bash' | grep -o version\.\.. | cut -d ' ' -f 2 | cut -d '.' -f 1)
# Associative arrays are only supported in bash 4 and up
if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then
declare -A SERVICES
SERVICES=( ["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
for SERVICE in "${!SERVICES[@]}"
do
if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then
(>&2 echo "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
fi
done
else
if [ "$CALDERA_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Caldera failed post-build tests and may not be functioning correctly.")
fi
if [ "$SPLUNK_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Splunk failed post-build tests and may not be functioning correctly.")
fi
if [ "$FLEET_CHECK" -lt 1 ]; then
(>&2 echo "Warning: Fleet failed post-build tests and may not be functioning correctly.")
fi
fi
}
main() {
# Get location of build_vagrant_only.sh
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
DL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROVIDER=""
LAB_HOSTS=("logger" "dc" "wef" "win10")
# If no argument was supplied, list available providers
if [ $# -eq 0 ]; then
PROVIDER=$(list_providers)
fi
# If more than one argument was supplied, print usage message
if [ $# -gt 1 ]; then
print_usage
exit 1
fi
if [ $# -eq 1 ]; then
# If the user specifies the provider as an agument, set the variable
# TODO: Check to make sure they actually have their provider installed
case "$1" in
virtualbox)
PROVIDER="$1"
;;
vmware_fusion)
PROVIDER="$1"
;;
*)
echo "\"$1\" is not a valid provider. Listing available providers:"
PROVIDER=$(list_providers)
;;
esac
fi
check_vagrant
preflight_checks $DL_DIR
download_boxes $DL_DIR $PROVIDER
# Vagrant up each box and attempt to reload one time if it fails
for VAGRANT_HOST in "${LAB_HOSTS[@]}"
do
RET=$(vagrant_up_host $PROVIDER $VAGRANT_HOST $DL_DIR)
if [ "$RET" -eq 0 ]; then
(>&2 echo "Good news! $VAGRANT_HOST was built successfully!")
fi
# Attempt to recover if the intial "vagrant up" fails
if [ "$RET" -ne 0 ]; then
(>&2 echo "Something went wrong while attempting to build the $VAGRANT_HOST box.")
(>&2 echo "Attempting to reload and reprovision the host...")
RETRY_STATUS=$(vagrant_reload_host $VAGRANT_HOST $DL_DIR)
if [ "$RETRY_STATUS" -ne 0 ]; then
(>&2 echo "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
exit 1
fi
fi
done
post_build_checks
}
main $@
exit 0

View File

@@ -1,6 +1,104 @@
# ci
# Continuous Integration
The files in this directory are used to bootstrap an Ubuntu 16.04 baremetal server
for continuous integration testing by installing the prerequisites needed for
Detection Lab. After the prerequisites are installed, the build script is called
and the build will begin in a tmux session.
## Understanding the build process
Once a PR is created, the contents of that PR will be copied to a CircleCI worker to be tested.
The CircleCI worker will evaluate which files have been modified and set environment variables accordingly. There are 4 possible options and 3 different tests:
1. Code in both the Packer and Vagrant directories was modified
* In this case, the CircleCI worker will execute `ci/circle_workflows/packer_and_vagrant_changes.sh`
2. Code in neither the Packer and Vagrant directories was modified
* In this case, the CircleCI worker will execute the default test `ci/circle_worker/vagrant_changes.sh`
3. Code in only the Packer directory was modified
* In this case, the CircleCI worker will execute `ci/circle_worker/packer_changes.sh`
4. Code in only the Vagrant directory was modified
* In this case, the CircleCI worker will execute `ci/circle_worker/vagrant_changes.sh`
## Test Case Walkthroughs
### packer_and_vagrant_changes.sh
1. Spins up a single Packet server
2. Bootstraps the Packet server by calling `ci/build_machine_bootstrap.sh` with no arguments
3. Builds the Windows10 and Windows2016 images one at a time
4. Moves the resulting boxes to the Boxes directory
5. Brings each Vagrant host online one-by-one
6. CircleCI records the build results from the Packet server
### vagrant_changes.sh
1. Spins up a single Packet server
2. Bootstraps the Packet server by calling `ci/build_machine_bootstrap.sh` with the `--vagrant-only` argument
3. Downloads the pre-build Windows10 and Windows2016 boxes from https://detectionlab.network directly to the Boxes directory
4. Brings each Vagrant host online one-by-one
5. CircleCI records the build results from the Packet server
### packer_changes.sh
1. Spins up two separate Packet servers to allow the Packer boxes to be built in parallel
2. Bootstraps each packet Server by calling `ci/build_machine_bootstrap.sh` with the `--packer-only` argument
3. Starts the Packer build process on each server
4. CircleCI records the build result from each Packet server
```
+------------+
| |
| |
| |
| Github |
| |
| |
+------+-----+
|
|
| Pull Request
|
v
+------+-----+
| |
| |
| Circle |
+----------------------------->| Worker |
| | |
| | |
| | |
| +------+-----+
| |
| | Code changes are evaluated
| | to determine which test suite
| | to run
| |
| v
| +----------------+--------------+
Circle Worker | | packer_and_vagrant_changes.sh |
quries for | | vagrant_changes.sh |
build results | | packer_changes.sh |
| +----------------+--------------+
| |
| |
| | |
| |
| |
| |
| |
| | 1. Provision Packet server(s)
| | 2. Copy repo to server
| | 3. Run server bootstrap
| | 4. Bootstrap calls build.sh with
| | the appropriate arguments
| |
| |
| +---------v---------+
| | |
| | |
| | |
+-------------------------->| Packet Server |
| |
| |
| |
+-------------------+
```

View File

@@ -1,54 +0,0 @@
#! /bin/bash
# This script is run on the Packet.net baremetal server for CI tests.
# This script will build the entire lab from scratch and takes 3-4 hours
# on a Packet.net host
# While building, the server will start a webserver on Port 80 that contains
# the text "building". Once the test is completed, the text will be replaced
# with "success" or "failed".
# Install Virtualbox 5.1
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
apt-get update
apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2
echo "building" > /var/www/html/index.html
# Set up firewall
ufw allow ssh
ufw allow http
ufw default allow outgoing
ufw --force enable
# Install Vagrant
mkdir /opt/vagrant
cd /opt/vagrant
wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb
dpkg -i vagrant_2.0.1_x86_64.deb
vagrant plugin install vagrant-reload
# Install Packer
mkdir /opt/packer
cd /opt/packer
wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip
unzip packer_1.1.3_linux_amd64.zip
cp packer /usr/local/bin/packer
# Make the packer images headless
for file in $(ls *.json); do
sed -i 's/"headless": false,/"headless": true,/g' "$file";
done
# Make the Vagrant instances headless
cd /opt/DetectionLab/Vagrant
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
# Ensure the script is executable
chmod +x /opt/DetectionLab/build.sh
cd /opt/DetectionLab
# Start the build in a tmux session
sn=tmuxsession
tmux new-session -s "$sn" -d
tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter

View File

@@ -1,42 +0,0 @@
#! /bin/bash
# This script is run on the Packet.net baremetal server for CI tests.
# This script will bootstrap the build by downloading pre-build Packer boxes
# and should take no more than 90 minutes on a Packet.net host.
# While building, the server will start a webserver on Port 80 that contains
# the text "building". Once the test is completed, the text will be replaced
# with "success" or "failed".
# Install Virtualbox 5.1
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
apt-get update
apt-get install -y virtualbox-5.1 build-essential unzip git ufw apache2
echo "building" > /var/www/html/index.html
# Set up firewall
ufw allow ssh
ufw allow http
ufw default allow outgoing
ufw --force enable
# Install Vagrant
mkdir /opt/vagrant
cd /opt/vagrant
wget https://releases.hashicorp.com/vagrant/2.0.1/vagrant_2.0.1_x86_64.deb
dpkg -i vagrant_2.0.1_x86_64.deb
vagrant plugin install vagrant-reload
# Make the Vagrant instances headless
cd /opt/DetectionLab/Vagrant
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
# Ensure the script is executable
chmod +x /opt/DetectionLab/build_vagrant_only.sh
cd /opt/DetectionLab
# Start the build in a tmux session
sn=tmuxsession
tmux new-session -s "$sn" -d
tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter

93
ci/build_machine_bootstrap.sh Executable file
View File

@@ -0,0 +1,93 @@
#! /bin/bash
# This script is run on the Packet.net baremetal server for CI tests.
# This script will build the entire lab from scratch and takes 3-4 hours
# on a Packet.net host
# While building, the server will start a webserver on Port 80 that contains
# the text "building". Once the test is completed, the text will be replaced
# with "success" or "failed".
ARGS="$1"
PACKER_ONLY=0
VAGRANT_ONLY=0
if [ ! -z "$1" ]; then
case "$1" in
--packer-only)
PACKER_ONLY=1
;;
--vagrant-only)
VAGRANT_ONLY=1
;;
*)
echo "\"$ARGS\" is not a supported argument to this script. Quitting"
exit 1
;;
esac
fi
echo "Args: $ARGS"
if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then
echo "Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong."
exit 1
fi
# Install Virtualbox 5.2
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
apt-get update
apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2
echo "building" > /var/www/html/index.html
# Set up firewall
ufw allow ssh
ufw allow http
ufw default allow outgoing
ufw --force enable
if [ "$PACKER_ONLY" -eq 0 ]; then
# Install Vagrant
mkdir /opt/vagrant
cd /opt/vagrant || exit 1
wget https://releases.hashicorp.com/vagrant/2.0.3/vagrant_2.0.3_x86_64.deb
dpkg -i vagrant_2.0.3_x86_64.deb
vagrant plugin install vagrant-reload
# Make the Vagrant instances headless
cd /opt/DetectionLab/Vagrant || exit 1
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
fi
if [ "$VAGRANT_ONLY" -eq 0 ]; then
# Install Packer
mkdir /opt/packer
cd /opt/packer || exit 1
wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip
unzip packer_1.1.3_linux_amd64.zip
cp packer /usr/local/bin/packer
# Make the Packer images headless
cd /opt/DetectionLab/Packer || exit 1
for file in *.json; do
sed -i 's/"headless": false,/"headless": true,/g' "$file";
done
fi
# Ensure the script is executable
chmod +x /opt/DetectionLab/build.sh
cd /opt/DetectionLab || exit 1
# Start the build in a tmux session
sn=tmuxsession
tmux new-session -s "$sn" -d
if [ "$PACKER_ONLY" -eq 1 ]; then
tmux send-keys -t "$sn:0" './build.sh virtualbox --packer-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
fi
if [ "$VAGRANT_ONLY" -eq 1 ]; then
tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
fi
if [[ "$PACKER_ONLY" -eq 0 ]] && [[ "$VAGRANT_ONLY" -eq 0 ]]; then
tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
fi

View File

@@ -0,0 +1,80 @@
#! /bin/bash
set -e
# Create artifacts directory
if [ ! -d "/tmp/artifacts" ]; then
mkdir /tmp/artifacts
fi
## Delete stale servers if they exist
DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then
echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID"
curl -X DELETE -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
fi
## Provision a Type1 baremetal Packet.net server
echo "Provisioning a server on Packet.net"
DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
# Make sure the device ID is sane.
# TODO: maybe make this a regex
if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
exit 1
fi
echo "Server successfully provisioned with ID: $DEVICE_ID"
echo "Sleeping 10 minutes to wait for Packet server to be provisioned"
sleep 300
echo "Sleeping 5 more minutes (CircleCI Keepalive)"
sleep 300
## Recording the IP address of the newly provisioned Packet server
IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
# Copy repo to Packet server
# TODO: Tar up the repo and expand it remotely
cd ~/repo
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
## Running install script on Packet server
ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh
echo "Sleeping 5 minutes to allow the build process to start"
sleep 300
## Waiting for Packet server to post build results
MINUTES_PAST=0
while [ "$MINUTES_PAST" -lt 400 ]; do
STATUS=$(curl $IP_ADDRESS)
if [ "$STATUS" == "building" ]; then
echo "$STATUS"
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present"
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
sleep 300
((MINUTES_PAST += 5))
else
break
fi
done
if [ "$MINUTES_PAST" -gt 400 ]; then
echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present"
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
## Recording the build results
echo $STATUS
if [ "$STATUS" != "success" ]; then
echo "Build failed. Cleaning up server with ID $DEVICE_ID"
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present"
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
echo "Build was successful. Cleaning up server with ID $DEVICE_ID"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 0

View File

@@ -0,0 +1,99 @@
#! /bin/bash
set -e
# Create artifacts directory
if [ ! -d "/tmp/artifacts" ]; then
mkdir /tmp/artifacts
fi
## Delete stale servers if they exist
echo "Deleting stale Packet.net servers"
DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then
echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID"
curl -X DELETE -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
fi
## Provision two Type1 baremetal Packet.net servers
echo "Provisioning packerwindows2016 on Packet.net"
SERVER1_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows2016", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
if [ "$(echo -n $SERVER1_ID | wc -c)" -ne 36 ]; then
echo "Server may have failed provisionining. Device ID is set to: $SERVER1_ID"
exit 1
fi
echo "packerwindows2016 successfully provisioned with ID: $SERVER1_ID"
sleep 5 # Wait a bit before issuing another provision command
echo "Provisioning packerwindows10 on Packet.net"
SERVER2_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows10", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
if [ "$(echo -n $SERVER2_ID | wc -c)" -ne 36 ]; then
echo "Server may have failed provisionining. Device ID is set to: $SERVER2_ID"
exit 1
fi
echo "packerwindows10 successfully provisioned with ID: $SERVER2_ID"
echo "Sleeping 10 minutes to wait for Packet servers to finish provisiong"
sleep 300
echo "Sleeping 5 more minutes (CircleCI Keepalive)"
sleep 300
## Recording the IP address of the newly provisioned Packet servers
SERVER1_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER1_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
SERVER2_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER2_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
# Copy repo to Packet servers
# TODO: Tar up the repo and expand it remotely
cd ~/repo
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab
## Running install script on Packet server
ssh -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only
ssh -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only
sleep 30
## Waiting for Packet server to post build results
MINUTES_PAST=0
while [ "$MINUTES_PAST" -lt 150 ]; do
SERVER1_STATUS=$(curl $SERVER1_IP_ADDRESS)
SERVER2_STATUS=$(curl $SERVER2_IP_ADDRESS)
if [[ "$SERVER1_STATUS" == "building" ]] || [[ "$SERVER2_STATUS" == "building" ]]; then
echo "$SERVER1_STATUS" :: "$SERVER2_STATUS"
scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log
scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log
sleep 300
((MINUTES_PAST += 5))
fi
if [[ "$SERVER1_STATUS" != "building" ]] && [[ "$SERVER2_STATUS" != "building" ]]; then
break
fi
if [ "$MINUTES_PAST" -gt 150 ]; then
echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
exit 1
fi
done
## Recording the build results
echo "Server1 Status: $SERVER1_STATUS"
echo "Server2 Status: $SERVER2_STATUS"
if [ "$SERVER1_STATUS" != "success" ]; then
echo "Build failed. Cleaning up server with ID $SERVER1_ID"
scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log || echo "Serveer1 packer_build.log not available yet"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
exit 1
fi
if [ "$SERVER2_STATUS" != "success" ]; then
echo "Build failed. Cleaning up server with ID $SERVER2_ID"
scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log || echo "Server2 packer_build.log not available yet"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
exit 1
fi
echo "Builds were successful. Cleaning up servers with IDs $SERVER1_ID and $SERVER2_ID"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
exit 0

View File

@@ -0,0 +1,74 @@
#! /bin/bash
set -e
# Create artifacts directory
if [ ! -d "/tmp/artifacts" ]; then
mkdir /tmp/artifacts
fi
## Delete stale servers if they exist
DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then
echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID"
curl -X -s DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
fi
## Provision a Type1 baremetal Packet.net server
echo "Provisioning a server on Packet.net"
DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
# Make sure the device ID is sane.
# TODO: maybe make this a regex
if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
exit 1
fi
echo "Server successfully provisioned with ID: $DEVICE_ID"
echo "Sleeping 10 minutes to wait for Packet server to be provisioned"
sleep 300
echo "Sleeping 5 more minutes (CircleCI Keepalive)"
sleep 300
## Recording the IP address of the newly provisioned Packet server
IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
# Copy repo to Packet server
# TODO: Tar up the repo and expand it remotely
cd ~/repo
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
## Running install script on Packet server
ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --vagrant-only
## Waiting for Packet server to post build results
MINUTES_PAST=0
while [ "$MINUTES_PAST" -lt 120 ]; do
STATUS=$(curl $IP_ADDRESS)
if [ "$STATUS" == "building" ]; then
echo "$STATUS"
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "vagrant_build.log not available yet"
sleep 300
((MINUTES_PAST += 5))
else
break
fi
if [ "$MINUTES_PAST" -gt 120 ]; then
echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
done
## Recording the build results
echo $STATUS
if [ "$STATUS" != "success" ]; then
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log
echo "Build failed. Cleaning up server with ID $DEVICE_ID"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
echo "Build was successful. Cleaning up server with ID $DEVICE_ID"
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 0