Merge pull request #77 from clong/update_packer_scripts
Add multiple continuous integration build suites and update Packer build
This commit is contained in:
Chris Long
@@ -9,79 +9,67 @@ jobs:
|
||||
- checkout
|
||||
|
||||
- run:
|
||||
name: Delete stale Packet servers
|
||||
name: Choose which test suite to run based on which files were modified
|
||||
command: |
|
||||
export DELETE_DEVICE_ID=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
|
||||
- run:
|
||||
name: Provisioning a baremetal Packet.net server
|
||||
command: |
|
||||
curl -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' > /tmp/device
|
||||
- run: cat /tmp/device | jq ."id" | tr -d '"' > /tmp/device_id
|
||||
- run: sleep 300
|
||||
- run: echo "Sleeping 5 more minutes while Packet server is provisioned"
|
||||
- run: sleep 300
|
||||
- run:
|
||||
name: Recording the IP address of the Packet server
|
||||
command: |
|
||||
export DEVICE_ID=$(cat /tmp/device_id);
|
||||
curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" > /tmp/ip_info;
|
||||
- run: cat /tmp/ip_info | jq ."ip_addresses[0].address" | tr -d '"' > /tmp/ip_address
|
||||
- run: cd ~/repo
|
||||
- run:
|
||||
name: Copying DetectionLab repo to Packet server
|
||||
command: |
|
||||
export IP_ADDRESS=$(cat /tmp/ip_address);
|
||||
scp -i ~/.ssh/id_rsa -r ~/repo root@"$IP_ADDRESS":/opt/DetectionLab
|
||||
- run:
|
||||
name: Copying install script to Packet server
|
||||
command: |
|
||||
export IP_ADDRESS=$(cat /tmp/ip_address);
|
||||
scp -i ~/.ssh/id_rsa ./ci/automated_install_vagrant_only.sh root@"$IP_ADDRESS":/root
|
||||
- run:
|
||||
name: Running install script on Packet server
|
||||
command: |
|
||||
export IP_ADDRESS=$(cat /tmp/ip_address)
|
||||
ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'chmod +x /root/automated_install_vagrant_only.sh; /bin/bash -c /root/automated_install_vagrant_only.sh'
|
||||
|
||||
- run:
|
||||
name: Waiting for Packet server to post build results
|
||||
shell: /bin/bash
|
||||
command: |
|
||||
export IP_ADDRESS=$(cat /tmp/ip_address)
|
||||
MINUTES_PAST=0
|
||||
while [ "$MINUTES_PAST" -lt 120 ]
|
||||
do
|
||||
export STATUS=$(curl $IP_ADDRESS)
|
||||
if [ "$STATUS" == "building" ]; then
|
||||
echo "$STATUS"
|
||||
sleep 300
|
||||
((MINUTES_PAST+=5))
|
||||
## Checking commits for changes to Packer files
|
||||
## This handles the cases where there are multiple commits
|
||||
if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then
|
||||
if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Packer/)" -gt 0 ]; then
|
||||
export PACKER_MODIFIED=1
|
||||
else
|
||||
export PACKER_MODIFIED=0
|
||||
fi
|
||||
else
|
||||
echo "$STATUS" > /tmp/status
|
||||
break
|
||||
## This handles the cases where there is only a single commit
|
||||
export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)"
|
||||
if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c ^Packer/)" -gt 0 ]; then
|
||||
export PACKER_MODIFIED=1
|
||||
else
|
||||
export PACKER_MODIFIED=0
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if [ "$MINUTES_PAST" -gt 120 ]; then
|
||||
export IP_ADDRESS=$(cat /tmp/ip_address);
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log .
|
||||
cat vagrant.log
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
|
||||
exit 1
|
||||
## Checking commits for changes to Vagrant files
|
||||
if echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then
|
||||
if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) $(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5) | grep -c ^Vagrant/)" -gt 0 ]; then
|
||||
export VAGRANT_MODIFIED=1
|
||||
else
|
||||
export VAGRANT_MODIFIED=0
|
||||
fi
|
||||
else
|
||||
export COMMIT_ID="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)"
|
||||
if [ "$(git diff-tree --no-commit-id --name-only -r $COMMIT_ID | grep -c \"^Vagrant/\")" -gt 0 ]; then
|
||||
export VAGRANT_MODIFIED=1
|
||||
else
|
||||
export VAGRANT_MODIFIED=0
|
||||
fi
|
||||
fi
|
||||
- run:
|
||||
name: Recording build results
|
||||
shell: /bin/bash
|
||||
command: |
|
||||
export DEVICE_ID=$(cat /tmp/device_id)
|
||||
export STATUS=$(cat /tmp/status)
|
||||
echo $STATUS
|
||||
if [ "$STATUS" != "success" ]; then
|
||||
export IP_ADDRESS=$(cat /tmp/ip_address);
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant.log .
|
||||
cat vagrant.log
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
|
||||
exit 1
|
||||
echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED"
|
||||
echo "PACKER_MODIFIED=$PACKER_MODIFIED"
|
||||
## Choosing which test suite to run based on the files that were changed
|
||||
if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then
|
||||
echo "Running the test suite for Packer and Vagrant changes"
|
||||
chmod +x ci/circle_workflows/packer_and_vagrant_changes.sh
|
||||
ci/circle_workflows/packer_and_vagrant_changes.sh
|
||||
exit 0
|
||||
fi
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
|
||||
exit 0
|
||||
if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then
|
||||
echo "Running the default test suite (Vagrant-only)"
|
||||
chmod +x ci/circle_workflows/vagrant_changes.sh
|
||||
ci/circle_workflows/vagrant_changes.sh
|
||||
exit 0
|
||||
fi
|
||||
if [ "$PACKER_MODIFIED" -eq 1 ]; then
|
||||
echo "Running the test suite for Packer-only changes"
|
||||
chmod +x ci/circle_workflows/packer_changes.sh
|
||||
ci/circle_workflows/packer_changes.sh
|
||||
exit 0
|
||||
fi
|
||||
if [ "$VAGRANT_MODIFIED" -eq 1 ]; then
|
||||
echo "Running the test suite for Vagrant-only changes"
|
||||
chmod +x ci/circle_workflows/vagrant_changes.sh
|
||||
ci/circle_workflows/vagrant_changes.sh
|
||||
exit 0
|
||||
fi
|
||||
|
||||
- store_artifacts:
|
||||
path: /tmp/artifacts
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,4 +1,6 @@
|
||||
Vagrant/.vagrant/*
|
||||
Vagrant/vagrant_build.log
|
||||
Packer/packer_cache/*
|
||||
Packer/packer_build.log
|
||||
Boxes/*
|
||||
.DS_Store
|
||||
|
||||
@@ -3,13 +3,15 @@
|
||||
|
||||
if "%PROCESSOR_ARCHITECTURE%"=="AMD64" goto 64BIT
|
||||
|
||||
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue
|
||||
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems
|
||||
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue > NUL
|
||||
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems > NUL
|
||||
|
||||
exit /b
|
||||
exit 0
|
||||
|
||||
:64BIT
|
||||
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue
|
||||
%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe update /force /queue
|
||||
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems
|
||||
%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe executequeueditems
|
||||
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe update /force /queue > NUL
|
||||
%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe update /force /queue > NUL
|
||||
%windir%\microsoft.net\framework\v4.0.30319\ngen.exe executequeueditems > NUL
|
||||
%windir%\microsoft.net\framework64\v4.0.30319\ngen.exe executequeueditems > NUL
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -1,7 +1,20 @@
|
||||
Try {
|
||||
Write-Output "Set power plan to high performance"
|
||||
|
||||
$HighPerf = powercfg -l | %{if($_.contains("High performance")) {$_.split()[3]}}
|
||||
|
||||
# $HighPerf cannot be $null, we try activate this power profile with powercfg
|
||||
#
|
||||
if ($HighPerf -eq $null)
|
||||
{
|
||||
throw "Error: HighPerf is null"
|
||||
}
|
||||
|
||||
$CurrPlan = $(powercfg -getactivescheme).split()[3]
|
||||
|
||||
if ($CurrPlan -ne $HighPerf) {powercfg -setactive $HighPerf}
|
||||
|
||||
} Catch {
|
||||
Write-Warning -Message "Unable to set power plan to high performance"
|
||||
Write-Warning $Error[0]
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
</settings>
|
||||
<settings pass="specialize">
|
||||
<component name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||
<ComputerName>vagrant-2012</ComputerName>
|
||||
<ComputerName>vagrant-2016</ComputerName>
|
||||
<CopyProfile>false</CopyProfile>
|
||||
</component>
|
||||
</settings>
|
||||
|
||||
@@ -15,7 +15,7 @@ if exist "C:\Users\vagrant\windows.iso" (
|
||||
)
|
||||
|
||||
if not exist "C:\Windows\Temp\windows.iso" (
|
||||
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://softwareupdate.vmware.com/cds/vmw-desktop/ws/12.5.7/5813279/windows/packages/tools-windows.tar', 'C:\Windows\Temp\vmware-tools.tar')" <NUL
|
||||
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://softwareupdate.vmware.com/cds/vmw-desktop/ws/14.1.1/7528167/windows/packages/tools-windows.tar', 'C:\Windows\Temp\vmware-tools.tar')" <NUL
|
||||
cmd /c ""C:\Program Files\7-Zip\7z.exe" x C:\Windows\Temp\vmware-tools.tar -oC:\Windows\Temp"
|
||||
FOR /r "C:\Windows\Temp" %%a in (VMware-tools-windows-*.iso) DO REN "%%~a" "windows.iso"
|
||||
rd /S /Q "C:\Program Files (x86)\VMWare"
|
||||
@@ -31,16 +31,12 @@ goto :done
|
||||
|
||||
:virtualbox
|
||||
|
||||
:: There needs to be Oracle CA (Certificate Authority) certificates installed in order
|
||||
:: to prevent user intervention popups which will undermine a silent installation.
|
||||
cmd /c certutil -addstore -f "TrustedPublisher" A:\oracle-cert.cer
|
||||
|
||||
if exist "C:\Users\vagrant\VBoxGuestAdditions.iso" (
|
||||
move /Y C:\Users\vagrant\VBoxGuestAdditions.iso C:\Windows\Temp
|
||||
)
|
||||
|
||||
if not exist "C:\Windows\Temp\VBoxGuestAdditions.iso" (
|
||||
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://download.virtualbox.org/virtualbox/5.1.28/VBoxGuestAdditions_5.1.28.iso', 'C:\Windows\Temp\VBoxGuestAdditions.iso')" <NUL
|
||||
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('http://download.virtualbox.org/virtualbox/5.2.4/VBoxGuestAdditions_5.2.4.iso', 'C:\Windows\Temp\VBoxGuestAdditions.iso')" <NUL
|
||||
)
|
||||
|
||||
cmd /c ""C:\Program Files\7-Zip\7z.exe" x C:\Windows\Temp\VBoxGuestAdditions.iso -oC:\Windows\Temp\virtualbox"
|
||||
|
||||
@@ -81,7 +81,7 @@
|
||||
"winrm_password": "vagrant",
|
||||
"winrm_timeout": "4h",
|
||||
"shutdown_command": "a:/sysprep.bat",
|
||||
"guest_os_type": "Windows2012_64",
|
||||
"guest_os_type": "Windows2016_64",
|
||||
"guest_additions_mode": "disable",
|
||||
"disk_size": 61440,
|
||||
"floppy_files": [
|
||||
|
||||
@@ -90,7 +90,7 @@ go build
|
||||
cd /home/vagrant
|
||||
|
||||
# Modify the config to work with config importer
|
||||
cat /home/vagrant/osquery-configuration/Endpoints/Windows/osquery.conf | sed 's#packs/#../packs/#g' | grep -v unwanted-chrome-extensions | grep -v security-tooling-checks | grep -v performance-metrics > /home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf
|
||||
cat /home/vagrant/osquery-configuration/Endpoints/Windows/osquery.conf | sed 's#packs/#../packs/#g' | grep -v unwanted-chrome-extensions | grep -v security-tooling-checks | grep -v performance-metrics | grep -v logger_snapshot_event_type > /home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf
|
||||
# Install configimporter
|
||||
echo "Installing configimporter"
|
||||
echo "Sleeping for 5"
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
# Purpose: Downloads and unzips a copy of the Palantir WEF Github Repo. This includes WEF subscriptions and custom WEF channels.
|
||||
|
||||
# GitHub requires TLS 1.2
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
||||
|
||||
Write-Host "Downloading and unzipping the Palantir Windows Event Forwarding Repo from Github..."
|
||||
|
||||
$wefRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\wef-Master.zip'
|
||||
|
||||
320
build.sh
320
build.sh
@@ -7,25 +7,13 @@
|
||||
# If you encounter issues, feel free to open an issue at
|
||||
# https://github.com/clong/DetectionLab/issues
|
||||
|
||||
set -e
|
||||
|
||||
print_usage() {
|
||||
echo "Usage: ./build.sh <virtualbox|vmware_fusion>"
|
||||
exit 0
|
||||
}
|
||||
|
||||
check_packer_and_vagrant() {
|
||||
# Check for existence of Vagrant in PATH
|
||||
if ! which vagrant >/dev/null; then
|
||||
(echo >&2 "Vagrant was not found in your PATH.")
|
||||
(echo >&2 "Please correct this before continuing. Quitting.")
|
||||
exit 1
|
||||
fi
|
||||
# Ensure Vagrant >= 2.0.0
|
||||
if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
|
||||
(echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
|
||||
fi
|
||||
# Check for existence of Packer in PATH
|
||||
check_packer_path() {
|
||||
# Check for existence of Packer in PATH
|
||||
if ! which packer >/dev/null; then
|
||||
(echo >&2 "Packer was not found in your PATH.")
|
||||
(echo >&2 "Please correct this before continuing. Quitting.")
|
||||
@@ -34,6 +22,19 @@ check_packer_and_vagrant() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_vagrant_path() {
|
||||
# Check for existence of Vagrant in PATH
|
||||
if ! which vagrant >/dev/null; then
|
||||
(echo >&2 "Vagrant was not found in your PATH.")
|
||||
(echo >&2 "Please correct this before continuing. Quitting.")
|
||||
exit 1
|
||||
fi
|
||||
# Ensure Vagrant >= 2.0.0
|
||||
if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
|
||||
(echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
|
||||
fi
|
||||
}
|
||||
|
||||
# Returns 0 if not installed or 1 if installed
|
||||
check_virtualbox_installed() {
|
||||
if which VBoxManage >/dev/null; then
|
||||
@@ -101,41 +102,40 @@ list_providers() {
|
||||
echo "$PROVIDER"
|
||||
}
|
||||
|
||||
# A series of checks to identify potential issues before starting the build
|
||||
preflight_checks() {
|
||||
DL_DIR="$1"
|
||||
|
||||
# Check to see if curl is in PATH
|
||||
if ! which curl >/dev/null; then
|
||||
(echo >&2 "Please install curl and make sure it is in your PATH.")
|
||||
exit 1
|
||||
fi
|
||||
# Check to see if boxes exist already
|
||||
# Check to see if boxes exist in the "Boxes" directory already
|
||||
check_boxes_built() {
|
||||
BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l)
|
||||
if [ "$BOXES_BUILT" -gt 0 ]; then
|
||||
(echo >&2 "You appear to have already built at least one box using Packer. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.")
|
||||
if [ "$VAGRANT_ONLY" -eq 1 ]; then
|
||||
(echo >&2 "WARNING: You seem to have at least one .box file present in $DL_DIR/Boxes already. If you would like fresh boxes downloaded, please remove all files from the Boxes directory and re-run this script.")
|
||||
DOWNLOAD_BOXES=0
|
||||
else
|
||||
(echo >&2 "You seem to have at least one .box file in $DL_DIR/Boxes. This script does not support pre-built boxes. Please either delete the existing boxes or follow the build steps in the README to continue.")
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Check to ensure either "md5" or "md5sum" is installed for verifying integrity of downloaded boxes
|
||||
check_md5_tool_exists() {
|
||||
if ! which md5 > /dev/null && ! which md5sum > /dev/null; then
|
||||
(echo >&2 "md5 or md5sum not found in PATH. Please install at least one of these utilities to continue.")
|
||||
exit 1
|
||||
fi
|
||||
# Check to see if any Vagrant instances exist already
|
||||
cd "$DL_DIR"/Vagrant/
|
||||
}
|
||||
|
||||
# Check to see if any Vagrant instances exist already
|
||||
check_vagrant_instances_exist() {
|
||||
cd "$DL_DIR"/Vagrant/ || exit 1
|
||||
# Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true"
|
||||
VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true
|
||||
if [ "$VAGRANT_BUILT" -ne 4 ]; then
|
||||
(echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
|
||||
exit 1
|
||||
fi
|
||||
# Check available disk space. Recommend 80GB free, warn if less.
|
||||
FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
|
||||
if [ "$FREE_DISK_SPACE" -lt 80000 ]; then
|
||||
(echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n")
|
||||
(df >&2 -m "$HOME")
|
||||
(echo >&2 "")
|
||||
fi
|
||||
# Check Packer version against known bad
|
||||
if [ "$(packer --version)" == '1.1.2' ]; then
|
||||
(echo >&2 "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.")
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_vagrant_reload_plugin() {
|
||||
# Ensure the vagrant-reload plugin is installed
|
||||
VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
|
||||
if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
|
||||
@@ -147,26 +147,70 @@ preflight_checks() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Check available disk space. Recommend 80GB free, warn if less.
|
||||
check_disk_free_space() {
|
||||
FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
|
||||
if [ "$FREE_DISK_SPACE" -lt 80000 ]; then
|
||||
(echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n")
|
||||
(df >&2 -m "$HOME")
|
||||
(echo >&2 "")
|
||||
fi
|
||||
}
|
||||
|
||||
# Check to see if curl is in PATH - needed for post-install checks
|
||||
check_curl(){
|
||||
if ! which curl >/dev/null; then
|
||||
(echo >&2 "Please install curl and make sure it is in your PATH.")
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check Packer version against known "bad" versions
|
||||
check_packer_known_bad() {
|
||||
if [ "$(packer --version)" == '1.1.2' ]; then
|
||||
(echo >&2 "Packer 1.1.2 is not supported. Please upgrade to a newer version and see https://github.com/hashicorp/packer/issues/5622 for more information.")
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# A series of checks to identify potential issues before starting the build
|
||||
preflight_checks() {
|
||||
# If it's not a Vagrant-only build, then run Packer-related checks
|
||||
if [ "$VAGRANT_ONLY" -eq 0 ]; then
|
||||
check_packer_path
|
||||
check_packer_known_bad
|
||||
else
|
||||
# If it is a Vagrant-only build, set appropriate checks
|
||||
DOWNLOAD_BOXES=1
|
||||
check_md5_tool_exists
|
||||
fi
|
||||
|
||||
# If it's not a Packer-only build, then run Vagrant-related checks
|
||||
if [ "$PACKER_ONLY" -eq 0 ]; then
|
||||
check_vagrant_path
|
||||
check_vagrant_instances_exist
|
||||
check_vagrant_reload_plugin
|
||||
fi
|
||||
|
||||
check_boxes_built
|
||||
check_disk_free_space
|
||||
check_curl
|
||||
}
|
||||
|
||||
# Builds a box using Packer
|
||||
packer_build_box() {
|
||||
PROVIDER="$1"
|
||||
BOX="$2"
|
||||
DL_DIR="$3"
|
||||
BOX="$1"
|
||||
if [ "$PROVIDER" == "vmware_fusion" ]; then
|
||||
PROVIDER="vmware"
|
||||
fi
|
||||
cd "$DL_DIR/Packer"
|
||||
cd "$DL_DIR/Packer" || exit 1
|
||||
(echo >&2 "Using Packer to build the $BOX Box. This can take 90-180 minutes depending on bandwidth and hardware.")
|
||||
if ! $(which packer) build --only="$PROVIDER-iso" "$BOX".json; then
|
||||
(echo >&2 "Something went wrong while attempting to build the $BOX box.")
|
||||
(echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
|
||||
fi
|
||||
PACKER_LOG=1 PACKER_LOG_PATH="$DL_DIR/Packer/packer_build.log" $(which packer) build --only="$PROVIDER-iso" "$BOX".json >&2
|
||||
echo "$?"
|
||||
}
|
||||
|
||||
# Moves the boxes from the Packer directory to the Boxes directory
|
||||
move_boxes() {
|
||||
PROVIDER="$1"
|
||||
DL_DIR="$2"
|
||||
# Hacky workaround for VMware
|
||||
if [ "$PROVIDER" == "vmware_fusion" ]; then
|
||||
PROVIDER="vmware"
|
||||
@@ -186,20 +230,17 @@ move_boxes() {
|
||||
|
||||
# Brings up a single host using Vagrant
|
||||
vagrant_up_host() {
|
||||
PROVIDER="$1"
|
||||
HOST="$2"
|
||||
DL_DIR="$3"
|
||||
HOST="$1"
|
||||
(echo >&2 "Attempting to bring up the $HOST host using Vagrant")
|
||||
cd "$DL_DIR"/Vagrant
|
||||
$(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2
|
||||
cd "$DL_DIR"/Vagrant || exit 1
|
||||
VAGRANT_LOG=info $(which vagrant) up "$HOST" --provider="$PROVIDER" 3>&1 1>&2 2>&3 | tee -a "$DL_DIR/Vagrant/vagrant_build.log"
|
||||
echo "$?"
|
||||
}
|
||||
|
||||
# Attempts to reload and re-provision a host if the intial "vagrant up" fails
|
||||
vagrant_reload_host() {
|
||||
HOST="$1"
|
||||
DL_DIR="$2"
|
||||
cd "$DL_DIR"/Vagrant
|
||||
cd "$DL_DIR"/Vagrant || exit 1
|
||||
# Attempt to reload the host if the vagrant up command didn't exit cleanly
|
||||
$(which vagrant) reload "$HOST" --provision 1>&2
|
||||
echo "$?"
|
||||
@@ -236,22 +277,17 @@ post_build_checks() {
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
# Get location of build.sh
|
||||
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
|
||||
DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROVIDER=""
|
||||
LAB_HOSTS=("logger" "dc" "wef" "win10")
|
||||
parse_cli_arguments() {
|
||||
# If no argument was supplied, list available providers
|
||||
if [ $# -eq 0 ]; then
|
||||
if [ "$#" -eq 0 ]; then
|
||||
PROVIDER=$(list_providers)
|
||||
fi
|
||||
# If more than one argument was supplied, print usage message
|
||||
if [ $# -gt 1 ]; then
|
||||
# If more than two arguments were supplied, print usage message
|
||||
if [ "$#" -gt 2 ]; then
|
||||
print_usage
|
||||
exit 1
|
||||
fi
|
||||
if [ $# -eq 1 ]; then
|
||||
if [ "$#" -ge 1 ]; then
|
||||
# If the user specifies the provider as an agument, set the variable
|
||||
# TODO: Check to make sure they actually have their provider installed
|
||||
case "$1" in
|
||||
@@ -267,12 +303,121 @@ main() {
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
if [ $# -eq 2 ]; then
|
||||
case "$2" in
|
||||
--packer-only)
|
||||
PACKER_ONLY=1
|
||||
;;
|
||||
--vagrant-only)
|
||||
VAGRANT_ONLY=1
|
||||
;;
|
||||
*)
|
||||
echo -e "\"$2\" is not recognized as an option. Available options are:\\n--packer-only\\n--vagrant-only"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
}
|
||||
|
||||
preflight_checks "$DL_DIR"
|
||||
packer_build_box "$PROVIDER" "windows_2016" "$DL_DIR"
|
||||
packer_build_box "$PROVIDER" "windows_10" "$DL_DIR"
|
||||
move_boxes "$PROVIDER" "$DL_DIR"
|
||||
build_packer_boxes() {
|
||||
PACKER_BOXES=("windows_2016" "windows_10")
|
||||
|
||||
if [ "$(hostname)" == "packerwindows10" ]; then # Workaround for CI environment
|
||||
(echo >&2 "CI Environment detected. If you are a user and are seeing this, please file an issue on GitHub.")
|
||||
RET=$(packer_build_box "windows_10")
|
||||
if [ "$RET" -eq 0 ]; then
|
||||
(echo >&2 "Good news! The windows_10 box was built with Packer successfully!")
|
||||
else
|
||||
(echo >&2 "Something went wrong while attempting to build the windows_10 box.")
|
||||
(echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
|
||||
exit 1
|
||||
fi
|
||||
elif [ "$(hostname)" == "packerwindows2016" ]; then # Workaround for CI environment
|
||||
(echo >&2 "CI Environment detected. If you are a user and are seeing this, please file an issue on GitHub.")
|
||||
RET=$(packer_build_box "windows_2016")
|
||||
if [ "$RET" -eq 0 ]; then
|
||||
(echo >&2 "Good news! The windows_2016 box was built with Packer successfully!")
|
||||
else
|
||||
(echo >&2 "Something went wrong while attempting to build the windows_2016 box.")
|
||||
(echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
for PACKER_BOX in "${PACKER_BOXES[@]}"; do # Normal user workflow
|
||||
RET=$(packer_build_box "$PACKER_BOX")
|
||||
if [ "$RET" -eq 0 ]; then
|
||||
(echo >&2 "Good news! $PACKER_BOX was built successfully!")
|
||||
else
|
||||
(echo >&2 "Something went wrong while attempting to build the $PACKER_BOX box.")
|
||||
(echo >&2 "To file an issue, please visit https://github.com/clong/DetectionLab/issues/")
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
choose_md5_tool() {
|
||||
if which md5; then
|
||||
MD5TOOL="$(which md5)"
|
||||
CUT_INDEX=4
|
||||
else
|
||||
MD5TOOL="$(which md5sum)"
|
||||
CUT_INDEX=1
|
||||
fi
|
||||
}
|
||||
|
||||
# Downloads pre-built Packer boxes from detectionlab.network to save time during CI builds
|
||||
download_boxes() {
|
||||
if [ "$PROVIDER" == "virtualbox" ]; then
|
||||
wget "https://www.detectionlab.network/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box
|
||||
wget "https://www.detectionlab.network/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box
|
||||
elif [ "$PROVIDER" == "vmware_fusion" ]; then
|
||||
wget "https://www.detectionlab.network/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box
|
||||
wget "https://www.detectionlab.network/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box
|
||||
fi
|
||||
|
||||
# Hacky workaround
|
||||
if [ "$PROVIDER" == "vmware_fusion" ]; then
|
||||
PROVIDER="vmware"
|
||||
fi
|
||||
|
||||
# Ensure Windows 10 box exists
|
||||
if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
|
||||
(echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.")
|
||||
exit 1
|
||||
fi
|
||||
# Ensure Windows 2016 box exists
|
||||
if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
|
||||
(echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.")
|
||||
exit 1
|
||||
fi
|
||||
# Verify hashes of VirtualBox boxes
|
||||
if [ "$PROVIDER" == "virtualbox" ]; then
|
||||
if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then
|
||||
(echo >&2 "Hash mismatch on windows_10_virtualbox.box")
|
||||
fi
|
||||
if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "614f984c82b51471b5bb753940b59d38" ]; then
|
||||
(echo >&2 "Hash mismatch on windows_2016_virtualbox.box")
|
||||
fi
|
||||
# Verify hashes of VMware boxes
|
||||
elif [ "$PROVIDER" == "vmware" ]; then
|
||||
if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then
|
||||
(echo >&2 "Hash mismatch on windows_10_vmware.box")
|
||||
exit 1
|
||||
fi
|
||||
if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then
|
||||
(echo >&2 "Hash mismatch on windows_2016_vmware.box")
|
||||
exit 1
|
||||
fi
|
||||
# Reset PROVIDER variable if using VMware
|
||||
if [ "$PROVIDER" == "vmware" ]; then
|
||||
PROVIDER="vmware_fusion"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
build_vagrant_hosts() {
|
||||
LAB_HOSTS=("logger" "dc" "wef" "win10")
|
||||
# Change provider back to original selection if using vmware_fusion
|
||||
if [ "$PROVIDER" == "vmware" ]; then
|
||||
PROVIDER="vmware_fusion"
|
||||
@@ -280,7 +425,7 @@ main() {
|
||||
|
||||
# Vagrant up each box and attempt to reload one time if it fails
|
||||
for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do
|
||||
RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR")
|
||||
RET=$(vagrant_up_host "$VAGRANT_HOST")
|
||||
if [ "$RET" -eq 0 ]; then
|
||||
(echo >&2 "Good news! $VAGRANT_HOST was built successfully!")
|
||||
fi
|
||||
@@ -288,16 +433,43 @@ main() {
|
||||
if [ "$RET" -ne 0 ]; then
|
||||
(echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.")
|
||||
(echo >&2 "Attempting to reload and reprovision the host...")
|
||||
RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR")
|
||||
RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST")
|
||||
if [ "$RETRY_STATUS" -ne 0 ]; then
|
||||
(echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
post_build_checks
|
||||
}
|
||||
|
||||
main() {
|
||||
# Get location of build.sh
|
||||
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
|
||||
DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PACKER_ONLY=0
|
||||
VAGRANT_ONLY=0
|
||||
|
||||
parse_cli_arguments "$@"
|
||||
preflight_checks
|
||||
if [[ "$DOWNLOAD_BOXES" -eq 1 ]] && [[ "$VAGRANT_ONLY" -eq 1 ]]; then
|
||||
download_boxes
|
||||
fi
|
||||
|
||||
# Build Packer boxes if this isn't a Vagrant-only build
|
||||
if [ "$VAGRANT_ONLY" -eq 0 ]; then
|
||||
build_packer_boxes
|
||||
# The only time we will need to move boxes is if we're doing a full build
|
||||
if [ "$PACKER_ONLY" -eq 0 ]; then
|
||||
move_boxes
|
||||
fi
|
||||
fi
|
||||
|
||||
# Build and Test Vagrant hosts if this isn't a Packer-only build
|
||||
if [ "$PACKER_ONLY" -eq 0 ]; then
|
||||
build_vagrant_hosts
|
||||
post_build_checks
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
exit 0
|
||||
|
||||
@@ -1,309 +0,0 @@
|
||||
#! /bin/bash
|
||||
|
||||
# This script skips the entire Packer box building process.
|
||||
# This script downloads pre-built VirtualBox Packer images from S3 and puts
|
||||
# them into the "Boxes" directory.
|
||||
# Only MacOS and Linux are supported.
|
||||
# If you encounter issues, feel free to open an issue at
|
||||
# https://github.com/clong/DetectionLab/issues
|
||||
|
||||
set -e
|
||||
|
||||
print_usage() {
|
||||
echo "Usage: ./build.sh <virtualbox|vmware_fusion>"
|
||||
exit 0
|
||||
}
|
||||
|
||||
check_vagrant() {
|
||||
# Check for existence of Vagrant in PATH
|
||||
if ! which vagrant >/dev/null; then
|
||||
(echo >&2 "Vagrant was not found in your PATH.")
|
||||
(echo >&2 "Please correct this before continuing. Quitting.")
|
||||
exit 1
|
||||
fi
|
||||
# Ensure Vagrant >= 2.0.0
|
||||
if [ "$(vagrant --version | grep -o "[0-9]" | head -1)" -lt 2 ]; then
|
||||
(echo >&2 "WARNING: It is highly recommended to use Vagrant 2.0.0 or above before continuing")
|
||||
fi
|
||||
}
|
||||
|
||||
# Returns 0 if not installed or 1 if installed
|
||||
check_virtualbox_installed() {
|
||||
if which VBoxManage >/dev/null; then
|
||||
echo "1"
|
||||
else
|
||||
echo "0"
|
||||
fi
|
||||
}
|
||||
|
||||
# Returns 0 if not installed or 1 if installed
|
||||
check_vmware_fusion_installed() {
|
||||
if [ -e "/Applications/VMware Fusion.app" ]; then
|
||||
echo "1"
|
||||
else
|
||||
echo "0"
|
||||
fi
|
||||
}
|
||||
|
||||
# Returns 0 if not installed or 1 if installed
|
||||
check_vmware_vagrant_plugin_installed() {
|
||||
VAGRANT_VMWARE_PLUGIN_PRESENT="$(vagrant plugin list | grep -c 'vagrant-vmware-fusion')"
|
||||
if [ "$VAGRANT_VMWARE_PLUGIN_PRESENT" -eq 0 ]; then
|
||||
(echo >&2 "VMWare Fusion is installed, but the Vagrant plugin is not.")
|
||||
(echo >&2 "Visit https://www.vagrantup.com/vmware/index.html#buy-now for more information on how to purchase and install it")
|
||||
(echo >&2 "VMWare Fusion will not be listed as a provider until the Vagrant plugin has been installed.")
|
||||
echo "0"
|
||||
else
|
||||
echo "$VAGRANT_VMWARE_PLUGIN_PRESENT"
|
||||
fi
|
||||
}
|
||||
|
||||
# List the available Vagrant providers present on the system
|
||||
list_providers() {
|
||||
VBOX_PRESENT=0
|
||||
VMWARE_FUSION_PRESENT=0
|
||||
|
||||
if [ "$(uname)" == "Darwin" ]; then
|
||||
# Detect Providers on OSX
|
||||
VBOX_PRESENT=$(check_virtualbox_installed)
|
||||
VMWARE_FUSION_PRESENT=$(check_vmware_fusion_installed)
|
||||
VAGRANT_VMWARE_PLUGIN_PRESENT=$(check_vmware_vagrant_plugin_installed)
|
||||
else
|
||||
# Assume the only other available provider is VirtualBox
|
||||
VBOX_PRESENT=$(check_virtualbox_installed)
|
||||
fi
|
||||
|
||||
(echo >&2 "Available Providers:")
|
||||
if [ "$VBOX_PRESENT" == "1" ]; then
|
||||
(echo >&2 "virtualbox")
|
||||
fi
|
||||
if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $VAGRANT_VMWARE_PLUGIN_PRESENT -eq 1 ]]; then
|
||||
(echo >&2 "vmware_fusion")
|
||||
fi
|
||||
if [[ $VBOX_PRESENT -eq 0 ]] && [[ $VMWARE_FUSION_PRESENT -eq 0 ]]; then
|
||||
(echo >&2 "You need to install a provider such as VirtualBox or VMware Fusion to continue.")
|
||||
exit 1
|
||||
fi
|
||||
(echo >&2 -e "\\nWhich provider would you like to use?")
|
||||
read -r PROVIDER
|
||||
# Sanity check
|
||||
if [[ "$PROVIDER" != "virtualbox" ]] && [[ "$PROVIDER" != "vmware_fusion" ]]; then
|
||||
(echo >&2 "Please choose a valid provider. \"$PROVIDER\" is not a valid option")
|
||||
exit 1
|
||||
fi
|
||||
echo "$PROVIDER"
|
||||
}
|
||||
|
||||
# A series of checks to identify potential issues before starting the build
|
||||
preflight_checks() {
|
||||
DL_DIR="$1"
|
||||
DOWNLOAD_BOXES=1
|
||||
|
||||
# Check to see if curl is in PATH
|
||||
if ! which curl >/dev/null; then
|
||||
(echo >&2 "Please install curl and make sure it is in your PATH.")
|
||||
exit 1
|
||||
fi
|
||||
# Check to see if wget is in PATH
|
||||
if ! which wget >/dev/null; then
|
||||
(echo >&2 "Please install curl and make sure it is in your PATH.")
|
||||
exit 1
|
||||
fi
|
||||
# Check to see if boxes exist already
|
||||
BOXES_BUILT=$(find "$DL_DIR"/Boxes -name "*.box" | wc -l)
|
||||
if [ "$BOXES_BUILT" -gt 0 ]; then
|
||||
(echo >&2 "WARNING: You seem to have boxes present in the Boxes directory already. If you would like fresh boxes downloaded, please remove all files from the Boxes directory and re-run this script.")
|
||||
DOWNLOAD_BOXES=0
|
||||
fi
|
||||
# Check to see if any Vagrant instances exist already
|
||||
cd "$DL_DIR"/Vagrant/
|
||||
# Vagrant status has the potential to return a non-zero error code, so we work around it with "|| true"
|
||||
VAGRANT_BUILT=$(vagrant status | grep -c 'not created') || true
|
||||
if [ "$VAGRANT_BUILT" -ne 4 ]; then
|
||||
(echo >&2 "You appear to have already created at least one Vagrant instance. This script does not support pre-created instances. Please either destroy the existing instances or follow the build steps in the README to continue.")
|
||||
exit 1
|
||||
fi
|
||||
# Check available disk space. Recommend 80GB free, warn if less.
|
||||
FREE_DISK_SPACE=$(df -m "$HOME" | tr -s ' ' | grep '/' | cut -d ' ' -f 4)
|
||||
if [ "$FREE_DISK_SPACE" -lt 80000 ]; then
|
||||
(echo >&2 -e "Warning: You appear to have less than 80GB of HDD space free on your primary partition. If you are using a separate parition, you may ignore this warning.\\n")
|
||||
(df >&2 -m "$HOME")
|
||||
(echo >&2 "")
|
||||
fi
|
||||
# Ensure the vagrant-reload plugin is installed
|
||||
VAGRANT_RELOAD_PLUGIN_INSTALLED=$(vagrant plugin list | grep -c 'vagrant-reload')
|
||||
if [ "$VAGRANT_RELOAD_PLUGIN_INSTALLED" != "1" ]; then
|
||||
(echo >&2 "The vagrant-reload plugin is required and not currently installed. This script will attempt to install it now.")
|
||||
if ! $(which vagrant) plugin install "vagrant-reload"; then
|
||||
(echo >&2 "Unable to install the vagrant-reload plugin. Please try to do so manually and re-run this script.")
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Downloads pre-built Packer boxes from detectionlab.network to save time during CI builds
|
||||
download_boxes() {
|
||||
DL_DIR="$1"
|
||||
PROVIDER="$2"
|
||||
|
||||
if [ "$PROVIDER" == "virtualbox" ]; then
|
||||
wget "https://www.detectionlab.network/windows_2016_virtualbox.box" -O "$DL_DIR"/Boxes/windows_2016_virtualbox.box
|
||||
wget "https://www.detectionlab.network/windows_10_virtualbox.box" -O "$DL_DIR"/Boxes/windows_10_virtualbox.box
|
||||
elif [ "$PROVIDER" == "vmware_fusion" ]; then
|
||||
wget "https://www.detectionlab.network/windows_2016_vmware.box" -O "$DL_DIR"/Boxes/windows_2016_vmware.box
|
||||
wget "https://www.detectionlab.network/windows_10_vmware.box" -O "$DL_DIR"/Boxes/windows_10_vmware.box
|
||||
fi
|
||||
|
||||
# Hacky workaround
|
||||
if [ "$PROVIDER" == "vmware_fusion" ]; then
|
||||
PROVIDER="vmware"
|
||||
fi
|
||||
|
||||
# Ensure Windows 10 box exists
|
||||
if [ ! -f "$DL_DIR"/Boxes/windows_10_"$PROVIDER".box ]; then
|
||||
(echo >&2 "Windows 10 box is missing from the Boxes directory. Qutting.")
|
||||
exit 1
|
||||
fi
|
||||
# Ensure Windows 2016 box exists
|
||||
if [ ! -f "$DL_DIR"/Boxes/windows_2016_"$PROVIDER".box ]; then
|
||||
(echo >&2 "Windows 2016 box is missing from the Boxes directory. Qutting.")
|
||||
exit 1
|
||||
fi
|
||||
# Verify hashes of VirtualBox boxes
|
||||
if [ "$PROVIDER" == "virtualbox" ]; then
|
||||
if [ "$(md5sum windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "30b06e30b36b02ccf1dc5c04017654aa" ]; then
|
||||
(echo >&2 "Hash mismatch on windows_10_virtualbox.box")
|
||||
fi
|
||||
if [ "$(md5sum windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "614f984c82b51471b5bb753940b59d38" ]; then
|
||||
(echo >&2 "Hash mismatch on windows_2016_virtualbox.box")
|
||||
fi
|
||||
# Verify hashes of VMware boxes
|
||||
elif [ "$PROVIDER" == "vmware" ]; then
|
||||
if [ "$(md5 windows_10_"$PROVIDER".box | cut -d ' ' -f 1)" != "174ad0f0fd2089ff74a880c6dadac74c" ]; then
|
||||
(echo >&2 "Hash mismatch on windows_10_vmware.box")
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(md5 windows_2016_"$PROVIDER".box | cut -d ' ' -f 1)" != "1511b9dc942c69c2cc5a8dc471fa8865" ]; then
|
||||
(echo >&2 "Hash mismatch on windows_2016_vmware.box")
|
||||
exit 1
|
||||
fi
|
||||
# Reset PROVIDER variable
|
||||
PROVIDER="vmware_fusion"
|
||||
fi
|
||||
}
|
||||
|
||||
# Brings up a single host using Vagrant
|
||||
vagrant_up_host() {
|
||||
PROVIDER="$1"
|
||||
HOST="$2"
|
||||
DL_DIR="$3"
|
||||
(echo >&2 "Attempting to bring up the $HOST host using Vagrant")
|
||||
cd "$DL_DIR"/Vagrant
|
||||
$(which vagrant) up "$HOST" --provider="$PROVIDER" 1>&2
|
||||
echo "$?"
|
||||
}
|
||||
|
||||
# Attempts to reload and re-provision a host if the intial "vagrant up" fails
|
||||
vagrant_reload_host() {
|
||||
HOST="$1"
|
||||
DL_DIR="$2"
|
||||
cd "$DL_DIR"/Vagrant
|
||||
# Attempt to reload the host if the vagrant up command didn't exit cleanly
|
||||
$(which vagrant) reload "$HOST" --provision 1>&2
|
||||
echo "$?"
|
||||
}
|
||||
|
||||
# A series of checks to ensure important services are responsive after the build completes.
|
||||
post_build_checks() {
|
||||
# If the curl operation fails, we'll just leave the variable equal to 0
|
||||
# This is needed to prevent the script from exiting if the curl operation fails
|
||||
CALDERA_CHECK=$(curl -ks -m 2 https://192.168.38.5:8888 | grep -c '302: Found' || echo "")
|
||||
SPLUNK_CHECK=$(curl -ks -m 2 https://192.168.38.5:8000/en-US/account/login?return_to=%2Fen-US%2F | grep -c 'This browser is not supported by Splunk' || echo "")
|
||||
FLEET_CHECK=$(curl -ks -m 2 https://192.168.38.5:8412 | grep -c 'Kolide Fleet' || echo "")
|
||||
|
||||
BASH_MAJOR_VERSION=$(/bin/bash --version | grep 'GNU bash' | grep -o version\.\.. | cut -d ' ' -f 2 | cut -d '.' -f 1)
|
||||
# Associative arrays are only supported in bash 4 and up
|
||||
if [ "$BASH_MAJOR_VERSION" -ge 4 ]; then
|
||||
declare -A SERVICES
|
||||
SERVICES=(["caldera"]="$CALDERA_CHECK" ["splunk"]="$SPLUNK_CHECK" ["fleet"]="$FLEET_CHECK")
|
||||
for SERVICE in "${!SERVICES[@]}"; do
|
||||
if [ "${SERVICES[$SERVICE]}" -lt 1 ]; then
|
||||
(echo >&2 "Warning: $SERVICE failed post-build tests and may not be functioning correctly.")
|
||||
fi
|
||||
done
|
||||
else
|
||||
if [ "$CALDERA_CHECK" -lt 1 ]; then
|
||||
(echo >&2 "Warning: Caldera failed post-build tests and may not be functioning correctly.")
|
||||
fi
|
||||
if [ "$SPLUNK_CHECK" -lt 1 ]; then
|
||||
(echo >&2 "Warning: Splunk failed post-build tests and may not be functioning correctly.")
|
||||
fi
|
||||
if [ "$FLEET_CHECK" -lt 1 ]; then
|
||||
(echo >&2 "Warning: Fleet failed post-build tests and may not be functioning correctly.")
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
# Get location of build_vagrant_only.sh
|
||||
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
|
||||
DL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROVIDER=""
|
||||
LAB_HOSTS=("logger" "dc" "wef" "win10")
|
||||
# If no argument was supplied, list available providers
|
||||
if [ $# -eq 0 ]; then
|
||||
PROVIDER=$(list_providers)
|
||||
fi
|
||||
# If more than one argument was supplied, print usage message
|
||||
if [ $# -gt 1 ]; then
|
||||
print_usage
|
||||
exit 1
|
||||
fi
|
||||
if [ $# -eq 1 ]; then
|
||||
# If the user specifies the provider as an agument, set the variable
|
||||
# TODO: Check to make sure they actually have their provider installed
|
||||
case "$1" in
|
||||
virtualbox)
|
||||
PROVIDER="$1"
|
||||
;;
|
||||
vmware_fusion)
|
||||
PROVIDER="$1"
|
||||
;;
|
||||
*)
|
||||
echo "\"$1\" is not a valid provider. Listing available providers:"
|
||||
PROVIDER=$(list_providers)
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
check_vagrant
|
||||
preflight_checks "$DL_DIR"
|
||||
if [ "$DOWNLOAD_BOXES" -eq 0 ]; then
|
||||
(echo >&2 "Skipping box downloads since .box files are already present in the Boxes/ directory.")
|
||||
else
|
||||
download_boxes "$DL_DIR" "$PROVIDER"
|
||||
fi
|
||||
|
||||
# Vagrant up each box and attempt to reload one time if it fails
|
||||
for VAGRANT_HOST in "${LAB_HOSTS[@]}"; do
|
||||
RET=$(vagrant_up_host "$PROVIDER" "$VAGRANT_HOST" "$DL_DIR")
|
||||
if [ "$RET" -eq 0 ]; then
|
||||
(echo >&2 "Good news! $VAGRANT_HOST was built successfully!")
|
||||
fi
|
||||
# Attempt to recover if the intial "vagrant up" fails
|
||||
if [ "$RET" -ne 0 ]; then
|
||||
(echo >&2 "Something went wrong while attempting to build the $VAGRANT_HOST box.")
|
||||
(echo >&2 "Attempting to reload and reprovision the host...")
|
||||
RETRY_STATUS=$(vagrant_reload_host "$VAGRANT_HOST" "$DL_DIR")
|
||||
if [ "$RETRY_STATUS" -ne 0 ]; then
|
||||
(echo >&2 "Failed to bring up $VAGRANT_HOST after a reload. Exiting.")
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
post_build_checks
|
||||
}
|
||||
|
||||
main "$@"
|
||||
exit 0
|
||||
100
ci/README.md
100
ci/README.md
@@ -1,6 +1,104 @@
|
||||
# ci
|
||||
# Continuous Integration
|
||||
|
||||
The files in this directory are used to bootstrap an Ubuntu 16.04 baremetal server
|
||||
for continuous integration testing by installing the prerequisites needed for
|
||||
Detection Lab. After the prerequisites are installed, the build script is called
|
||||
and the build will begin in a tmux session.
|
||||
|
||||
## Understanding the build process
|
||||
|
||||
Once a PR is created, the contents of that PR will be copied to a CircleCI worker to be tested.
|
||||
The CircleCI worker will evaluate which files have been modified and set environment variables accordingly. There are 4 possible options and 3 different tests:
|
||||
|
||||
1. Code in both the Packer and Vagrant directories was modified
|
||||
* In this case, the CircleCI worker will execute `ci/circle_workflows/packer_and_vagrant_changes.sh`
|
||||
2. Code in neither the Packer and Vagrant directories was modified
|
||||
* In this case, the CircleCI worker will execute the default test `ci/circle_worker/vagrant_changes.sh`
|
||||
3. Code in only the Packer directory was modified
|
||||
* In this case, the CircleCI worker will execute `ci/circle_worker/packer_changes.sh`
|
||||
4. Code in only the Vagrant directory was modified
|
||||
* In this case, the CircleCI worker will execute `ci/circle_worker/vagrant_changes.sh`
|
||||
|
||||
## Test Case Walkthroughs
|
||||
|
||||
### packer_and_vagrant_changes.sh
|
||||
1. Spins up a single Packet server
|
||||
2. Bootstraps the Packet server by calling `ci/build_machine_bootstrap.sh` with no arguments
|
||||
3. Builds the Windows10 and Windows2016 images one at a time
|
||||
4. Moves the resulting boxes to the Boxes directory
|
||||
5. Brings each Vagrant host online one-by-one
|
||||
6. CircleCI records the build results from the Packet server
|
||||
|
||||
### vagrant_changes.sh
|
||||
1. Spins up a single Packet server
|
||||
2. Bootstraps the Packet server by calling `ci/build_machine_bootstrap.sh` with the `--vagrant-only` argument
|
||||
3. Downloads the pre-build Windows10 and Windows2016 boxes from https://detectionlab.network directly to the Boxes directory
|
||||
4. Brings each Vagrant host online one-by-one
|
||||
5. CircleCI records the build results from the Packet server
|
||||
|
||||
|
||||
### packer_changes.sh
|
||||
1. Spins up two separate Packet servers to allow the Packer boxes to be built in parallel
|
||||
2. Bootstraps each packet Server by calling `ci/build_machine_bootstrap.sh` with the `--packer-only` argument
|
||||
3. Starts the Packer build process on each server
|
||||
4. CircleCI records the build result from each Packet server
|
||||
|
||||
```
|
||||
+------------+
|
||||
| |
|
||||
| |
|
||||
| |
|
||||
| Github |
|
||||
| |
|
||||
| |
|
||||
+------+-----+
|
||||
|
|
||||
|
|
||||
| Pull Request
|
||||
|
|
||||
v
|
||||
+------+-----+
|
||||
| |
|
||||
| |
|
||||
| Circle |
|
||||
+----------------------------->| Worker |
|
||||
| | |
|
||||
| | |
|
||||
| | |
|
||||
| +------+-----+
|
||||
| |
|
||||
| | Code changes are evaluated
|
||||
| | to determine which test suite
|
||||
| | to run
|
||||
| |
|
||||
| v
|
||||
| +----------------+--------------+
|
||||
Circle Worker | | packer_and_vagrant_changes.sh |
|
||||
quries for | | vagrant_changes.sh |
|
||||
build results | | packer_changes.sh |
|
||||
| +----------------+--------------+
|
||||
| |
|
||||
| |
|
||||
| | |
|
||||
| |
|
||||
| |
|
||||
| |
|
||||
| |
|
||||
| | 1. Provision Packet server(s)
|
||||
| | 2. Copy repo to server
|
||||
| | 3. Run server bootstrap
|
||||
| | 4. Bootstrap calls build.sh with
|
||||
| | the appropriate arguments
|
||||
| |
|
||||
| |
|
||||
| +---------v---------+
|
||||
| | |
|
||||
| | |
|
||||
| | |
|
||||
+-------------------------->| Packet Server |
|
||||
| |
|
||||
| |
|
||||
| |
|
||||
+-------------------+
|
||||
|
||||
```
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
#! /bin/bash
|
||||
|
||||
# This script is run on the Packet.net baremetal server for CI tests.
|
||||
# This script will build the entire lab from scratch and takes 3-4 hours
|
||||
# on a Packet.net host
|
||||
# While building, the server will start a webserver on Port 80 that contains
|
||||
# the text "building". Once the test is completed, the text will be replaced
|
||||
# with "success" or "failed".
|
||||
|
||||
# Install Virtualbox 5.2
|
||||
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
|
||||
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
|
||||
apt-get update
|
||||
apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2
|
||||
|
||||
echo "building" > /var/www/html/index.html
|
||||
|
||||
# Set up firewall
|
||||
ufw allow ssh
|
||||
ufw allow http
|
||||
ufw default allow outgoing
|
||||
ufw --force enable
|
||||
|
||||
# Install Vagrant
|
||||
mkdir /opt/vagrant
|
||||
cd /opt/vagrant || exit 1
|
||||
wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb
|
||||
dpkg -i vagrant_2.0.2_x86_64.deb
|
||||
vagrant plugin install vagrant-reload
|
||||
|
||||
# Install Packer
|
||||
mkdir /opt/packer
|
||||
cd /opt/packer || exit 1
|
||||
wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip
|
||||
unzip packer_1.1.3_linux_amd64.zip
|
||||
cp packer /usr/local/bin/packer
|
||||
|
||||
# Make the packer images headless
|
||||
for file in $(ls *.json); do
|
||||
sed -i 's/"headless": false,/"headless": true,/g' "$file";
|
||||
done
|
||||
|
||||
# Make the Vagrant instances headless
|
||||
cd /opt/DetectionLab/Vagrant || exit 1
|
||||
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
|
||||
|
||||
# Ensure the script is executable
|
||||
chmod +x /opt/DetectionLab/build.sh
|
||||
cd /opt/DetectionLab || exit 1
|
||||
|
||||
# Start the build in a tmux session
|
||||
sn=tmuxsession
|
||||
tmux new-session -s "$sn" -d
|
||||
tmux send-keys -t "$sn:0" './build.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
|
||||
@@ -1,42 +0,0 @@
|
||||
#! /bin/bash
|
||||
|
||||
# This script is run on the Packet.net baremetal server for CI tests.
|
||||
# This script will bootstrap the build by downloading pre-build Packer boxes
|
||||
# and should take no more than 90 minutes on a Packet.net host.
|
||||
# While building, the server will start a webserver on Port 80 that contains
|
||||
# the text "building". Once the test is completed, the text will be replaced
|
||||
# with "success" or "failed".
|
||||
|
||||
# Install Virtualbox 5.2
|
||||
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
|
||||
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
|
||||
apt-get update
|
||||
apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2
|
||||
|
||||
echo "building" > /var/www/html/index.html
|
||||
|
||||
# Set up firewall
|
||||
ufw allow ssh
|
||||
ufw allow http
|
||||
ufw default allow outgoing
|
||||
ufw --force enable
|
||||
|
||||
# Install Vagrant
|
||||
mkdir /opt/vagrant
|
||||
cd /opt/vagrant || exit 1
|
||||
wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb
|
||||
dpkg -i vagrant_2.0.2_x86_64.deb
|
||||
vagrant plugin install vagrant-reload
|
||||
|
||||
# Make the Vagrant instances headless
|
||||
cd /opt/DetectionLab/Vagrant || exit 1
|
||||
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
|
||||
|
||||
# Ensure the script is executable
|
||||
chmod +x /opt/DetectionLab/build_vagrant_only.sh
|
||||
cd /opt/DetectionLab || exit 1
|
||||
|
||||
# Start the build in a tmux session
|
||||
sn=tmuxsession
|
||||
tmux new-session -s "$sn" -d
|
||||
tmux send-keys -t "$sn:0" './build_vagrant_only.sh virtualbox | tee -a /opt/DetectionLab/Vagrant/vagrant.log && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
|
||||
93
ci/build_machine_bootstrap.sh
Executable file
93
ci/build_machine_bootstrap.sh
Executable file
@@ -0,0 +1,93 @@
|
||||
#! /bin/bash
|
||||
|
||||
# This script is run on the Packet.net baremetal server for CI tests.
|
||||
# This script will build the entire lab from scratch and takes 3-4 hours
|
||||
# on a Packet.net host
|
||||
# While building, the server will start a webserver on Port 80 that contains
|
||||
# the text "building". Once the test is completed, the text will be replaced
|
||||
# with "success" or "failed".
|
||||
|
||||
ARGS="$1"
|
||||
PACKER_ONLY=0
|
||||
VAGRANT_ONLY=0
|
||||
|
||||
if [ ! -z "$1" ]; then
|
||||
case "$1" in
|
||||
--packer-only)
|
||||
PACKER_ONLY=1
|
||||
;;
|
||||
--vagrant-only)
|
||||
VAGRANT_ONLY=1
|
||||
;;
|
||||
*)
|
||||
echo "\"$ARGS\" is not a supported argument to this script. Quitting"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
echo "Args: $ARGS"
|
||||
|
||||
if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then
|
||||
echo "Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install Virtualbox 5.2
|
||||
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
|
||||
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
|
||||
apt-get update
|
||||
apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2
|
||||
|
||||
echo "building" > /var/www/html/index.html
|
||||
|
||||
# Set up firewall
|
||||
ufw allow ssh
|
||||
ufw allow http
|
||||
ufw default allow outgoing
|
||||
ufw --force enable
|
||||
|
||||
if [ "$PACKER_ONLY" -eq 0 ]; then
|
||||
# Install Vagrant
|
||||
mkdir /opt/vagrant
|
||||
cd /opt/vagrant || exit 1
|
||||
wget https://releases.hashicorp.com/vagrant/2.0.3/vagrant_2.0.3_x86_64.deb
|
||||
dpkg -i vagrant_2.0.3_x86_64.deb
|
||||
vagrant plugin install vagrant-reload
|
||||
|
||||
# Make the Vagrant instances headless
|
||||
cd /opt/DetectionLab/Vagrant || exit 1
|
||||
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
|
||||
fi
|
||||
|
||||
if [ "$VAGRANT_ONLY" -eq 0 ]; then
|
||||
# Install Packer
|
||||
mkdir /opt/packer
|
||||
cd /opt/packer || exit 1
|
||||
wget https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip
|
||||
unzip packer_1.1.3_linux_amd64.zip
|
||||
cp packer /usr/local/bin/packer
|
||||
|
||||
# Make the Packer images headless
|
||||
cd /opt/DetectionLab/Packer || exit 1
|
||||
for file in *.json; do
|
||||
sed -i 's/"headless": false,/"headless": true,/g' "$file";
|
||||
done
|
||||
fi
|
||||
|
||||
# Ensure the script is executable
|
||||
chmod +x /opt/DetectionLab/build.sh
|
||||
cd /opt/DetectionLab || exit 1
|
||||
|
||||
# Start the build in a tmux session
|
||||
sn=tmuxsession
|
||||
tmux new-session -s "$sn" -d
|
||||
if [ "$PACKER_ONLY" -eq 1 ]; then
|
||||
tmux send-keys -t "$sn:0" './build.sh virtualbox --packer-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
|
||||
fi
|
||||
if [ "$VAGRANT_ONLY" -eq 1 ]; then
|
||||
tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
|
||||
fi
|
||||
if [[ "$PACKER_ONLY" -eq 0 ]] && [[ "$VAGRANT_ONLY" -eq 0 ]]; then
|
||||
tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
|
||||
fi
|
||||
80
ci/circle_workflows/packer_and_vagrant_changes.sh
Normal file
80
ci/circle_workflows/packer_and_vagrant_changes.sh
Normal file
@@ -0,0 +1,80 @@
|
||||
#! /bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# Create artifacts directory
|
||||
if [ ! -d "/tmp/artifacts" ]; then
|
||||
mkdir /tmp/artifacts
|
||||
fi
|
||||
|
||||
## Delete stale servers if they exist
|
||||
DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
|
||||
if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then
|
||||
echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID"
|
||||
curl -X DELETE -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
|
||||
fi
|
||||
|
||||
## Provision a Type1 baremetal Packet.net server
|
||||
echo "Provisioning a server on Packet.net"
|
||||
DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
|
||||
# Make sure the device ID is sane.
|
||||
# TODO: maybe make this a regex
|
||||
if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
|
||||
echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
|
||||
exit 1
|
||||
fi
|
||||
echo "Server successfully provisioned with ID: $DEVICE_ID"
|
||||
|
||||
echo "Sleeping 10 minutes to wait for Packet server to be provisioned"
|
||||
sleep 300
|
||||
echo "Sleeping 5 more minutes (CircleCI Keepalive)"
|
||||
sleep 300
|
||||
|
||||
## Recording the IP address of the newly provisioned Packet server
|
||||
IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
|
||||
|
||||
# Copy repo to Packet server
|
||||
# TODO: Tar up the repo and expand it remotely
|
||||
cd ~/repo
|
||||
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
|
||||
|
||||
## Running install script on Packet server
|
||||
ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh
|
||||
|
||||
echo "Sleeping 5 minutes to allow the build process to start"
|
||||
sleep 300
|
||||
|
||||
## Waiting for Packet server to post build results
|
||||
MINUTES_PAST=0
|
||||
while [ "$MINUTES_PAST" -lt 400 ]; do
|
||||
STATUS=$(curl $IP_ADDRESS)
|
||||
if [ "$STATUS" == "building" ]; then
|
||||
echo "$STATUS"
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present"
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
|
||||
sleep 300
|
||||
((MINUTES_PAST += 5))
|
||||
else
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$MINUTES_PAST" -gt 400 ]; then
|
||||
echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present"
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
## Recording the build results
|
||||
echo $STATUS
|
||||
if [ "$STATUS" != "success" ]; then
|
||||
echo "Build failed. Cleaning up server with ID $DEVICE_ID"
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "Vagrant log not yet present"
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
|
||||
exit 1
|
||||
fi
|
||||
echo "Build was successful. Cleaning up server with ID $DEVICE_ID"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
|
||||
exit 0
|
||||
99
ci/circle_workflows/packer_changes.sh
Normal file
99
ci/circle_workflows/packer_changes.sh
Normal file
@@ -0,0 +1,99 @@
|
||||
#! /bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# Create artifacts directory
|
||||
if [ ! -d "/tmp/artifacts" ]; then
|
||||
mkdir /tmp/artifacts
|
||||
fi
|
||||
|
||||
## Delete stale servers if they exist
|
||||
echo "Deleting stale Packet.net servers"
|
||||
DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
|
||||
if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then
|
||||
echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID"
|
||||
curl -X DELETE -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
|
||||
fi
|
||||
|
||||
## Provision two Type1 baremetal Packet.net servers
|
||||
echo "Provisioning packerwindows2016 on Packet.net"
|
||||
SERVER1_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows2016", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
|
||||
if [ "$(echo -n $SERVER1_ID | wc -c)" -ne 36 ]; then
|
||||
echo "Server may have failed provisionining. Device ID is set to: $SERVER1_ID"
|
||||
exit 1
|
||||
fi
|
||||
echo "packerwindows2016 successfully provisioned with ID: $SERVER1_ID"
|
||||
|
||||
sleep 5 # Wait a bit before issuing another provision command
|
||||
|
||||
echo "Provisioning packerwindows10 on Packet.net"
|
||||
SERVER2_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows10", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
|
||||
if [ "$(echo -n $SERVER2_ID | wc -c)" -ne 36 ]; then
|
||||
echo "Server may have failed provisionining. Device ID is set to: $SERVER2_ID"
|
||||
exit 1
|
||||
fi
|
||||
echo "packerwindows10 successfully provisioned with ID: $SERVER2_ID"
|
||||
|
||||
echo "Sleeping 10 minutes to wait for Packet servers to finish provisiong"
|
||||
sleep 300
|
||||
echo "Sleeping 5 more minutes (CircleCI Keepalive)"
|
||||
sleep 300
|
||||
|
||||
## Recording the IP address of the newly provisioned Packet servers
|
||||
SERVER1_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER1_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
|
||||
SERVER2_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER2_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
|
||||
|
||||
# Copy repo to Packet servers
|
||||
# TODO: Tar up the repo and expand it remotely
|
||||
cd ~/repo
|
||||
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab
|
||||
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab
|
||||
|
||||
## Running install script on Packet server
|
||||
ssh -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only
|
||||
ssh -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only
|
||||
|
||||
sleep 30
|
||||
|
||||
## Waiting for Packet server to post build results
|
||||
MINUTES_PAST=0
|
||||
while [ "$MINUTES_PAST" -lt 150 ]; do
|
||||
SERVER1_STATUS=$(curl $SERVER1_IP_ADDRESS)
|
||||
SERVER2_STATUS=$(curl $SERVER2_IP_ADDRESS)
|
||||
if [[ "$SERVER1_STATUS" == "building" ]] || [[ "$SERVER2_STATUS" == "building" ]]; then
|
||||
echo "$SERVER1_STATUS" :: "$SERVER2_STATUS"
|
||||
scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log
|
||||
scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log
|
||||
sleep 300
|
||||
((MINUTES_PAST += 5))
|
||||
fi
|
||||
if [[ "$SERVER1_STATUS" != "building" ]] && [[ "$SERVER2_STATUS" != "building" ]]; then
|
||||
break
|
||||
fi
|
||||
if [ "$MINUTES_PAST" -gt 150 ]; then
|
||||
echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
## Recording the build results
|
||||
echo "Server1 Status: $SERVER1_STATUS"
|
||||
echo "Server2 Status: $SERVER2_STATUS"
|
||||
if [ "$SERVER1_STATUS" != "success" ]; then
|
||||
echo "Build failed. Cleaning up server with ID $SERVER1_ID"
|
||||
scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log || echo "Serveer1 packer_build.log not available yet"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$SERVER2_STATUS" != "success" ]; then
|
||||
echo "Build failed. Cleaning up server with ID $SERVER2_ID"
|
||||
scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log || echo "Server2 packer_build.log not available yet"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
|
||||
exit 1
|
||||
fi
|
||||
echo "Builds were successful. Cleaning up servers with IDs $SERVER1_ID and $SERVER2_ID"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
|
||||
exit 0
|
||||
74
ci/circle_workflows/vagrant_changes.sh
Normal file
74
ci/circle_workflows/vagrant_changes.sh
Normal file
@@ -0,0 +1,74 @@
|
||||
#! /bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# Create artifacts directory
|
||||
if [ ! -d "/tmp/artifacts" ]; then
|
||||
mkdir /tmp/artifacts
|
||||
fi
|
||||
|
||||
## Delete stale servers if they exist
|
||||
DELETE_DEVICE_ID=$(curl -X GET -s --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."devices[0].id" | tr -d '"')
|
||||
if [ "$(echo -n $DELETE_DEVICE_ID | wc -c)" -eq 36 ]; then
|
||||
echo "Requesting deletion for Packet server with ID $DELETE_DEVICE_ID"
|
||||
curl -X -s DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DELETE_DEVICE_ID"
|
||||
fi
|
||||
|
||||
## Provision a Type1 baremetal Packet.net server
|
||||
echo "Provisioning a server on Packet.net"
|
||||
DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
|
||||
# Make sure the device ID is sane.
|
||||
# TODO: maybe make this a regex
|
||||
if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
|
||||
echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
|
||||
exit 1
|
||||
fi
|
||||
echo "Server successfully provisioned with ID: $DEVICE_ID"
|
||||
|
||||
echo "Sleeping 10 minutes to wait for Packet server to be provisioned"
|
||||
sleep 300
|
||||
echo "Sleeping 5 more minutes (CircleCI Keepalive)"
|
||||
sleep 300
|
||||
|
||||
## Recording the IP address of the newly provisioned Packet server
|
||||
IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
|
||||
|
||||
# Copy repo to Packet server
|
||||
# TODO: Tar up the repo and expand it remotely
|
||||
cd ~/repo
|
||||
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
|
||||
|
||||
## Running install script on Packet server
|
||||
ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --vagrant-only
|
||||
|
||||
## Waiting for Packet server to post build results
|
||||
MINUTES_PAST=0
|
||||
while [ "$MINUTES_PAST" -lt 120 ]; do
|
||||
STATUS=$(curl $IP_ADDRESS)
|
||||
if [ "$STATUS" == "building" ]; then
|
||||
echo "$STATUS"
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log || echo "vagrant_build.log not available yet"
|
||||
sleep 300
|
||||
((MINUTES_PAST += 5))
|
||||
else
|
||||
break
|
||||
fi
|
||||
if [ "$MINUTES_PAST" -gt 120 ]; then
|
||||
echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
## Recording the build results
|
||||
echo $STATUS
|
||||
if [ "$STATUS" != "success" ]; then
|
||||
scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_build.log /tmp/artifacts/vagrant_build.log
|
||||
echo "Build failed. Cleaning up server with ID $DEVICE_ID"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
|
||||
exit 1
|
||||
fi
|
||||
echo "Build was successful. Cleaning up server with ID $DEVICE_ID"
|
||||
curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user