trinitor/DetectionLab
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Simplify CI build and support Packet external storage

Browse Source
This commit is contained in:
Chris Long
2019-05-20 00:01:25 -07:00
parent 2757ca9bbc
commit 21df017d54
7 changed files with 192 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
.circleci/config.yml
View File

@@ -10,67 +10,127 @@ jobs:
steps: steps:
- checkout - checkout
- run: - run:
name: Choose which test suite to run based on which files were modified name: Create directory for artifacts
command: | command: |
## As far as I'm aware, there are three possibile cases to check for here. The CIRCLE_COMPARE_URL environment variable will either contain: # Create artifacts directory
## - A SHA1 hash for the commit (If there is only a single commit on the branch) if [ ! -d "/tmp/artifacts" ]; then
## - Two SHA1 hashes in a "xxxxxxxx..yyyyyyyy" (comparing the two most recent commits mkdir /tmp/artifacts
## - Nothing fi
## There is certainly a better way to accomplish all of this, but my limited knowledge of git has lead me to write this awful spaghetti code :-/
## We'll handle all 3 of these cases below: - run:
## Checking commits for changes to Packer files name: Creating a Packet Server
COMMIT_SHA1="" command: |
PACKER_MODIFIED=0 ## Provision a Type1 baremetal Packet.net server
VAGRANT_MODIFIED=0 echo "[$(date +%H:%M:%S)]: Provisioning a server on Packet.net"
## Check for empty CIRCLE_COMPARE_URL. If it is, set the SHA1 hash to the CIRCLE_SHA1 environment variable DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
if [ "$(echo -n $CIRCLE_COMPARE_URL | wc -c)" -eq 0 ]; then echo $DEVICE_ID > /tmp/device_id
export COMMIT_SHA1=$CIRCLE_SHA1 # Make sure the device ID is sane.
## Check for two short-hashes in the CIRCLE_COMPARE_URL by searching for '..'. If it exists, use the second short-hash # TODO: maybe make this a regex
elif echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
export COMMIT_SHA1="$(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5)" echo "[$(date +%H:%M:%S)]: Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
## Check for a single short hash in the CIRCLE_COMPARE_URL and use it if it exists echo "[$(date +%H:%M:%S)]: This usually happens if there are no servers available in the selected datacenter."
## TODO: This check may not be needed. echo "[$(date +%H:%M:%S)]: Attempting to retry in another datacenter..."
export DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "ewr1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
echo "[$(date +%H:%M:%S)]: This script was still unable to successfully provision a server. Exiting."
exit 1
fi
fi
echo "[$(date +%H:%M:%S)]: Server successfully created with ID: $DEVICE_ID"
- run:
name: Waiting for Packet server provisioning to complete
command: |
DEVICE_ID=$(cat /tmp/device_id)
echo "[$(date +%H:%M:%S)]: Waiting for server to finish provisioning..."
# Continue to poll the API until the state of the host is "active"
export STATE="provisioning"
while [ "$STATE" != "active" ]; do
sleep 10
echo "[$(date +%H:%M:%S)]: Sleeping for 10 seconds. Server is still $STATE."
export STATE="$(curl -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID" | jq .state | tr -d '"')"
done
echo "[$(date +%H:%M:%S)]: Device with ID $DEVICE_ID has finished provisioning! Onto the build process..."
- run:
name: Mount external storage
command: |
DEVICE_ID=$(cat /tmp/device_id)
## Mount external storage containing Vagrant boxes
echo "[$(date +%H:%M:%S)]: Attempting to mount external storage to this server..."
MOUNT_STATUS=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{"device_id": "'"$DEVICE_ID"'"}' 'https://api.packet.net/storage/fed37d73-6719-451a-9160-df8b0addc915/attachments' | jq .id | wc -c)
# Stupid check to make sure MOUNT_STATUS contains a UUID
if [ "$MOUNT_STATUS" != "39" ]; then
echo "[$(date +%H:%M:%S)]: Mounting may have failed. ID is $MOUNT_STATUS"
else else
export COMMIT_SHA1="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" echo "[$(date +%H:%M:%S)]: External storage successfully mounted!"
fi fi
## Display the files that were modified in this branch
echo "Files modified since origin/Master:" - run:
git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" name: Record the IP address of the Packet server
## Check to see if Packer files were modified command: |
if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" | grep -c ^Packer/)" -gt 0 ]; then ## Recording the IP address of the newly provisioned Packet server
export PACKER_MODIFIED=1 DEVICE_ID=$(cat /tmp/device_id)
fi IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" | grep -c ^Vagrant/)" -gt 0 ]; then echo $IP_ADDRESS > /tmp/ip_address
export VAGRANT_MODIFIED=1
fi - run:
echo "Displaying the values of the modifier environment variables:" name: SCP Repo to Packet Server
echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED" command: |
echo "PACKER_MODIFIED=$PACKER_MODIFIED" # Copy repo to Packet server
## Choosing which test suite to run based on the files that were changed # TODO: Tar up the repo and expand it remotely
if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then IP_ADDRESS=$(cat /tmp/ip_address)
echo "Running the test suite for Packer and Vagrant changes" cd ~/repo && rsync -Paq -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
chmod +x ci/circle_workflows/packer_and_vagrant_changes.sh
ci/circle_workflows/packer_and_vagrant_changes.sh - run:
exit 0 name: Run the build machine bootstrap script
fi command: |
if [[ "$PACKER_MODIFIED" -eq 0 ]] && [[ "$VAGRANT_MODIFIED" -eq 0 ]]; then IP_ADDRESS=$(cat /tmp/ip_address)
echo "Running the default test suite (Vagrant-only)" ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh
chmod +x ci/circle_workflows/vagrant_changes.sh
ci/circle_workflows/vagrant_changes.sh - run:
exit 0 name: Wait for build results
fi command: |
if [ "$PACKER_MODIFIED" -eq 1 ]; then export MINUTES_PAST=0
echo "Running the test suite for Packer-only changes" IP_ADDRESS=$(cat /tmp/ip_address)
chmod +x ci/circle_workflows/packer_changes.sh DEVICE_ID=$(cat /tmp/device_id)
ci/circle_workflows/packer_changes.sh while [ "$MINUTES_PAST" -le 240 ]; do
exit 0 export STATUS=$(curl -s $IP_ADDRESS)
fi if [ "$STATUS" == "building" ]; then
if [ "$VAGRANT_MODIFIED" -eq 1 ]; then echo "[$(date +%H:%M:%S)]: $STATUS"
echo "Running the test suite for Vagrant-only changes" scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present"
chmod +x ci/circle_workflows/vagrant_changes.sh sleep 300
ci/circle_workflows/vagrant_changes.sh ((MINUTES_PAST += 5))
exit 0 else
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present"
echo "$STATUS" > /tmp/status
break
fi
if [ "$MINUTES_PAST" -gt 240 ]; then
echo "[$(date +%H:%M:%S)]: Serer timed out. Uptime: $MINUTES_PAST minutes."
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
done
- run:
name: Wait for build results
command: |
## Recording the build results
STATUS=$(cat /tmp/status)
IP_ADDRESS=$(cat /tmp/ip_address)
DEVICE_ID=$(cat /tmp/device_id)
echo "[$(date +%H:%M:%S)]: $STATUS"
if [ "$STATUS" != "success" ]; then
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/
echo "Build failed. Cleaning up server with ID $DEVICE_ID"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"'?force_delete=true'
exit 1
fi fi
echo "[$(date +%H:%M:%S)]: Build was successful. Cleaning up server with ID $DEVICE_ID"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"'?force_delete=true'
exit 0
- store_artifacts: - store_artifacts:
path: /tmp/artifacts path: /tmp/artifacts

139
ci/build_machine_bootstrap.sh
View File

@@ -1,42 +1,59 @@
#! /bin/bash #! /bin/bash
# This script is run on the Packet.net baremetal server for CI tests. # This script is run on the Packet.net baremetal server for CI tests.
# This script will build the entire lab from scratch and takes 3-4 hours
# on a Packet.net host
# While building, the server will start a webserver on Port 80 that contains # While building, the server will start a webserver on Port 80 that contains
# the text "building". Once the test is completed, the text will be replaced # the text "building". Once the test is completed, the text will be replaced
# with "success" or "failed". # with "success" or "failed".
ARGS="$1" # Download Packet.net storage utilities
PACKER_ONLY=0 echo "[$(date +%H:%M:%S)]: Downloading Packet external storage utilities..."
VAGRANT_ONLY=0 wget -q -O /usr/local/bin/packet-block-storage-attach "https://raw.githubusercontent.com/packethost/packet-block-storage/master/packet-block-storage-attach"
chmod +x /usr/local/bin/packet-block-storage-attach
wget -q -O /usr/local/bin/packet-block-storage-detach "https://raw.githubusercontent.com/packethost/packet-block-storage/master/packet-block-storage-detach"
chmod +x /usr/local/bin/packet-block-storage-detach
if [ ! -z "$1" ]; then # Set a flag to determine if the boxes are available on external Packet storage
case "$1" in BOXES_PRESENT=0
--packer-only) # Attempt to mount the block storage
PACKER_ONLY=1 echo "[$(date +%H:%M:%S)]: Attempting to mount external storage..."
;; /usr/local/bin/packet-block-storage-attach
--vagrant-only) sleep 10
VAGRANT_ONLY=1 # Check if it was successful by looking for volume* in /dev/mapper
;; if ls -al /dev/mapper/volume* > /dev/null 2>&1; then
*) echo "[$(date +%H:%M:%S)]: Mounting of external storage was successful."
echo "\"$ARGS\" is not a supported argument to this script. Quitting" sleep 5
exit 1 if mount /dev/mapper/volume-fed37d73-part1 /mnt; then
;; echo "[$(date +%H:%M:%S)]: External storage successfully mounted to /mnt"
esac else
echo "[$(date +%H:%M:%S)]: Something went wrong mounting the filesystem from the external storage."
fi
if ls -al /mnt/*.box > /dev/null 2>&1; then
BOXES_PRESENT=1
fi
else
echo "[$(date +%H:%M:%S)]: No volumes found after attempting to mount storage. Trying again..."
/usr/local/bin/packet-block-storage-attach
sleep 15
if ! ls -al /dev/mapper/volume* > /dev/null 2>&1; then
echo "[$(date +%H:%M:%S)]: Failed to mount volumes even after a retry. Giving up..."
else
echo "[$(date +%H:%M:%S)]: Successfully mounted the external storage after a retry."
sleep 10
if mount /dev/mapper/volume-fed37d73-part1 /mnt; then
echo "[$(date +%H:%M:%S)]: External storage successfully mounted to /mnt"
else
echo "[$(date +%H:%M:%S)]: Something went wrong mounting the filesystem from the external storage."
fi
if ls -al /mnt/*.box > /dev/null 2>&1; then
BOXES_PRESENT=1
fi
fi
fi fi
echo "Args: $ARGS"
# Disable IPv6 - may help with the vagrant-reload plugin: https://github.com/hashicorp/vagrant/issues/8795#issuecomment-468945063 # Disable IPv6 - may help with the vagrant-reload plugin: https://github.com/hashicorp/vagrant/issues/8795#issuecomment-468945063
echo "net.ipv6.conf.all.disable_ipv6=1" >> /etc/sysctl.conf echo "net.ipv6.conf.all.disable_ipv6=1" >> /etc/sysctl.conf
sysctl -p /etc/sysctl.conf > /dev/null sysctl -p /etc/sysctl.conf > /dev/null
if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then
echo "[$(date +%H:%M:%S)]: Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong."
exit 1
fi
# Install Virtualbox 5.2 # Install Virtualbox 5.2
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
sed -i "2ideb mirror://mirrors.ubuntu.com/mirrors.txt xenial main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-updates main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-backports main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-security main restricted universe multiverse" /etc/apt/sources.list sed -i "2ideb mirror://mirrors.ubuntu.com/mirrors.txt xenial main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-updates main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-backports main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-security main restricted universe multiverse" /etc/apt/sources.list
@@ -54,60 +71,42 @@ ufw allow http
ufw default allow outgoing ufw default allow outgoing
ufw --force enable ufw --force enable
if [ "$PACKER_ONLY" -eq 0 ]; then # Install Vagrant
# Install Vagrant echo "[$(date +%H:%M:%S)]: Installing Vagrant..."
echo "[$(date +%H:%M:%S)]: Installing Vagrant..." mkdir /opt/vagrant
mkdir /opt/vagrant cd /opt/vagrant || exit 1
cd /opt/vagrant || exit 1 wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.4/vagrant_2.2.4_x86_64.deb
wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.4/vagrant_2.2.4_x86_64.deb dpkg -i vagrant_2.2.4_x86_64.deb
dpkg -i vagrant_2.2.4_x86_64.deb echo "[$(date +%H:%M:%S)]: Installing vagrant-reload plugin..."
echo "[$(date +%H:%M:%S)]: Installing vagrant-reload plugin..." vagrant plugin install vagrant-reload
# Make sure the plugin installed correctly. Retry if not.
if [ "$(vagrant plugin list | grep -c vagrant-reload)" -ne "1" ]; then
echo "[$(date +%H:%M:%S)]: The first attempt to install the vagrant-reload plugin failed. Trying again."
vagrant plugin install vagrant-reload vagrant plugin install vagrant-reload
# Make sure the plugin installed correctly. Retry if not.
if [ "$(vagrant plugin list | grep -c vagrant-reload)" -ne "1" ]; then
echo "[$(date +%H:%M:%S)]: The first attempt to install the vagrant-reload plugin failed. Trying again."
vagrant plugin install vagrant-reload
fi
# Re-enable IPv6 - may help with the Vagrant Cloud slowness
echo "net.ipv6.conf.all.disable_ipv6=0" >> /etc/sysctl.conf
sysctl -p /etc/sysctl.conf > /dev/null
# Make the Vagrant instances headless
cd /opt/DetectionLab/Vagrant || exit 1
sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
fi fi
if [ "$VAGRANT_ONLY" -eq 0 ]; then # Re-enable IPv6 - may help with the Vagrant Cloud slowness
echo "[$(date +%H:%M:%S)]: Installing Packer..." echo "net.ipv6.conf.all.disable_ipv6=0" >> /etc/sysctl.conf
# Install Packer sysctl -p /etc/sysctl.conf > /dev/null
mkdir /opt/packer
cd /opt/packer || exit 1
wget --progress=bar:force https://releases.hashicorp.com/packer/1.4.0/packer_1.4.0_linux_amd64.zip
unzip packer_1.4.0_linux_amd64.zip
cp packer /usr/local/bin/packer
# Make the Packer images headless # Make the Vagrant instances headless
cd /opt/DetectionLab/Packer || exit 1 cd /opt/DetectionLab/Vagrant || exit 1
for file in *.json; do sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
sed -i 's/"headless": false,/"headless": true,/g' "$file";
done # If the boxes are present on external storage, we can modify the Vagrantfile to
# point to the boxes on disk so we don't have to download them
if [ $BOXES_PRESENT -eq 1 ]; then
echo "[$(date +%H:%M:%S)]: Updating the Vagrantfile to point to the boxes mounted on external storage..."
sed -i 's#"detectionlab/win2016"#"/mnt/windows_2016_virtualbox.box"#g' /opt/DetectionLab/Vagrant/Vagrantfile
sed -i 's#"detectionlab/win10"#"/mnt/windows_10_virtualbox.box"#g' /opt/DetectionLab/Vagrant/Vagrantfile
fi fi
# Ensure the script is executable # Make the build script is executable
chmod +x /opt/DetectionLab/build.sh chmod +x /opt/DetectionLab/build.sh
cd /opt/DetectionLab || exit 1 cd /opt/DetectionLab || exit 1
# Start the build in a tmux session # Start the build in a tmux session
sn=tmuxsession sn=tmuxsession
tmux new-session -s "$sn" -d tmux new-session -s "$sn" -d
if [ "$PACKER_ONLY" -eq 1 ]; then tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html; umount /mnt && /usr/local/bin/packet-block-storage-detach' Enter
tmux send-keys -t "$sn:0" './build.sh virtualbox --packer-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
fi
if [ "$VAGRANT_ONLY" -eq 1 ]; then
tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
fi
if [[ "$PACKER_ONLY" -eq 0 ]] && [[ "$VAGRANT_ONLY" -eq 0 ]]; then
tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
fi

73
ci/circle_workflows/packer_and_vagrant_changes.sh
View File

@@ -1,73 +0,0 @@
#! /bin/bash
set -e
# Create artifacts directory
if [ ! -d "/tmp/artifacts" ]; then
mkdir /tmp/artifacts
fi
## Provision a Type1 baremetal Packet.net server
echo "Provisioning a server on Packet.net"
DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
# Make sure the device ID is sane.
# TODO: maybe make this a regex
if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
exit 1
fi
echo "Server successfully provisioned with ID: $DEVICE_ID"
echo "Sleeping 10 minutes to wait for Packet server to be provisioned"
sleep 300
echo "Sleeping 5 more minutes (CircleCI Keepalive)"
sleep 300
## Recording the IP address of the newly provisioned Packet server
IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
# Copy repo to Packet server
# TODO: Tar up the repo and expand it remotely
cd ~/repo
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
## Running install script on Packet server
ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh
echo "Sleeping 5 minutes to allow the build process to start"
sleep 300
## Waiting for Packet server to post build results
MINUTES_PAST=0
while [ "$MINUTES_PAST" -lt 400 ]; do
STATUS=$(curl $IP_ADDRESS)
if [ "$STATUS" == "building" ]; then
echo "$STATUS"
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/|| echo "Vagrant log not yet present"
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
sleep 300
((MINUTES_PAST += 5))
else
break
fi
done
if [ "$MINUTES_PAST" -gt 400 ]; then
echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present"
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
## Recording the build results
echo $STATUS
if [ "$STATUS" != "success" ]; then
echo "Build failed. Cleaning up server with ID $DEVICE_ID"
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present"
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
echo "Build was successful. Cleaning up server with ID $DEVICE_ID"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 0

91
ci/circle_workflows/packer_changes.sh
View File

@@ -1,91 +0,0 @@
#! /bin/bash
set -e
# Create artifacts directory
if [ ! -d "/tmp/artifacts" ]; then
mkdir /tmp/artifacts
fi
## Provision two Type1 baremetal Packet.net servers
echo "Provisioning packerwindows2016 on Packet.net"
SERVER1_ID=$(curl -s -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows2016", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
if [ "$(echo -n $SERVER1_ID | wc -c)" -ne 36 ]; then
echo "Server may have failed provisionining. Device ID is set to: $SERVER1_ID"
exit 1
fi
echo "packerwindows2016 successfully provisioned with ID: $SERVER1_ID"
sleep 5 # Wait a bit before issuing another provision command
echo "Provisioning packerwindows10 on Packet.net"
SERVER2_ID=$(curl -s -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows10", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
if [ "$(echo -n $SERVER2_ID | wc -c)" -ne 36 ]; then
echo "Server may have failed provisionining. Device ID is set to: $SERVER2_ID"
exit 1
fi
echo "packerwindows10 successfully provisioned with ID: $SERVER2_ID"
echo "Sleeping 10 minutes to wait for Packet servers to finish provisiong"
sleep 300
echo "Sleeping 5 more minutes (CircleCI Keepalive)"
sleep 300
## Recording the IP address of the newly provisioned Packet servers
SERVER1_IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER1_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
SERVER2_IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER2_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
# Copy repo to Packet servers
# TODO: Tar up the repo and expand it remotely
cd ~/repo
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab
rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab
## Running install script on Packet server
ssh -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only
ssh -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only
sleep 30
## Waiting for Packet server to post build results
MINUTES_PAST=0
while [ "$MINUTES_PAST" -lt 150 ]; do
SERVER1_STATUS=$(curl $SERVER1_IP_ADDRESS)
SERVER2_STATUS=$(curl $SERVER2_IP_ADDRESS)
if [[ "$SERVER1_STATUS" == "building" ]] || [[ "$SERVER2_STATUS" == "building" ]]; then
echo "$SERVER1_STATUS" :: "$SERVER2_STATUS"
scp -q -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log
scp -q -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log
sleep 300
((MINUTES_PAST += 5))
fi
if [[ "$SERVER1_STATUS" != "building" ]] && [[ "$SERVER2_STATUS" != "building" ]]; then
break
fi
if [ "$MINUTES_PAST" -gt 150 ]; then
echo "Serer timed out. Uptime: $MINUTES_PAST minutes."
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
exit 1
fi
done
## Recording the build results
echo "Server1 Status: $SERVER1_STATUS"
echo "Server2 Status: $SERVER2_STATUS"
if [ "$SERVER1_STATUS" != "success" ]; then
echo "Build failed. Cleaning up server with ID $SERVER1_ID"
scp -q -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log || echo "Serveer1 packer_build.log not available yet"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
exit 1
fi
if [ "$SERVER2_STATUS" != "success" ]; then
echo "Build failed. Cleaning up server with ID $SERVER2_ID"
scp -q -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log || echo "Server2 packer_build.log not available yet"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
exit 1
fi
echo "Builds were successful. Cleaning up servers with IDs $SERVER1_ID and $SERVER2_ID"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID"
exit 0

79
ci/circle_workflows/vagrant_changes.sh
View File

@@ -1,79 +0,0 @@
#! /bin/bash
set -e
# Create artifacts directory
if [ ! -d "/tmp/artifacts" ]; then
mkdir /tmp/artifacts
fi
## Provision a Type1 baremetal Packet.net server
echo "[$(date +%H:%M:%S)]: Provisioning a server on Packet.net"
DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
# Make sure the device ID is sane.
# TODO: maybe make this a regex
if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
echo "[$(date +%H:%M:%S)]: Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
echo "[$(date +%H:%M:%S)]: This usually happens if there are no servers available in the selected datacenter."
echo "[$(date +%H:%M:%S)]: Attempting to retry in another datacenter..."
DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "ewr1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
echo "[$(date +%H:%M:%S)]: This script was still unable to successfully provision a server. Exiting."
exit 1
fi
fi
echo "[$(date +%H:%M:%S)]: Server successfully created with ID: $DEVICE_ID"
echo "[$(date +%H:%M:%S)]: Waiting for server to finish provisioning..."
# Continue to poll the API until the state of the host is "active"
STATE="provisioning"
while [ "$STATE" != "active" ]; do
sleep 10
echo "[$(date +%H:%M:%S)]: Sleeping for 10 seconds. Server is still $STATE."
STATE="$(curl -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID" | jq .state | tr -d '"')"
done
echo "[$(date +%H:%M:%S)]: Device with ID $DEVICE_ID has finished provisioning! Onto the build process..."
## Recording the IP address of the newly provisioned Packet server
IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
# Copy repo to Packet server
# TODO: Tar up the repo and expand it remotely
cd ~/repo
rsync -Paq -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
## Running install script on Packet server
ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --vagrant-only
## Waiting for Packet server to post build results
MINUTES_PAST=0
while [ "$MINUTES_PAST" -lt 180 ]; do
STATUS=$(curl $IP_ADDRESS)
if [ "$STATUS" == "building" ]; then
echo "[$(date +%H:%M:%S)]: $STATUS"
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present"
sleep 300
((MINUTES_PAST += 5))
else
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present"
break
fi
if [ "$MINUTES_PAST" -gt 180 ]; then
echo "[$(date +%H:%M:%S)]: Serer timed out. Uptime: $MINUTES_PAST minutes."
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
done
## Recording the build results
echo "[$(date +%H:%M:%S)]: $STATUS"
if [ "$STATUS" != "success" ]; then
scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/
echo "Build failed. Cleaning up server with ID $DEVICE_ID"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 1
fi
echo "[$(date +%H:%M:%S)]: Build was successful. Cleaning up server with ID $DEVICE_ID"
curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"
exit 0

7
ci/manual_machine_bootstrap.sh
View File

@@ -4,18 +4,13 @@
sed -i 's/archive.ubuntu.com/us.archive.ubuntu.com/g' /etc/apt/sources.list sed -i 's/archive.ubuntu.com/us.archive.ubuntu.com/g' /etc/apt/sources.list
if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then
echo "Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong."
exit 1
fi
# Install Virtualbox 5.2 # Install Virtualbox 5.2
echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -
apt-get update apt-get update
apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2 python-pip apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2 python-pip
pip install awscli --upgrade --user pip install awscli --upgrade --user
export PATH=$PATH:/root/.local/bin cp /root/.local/bin/aws /usr/local/bin/aws && chmod +x /usr/local/bin/aws
# Set up firewall # Set up firewall
ufw allow ssh ufw allow ssh

14
ci/manual_machine_bootstrap_vmware.sh
View File

@@ -1,22 +1,18 @@
#! /bin/bash #! /bin/bash
# This script is used to manually prepare an Ubuntu 16.04 server for DetectionLab building # This script is used to manually prepare an Ubuntu 16.04 server for DetectionLab building
export DEBIAN_FRONTEND=noninteractive export DEBIAN_FRONTEND=noninteractive
export SERIALNUMBER="SECRET" export SERIALNUMBER="SECRET"
export LICENSEFILE="SECRET" export LICENSEFILE="SECRET"
sed -i 's/archive.ubuntu.com/us.archive.ubuntu.com/g' /etc/apt/sources.list sed -i 's/archive.ubuntu.com/us.archive.ubuntu.com/g' /etc/apt/sources.list
if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then
echo "Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong."
exit 1
fi
# Install VMWare Workstation 15 # Install VMWare Workstation 15
apt-get update apt-get update
apt-get install -y linux-headers-"$(uname -r)" build-essential unzip git ufw apache2 python-pip ubuntu-desktop apt-get install -y linux-headers-"$(uname -r)" build-essential unzip git ufw apache2 python-pip ubuntu-desktop python-pip
pip install awscli --upgrade --user pip install awscli --upgrade --user
export PATH=$PATH:/root/.local/bin cp /root/.local/bin/aws /usr/local/bin/aws && chmod +x /usr/local/bin/aws
wget -O VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle "https://download3.vmware.com/software/wkst/file/VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle?HashKey=6f83753e4d9e94da7f920c32b5808033&params=%7B%22custnumber%22%3A%22KipkcHRoJWVlZA%3D%3D%22%2C%22sourcefilesize%22%3A%22472.70+MB%22%2C%22dlgcode%22%3A%22WKST-1504-LX%22%2C%22languagecode%22%3A%22en%22%2C%22source%22%3A%22DOWNLOADS%22%2C%22downloadtype%22%3A%22manual%22%2C%22eula%22%3A%22Y%22%2C%22downloaduuid%22%3A%225caee685-d5ad-4f6b-94db-2ddc4f7f3a97%22%2C%22purchased%22%3A%22N%22%2C%22dlgtype%22%3A%22Product+Binaries%22%2C%22productversion%22%3A%2215.0.4%22%2C%22productfamily%22%3A%22VMware+Workstation+Pro%22%7D&AuthKey=1556427011_a994b5252f29429710c077c8dcab1c19" wget -O VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle "https://download3.vmware.com/software/wkst/file/VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle?HashKey=6f83753e4d9e94da7f920c32b5808033&params=%7B%22custnumber%22%3A%22KipkcHRoJWVlZA%3D%3D%22%2C%22sourcefilesize%22%3A%22472.70+MB%22%2C%22dlgcode%22%3A%22WKST-1504-LX%22%2C%22languagecode%22%3A%22en%22%2C%22source%22%3A%22DOWNLOADS%22%2C%22downloadtype%22%3A%22manual%22%2C%22eula%22%3A%22Y%22%2C%22downloaduuid%22%3A%225caee685-d5ad-4f6b-94db-2ddc4f7f3a97%22%2C%22purchased%22%3A%22N%22%2C%22dlgtype%22%3A%22Product+Binaries%22%2C%22productversion%22%3A%2215.0.4%22%2C%22productfamily%22%3A%22VMware+Workstation+Pro%22%7D&AuthKey=1556427011_a994b5252f29429710c077c8dcab1c19"
chmod +x VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle chmod +x VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle
@@ -48,8 +44,8 @@ sed -i 's/v.gui = true/v.gui = false/g' Vagrantfile
# Install Packer # Install Packer
mkdir /opt/packer mkdir /opt/packer
cd /opt/packer || exit 1 cd /opt/packer || exit 1
wget --progress=bar:force https://releases.hashicorp.com/packer/1.3.2/packer_1.3.2_linux_amd64.zip wget --progress=bar:force https://releases.hashicorp.com/packer/1.4.0/packer_1.4.0_linux_amd64.zip
unzip packer_1.3.2_linux_amd64.zip unzip packer_1.4.0_linux_amd64.zip
cp packer /usr/local/bin/packer cp packer /usr/local/bin/packer
# Make the Packer images headless # Make the Packer images headless