diff --git a/.circleci/config.yml b/.circleci/config.yml index fb7ec4f..af01275 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,67 +10,127 @@ jobs: steps: - checkout - run: - name: Choose which test suite to run based on which files were modified + name: Create directory for artifacts command: | - ## As far as I'm aware, there are three possibile cases to check for here. The CIRCLE_COMPARE_URL environment variable will either contain: - ## - A SHA1 hash for the commit (If there is only a single commit on the branch) - ## - Two SHA1 hashes in a "xxxxxxxx..yyyyyyyy" (comparing the two most recent commits - ## - Nothing - ## There is certainly a better way to accomplish all of this, but my limited knowledge of git has lead me to write this awful spaghetti code :-/ - ## We'll handle all 3 of these cases below: - ## Checking commits for changes to Packer files - COMMIT_SHA1="" - PACKER_MODIFIED=0 - VAGRANT_MODIFIED=0 - ## Check for empty CIRCLE_COMPARE_URL. If it is, set the SHA1 hash to the CIRCLE_SHA1 environment variable - if [ "$(echo -n $CIRCLE_COMPARE_URL | wc -c)" -eq 0 ]; then - export COMMIT_SHA1=$CIRCLE_SHA1 - ## Check for two short-hashes in the CIRCLE_COMPARE_URL by searching for '..'. If it exists, use the second short-hash - elif echo "$CIRCLE_COMPARE_URL" | grep '\.\.'; then - export COMMIT_SHA1="$(echo $CIRCLE_COMPARE_URL | cut -d '.' -f 5)" - ## Check for a single short hash in the CIRCLE_COMPARE_URL and use it if it exists - ## TODO: This check may not be needed. + # Create artifacts directory + if [ ! -d "/tmp/artifacts" ]; then + mkdir /tmp/artifacts + fi + + - run: + name: Creating a Packet Server + command: | + ## Provision a Type1 baremetal Packet.net server + echo "[$(date +%H:%M:%S)]: Provisioning a server on Packet.net" + DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') + echo $DEVICE_ID > /tmp/device_id + # Make sure the device ID is sane. + # TODO: maybe make this a regex + if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then + echo "[$(date +%H:%M:%S)]: Server may have failed provisionining. Device ID is set to: $DEVICE_ID" + echo "[$(date +%H:%M:%S)]: This usually happens if there are no servers available in the selected datacenter." + echo "[$(date +%H:%M:%S)]: Attempting to retry in another datacenter..." + export DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "ewr1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') + if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then + echo "[$(date +%H:%M:%S)]: This script was still unable to successfully provision a server. Exiting." + exit 1 + fi + fi + echo "[$(date +%H:%M:%S)]: Server successfully created with ID: $DEVICE_ID" + + - run: + name: Waiting for Packet server provisioning to complete + command: | + DEVICE_ID=$(cat /tmp/device_id) + echo "[$(date +%H:%M:%S)]: Waiting for server to finish provisioning..." + # Continue to poll the API until the state of the host is "active" + export STATE="provisioning" + while [ "$STATE" != "active" ]; do + sleep 10 + echo "[$(date +%H:%M:%S)]: Sleeping for 10 seconds. Server is still $STATE." + export STATE="$(curl -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID" | jq .state | tr -d '"')" + done + echo "[$(date +%H:%M:%S)]: Device with ID $DEVICE_ID has finished provisioning! Onto the build process..." + + - run: + name: Mount external storage + command: | + DEVICE_ID=$(cat /tmp/device_id) + ## Mount external storage containing Vagrant boxes + echo "[$(date +%H:%M:%S)]: Attempting to mount external storage to this server..." + MOUNT_STATUS=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{"device_id": "'"$DEVICE_ID"'"}' 'https://api.packet.net/storage/fed37d73-6719-451a-9160-df8b0addc915/attachments' | jq .id | wc -c) + # Stupid check to make sure MOUNT_STATUS contains a UUID + if [ "$MOUNT_STATUS" != "39" ]; then + echo "[$(date +%H:%M:%S)]: Mounting may have failed. ID is $MOUNT_STATUS" else - export COMMIT_SHA1="$(echo $CIRCLE_COMPARE_URL | cut -d '/' -f 7)" + echo "[$(date +%H:%M:%S)]: External storage successfully mounted!" fi - ## Display the files that were modified in this branch - echo "Files modified since origin/Master:" - git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" - ## Check to see if Packer files were modified - if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" | grep -c ^Packer/)" -gt 0 ]; then - export PACKER_MODIFIED=1 - fi - if [ "$(git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/HEAD) "$COMMIT_SHA1" | grep -c ^Vagrant/)" -gt 0 ]; then - export VAGRANT_MODIFIED=1 - fi - echo "Displaying the values of the modifier environment variables:" - echo "VAGRANT_MODIFIED=$VAGRANT_MODIFIED" - echo "PACKER_MODIFIED=$PACKER_MODIFIED" - ## Choosing which test suite to run based on the files that were changed - if [[ "$PACKER_MODIFIED" -eq 1 ]] && [[ "$VAGRANT_MODIFIED" -eq 1 ]]; then - echo "Running the test suite for Packer and Vagrant changes" - chmod +x ci/circle_workflows/packer_and_vagrant_changes.sh - ci/circle_workflows/packer_and_vagrant_changes.sh - exit 0 - fi - if [[ "$PACKER_MODIFIED" -eq 0 ]] && [[ "$VAGRANT_MODIFIED" -eq 0 ]]; then - echo "Running the default test suite (Vagrant-only)" - chmod +x ci/circle_workflows/vagrant_changes.sh - ci/circle_workflows/vagrant_changes.sh - exit 0 - fi - if [ "$PACKER_MODIFIED" -eq 1 ]; then - echo "Running the test suite for Packer-only changes" - chmod +x ci/circle_workflows/packer_changes.sh - ci/circle_workflows/packer_changes.sh - exit 0 - fi - if [ "$VAGRANT_MODIFIED" -eq 1 ]; then - echo "Running the test suite for Vagrant-only changes" - chmod +x ci/circle_workflows/vagrant_changes.sh - ci/circle_workflows/vagrant_changes.sh - exit 0 + + - run: + name: Record the IP address of the Packet server + command: | + ## Recording the IP address of the newly provisioned Packet server + DEVICE_ID=$(cat /tmp/device_id) + IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') + echo $IP_ADDRESS > /tmp/ip_address + + - run: + name: SCP Repo to Packet Server + command: | + # Copy repo to Packet server + # TODO: Tar up the repo and expand it remotely + IP_ADDRESS=$(cat /tmp/ip_address) + cd ~/repo && rsync -Paq -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab + + - run: + name: Run the build machine bootstrap script + command: | + IP_ADDRESS=$(cat /tmp/ip_address) + ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh + + - run: + name: Wait for build results + command: | + export MINUTES_PAST=0 + IP_ADDRESS=$(cat /tmp/ip_address) + DEVICE_ID=$(cat /tmp/device_id) + while [ "$MINUTES_PAST" -le 240 ]; do + export STATUS=$(curl -s $IP_ADDRESS) + if [ "$STATUS" == "building" ]; then + echo "[$(date +%H:%M:%S)]: $STATUS" + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" + sleep 300 + ((MINUTES_PAST += 5)) + else + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" + echo "$STATUS" > /tmp/status + break + fi + if [ "$MINUTES_PAST" -gt 240 ]; then + echo "[$(date +%H:%M:%S)]: Serer timed out. Uptime: $MINUTES_PAST minutes." + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + exit 1 + fi + done + + - run: + name: Wait for build results + command: | + ## Recording the build results + STATUS=$(cat /tmp/status) + IP_ADDRESS=$(cat /tmp/ip_address) + DEVICE_ID=$(cat /tmp/device_id) + echo "[$(date +%H:%M:%S)]: $STATUS" + if [ "$STATUS" != "success" ]; then + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ + echo "Build failed. Cleaning up server with ID $DEVICE_ID" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"'?force_delete=true' + exit 1 fi + echo "[$(date +%H:%M:%S)]: Build was successful. Cleaning up server with ID $DEVICE_ID" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"'?force_delete=true' + exit 0 - store_artifacts: path: /tmp/artifacts diff --git a/ci/build_machine_bootstrap.sh b/ci/build_machine_bootstrap.sh index 1794864..9b39870 100755 --- a/ci/build_machine_bootstrap.sh +++ b/ci/build_machine_bootstrap.sh @@ -1,42 +1,59 @@ #! /bin/bash # This script is run on the Packet.net baremetal server for CI tests. -# This script will build the entire lab from scratch and takes 3-4 hours -# on a Packet.net host # While building, the server will start a webserver on Port 80 that contains # the text "building". Once the test is completed, the text will be replaced # with "success" or "failed". -ARGS="$1" -PACKER_ONLY=0 -VAGRANT_ONLY=0 +# Download Packet.net storage utilities +echo "[$(date +%H:%M:%S)]: Downloading Packet external storage utilities..." +wget -q -O /usr/local/bin/packet-block-storage-attach "https://raw.githubusercontent.com/packethost/packet-block-storage/master/packet-block-storage-attach" +chmod +x /usr/local/bin/packet-block-storage-attach +wget -q -O /usr/local/bin/packet-block-storage-detach "https://raw.githubusercontent.com/packethost/packet-block-storage/master/packet-block-storage-detach" +chmod +x /usr/local/bin/packet-block-storage-detach -if [ ! -z "$1" ]; then - case "$1" in - --packer-only) - PACKER_ONLY=1 - ;; - --vagrant-only) - VAGRANT_ONLY=1 - ;; - *) - echo "\"$ARGS\" is not a supported argument to this script. Quitting" - exit 1 - ;; - esac +# Set a flag to determine if the boxes are available on external Packet storage +BOXES_PRESENT=0 +# Attempt to mount the block storage +echo "[$(date +%H:%M:%S)]: Attempting to mount external storage..." +/usr/local/bin/packet-block-storage-attach +sleep 10 +# Check if it was successful by looking for volume* in /dev/mapper +if ls -al /dev/mapper/volume* > /dev/null 2>&1; then + echo "[$(date +%H:%M:%S)]: Mounting of external storage was successful." + sleep 5 + if mount /dev/mapper/volume-fed37d73-part1 /mnt; then + echo "[$(date +%H:%M:%S)]: External storage successfully mounted to /mnt" + else + echo "[$(date +%H:%M:%S)]: Something went wrong mounting the filesystem from the external storage." + fi + if ls -al /mnt/*.box > /dev/null 2>&1; then + BOXES_PRESENT=1 + fi +else + echo "[$(date +%H:%M:%S)]: No volumes found after attempting to mount storage. Trying again..." + /usr/local/bin/packet-block-storage-attach + sleep 15 + if ! ls -al /dev/mapper/volume* > /dev/null 2>&1; then + echo "[$(date +%H:%M:%S)]: Failed to mount volumes even after a retry. Giving up..." + else + echo "[$(date +%H:%M:%S)]: Successfully mounted the external storage after a retry." + sleep 10 + if mount /dev/mapper/volume-fed37d73-part1 /mnt; then + echo "[$(date +%H:%M:%S)]: External storage successfully mounted to /mnt" + else + echo "[$(date +%H:%M:%S)]: Something went wrong mounting the filesystem from the external storage." + fi + if ls -al /mnt/*.box > /dev/null 2>&1; then + BOXES_PRESENT=1 + fi + fi fi -echo "Args: $ARGS" - # Disable IPv6 - may help with the vagrant-reload plugin: https://github.com/hashicorp/vagrant/issues/8795#issuecomment-468945063 echo "net.ipv6.conf.all.disable_ipv6=1" >> /etc/sysctl.conf sysctl -p /etc/sysctl.conf > /dev/null -if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then - echo "[$(date +%H:%M:%S)]: Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong." - exit 1 -fi - # Install Virtualbox 5.2 echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list sed -i "2ideb mirror://mirrors.ubuntu.com/mirrors.txt xenial main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-updates main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-backports main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-security main restricted universe multiverse" /etc/apt/sources.list @@ -54,60 +71,42 @@ ufw allow http ufw default allow outgoing ufw --force enable -if [ "$PACKER_ONLY" -eq 0 ]; then - # Install Vagrant - echo "[$(date +%H:%M:%S)]: Installing Vagrant..." - mkdir /opt/vagrant - cd /opt/vagrant || exit 1 - wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.4/vagrant_2.2.4_x86_64.deb - dpkg -i vagrant_2.2.4_x86_64.deb - echo "[$(date +%H:%M:%S)]: Installing vagrant-reload plugin..." +# Install Vagrant +echo "[$(date +%H:%M:%S)]: Installing Vagrant..." +mkdir /opt/vagrant +cd /opt/vagrant || exit 1 +wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.4/vagrant_2.2.4_x86_64.deb +dpkg -i vagrant_2.2.4_x86_64.deb +echo "[$(date +%H:%M:%S)]: Installing vagrant-reload plugin..." +vagrant plugin install vagrant-reload + +# Make sure the plugin installed correctly. Retry if not. +if [ "$(vagrant plugin list | grep -c vagrant-reload)" -ne "1" ]; then + echo "[$(date +%H:%M:%S)]: The first attempt to install the vagrant-reload plugin failed. Trying again." vagrant plugin install vagrant-reload - - # Make sure the plugin installed correctly. Retry if not. - if [ "$(vagrant plugin list | grep -c vagrant-reload)" -ne "1" ]; then - echo "[$(date +%H:%M:%S)]: The first attempt to install the vagrant-reload plugin failed. Trying again." - vagrant plugin install vagrant-reload - fi - - # Re-enable IPv6 - may help with the Vagrant Cloud slowness - echo "net.ipv6.conf.all.disable_ipv6=0" >> /etc/sysctl.conf - sysctl -p /etc/sysctl.conf > /dev/null - - # Make the Vagrant instances headless - cd /opt/DetectionLab/Vagrant || exit 1 - sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile fi -if [ "$VAGRANT_ONLY" -eq 0 ]; then - echo "[$(date +%H:%M:%S)]: Installing Packer..." - # Install Packer - mkdir /opt/packer - cd /opt/packer || exit 1 - wget --progress=bar:force https://releases.hashicorp.com/packer/1.4.0/packer_1.4.0_linux_amd64.zip - unzip packer_1.4.0_linux_amd64.zip - cp packer /usr/local/bin/packer +# Re-enable IPv6 - may help with the Vagrant Cloud slowness +echo "net.ipv6.conf.all.disable_ipv6=0" >> /etc/sysctl.conf +sysctl -p /etc/sysctl.conf > /dev/null - # Make the Packer images headless - cd /opt/DetectionLab/Packer || exit 1 - for file in *.json; do - sed -i 's/"headless": false,/"headless": true,/g' "$file"; - done +# Make the Vagrant instances headless +cd /opt/DetectionLab/Vagrant || exit 1 +sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile + +# If the boxes are present on external storage, we can modify the Vagrantfile to +# point to the boxes on disk so we don't have to download them +if [ $BOXES_PRESENT -eq 1 ]; then + echo "[$(date +%H:%M:%S)]: Updating the Vagrantfile to point to the boxes mounted on external storage..." + sed -i 's#"detectionlab/win2016"#"/mnt/windows_2016_virtualbox.box"#g' /opt/DetectionLab/Vagrant/Vagrantfile + sed -i 's#"detectionlab/win10"#"/mnt/windows_10_virtualbox.box"#g' /opt/DetectionLab/Vagrant/Vagrantfile fi -# Ensure the script is executable +# Make the build script is executable chmod +x /opt/DetectionLab/build.sh cd /opt/DetectionLab || exit 1 # Start the build in a tmux session sn=tmuxsession tmux new-session -s "$sn" -d -if [ "$PACKER_ONLY" -eq 1 ]; then - tmux send-keys -t "$sn:0" './build.sh virtualbox --packer-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter -fi -if [ "$VAGRANT_ONLY" -eq 1 ]; then - tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter -fi -if [[ "$PACKER_ONLY" -eq 0 ]] && [[ "$VAGRANT_ONLY" -eq 0 ]]; then - tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter -fi +tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html; umount /mnt && /usr/local/bin/packet-block-storage-detach' Enter diff --git a/ci/circle_workflows/packer_and_vagrant_changes.sh b/ci/circle_workflows/packer_and_vagrant_changes.sh deleted file mode 100644 index e9ee1c2..0000000 --- a/ci/circle_workflows/packer_and_vagrant_changes.sh +++ /dev/null @@ -1,73 +0,0 @@ -#! /bin/bash - -set -e - -# Create artifacts directory -if [ ! -d "/tmp/artifacts" ]; then - mkdir /tmp/artifacts -fi - -## Provision a Type1 baremetal Packet.net server -echo "Provisioning a server on Packet.net" -DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') -# Make sure the device ID is sane. -# TODO: maybe make this a regex -if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then - echo "Server may have failed provisionining. Device ID is set to: $DEVICE_ID" - exit 1 -fi -echo "Server successfully provisioned with ID: $DEVICE_ID" - -echo "Sleeping 10 minutes to wait for Packet server to be provisioned" -sleep 300 -echo "Sleeping 5 more minutes (CircleCI Keepalive)" -sleep 300 - -## Recording the IP address of the newly provisioned Packet server -IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') - -# Copy repo to Packet server -# TODO: Tar up the repo and expand it remotely -cd ~/repo -rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab - -## Running install script on Packet server -ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh - -echo "Sleeping 5 minutes to allow the build process to start" -sleep 300 - -## Waiting for Packet server to post build results -MINUTES_PAST=0 -while [ "$MINUTES_PAST" -lt 400 ]; do - STATUS=$(curl $IP_ADDRESS) - if [ "$STATUS" == "building" ]; then - echo "$STATUS" - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/|| echo "Vagrant log not yet present" - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" - sleep 300 - ((MINUTES_PAST += 5)) - else - break - fi -done -if [ "$MINUTES_PAST" -gt 400 ]; then - echo "Serer timed out. Uptime: $MINUTES_PAST minutes." - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" - curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" - exit 1 -fi - -## Recording the build results -echo $STATUS -if [ "$STATUS" != "success" ]; then - echo "Build failed. Cleaning up server with ID $DEVICE_ID" - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" - curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" - exit 1 -fi -echo "Build was successful. Cleaning up server with ID $DEVICE_ID" -curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" -exit 0 diff --git a/ci/circle_workflows/packer_changes.sh b/ci/circle_workflows/packer_changes.sh deleted file mode 100644 index 704dba2..0000000 --- a/ci/circle_workflows/packer_changes.sh +++ /dev/null @@ -1,91 +0,0 @@ -#! /bin/bash - -set -e - -# Create artifacts directory -if [ ! -d "/tmp/artifacts" ]; then - mkdir /tmp/artifacts -fi - -## Provision two Type1 baremetal Packet.net servers -echo "Provisioning packerwindows2016 on Packet.net" -SERVER1_ID=$(curl -s -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows2016", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') -if [ "$(echo -n $SERVER1_ID | wc -c)" -ne 36 ]; then - echo "Server may have failed provisionining. Device ID is set to: $SERVER1_ID" - exit 1 -fi -echo "packerwindows2016 successfully provisioned with ID: $SERVER1_ID" - -sleep 5 # Wait a bit before issuing another provision command - -echo "Provisioning packerwindows10 on Packet.net" -SERVER2_ID=$(curl -s -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows10", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') -if [ "$(echo -n $SERVER2_ID | wc -c)" -ne 36 ]; then - echo "Server may have failed provisionining. Device ID is set to: $SERVER2_ID" - exit 1 -fi -echo "packerwindows10 successfully provisioned with ID: $SERVER2_ID" - -echo "Sleeping 10 minutes to wait for Packet servers to finish provisiong" -sleep 300 -echo "Sleeping 5 more minutes (CircleCI Keepalive)" -sleep 300 - -## Recording the IP address of the newly provisioned Packet servers -SERVER1_IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER1_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') -SERVER2_IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER2_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') - -# Copy repo to Packet servers -# TODO: Tar up the repo and expand it remotely -cd ~/repo -rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab -rsync -Pav -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab - -## Running install script on Packet server -ssh -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only -ssh -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --packer-only - -sleep 30 - -## Waiting for Packet server to post build results -MINUTES_PAST=0 -while [ "$MINUTES_PAST" -lt 150 ]; do - SERVER1_STATUS=$(curl $SERVER1_IP_ADDRESS) - SERVER2_STATUS=$(curl $SERVER2_IP_ADDRESS) - if [[ "$SERVER1_STATUS" == "building" ]] || [[ "$SERVER2_STATUS" == "building" ]]; then - echo "$SERVER1_STATUS" :: "$SERVER2_STATUS" - scp -q -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log - scp -q -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log - sleep 300 - ((MINUTES_PAST += 5)) - fi - if [[ "$SERVER1_STATUS" != "building" ]] && [[ "$SERVER2_STATUS" != "building" ]]; then - break - fi - if [ "$MINUTES_PAST" -gt 150 ]; then - echo "Serer timed out. Uptime: $MINUTES_PAST minutes." - curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" - curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" - exit 1 - fi -done - -## Recording the build results -echo "Server1 Status: $SERVER1_STATUS" -echo "Server2 Status: $SERVER2_STATUS" -if [ "$SERVER1_STATUS" != "success" ]; then - echo "Build failed. Cleaning up server with ID $SERVER1_ID" - scp -q -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log || echo "Serveer1 packer_build.log not available yet" - curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" - exit 1 -fi -if [ "$SERVER2_STATUS" != "success" ]; then - echo "Build failed. Cleaning up server with ID $SERVER2_ID" - scp -q -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log || echo "Server2 packer_build.log not available yet" - curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" - exit 1 -fi -echo "Builds were successful. Cleaning up servers with IDs $SERVER1_ID and $SERVER2_ID" -curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" -curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" -exit 0 diff --git a/ci/circle_workflows/vagrant_changes.sh b/ci/circle_workflows/vagrant_changes.sh deleted file mode 100644 index 089fb86..0000000 --- a/ci/circle_workflows/vagrant_changes.sh +++ /dev/null @@ -1,79 +0,0 @@ -#! /bin/bash - -set -e - -# Create artifacts directory -if [ ! -d "/tmp/artifacts" ]; then - mkdir /tmp/artifacts -fi - -## Provision a Type1 baremetal Packet.net server -echo "[$(date +%H:%M:%S)]: Provisioning a server on Packet.net" -DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') -# Make sure the device ID is sane. -# TODO: maybe make this a regex -if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then - echo "[$(date +%H:%M:%S)]: Server may have failed provisionining. Device ID is set to: $DEVICE_ID" - echo "[$(date +%H:%M:%S)]: This usually happens if there are no servers available in the selected datacenter." - echo "[$(date +%H:%M:%S)]: Attempting to retry in another datacenter..." - DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "ewr1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') - if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then - echo "[$(date +%H:%M:%S)]: This script was still unable to successfully provision a server. Exiting." - exit 1 - fi -fi -echo "[$(date +%H:%M:%S)]: Server successfully created with ID: $DEVICE_ID" - -echo "[$(date +%H:%M:%S)]: Waiting for server to finish provisioning..." -# Continue to poll the API until the state of the host is "active" -STATE="provisioning" -while [ "$STATE" != "active" ]; do - sleep 10 - echo "[$(date +%H:%M:%S)]: Sleeping for 10 seconds. Server is still $STATE." - STATE="$(curl -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID" | jq .state | tr -d '"')" -done -echo "[$(date +%H:%M:%S)]: Device with ID $DEVICE_ID has finished provisioning! Onto the build process..." - -## Recording the IP address of the newly provisioned Packet server -IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') - -# Copy repo to Packet server -# TODO: Tar up the repo and expand it remotely -cd ~/repo -rsync -Paq -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab - -## Running install script on Packet server -ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh --vagrant-only - -## Waiting for Packet server to post build results -MINUTES_PAST=0 -while [ "$MINUTES_PAST" -lt 180 ]; do - STATUS=$(curl $IP_ADDRESS) - if [ "$STATUS" == "building" ]; then - echo "[$(date +%H:%M:%S)]: $STATUS" - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" - sleep 300 - ((MINUTES_PAST += 5)) - else - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" - break - fi - if [ "$MINUTES_PAST" -gt 180 ]; then - echo "[$(date +%H:%M:%S)]: Serer timed out. Uptime: $MINUTES_PAST minutes." - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ - curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" - exit 1 - fi -done - -## Recording the build results -echo "[$(date +%H:%M:%S)]: $STATUS" -if [ "$STATUS" != "success" ]; then - scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ - echo "Build failed. Cleaning up server with ID $DEVICE_ID" - curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" - exit 1 -fi -echo "[$(date +%H:%M:%S)]: Build was successful. Cleaning up server with ID $DEVICE_ID" -curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" -exit 0 diff --git a/ci/manual_machine_bootstrap.sh b/ci/manual_machine_bootstrap.sh index f96e0fa..1ca8521 100644 --- a/ci/manual_machine_bootstrap.sh +++ b/ci/manual_machine_bootstrap.sh @@ -4,18 +4,13 @@ sed -i 's/archive.ubuntu.com/us.archive.ubuntu.com/g' /etc/apt/sources.list -if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then - echo "Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong." - exit 1 -fi - # Install Virtualbox 5.2 echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - apt-get update apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential unzip git ufw apache2 python-pip pip install awscli --upgrade --user -export PATH=$PATH:/root/.local/bin +cp /root/.local/bin/aws /usr/local/bin/aws && chmod +x /usr/local/bin/aws # Set up firewall ufw allow ssh diff --git a/ci/manual_machine_bootstrap_vmware.sh b/ci/manual_machine_bootstrap_vmware.sh index 13e8567..a0e56ac 100644 --- a/ci/manual_machine_bootstrap_vmware.sh +++ b/ci/manual_machine_bootstrap_vmware.sh @@ -1,22 +1,18 @@ #! /bin/bash # This script is used to manually prepare an Ubuntu 16.04 server for DetectionLab building + export DEBIAN_FRONTEND=noninteractive export SERIALNUMBER="SECRET" export LICENSEFILE="SECRET" sed -i 's/archive.ubuntu.com/us.archive.ubuntu.com/g' /etc/apt/sources.list -if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then - echo "Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong." - exit 1 -fi - # Install VMWare Workstation 15 apt-get update -apt-get install -y linux-headers-"$(uname -r)" build-essential unzip git ufw apache2 python-pip ubuntu-desktop +apt-get install -y linux-headers-"$(uname -r)" build-essential unzip git ufw apache2 python-pip ubuntu-desktop python-pip pip install awscli --upgrade --user -export PATH=$PATH:/root/.local/bin +cp /root/.local/bin/aws /usr/local/bin/aws && chmod +x /usr/local/bin/aws wget -O VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle "https://download3.vmware.com/software/wkst/file/VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle?HashKey=6f83753e4d9e94da7f920c32b5808033¶ms=%7B%22custnumber%22%3A%22KipkcHRoJWVlZA%3D%3D%22%2C%22sourcefilesize%22%3A%22472.70+MB%22%2C%22dlgcode%22%3A%22WKST-1504-LX%22%2C%22languagecode%22%3A%22en%22%2C%22source%22%3A%22DOWNLOADS%22%2C%22downloadtype%22%3A%22manual%22%2C%22eula%22%3A%22Y%22%2C%22downloaduuid%22%3A%225caee685-d5ad-4f6b-94db-2ddc4f7f3a97%22%2C%22purchased%22%3A%22N%22%2C%22dlgtype%22%3A%22Product+Binaries%22%2C%22productversion%22%3A%2215.0.4%22%2C%22productfamily%22%3A%22VMware+Workstation+Pro%22%7D&AuthKey=1556427011_a994b5252f29429710c077c8dcab1c19" chmod +x VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle @@ -48,8 +44,8 @@ sed -i 's/v.gui = true/v.gui = false/g' Vagrantfile # Install Packer mkdir /opt/packer cd /opt/packer || exit 1 -wget --progress=bar:force https://releases.hashicorp.com/packer/1.3.2/packer_1.3.2_linux_amd64.zip -unzip packer_1.3.2_linux_amd64.zip +wget --progress=bar:force https://releases.hashicorp.com/packer/1.4.0/packer_1.4.0_linux_amd64.zip +unzip packer_1.4.0_linux_amd64.zip cp packer /usr/local/bin/packer # Make the Packer images headless