Simplify CI build and support Packet external storage

2019-05-20 00:01:25 -07:00
parent 2757ca9bbc
commit 21df017d54
7 changed files with 192 additions and 385 deletions
--- a/ci/build_machine_bootstrap.sh
+++ b/ci/build_machine_bootstrap.sh
@@ -1,42 +1,59 @@
 #! /bin/bash

 # This script is run on the Packet.net baremetal server for CI tests.
-# This script will build the entire lab from scratch and takes 3-4 hours
-# on a Packet.net host
 # While building, the server will start a webserver on Port 80 that contains
 # the text "building". Once the test is completed, the text will be replaced
 # with "success" or "failed".

-ARGS="$1"
-PACKER_ONLY=0
-VAGRANT_ONLY=0
+# Download Packet.net storage utilities
+echo "[$(date +%H:%M:%S)]: Downloading Packet external storage utilities..."
+wget -q -O /usr/local/bin/packet-block-storage-attach "https://raw.githubusercontent.com/packethost/packet-block-storage/master/packet-block-storage-attach"
+chmod +x /usr/local/bin/packet-block-storage-attach
+wget -q -O /usr/local/bin/packet-block-storage-detach "https://raw.githubusercontent.com/packethost/packet-block-storage/master/packet-block-storage-detach"
+chmod +x /usr/local/bin/packet-block-storage-detach

-if [ ! -z "$1" ]; then
-  case "$1" in
-    --packer-only)
-    PACKER_ONLY=1
-    ;;
-    --vagrant-only)
-    VAGRANT_ONLY=1
-    ;;
-    *)
-    echo "\"$ARGS\" is not a supported argument to this script. Quitting"
-    exit 1
-    ;;
-  esac
+# Set a flag to determine if the boxes are available on external Packet storage
+BOXES_PRESENT=0
+# Attempt to mount the block storage
+echo "[$(date +%H:%M:%S)]: Attempting to mount external storage..."
+/usr/local/bin/packet-block-storage-attach
+sleep 10
+# Check if it was successful by looking for volume* in /dev/mapper
+if ls -al /dev/mapper/volume* > /dev/null 2>&1; then
+  echo "[$(date +%H:%M:%S)]: Mounting of external storage was successful."
+  sleep 5
+  if mount /dev/mapper/volume-fed37d73-part1 /mnt; then
+    echo "[$(date +%H:%M:%S)]: External storage successfully mounted to /mnt"
+  else
+    echo "[$(date +%H:%M:%S)]: Something went wrong mounting the filesystem from the external storage."
+  fi
+  if ls -al /mnt/*.box > /dev/null 2>&1; then
+    BOXES_PRESENT=1
+  fi
+else
+  echo "[$(date +%H:%M:%S)]: No volumes found after attempting to mount storage. Trying again..."
+  /usr/local/bin/packet-block-storage-attach
+  sleep 15
+  if ! ls -al /dev/mapper/volume* > /dev/null 2>&1; then
+    echo "[$(date +%H:%M:%S)]: Failed to mount volumes even after a retry. Giving up..."
+  else
+    echo "[$(date +%H:%M:%S)]: Successfully mounted the external storage after a retry."
+    sleep 10
+    if mount /dev/mapper/volume-fed37d73-part1 /mnt; then
+      echo "[$(date +%H:%M:%S)]: External storage successfully mounted to /mnt"
+    else
+      echo "[$(date +%H:%M:%S)]: Something went wrong mounting the filesystem from the external storage."
+    fi
+    if ls -al /mnt/*.box > /dev/null 2>&1; then
+      BOXES_PRESENT=1
+    fi
+  fi
 fi

-echo "Args: $ARGS"
-
 # Disable IPv6 - may help with the vagrant-reload plugin: https://github.com/hashicorp/vagrant/issues/8795#issuecomment-468945063
 echo "net.ipv6.conf.all.disable_ipv6=1" >> /etc/sysctl.conf
 sysctl -p /etc/sysctl.conf > /dev/null

-if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then
-  echo "[$(date +%H:%M:%S)]: Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong."
-  exit 1
-fi
-
 # Install Virtualbox 5.2
 echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list
 sed -i "2ideb mirror://mirrors.ubuntu.com/mirrors.txt xenial main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-updates main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-backports main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-security main restricted universe multiverse" /etc/apt/sources.list
@@ -54,60 +71,42 @@ ufw allow http
 ufw default allow outgoing
 ufw --force enable

-if [ "$PACKER_ONLY" -eq 0 ]; then
-  # Install Vagrant
-  echo "[$(date +%H:%M:%S)]: Installing Vagrant..."
-  mkdir /opt/vagrant
-  cd /opt/vagrant || exit 1
-  wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.4/vagrant_2.2.4_x86_64.deb
-  dpkg -i vagrant_2.2.4_x86_64.deb
-  echo "[$(date +%H:%M:%S)]: Installing vagrant-reload plugin..."
+# Install Vagrant
+echo "[$(date +%H:%M:%S)]: Installing Vagrant..."
+mkdir /opt/vagrant
+cd /opt/vagrant || exit 1
+wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.4/vagrant_2.2.4_x86_64.deb
+dpkg -i vagrant_2.2.4_x86_64.deb
+echo "[$(date +%H:%M:%S)]: Installing vagrant-reload plugin..."
+vagrant plugin install vagrant-reload
+
+# Make sure the plugin installed correctly. Retry if not.
+if [ "$(vagrant plugin list | grep -c vagrant-reload)" -ne "1" ]; then
+  echo "[$(date +%H:%M:%S)]: The first attempt to install the vagrant-reload plugin failed. Trying again."
  vagrant plugin install vagrant-reload
-
-  # Make sure the plugin installed correctly. Retry if not.
-  if [ "$(vagrant plugin list | grep -c vagrant-reload)" -ne "1" ]; then
-    echo "[$(date +%H:%M:%S)]: The first attempt to install the vagrant-reload plugin failed. Trying again."
-    vagrant plugin install vagrant-reload
-  fi
-
-  # Re-enable IPv6 - may help with the Vagrant Cloud slowness
-  echo "net.ipv6.conf.all.disable_ipv6=0" >> /etc/sysctl.conf
-  sysctl -p /etc/sysctl.conf > /dev/null
-
-  # Make the Vagrant instances headless
-  cd /opt/DetectionLab/Vagrant || exit 1
-  sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
 fi

-if [ "$VAGRANT_ONLY" -eq 0 ]; then
-  echo "[$(date +%H:%M:%S)]: Installing Packer..."
-  # Install Packer
-  mkdir /opt/packer
-  cd /opt/packer || exit 1
-  wget --progress=bar:force https://releases.hashicorp.com/packer/1.4.0/packer_1.4.0_linux_amd64.zip
-  unzip packer_1.4.0_linux_amd64.zip
-  cp packer /usr/local/bin/packer
+# Re-enable IPv6 - may help with the Vagrant Cloud slowness
+echo "net.ipv6.conf.all.disable_ipv6=0" >> /etc/sysctl.conf
+sysctl -p /etc/sysctl.conf > /dev/null

-  # Make the Packer images headless
-  cd /opt/DetectionLab/Packer || exit 1
-  for file in *.json; do
-    sed -i 's/"headless": false,/"headless": true,/g' "$file";
-  done
+# Make the Vagrant instances headless
+cd /opt/DetectionLab/Vagrant || exit 1
+sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
+
+# If the boxes are present on external storage, we can modify the Vagrantfile to
+# point to the boxes on disk so we don't have to download them
+if [ $BOXES_PRESENT -eq 1 ]; then
+  echo "[$(date +%H:%M:%S)]: Updating the Vagrantfile to point to the boxes mounted on external storage..."
+  sed -i 's#"detectionlab/win2016"#"/mnt/windows_2016_virtualbox.box"#g' /opt/DetectionLab/Vagrant/Vagrantfile
+  sed -i 's#"detectionlab/win10"#"/mnt/windows_10_virtualbox.box"#g' /opt/DetectionLab/Vagrant/Vagrantfile
 fi

-# Ensure the script is executable
+# Make the build script is executable
 chmod +x /opt/DetectionLab/build.sh
 cd /opt/DetectionLab || exit 1

 # Start the build in a tmux session
 sn=tmuxsession
 tmux new-session -s "$sn" -d
-if [ "$PACKER_ONLY" -eq 1 ]; then
-  tmux send-keys -t "$sn:0" './build.sh virtualbox --packer-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
-fi
-if [ "$VAGRANT_ONLY" -eq 1 ]; then
-  tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
-fi
-if [[ "$PACKER_ONLY" -eq 0 ]] && [[ "$VAGRANT_ONLY" -eq 0 ]]; then
-  tmux send-keys -t "$sn:0" './build.sh virtualbox && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html' Enter
-fi
+tmux send-keys -t "$sn:0" './build.sh virtualbox --vagrant-only && echo "success" > /var/www/html/index.html || echo "failed" > /var/www/html/index.html; umount /mnt && /usr/local/bin/packet-block-storage-detach' Enter