diff --git a/README.md b/README.md index e0c51a1..adcf420 100644 --- a/README.md +++ b/README.md @@ -122,9 +122,9 @@ Vagrant commands must be run from the "Vagrant" folder. ## Lab Information * Domain Name: windomain.local * Admininstrator login: vagrant:vagrant -* Fleet login: https://192.168.38.105:8412 - admin@detectionlab.network:admin123# +* Fleet login: https://192.168.38.105:8412 - admin@:admin123# * Splunk login: https://192.168.38.105:8000 - admin:changeme -* Caldera login: https://192.168.38.105:8888 - admin:caldera +* Caldera login: https://192.168.38.105:8888 - admin:admin * MS ATA login: https://192.168.38.103 - wef\vagrant:vagrant ## Lab Hosts diff --git a/Vagrant/Vagrantfile_Minimum b/Vagrant/Vagrantfile_Minimum index 2054745..62f3557 100644 --- a/Vagrant/Vagrantfile_Minimum +++ b/Vagrant/Vagrantfile_Minimum @@ -6,13 +6,6 @@ Vagrant.configure("2") do |config| config.vm.provision :shell, path: "bootstrap.sh" cfg.vm.network :private_network, ip: "192.168.38.105", gateway: "192.168.38.1", dns: "8.8.8.8" - cfg.vm.provider "vmware_fusion" do |v, override| - v.vmx["displayname"] = "logger" - v.memory = 2048 - v.cpus = 1 - v.gui = true - end - cfg.vm.provider "vmware_desktop" do |v, override| v.vmx["displayname"] = "logger" v.memory = 4096 @@ -33,13 +26,9 @@ Vagrant.configure("2") do |config| end config.vm.define "dc" do |cfg| - cfg.vm.box = "../Boxes/windows_2016_virtualbox.box" + cfg.vm.box = "detectionlab/win2016" cfg.vm.hostname = "dc" cfg.vm.boot_timeout = 600 - # use the plaintext WinRM transport and force it to use basic authentication. - # NB this is needed because the default negotiate transport stops working - # after the domain controller is installed. - # see https://groups.google.com/forum/#!topic/vagrant-up/sZantuCM0q4 cfg.winrm.transport = :plaintext cfg.vm.communicator = "winrm" cfg.winrm.basic_auth_only = true @@ -63,22 +52,14 @@ Vagrant.configure("2") do |config| cfg.vm.provision "shell", path: "scripts/configure-wef-gpo.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/configure-powershelllogging.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/configure-AuditingPolicyGPOs.ps1", privileged: false + cfg.vm.provision "shell", path: "scripts/configure-rdp-user-gpo.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/install-autorunstowineventlog.ps1", privileged: false cfg.vm.provision "shell", inline: 'wevtutil el | Select-String -notmatch "Microsoft-Windows-LiveId" | Foreach-Object {wevtutil cl "$_"}', privileged: false cfg.vm.provision "shell", inline: "Set-SmbServerConfiguration -AuditSmb1Access $true -Force", privileged: false - cfg.vm.provider "vmware_fusion" do |v, override| - override.vm.box = "../Boxes/windows_2016_vmware.box" - v.vmx["displayname"] = "dc.windomain.local" - v.memory = 2560 - v.cpus = 2 - v.gui = true - end - cfg.vm.provider "vmware_desktop" do |v, override| - override.vm.box = "../Boxes/windows_2016_vmware.box" v.vmx["displayname"] = "dc.windomain.local" - v.memory = 2560 + v.memory = 3072 v.cpus = 2 v.gui = true v.enable_vmrun_ip_lookup = false @@ -88,7 +69,7 @@ Vagrant.configure("2") do |config| vb.gui = true vb.name = "dc.windomain.local" vb.default_nic_type = "82545EM" - vb.customize ["modifyvm", :id, "--memory", 2560] + vb.customize ["modifyvm", :id, "--memory", 3072] vb.customize ["modifyvm", :id, "--cpus", 2] vb.customize ["modifyvm", :id, "--vram", "32"] vb.customize ["modifyvm", :id, "--clipboard", "bidirectional"] @@ -97,7 +78,7 @@ Vagrant.configure("2") do |config| end config.vm.define "wef" do |cfg| - cfg.vm.box = "../Boxes/windows_2016_virtualbox.box" + cfg.vm.box = "detectionlab/win2016" cfg.vm.hostname = "wef" cfg.vm.boot_timeout = 600 cfg.vm.communicator = "winrm" @@ -125,16 +106,7 @@ Vagrant.configure("2") do |config| cfg.vm.provision "shell", path: "scripts/install-autorunstowineventlog.ps1", privileged: false cfg.vm.provision "shell", inline: "Set-SmbServerConfiguration -AuditSmb1Access $true -Force", privileged: false - cfg.vm.provider "vmware_fusion" do |v, override| - override.vm.box = "../Boxes/windows_2016_vmware.box" - v.vmx["displayname"] = "wef.windomain.local" - v.memory = 2048 - v.cpus = 2 - v.gui = true - end - cfg.vm.provider "vmware_desktop" do |v, override| - override.vm.box = "../Boxes/windows_2016_vmware.box" v.vmx["displayname"] = "wef.windomain.local" v.memory = 2048 v.cpus = 2 @@ -155,7 +127,7 @@ Vagrant.configure("2") do |config| end config.vm.define "win10" do |cfg| - cfg.vm.box = "../Boxes/windows_10_virtualbox.box" + cfg.vm.box = "detectionlab/win10" cfg.vm.hostname = "win10" cfg.vm.boot_timeout = 600 cfg.vm.communicator = "winrm" @@ -165,12 +137,12 @@ Vagrant.configure("2") do |config| cfg.vm.network :private_network, ip: "192.168.38.104", gateway: "192.168.38.1", dns: "192.168.38.102" cfg.vm.provision "shell", path: "scripts/fix-second-network.ps1", privileged: false, args: "-ip 192.168.38.104 -dns 192.168.38.102" + cfg.vm.provision "shell", path: "scripts/MakeWindows10GreatAgain.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/provision.ps1", privileged: false cfg.vm.provision "reload" cfg.vm.provision "shell", path: "scripts/provision.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/download_palantir_wef.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/download_palantir_osquery.ps1", privileged: false - cfg.vm.provision "shell", path: "scripts/MakeWindows10GreatAgain.ps1", privileged: false cfg.vm.provision "shell", inline: 'wevtutil el | Select-String -notmatch "Microsoft-Windows-LiveId" | Foreach-Object {wevtutil cl "$_"}', privileged: false cfg.vm.provision "shell", path: "scripts/install-splunkuf.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/install-utilities.ps1", privileged: false @@ -180,19 +152,7 @@ Vagrant.configure("2") do |config| cfg.vm.provision "shell", path: "scripts/install-sysinternals.ps1", privileged: false cfg.vm.provision "shell", path: "scripts/install-autorunstowineventlog.ps1", privileged: false - cfg.vm.provider "vmware_fusion" do |v, override| - override.vm.box = "../Boxes/windows_10_vmware.box" - v.vmx["displayname"] = "win10.windomain.local" - v.vmx["gui.fullscreenatpoweron"] = "FALSE" - v.vmx["gui.viewModeAtPowerOn"] = "windowed" - v.vmx["gui.fitguestusingnativedisplayresolution"] = "FALSE" - v.memory = 2048 - v.cpus = 1 - v.gui = true - end - cfg.vm.provider "vmware_desktop" do |v, override| - override.vm.box = "../Boxes/windows_10_vmware.box" v.vmx["displayname"] = "win10.windomain.local" v.vmx["gui.fullscreenatpoweron"] = "FALSE" v.vmx["gui.viewModeAtPowerOn"] = "windowed" diff --git a/Vagrant/bootstrap.sh b/Vagrant/bootstrap.sh index 5c08ef7..348429a 100644 --- a/Vagrant/bootstrap.sh +++ b/Vagrant/bootstrap.sh @@ -5,12 +5,6 @@ echo "apt-fast apt-fast/maxdownloads string 10" | debconf-set-selections; echo "apt-fast apt-fast/dlflag boolean true" | debconf-set-selections; sed -i "2ideb mirror://mirrors.ubuntu.com/mirrors.txt xenial main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-updates main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-backports main restricted universe multiverse\ndeb mirror://mirrors.ubuntu.com/mirrors.txt xenial-security main restricted universe multiverse" /etc/apt/sources.list -install_mongo_db_apt_key() { - # Install key and apt source for MongoDB - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 - echo "deb http://repo.mongodb.org/apt/ubuntu $(lsb_release -sc)/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list -} - install_python_apt_source() { # Install apt source for Python3.6 add-apt-repository -y ppa:jonathonf/python-3.6 @@ -23,14 +17,14 @@ apt_install_prerequisites() { apt-get -qq update apt-get -qq install -y apt-fast echo "Running apt-fast install..." - apt-fast -qq install -y jq whois build-essential git docker docker-compose unzip mongodb-org python3.6 python3.6-dev + apt-fast -qq install -y jq whois build-essential git docker docker-compose unzip python3.6 python3.6-dev # Install pip for Python 3.6 echo "Installing Pip3.6..." - curl https://bootstrap.pypa.io/get-pip.py | sudo -H python3.6 + curl -s https://bootstrap.pypa.io/get-pip.py | sudo -H python3.6 } test_prerequisites() { - for package in jq whois build-essential git docker docker-compose unzip mongodb-org python3.6 python3.6-dev + for package in jq whois build-essential git docker docker-compose unzip python3.6 python3.6-dev do echo "[TEST] Validating that $package is correctly installed..." # Loop through each package using dpkg @@ -113,9 +107,22 @@ install_splunk() { echo "Installing Splunk..." # Get Splunk.com into the DNS cache. Sometimes resolution randomly fails during wget below dig @8.8.8.8 splunk.com - # Download Splunk - wget --progress=bar:force -O splunk-7.2.5.1-962d9a8e1586-linux-2.6-amd64.deb 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=7.2.5.1&product=splunk&filename=splunk-7.2.5.1-962d9a8e1586-linux-2.6-amd64.deb&wget=true' - dpkg -i splunk-7.2.5.1-962d9a8e1586-linux-2.6-amd64.deb + mkdir splunk + + # Try to resolve the latest version of Splunk by parsing the HTML on the downloads page + LATEST_SPLUNK=$(curl https://www.splunk.com/en_us/download/splunk-enterprise.html | grep -i deb | grep -Eo "data-link=\"................................................................................................................................" | cut -d '"' -f 2) + # Sanity check what was returned from the auto-parse attempt + echo "Attempting to autoresolve the latest version of Splunk..." + if [[ "$(echo $LATEST_SPLUNK | grep -c "^https:")" -eq 1 ]] && [[ "$(echo $LATEST_SPLUNK | grep -c "\.deb$")" -eq 1 ]]; then + echo "The URL to the latest Splunk version was automatically resolved as: $LATEST_SPLUNK" + echo "Attempting to download..." + wget --progress=bar:force -P splunk/ "$LATEST_SPLUNK" + else + echo "Unable to auto-resolve the latest Splunk version. Falling back to hardcoded URL..." + # Download Hardcoded Splunk + wget --progress=bar:force -O splunk/splunk-7.2.6-c0bf0f679ce9-linux-2.6-amd64.deb 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=7.2.6&product=splunk&filename=splunk-7.2.6-c0bf0f679ce9-linux-2.6-amd64.deb&wget=true' + fi + dpkg -i splunk/*.deb /opt/splunk/bin/splunk start --accept-license --answer-yes --no-prompt --seed-passwd changeme /opt/splunk/bin/splunk add index wineventlog -auth 'admin:changeme' /opt/splunk/bin/splunk add index osquery -auth 'admin:changeme' @@ -192,12 +199,12 @@ download_palantir_osquery_config() { } import_osquery_config_into_fleet() { - wget --progress=bar:force https://github.com/kolide/fleet/releases/download/2.0.1/fleet_2.0.1.zip - unzip fleet_2.0.1.zip -d fleet_2.0.1 - cp fleet_2.0.1/linux/fleetctl /usr/local/bin/fleetctl && chmod +x /usr/local/bin/fleetctl + wget --progress=bar:force https://github.com/kolide/fleet/releases/download/2.1.1/fleet_2.1.1.zip + unzip fleet_2.1.1.zip -d fleet_2.1.1 + cp fleet_2.1.1/linux/fleetctl /usr/local/bin/fleetctl && chmod +x /usr/local/bin/fleetctl fleetctl config set --address https://192.168.38.105:8412 fleetctl config set --tls-skip-verify true - fleetctl setup --email admin@detectionlab.network --password 'admin123#' --org-name DetectionLab + fleetctl setup --email admin@detectionlab.network --username admin --password 'admin123#' --org-name DetectionLab fleetctl login --email admin@detectionlab.network --password 'admin123#' # Use fleetctl to import YAML files @@ -220,23 +227,20 @@ install_caldera() { echo "Installing Caldera..." cd /home/vagrant || exit git clone https://github.com/mitre/caldera.git - cd /home/vagrant/caldera/caldera || exit + cd /home/vagrant/caldera/plugins || exit + git clone https://github.com/mitre/adversary.git adversary + git clone https://github.com/mitre/chain.git chain + git clone https://github.com/mitre/gui.git gui + git clone https://github.com/mitre/sandcat.git sandcat + git clone https://github.com/mitre/stockpile.git stockpile + cd /home/vagrant/caldera || exit pip3.6 install -r requirements.txt - - # Add a Systemd service for MongoDB - # https://www.howtoforge.com/tutorial/install-mongodb-on-ubuntu-16.04/ - cp /vagrant/resources/caldera/mongod.service /lib/systemd/system/mongod.service + # Make Caldera accessible on all ports + sed -i 's/127.0.0.1/0.0.0.0/g' conf/local.yml # Create Systemd service for Caldera cp /vagrant/resources/caldera/caldera.service /lib/systemd/system/caldera.service - # Enable replication - echo 'replication: - replSetName: caldera' >> /etc/mongod.conf - service mongod start - systemctl enable mongod.service cd /home/vagrant/caldera || exit - mkdir -p dep/crater/crater - wget --progress=bar:force https://github.com/mitre/caldera-crater/releases/download/v0.1.0/CraterMainWin8up.exe -O /home/vagrant/caldera/dep/crater/crater/CraterMain.exe - cp /vagrant/resources/caldera/cert.pem /vagrant/resources/caldera/key.pem /vagrant/resources/caldera/settings.yml /home/vagrant/caldera/caldera/conf + cp /vagrant/resources/caldera/cert.pem /vagrant/resources/caldera/key.pem /home/vagrant/caldera/conf service caldera start systemctl enable caldera.service fi @@ -423,7 +427,6 @@ test_suricata_prerequisites() { } main() { - install_mongo_db_apt_key install_python_apt_source apt_install_prerequisites test_prerequisites diff --git a/Vagrant/resources/caldera/caldera.service b/Vagrant/resources/caldera/caldera.service index 044bd26..fade6b1 100644 --- a/Vagrant/resources/caldera/caldera.service +++ b/Vagrant/resources/caldera/caldera.service @@ -1,11 +1,11 @@ [Unit] -Description=My Script Service +Description=Caldera After=multi-user.target [Service] Type=idle -WorkingDirectory=/home/vagrant/caldera/caldera -ExecStart=/usr/bin/python3.6 caldera.py +WorkingDirectory=/home/vagrant/caldera +ExecStart=/usr/bin/python3.6 server.py -E local [Install] WantedBy=multi-user.target diff --git a/Vagrant/resources/windows/MenuSettings.xml b/Vagrant/resources/windows/MenuSettings.xml new file mode 100755 index 0000000..05144c1 --- /dev/null +++ b/Vagrant/resources/windows/MenuSettings.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/Vagrant/resources/windows/classic_shell_win7.reg b/Vagrant/resources/windows/classic_shell_win7.reg deleted file mode 100644 index 83fb24e..0000000 --- a/Vagrant/resources/windows/classic_shell_win7.reg +++ /dev/null @@ -1,4 +0,0 @@ -Windows Registry Editor Version 5.00 - -[HKEY_LOCAL_MACHINE\SOFTWARE\IvoSoft\ClassicStartMenu] -"MenuStyle_Default"="Win7" diff --git a/Vagrant/scripts/install-caldera-agent.ps1 b/Vagrant/scripts/install-caldera-agent.ps1 index 65c72c3..bb9c202 100644 --- a/Vagrant/scripts/install-caldera-agent.ps1 +++ b/Vagrant/scripts/install-caldera-agent.ps1 @@ -1,4 +1,6 @@ # Purpose: Installs the Caldera agent on the host +Write-Host "Installing the Caldera agent..." +$url="https://192.168.38.105:8888/file/render"; $ps_table = $PSVersionTable.PSVersion;If([double]$ps_table.Major -ge 6){iex (irm -Method Post -Uri $url -Headers @{"file"="54ndc47.ps1"} -SkipCertificateCheck);}else{[System.Net.ServicePointManager]::ServerCertificateValidationCallback={$True};$web=New-Object System.Net.WebClient;$web.Headers.Add("file","54ndc47.ps1");$resp=$web.UploadString("$url",'');iex($resp);} If (-not (Test-Path 'C:\Program Files\cagent\cagent.exe')) { # Add /etc/hosts entry diff --git a/Vagrant/scripts/install-choco-extras.ps1 b/Vagrant/scripts/install-choco-extras.ps1 index d7f8c2f..b26905a 100644 --- a/Vagrant/scripts/install-choco-extras.ps1 +++ b/Vagrant/scripts/install-choco-extras.ps1 @@ -10,7 +10,7 @@ If (-not (Test-Path "C:\ProgramData\chocolatey")) { } Write-Host "Installing Chocolatey extras..." -choco install -y wireshark -choco install -y winpcap +choco install -y --limit-output wireshark +choco install -y --limit-output winpcap Write-Host "Choco addons complete!" diff --git a/Vagrant/scripts/install-utilities.ps1 b/Vagrant/scripts/install-utilities.ps1 index 2b2a419..5755e61 100755 --- a/Vagrant/scripts/install-utilities.ps1 +++ b/Vagrant/scripts/install-utilities.ps1 @@ -10,11 +10,11 @@ If (-not (Test-Path "C:\ProgramData\chocolatey")) { Write-Host "Installing utilities..." If ($(hostname) -eq "win10") { # Because the Windows10 start menu sucks - choco install -y classic-shell -installArgs ADDLOCAL=ClassicStartMenu - reg import "c:\vagrant\resources\windows\classic_shell_win7.reg" + choco install -y --limit-output classic-shell -installArgs ADDLOCAL=ClassicStartMenu + & "C:\Program Files\Classic Shell\ClassicStartMenu.exe" "-xml" "c:\vagrant\resources\windows\MenuSettings.xml" } -choco install -y NotepadPlusPlus -choco install -y GoogleChrome -choco install -y WinRar +choco install -y --limit-output NotepadPlusPlus +choco install -y --limit-output GoogleChrome +choco install -y --limit-output WinRar Write-Host "Utilties installation complete!" diff --git a/ci/circle_workflows/packer_and_vagrant_changes.sh b/ci/circle_workflows/packer_and_vagrant_changes.sh index 93e4842..e9ee1c2 100644 --- a/ci/circle_workflows/packer_and_vagrant_changes.sh +++ b/ci/circle_workflows/packer_and_vagrant_changes.sh @@ -43,8 +43,8 @@ while [ "$MINUTES_PAST" -lt 400 ]; do STATUS=$(curl $IP_ADDRESS) if [ "$STATUS" == "building" ]; then echo "$STATUS" - scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/|| echo "Vagrant log not yet present" - scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/|| echo "Vagrant log not yet present" + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" sleep 300 ((MINUTES_PAST += 5)) else @@ -53,9 +53,9 @@ while [ "$MINUTES_PAST" -lt 400 ]; do done if [ "$MINUTES_PAST" -gt 400 ]; then echo "Serer timed out. Uptime: $MINUTES_PAST minutes." - scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" - scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" exit 1 fi @@ -63,11 +63,11 @@ fi echo $STATUS if [ "$STATUS" != "success" ]; then echo "Build failed. Cleaning up server with ID $DEVICE_ID" - scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" - scp -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present" + scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/packer_build.log || echo "Packer log not yet present" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" exit 1 fi echo "Build was successful. Cleaning up server with ID $DEVICE_ID" -curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" +curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" exit 0 diff --git a/ci/circle_workflows/packer_changes.sh b/ci/circle_workflows/packer_changes.sh index 88c348d..704dba2 100644 --- a/ci/circle_workflows/packer_changes.sh +++ b/ci/circle_workflows/packer_changes.sh @@ -9,7 +9,7 @@ fi ## Provision two Type1 baremetal Packet.net servers echo "Provisioning packerwindows2016 on Packet.net" -SERVER1_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows2016", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') +SERVER1_ID=$(curl -s -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows2016", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') if [ "$(echo -n $SERVER1_ID | wc -c)" -ne 36 ]; then echo "Server may have failed provisionining. Device ID is set to: $SERVER1_ID" exit 1 @@ -19,7 +19,7 @@ echo "packerwindows2016 successfully provisioned with ID: $SERVER1_ID" sleep 5 # Wait a bit before issuing another provision command echo "Provisioning packerwindows10 on Packet.net" -SERVER2_ID=$(curl -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows10", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') +SERVER2_ID=$(curl -s -X POST -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "packerwindows10", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys":["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"') if [ "$(echo -n $SERVER2_ID | wc -c)" -ne 36 ]; then echo "Server may have failed provisionining. Device ID is set to: $SERVER2_ID" exit 1 @@ -32,8 +32,8 @@ echo "Sleeping 5 more minutes (CircleCI Keepalive)" sleep 300 ## Recording the IP address of the newly provisioned Packet servers -SERVER1_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER1_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') -SERVER2_IP_ADDRESS=$(curl -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER2_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') +SERVER1_IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER1_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') +SERVER2_IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$SERVER2_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"') # Copy repo to Packet servers # TODO: Tar up the repo and expand it remotely @@ -54,8 +54,8 @@ while [ "$MINUTES_PAST" -lt 150 ]; do SERVER2_STATUS=$(curl $SERVER2_IP_ADDRESS) if [[ "$SERVER1_STATUS" == "building" ]] || [[ "$SERVER2_STATUS" == "building" ]]; then echo "$SERVER1_STATUS" :: "$SERVER2_STATUS" - scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log - scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log + scp -q -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log + scp -q -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log sleep 300 ((MINUTES_PAST += 5)) fi @@ -64,8 +64,8 @@ while [ "$MINUTES_PAST" -lt 150 ]; do fi if [ "$MINUTES_PAST" -gt 150 ]; then echo "Serer timed out. Uptime: $MINUTES_PAST minutes." - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" exit 1 fi done @@ -75,17 +75,17 @@ echo "Server1 Status: $SERVER1_STATUS" echo "Server2 Status: $SERVER2_STATUS" if [ "$SERVER1_STATUS" != "success" ]; then echo "Build failed. Cleaning up server with ID $SERVER1_ID" - scp -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log || echo "Serveer1 packer_build.log not available yet" - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" + scp -q -i ~/.ssh/id_rsa root@"$SERVER1_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server1_packer.log || echo "Serveer1 packer_build.log not available yet" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" exit 1 fi if [ "$SERVER2_STATUS" != "success" ]; then echo "Build failed. Cleaning up server with ID $SERVER2_ID" - scp -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log || echo "Server2 packer_build.log not available yet" - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" + scp -q -i ~/.ssh/id_rsa root@"$SERVER2_IP_ADDRESS":/opt/DetectionLab/Packer/packer_build.log /tmp/artifacts/server2_packer.log || echo "Server2 packer_build.log not available yet" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" exit 1 fi echo "Builds were successful. Cleaning up servers with IDs $SERVER1_ID and $SERVER2_ID" -curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" -curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" +curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER1_ID" +curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$SERVER2_ID" exit 0 diff --git a/ci/circle_workflows/vagrant_changes.sh b/ci/circle_workflows/vagrant_changes.sh index 6d6f56d..3eff371 100644 --- a/ci/circle_workflows/vagrant_changes.sh +++ b/ci/circle_workflows/vagrant_changes.sh @@ -56,7 +56,7 @@ while [ "$MINUTES_PAST" -lt 180 ]; do if [ "$MINUTES_PAST" -gt 180 ]; then echo "Serer timed out. Uptime: $MINUTES_PAST minutes." scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" exit 1 fi done @@ -66,9 +66,9 @@ echo $STATUS if [ "$STATUS" != "success" ]; then scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ echo "Build failed. Cleaning up server with ID $DEVICE_ID" - curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" + curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" exit 1 fi echo "Build was successful. Cleaning up server with ID $DEVICE_ID" -curl -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" +curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID" exit 0