From f20589be4de374b094b15289440873fcb3825bd8 Mon Sep 17 00:00:00 2001 From: Chris Long Date: Thu, 9 May 2019 20:58:06 -0700 Subject: [PATCH] Terraform AMI Refresh, Windows 10 box refresh --- Terraform/Pre-Built_AMIs.md | 2 +- Terraform/README.md | 11 ++++++++ Terraform/VM_to_AMIs.md | 10 +++---- Terraform/main.tf | 11 +++++--- Terraform/outputs.tf | 16 +++++++++++ Terraform/variables.tf | 39 ++++++++++++++++++++++----- Terraform/vm_import/logger.json | 9 ------- ci/build_machine_bootstrap.sh | 10 ++++--- ci/manual_machine_bootstrap.sh | 7 ++--- ci/manual_machine_bootstrap_vmware.sh | 14 +++++----- 10 files changed, 87 insertions(+), 42 deletions(-) delete mode 100644 Terraform/vm_import/logger.json diff --git a/Terraform/Pre-Built_AMIs.md b/Terraform/Pre-Built_AMIs.md index d04f365..06c0881 100644 --- a/Terraform/Pre-Built_AMIs.md +++ b/Terraform/Pre-Built_AMIs.md @@ -19,7 +19,7 @@ The supplied Terraform configuration can then be used to create EC2 instances an 3. Create a private/public keypair to use to SSH into logger: `ssh-keygen -b 2048 -f ~/.ssh/id_logger` 4. Copy the file at [/DetectionLab/Terraform/terraform.tfvars.example](./terraform.tfvars.example) to `/DetectionLab/Terraform/terraform.tfvars` 5. In `terraform.tfvars`, provide overrides for the variables specified in [variables.tf](./variables.tf) -6. From the `/DetectionLab/Terraform/` directory, run `terraform init` to setup the initial Terraform configuration +6. From the `/DetectionLab/Terraform` directory, run `terraform init` to setup the initial Terraform configuration 7. Run `terraform apply` to begin the provisioning process [![DetectionLab - Terraform](https://i.vimeocdn.com/video/777172792_640.webp)](https://vimeo.com/331695321) diff --git a/Terraform/README.md b/Terraform/README.md index d377023..d52a371 100644 --- a/Terraform/README.md +++ b/Terraform/README.md @@ -19,3 +19,14 @@ One method for spinning up DetectionLab in AWS is to begin by using Virtualbox o This method has the benefit of allowing users to customize the VMs before importing them to AWS. The instructions for deploying DetectionLab in AWS via this method are available here: [Build Your Own AMIs README](./VM_to_AMIs.md) + + +### Current AMI Listing +| Region | Name | AMI-ID | +|--------|------|--------| +| us-west-1 | detectionlab-dc | ami-03e2df055c632a0dd | +| us-west-1 | detectionlab-wef | ami-03c82482c03a740c5 | +| us-west-1 | detectionlab-win10 | ami-0a4644e74768900f7 | +| us-east-1 | detectionlab-dc | ami-0eba8a430eb9c0d92 | +| us-east-1 | detectionlab-wef | ami-077981880d8b81b6b | +| us-east-1 | detectionlab-win10 | ami-0d1b75d4a41ff0e0a | diff --git a/Terraform/VM_to_AMIs.md b/Terraform/VM_to_AMIs.md index 35bbe04..34956af 100644 --- a/Terraform/VM_to_AMIs.md +++ b/Terraform/VM_to_AMIs.md @@ -28,18 +28,18 @@ The supplied Terraform configuration can then be used to create EC2 instances an 7. Export the DetectionLab VMs as single file OVA files if they are not already in that format 8. [Upload the OVAs to the S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/upload-objects.html) you created in step three -9. Edit the `logger.json`, `dc.json`, `wef.json` and `win10.json` files and modify the S3Bucket and S3Key headers to match the location of the OVA files in your S3 bucket. +9. Edit the `dc.json`, `wef.json` and `win10.json` files and modify the S3Bucket and S3Key headers to match the location of the OVA files in your S3 bucket. 10. Import the VMs from S3 as AMIs by running the following commands: ``` aws ec2 import-image --description "dc" --license-type byol --disk-containers file:///path/to/DetectionLab/Terraform/vm_import/dc.json aws ec2 import-image --description "wef" --license-type byol --disk-containers file:///path/to/DetectionLab/Terraform/vm_import/wef.json aws ec2 import-image --description "win10" --license-type byol --disk-containers file:///path/to/DetectionLab/Terraform/vm_import/win10.json -aws ec2 import-image --description "logger" --license-type byol --disk-containers file:///path/to/DetectionLab/Terraform/vm_import/logger.json ``` 11. Check on the status of the importation with the following command: ```aws ec2 describe-import-image-tasks --import-task-ids ``` -12. Fill out the variables in `/path/to/DetectionLab/Terraform/terraform.tfvars` -13. Run `terraform init` to setup the initial Terraform configuration -14. `cd /path/to/DetectionLab/Terraform/Method1 && terraform apply` +12. Copy the file at [/DetectionLab/Terraform/terraform.tfvars.example](./terraform.tfvars.example) to `/DetectionLab/Terraform/terraform.tfvars` +13. Fill out the variables in `/DetectionLab/Terraform/terraform.tfvars` +14. Run `terraform init` to setup the initial Terraform configuration +15. cd to `DetectionLab/Terraform` and run `terraform apply` diff --git a/Terraform/main.tf b/Terraform/main.tf index 615d577..f1234d2 100644 --- a/Terraform/main.tf +++ b/Terraform/main.tf @@ -152,7 +152,7 @@ resource "aws_instance" "logger" { # Provision the AWS Ubuntu 16.04 AMI from scratch. provisioner "remote-exec" { inline = [ - "sudo add-apt-repository universe && sudo apt-get update && sudo apt-get install -y git", + "sudo add-apt-repository universe && sudo apt-get -qq update && sudo apt-get -qq install -y git", "echo 'logger' | sudo tee /etc/hostname && sudo hostnamectl set-hostname logger", "sudo adduser --disabled-password --gecos \"\" vagrant && echo 'vagrant:vagrant' | sudo chpasswd", "sudo mkdir /home/vagrant/.ssh && sudo cp /home/ubuntu/.ssh/authorized_keys /home/vagrant/.ssh/authorized_keys && sudo chown -R vagrant:vagrant /home/vagrant/.ssh", @@ -180,7 +180,8 @@ resource "aws_instance" "logger" { resource "aws_instance" "dc" { instance_type = "t2.medium" - ami = "${var.dc_ami}" + # Change the below variable to "${var.dc_ami}" if using hardcoded AMIs + ami = "${data.aws_ami.dc_ami.image_id}" tags { Name = "dc.windomain.local" } @@ -194,7 +195,8 @@ resource "aws_instance" "dc" { resource "aws_instance" "wef" { instance_type = "t2.medium" - ami = "${var.wef_ami}" + # Change the below variable to "${var.wef_ami}" if using hardcoded AMIs + ami = "${data.aws_ami.wef_ami.image_id}" tags { Name = "wef.windomain.local" } @@ -208,7 +210,8 @@ resource "aws_instance" "wef" { resource "aws_instance" "win10" { instance_type = "t2.medium" - ami = "${var.win10_ami}" + # Change the below variable to "${var.win10_ami}" if using hardcoded AMIs + ami = "${data.aws_ami.win10_ami.image_id}" tags { Name = "win10.windomain.local" } diff --git a/Terraform/outputs.tf b/Terraform/outputs.tf index 0fb7147..9677bb8 100644 --- a/Terraform/outputs.tf +++ b/Terraform/outputs.tf @@ -1,3 +1,7 @@ +output region { + value = "${var.region}" +} + output "logger_public_ip" { value = "${aws_instance.logger.public_ip}" } @@ -13,3 +17,15 @@ output "wef_public_ip" { output "win10_public_ip" { value = "${aws_instance.win10.public_ip}" } + +output "latest_dc_ami_id" { + value = "${data.aws_ami.dc_ami.image_id}" +} + +output "latest_wef_ami_id" { + value = "${data.aws_ami.wef_ami.image_id}" +} + +output "latest_win10_ami_id" { + value = "${data.aws_ami.wef_ami.image_id}" +} diff --git a/Terraform/variables.tf b/Terraform/variables.tf index 041a8b3..c501bb8 100644 --- a/Terraform/variables.tf +++ b/Terraform/variables.tf @@ -35,22 +35,47 @@ variable "external_dns_servers" { default = ["8.8.8.8"] } +# Use Data Sources to resolve the AMI-ID for the pre-built DC host +data "aws_ami" "dc_ami" { + owners = ["505638924199"] + filter { + name = "name" + values = ["detectionlab-dc"] + } +} + + # Use Data Sources to resolve the AMI-ID for the pre-built WEF host +data "aws_ami" "wef_ami" { + owners = ["505638924199"] + most_recent = true + filter { + name = "name" + values = ["detectionlab-wef"] + } +} + + # Use Data Sources to resolve the AMI-ID for the pre-built Win10 host +data "aws_ami" "win10_ami" { + owners = ["505638924199"] + most_recent = true + filter { + name = "name" + values = ["detectionlab-win10"] + } +} + # The logger host uses the Amazon Ubuntu 16.04 image # If you are building your own AMIs, replace the default values below with # the AMI IDs -variable "logger_ami" { - type = "string" - default = "ami-0693b32d066fade8a" -} variable "dc_ami" { type = "string" - default = "ami-0f0f0aaba01986b10" + default = "ami-03e2df055c632a0dd" } variable "wef_ami" { type = "string" - default = "ami-02566cd6ca3b7c6ae" + default = "ami-03c82482c03a740c5" } variable "win10_ami" { type = "string" - default = "ami-06a8a101dac68a81a" + default = "ami-0a4644e74768900f7" } diff --git a/Terraform/vm_import/logger.json b/Terraform/vm_import/logger.json deleted file mode 100644 index 1ea443a..0000000 --- a/Terraform/vm_import/logger.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "Description": "logger", - "Format": "ova", - "UserBucket": { - "S3Bucket": "YOUR_BUCKET_GOES_HERE", - "S3Key": "logger.ova" - } -}] diff --git a/ci/build_machine_bootstrap.sh b/ci/build_machine_bootstrap.sh index 51ea332..1794864 100755 --- a/ci/build_machine_bootstrap.sh +++ b/ci/build_machine_bootstrap.sh @@ -30,7 +30,7 @@ echo "Args: $ARGS" # Disable IPv6 - may help with the vagrant-reload plugin: https://github.com/hashicorp/vagrant/issues/8795#issuecomment-468945063 echo "net.ipv6.conf.all.disable_ipv6=1" >> /etc/sysctl.conf -sysctl -p /etc/sysctl.conf +sysctl -p /etc/sysctl.conf > /dev/null if [[ "$VAGRANT_ONLY" -eq 1 ]] && [[ "$PACKER_ONLY" -eq 1 ]]; then echo "[$(date +%H:%M:%S)]: Somehow this build is configured as both packer-only and vagrant-only. This means something has gone horribly wrong." @@ -70,6 +70,10 @@ if [ "$PACKER_ONLY" -eq 0 ]; then vagrant plugin install vagrant-reload fi + # Re-enable IPv6 - may help with the Vagrant Cloud slowness + echo "net.ipv6.conf.all.disable_ipv6=0" >> /etc/sysctl.conf + sysctl -p /etc/sysctl.conf > /dev/null + # Make the Vagrant instances headless cd /opt/DetectionLab/Vagrant || exit 1 sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile @@ -80,8 +84,8 @@ if [ "$VAGRANT_ONLY" -eq 0 ]; then # Install Packer mkdir /opt/packer cd /opt/packer || exit 1 - wget --progress=bar:force https://releases.hashicorp.com/packer/1.3.2/packer_1.3.2_linux_amd64.zip - unzip packer_1.3.2_linux_amd64.zip + wget --progress=bar:force https://releases.hashicorp.com/packer/1.4.0/packer_1.4.0_linux_amd64.zip + unzip packer_1.4.0_linux_amd64.zip cp packer /usr/local/bin/packer # Make the Packer images headless diff --git a/ci/manual_machine_bootstrap.sh b/ci/manual_machine_bootstrap.sh index 3ab779a..f96e0fa 100644 --- a/ci/manual_machine_bootstrap.sh +++ b/ci/manual_machine_bootstrap.sh @@ -17,11 +17,8 @@ apt-get install -y linux-headers-"$(uname -r)" virtualbox-5.2 build-essential un pip install awscli --upgrade --user export PATH=$PATH:/root/.local/bin -echo "building" > /var/www/html/index.html - # Set up firewall ufw allow ssh -ufw allow http ufw default allow outgoing ufw --force enable @@ -41,8 +38,8 @@ sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile # Install Packer mkdir /opt/packer cd /opt/packer || exit 1 -wget https://releases.hashicorp.com/packer/1.3.2/packer_1.3.2_linux_amd64.zip -unzip packer_1.3.2_linux_amd64.zip +wget --progress=bar:force https://releases.hashicorp.com/packer/1.4.0/packer_1.4.0_linux_amd64.zip +unzip packer_1.4.0_linux_amd64.zip cp packer /usr/local/bin/packer # Make the Packer images headless diff --git a/ci/manual_machine_bootstrap_vmware.sh b/ci/manual_machine_bootstrap_vmware.sh index f6ff1cf..13e8567 100644 --- a/ci/manual_machine_bootstrap_vmware.sh +++ b/ci/manual_machine_bootstrap_vmware.sh @@ -1,9 +1,9 @@ #! /bin/bash # This script is used to manually prepare an Ubuntu 16.04 server for DetectionLab building - -SERIALNUMBER="TODO" -LICENSEFILE="TODO" +export DEBIAN_FRONTEND=noninteractive +export SERIALNUMBER="SECRET" +export LICENSEFILE="SECRET" sed -i 's/archive.ubuntu.com/us.archive.ubuntu.com/g' /etc/apt/sources.list @@ -14,7 +14,7 @@ fi # Install VMWare Workstation 15 apt-get update -apt-get install -y linux-headers-"$(uname -r)" build-essential unzip git ufw apache2 python-pip +apt-get install -y linux-headers-"$(uname -r)" build-essential unzip git ufw apache2 python-pip ubuntu-desktop pip install awscli --upgrade --user export PATH=$PATH:/root/.local/bin @@ -22,11 +22,8 @@ wget -O VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle "https://download3 chmod +x VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle sudo sh VMware-Workstation-Full-15.0.4-12990004.x86_64.bundle --console --required --eulas-agreed --set-setting vmware-workstation serialNumber $SERIALNUMBER -echo "building" > /var/www/html/index.html - # Set up firewall ufw allow ssh -ufw allow http ufw default allow outgoing ufw --force enable @@ -39,7 +36,8 @@ wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.4/vagrant_2 dpkg -i vagrant_2.2.4_x86_64.deb vagrant plugin install vagrant-reload vagrant plugin install vagrant-vmware-desktop -vagrant plugin license vagrant-vmware-desktop $LICENSEFILE +echo $LICENSEFILE | base64 -d > /tmp/license.lic +vagrant plugin license vagrant-vmware-desktop /tmp/license.lic wget --progress=bar:force "https://releases.hashicorp.com/vagrant-vmware-utility/1.0.7/vagrant-vmware-utility_1.0.7_x86_64.deb" dpkg -i vagrant-vmware-utility_1.0.7_x86_64.deb