Merge pull request #1 from clong/master

Pull Master from Origin

.gitignore

@@ -1,3 +1,4 @@
+Azure/Ansible/inventory.yml
 Vagrant/.vagrant/*
 Vagrant/vagrant*.log
 Packer/packer_cache/*
@@ -13,4 +14,5 @@ inventory.yml.bak
 *.box
 manifest.xml
 HyperV/.vagrant/*
-logger_variables.sh
+logger_variables.sh
+ESXi/Packer/variables.json

AWS/Terraform/main.tf

@@ -193,6 +193,7 @@ resource "aws_instance" "logger" {
       "sudo sed -i 's/eth1/ens5/g' /opt/DetectionLab/Vagrant/logger_bootstrap.sh",
       "sudo sed -i 's/ETH1/ens5/g' /opt/DetectionLab/Vagrant/logger_bootstrap.sh",
       "sudo sed -i 's/eth1/ens5/g' /opt/DetectionLab/Vagrant/resources/suricata/suricata.yaml",
+      "sudo sed -i -e '127,130d' /opt/DetectionLab/Vagrant/resources/suricata/suricata.yaml",
       "sudo sed -i 's#/vagrant/resources#/opt/DetectionLab/Vagrant/resources#g' /opt/DetectionLab/Vagrant/logger_bootstrap.sh",
       "sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config",
       "sudo service ssh restart",

ESXi/Packer/ubuntu1804_esxi.json

@@ -26,36 +26,45 @@
                 "<enter><wait>"
             ],
             "boot_wait": "10s",
-            "keep_registered": true,
+            "cpus": "{{ user `cpus` }}",
-            "remote_datastore": "{{user `esxi_datastore`}}",
-            "remote_host": "{{user `esxi_host`}}",
-            "remote_username": "{{user `esxi_username`}}",
-            "remote_password": "{{user `esxi_password`}}",
-            "remote_type": "esx5",
-            "vnc_disable_password": true,
-            "vnc_port_min": 5900,
-            "vnc_port_max": 5980,
             "disk_size": "{{user `disk_size`}}",
             "guest_os_type": "ubuntu-64",
             "http_directory": "{{user `http_directory`}}",
             "iso_checksum": "{{user `iso_checksum`}}",
             "iso_url": "{{user `mirror`}}/{{user `mirror_directory`}}/{{user `iso_name`}}",
+            "keep_registered": true,
             "shutdown_command": "echo 'vagrant' | sudo -S shutdown -P now",
             "ssh_password": "vagrant",
             "ssh_port": 22,
             "ssh_username": "vagrant",
             "ssh_timeout": "10000s",
-            "pause_before_connecting": "10m",
+            "memory": "{{ user `memory` }}",
+            "pause_before_connecting": "1m",
+            "remote_datastore": "{{user `esxi_datastore`}}",
+            "remote_host": "{{user `esxi_host`}}",
+            "remote_username": "{{user `esxi_username`}}",
+            "remote_password": "{{user `esxi_password`}}",
+            "remote_type": "esx5",
+            "skip_export": true,
             "tools_upload_flavor": "linux",
             "type": "vmware-iso",
             "vm_name": "Ubuntu1804",
-            "memory": "{{ user `memory` }}",
-            "cpus": "{{ user `cpus` }}",
             "vmx_data": {
                 "ethernet0.networkName": "{{user `esxi_network_with_dhcp_and_internet` }}",
                 "cpuid.coresPerSocket": "1",
-                "ethernet0.pciSlotNumber": "32"
+                "ethernet0.pciSlotNumber": "32",
-            }
+                "tools.syncTime": "0",
+                "time.synchronize.continue": "0",
+                "time.synchronize.restore": "0",
+                "time.synchronize.resume.disk": "0",
+                "time.synchronize.shrink": "0",
+                "time.synchronize.tools.startup": "0",
+                "time.synchronize.tools.enable": "0",
+                "time.synchronize.resume.host": "0"
+            },
+            "vnc_disable_password": true,
+            "vnc_port_min": 5900,
+            "vnc_port_max": 5980
         }
     ],
     "provisioners": [

ESXi/Packer/windows_10_esxi.json

@@ -1,31 +1,10 @@
 {
   "builders": [
     {
-      "vnc_disable_password": true,
-      "keep_registered": true,
-      "remote_datastore": "{{user `esxi_datastore`}}",
-      "remote_host": "{{user `esxi_host`}}",
-      "remote_username": "{{user `esxi_username`}}",
-      "remote_password": "{{user `esxi_password`}}",
-      "remote_type": "esx5",
-      "type": "vmware-iso",
-      "vm_name":"Windows10",
-      "communicator": "winrm",
-      "iso_url": "{{user `iso_url`}}",
-      "iso_checksum": "{{user `iso_checksum`}}",
-      "headless": false,
       "boot_wait": "6m",
       "boot_command": "",
-      "winrm_username": "vagrant",
+      "communicator": "winrm",
-      "winrm_password": "vagrant",
+      "disk_size": 61440,
-      "winrm_timeout": "4h",
-      "shutdown_timeout": "2h",
-      "shutdown_command": "a:/sysprep.bat",
-      "guest_os_type": "windows9-64",
-      "disk_size": "{{user `disk_size`}}",
-      "vnc_port_min": 5900,
-      "vnc_port_max": 5980,
-      "version": 11,
       "floppy_files": [
         "{{user `autounattend`}}",
         "../../Packer/floppy/WindowsPowershell.lnk",
@@ -40,12 +19,42 @@
         "../../Packer/scripts/unattend.xml",
         "../../Packer/scripts/sysprep.bat"
       ],
+      "guest_os_type": "windows9-64",
+      "keep_registered": true,
+      "headless": false,
+      "iso_url": "{{user `iso_url`}}",
+      "iso_checksum": "{{user `iso_checksum`}}",
+      "shutdown_timeout": "2h",
+      "shutdown_command": "a:/sysprep.bat",
+      "skip_export": true,
+      "remote_datastore": "{{user `esxi_datastore`}}",
+      "remote_host": "{{user `esxi_host`}}",
+      "remote_username": "{{user `esxi_username`}}",
+      "remote_password": "{{user `esxi_password`}}",
+      "remote_type": "esx5",
+      "type": "vmware-iso",
+      "version": 11,
+      "vm_name": "Windows10",
+      "vnc_disable_password": true,
+      "vnc_port_min": 5900,
+      "vnc_port_max": 5980,
       "vmx_data": {
         "ethernet0.networkName": "{{user `esxi_network_with_dhcp_and_internet`}}",
         "memsize": "2048",
         "numvcpus": "2",
-        "scsi0.virtualDev": "lsisas1068"
+        "scsi0.virtualDev": "lsisas1068",
-      }
+        "tools.syncTime": "0",
+        "time.synchronize.continue": "0",
+        "time.synchronize.restore": "0",
+        "time.synchronize.resume.disk": "0",
+        "time.synchronize.shrink": "0",
+        "time.synchronize.tools.startup": "0",
+        "time.synchronize.tools.enable": "0",
+        "time.synchronize.resume.host": "0"
+      },
+    "winrm_username": "vagrant",
+    "winrm_password": "vagrant",
+    "winrm_timeout": "4h"
     }
   ],
   "provisioners": [
@@ -93,8 +102,7 @@
     "esxi_password": "",
     "iso_checksum": "sha256:ab4862ba7d1644c27f27516d24cb21e6b39234eb3301e5f1fb365a78b22f79b3",
     "iso_url": "https://software-download.microsoft.com/download/pr/18362.30.190401-1528.19h1_release_svc_refresh_CLIENTENTERPRISEEVAL_OEMRET_x64FRE_en-us.iso",
-    "autounattend": "../../Packer/answer_files/10/Autounattend.xml",
+    "autounattend": "../../Packer/answer_files/10/Autounattend.xml"
-    "disk_size": "61440"
   }
 }
 

ESXi/Packer/windows_2016_esxi.json

@@ -1,28 +1,9 @@
 {
   "builders": [
     {
-      "vnc_disable_password": true,
-      "keep_registered": true,
-      "remote_datastore": "{{user `esxi_datastore`}}",
-      "remote_host": "{{user `esxi_host`}}",
-      "remote_username": "{{user `esxi_username`}}",
-      "remote_password": "{{user `esxi_password`}}",
-      "remote_type": "esx5",
-      "vm_name":"WindowsServer2016",
-      "type": "vmware-iso",
-      "communicator": "winrm",
-      "iso_url": "{{user `iso_url`}}",
-      "iso_checksum": "{{user `iso_checksum`}}",
-      "headless": false,
       "boot_wait": "2m",
-      "winrm_username": "vagrant",
+      "communicator": "winrm",
-      "winrm_password": "vagrant",
-      "winrm_timeout": "4h",
-      "shutdown_timeout": "2h",
-      "shutdown_command": "a:/sysprep.bat",
-      "guest_os_type": "windows8srv-64",
       "disk_size": 61440,
-      "version": 11,
       "floppy_files": [
         "{{user `autounattend`}}",
         "../../Packer/floppy/WindowsPowershell.lnk",
@@ -35,12 +16,40 @@
         "../../Packer/scripts/microsoft-updates.bat",
         "../../Packer/scripts/win-updates.ps1"
       ],
+      "guest_os_type": "windows8srv-64",
+      "headless": false,
+      "iso_url": "{{user `iso_url`}}",
+      "iso_checksum": "{{user `iso_checksum`}}",
+      "keep_registered": true,
+      "shutdown_timeout": "2h",
+      "shutdown_command": "a:/sysprep.bat",
+      "skip_export": true,
+      "remote_datastore": "{{user `esxi_datastore`}}",
+      "remote_host": "{{user `esxi_host`}}",
+      "remote_username": "{{user `esxi_username`}}",
+      "remote_password": "{{user `esxi_password`}}",
+      "remote_type": "esx5",
+      "type": "vmware-iso",
+      "version": 11,
+      "vm_name": "WindowsServer2016",
+      "vnc_disable_password": true,
       "vmx_data": {
         "ethernet0.networkName": "{{user `esxi_network_with_dhcp_and_internet`}}",
         "memsize": "2048",
         "numvcpus": "2",
-        "scsi0.virtualDev": "lsisas1068"
+        "scsi0.virtualDev": "lsisas1068",
-      }
+        "tools.syncTime": "0",
+        "time.synchronize.continue": "0",
+        "time.synchronize.restore": "0",
+        "time.synchronize.resume.disk": "0",
+        "time.synchronize.shrink": "0",
+        "time.synchronize.tools.startup": "0",
+        "time.synchronize.tools.enable": "0",
+        "time.synchronize.resume.host": "0"
+      },
+      "winrm_username": "vagrant",
+      "winrm_password": "vagrant",
+      "winrm_timeout": "4h"
     }
   ],
   "provisioners": [

ESXi/ansible/inventory.yml

@@ -7,7 +7,7 @@ logger:
       ansible_password: vagrant
       ansible_port: 22
       ansible_connection: ssh
-      ansible_ssh_common_args: '-o StrictHostKeyChecking=no'
+      ansible_ssh_common_args: '-o UserKnownHostsFile=/dev/null'
 
 dc:
   hosts:

ESXi/ansible/roles/logger/tasks/main.yml

@@ -300,7 +300,7 @@
     executable: /bin/bash
   become: yes
   shell: |
-    if [ -f "/opt/fleet" ]; then
+    if [ -d "/opt/fleet" ]; then
       echo "[$(date +%H:%M:%S)]: Fleet is already installed"
     else
       cd /opt || exit 1
@@ -314,7 +314,7 @@
       mysql -uroot -pkolide -e "create database kolide;"
 
       # Always download the latest release of Fleet
-      curl -s https://api.github.com/repos/fleetdm/fleet/releases/latest | grep 'https://github.com' | grep "/fleet.zip" | cut -d ':' -f 2,3 | tr -d '"' | wget --progress=bar:force -i -
+      curl -s https://api.github.com/repos/fleetdm/fleet/releases | grep 'https://github.com' | grep "/fleet.zip" | cut -d ':' -f 2,3 | tr -d '"' | tr -d ' '  | head -1 | wget --progress=bar:force -i -
       unzip fleet.zip -d fleet
       cp fleet/linux/fleetctl /usr/local/bin/fleetctl && chmod +x /usr/local/bin/fleetctl
       cp fleet/linux/fleet /usr/local/bin/fleet && chmod +x /usr/local/bin/fleet

ESXi/main.tf

@@ -29,9 +29,8 @@ resource "esxi_guest" "logger" {
 
     provisioner "remote-exec" {
     inline = [
-      "sudo ifconfig eth1 up || echo 'eth1 up'",
+      "sudo ifconfig eth0 up && echo 'eth0 up' || echo 'unable to bring eth0 interface up",
-      "sudo ifconfig eth2 up || echo 'eth2 up'",
+      "sudo ifconfig eth1 up && echo 'eth1 up' || echo 'unable to bring eth1 interface up"
-      "sudo route add default gw 192.168.76.1 || echo 'route exists'"
     ]
 
     connection {
@@ -68,11 +67,10 @@ resource "esxi_guest" "logger" {
 
 resource "esxi_guest" "dc" {
   guest_name = "dc"
-  disk_store = "datastore2"
+  disk_store = var.esxi_datastore
   guestos    = "windows9srv-64"
 
   boot_disk_type = "thin"
-  boot_disk_size = "35"
 
   memsize            = "4096"
   numvcpus           = "2"
@@ -97,11 +95,10 @@ resource "esxi_guest" "dc" {
 
 resource "esxi_guest" "wef" {
   guest_name = "wef"
-  disk_store = "datastore2"
+  disk_store = var.esxi_datastore
   guestos    = "windows9srv-64"
 
   boot_disk_type = "thin"
-  boot_disk_size = "35"
 
   memsize            = "2048"
   numvcpus           = "2"
@@ -126,11 +123,10 @@ resource "esxi_guest" "wef" {
 
 resource "esxi_guest" "win10" {
   guest_name = "win10"
-  disk_store = "datastore2"
+  disk_store = var.esxi_datastore
   guestos    = "windows9-64"
 
   boot_disk_type = "thin"
-  boot_disk_size = "35"
 
   memsize            = "2048"
   numvcpus           = "2"

ESXi/variables.tf

@@ -1,7 +1,8 @@
 #
 #  See https://www.terraform.io/intro/getting-started/variables.html for more details.
 #
-#  Change these defaults to fit your needs!
+#  Don't change the variables in this file! 
+#  Instead, create a terrform.tfvars file to override them.
 
 variable "esxi_hostname" {
   default = ""

HyperV/hyperv-create-nat-switch.ps1

@@ -1,32 +1,40 @@
 # See: https://www.petri.com/using-nat-virtual-switch-hyper-v
 
-If ("NATSwitch" -in (Get-VMSwitch | Select-Object -ExpandProperty Name) -eq $FALSE) {
+$NATHostIP = "192.168.38.1"
-    'Creating Internal-only switch named "NATSwitch" on Windows Hyper-V host...'
+$NATNetPrefixLength = 24
+$NATNet = "192.168.38.0/$NATNetPrefixLength"
+$NATNetName = "NATNetwork"
+$NATSwitchName = "NATSwitch"
+$NATSwitchNameAlias = "vEthernet ($NATSwitchName)"
 
-    New-VMSwitch -SwitchName "NATSwitch" -SwitchType Internal
+# Check our NAT switch exists, create it and configure it if it doesn't.
+If ("$NATSwitchName" -in (Get-VMSwitch | Select-Object -ExpandProperty Name) -eq $FALSE) {
+    "Creating Internal-only switch named ""$NatSwitchName"" on Windows Hyper-V host..."
 
-    New-NetIPAddress -IPAddress 192.168.38.1 -PrefixLength 24 -InterfaceAlias "vEthernet (NATSwitch)"
+    New-VMSwitch -SwitchName $NATSwitchName -SwitchType Internal
+    New-NetIPAddress -IPAddress $NATHostIP -PrefixLength $NATNetPrefixLength -InterfaceAlias $NATSwitchNameAlias
+    New-NetNAT -Name $NATNetName -InternalIPInterfaceAddressPrefix $NATNet
 
-    New-NetNAT -Name "NATNetwork" -InternalIPInterfaceAddressPrefix 192.168.38.0/24
+} else {
-}
+    """$NATSwitchName"" VM Switch on Hyper-V host for guest static IP configuration already exists; skipping..."
-else {
-    '"NATSwitch" for static IP configuration already exists; skipping'
 }
 
-If ("192.168.38.1" -in (Get-NetIPAddress | Select-Object -ExpandProperty IPAddress) -eq $FALSE) {
+# Check that our Hyper-V host has the proper gateway address for the NAT Network.
-    'Registering new IP address 192.168.38.1 on Windows Hyper-V host...'
+If (@(Get-NetIPAddress | Where-Object {$_.IPAddress -eq "$NATHostIP" -and $_.InterfaceAlias -eq "$NATSwitchNameAlias"}).Count -eq 1) {
+    "Registering new IP address $NATHostIP on Windows Hyper-V host..."
 
-    New-NetIPAddress -IPAddress 192.168.38.1 -PrefixLength 24 -InterfaceAlias "vEthernet (NATSwitch)"
+    New-NetIPAddress -IPAddress $NATHostIP -PrefixLength $NATNetPrefixLength -InterfaceAlias $NATSwitchNameAlias
-}
+
-else {
+} else {
-    '"192.168.38.1" for static IP configuration already registered; skipping'
+    """$NATHostIP"" Hyper-V host gateway address for guest static IP configuration already registered; skipping..."
 }
 
-If ("192.168.38.0/24" -in (Get-NetNAT | Select-Object -ExpandProperty InternalIPInterfaceAddressPrefix) -eq $FALSE) {
+# Check that our Hyper-V host has the proper NAT Network setup
-    'Registering new NAT adapter for 192.168.38.0/24 on Windows Hyper-V host...'
+If ("$NATNet" -in (Get-NetNAT | Select-Object -ExpandProperty InternalIPInterfaceAddressPrefix) -eq $FALSE) {
+    "Registering new NAT adapter for $NATNet on Windows Hyper-V host..."
 
-    New-NetNAT -Name "NATNetwork" -InternalIPInterfaceAddressPrefix 192.168.38.0/24
+    New-NetNAT -Name $NATNetName -InternalIPInterfaceAddressPrefix $NATNet
-}
+
-else {
+} else {
-    '"192.168.38.0/24" for static IP configuration already registered; skipping'
+    """$NATNet"" Hyper-V host NAT Network for guest static IP configuration already registered; skipping"
 }

Packer/windows_2016.json

@@ -83,7 +83,7 @@
     },
     {
       "type": "qemu",
-      "vm_name": "windows_10",
+      "vm_name": "WindowsServer2016",
       "communicator": "winrm",
       "iso_url": "{{user `iso_url`}}",
       "iso_checksum": "{{user `iso_checksum`}}",

README.md

@@ -6,9 +6,10 @@ DetectionLab is tested weekly on Saturdays via a scheduled CircleCI workflow to
 [![CircleCI](https://circleci.com/gh/clong/DetectionLab/tree/master.svg?style=shield)](https://circleci.com/gh/clong/DetectionLab/tree/master)
 ![Lint Code Base](https://github.com/clong/DetectionLab/workflows/Lint%20Code%20Base/badge.svg)
 [![license](https://img.shields.io/github/license/clong/DetectionLab.svg?style=flat-square)](https://github.com/clong/DetectionLab/blob/master/license.md)
-![Maintenance](https://img.shields.io/maintenance/yes/2020.svg?style=flat-square)
+![Maintenance](https://img.shields.io/maintenance/yes/2021.svg?style=flat-square)
 [![GitHub last commit](https://img.shields.io/github/last-commit/clong/DetectionLab.svg?style=flat-square)](https://github.com/clong/DetectionLab/commit/master)
 [![Twitter](https://img.shields.io/twitter/follow/DetectionLab.svg?style=social)](https://twitter.com/DetectionLab)
+[![Slack](https://img.shields.io/badge/Slack-DetectionLab-blue)](https://join.slack.com/t/detectionlab/shared_invite/zt-mv1qnw9f-3qo2ZrB0IbIKhvinfsgYhg)
 
 #### Donate to the project:
 
@@ -103,13 +104,13 @@ A sizable percentage of this code was borrowed and adapted from [Stefan Scherer]
 * [EVTX-ATTACK-SAMPLES](https://github.com/sbousseaden/EVTX-ATTACK-SAMPLES)
 
 # DetectionLab Sponsors
-#### Lated updated: 9/16/2020
+#### Lated updated: 02/04/2020
 I would like to extend thanks to the following sponsors for funding DetectionLab development. If you are interested in becoming a sponsor, please visit the [sponsors page](https://github.com/sponsors/clong).
 
 ### Diamond Sponsors:
 * [Veramine](https://github.com/veramine)
 * [Thinkst](https://github.com/ThinkstAppliedResearch)
-* [csterner82](https://github.com/csterner82)
+* [kungskal](https://github.com/kungskal)
 * [swizzlez](https://github.com/swizzlez)
 * [CyDefUnicorn](https://github.com/CyDefUnicorn)
 * [olliencc](https://github.com/olliencc)
@@ -118,17 +119,18 @@ I would like to extend thanks to the following sponsors for funding DetectionLab
 * [mikeesparza](https://github.com/mikeesparza)
 * [dlee35](https://github.com/dlee35)
 * [chrissanders](https://github.com/chrissanders)
-* [punchdrunktux](https://github.com/punchdrunktux)
 * [jaredhaight](https://github.com/jaredhaight)
 * [iamfuntime](https://github.com/iamfuntime)
 * [Luct0r](https://github.com/Luct0r)
+* [zhuma549](https://github.com/zhuma549)
+* +1 private sponsor
 
 ### Standard Sponsors:
-* [dtonomy](https://github.com/dtonomy)
 * [braimee](https://github.com/braimee)
-* [iLoC0dez](https://github.com/iLoC0dez)
 * [defensivedepth](https://github.com/defensivedepth)
-* [elreydetoda](https://github.com/elreydetoda)
 * [kafkaesqu3](https://github.com/kafkaesqu3)
-* [anthonysecurity](https://github.com/anthonysecurity)
+* [mdtro](https://github.com/mdtro)
+* [ealaney](https://github.com/ealaney)
+* [elreydetoda](https://github.com/elreydetoda)
+* [DevBits1702](https://github.com/DevBits1702)
 * +2 private sponsors

Vagrant/logger_bootstrap.sh

@@ -195,6 +195,10 @@ install_splunk() {
     cp /vagrant/resources/splunk_server/windows_ta_props.conf /opt/splunk/etc/apps/Splunk_TA_windows/default/props.conf
     cp /vagrant/resources/splunk_server/sysmon_ta_props.conf /opt/splunk/etc/apps/TA-microsoft-sysmon/default/props.conf
 
+    # Add props.conf to Splunk Zeek TA to properly parse timestamp
+    # and avoid grouping events as a single event
+    cp /vagrant/resources/splunk_server/zeek_ta_props.conf /opt/splunk/etc/apps/Splunk_TA_bro/local/props.conf
+
     # Add custom Macro definitions for ThreatHunting App
     cp /vagrant/resources/splunk_server/macros.conf /opt/splunk/etc/apps/ThreatHunting/default/macros.conf
     # Fix props.conf in ThreatHunting App
@@ -267,7 +271,7 @@ install_fleet_import_osquery_config() {
     mysql -uroot -pkolide -e "create database kolide;"
 
     # Always download the latest release of Fleet
-    curl -s https://api.github.com/repos/fleetdm/fleet/releases/latest | grep 'https://github.com' | grep "/fleet.zip" | cut -d ':' -f 2,3 | tr -d '"' | wget --progress=bar:force -i -
+    curl -s https://api.github.com/repos/fleetdm/fleet/releases | grep 'https://github.com' | grep "/fleet.zip" | cut -d ':' -f 2,3 | tr -d '"' | tr -d ' '  | head -1 | wget --progress=bar:force -i -
     unzip fleet.zip -d fleet
     cp fleet/linux/fleetctl /usr/local/bin/fleetctl && chmod +x /usr/local/bin/fleetctl
     cp fleet/linux/fleet /usr/local/bin/fleet && chmod +x /usr/local/bin/fleet
@@ -377,6 +381,17 @@ install_zeek() {
   crudini --set $NODECFG proxy host localhost
 
   # Setup $CPUS numbers of Zeek workers
+  # AWS only has a single interface (eth1), so don't monitor eth0 if we're in AWS
+  if ! curl -s 169.254.169.254 --connect-timeout 2 >/dev/null; then 
+  # TL;DR of ^^^: if you can't reach the AWS metadata service, you're not running in AWS
+  # Therefore, it's ok to add this.
+    crudini --set $NODECFG worker-eth0 type worker
+    crudini --set $NODECFG worker-eth0 host localhost
+    crudini --set $NODECFG worker-eth0 interface eth0
+    crudini --set $NODECFG worker-eth0 lb_method pf_ring
+    crudini --set $NODECFG worker-eth0 lb_procs "$(nproc)"
+  fi
+  
   crudini --set $NODECFG worker-eth1 type worker
   crudini --set $NODECFG worker-eth1 host localhost
   crudini --set $NODECFG worker-eth1 interface eth1
@@ -391,7 +406,7 @@ install_zeek() {
   # Configure the Splunk inputs
   mkdir -p /opt/splunk/etc/apps/Splunk_TA_bro/local && touch /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf
   crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager index zeek
-  crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager sourcetype bro:json
+  crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager sourcetype zeek:json
   crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager whitelist '.*\.log$'
   crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager blacklist '.*(communication|stderr)\.log$'
   crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager disabled 0
@@ -464,12 +479,11 @@ install_suricata() {
   suricata-update enable-source ptresearch/attackdetection
 
   # Configure the Splunk inputs
-  mkdir -p /opt/splunk/etc/apps/SplunkLightForwarder/local && touch /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf
+  crudini --set /opt/splunk/etc/apps/search/local/inputs.conf monitor:///var/log/suricata index suricata
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata index suricata
+  crudini --set /opt/splunk/etc/apps/search/local/inputs.conf monitor:///var/log/suricata sourcetype suricata:json
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata sourcetype suricata:json
+  crudini --set /opt/splunk/etc/apps/search/local/inputs.conf monitor:///var/log/suricata whitelist 'eve.json'
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata whitelist 'eve.json'
+  crudini --set /opt/splunk/etc/apps/search/local/inputs.conf monitor:///var/log/suricata disabled 0
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata disabled 0
+  crudini --set /opt/splunk/etc/apps/search/local/props.conf suricata:json TRUNCATE 0
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/props.conf json_suricata TRUNCATE 0
 
   # Update suricata and restart
   suricata-update

Vagrant/prepare.ps1

@@ -253,6 +253,14 @@ function preflight_checks {
   else {
     Write-Host '  ['$($checkmark)'] The vagrant-reload plugin is installed' -ForegroundColor green
   }
+
+  # Warn if Virtual Machine Platform is enabled
+  Write-Host ''
+  Write-Host '[+] Checking if Virtual Machine Platform is enabled...'
+  if ((Get-WindowsOptionalFeature -Online -FeatureName "VirtualMachinePlatform").State -eq "Enabled") {
+    Write-Host '  [-] The "Virtual Machine Platform" Windows feature is enabled on your computer and is known to cause issues with starting Virtualbox VMs' -ForegroundColor yellow
+    Write-Host '  [-] If you experience issues with Virtualbox, it is recommended to disable it by running "optionalfeatures" from the run prompt and unchecking "Virtual Machine Platform".' -ForegroundColor yellow
+  }
 }
 
 

Vagrant/prepare.sh

@@ -119,6 +119,14 @@ check_vagrant_vmware_utility_installed() {
   fi
 }
 
+check_macos_bigsur() {
+  if sw_vers | grep ProductVersion | grep -c 11\. > /dev/null; then
+    echo "1"
+  else
+    echo "0"
+  fi
+}
+
 # List the available Vagrant providers present on the system
 list_providers() {
   VBOX_PRESENT=0
@@ -131,6 +139,7 @@ list_providers() {
     VMWARE_WORKSTATION_PRESENT=0 # Workstation doesn't exist on Darwain-based OS
     VAGRANT_VMWARE_DESKTOP_PLUGIN_PRESENT=$(check_vmware_desktop_vagrant_plugin_installed)
     VAGRANT_VMWARE_UTILITY_PRESENT=$(check_vagrant_vmware_utility_installed)
+    IS_BIGSUR=$(check_macos_bigsur)
   else
     VBOX_PRESENT=$(check_virtualbox_installed)
     VMWARE_WORKSTATION_PRESENT=$(check_vmware_workstation_installed)
@@ -159,6 +168,10 @@ list_providers() {
     (echo >&2  "${INFO} More details can be found here: https://www.vagrantup.com/docs/providers/default" )
     (echo >&2  "${INFO} Additionally, please ensure only one providers' network adapters are active at any given time." )
   fi
+  if [[ $VMWARE_FUSION_PRESENT -eq 1 ]] && [[ $IS_BIGSUR -eq 1 ]]; then
+    (echo >&2 "${INFO} A workaround is currently required to use VMware Fusion with Big Sur.")
+    (echo >&2 "${INFO} See https://github.com/clong/DetectionLab/issues/539 for more info.")
+  fi
 }
 
 # Check to see if boxes exist in the "Boxes" directory already

Vagrant/resources/splunk_server/logger_dashboard.xml

@@ -69,13 +69,14 @@
       <title>Zeek Network Traffic by Type</title>
       <chart>
         <search>
-          <query>index=zeek | stats count by _time, tag::eventtype | timechart span=1h count by tag::eventtype</query>
+          <query>| tstats count where index=zeek by source, _time span=1h prestats=t | timechart span=1h count by source useother=f</query>
           <earliest>-24h@h</earliest>
           <latest>now</latest>
         </search>
         <option name="charting.chart">column</option>
         <option name="charting.chart.stackMode">stacked</option>
         <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
       </chart>
     </panel>
   </row>
@@ -125,16 +126,18 @@
       <table>
         <title>http://findingbad.blogspot.com/2020/05/hunting-for-beacons-part-2.html</title>
         <search>
-          <query>index=zeek (dest_port=443 OR dest_port=80)  
+          <query>index=zeek (dest_port=443 OR dest_port=80)  dest_ip!=192.168.0.0/16
-| rename orig_bytes as bytes_out resp_bytes as bytes_in 
+| rename orig_bytes as bytes_out resp_bytes as bytes_in
-| stats count(bytes_out) as "beacon_count" values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out |eventstats sum(beacon_count) as total_count dc(bytes_out) as unique_count by src_ip,dest_ip 
+| stats count(bytes_out) as "beacon_count" values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out |eventstats sum(beacon_count) as total_count dc(bytes_out) as unique_count by src_ip,dest_ip
-| eval beacon_avg=('beacon_count' / 'total_count') 
+| eval beacon_avg=('beacon_count' / 'total_count')
-| stats values(beacon_count) as beacon_count values(unique_count) as unique_count values(beacon_avg) as beacon_avg values(total_count) as total_count values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out 
+| stats values(beacon_count) as beacon_count values(unique_count) as unique_count values(beacon_avg) as beacon_avg values(total_count) as total_count values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out
-| head 100
+| eval beacon_avg=('beacon_count' / 'total_count')
-| eval incount=mvcount(bytes_in) 
+| stats values(beacon_count) as beacon_count values(unique_count) as unique_count values(beacon_avg) as beacon_avg values(total_count) as total_count values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out
-| eventstats avg(beacon_count) as overall_average 
+| eval incount=mvcount(bytes_in)
-| eval beacon_percentage=('beacon_count' / 'overall_average') 
+| eventstats avg(beacon_count) as overall_average
-| sort - beacon_percentage</query>
+| eval beacon_percentage=('beacon_count' / 'overall_average')
+| sort - beacon_percentage
+| fields - incount,overall_average</query>
           <earliest>-24h@h</earliest>
           <latest>now</latest>
         </search>
@@ -215,4 +218,4 @@
       </chart>
     </panel>
   </row>
-</dashboard>
+</dashboard>

Vagrant/resources/splunk_server/zeek_ta_props.conf

@@ -0,0 +1,12 @@
+[zeek:json]
+DATETIME_CONFIG =
+INDEXED_EXTRACTIONS = json
+KV_MODE = none
+LINE_BREAKER = ([\r\n]+)
+NO_BINARY_CHECK = true
+category = Structured
+description = Zeek JSON sourcetype with fixed timestamp parsing.
+disabled = false
+pulldown_type = true
+TIMESTAMP_FIELDS = ts
+TIME_FORMAT = %s.%6N

Vagrant/resources/suricata/suricata.yaml

@@ -124,11 +124,14 @@ logging:
       facility: local5
       format: "[%i] <%d> -- "
 af-packet:
+  - interface: eth0
+    cluster-id: 98
+    cluster-type: cluster_flow
+    defrag: yes
   - interface: eth1
     cluster-id: 99
     cluster-type: cluster_flow
     defrag: yes
-  - interface: default
 pcap-file:
   checksum-checks: auto
 app-layer:

Vagrant/scripts/install-autorunstowineventlog.ps1

@@ -3,6 +3,14 @@
 Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Installing AutorunsToWinEventLog..."
 If ((Get-ScheduledTask -TaskName "AutorunsToWinEventLog" -ea silent) -eq $null)
 {
+    # Modify the installer to add an HTTP fallback until this gets fixed upstream in the windows-event-fowarding repo
+    # See https://github.com/clong/DetectionLab/issues/597
+    (Get-Content c:\Users\vagrant\AppData\Local\Temp\windows-event-forwarding-master\AutorunsToWinEventLog\Install.ps1 -Raw) -replace 'Invoke-WebRequest -Uri "https://live.sysinternals.com/autorunsc64.exe" -OutFile "\$autorunsPath"', 'Try {
+    (New-Object System.Net.WebClient).DownloadFile(''https://live.sysinternals.com/Autoruns64.exe'', $autorunsPath)
+  } Catch {
+    Write-Host "HTTPS connection failed. Switching to HTTP :("
+    (New-Object System.Net.WebClient).DownloadFile(''http://live.sysinternals.com/Autoruns64.exe'', $autorunsPath)
+  }' | Set-Content -Path "c:\Users\vagrant\AppData\Local\Temp\windows-event-forwarding-master\AutorunsToWinEventLog\Install.ps1"
     . c:\Users\vagrant\AppData\Local\Temp\windows-event-forwarding-master\AutorunsToWinEventLog\Install.ps1
     Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) AutorunsToWinEventLog installed. Starting the scheduled task. Future runs will begin at 11am"
     Start-ScheduledTask -TaskName "AutorunsToWinEventLog"

Vagrant/scripts/install-sysinternals.ps1

@@ -93,7 +93,9 @@ $Shortcut.TargetPath = $tcpviewPath
 $Shortcut.Save()
 
 # Restart Explorer so the taskbar shortcuts show up
-Stop-Process -ProcessName explorer -Force
+if (Get-Process -ProcessName explorer -ErrorAction 'silentlycontinue') {
+  Stop-Process -ProcessName explorer -Force
+}
 
 # Download Olaf Hartongs Sysmon config
 Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Downloading Olaf Hartong's Sysmon config..."

Vagrant/scripts/install-utilities.ps1

@@ -15,6 +15,9 @@ If ($(hostname) -eq "win10") {
   & "C:\Program Files\Classic Shell\ClassicStartMenu.exe" "-xml" "c:\vagrant\resources\windows\MenuSettings.xml"
   regedit /s c:\vagrant\resources\windows\MenuStyle_Default_Win7.reg
 }
-choco install -y --limit-output --no-progress NotepadPlusPlus GoogleChrome WinRar 
+choco install -y --limit-output --no-progress NotepadPlusPlus WinRar processhacker
+
+# This repo often causes failures due to incorrect checksums, so we ignore them for Chrome
+choco install -y --limit-output --no-progress --ignore-checksums GoogleChrome 
 
 Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Utilties installation complete!"

ci/build_machine_bootstrap.sh

@@ -79,8 +79,8 @@ ufw --force enable
 echo "[$(date +%H:%M:%S)]: Installing Vagrant..."
 mkdir /opt/vagrant
 cd /opt/vagrant || exit 1
-wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.10/vagrant_2.2.10_x86_64.deb
+wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.14/vagrant_2.2.14_x86_64.deb
-dpkg -i vagrant_2.2.10_x86_64.deb
+dpkg -i vagrant_2.2.14_x86_64.deb
 echo "[$(date +%H:%M:%S)]: Installing vagrant-reload plugin..."
 vagrant plugin install vagrant-reload
 

ci/copy_to_s3.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+
+# This script is used to prepare DetectionLab to be imported as VM in AWS
+
+# Configure credentials for awscli
+aws configure set aws_access_key_id $AWS_ACCESS_KEY
+aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+aws configure set default.region us-west-1
+export BUCKET_NAME="FILL_ME_IN"
+
+cd /opt/DetectionLab/Vagrant || exit 1
+echo "Running WinRM Commands to open WinRM on the firewall..."
+for host in dc wef win10;
+do
+  echo "Running 'Set-NetFirewallRule -Name WINRM-HTTP-In-TCP -Profile Any' on $host..."
+  vagrant winrm -e -c "Set-NetFirewallRule -Name 'WINRM-HTTP-In-TCP' -Profile Any" -s powershell $host; sleep 2
+done
+echo "Running 'Set-NetFirewallRule -Name WINRM-HTTP-In-TCP-NoScope -Profile Any' on win10..."
+vagrant winrm -c "Set-NetFirewallRule -Name 'WINRM-HTTP-In-TCP-NoScope' -Profile Any" -s powershell win10; sleep 2
+
+echo "Running WinRM Commands to clear the event logs..."
+for host in dc wef win10;
+do
+  echo "Clearing event logs on $host..."
+  vagrant winrm -e -s powershell -c "Clear-Eventlog -Log Application, System" $host
+  sleep 2
+done
+
+echo "Printing activivation status of all hosts..."
+for host in dc wef win10;
+do
+  echo "$host"
+  vagrant winrm -s powershell -c "cscript c:\windows\system32\slmgr.vbs /dlv" $host
+  sleep 2
+done
+echo "If you're ready to continue, type y:"
+read READY
+
+if [ "$READY" != "y" ]; then
+  echo "Okay, quitting"
+  exit 1
+fi
+
+#echo "Re-arming WEF"
+#vagrant winrm -e -s powershell -c "cscript c:\windows\system32\slmgr.vbs /rearm" wef
+#echo "Activating Win10..."
+#vagrant winrm -e -s powershell -c "Set-Service TrustedInstaller -StartupType Automatic" win10
+#sleep 2
+#vagrant winrm -e -s powershell -c "Start-Service TrustedInstaller" win10
+#sleep 10
+#vagrant winrm -e -s powershell -c "cscript c:\windows\system32\slmgr.vbs /ato " win10
+
+# Stop vagrant and export each box as an OVA
+cd /opt/DetectionLab/Vagrant || exit 1
+echo "Halting all VMs..."
+vagrant halt
+
+echo "Creating a new tmux session..."
+sn=tmuxsession
+tmux new-session -s "$sn" -d
+tmux new-window -t "$sn:2" -n "dc" -d
+tmux new-window -t "$sn:3" -n "wef" -d
+tmux new-window -t "$sn:4" -n "win10" -d
+if which vmrun; then
+  tmux send-keys -t "$sn:2" 'ovftool /opt/DetectionLab/Vagrant/.vagrant/machines/dc/vmware_desktop/*/WindowsServer2016.vmx /root/dc.ova && echo -n "success" > /root/dc.export || echo "failed" > /root/dc.export' Enter
+  tmux send-keys -t "$sn:3" 'ovftool /opt/DetectionLab/Vagrant/.vagrant/machines/wef/vmware_desktop/*/WindowsServer2016.vmx /root/wef.ova && echo -n "success" > /root/wef.export || echo "failed" > /root/wef.export' Enter
+  tmux send-keys -t "$sn:4" 'ovftool /opt/DetectionLab/Vagrant/.vagrant/machines/win10/vmware_desktop/*/windows_10.vmx /root/win10.ova && echo -n "success" > /root/win10.export || echo "failed" > /root/win10.export' Enter
+else
+  tmux send-keys -t "$sn:2" 'vboxmanage export dc.windomain.local -o /root/dc.ova && echo -n "success" > /root/dc.export || echo "failed" > /root/dc.export' Enter
+  tmux send-keys -t "$sn:3" 'vboxmanage export wef.windomain.local -o /root/wef.ova && echo -n "success" > /root/wef.export || echo "failed" > /root/wef.export' Enter
+  tmux send-keys -t "$sn:4" 'vboxmanage export win10.windomain.local -o /root/win10.ova && echo -n "success" > /root/win10.export || echo "failed" > /root/win10.export' Enter
+fi
+
+# Sleep until all exports are complete
+while [[ ! -f /root/dc.export || ! -f /root/wef.export || ! -f /root/win10.export ]];
+  do sleep 5
+  echo "Waiting for the OVA export to complete. Sleeping for 5."
+done
+
+# Copy each OVA into S3
+if [[ "$(cat /root/dc.export)" == "success" && "$(cat /root/wef.export)" == "success" && "$(cat /root/win10.export)" == "success" ]]; then
+  for file in dc wef win10
+  do
+    aws s3 cp /root/$file.ova s3://$BUCKET_NAME/disks/
+  done
+fi
+
+# Fix the bucket
+cd /opt/DetectionLab/AWS/Terraform/vm_import || exit 1
+for file in *.json;
+  do sed -i "s/YOUR_BUCKET_GOES_HERE/$BUCKET_NAME/g" "$file";
+done
+
+# Fix the key names
+for file in *.json;
+  do sed -i 's#"S3Key": "#"S3Key": "disks/#g' "$file";
+done
+
+aws ec2 import-image --description "dc" --license-type byol --disk-containers file:///opt/DetectionLab/AWS/Terraform/vm_import/dc.json
+aws ec2 import-image --description "wef" --license-type byol --disk-containers file:///opt/DetectionLab/AWS/Terraform/vm_import/wef.json
+aws ec2 import-image --description "win10" --license-type byol --disk-containers file:///opt/DetectionLab/AWS/Terraform/vm_import/win10.json

ci/manual_machine_bootstrap.sh

@@ -25,8 +25,8 @@ git clone https://github.com/clong/DetectionLab.git /opt/DetectionLab
 # Install Vagrant
 mkdir /opt/vagrant
 cd /opt/vagrant || exit 1
-wget https://releases.hashicorp.com/vagrant/2.2.10/vagrant_2.2.10_x86_64.deb
+wget https://releases.hashicorp.com/vagrant/2.2.14/vagrant_2.2.14_x86_64.deb
-dpkg -i vagrant_2.2.10_x86_64.deb
+dpkg -i vagrant_2.2.14_x86_64.deb
 
 # Disable IPv6 - may help with the vagrant-reload plugin: https://github.com/hashicorp/vagrant/issues/8795#issuecomment-468945063
 echo "net.ipv6.conf.all.disable_ipv6=1" >> /etc/sysctl.conf
@@ -40,8 +40,8 @@ sed -i 's/vb.gui = true/vb.gui = false/g' Vagrantfile
 # Install Packer
 mkdir /opt/packer
 cd /opt/packer || exit 1
-wget --progress=bar:force https://releases.hashicorp.com/packer/1.6.3/packer_1.6.3_linux_amd64.zip
+wget --progress=bar:force https://releases.hashicorp.com/packer/1.6.6/packer_1.6.6_linux_amd64.zip
-unzip packer_1.6.3_linux_amd64.zip
+unzip packer_1.6.6_linux_amd64.zip
 cp packer /usr/local/bin/packer
 
 # Make the Packer images headless

ci/manual_machine_bootstrap_vmware.sh

@@ -28,8 +28,8 @@ git clone https://github.com/clong/DetectionLab.git /opt/DetectionLab
 # Install Vagrant
 mkdir /opt/vagrant
 cd /opt/vagrant || exit 1
-wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.10/vagrant_2.2.10_x86_64.deb
+wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.14/vagrant_2.2.14_x86_64.deb
-dpkg -i vagrant_2.2.10_x86_64.deb
+dpkg -i vagrant_2.2.14_x86_64.deb
 # Disable IPv6 - may help with the vagrant-reload plugin: https://github.com/hashicorp/vagrant/issues/8795#issuecomment-468945063
 echo "net.ipv6.conf.all.disable_ipv6=1" >> /etc/sysctl.conf
 sysctl -p /etc/sysctl.conf > /dev/null
@@ -47,8 +47,8 @@ sed -i 's/v.gui = true/v.gui = false/g' Vagrantfile
 # Install Packer
 mkdir /opt/packer
 cd /opt/packer || exit 1
-wget --progress=bar:force https://releases.hashicorp.com/packer/1.6.3/packer_1.6.3_linux_amd64.zip
+wget --progress=bar:force https://releases.hashicorp.com/packer/1.6.6/packer_1.6.6_linux_amd64.zip
-unzip packer_1.6.3_linux_amd64.zip
+unzip packer_1.6.6_linux_amd64.zip
 cp packer /usr/local/bin/packer
 
 # Make the Packer images headless