Merge pull request #468 from clong/osquery_refactor

Refactor osquery and add retry-loop for OU
This commit is contained in:
Chris Long
2020-06-21 14:04:30 -07:00
committed by GitHub
9 changed files with 124 additions and 124 deletions

View File

@@ -1,9 +1,6 @@
* Operating System Version:
* Provider (VirtualBox/VMWare):
* Vagrant Version:
* Packer Version:
* Are you using stock boxes (downloaded) or were they built from scratch using Packer?
* Is the issue reproducible or intermittent?
* Deploying via (VirtualBox/VMWare/AWS/Azure/ESXi):
* Vagrant Version (if applicable):
Please verify that you are building from an updated Master branch before filing an issue.

45
.github/workflows/linter.yml vendored Normal file
View File

@@ -0,0 +1,45 @@
---
###########################
###########################
## Linter GitHub Actions ##
###########################
###########################
name: Lint Code Base
#
# Documentation:
# https://help.github.com/en/articles/workflow-syntax-for-github-actions
#
######################################
# Start the job on all pull requests #
######################################
on: pull_request
###############
# Set the Job #
###############
jobs:
build:
# Name the Job
name: Lint Code Base
# Set the agent to run on
runs-on: ubuntu-latest
##################
# Load all steps #
##################
steps:
##########################
# Checkout the code base #
##########################
- name: Checkout Code
uses: actions/checkout@v2
################################
# Run Linter against code base #
################################
- name: Lint Code Base
uses: docker://github/super-linter:v2.1.1
env:
VALIDATE_ALL_CODEBASE: true

View File

@@ -9,15 +9,6 @@
- debug: msg="{{ palantir_wef.stdout_lines }}"
- name: Downloading the Palantir osquery Configuration
win_shell: ".\\download_palantir_osquery.ps1"
args:
chdir: 'c:\vagrant\scripts'
register: palantir_osquery
failed_when: "'Exception' in palantir_osquery.stdout"
- debug: msg="{{ palantir_osquery.stdout_lines }}"
- name: Installing osquery
win_shell: ".\\install-osquery.ps1"
args:

View File

@@ -9,15 +9,6 @@
- debug: msg="{{ palantir_wef.stdout_lines }}"
- name: Downloading the Palantir osquery Configuration
win_shell: ".\\download_palantir_osquery.ps1"
args:
chdir: 'c:\vagrant\scripts'
register: palantir_osquery
failed_when: "'Exception' in palantir_osquery.stdout"
- debug: msg="{{ palantir_osquery.stdout_lines }}"
- name: Installing osquery
win_shell: ".\\install-osquery.ps1"
args:

3
Vagrant/Vagrantfile vendored
View File

@@ -63,7 +63,6 @@ Vagrant.configure("2") do |config|
cfg.vm.provision "reload"
cfg.vm.provision "shell", path: "scripts/provision.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/download_palantir_wef.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/download_palantir_osquery.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/install-utilities.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/install-redteam.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/install-choco-extras.ps1", privileged: false
@@ -130,7 +129,6 @@ Vagrant.configure("2") do |config|
cfg.vm.provision "reload"
cfg.vm.provision "shell", path: "scripts/provision.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/download_palantir_wef.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/download_palantir_osquery.ps1", privileged: false
cfg.vm.provision "shell", inline: 'wevtutil el | Select-String -notmatch "Microsoft-Windows-LiveId" | Foreach-Object {wevtutil cl "$_"}', privileged: false
cfg.vm.provision "shell", path: "scripts/install-wefsubscriptions.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/install-splunkuf.ps1", privileged: false
@@ -194,7 +192,6 @@ Vagrant.configure("2") do |config|
cfg.vm.provision "reload"
cfg.vm.provision "shell", path: "scripts/provision.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/download_palantir_wef.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/download_palantir_osquery.ps1", privileged: false
cfg.vm.provision "shell", inline: 'wevtutil el | Select-String -notmatch "Microsoft-Windows-LiveId" | Foreach-Object {wevtutil cl "$_"}', privileged: false
cfg.vm.provision "shell", path: "scripts/install-utilities.ps1", privileged: false
cfg.vm.provision "shell", path: "scripts/install-redteam.ps1", privileged: false

View File

@@ -10,7 +10,7 @@ sed -i 's/nameserver 127.0.0.53/nameserver 8.8.8.8/g' /etc/resolv.conf && chattr
# Get a free Maxmind license here: https://www.maxmind.com/en/geolite2/signup
# Required for the ASNgen app to work: https://splunkbase.splunk.com/app/3531/
export MAXMIND_LICENSE=
if [ -z $MAXMIND_LICENSE ]; then
if [ -z "$MAXMIND_LICENSE" ]; then
echo "Note: You have not entered a MaxMind license key on line 5 of bootstrap.sh, so the ASNgen Splunk app may not work correctly."
echo "However, it is not required and everything else should function correctly."
fi
@@ -72,7 +72,7 @@ test_prerequisites() {
fix_eth1_static_ip() {
USING_KVM=$(sudo lsmod | grep kvm)
if [ ! -z "$USING_KVM" ]; then
if [ -n "$USING_KVM" ]; then
echo "[*] Using KVM, no need to fix DHCP for eth1 iface"
return 0
fi
@@ -127,7 +127,7 @@ install_splunk() {
echo "[$(date +%H:%M:%S)]: Attempting to autoresolve the latest version of Splunk..."
LATEST_SPLUNK=$(curl https://www.splunk.com/en_us/download/splunk-enterprise.html | grep -i deb | grep -Eo "data-link=\"................................................................................................................................" | cut -d '"' -f 2)
# Sanity check what was returned from the auto-parse attempt
if [[ "$(echo $LATEST_SPLUNK | grep -c "^https:")" -eq 1 ]] && [[ "$(echo $LATEST_SPLUNK | grep -c "\.deb$")" -eq 1 ]]; then
if [[ "$(echo "$LATEST_SPLUNK" | grep -c "^https:")" -eq 1 ]] && [[ "$(echo "$LATEST_SPLUNK" | grep -c "\.deb$")" -eq 1 ]]; then
echo "[$(date +%H:%M:%S)]: The URL to the latest Splunk version was automatically resolved as: $LATEST_SPLUNK"
echo "[$(date +%H:%M:%S)]: Attempting to download..."
wget --progress=bar:force -P /opt "$LATEST_SPLUNK"
@@ -166,7 +166,7 @@ install_splunk() {
/opt/splunk/bin/splunk install app /vagrant/resources/splunk_server/threathunting_141.tgz -auth 'admin:changeme'
# Install the Maxmind license key for the ASNgen App
if [ ! -z $MAXMIND_LICENSE ]; then
if [ ! -z "$MAXMIND_LICENSE" ]; then
mkdir /opt/splunk/etc/apps/TA-asngen/local
cp /opt/splunk/etc/apps/TA-asngen/default/asngen.conf /opt/splunk/etc/apps/TA-asngen/local/asngen.conf
sed -i "s/license_key =/license_key = $MAXMIND_LICENSE/g" /opt/splunk/etc/apps/TA-asngen/local/asngen.conf
@@ -249,7 +249,7 @@ download_palantir_osquery_config() {
}
import_osquery_config_into_fleet() {
cd /opt
cd /opt || exit 1
wget --progress=bar:force https://github.com/kolide/fleet/releases/download/2.4.0/fleet.zip
unzip fleet.zip -d fleet
cp fleet/linux/fleetctl /usr/local/bin/fleetctl && chmod +x /usr/local/bin/fleetctl
@@ -289,10 +289,6 @@ install_zeek() {
echo "[$(date +%H:%M:%S)]: Installing Zeek..."
# Environment variables
NODECFG=/opt/zeek/etc/node.cfg
SPLUNK_ZEEK_JSON=/opt/splunk/etc/apps/Splunk_TA_bro
SPLUNK_ZEEK_MONITOR='monitor:///opt/zeek/spool/manager'
SPLUNK_SURICATA_MONITOR='monitor:///var/log/suricata'
SPLUNK_SURICATA_SOURCETYPE='json_suricata'
sh -c "echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_18.04/ /' > /etc/apt/sources.list.d/security:zeek.list"
wget -nv https://download.opensuse.org/repositories/security:zeek/xUbuntu_18.04/Release.key -O /tmp/Release.key
apt-key add - </tmp/Release.key &>/dev/null
@@ -346,22 +342,16 @@ install_zeek() {
systemctl enable zeek
systemctl start zeek
mkdir -p $SPLUNK_ZEEK_JSON/local
cp $SPLUNK_ZEEK_JSON/default/inputs.conf $SPLUNK_ZEEK_JSON/local/inputs.conf
crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR index zeek
crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR sourcetype bro:json
crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR whitelist '.*\.log$'
crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR blacklist '.*(communication|stderr)\.log$'
crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_ZEEK_MONITOR disabled 0
crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR index suricata
crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR sourcetype suricata:json
crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR whitelist 'eve.json'
crudini --set $SPLUNK_ZEEK_JSON/local/inputs.conf $SPLUNK_SURICATA_MONITOR disabled 0
crudini --set $SPLUNK_ZEEK_JSON/local/props.conf $SPLUNK_SURICATA_SOURCETYPE TRUNCATE 0
# Configure the Splunk inputs
mkdir -p /opt/splunk/etc/apps/Splunk_TA_bro/local && touch /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf
crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager index zeek
crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager sourcetype bro:json
crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager whitelist '.*\.log$'
crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager blacklist '.*(communication|stderr)\.log$'
crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager disabled 0
# Ensure permissions are correct and restart splunk
chown -R splunk $SPLUNK_ZEEK_JSON
chown -R splunk /opt/splunk/etc/apps/Splunk_TA_bro
/opt/splunk/bin/splunk restart
# Verify that Zeek is running
@@ -394,6 +384,14 @@ install_suricata() {
suricata-update enable-source et/open
suricata-update enable-source ptresearch/attackdetection
# Configure the Splunk inputs
mkdir -p /opt/splunk/etc/apps/SplunkLightForwarder/local && touch /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf
crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata index suricata
crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata sourcetype suricata:json
crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata whitelist 'eve.json'
crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata disabled 0
crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/props.conf json_suricata TRUNCATE 0
# Update suricata and restart
suricata-update
service suricata stop
@@ -428,13 +426,13 @@ test_suricata_prerequisites() {
install_guacamole() {
echo "[$(date +%H:%M:%S)]: Installing Guacamole..."
cd /opt
cd /opt || exit 1
apt-get -qq install -y libcairo2-dev libjpeg62-dev libpng-dev libossp-uuid-dev libfreerdp-dev libpango1.0-dev libssh2-1-dev libssh-dev tomcat8 tomcat8-admin tomcat8-user
wget --progress=bar:force "http://apache.org/dyn/closer.cgi?action=download&filename=guacamole/1.0.0/source/guacamole-server-1.0.0.tar.gz" -O guacamole-server-1.0.0.tar.gz
tar -xf guacamole-server-1.0.0.tar.gz && cd guacamole-server-1.0.0
tar -xf guacamole-server-1.0.0.tar.gz && cd guacamole-server-1.0.0 || echo "[-] Unable to find the Guacamole folder."
./configure &>/dev/null && make --quiet &>/dev/null && make --quiet install &>/dev/null || echo "[-] An error occurred while installing Guacamole."
ldconfig
cd /var/lib/tomcat8/webapps
cd /var/lib/tomcat8/webapps || echo "[-] Unable to find the tomcat8/webapps folder."
wget --progress=bar:force "http://apache.org/dyn/closer.cgi?action=download&filename=guacamole/1.0.0/binary/guacamole-1.0.0.war" -O guacamole.war
mkdir /etc/guacamole
mkdir /usr/share/tomcat8/.guacamole

View File

@@ -1,55 +1,53 @@
# Purpose: Sets up the Server and Workstations OUs
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Checking AD services status..."
$svcs = "adws","dns","kdc","netlogon"
Get-Service -name $svcs -ComputerName localhost | Select Machinename,Name,Status
# Hardcoding DC hostname in hosts file
# Hardcoding DC hostname in hosts file to sidestep any DNS issues
Add-Content "c:\windows\system32\drivers\etc\hosts" " 192.168.38.102 dc.windomain.local"
# Force DNS resolution of the domain
ping /n 1 dc.windomain.local
ping /n 1 windomain.local
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Creating Server and Workstation OUs..."
# Create the Servers OU if it doesn't exist
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Creating Server OU"
try {
Get-ADOrganizationalUnit -Identity 'OU=Servers,DC=windomain,DC=local' | Out-Null
Write-Host "Servers OU already exists. Moving On."
}
catch [Microsoft.ActiveDirectory.Management.ADIdentityNotFoundException] {
New-ADOrganizationalUnit -Name "Servers" -Server "dc.windomain.local"
Write-Host "Created Servers OU."
}
catch [Microsoft.ActiveDirectory.Management.ADServerDownException] {
Write-Host "Unable to reach Active Directory. Sleeping for 10 and attmepting one more time..."
Start-Sleep 10
New-ADOrganizationalUnit -Name "Servers" -Server "dc.windomain.local"
Write-Host "Created Servers OU after a retry."
}
catch {
Write-Host "Something went wrong attempting to reach AD or create the OU."
$servers_ou_created = 0
while ($servers_ou_created -ne 1) {
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Creating Server OU..."
try {
Get-ADOrganizationalUnit -Identity 'OU=Servers,DC=windomain,DC=local' | Out-Null
Write-Host "Servers OU already exists. Moving On."
$servers_ou_created = 1
}
catch [Microsoft.ActiveDirectory.Management.ADIdentityNotFoundException] {
New-ADOrganizationalUnit -Name "Servers" -Server "dc.windomain.local"
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Created Servers OU."
$servers_ou_created = 1
}
catch [Microsoft.ActiveDirectory.Management.ADServerDownException] {
Write-Host "Unable to reach Active Directory. Sleeping for 5 and trying again..."
Start-Sleep 5
}
catch {
Write-Host "Something went wrong attempting to reach AD or create the OU."
}
}
# Create the Workstations OU if it doesn't exist
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Creating Workstations OU"
try {
Get-ADOrganizationalUnit -Identity 'OU=Workstations,DC=windomain,DC=local' | Out-Null
Write-Host "Workstations OU already exists. Moving On."
}
catch [Microsoft.ActiveDirectory.Management.ADIdentityNotFoundException] {
New-ADOrganizationalUnit -Name "Workstations" -Server "dc.windomain.local"
Write-Host "Created Workstations OU."
}
catch [Microsoft.ActiveDirectory.Management.ADServerDownException] {
Write-Host "Unable to reach Active Directory. Sleeping for 10 and attmepting one more time..."
Start-Sleep 10
New-ADOrganizationalUnit -Name "Workstations" -Server "dc.windomain.local"
Write-Host "Created Workstations OU after a retry."
}
catch {
Write-Host "Something went wrong attempting to reach AD or create the OU."
$workstations_ou_created = 0
while ($workstations_ou_created -ne 1) {
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Creating Workstations OU..."
try {
Get-ADOrganizationalUnit -Identity 'OU=Workstations,DC=windomain,DC=local' | Out-Null
Write-Host "Workstations OU already exists. Moving On."
$workstations_ou_created = 1
}
catch [Microsoft.ActiveDirectory.Management.ADIdentityNotFoundException] {
New-ADOrganizationalUnit -Name "Workstations" -Server "dc.windomain.local"
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Created Workstations OU."
$workstations_ou_created = 1
}
catch [Microsoft.ActiveDirectory.Management.ADServerDownException] {
Write-Host "Unable to reach Active Directory. Sleeping for 5 and trying again..."
Start-Sleep 5
}
catch {
Write-Host "Something went wrong attempting to reach AD or create the OU."
}
}
# Sysprep breaks auto-login. Let's restore it here:

View File

@@ -1,17 +0,0 @@
# Purpose: Downloads and unzips a copy of the Palantir osquery Github Repo. These configs are added to the Fleet server in bootstrap.sh.
# The items from this config file are used later in install-osquery.ps1
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Downloading and unzipping the Palantir osquery Repo from Github..."
$osqueryRepoPath = 'C:\Users\vagrant\AppData\Local\Temp\osquery-Master.zip'
if (-not (Test-Path $osqueryRepoPath))
{
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Invoke-WebRequest -Uri "https://github.com/palantir/osquery-configuration/archive/master.zip" -OutFile $osqueryRepoPath
Expand-Archive -path "$osqueryRepoPath" -destinationpath 'c:\Users\vagrant\AppData\Local\Temp' -Force
}
else
{
Write-Host "$osqueryRepoPath already exists. Moving On."
}
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Palantir osquery config download complete!"

View File

@@ -1,18 +1,18 @@
# Purpose: Installs osquery on the host
# Note: by default, osquery will be configured to connect to the Fleet server on the "logger" host via TLS.
# If you would like to have osquery run without TLS & Fleet, uncomment line 15 and comment lines 21-30.
Write-Host "$('[{0:HH:mm}]' -f (Get-Date)) Installing osquery..."
$packsDir = "c:\Program Files\osquery\packs"
$flagfile = "c:\Program Files\osquery\osquery.flags"
choco install -y --limit-output --no-progress osquery | Out-String # Apparently Out-String makes the process wait
$service = Get-WmiObject -Class Win32_Service -Filter "Name='osqueryd'"
If (-not ($service)) {
Write-Host "Setting osquery to run as a service"
New-Service -Name "osqueryd" -BinaryPathName "C:\Program Files\osquery\osqueryd\osqueryd.exe --flagfile=`"C:\Program Files\osquery\osquery.flags`""
# Copy over the config and packs from the Palantir repo
Copy-Item "c:\Users\vagrant\AppData\Local\Temp\osquery-configuration-master\Classic\Endpoints\Windows\*" "c:\Program Files\osquery"
Copy-Item "c:\Users\vagrant\AppData\Local\Temp\osquery-configuration-master\Classic\Endpoints\packs" -Path "c:\Program Files\osquery"
# Download the flags file from the Palantir osquery-configuration Github
# GitHub requires TLS 1.2 as of 2/1/2018
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/palantir/osquery-configuration/master/Classic/Endpoints/Windows/osquery.flags" -OutFile $flagfile
## Use the TLS config
## Add entry to hosts file for Kolide for SSL validation
@@ -20,14 +20,14 @@ If (-not ($service)) {
## Add kolide secret and avoid BOM
$Utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $False
[System.IO.File]::WriteAllLines("c:\Program Files\osquery\kolide_secret.txt", "enrollmentsecret", $Utf8NoBomEncoding)
## Change TLS server hostname
(Get-Content "c:\Program Files\osquery\osquery.flags") -replace 'tls.endpoint.server.com', 'kolide:8412' | Set-Content "c:\Program Files\osquery\osquery.flags"
## Change path to secrets
(Get-Content "c:\Program Files\osquery\osquery.flags") -replace 'path\\to\\file\\containing\\secret.txt', 'Program Files\osquery\kolide_secret.txt' | Set-Content "c:\Program Files\osquery\osquery.flags"
## Change path to certfile
(Get-Content "c:\Program Files\osquery\osquery.flags") -replace 'c:\\ProgramData\\osquery\\certfile.crt', 'c:\Program Files\osquery\certfile.crt' | Set-Content "c:\Program Files\osquery\osquery.flags"
## Change TLS server hostname in the flags file
(Get-Content $flagfile) -replace 'tls.endpoint.server.com', 'kolide:8412' | Set-Content $flagfile
## Change path to secrets in the flags file
(Get-Content $flagfile) -replace 'path\\to\\file\\containing\\secret.txt', 'Program Files\osquery\kolide_secret.txt' | Set-Content $flagfile
## Change path to certfile in the flags file
(Get-Content $flagfile) -replace 'c:\\ProgramData\\osquery\\certfile.crt', 'c:\Program Files\osquery\certfile.crt' | Set-Content $flagfile
## Remove the verbose flag and replace it with the logger_min_status=1 option (See https://github.com/osquery/osquery/issues/5212)
(Get-Content "c:\Program Files\osquery\osquery.flags") -replace '--verbose=true', '--logger_min_status=1' | Set-Content "c:\Program Files\osquery\osquery.flags"
(Get-Content $flagfile) -replace '--verbose=true', '--logger_min_status=1' | Set-Content $flagfile
## Add certfile.crt
Copy-Item "c:\vagrant\resources\fleet\server.crt" "c:\Program Files\osquery\certfile.crt"
## Start the service