Merge pull request #181 from clong/fleet_update

Update Fleet and use Fleetctl for pack imports.

README.md

@@ -118,7 +118,7 @@ Vagrant commands must be run from the "Vagrant" folder.
 ## Lab Information
 * Domain Name: windomain.local
 * Admininstrator login: vagrant:vagrant
-* Fleet login: https://192.168.38.105:8412 - admin:admin123#
+* Fleet login: https://192.168.38.105:8412 - admin@detectionlab.network:admin123#
 * Splunk login: https://192.168.38.105:8000 - admin:changeme
 * Caldera login: https://192.168.38.105:8888 - admin:caldera
 * MS ATA login: https://192.168.38.103 - wef\vagrant:vagrant

Vagrant/bootstrap.sh

@@ -51,30 +51,6 @@ install_python() {
   fi
 }
 
-install_golang() {
-  if [ ! -f "go1.8.linux-amd64.tar.gz" ]; then
-    # Install Golang v1.8
-    echo "Installing GoLang v1.8..."
-    wget https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz
-    tar -xvf go1.8.linux-amd64.tar.gz
-    mv go /usr/local
-    mkdir /home/vagrant/.go
-    chown vagrant:vagrant /home/vagrant/.go
-    mkdir /root/.go
-    echo 'export GOPATH=$HOME/.go' >> /home/vagrant/.
-    echo 'export GOROOT=/usr/local/go' >> /home/vagrant/.bashrc
-    echo 'export GOPATH=$HOME/.go' >> /root/.bashrc
-    echo 'export GOROOT=/usr/local/go' >> /root/.bashrc
-    echo 'export PATH=$PATH:/opt/splunk/bin' >> /root/.bashrc
-    source /root/.bashrc
-    sudo update-alternatives --install "/usr/bin/go" "go" "/usr/local/go/bin/go" 0
-    sudo update-alternatives --set go /usr/local/go/bin/go
-    /usr/bin/go get -u github.com/howeyc/gopass
-  else
-    echo "GoLang seems to be downloaded already.. Skipping."
-  fi
-}
-
 install_splunk() {
   # Check if Splunk is already installed
   if [ -f "/opt/splunk/bin/splunk" ]; then
@@ -138,7 +114,6 @@ install_fleet() {
     cd kolide-quickstart || echo "Something went wrong while trying to clone the kolide-quickstart repository"
     cp /vagrant/resources/fleet/server.* .
     sed -i 's/ -it//g' demo.sh
-    sed -i 's#kolide/fleet:latest#kolide/fleet:1.0.8#g' docker-compose.yml
     ./demo.sh up simple
     # Set the enrollment secret to match what we deploy to Windows hosts
     docker run --rm --network=kolidequickstart_default mysql:5.7 mysql -h mysql -u kolide --password=kolide -e 'update app_configs set osquery_enroll_secret = "enrollmentsecret" where id=1;' --batch kolide
@@ -156,55 +131,28 @@ download_palantir_osquery_config() {
     # Import Palantir osquery configs into Fleet
     echo "Downloading Palantir configs..."
     git clone https://github.com/palantir/osquery-configuration.git
-    git clone https://github.com/kolide/configimporter.git
-    cd configimporter || exit
-    go build
-    cd /home/vagrant || exit
   fi
 }
 
 import_osquery_config_into_fleet() {
-  if [ -f "/home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf" ]; then
+  wget https://github.com/kolide/fleet/releases/download/2.0.1/fleet_2.0.1.zip
-    echo "The osquery configuration has already been imported into Fleet"
+  unzip fleet_2.0.1.zip -d fleet_2.0.1
-  else
+  cp fleet_2.0.1/linux/fleetctl /usr/local/bin/fleetctl && chmod +x /usr/local/bin/fleetctl
-    # Modify the config to work with config importer
+  fleetctl config set --address https://192.168.38.105:8412
-    cat /home/vagrant/osquery-configuration/Endpoints/Windows/osquery.conf  | sed 's#packs/#../packs/#g' | grep -v unwanted-chrome-extensions | grep -v security-tooling-checks | grep -v performance-metrics | grep -v logger_snapshot_event_type > /home/vagrant/osquery-configuration/Endpoints/Windows/osquery_to_import.conf
+  fleetctl config set --tls-skip-verify true
-    # Install configimporter
+  fleetctl setup --email admin@detectionlab.network --password 'admin123#' --org-name DetectionLab
-    echo "Installing configimporter"
+  fleetctl login --email admin@detectionlab.network --password 'admin123#'
-    echo "Sleeping for 5"
-    sleep 5
-    export CONFIGIMPORTER_PASSWORD='admin123#'
-    cd /home/vagrant/osquery-configuration/Endpoints/Windows/ || exit
-    # Fleet requires you to login before importing packs
-    # Login
-    curl 'https://192.168.38.105:8412/api/v1/kolide/login' -H 'origin: https://192.168.38.105:8412' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' -H 'content-type: application/json' -H 'accept: application/json' -H 'referer: https://192.168.38.105:8412/login' -H 'authority: 192.168.38.105:8412' --data-binary '{"username":"admin","password":"admin123#"}' --compressed --insecure
-    sleep 1
-    curl 'https://192.168.38.105:8412/setup' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H 'upgrade-insecure-requests: 1' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' -H 'authority: 192.168.38.105:8412' --compressed --insecure
-    sleep 1
-    # Setup organization name and email address
-    curl 'https://192.168.38.105:8412/api/v1/setup' -H 'origin: https://192.168.38.105:8412' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' -H 'content-type: application/json' -H 'accept: application/json' -H 'referer: https://192.168.38.105:8412/setup' -H 'authority: 192.168.38.105:8412' --data-binary '{"kolide_server_url":"https://192.168.38.105:8412","org_info":{"org_name":"detectionlab"},"admin":{"admin":true,"email":"example@example.com","password":"admin123#","password_confirmation":"admin123#","username":"admin"}}' --compressed --insecure
-    sleep 3
-    # Import all Windows configs
-    /home/vagrant/configimporter/configimporter -host https://localhost:8412 -user 'admin' -config osquery_to_import.conf
 
-    # Get auth token
+  # Use fleetctl to import YAML files
-    TOKEN=$(curl 'https://192.168.38.105:8412/api/v1/kolide/login' -H 'origin: https://192.168.38.105:8412' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' -H 'content-type: application/json' -H 'accept: application/json' -H 'referer: https://192.168.38.105:8412/login' -H 'authority: 192.168.38.105:8412' --data-binary '{"username":"admin","password":"admin123#"}' --compressed --insecure | grep token | cut -d '"' -f 4)
+  fleetctl apply -f osquery-configuration/Fleet/Endpoints/MacOS/osquery.yaml
-    # Set all packs to be targeted to Windows hosts
+  fleetctl apply -f osquery-configuration/Fleet/Endpoints/Windows/osquery.yaml
-    curl 'https://192.168.38.105:8412/api/v1/kolide/packs/1' -X PATCH -H 'origin: https://192.168.38.105:8412' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -H 'accept: application/json' -H 'referer: https://192.168.38.105:8412/packs/3/edit' -H 'authority: 192.168.38.105:8412' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' --data-binary '{"label_ids":[10]}' --compressed --insecure
+  for pack in osquery-configuration/Fleet/Endpoints/packs/*.yaml
-    sleep 1
+    do fleetctl apply -f "$pack"
-    curl 'https://192.168.38.105:8412/api/v1/kolide/packs/2' -X PATCH -H 'origin: https://192.168.38.105:8412' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -H 'accept: application/json' -H 'referer: https://192.168.38.105:8412/packs/3/edit' -H 'authority: 192.168.38.105:8412' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' --data-binary '{"label_ids":[10]}' --compressed --insecure
+  done
-    sleep 1
+
-    curl 'https://192.168.38.105:8412/api/v1/kolide/packs/3' -X PATCH -H 'origin: https://192.168.38.105:8412' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -H 'accept: application/json' -H 'referer: https://192.168.38.105:8412/packs/3/edit' -H 'authority: 192.168.38.105:8412' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' --data-binary '{"label_ids":[10]}' --compressed --insecure
+  # Add Splunk monitors for Fleet
-    sleep 1
+  /opt/splunk/bin/splunk add monitor "/home/vagrant/kolide-quickstart/osquery_result" -index osquery -sourcetype 'osquery:json' -auth 'admin:changeme'
-    curl 'https://192.168.38.105:8412/api/v1/kolide/packs/4' -X PATCH -H 'origin: https://192.168.38.105:8412' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -H 'accept: application/json' -H 'referer: https://192.168.38.105:8412/packs/3/edit' -H 'authority: 192.168.38.105:8412' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' --data-binary '{"label_ids":[10]}' --compressed --insecure
+  /opt/splunk/bin/splunk add monitor "/home/vagrant/kolide-quickstart/osquery_status" -index osquery-status -sourcetype 'osquery:status' -auth 'admin:changeme'
-    sleep 1
-    curl 'https://192.168.38.105:8412/api/v1/kolide/packs/5' -X PATCH -H 'origin: https://192.168.38.105:8412' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -H 'accept: application/json' -H 'referer: https://192.168.38.105:8412/packs/3/edit' -H 'authority: 192.168.38.105:8412' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' --data-binary '{"label_ids":[10]}' --compressed --insecure
-    # Rename primary pack
-    curl 'https://192.168.38.105:8412/api/v1/kolide/packs/5' -X PATCH -H 'origin: https://192.168.38.105:8412' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9' -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -H 'accept: application/json' -H 'referer: https://192.168.38.105:8412/packs/5/edit' -H 'authority: 192.168.38.105:8412' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' --data-binary '{"name":"windows-pack"}' --compressed --insecure
-    # Add Splunk monitors for Fleet
-    /opt/splunk/bin/splunk add monitor "/home/vagrant/kolide-quickstart/osquery_result" -index osquery -sourcetype 'osquery:json' -auth 'admin:changeme'
-    /opt/splunk/bin/splunk add monitor "/home/vagrant/kolide-quickstart/osquery_status" -index osquery-status -sourcetype 'osquery:status' -auth 'admin:changeme'
-  fi
 }
 
 install_caldera() {
@@ -383,7 +331,6 @@ main() {
   apt_install_prerequisites
   fix_eth1_static_ip
   install_python
-  install_golang
   install_splunk
   install_fleet
   download_palantir_osquery_config

Vagrant/scripts/install-osquery.ps1

@@ -10,8 +10,8 @@ If (-not ($service)) {
   Write-Host "Setting osquery to run as a service"
   Start-Process -FilePath "c:\programdata\osquery\osqueryd\osqueryd.exe" -ArgumentList "--install" -Wait
   # Copy over the config and packs from the Palantir repo
-  Copy-Item "c:\Users\vagrant\AppData\Local\Temp\osquery-configuration-master\Endpoints\Windows\*" "c:\ProgramData\osquery"
+  Copy-Item "c:\Users\vagrant\AppData\Local\Temp\osquery-configuration-master\Classic\Endpoints\Windows\*" "c:\ProgramData\osquery"
-  Copy-Item "c:\Users\vagrant\AppData\Local\Temp\osquery-configuration-master\Endpoints\packs" -Path "c:\ProgramData\osquery"
+  Copy-Item "c:\Users\vagrant\AppData\Local\Temp\osquery-configuration-master\Classic\Endpoints\packs" -Path "c:\ProgramData\osquery"
 
   ## Use the TLS config by default. Un-comment the line below to use the local configuration and avoid connecting to Fleet.
   # Copy-Item "c:\ProgramData\osquery\osquery_no_tls.flags" -Path "c:\ProgramData\osquery\osquery.flags" -Force

build.ps1

@@ -215,10 +215,12 @@ function download_boxes {
   Write-Host '[download_boxes] Checking Filehashes..'
   if ($win10hash -ne $win10Filehash) {
     Write-Error 'Hash mismatch on windows_10_virtualbox.box'
+    Write-Error 'The boxes may have been updated since you last ran the build script. Try updating the git repository to retrieve the latest hashes.'
     break
   }
   if ($win2016hash -ne $win2016Filehash) {
     Write-Error 'Hash mismatch on windows_2016_virtualbox.box'
+    Write-Error 'The boxes may have been updated since you last ran the build script. Try updating the git repository to retrieve the latest hashes.'
     break
   }
   Write-Host '[download_boxes] Finished.'

build.sh

@@ -399,18 +399,22 @@ download_boxes() {
   if [ "$PACKER_PROVIDER" == "virtualbox" ]; then
     if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PACKER_PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "54b9c596486a47f208d952337748079b" ]; then
       (echo >&2 "Hash mismatch on windows_10_virtualbox.box")
+      (echo >&2 "The boxes may have been updated since you last ran the build script. Try updating the git repository to retrieve the latest hashes.")
     fi
     if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PACKER_PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "231b54077d4396cad01e4cd60651b1e0" ]; then
       (echo >&2 "Hash mismatch on windows_2016_virtualbox.box")
+      (echo >&2 "The boxes may have been updated since you last ran the build script. Try updating the git repository to retrieve the latest hashes.")
     fi
     # Verify hashes of VMware boxes
   elif [ "$PACKER_PROVIDER" == "vmware" ]; then
     if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_10_"$PACKER_PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "e754a214c548312746632c870cc40c24" ]; then
       (echo >&2 "Hash mismatch on windows_10_vmware.box")
+      (echo >&2 "The boxes may have been updated since you last ran the build script. Try updating the git repository to retrieve the latest hashes.")
       exit 1
     fi
     if [ "$("$MD5TOOL" "$DL_DIR"/Boxes/windows_2016_"$PACKER_PROVIDER".box | cut -d ' ' -f "$CUT_INDEX")" != "2bbaf5a1177e0499dc3aacdb0246eb38" ]; then
       (echo >&2 "Hash mismatch on windows_2016_vmware.box")
+      (echo >&2 "The boxes may have been updated since you last ran the build script. Try updating the git repository to retrieve the latest hashes.")
       exit 1
     fi
   fi