From 7ae0b0024164695a7e7f2427db58c844b955e327 Mon Sep 17 00:00:00 2001 From: Trinitor Date: Tue, 27 Dec 2022 21:59:06 +0100 Subject: [PATCH] first commit --- .gitignore | 19 + README.md | 99 +++++ Vagrantfile | 37 ++ data/apidemo-cron/build/Dockerfile | 9 + data/apidemo-cron/build/entrypoint.sh | 8 + .../scripts/get_cryptocurrency.sh | 5 + data/apidemo-filebeat/config/filebeat.yml | 13 + data/apidemo-logstash/config/logstash.conf | 26 ++ data/beats-logstash/config/logstash.conf | 23 + data/mdns/build/Dockerfile | 9 + data/mdns/build/entrypoint.sh | 15 + data/mdns/config/names.csv | 3 + .../config/internal_users_example.yml | 59 +++ data/opensearch-node1/config/opensearch.yml | 17 + data/opensearch-node2/config/opensearch.yml | 17 + data/setup/build/01_precreate_folders.sh | 30 ++ data/setup/build/02_generate_certificates.sh | 109 +++++ data/setup/build/03_configure_opensearch.sh | 42 ++ data/setup/build/04_configure_grafana.sh | 56 +++ data/setup/build/Dockerfile | 9 + data/setup/build/entrypoint.sh | 11 + data/syslog-filebeat/config/filebeat.yml | 51 +++ data/syslog-logstash/config/logstash.conf | 28 ++ data/traefik/config/encryption.toml | 9 + docker-compose.yml | 412 ++++++++++++++++++ docs/ohunt_overview.drawio.png | Bin 0 -> 32548 bytes env_example | 1 + reset.ps1 | 19 + reset.sh | 21 + 29 files changed, 1157 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 Vagrantfile create mode 100644 data/apidemo-cron/build/Dockerfile create mode 100644 data/apidemo-cron/build/entrypoint.sh create mode 100644 data/apidemo-cron/scripts/get_cryptocurrency.sh create mode 100644 data/apidemo-filebeat/config/filebeat.yml create mode 100644 data/apidemo-logstash/config/logstash.conf create mode 100644 data/beats-logstash/config/logstash.conf create mode 100644 data/mdns/build/Dockerfile create mode 100644 data/mdns/build/entrypoint.sh create mode 100644 data/mdns/config/names.csv create mode 100644 data/opensearch-node1/config/internal_users_example.yml create mode 100644 data/opensearch-node1/config/opensearch.yml create mode 100644 data/opensearch-node2/config/opensearch.yml create mode 100644 data/setup/build/01_precreate_folders.sh create mode 100644 data/setup/build/02_generate_certificates.sh create mode 100644 data/setup/build/03_configure_opensearch.sh create mode 100644 data/setup/build/04_configure_grafana.sh create mode 100644 data/setup/build/Dockerfile create mode 100644 data/setup/build/entrypoint.sh create mode 100644 data/syslog-filebeat/config/filebeat.yml create mode 100644 data/syslog-logstash/config/logstash.conf create mode 100644 data/traefik/config/encryption.toml create mode 100644 docker-compose.yml create mode 100644 docs/ohunt_overview.drawio.png create mode 100644 env_example create mode 100644 reset.ps1 create mode 100644 reset.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d5ef649 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +.env +.vagrant/ +auditbeat-*.deb +data/certificates/certs/ +data/opensearch-dashboards/certs/ +data/opensearch-node1/certs/ +data/opensearch-node1/data/ +data/opensearch-node1/config/internal_users.yml +data/opensearch-node2/certs/ +data/opensearch-node2/data/ +data/opensearch-node2/config/internal_users.yml +data/grafana/data/ +data/traefik/certs/ +data/apidemo-cron/output/ +data/apidemo-filebeat/data/ +data/syslog-filebeat/data/ +data/graylog/data/ +data/graylog-mongodb/configdb/ +data/graylog-mongodb/db/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..40f979b --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +# Log Collection Lab + +## Overview +This repository creates a two node opensearch cluster in an fully automated way. +Certificates created (no demo certs) and default passwords changed. +It includes demo configurations for logstash and filebeats. + +Full feature list: +- vagrant to download and create an Ubuntu VM and installs docker +- setup container to create a CA and certificates for the services +- two node opensearch cluster + - uses your own certificates + - default passwords changed +- opensearch-dashboard + - creates index pattern automatically in global tenant +- Grafana + - creates datasource automatically +- traefik reverse proxy for opensearch-dashboard +- mdns responder to allow connection by hostname instead of IP +- beats logstash container to other computers in your lab can send information (winlogbeat, auditbeat, or packetbeat) +- log receiver for Cisco ASA, Cisco IOS, Checkpoint, Snort, CEF, Syslog, and Netflow +- cron container to download json information periodically from API including filebeat/logstash pipeline + +The repository was created to give you a starting point for your own opensearch installation. +It shows you how to change common settings and replace default passwords and certificates. +The demo configuration give you a start into file injections, syslog, grok, beats, logstash configs, certificate creation, docker, vagrant, and more. + +## Requirements +Install virtualbox and vagrant on your computer. + +## Start +Run +``` +vagrant up +```` +in the directory. + +## Opensearch Dashboard login +URL: https://opensearch.local (or http://192.168.57.2:5601) +Username: admin +Password: vagrant + +## Grafana Dashboard login +URL: https://grafana.local +Username: admin +Password: vagrant + +# Network +The logger virtual machine has three network interfaces. +1. NAT +2. private network with static IP 192.168.57.2 (only reachable from your host) +3. bridged network with dhcp + +you can send beats from other hosts to the bridged IP address. + +## Beats +- install a beats collector (https://www.elastic.co/beats/) on your computer + - winlogbeat, auditbeat, or packetbeat are easy to get started +- change output settings in the configuration file to: +``` +output.logstash: + hosts: ["192.168.57.2:5044"] +``` +You will need to remove the output.elasticsearch section. +- information will be stored in opensearch-logstash-beats-$date +- you just need to create the index pattern + +## Password changes +All passwords are set to "vagrant" in this repository. +The password hashes are stored in internal_users.yml and the logstash clear text password is in the .env file (used by logstash containers) +If you want to change the password your need to replace the hashes and tell opensearch to read the configuration. +The securityadmin command must be executed in both opensearch nodes +``` +docker exec -it opensearch-node1 /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh --clustername opensearch-cluster --configdir /usr/share/opensearch/config/opensearch-security -cacert /usr/share/opensearch/config/certs/opensearch-ca.pem -key /usr/share/opensearch/config/certs/opensearch-admin.key -cert /usr/share/opensearch/config/certs/opensearch-admin.pem -h `cat /etc/hostname` + +docker exec -it opensearch-node2 /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh --clustername opensearch-cluster --configdir /usr/share/opensearch/config/opensearch-security -cacert /usr/share/opensearch/config/certs/opensearch-ca.pem -key /usr/share/opensearch/config/certs/opensearch-admin.key -cert /usr/share/opensearch/config/certs/opensearch-admin.pem -h `cat /etc/hostname` +``` + +# Troubleshooting +## Docker +``` +vagrant ssh +``` +-> logs you into the VM + +``` +sudo -s && cd /vagrant && docker-compose logs -f +``` +-> all files are mapped inth this folder of the VM. you can use all docker and docker-compose commands as usual (ps, exec, ...) + +## Elasticsearch +GET _cat/shards?v=true&h=index,shard,prirep,state,node,unassigned.reason&s=state + +GET _cluster/allocation/explain + +GET _cluster/settings?flat_settings=true&include_defaults=true + +PUT _cluster/settings +{ "persistent" : { "cluster.routing.allocation.enable" : "all" } } diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..3f2cd68 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,37 @@ +Vagrant.configure("2") do |config| + config.vm.define "opensearch", autostart: true do |cfg| + cfg.vm.box = "ubuntu/jammy64" + cfg.vm.hostname = "opensearch" + cfg.vm.network :private_network, ip: "192.168.57.2", gateway: "192.168.57.1", dns: "8.8.8.8" + cfg.vm.network "public_network" + cfg.vm.boot_timeout = 1200 + cfg.vm.provider "virtualbox" do |vb| + vb.gui = true + vb.name = "opensearch" + vb.cpus = 2 + vb.memory = "8192" + end + + cfg.vm.provision "shell", run: "once", inline: <<-SHELL + export DEBIAN_FRONTEND=noninteractive + rm -rf /var/lib/apt/lists/* + apt update + apt -y upgrade + apt -y install docker.io docker-compose + apt -y autoremove + apt clean + echo vm.max_map_count=262144 >> /etc/sysctl.conf + sysctl -p + cd /vagrant + docker-compose up -d + mkdir /opt/install && cd /opt/install + wget https://artifacts.elastic.co/downloads/beats/auditbeat/auditbeat-oss-7.12.1-amd64.deb + dpkg -i auditbeat-oss-7.12.1-amd64.deb + echo "give opensearch some time to start" + echo "connect to opensearch-dashboards afterwards with" + echo "URL: https://opensearch.local/ (or http://192.168.57.2:5601)" + echo "Username: admin" + echo "Password: vagrant" + SHELL + end +end diff --git a/data/apidemo-cron/build/Dockerfile b/data/apidemo-cron/build/Dockerfile new file mode 100644 index 0000000..a03295c --- /dev/null +++ b/data/apidemo-cron/build/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:22.04 + +RUN apt-get update && apt-get -y upgrade +RUN apt-get -y install cron curl jq dos2unix + +COPY entrypoint.sh /opt/entrypoint.sh +RUN dos2unix /opt/entrypoint.sh ; chmod +x /opt/entrypoint.sh + +CMD ["sh", "/opt/entrypoint.sh"] diff --git a/data/apidemo-cron/build/entrypoint.sh b/data/apidemo-cron/build/entrypoint.sh new file mode 100644 index 0000000..e358f8f --- /dev/null +++ b/data/apidemo-cron/build/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +dos2unix $COMMAND + +echo "$SCHEDULE $USER $COMMAND" > /etc/cron.d/api-cronjob +chmod 0644 /etc/cron.d/api-cronjob +crontab /etc/cron.d/api-cronjob +touch /var/log/cron.log +env > /etc/environment && cron -f \ No newline at end of file diff --git a/data/apidemo-cron/scripts/get_cryptocurrency.sh b/data/apidemo-cron/scripts/get_cryptocurrency.sh new file mode 100644 index 0000000..3d64bd9 --- /dev/null +++ b/data/apidemo-cron/scripts/get_cryptocurrency.sh @@ -0,0 +1,5 @@ +#!/bin/bash +DATE=`date +"%Y-%m-%d"` +curl https://api.coindesk.com/v1/bpi/currentprice.json > /tmp/cryptocurrency.json +jq -c 'del(.disclaimer)' /tmp/cryptocurrency.json >> /opt/output/cryptocurrency_$DATE.json +find /opt/output/ -mtime +5 -delete diff --git a/data/apidemo-filebeat/config/filebeat.yml b/data/apidemo-filebeat/config/filebeat.yml new file mode 100644 index 0000000..7445824 --- /dev/null +++ b/data/apidemo-filebeat/config/filebeat.yml @@ -0,0 +1,13 @@ +filebeat.inputs: +- type: log + enabled: true + paths: + - ${INPUT_PATH} + +filebeat.config.modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +output.logstash: + enabled: true + hosts: ["${LOGSTASH_HOST}"] diff --git a/data/apidemo-logstash/config/logstash.conf b/data/apidemo-logstash/config/logstash.conf new file mode 100644 index 0000000..9edd511 --- /dev/null +++ b/data/apidemo-logstash/config/logstash.conf @@ -0,0 +1,26 @@ +input { + beats { + port => 5044 + } +} + +filter { + json { + source => "message" + } +} + +output { + #stdout {} + #file { + # path => "/tmp/output.json" + #} + opensearch { + hosts => ["${OPENSEARCH_HOST}"] + index => "${OPENSEARCH_INDEX}-%{+YYYY-MM-dd}" + user => "${LOGSTASH_USER}" + password => "${LOGSTASH_PASSWORD}" + ssl => true + ssl_certificate_verification => false + } +} \ No newline at end of file diff --git a/data/beats-logstash/config/logstash.conf b/data/beats-logstash/config/logstash.conf new file mode 100644 index 0000000..ee3af11 --- /dev/null +++ b/data/beats-logstash/config/logstash.conf @@ -0,0 +1,23 @@ +input { + beats { + port => 5044 + } +} + +filter { +} + +output { + #stdout {} + #file { + # path => "/tmp/output.json" + #} + opensearch { + hosts => ["${OPENSEARCH_HOST}"] + index => "${OPENSEARCH_INDEX}-%{+YYYY-MM-dd}" + user => "${LOGSTASH_USER}" + password => "${LOGSTASH_PASSWORD}" + ssl => true + ssl_certificate_verification => false + } +} \ No newline at end of file diff --git a/data/mdns/build/Dockerfile b/data/mdns/build/Dockerfile new file mode 100644 index 0000000..608ff6d --- /dev/null +++ b/data/mdns/build/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:22.04 + +RUN apt-get update && apt-get -y upgrade +RUN apt-get -y install avahi-utils libnss-mdns dos2unix + +ADD entrypoint.sh /opt/entrypoint.sh +RUN dos2unix /opt/entrypoint.sh ; chmod +x /opt/entrypoint.sh + +CMD ["sh", "/opt/entrypoint.sh"] \ No newline at end of file diff --git a/data/mdns/build/entrypoint.sh b/data/mdns/build/entrypoint.sh new file mode 100644 index 0000000..06e2f2a --- /dev/null +++ b/data/mdns/build/entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +service dbus start +service avahi-daemon start + +dos2unix /opt/config/names.csv + +while read LINE; do + PUBLISH_HOSTNAME=$(echo $LINE | cut -d ";" -f 1) + PUBLISH_IP=$(echo $LINE | cut -d ";" -f 2) + echo "$PUBLISH_HOSTNAME - $PUBLISH_IP" + /usr/bin/avahi-publish -a -R $PUBLISH_HOSTNAME $PUBLISH_IP & +done < /opt/config/names.csv + +tail -f /dev/null diff --git a/data/mdns/config/names.csv b/data/mdns/config/names.csv new file mode 100644 index 0000000..7f22994 --- /dev/null +++ b/data/mdns/config/names.csv @@ -0,0 +1,3 @@ +opensearch.local;192.168.57.2 +traefik.local;192.168.57.2 +grafana.local;192.168.57.2 diff --git a/data/opensearch-node1/config/internal_users_example.yml b/data/opensearch-node1/config/internal_users_example.yml new file mode 100644 index 0000000..050255e --- /dev/null +++ b/data/opensearch-node1/config/internal_users_example.yml @@ -0,0 +1,59 @@ +--- +# This is the internal user database +# The hash value is a bcrypt hash and can be generated with /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh + +_meta: + type: "internalusers" + config_version: 2 + +admin: + hash: "$2y$12$x22en27Ec7WS8OmtW1MxMeu7l0GHHrSwEn3HMH/o4JcKeeAQ.UGFK" + reserved: true + backend_roles: + - "admin" + description: "Demo admin user" + +anomalyadmin: + hash: "$2y$12$x22en27Ec7WS8OmtW1MxMeu7l0GHHrSwEn3HMH/o4JcKeeAQ.UGFK" + reserved: false + opendistro_security_roles: + - "anomaly_full_access" + description: "Demo anomaly admin user, using internal role" + +kibanaserver: + hash: "$2y$12$x22en27Ec7WS8OmtW1MxMeu7l0GHHrSwEn3HMH/o4JcKeeAQ.UGFK" + reserved: true + description: "Demo OpenSearch Dashboards user" + +kibanaro: + hash: "$2y$12$x22en27Ec7WS8OmtW1MxMeu7l0GHHrSwEn3HMH/o4JcKeeAQ.UGFK" + reserved: false + backend_roles: + - "kibanauser" + - "readall" + attributes: + attribute1: "value1" + attribute2: "value2" + attribute3: "value3" + description: "Demo OpenSearch Dashboards read only user, using external role mapping" + +logstash: + hash: "$2y$12$x22en27Ec7WS8OmtW1MxMeu7l0GHHrSwEn3HMH/o4JcKeeAQ.UGFK" + reserved: false + backend_roles: + - "logstash" + description: "Demo logstash user, using external role mapping" + +readall: + hash: "$2y$12$x22en27Ec7WS8OmtW1MxMeu7l0GHHrSwEn3HMH/o4JcKeeAQ.UGFK" + reserved: false + backend_roles: + - "readall" + description: "Demo readall user, using external role mapping" + +snapshotrestore: + hash: "$2y$12$x22en27Ec7WS8OmtW1MxMeu7l0GHHrSwEn3HMH/o4JcKeeAQ.UGFK" + reserved: false + backend_roles: + - "snapshotrestore" + description: "Demo snapshotrestore user, using external role mapping" \ No newline at end of file diff --git a/data/opensearch-node1/config/opensearch.yml b/data/opensearch-node1/config/opensearch.yml new file mode 100644 index 0000000..4755fe7 --- /dev/null +++ b/data/opensearch-node1/config/opensearch.yml @@ -0,0 +1,17 @@ +cluster.name: docker-cluster +network.host: 0.0.0.0 +plugins.security.authcz.admin_dn: + - "CN=admin,O=security,L=IT,ST=NY,C=US" +plugins.security.nodes_dn: + - "CN=opensearch-node*" +plugins.security.ssl.transport.enforce_hostname_verification: false +plugins.security.ssl.transport.resolve_hostname: false +plugins.security.ssl.http.enabled: true +plugins.security.allow_unsafe_democertificates: true +plugins.security.allow_default_init_securityindex: true +plugins.security.audit.type: internal_opensearch +plugins.security.enable_snapshot_restore_privilege: true +plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +plugins.security.system_indices.enabled: true +plugins.security.system_indices.indices: [".plugins-ml-model", ".plugins-ml-task", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"] \ No newline at end of file diff --git a/data/opensearch-node2/config/opensearch.yml b/data/opensearch-node2/config/opensearch.yml new file mode 100644 index 0000000..4755fe7 --- /dev/null +++ b/data/opensearch-node2/config/opensearch.yml @@ -0,0 +1,17 @@ +cluster.name: docker-cluster +network.host: 0.0.0.0 +plugins.security.authcz.admin_dn: + - "CN=admin,O=security,L=IT,ST=NY,C=US" +plugins.security.nodes_dn: + - "CN=opensearch-node*" +plugins.security.ssl.transport.enforce_hostname_verification: false +plugins.security.ssl.transport.resolve_hostname: false +plugins.security.ssl.http.enabled: true +plugins.security.allow_unsafe_democertificates: true +plugins.security.allow_default_init_securityindex: true +plugins.security.audit.type: internal_opensearch +plugins.security.enable_snapshot_restore_privilege: true +plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +plugins.security.system_indices.enabled: true +plugins.security.system_indices.indices: [".plugins-ml-model", ".plugins-ml-task", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"] \ No newline at end of file diff --git a/data/setup/build/01_precreate_folders.sh b/data/setup/build/01_precreate_folders.sh new file mode 100644 index 0000000..232a45c --- /dev/null +++ b/data/setup/build/01_precreate_folders.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +mkdir /data/graylog-mongodb/configdb +mkdir /data/graylog-mongodb/db +chmod 777 /data/graylog-mongodb/configdb +chmod 777 /data/graylog-mongodb/db + +if [ ! -f /data/.env ] +then + cp /data/env_example /data/.env +fi + +if [ ! -f /data/opensearch-node1/config/internal_users.yml ] +then + cp /data/opensearch-node1/config/internal_users_example.yml /data/opensearch-node1/config/internal_users.yml + mkdir /data/opensearch-node2/config/ + cp /data/opensearch-node1/config/internal_users_example.yml /data/opensearch-node2/config/internal_users.yml +fi + +if [ ! -d "/data/opensearch-node1/data/" ] +then + echo "creating opensearch node1 data directoy" + mkdir -p /data/opensearch-node1/data/ +fi + +if [ ! -d "/data/opensearch-node2/data/" ] +then + echo "creating opensearch node2 data directoy" + mkdir -p /data/opensearch-node2/data/ +fi diff --git a/data/setup/build/02_generate_certificates.sh b/data/setup/build/02_generate_certificates.sh new file mode 100644 index 0000000..dc82b6b --- /dev/null +++ b/data/setup/build/02_generate_certificates.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +if [ ! -f /data/certificates/certs/opensearch-ca.key ] +then + echo "generating CA" + mkdir -p /data/certificates/certs/ + openssl genrsa -out /data/certificates/certs/opensearch-ca.key 2048 + openssl req -new -x509 -sha256 -days 3650 -subj "/C=US/ST=NY/L=IT/O=security/CN=opensearch-ca" -key /data/certificates/certs/opensearch-ca.key -out /data/certificates/certs/opensearch-ca.pem + openssl x509 -noout -subject -in /data/certificates/certs/opensearch-ca.pem +fi + +if [ ! -f /data/certificates/certs/opensearch-admin.key ] +then + echo "generating admin user key" + mkdir -p /data/certificates/certs/ + openssl genrsa -out /data/certificates/certs/opensearch-admin_rsa.key 2048 + openssl pkcs8 -v1 PBE-SHA1-3DES -nocrypt -in /data/certificates/certs/opensearch-admin_rsa.key -topk8 -out /data/certificates/certs/opensearch-admin.key + openssl req -new -inform PEM -outform PEM -subj "/C=US/ST=NY/L=IT/O=security/CN=admin" -key /data/certificates/certs/opensearch-admin.key -out /data/certificates/certs/opensearch-admin.csr + openssl x509 -req -days 3650 -in /data/certificates/certs/opensearch-admin.csr -CA /data/certificates/certs/opensearch-ca.pem -CAkey /data/certificates/certs/opensearch-ca.key -CAcreateserial -sha256 -out /data/certificates/certs/opensearch-admin.pem + #openssl verify -CAfile /data/certificates/certs/opensearch-ca.pem /data/certificates/certs/opensearch-admin.pem + #openssl x509 -noout -subject -in /data/certificates/certs/opensearch-admin.pem +fi + +if [ ! -f /data/opensearch-node1/certs/opensearch-node1.key ] +then + for NODE_NAME in "node1" "node2" + do + echo "generating certificate opensearch-$NODE_NAME" + mkdir -p /data/opensearch-$NODE_NAME/certs/ + + cat << EOF > /tmp/request.conf + [req] + distinguished_name = req_distinguished_name + req_extensions = v3_req + prompt = no + [req_distinguished_name] + C = US + ST = NY + L = IT + O = security + CN = opensearch-$NODE_NAME + [v3_req] + keyUsage = keyEncipherment, dataEncipherment, digitalSignature, nonRepudiation + extendedKeyUsage = serverAuth, clientAuth + subjectAltName = @alt_names + [alt_names] + DNS.1 = docker-cluster + DNS.2 = opensearch-$NODE_NAME + RID.1 = 1.2.3.4.5.5 +EOF + + openssl genrsa -out /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME-rsa.key 2048 + openssl pkcs8 -inform PEM -outform PEM -in /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME-rsa.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.key + openssl req -new -config /tmp/request.conf -key /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.key -out /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.csr + openssl x509 -req -days 3650 -extfile /tmp/request.conf -extensions v3_req -in /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.csr -CA /data/certificates/certs/opensearch-ca.pem -CAkey /data/certificates/certs/opensearch-ca.key -CAcreateserial -sha256 -out /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.pem + + cp /data/certificates/certs/opensearch-ca.pem /data/opensearch-$NODE_NAME/certs/ + cp /data/certificates/certs/opensearch-admin.pem /data/opensearch-$NODE_NAME/certs/ + cp /data/certificates/certs/opensearch-admin.key /data/opensearch-$NODE_NAME/certs/ + + #openssl verify -CAfile /data/opensearch-$NODE_NAME/certs/opensearch-ca.pem /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.pem + #openssl x509 -text -in /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.pem + done +fi + +if [ ! -f /data/traefik/certs/traefik.key ] +then + echo "generating certificate traefik" + mkdir -p /data/traefik/certs/ + + cat << EOF > /tmp/request.conf + [req] + distinguished_name = req_distinguished_name + req_extensions = v3_req + prompt = no + [req_distinguished_name] + C = US + ST = NY + L = IT + O = security + CN = opensearch-lab + [v3_req] + keyUsage = keyEncipherment, dataEncipherment, digitalSignature, nonRepudiation + extendedKeyUsage = serverAuth, clientAuth + subjectAltName = @alt_names + [alt_names] + DNS.1 = traefik.local + DNS.2 = opensearch.local + DNS.3 = grafana.local +EOF + + ##openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /data/traefik/certs/server.key -out /data/traefik/certs/server.crt -subj "/C=US/ST=NY/L=IT/O=security/CN=logger" + #openssl genrsa -out /data/traefik/certs/traefik_rsa.key 2048 + #openssl pkcs8 -inform PEM -outform PEM -in /data/traefik/certs/traefik_rsa.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out /data/traefik/certs/traefik.key + #openssl req -new -subj "/C=US/ST=NY/L=IT/O=security/CN=traefik" -key /data/traefik/certs/traefik.key -out /data/traefik/certs/traefik.csr + #openssl x509 -req -days 3650 -in /data/traefik/certs/traefik.csr -CA /data/certificates/certs/opensearch-ca.pem -CAkey /data/certificates/certs/opensearch-ca.key -CAcreateserial -sha256 -out /data/traefik/certs/traefik.pem + #openssl verify -CAfile /data/certificates/certs/opensearch-ca.pem /data/traefik/certs/traefik.pem + #openssl x509 -noout -subject -in /data/traefik/certs/traefik.pem + + openssl genrsa -out /data/traefik/certs/server_rsa.key 2048 + openssl pkcs8 -inform PEM -outform PEM -in /data/traefik/certs/server_rsa.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out /data/traefik/certs/server.key + openssl req -new -config /tmp/request.conf -key /data/traefik/certs/server.key -out /data/traefik/certs/server.csr + openssl x509 -req -days 3650 -extfile /tmp/request.conf -extensions v3_req -in /data/traefik/certs/server.csr -CA /data/certificates/certs/opensearch-ca.pem -CAkey /data/certificates/certs/opensearch-ca.key -CAcreateserial -sha256 -out /data/traefik/certs/server.pem + + #openssl verify -CAfile /data/traefik/certs/server.pem /data/traefik/certs/server.pem + #openssl x509 -text -in /data/traefik/certs/server.pem +fi + +sleep 2 \ No newline at end of file diff --git a/data/setup/build/03_configure_opensearch.sh b/data/setup/build/03_configure_opensearch.sh new file mode 100644 index 0000000..20da56b --- /dev/null +++ b/data/setup/build/03_configure_opensearch.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +## run security_admin in each node +#for NODE_NAME in "node1" "node2" +#do +# +# COMMAND=(docker exec -it opensearch-$NODE_NAME /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh --clustername opensearch-cluster --configdir /usr/share/opensearch/config/opensearch-security -cacert /usr/share/opensearch/config/certs/opensearch-ca.pem -key /usr/share/opensearch/config/certs/opensearch-admin.key -cert /usr/share/opensearch/config/certs/opensearch-admin.pem -h opensearch-$NODE_NAME) +# +# until "${COMMAND[@]}" ; do +# echo "opensearch not up yet. retrying in 10 seconds..." +# sleep 10 +# done +#done + +# use opensearch-dashboards api to create index pattern logstash-* for global tennant until it succeeds. (this will not create it for you personal tenant) +cat > /tmp/opensearch_create_index_pattern.sh << EOF + curl -k \ + -X POST "http://opensearch-dashboards:5601/api/saved_objects/index-pattern/logstash-*" \ + -u 'admin:vagrant' \ + -H "securitytenant:global" \ + -H "osd-xsrf:true" \ + -H "content-type:application/json" \ + -d "{ \"attributes\": { \"title\": \"logstash-*\", \"timeFieldName\": \"@timestamp\" } }" +EOF + +cat > /tmp/opensearch_check_index_pattern.sh << EOF + curl -k \ + -X GET "http://opensearch-dashboards:5601/api/saved_objects/index-pattern/logstash-*" \ + -u 'admin:vagrant' \ + -H "securitytenant:global" \ + -H "osd-xsrf:true" \ + -H "content-type:application/json" \ + | grep "namespace" +EOF +chmod +x /tmp/opensearch_*.sh + +until "/tmp/opensearch_check_index_pattern.sh" ; do + echo "opensearch index-pattern does not exist; trying to create logstash-*" + /tmp/opensearch_create_index_pattern.sh + sleep 10 +done +echo "opensearch index-pattern created" diff --git a/data/setup/build/04_configure_grafana.sh b/data/setup/build/04_configure_grafana.sh new file mode 100644 index 0000000..dee9e28 --- /dev/null +++ b/data/setup/build/04_configure_grafana.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +cat > /tmp/grafana_check.sh << EOF +curl -k \ + -X GET "http://grafana:3000/api/datasources" \ + -u 'admin:vagrant' \ + -H "content-type:application/json" \ +| grep '"name":"OpenSearch"' +EOF + +cat > /tmp/grafana_initial_setup.sh << EOF +curl -k \ + -X POST "http://grafana:3000/api/datasources" \ + -u 'admin:vagrant' \ + -H "content-type:application/json" \ + -d ' + { + "orgId": 1, + "name": "OpenSearch", + "type": "grafana-opensearch-datasource", + "typeName": "OpenSearch", + "typeLogoUrl": "public/plugins/grafana-opensearch-datasource/img/logo.svg", + "access": "proxy", + "url": "https://opensearch-node1:9200", + "basicAuth": true, + "basicAuthUser": "admin", + "isDefault": true, + "secureJsonData": { + "basicAuthPassword": "vagrant" + }, + "jsonData": { + "database": "logstash-*", + "esVersion": "8.0.0", + "flavor": "opensearch", + "logLevelField": "fields.level", + "logMessageField": "message", + "maxConcurrentShardRequests": 5, + "pplEnabled": true, + "timeField": "@timestamp", + "tlsAuthWithCACert": false, + "tlsSkipVerify": true, + "version": "1.0.0" + }, + "readOnly": false + } + ' +EOF + +chmod +x /tmp/grafana*.sh + +until "/tmp/grafana_check.sh" ; do + echo "Grafana settings not applied; retrying" + /tmp/grafana_initial_setup.sh + sleep 10 +done +echo "Grafana settings applied" diff --git a/data/setup/build/Dockerfile b/data/setup/build/Dockerfile new file mode 100644 index 0000000..ccef220 --- /dev/null +++ b/data/setup/build/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:22.04 + +RUN apt-get update && apt-get -y upgrade +RUN apt-get -y install openssl docker.io curl dos2unix + +COPY *.sh /opt/ +RUN chmod +x /opt/*.sh ; dos2unix /opt/*.sh + +CMD ["bash", "/opt/entrypoint.sh"] diff --git a/data/setup/build/entrypoint.sh b/data/setup/build/entrypoint.sh new file mode 100644 index 0000000..d8c1b47 --- /dev/null +++ b/data/setup/build/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +/opt/01_precreate_folders.sh +/opt/02_generate_certificates.sh +echo "initial setup done. tag setup container as healthy to start other containers" +touch /tmp/healthcheck.txt + +/opt/03_configure_opensearch.sh +/opt/04_configure_grafana.sh + +sleep infinity diff --git a/data/syslog-filebeat/config/filebeat.yml b/data/syslog-filebeat/config/filebeat.yml new file mode 100644 index 0000000..162db13 --- /dev/null +++ b/data/syslog-filebeat/config/filebeat.yml @@ -0,0 +1,51 @@ +# for more modules visit https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules-overview.html + +filebeat.inputs: +- type: udp + max_message_size: 10KiB + host: "0.0.0.0:514" + tags: ["udp-514"] +- type: tcp + max_message_size: 10MiB + host: "0.0.0.0:514" + tags: ["tcp-514"] + +filebeat.modules: +#- module: cisco +# asa: +# var.syslog_host: 0.0.0.0 +# var.syslog_port: 9001 +# var.log_level: 5 +# +#- module: cisco +# ios: +# var.syslog_host: 0.0.0.0 +# var.syslog_port: 9002 +# var.log_level: 5 +# +#- module: cef +# log: +# var.syslog_host: 0.0.0.0 +# var.syslog_port: 9003 +# +#- module: checkpoint +# firewall: +# var.syslog_host: 0.0.0.0 +# var.syslog_port: 9004 +# +- module: netflow + log: + enabled: true + var: + netflow_host: 0.0.0.0 + netflow_port: 2055 + tags: ["netflow"] + +#- module: snort +# snort: +# var.syslog_host: 0.0.0.0 +# var.syslog_port: 9532 + +output.logstash: + enabled: true + hosts: ["${LOGSTASH_HOST}"] diff --git a/data/syslog-logstash/config/logstash.conf b/data/syslog-logstash/config/logstash.conf new file mode 100644 index 0000000..9e290b9 --- /dev/null +++ b/data/syslog-logstash/config/logstash.conf @@ -0,0 +1,28 @@ +input { + beats { + port => 5044 + } +} + +filter { + grok { + match => ["message", "<%{DATA:event_priority}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{GREEDYDATA:syslog_process}\[%{NUMBER:syslog_uid}\]: %{DATA:SYSLOGMESSAGE}"] + add_tag => [ "syslog" ] + } + +} + +output { + #stdout {} + #file { + # path => "/tmp/output.json" + #} + opensearch { + hosts => ["${OPENSEARCH_HOST}"] + index => "${OPENSEARCH_INDEX}-%{+YYYY-MM-dd}" + user => "${LOGSTASH_USER}" + password => "${LOGSTASH_PASSWORD}" + ssl => true + ssl_certificate_verification => false + } +} diff --git a/data/traefik/config/encryption.toml b/data/traefik/config/encryption.toml new file mode 100644 index 0000000..51bc716 --- /dev/null +++ b/data/traefik/config/encryption.toml @@ -0,0 +1,9 @@ +[tls.stores] + [tls.stores.default] + [tls.stores.default.defaultCertificate] + certFile = "/etc/traefik/certs/server.pem" + keyFile = "/etc/traefik/certs/server.key" + +[[tls.certificates]] + certFile = "/etc/traefik/certs/server.pem" + keyFile = "/etc/traefik/certs/server.key" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b780ae0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,412 @@ +version: '3' +services: + # this container creates certificates used by other services + setup: + build: ./data/setup/build/. + container_name: "setup" + restart: "no" + hostname: setup + volumes: + - "./data:/data" + networks: + - setup-net + healthcheck: + test: ["CMD-SHELL", "test -f /tmp/healthcheck.txt"] + interval: 10s + timeout: 5s + retries: 5 + logging: + driver: "json-file" + options: + max-size: "50m" + + # avahi mdns broadcasts the name opensearch.local to make the dashboard accessable by this name in your browser + mdns: + build: ./data/mdns/build/. + container_name: "mdns" + restart: "no" + hostname: mdns + volumes: + - "./data/mdns/config:/opt/config" + network_mode: "host" + logging: + driver: "json-file" + options: + max-size: "50m" + + # reverse proxy used to accept traffic for http/https and nd forward it to the containers + traefik: + image: "traefik:v2.9.1" + container_name: "traefik" + hostname: traefik + restart: always + depends_on: + - setup + command: + #- "--log.level=DEBUG" + - "--api.dashboard=true" # enable traefik dashboard + - "--api.insecure=true" # URL for traefik dashboard = http://opensearch.local:8080/dashboard/ (needs ports: 8080 to be enabled) + - "--global.sendAnonymousUsage=false" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.http.address=:80" + - "--entrypoints.https.address=:443" + - "--providers.file.filename=/etc/traefik/encryption.toml" + - "--providers.file.watch=true" + labels: + - traefik.enable=true + - traefik.http.routers.traefik.rule=Host(`traefik.local`) + - traefik.http.routers.traefik.tls=true + - traefik.http.routers.traefik.entrypoints=https + - traefik.http.routers.traefik.service=api@internal + - traefik.http.routers.traefik.middlewares=traefik-auth-middleware + - traefik.http.middlewares.traefik-auth-middleware.basicauth.users=admin:$$apr1$$QIHSR7rW$$fW5DzBnqnCbHP5L2k6kfY0 #admin:vagrant + - traefik.http.services.traefik.loadbalancer.server.scheme=http + - traefik.http.services.traefik.loadbalancer.server.port=8080 + networks: + - traefik-net + ports: + - "80:80" + - "443:443" + #- "8080:8080" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./data/traefik/config/encryption.toml:/etc/traefik/encryption.toml:ro + - ./data/traefik/certs/:/etc/traefik/certs/:ro + logging: + driver: "json-file" + options: + max-size: "50m" + + # Opensearch two node cluster + opensearch-node1: + image: opensearchproject/opensearch:2.3.0 + container_name: opensearch-node1 + hostname: opensearch-node1 + restart: always + depends_on: + setup: + condition: service_healthy + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node1 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - plugins.security.ssl.transport.pemkey_filepath=certs/opensearch-node1.key + - plugins.security.ssl.transport.pemcert_filepath=certs/opensearch-node1.pem + - plugins.security.ssl.transport.pemtrustedcas_filepath=certs/opensearch-ca.pem + - plugins.security.ssl.http.pemkey_filepath=certs/opensearch-node1.key + - plugins.security.ssl.http.pemcert_filepath=certs/opensearch-node1.pem + - plugins.security.ssl.http.pemtrustedcas_filepath=certs/opensearch-ca.pem + - cluster.routing.allocation.disk.threshold_enabled=true + - cluster.routing.allocation.disk.watermark.low=97% + - cluster.routing.allocation.disk.watermark.high=98% + - cluster.routing.allocation.disk.watermark.flood_stage=99% + #- network.publish_host=192.168.57.2 + - DISABLE_INSTALL_DEMO_CONFIG=true + - bootstrap.memory_lock=true + - plugins.security.ssl.transport.enforce_hostname_verification=false + - plugins.security.ssl.transport.resolve_hostname=false + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems + hard: 65536 + volumes: + - ./data/opensearch-node1/data/:/usr/share/opensearch/data + - ./data/opensearch-node1/certs/:/usr/share/opensearch/config/certs:ro + - ./data/opensearch-node1/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml:ro + - ./data/opensearch-node1/config/internal_users.yml:/usr/share/opensearch/config/opensearch-security/internal_users.yml:ro + #ports: + # - 9200:9200 + # - 9600:9600 # required for Performance Analyzer + networks: + - opensearch-db-net + logging: + driver: "json-file" + options: + max-size: "50m" + + opensearch-node2: + image: opensearchproject/opensearch:2.3.0 + container_name: opensearch-node2 + hostname: opensearch-node2 + restart: always + depends_on: + setup: + condition: service_healthy + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node2 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - plugins.security.ssl.transport.pemkey_filepath=certs/opensearch-node2.key + - plugins.security.ssl.transport.pemcert_filepath=certs/opensearch-node2.pem + - plugins.security.ssl.transport.pemtrustedcas_filepath=certs/opensearch-ca.pem + - plugins.security.ssl.http.pemkey_filepath=certs/opensearch-node2.key + - plugins.security.ssl.http.pemcert_filepath=certs/opensearch-node2.pem + - plugins.security.ssl.http.pemtrustedcas_filepath=certs/opensearch-ca.pem + - cluster.routing.allocation.disk.threshold_enabled=true + - cluster.routing.allocation.disk.watermark.low=97% + - cluster.routing.allocation.disk.watermark.high=98% + - cluster.routing.allocation.disk.watermark.flood_stage=99% + #- network.publish_host=192.168.57.2 + - DISABLE_INSTALL_DEMO_CONFIG=true + - bootstrap.memory_lock=true + - plugins.security.ssl.transport.enforce_hostname_verification=false + - plugins.security.ssl.transport.resolve_hostname=false + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - ./data/opensearch-node2/data/:/usr/share/opensearch/data + - ./data/opensearch-node2/certs/:/usr/share/opensearch/config/certs:ro + - ./data/opensearch-node2/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml:ro + - ./data/opensearch-node2/config/internal_users.yml:/usr/share/opensearch/config/opensearch-security/internal_users.yml:ro + networks: + - opensearch-db-net + logging: + driver: "json-file" + options: + max-size: "50m" + + # opensearch dashboards for search and dashboarding + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:2.3.0 + container_name: opensearch-dashboards + hostname: opensearch-node2 + restart: always + depends_on: + setup: + condition: service_healthy + opensearch-node1: + condition: service_started + opensearch-node2: + condition: service_started + environment: + OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' + OPENSEARCH_USERNAME: "kibanaserver" + OPENSEARCH_PASSWORD: "vagrant" + labels: + - "traefik.enable=true" + - "traefik.http.routers.opensearch-dashboards.service=opensearch-dashboards" + - "traefik.http.routers.opensearch-dashboards.entrypoints=https" + - "traefik.http.routers.opensearch-dashboards.tls=true" + - "traefik.http.routers.opensearch-dashboards.rule=Host(`opensearch.local`)" + - "traefik.http.services.opensearch-dashboards.loadbalancer.server.port=5601" + - "traefik.http.services.opensearch-dashboards.loadbalancer.server.scheme=http" + - "traefik.docker.network=traefik-net" + volumes: + - ./data/opensearch-dashboards/certs/:/usr/share/opensearch-dashboards/config/certs:ro + #ports: + # - 5601:5601 + expose: + - "5601" + networks: + - setup-net + - traefik-net + - opensearch-db-net + logging: + driver: "json-file" + options: + max-size: "50m" + + # simple logstash listening on port 5044. Install winlogbeat, auditbeat, or packetbeat and send data to this container (5044/tcp -> logstash -> opensearch) + beats-logstash: + image: opensearchproject/logstash-oss-with-opensearch-output-plugin:8.4.0 + container_name: beats-logstash + hostname: beats-logstash + restart: always + depends_on: + - opensearch-node1 + environment: + - OPENSEARCH_HOST=https://opensearch-node1:9200 + - LOGSTASH_USER=logstash + - LOGSTASH_PASSWORD=${LOGSTASH_PASSWORD:-vagrant} + - OPENSEARCH_INDEX=logstash-beats + volumes: + - ./data/beats-logstash/config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro + networks: + - external-net + - opensearch-db-net + ports: + - 5044:5044 + logging: + driver: "json-file" + options: + max-size: "50m" + + + # uses filebeats modules to open syslog ports (network -> filebeat -> logstash -> opensearch) + syslog-filebeat: + image: elastic/filebeat:8.4.3 + container_name: "syslog-filebeat" + hostname: syslog-filebeat + restart: always + depends_on: + - syslog-logstash + environment: + - LOGSTASH_HOST=syslog-logstash:5044 + command: ["--strict.perms=false"] + volumes: + - ./data/syslog-filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml + #- ./data/syslog-filebeat/data:/usr/share/filebeat/data # not needed for test environments + networks: + - external-net + - syslog-net + ports: + - 514:514 # TCP input + - 514:514/udp # UDP input + - 9001:9001 # Cisco ASA + - 9002:9002 # Cisco IOS + - 9003:9003 # CEF + - 9004:9004 # Checkpoint + - 2055:2055 # NetFlow + - 2055:2055/udp # NetFlow + - 9532:9532 # Snort + logging: + driver: "json-file" + options: + max-size: "50m" + + syslog-logstash: + image: opensearchproject/logstash-oss-with-opensearch-output-plugin:8.4.0 + container_name: syslog-logstash + hostname: syslog-logstash + restart: always + depends_on: + - opensearch-node1 + environment: + - OPENSEARCH_HOST=https://opensearch-node1:9200 + - LOGSTASH_USER=logstash + - LOGSTASH_PASSWORD=${LOGSTASH_PASSWORD:-vagrant} + - OPENSEARCH_INDEX=logstash-syslog + volumes: + - ./data/syslog-logstash/config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro + networks: + - syslog-net + - opensearch-db-net + expose: + - "5044" + logging: + driver: "json-file" + options: + max-size: "50m" + + # api demo example. connects to coindesk free api every minute, uses jq as a parsing example, and sends it it through filesbeats to logstash (cron -> file -> filebeat -> logstash -> opensearch) + apidemo-cron: + build: ./data/apidemo-cron/build/. + container_name: "apidemo-cron" + hostname: apidemo-cron + restart: always + depends_on: + - apidemo-filebeat + environment: + - SCHEDULE=* * * * * + - USER=root + - COMMAND=bash /opt/scripts/get_cryptocurrency.sh + volumes: + - ./data/apidemo-cron/scripts:/opt/scripts/ + - ./data/apidemo-cron/output:/opt/output/ + networks: + - apidemo-net + logging: + driver: "json-file" + options: + max-size: "50m" + + apidemo-filebeat: + image: elastic/filebeat:8.4.3 + container_name: "apidemo-filebeat" + hostname: apidemo-filebeat + restart: always + depends_on: + - apidemo-logstash + environment: + - INPUT_PATH=/opt/input/*.json + - LOGSTASH_HOST=apidemo-logstash:5044 + command: ["--strict.perms=false"] + volumes: + - ./data/apidemo-filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml + - ./data/apidemo-cron/output:/opt/input/ + #- ./data/apidemo-filebeat/data:/usr/share/filebeat/data # not needed for test environments + networks: + - apidemo-net + logging: + driver: "json-file" + options: + max-size: "50m" + + apidemo-logstash: + image: opensearchproject/logstash-oss-with-opensearch-output-plugin:8.4.0 + container_name: apidemo-logstash + hostname: apidemo-logstash + restart: always + depends_on: + - opensearch-node1 + environment: + - OPENSEARCH_HOST=https://opensearch-node1:9200 + - LOGSTASH_USER=logstash + - LOGSTASH_PASSWORD=${LOGSTASH_PASSWORD:-vagrant} + - OPENSEARCH_INDEX=logstash-demoapi + volumes: + - ./data/apidemo-logstash/config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro + networks: + - apidemo-net + - opensearch-db-net + expose: + - "5044" + logging: + driver: "json-file" + options: + max-size: "50m" + + grafana: + image: grafana/grafana + container_name: grafana + hostname: grafana + restart: always + user: root + labels: + - "traefik.enable=true" + - "traefik.http.routers.grafana.service=grafana" + - "traefik.http.routers.grafana.entrypoints=https" + - "traefik.http.routers.grafana.tls=true" + - "traefik.http.routers.grafana.rule=Host(`grafana.local`)" + - "traefik.http.services.grafana.loadbalancer.server.port=3000" + - "traefik.http.services.grafana.loadbalancer.server.scheme=http" + - "traefik.docker.network=traefik-net" + volumes: + - ./data/grafana/data:/var/lib/grafana + environment: + default_timezone: 'Europe/Amsterdam' + GF_INSTALL_PLUGINS: grafana-piechart-panel,grafana-clock-panel,grafana-simple-json-datasource,grafana-opensearch-datasource + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: vagrant + networks: + - setup-net + - traefik-net + - opensearch-db-net + expose: + - 3000 + #ports: + # - 3000:3000 + +networks: + setup-net: + external-net: + traefik-net: + name: traefik-net + opensearch-dashboards-net: + opensearch-db-net: + graylog-net: + apidemo-net: + syslog-net: diff --git a/docs/ohunt_overview.drawio.png b/docs/ohunt_overview.drawio.png new file mode 100644 index 0000000000000000000000000000000000000000..a64637e9091413c95dc193438f25d718bc97e164 GIT binary patch literal 32548 zcmeFZcUaTe)&`0N5gXD~dJq8-=>(Kc5+IaNq&Gw8C7}jF3n)qx5Cy4H?KEj30zm}? z>8MBtrGpgd9qz`NIp4sUnRD*_^FC*uXJQEX?Xvc2`(5uMR8vFg z$;b|Kk&zu}I(h_r!pr@x8~k&?O)bA8liW{e2*@%YtA49(_BkCC<|gyw$h{UW5d}$07>g9r%T~=x=Ym#6@|* zC-TnDj#m0s7OHmcT*3%35$I(RaquyZs`52;H7;RA@ZQnR!3zANWQB3S?LK1ZiuSy0 zhXY?17Q8HcSps~dXzPk|0$)N12?}2pyeuXtD0W#$914E(e_KNcNrGR2JqQcxpsmra zc7K|0ciUF(Xq(@*Y$GPxw zHiR46Y1@Li5Lh8$Z#6|RCj^#&K_i{uSTP$7eI1;Jfuy7*#$8Zd!P3jxO;g^*THDA= zR7=AFW^V|S^tABAsGzhw1Z`A2^u<(k;o3+IMYs(P?Sz11J#{oCwG6#HEmXCPV1{;X zFg+cNx}d=|4|f<=M;9rfXQW}PVr}Q4>EsR5w-$C*)e=MMfR(^-hLUJ~@Ww$GhR`<9 zK^eJNiYZ7SVQ5VP0Rz=_h1n~K=_`7}ZLzjCCKgwX-IuIT=XqT7#;fg^IYOn}~Q|S zT;WF%h9n5udnh`33cKiO8liNIU`ArLz%M8`I>R-vj$Y2n-WsA{8sLNwSO;feU13)O zO2QSbX5k@f?*Y@-bGT-qNU+B$hzQ%GR8W!{B2F$ISY>TxcNL_E7S>rDCaI|?kH#3m z2+A0O8&1r5cO+d6K|!b*(nA6!2t}csG>zP}u&(+xuG+c=>hekmDB4L&L&HE?79uVPao|!Ao>ro0A$@UgsG_crl7z0fo0yQHILtPzyu2f~CHMh@QQmyrh+?qz%?}_sB#Q#6@gWg`AzNjSK_{_8$5| z>W;u`DL@U>b#29kj9m4wa7i}}muv3&67sqTcW*U$H6xUyrjdw*0@TtTE2yB27Zs8O z(~9BHqIzzY!j{Uy&IVqJ!dQZnBpi&Sr|j&e>*?j>4AYY^v{OPjiHYJI(HM+@lA)-& z8s5Oe!xJX2i&BMQg_JBbU~U*e7f-10u65dqU{tJaLh^ zgYa}jYPnsrMOxbkX=$j5Yr+uDHfSvgsE~#|N>kg-9uD)iM8NQZ2#gqratiifKL{m9pA`TDOg|^IC}Kx zhcogaopPLvwt+{A0!_YuCg-seNV71q>!K#4l zl`;D1$n+T)_0J=;^8w{})~o1L+@+4*=4)S#%JiBHi>-CHI#%WmbcxwCtw(6Ziq)p* zwsZNOmLofG_|Grw59A(eC$zS=FKv@Zq%D|bc1ZbW@KYA}sUxd>(jor}I@3S!@})mN z^lOZ?Qt)KkU=;zPO-;>@5Z%~B!bseZ6}_x>PeB%gfQ;(52**IOgGbo|{`I1%9f*AF z`Zy%n+?*cGSAH4tM%dy;NV)9$O!82iepdTgE~{|APC70NoqUcx4+h8?9-hqNOv%?n zWM}8yST!p)YNw|*_PrPzmYcR}v6U*c#5RdnDzMKWeASHMX*gTs0{Opbci&ahw2mP{ zLKx034y>1|z@F6dUj7n(_VbStA6sKn^|=z4?hofo-ic@MXuS+@mH6|b!P6hfA$;fE zJ*wB*?R8UW+rB>4em?#p6c-+nn|QqjMk^ggYERc2VhzzKJa(R5wf4Pn?`hH6Y{g#NMaa0|H1!mYBki@BJLB?ilB08-#O{V6?g-1#K=i?Q`rmx9ew9t^)rAl1?$iF8~Fp@)a|Y1?@GuFAwxr!qwQ zWVOE_Nml)+o!r@Iw=W(nUSRe_XhCY-Pm3r^;lgVHERV4^&QP6SGwC9WVNjxc7@A!&H z`TW`Xq@Q|EVMt1br>wRZ_gbpnEGaQymE_9(FpY&B{ zr%gX}Qg+$qac=H2y{zsBsN)C1XdY4QRVT88>W3%!E|AzEM%M-T*o_DiVaNKG&GeK{ zBU1%CT`A|KO2ZlFR?tobq#o>-lU?N86}xKT8l^BeE01ypB&g5 zHN^EXiNDHz(R(X`d0j8E<3;<6?Xn#mbM^exka%LzUG%_Ade4uW!X>thKAPnHcY)>K zY)I$WYbe#U&lnkI+9-64_}ydMY}cV%dzS-scnvT~N)&Xss<>IP-N$`Wdp_pRX; z4Uo*6_uqwn1jep^y(6FpJlfsIpP!QE6;g%Jl{G_Ndg`0EVl#qtyxY#~wH5Xjpl)#U z?*z;i$Ak5+C-@OX6~Y$mH`o11PJLg~f#9H;Wf?ok8L&3a#<_h97NvW}cx>^($*sl? zW2kR-r0oY&U?c76Mn5}CNa_>EY3aBmB=?;nl3TMfXYgzmy9WcxX8+Y=6M;qJAc;sBoMH^MmiaFt2j_JEdc^vmtfSz>Dacow{uEP&1~^ z@a>t)dyhUK37Gm?%leP2OS-8WR{08$D}&x>1-3`e^G>yQ-0^=q{cOG~F#{$u^x@gQ zRni;&56>2R6i5Euvfzl%>EMjyESrZLIrS|NrG3lpAURUZ@KSP8PaRK>slT^ zUIUR+{h6fZo=4Nl4@T?K$XA4k%-es`&UpYFNWa@XPNh92isZcBcY(U`A|+ELh>VH? z9>vwb6D(~O9KM|aGlVqllPwnm&>dWd`GIr8hq(A+>r3FnO7gqZL^b_|UU)E@d9yDH~}u+c}$#_7GCd;2%;!RXGaF}0z|KMZ0kWFZ?y z^Kl47#ZngjeI{Sep9T;0gIlVUp1sE=4G$g07s2+#wbMK$0SuhW={(U=lsw4uC7E3N z{7ET!5zibHGOza^Mqr43JYucv54_90o;I3&Zm#zr*oPpPUPbP2SE&Li$nHJyF)Aek z=6cm=Zx(Pg##k<#+9#}T-c{thEbxgRr)@*`Mhi{st_Pd@8%d8J3zddh!tKbJsgnQN zHKx^s^zkCkN6G%3kho-cAh@_BjL1?VH*p5K9=8=5JOr87_y;-ddhOM} z@!IaUsyE=4R{y-hIUwEI)V~uD%`9!S?mr(wr!GwIKkT6b%)x!&Z>4~zju!h5b0lQI z=Fg|`ME6EM*ca1A7+N!`H6R7A{`sJt+}O4Lji+baf61W=ch!+&WOCI=Zq>O3H>2_e2& zW_QkTLChJVabb0_5aBYD^FYP4-Zw>k-7iAwGZ!EYaM;+V(XSEokb;~L_wYI&7;GmM zr3NfOD82K7yUQ zhNoRW7ZHI$cSAGD|II60W{lp^Gv`mwI*)y^LoC&$VFvEyur0aWJT-NfDGW~^?Xq-n zfStPQ62#%hiu{%#%#1cV4sNL~ADL9O7j`fqUpz~r3b%Y6VymT)%9a1A__OecJ($d1>mkfjJ+Gz3w9I^uhIutv;|}RY<{$ z(@I|gZ-NZ?z~UYJ+Q99TO!yj?T9yuvuZA`e5`!U}La|z?h<(iKP?3u z9aibaJkQOAkbI> zJGb~5)iBjnxU+dbbDg9aMww$>+TgTQJjndvnSy}kdQ)^e$ zw?~8v2Tvw$hL2V2O&q;ToOu1oK`M}W*u4DEM6Vv0JK1m2M`LL7=vo7IGL5|V9Tz9! z&g;dQvRPIct9m*=TdOCj`ep=pZrRKm&zZ`dZ6_ZepQqs={g@hV(*u{+C)V*rE25Xz zQ>a2V7fzhh_8c}>ZsQp+sH>c>k$S}0EWdrLy507Yuk2B2g;PVSLz^IpK7IFXV*hG% zl<#m107ra-=Z0!M-EAwkrg2u?MRpA-J~O2rK%Q5WV5ET$0gKOzF>uTHka3y-k0oee zwrysVuv0y}@@<>OP{wRR_}@+o*McnCda;_H8&e``4XzGF$(HBkwanh6m?SebHLdiX z%CGHCi1&WN=j&N0v)&fvKW8Jm(WL>@^BwclO6889_) zxO|RN$w(AHM3t_bg>1BDzI=Uy}@#-h9nAT$lm6{{{$(;v~zJMHB#k zj(^QDUQwlPBc^0kl0@S%GjHYM>F{LIS*A~1+E%3S^DQE_%6-v94tS<7hGVCmap7*H z1jQO74I@0XHS;^c*Djf@I0FW-OzzwMZ3|wReSUdHIVlh3W$HV(;yy&N$!>I1ZZF|8 zm7-3~tb&=XCcixAWA$5X8*%8!mDYh>*fm?Wj%-7>Y9-nAw%DDYR6BxGZx2y1zgp?{ zX{ZjRwW_-SsfGfKWY6nS-Kx`nvC(Bn6MKh<%g_)*)T-k0c`QghX`^WSb#p&h{tT#JE0`rfP#g)`Ow5bR@PqdItXwtRu= zsm4R^;dXs<>Gz7Br85@oFLYhU9(qpR6z_2N^pvginj1V0vqa}oUF`psK=;9|enijC zWKml2E-^)3aWZoAIPbX;rsHe+r1)U>YRS6FyRUhWL98cRem4V2<+FItpZFf9l_7dQ zy*u^JsimFmV77|yVSKq~K6P|>CWJKB*QDL2x~GD6E&FD?t17;&!>Kl{ci(XCD1;%_ zJ}7L*3XF2FxSX07IDa{eCS9p&VfB-QEDzG_l(t@KQ!mzPfg zw_BKduyb^}WZ*$QW@$RXH<8lB^&=M(wshn{4N%LiM$iwaB5JS0PhO6E|&;qDp;mm_|47(kRncj$KMCT{fxh6fEwoX*DG%F67vXCA|a?i87XcbnH}wl%Fdx)p&)`j=wL*b=LYE zPg#sxRW>gmf@absW8~|M;A+8ee&;#^TXT4(HEZ>}CJv}t)T@%ibFJ+L_E_ZSC-oG;euRDmSA#APP^`~|8jB%$h>sz z98n8)K{Lko#p?zTKCCM?X3DtDeCKMCT4Z-N+dyP?KmU}#7s#vgBs=-4Cw3_U+TYrH zsXkZ*tp(}&Ch_U@xX&nLN9yM7FDz?}+M@MPhWR{SA*zWGx4T3R7l6b_pG&EY`>^&- zgiDHo*V@yZ`HK5^j`{X{9MpG+X`s`;I^ksq}vR8;v1Sebh80z6Z^QE7#y!SSBJ=&`O}20RoGHbt;9T+9m1>?plG9Q5Wp^eHHg`l7qU}VsR5FwhhI>H z0h1l=BTqIS90X3al{0=~$&h;{nAO|1X&59(c{vwbTgVwr*f(Y?D|`JnriF>yzNI_c z8zaN_QMp@FBXQ;rfDwGz-i*gYMpLkQ$$f}m;_mLqDd{)r8GT2=76ZbMKCAaQN0(cR zw1ce3Mt)DWaY^p0ysI;>Gr!KPjioX!G}x9!Y>8|wWaYj=f~*ok!RG7i)QBseL+r#i zx|FXDh8M@ufMERKM_#3Wcs~9jCGg4yywR?rN>Qv^NkD*(vr)v&A>P^0nW1Bw<4Usk zFd&Pjq+YY^6+eKBn}Ew+4^}wE9}!Nn;rUyw(9g`bT;6(1$A~SPer0q(A^Z@?(y~n4 z+u+4J{?o-B_T<4<4G*Tc+ywvN0(>kTOW_0gc%jT9NiX^J=6r)g&CLW6oB9lRxc#jI z{0D8n%O5|EC)%VCo+SG%cL@WfG741K_!708IlYv*WtaawyRi8Mz3?Jt*{)k^rPpn1 zL#St><$-`X3BJ=kwwp)1528F-o4GOhg?rbCVQ<=)MJ(&?S}{Sf%*iv@@d}ne!$NER zvf+EGi%1vz)#-zM;)UVeS2a zcIsuA`A$!EWb6Ce3{2qO{{Gf+hv~+a#eqE1`U_Y%&1hP2{B=f&#uuSFDCuHiaZuM+ z)+`3usV&hm^(tnRyA&N)uSQ@&A1QwpWOdt{e0k-4L5#&4L zJ`&VV(>4{s(6&955#*P4C?~DeDKm>XgO6)UsK|)7EnE51^YhBPULqPR>e&NS(9AI3 zqUNQ$r{b&&Icd2#kH2(nG%3}j(tnF_=oO`JS>xq(7nVfRYncr_gUkA()4Qu#zVndS z=EEf&m1*TUT0*DuR3pb}_mbB>!!4#Nw)9Rf-G7Sfv0+r9^4b_5DQB>4s>Fn}+?UO^ zalpZQcHS6Q(O!9&kk(PR)+~@ZM(A3|d)VoIapjXYpWM~j{_*mkLb)Ju>5gOQ*v1fL zo%`D4T4Gwan-tF{N$V)Ia8#I0RxT|^Am9P_HLlo&&z@{b;YCL&tvF%Z&Inn+rq423 zON}0QVEs7gRbi?uIz=a)55j|dZT<1$;Ta@ zo5+5fgd_0|)ke+sfh^h|Qjk`dIrnM>NFbN<2~Qj8I+=MG=_154tZwo`X4FHX3^(7j z@?_9-JzpLjzw-FaM~$^hs<^8d=dM@xhc0PvmsYc5doF#i%L+*2F$$sBK zwMO5uxR!eL&s2wdylY2Bv;o>2eb7v`ebWyL6;wi=U1|=o^j7A!GTg) z(=_d}vCx#UPzJ8kR_}1D+jNYM#`BY1y|>d#uE7RKXl&oh9g`Iy%!fPOOwx^od>OZf z5Vh3&W}?QW->ZbMY(V!ovSV5;svko#%>AL^bw6LT6FRXgeV#kBN^E|MA4~f1#_5rK zRLgBb{)4ost2?Rw>l5I1od?B_%*nlCc6-6-r=u*m@^K<7yiwX?-FWm6m6YaX5QlQi z;w�T)DOocD};v2kqUa`;o>KVZ7^;oKBf2o=8&j^R^n}9ar1M{Mm`Z)MuOEKJq2y zt|PFNSdG!+pW=lu=SVZWIikp+jDb;uj?lU^@BG#7BQ>G7*KCX(n&Y{zXJxI06E|1q zPMW7$`Rk+&eHntH8nKJrxz^?f8-)~9mO zUuMKDFvVDxc8njtvu0u8o0%v5nZj*?G5>WQfA}cE3@9ISN#%8fTVb2SaK?SE`&8Y_ zh{J$Q?V-1&1vRg$p&`WGobYAn{oxAfFG%~*JhOUpZm*3 zxWRH*CC)yT?h$1KhCFVm^`4H?oh5wxF(5~&!iJyR;Ck15lY+{BI<9haq;+fgvw7>n zM9|yf4zn_g(Z+v_-gXM0AyjNy5mS$s?vebrbut%5>(!m#PpF=XKct70;Baq$GYgJDg=S!c`zcN93 zqMr11XOi2^q$!MD#vIh1(2z|)-;nEC^mNdmwhi}4i z+W@48&{bdM5An(z=#T%HHdE{WFv0(w?DrsA`x>@|#X0Ldeg{!PBWAn!{y>|LUxa#A zN1ffy1=fgM`3pz9{n}8O9aZo=t|+PbI=#F|k-V(WCSrL>4)~`0g?iT13gDgQMBhB9 zocqAEr5ZZD@P0KuxnKtyk}@`3&H2VDdcH-hBclW>_J+X&vNH9CLT z>+Gi7@7~z1F~dUri{cKrurqD?S6(f=xbCQj6Gs|N0-aNK#bi_6RYYB2w# z5t2x!zBS|4IHUQ7>WwXtg)f5+c@=t>D;7U(KV|J9NYR-+c`4k1%2Ye`*HA|vaMwd$sFVa?a`P0~Isw81aM!#u;LawQ0mqd%ebDcFb z{M9h&L2z~|8C&Hi+)gD7$R)NT5BZ@RFxy;pv)dQiV!0#CIG=Xh(Cc95KlePZ|MHhQ zLvGtP$_J(fB4Z(^>LbyMW|$2Lii^l15!SLi!oq0H6@e*nay}%Fn{h|w>$MJSE8>E7PnV=wMNlKbvwvuk_s?=Djh;T*?|rskgm;!r62H zvcw%m!$uvA3az_)ug^XGU&MKUu(Y~uFsx+)q|Y`XsL9$_H)N1kAtxuMB@7A z)Zy}l=8c8QosCM*9K*sB$lt1%vAM#>h_t&>6HVTU&oXk7G6cwm_W8utE)u{0Mvjp(@= zGwyLS~nSzM}N%bNhdxx2MGSwicHT69P0BV=yH>2WmDV6)DpDcJIZD;fhNbS=1VqD zTeAq))W&#I+ziftA*pyk0zqW5mpxSF{{(g~w7?+}r1>~EwRN@7ACpvNcb3#BHyTp6 zM#J$S_cB)Emc!@%y0*CtR(D@KWZn`Zm?dzouTr*Eglmuv99Dcc>Oj$Ae7=7Ty{ym^>IiM%#BGYOO=Vzo#PyEZ&Ho|c%Gsa*7ak1Yz9oAhGorxIvDJ*M4!;q1>50EeXt zId;CJOkMx-)cc8ALP1u)57c#7IZs8=#`xHH(DnKlUfj;4e&za5>h|K(sPMY_WTwY@ zF`A4%XGxb?+TY*Ux+}LJe$3~&&!T+rau0Oz>ths9=VF0v7xP^?qp_KSsvFb#B5=$( z(<=_LoAsBGqTS7RiXXv4U!}^(wmRgGHa|4xb1e4Wxb2ja@A!bP4NBH|-}n9@7wkaD zsCK3?RE?O^PojRxgKp+qkZ~933Q6*@eqIo%wF53`>c;TUzw+SJjaq&P;mR}p#94$I3Y`|`{BerhQ9@U#Dh|h{%hNN#X`pRT1lT#kSa<{E2HbVjCM~wf5#w z)6iSVgwcfJS(?PSCueN29>Pv{;+8Rb83`rOuDoksQXSBO-Tgz;O!9fK>yKeS`25{( zX}68 zj=X`KygXE8(A;`!Oy(7*?xKEh&&Pb9Y|mQqm-4H#HcfQOHLii^087pDzJ(6Sd7@t5pObE1&zdFUd`K^e3;xWKE@ukP&FtO`_7Ro zW_rjI?8CwdPkGwTK_eD8A7xa_ahq!qw`-y~m6}te2=i$GOt7DNq;fyZJNX?;2e_%1 zt?uIOTc8AW>NwVUj^azS3I{xN^?+-aFaQP`yv`L(D{m8Ij=XVzXnx(~8-fW{-%mTS zrhnU{P=C`l1u)jnXtR!T>=^Lodwl53a*dQ(w>2!b*R9f(T-U^Gc>Elde=*w9v|>^i z)+p7euTM;fm3XtRk8^mXA1!9Q^o%n?Myfyi#3$K~t;>c5uv0Rt-}ud@b2f^js(O#* zY=Q9;mP=zktjTLZrk{z;5`Vr#SIwN2_20}N*qZSy`D_ebwG?pfb*bI@K3B`O^L$%6 zVXMDlDq1C-@q^%16#%BKDS!MctXXQ94Iaub@}$iz)&01eYRXo(<3fP>M*6ci)3N$y zhCYuA{invJVW}IBN=_y9%ult;=A=;3`>_;lHHs~i^K5*r0Qb?Ho!IYJQpOcyJh(#q z7a5*p%Yusw9all*p!qV?71R$=te(9tY+eA8>+eb6c&X3%@<(nNX=yYGmm&b!;S9d0 zcFgTe(Q8FeXJYdl5Gvu^u}~Os5t$c=(&B55|5!O#EgYqEFL*PPC*R6i%v(_}6IvnW z9Z^Am=S;Nu(?(ZZQ7HM`er%Zu-T30z!g@af=Mv6vYAPSG;ecn(>7R;Ol6>$>ct3c; zltF!RXZ<}R$@$wu&;GTPQ&TCXh{OJyLu?JZRVmp#qvDHEA6B=0qwEggxSu7;Q-%Xfsa#RYrTW%uts^uJ1Xn)zuznjDy;Xr=-)^SFx%f{_U z&Sn?)dG%c^sYwqh1vt4;&ZTwN{wQJaAC_6mkMf%$F`9XyeHe|qpKuD=eR@>FaM@(l zka{i3efcgjkB&)&!*W*m_hM@ds;nzAcJLJe#Ck_se5_Ya?SLX=He_6z0osY_ZDWBEJ&4098Jgbuk z#skNw4**9}UzFr%{GFA?86(O~u=qGkq@?v+eJr2n#{^IuOD4~L_#Kq~a5g(auef$B z*hf5Q=LPbj`98ew=;?rs{WIf8MSAS*9{RWpl0L(3jrXTqqCZ^x1gg zlMGDu9>5nTk5I2!%TC-1-v z0PgK+P=+b!7?a&0C|M+KoL-G1uEi~VexmN<*OVycvobm0cS< zEBa^#6hkw#Ncne(BjE&PrR3cWP-}mPakvC@vD`0HMAE%8P5x#|!d1_iv_q#P(nL6~ z=ZOXS%vR`==411+yI#9wl1R%n%msqHjp)|FQ<5Mm_xjcLMs9{v#@05~RZ1apBrq3t z!e|?$SXF+>o>e643#T`)qfa7G>$1{6&L(W4MmC)sFPKdK{2IXFJHIVttgUtM(@On7E zi{lXh-5cS53A*o26BHvfa!&P^0~U0(9m~hm_6xZV*Hxy9IehKBkCKtRqcvhn2RJzo z+OzZ1uBZ>87o(7K7c#-$z33H67=}7EiNu@W`#(lW-2n-K2tkJ zK~q21@)_vHDEoi5^9-1Q%nQJ2gL|#zuNptt`SUP!5)b=w_v9DCDQeLqkmkdj=eSEW| zjTt^jfgEgGmU`{sSYPlbmizBDA<^%DRYzGb{Ab;z<9gqiUb&kG6SqG-R*fQnca;;J zm?##%R(Xn#fLMXj@|XMmD+uN;0d80b z-_v|y*!-%IWY2@m2E(_<#7*Y&t{-UX`ab+iX)ZlKCfW;>h9Aky0a-^eq)LM5q``3Y zVyn!y8SGRaD45BTzA8E*=rQQmTItpyV zM*i<93H!V`kJ71!SKY^3as8lVs3;3)X(RI?HqFVUcmR{$^!D*V(MD6a5l&B>0A=ti zx$=vNAK(KEfzkibf5>ZPNJ+X6kGEyct^uB#&?1OLJp>SbKGsD4m<%Cqj{$znU`;r& z%ypzbFQksmRp{r`k*>JD+)ka~!wMvg`$QIG*a=V7ZL5%`+*dU0JIY}i$AVIb=}~$2 z^DzSx$Gw_l^Bz7XPEEbL`PKt~y`*sm^9`ex$8ly?yg`o4`et$97HMYTld6E};``$Q zCyYw44+&MbcUamu|HMIDqiHoiCKz>`(>ik!ckoMlZ&uYA?QMj-@R)$L3_{8d=zAQl z_x0SW0CBCsp+#~m*#)q?o<{jU-0&KXRvHHGg17+YJXe72djlw!j$;}Vf8s?44gvCZ4RzA<3b~<>_dZG?DS$< zpR#JuM-%Cz^Fr&ozQPK$v=sDplP^L5?H8UMqkQIM;-3ZpZHqZzWA5UL?y*7r94`%F zU&HENukZYDI7!j5lL~byqRzC2Q<|=&u~m>OhMz`-*BxJyb+{Jua_3Ggjnm-(^*!YO zK=H$%BP=lX{gd@q5n1qI3u@35t|rw?8*K=gUs~^x(C+jQDs#LRBUpm%)g?X%%Du7r z+M^7SX@l(5GUv*Vv_aB|Xh~fBGf(!Y0?^je=yA5gbIx~^G6DMkX6|mtl*1AW=#1R1 z^M4W~eL@c@apcw?%}fF`>FIxP0e)-Cl%qXx;T`(%k>3V@750cUGKrnr0<3%tyY1et zr0UbxX_yc3R;h;sbK;A5(y1m|$CSq}9$h#=*>_6@Gbs2&{=vEpC`(sAmHd-$ILhAo zgzWw|@psz&$zBpbV`d1V)LFz-#+b{e@uazaw z+w&5NQ)kK`tU`A|uNI_jZ&TP&7?^`S@EO%jFNqHzTl}~7FJZdtG|3$JJEqi8XOwfq z)Bn9^i(L`O_^d^3C;ePVAPlTB^pErS{S4T_D!~Q9?@V=-5GxSi{x-6^=|F%d@-b6k zUe$8;#gu>1`*rr1AZ7FEs~>Vhlun!nofpdcWOa}l9G)`!WEDGWxH?TBiQAn*c({}WyH<=dEMxC6pAG@5lsl-!wyR7OeE;6o zwnx-PP5_kK-)6Vxyg)UJlV+bz$Q_P9OhFT}E*KJwkuu=@xAwR_qW)q}3na0>U2o5M z0fmU(r{;$tKq2nReF>JAVjg;j{PzyRKa2X2UJ~fW`^WRj;Q1Z`+uOv)ei|r5;Nc9J z6NBH+Pyq}5M^P7d2mO1G%%1bw?Q=Z8ug@{?s37I3^QV{4r>SbaUTXidsQ=D1zWLB( zZ_UZ4VL*_&f@q1{>(O43mosv$uRl{QV!7pgGdK70JAvmZA?sQ($}{`=ao8O|FN9%u zNhceevgA6D*g~e*9v5-9727W^@Jdr|^H^ur@WInH+N{&;Uh`#-<`5 z(orRcKYK#C|IP6Ke<_V82l(@j={N@F22e$99+!|vxy|6cy~b>AZk}2h^Ht?Ss>Siu zW=7#Boonygev9Oq*AC>FXM~5vbPT;f*(+?PAh$=v04RBtYKt2SM&-ef zx^Hb3&xQkbHSx z{2xs_@{}M$y~ylOaX%P<^purCRL@I(0d~JGHR(KBxn6EXzbBflCLnAF7IISER|uo2 zQGfgEdxN9q<4PbqT0ANw^Sfr(A;I_bKCyJ7STEcH26*?(*=gb31wb+_uS1Q&jO@IB zG^Gbtf)ts@`NA+-TUjgPHD&xRrlQrmd3+1BXa})!d6ZXyaqmBk&g* zC}=n{O+k$SWN%ypEr5|Iofq70$Ts`d(}gAfMjqpnm7jnyp_cjT#D)eYmf^xKV)4pdF}jm zs~oUkcX_Ve%R?U`+*lg`2rvRLmfR(9ee$`o((&x3l-e;xKICe%f z8Q@5_4=qXCv1oF(p7b^Mp}5kB52LyA3ecJBKv99YECjtgdzI8}-XTX(+ww=O4kRPj zPN3`Nqgjjx;HJOiF0m<#PD6U>+Ag8qo^HM8*)A2i08pe?@~#W^r25W(v_Oq7wlrO& z*qQ}>IiyOqzL{1U483sATiF(3t3hTuJ68CF!^esFdE+hFWGfH7DUmS$>U@*HhUdOs3o%09xz1wbT(l zQCoYF-RIK?;nVEGVk_y2QOvW{)cO7ul0Lv)3UysF9;##|Ft?=262lD=Hy!B4Y_ahU zFG{~goE3TU>cu4unFJH4q(rOO&-n%c#=fXrgU?Z{S)=*>jUL^@!v$6i_I&k;rase; zT8G-(9oGqXz=^(hR>bior$#>kt5(^m|&E zaOGMXu6_$m(1z8yAqVncQ_ntKwYl#vt4?J_wE0qYOuP4Tz4jZDqE6s~^nBqCfFi9F%F8ZV|KpkJ;_T_37QpdvYaqSq z8B_Tb%j`0ARg)$uRVOBduLl2n_&l8$P@;k^%m~^VTOXMQory2DtW_#6$%1yhEm~il z>X#)@+Bg5Aujw+3He_imbF0O_tXW_;;*Vo{#*=O?3VOSYm~<9Z3mB-ULR0%X0Q9ha zoNk?~wCl{2I2RzKNyLSQ5RTSdm=t$!^V|9y#InQFx9^ zGh&)iU8E=CNxeO_neY<5j#X=d)3NF&-_ScOu5S-eIL?)``!E|RLusY!!jeD)jS4L6p;zwK7VlU4y+y{gz4I%O=G(UT;Q+GY_rK^&|O zE#7AGvlG2_qHweGk`FYCw#d}|aZp!O{~(%JP{iW`>W7YPPr3@c-O3QR;i0En z0?eq_0z;C|*nFa2Aya6S)Dcy^S7lv{${Y&Q2%2*G(qJYt@XXKMBF;h!f?e%wrfH>o zX(lj~%+?nA+Ry0|IGdN>09(Jg_;Mm&#JtMF(>{kZzms2kZYN4Na3ua{Q%LoTh+=rI zZk7r-9$S2qQnnk3YQF0chZtFdA!vHi|2gaWKCh zxBR2SWt_EpMAsD_jvy(wB8)Pj7BHf*%}!-*jBU}Us!Lhod;$i%9M5xNUZQS-v-5Zy zR3TN-j%4%Ernh;l;a-pk);?@A>#?|evZ*Uu9~mFI1BzMNeCMELU%RrL2Zwd;Ia~*8 zv40Fpp0g!3g*jd3<2q;BGyUTvxWx!?6#eUtqRopOKdeGL^ms*3X}}B3%440a^K)_{9HQZw2=GLEY)j*z+5Ds?>bd`ryh-0hdD56PY^tA%R1 zH3O>AI&&Ce?B13dr8G)S%L1UV#WsoKO8Q2k4s3oWFT*JXh-ZsDq?_gwXsnA9x0oq6 zDlj$51?tUe^6khCd z;6tQsZ7*Q6mUEfg2JT53E{V8k`fDbLmP~PEl4xS&xyhZ4&3jn+`XaSATQ#*zivn!D z4HKV<88(}_5ppBxMajbtHSd#bw5FUGsMMxJIB2nmdJNLc0d z-V;Jiib)F-jQm(OKiPfrrwUmXD|+~B7E^VCQBU7k$MUFAr?^U&FsMgfQt7Xw$)FbN z5AQeXwn;}yasX%B&fH;Y<^k}$^*~1t1y?<1ChYH_QW$97^bQO%%=u=>Ds~eCt%}HN zDpHL*E3RKa-@}xo*UH?inSMV)N;NZ@!B874);TT0QR0(KMLi$m-HQCUFSocRwYZ0S zf{X%l3^S7D4y8dVShdyml3X8X4cMM1GyX`jO7ZqgpS4p9aElV(1*hj@qafngAn7jC zZ`9z=66G)F#C00xub}vSEp=7J3+i&O0_*-dY42P;to(J-rV#lQ?=$`A_$A2Z1wOK) zomq`qaEu34RSRn<@Q4cJ-nLp4ME$23*23(bqwps~B>n0kA8AFs8Z@vfi1gAY=b!Wc9nl-!{bm4=o#VwYl0z z@-=D$&W>-dqz|poh*vGL#STQ*Puw-Qi#j^gFS~rd#M`YtF7=Cl{q;k)*43&gYL|ly zXziK-nK}z?-YzAklvzp*9;!WufSLF3KD*fA)YqS{i?Z_BupsYk6zKT?rHw5?459iq zIOkzKQc#BLbc#*a?bfX)o;&XV+8(j*9s1|qyXpe`Sg88l(u3L{iZX%9^a(rid8ir* zr0}tn0<4=_^zlaZO-@$O&^YBxurT1We!D}W0$7%bg@t;+@=-*cJ?OiQqD~@uYArFa z`u<8G!E1TW6<9bu^4iOg!WXm^IzAzxyRj$}1wN}mV1B_Bs!cbLP;Fvk1?`wlwD-@% z1;y3dQF15A{lh5-DfCWHnjJxu(hXUrPfO1Uo6WVaE?g*&W z*d1elN=*iKY4`*XbR?mcsLbCCS`Ic%8WNSD0CEICVz);vIh^|-qv8wt8XF#s0usES zBF^hHR+~3w|7UX~JcM1sH2xbVNvDEy3wi~v34J=Bok0bs@9qCLy+sPf39>6z4WOIx zpQJod>eq$3{{SRZr^l&(s?#X6P&_ePGBWTnP|`o32RF)Mo>l=99Zs#3GD!q0H~GbR zMR2EWv_mSTviEz0-&S(_y_Ud5x&MA25dWLJM()gP8ph=;ad|qUI`s;HLIc{VgU=gG zis#f8tnU?2WUp3DXo)4bpAc$E!XA#6Vm z-v~H!kO}spf2a{k?S~C{=g^LsXDNJBmK+N?JD%_b9)YbaD8FH&Dk~We^*hZtPWV&n#nu99xuO_<7OBd9sOJ#nu%QZ23aQ%_ zuom(l6nHq@<)V$Mn)dP*AKLX)>$%dCE3`)Vp6{n0yY%KZeS2%_ETHbiZL8}0b4x`$ zfk}M*qt8+qHbkV0mTs4-k_CyH_z9vZ2QTtk(1TEj*$&2H;4p!W^)=*EZ|~cY=CW-r z;HU~ci8)UG*JMwu)>QBU>UmpRXA_z&MYCpvdZdZ0e9mgV%RfVoEmb^}XiGZ@*~_L+ z3P$BNyl>BS8=t#?|H`?4LFwmnMS8;YsMW&by!3AUp4_VWep~MU=$h*(MovIdy?Sb} z>2r%7<`MW$$$i2`MEyFs#Sz_T0RNe$F0U14A{~dG0bbpw)O@EyJVlKJWJD-xEdd<~ z*h10rW;DR>+8G8AV^V*Q|dK)?5HLxgNS7l8glS`n)iiqUb3JPT&nd6tus z^-&3lUZ?$>Nr4!M6FlOSBd(@NfHtS_Ls`dKQ+Qz?mWP{N+Rw(9b@}c`lh8`BR(%~o z{ZgUpZMP_{gSRQ``Z|c9BXu(HR9phx5sC{^o*QV|kwfmR&=H^jcOr8UXB{)Qi!H7i6(Jr;~IpaQ@rC z*{lc4;xiOchH30ngBnH7sUK{aplG1iFSk9K~Uljcf+Gq&H+a9R>syu2@)o1#{6 zXNH8u=7Z*y#i!#-34vOGB+{`oZEQbMF3JQ%1G&Ajj#>IZ;MD7nuGFc;EQOJ#B5ct$ zVaKyObFple*9Nqt2SUE&jSY0#)~8Kplt0f;<+kN|T-0i-dFim2bh4!hr)PRlW4$Mn zT;4m$hLJn9x8Fv_JC0e%-=NT7Ud~#ib35u=UsB87Nje#>tj!z1(y=Zm%uJ-DuA}rL zK^-8wvRLy7Md?p>q8DU`o@#hl5$A6Jcg)$+g9xDencdusAnwxcRdFuFu8`BdEp`RC zEG5$@Mkf{%?hpff8zlJ6U53&)7BB{bndpYNw6b`Y;riQ(wt(Wdvaa?m<_2!gR4!pb zWJ%Nl`-YzXL%Tv;otLeV9nb^)gRmr5|Eqx&ZTwxpywr`g) zwS2+bq|9Y~V2jVWSSJV2eF`$NrB28x+l`H}T{gGIbyakyTh0IOvE4)HzpH%j?Hto!ItkyW8Ym z!jNzQe=PVw6h~XVSC|CG5bLhUFY+a~;zqXD-jJ@Oi`sElct$H9<>W?N8*3Tb$=vWr zb_qXoIqT`w@#N;weVel-KAm_Mda%A4&3{)wJu5zYI>(wnt)XMz2uIVI>?P+dgtT;; zlLL{q=I^}Grxa{k>CRm{k0z=DhQM(T}X!zQV;9k z-v0OsFY#9I8zC-TQ!8)Z^nlFJ%7_m68cck)*628;extGFHgy8z<t>oQA;^UMn<|WmX1Ex+YWXy9oG_ptec33w9 z{*h?RfN21pyFuk1vY#LMpsene_Z&r(N{I##w`MGr;WoSK^=zEp6`F5hRoz1oJa=0U` zbk|K8P^XC0){Z+@QZ)Uz9-h@`D*o8c6+Nr`MAT*U6e^sO)ED%NOL6Us)N6=SPI<)A z(C2M{eoEik~x4$0KPe|0@< zpMYc@fO3P162VkGY?s1$1I&GD%%- z{my8&c+Viymu-rjQg<*a>ugC-eE{8Zhh9lEZa-KFNX0WB(4Id}aBgl!R2ji1Id!K- z_ci1ih&l>-3%|_?jUL`dI~B4$swl$eb8VD#E-K+Yi7rA&zbH$Y`lU3~_2874I8gD!wiebW;F!!oW*C2~5@7=I_{kjX6tl zHvIOn9EN{RO`_Gpj?zO3p;*n2Wptjf9ujxB=Qc~I#DztPYAO3~e^wQ7suAED5x4vk z9-a||>eHTtkL_x308ZGuf8l2t3IU(4c;?Td!gLJ&gKqIjR>6aNEyS&>$R^?eW$dqc znII^McO+&Yrgwrbd^aY&q9caiNY zWvBiMcG%VwjTEO4fSd(D?3XU zh+sj*sw&1`I>5|~s}pCy)#czzjEKRJ5w_)ZU4jt4}QoxjXW-9kQlfAVjC-5Ov-d5pQKjAS>yGUel()AcK^5w zp<+>6o}MaJnZF=5`Vm5X zia^SK{MU{B)$9tiGWc(|x}koDR(Cf-J*#``;Wa~1`g2EZt-+%om5{u*@hx<%=xVK- zJXNfhLyQU zbkWdSnVY!gIMJu#_El@f-LAjp5xbgk~;SAB_p?-zKcAN zQLmyDYeao5x{IAIE!i*5jM^;KyBgQVPiduITt1bd0V%E%=Xmad;4 zG&4?$?k>99bkZU=t73X?VcJ4Wrw=+9Y|pkBgqZT8-Y|g&c-_HPoq5Nf4nWn5RyDuV zpwM2dpt*z463%I{RE|(jy;$e6ZfPOHcDUnwSnj^(-aNYJefg_})(oSINQCDE9fQps zM9DjTMfXXkEer$o(HS(06F9ePR#s!t1H~?j;!ATA@dff#QPI&_4&&V#%p9d|-{FKI zz|;=J&G5;X<%+-q%>)|+bL(fDC$dhcty?6&R=A(?V%2T|ar1q{I!1mk^eZw=>z`yf z&5rF-!l9K6`*d3&{QTKVq z@7H2DJ;^e*yhi&7yMzskE_-jAR^XxOmAvXJ%UZRxr|Nv%!G{AtKy^JP9cRYmb3)X2 zvD)Twh#7wLS!*(vy6B^b6*==H<-O|odZA$sapv^yJFG$0L<++9G^Iy(&#-*i)hew4=QP zhYf}KKIURbSLan-1k^sZJZSsVwm9R(`9F$1#`xF0d#~)aos+@uG1bD!#K)Rx;ff@$ zkN6-qk0&8|U}+1?pndtaYdEQw-uEoZQ1N1Zaa4(s?~g-@H}k@0(~;HBX!qPbQ;}~w z(73cXORMG)Ak|IaI?+GFCF^L1*21-o^A7#1C3!oFcXb49E6buXMV$DDM`zY~1@|#J zn1OH7Ln)VQ#>TOYcj47||7?#2zuEoA0u!(jV}nI)+L-H)YLSpyn*>6YOE37hIg5xB zQhm5JMk|sr%FOV literal 0 HcmV?d00001 diff --git a/env_example b/env_example new file mode 100644 index 0000000..b74b828 --- /dev/null +++ b/env_example @@ -0,0 +1 @@ +LOGSTASH_PASSWORD=vagrant \ No newline at end of file diff --git a/reset.ps1 b/reset.ps1 new file mode 100644 index 0000000..cd3e711 --- /dev/null +++ b/reset.ps1 @@ -0,0 +1,19 @@ +Remove-Item -recurse -path .\data\.env + +Remove-Item -recurse -path .\data\certificates\certs + +Remove-Item -recurse -path .\data\opensearch-ca +Remove-Item -recurse -path .\data\opensearch-node1\certs +Remove-Item -recurse -path .\data\opensearch-node1\data +Remove-Item -recurse -path .\data\opensearch-node1\config\internal_users.yml +Remove-Item -recurse -path .\data\opensearch-node2\certs +Remove-Item -recurse -path .\data\opensearch-node2\data +Remove-Item -recurse -path .\data\opensearch-node2\config\internal_users.yml +Remove-Item -recurse -path .\data\opensearch-dashboards\certs +Remove-Item -recurse -path .\data\traefik\certs + +Remove-Item -recurse -path .\data\apidemo-cron\output +Remove-Item -recurse -path .\data\apidemo-filebeat\data +Remove-Item -recurse -path .\data\syslog-filebeat\data + +Remove-Item -recurse -path .\data\grafana\data diff --git a/reset.sh b/reset.sh new file mode 100644 index 0000000..8f86482 --- /dev/null +++ b/reset.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +rm -rf data/.env + +rm -rf data/certificates/certs/ + +rm -rf data/opensearch-ca +rm -rf data/opensearch-node1/certs +rm -rf data/opensearch-node1/data +rm -rf data/opensearch-node1/config/internal_users.yml +rm -rf data/opensearch-node2/certs +rm -rf data/opensearch-node2/data +rm -rf data/opensearch-node2/config/internal_users.yml +rm -rf data/opensearch-dashboards/certs +rm -rf data/traefik/certs + +rm -rf data/apidemo-cron/output +rm -rf data/apidemo-filebeat/data +rm -rf data/syslog-filebeat/data + +rm -rf data/grafana/data