first commit

2022-12-27 21:59:06 +01:00
commit 7ae0b00241
29 changed files with 1157 additions and 0 deletions
--- a/data/setup/build/01_precreate_folders.sh
+++ b/data/setup/build/01_precreate_folders.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+mkdir /data/graylog-mongodb/configdb
+mkdir /data/graylog-mongodb/db
+chmod 777 /data/graylog-mongodb/configdb
+chmod 777 /data/graylog-mongodb/db
+
+if [ ! -f /data/.env ]
+then
+    cp /data/env_example /data/.env
+fi
+
+if [ ! -f /data/opensearch-node1/config/internal_users.yml ]
+then
+    cp /data/opensearch-node1/config/internal_users_example.yml /data/opensearch-node1/config/internal_users.yml
+    mkdir /data/opensearch-node2/config/
+    cp /data/opensearch-node1/config/internal_users_example.yml /data/opensearch-node2/config/internal_users.yml
+fi
+
+if [ ! -d "/data/opensearch-node1/data/" ]
+then
+    echo "creating opensearch node1 data directoy"
+    mkdir -p /data/opensearch-node1/data/
+fi
+
+if [ ! -d "/data/opensearch-node2/data/" ]
+then
+    echo "creating opensearch node2 data directoy"
+    mkdir -p /data/opensearch-node2/data/
+fi
--- a/data/setup/build/02_generate_certificates.sh
+++ b/data/setup/build/02_generate_certificates.sh
@@ -0,0 +1,109 @@
+#!/bin/bash
+
+if [ ! -f /data/certificates/certs/opensearch-ca.key ]
+then
+    echo "generating CA"
+    mkdir -p /data/certificates/certs/
+    openssl genrsa -out /data/certificates/certs/opensearch-ca.key 2048
+    openssl req -new -x509 -sha256 -days 3650 -subj "/C=US/ST=NY/L=IT/O=security/CN=opensearch-ca" -key /data/certificates/certs/opensearch-ca.key -out /data/certificates/certs/opensearch-ca.pem
+    openssl x509 -noout -subject -in /data/certificates/certs/opensearch-ca.pem
+fi
+
+if [ ! -f /data/certificates/certs/opensearch-admin.key ]
+then
+    echo "generating admin user key"
+    mkdir -p /data/certificates/certs/
+    openssl genrsa -out /data/certificates/certs/opensearch-admin_rsa.key 2048
+    openssl pkcs8 -v1 PBE-SHA1-3DES -nocrypt -in /data/certificates/certs/opensearch-admin_rsa.key -topk8 -out /data/certificates/certs/opensearch-admin.key
+    openssl req -new -inform PEM -outform PEM -subj "/C=US/ST=NY/L=IT/O=security/CN=admin" -key /data/certificates/certs/opensearch-admin.key -out /data/certificates/certs/opensearch-admin.csr
+    openssl x509 -req -days 3650 -in /data/certificates/certs/opensearch-admin.csr -CA /data/certificates/certs/opensearch-ca.pem -CAkey /data/certificates/certs/opensearch-ca.key -CAcreateserial -sha256 -out /data/certificates/certs/opensearch-admin.pem
+    #openssl verify -CAfile /data/certificates/certs/opensearch-ca.pem /data/certificates/certs/opensearch-admin.pem
+    #openssl x509 -noout -subject -in /data/certificates/certs/opensearch-admin.pem
+fi
+
+if [ ! -f /data/opensearch-node1/certs/opensearch-node1.key ]
+then
+    for NODE_NAME in "node1" "node2" 
+    do
+        echo "generating certificate opensearch-$NODE_NAME"
+        mkdir -p /data/opensearch-$NODE_NAME/certs/
+
+        cat << EOF > /tmp/request.conf
+        [req]
+        distinguished_name = req_distinguished_name
+        req_extensions = v3_req
+        prompt = no
+        [req_distinguished_name]
+        C = US
+        ST = NY
+        L = IT
+        O = security
+        CN = opensearch-$NODE_NAME
+        [v3_req]
+        keyUsage = keyEncipherment, dataEncipherment, digitalSignature, nonRepudiation
+        extendedKeyUsage = serverAuth, clientAuth
+        subjectAltName = @alt_names
+        [alt_names]
+        DNS.1 = docker-cluster
+        DNS.2 = opensearch-$NODE_NAME
+        RID.1 = 1.2.3.4.5.5
+EOF
+
+        openssl genrsa -out /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME-rsa.key 2048
+        openssl pkcs8 -inform PEM -outform PEM -in /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME-rsa.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.key
+        openssl req -new -config /tmp/request.conf -key /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.key -out /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.csr
+        openssl x509 -req -days 3650 -extfile /tmp/request.conf -extensions v3_req -in /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.csr -CA /data/certificates/certs/opensearch-ca.pem -CAkey /data/certificates/certs/opensearch-ca.key -CAcreateserial -sha256 -out /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.pem
+
+        cp /data/certificates/certs/opensearch-ca.pem /data/opensearch-$NODE_NAME/certs/
+        cp /data/certificates/certs/opensearch-admin.pem /data/opensearch-$NODE_NAME/certs/
+        cp /data/certificates/certs/opensearch-admin.key /data/opensearch-$NODE_NAME/certs/
+
+        #openssl verify -CAfile /data/opensearch-$NODE_NAME/certs/opensearch-ca.pem /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.pem
+        #openssl x509 -text -in /data/opensearch-$NODE_NAME/certs/opensearch-$NODE_NAME.pem
+    done
+fi
+
+if [ ! -f /data/traefik/certs/traefik.key ]
+then
+    echo "generating certificate traefik"
+    mkdir -p /data/traefik/certs/
+
+    cat << EOF > /tmp/request.conf
+    [req]
+    distinguished_name = req_distinguished_name
+    req_extensions = v3_req
+    prompt = no
+    [req_distinguished_name]
+    C = US
+    ST = NY
+    L = IT
+    O = security
+    CN = opensearch-lab
+    [v3_req]
+    keyUsage = keyEncipherment, dataEncipherment, digitalSignature, nonRepudiation
+    extendedKeyUsage = serverAuth, clientAuth
+    subjectAltName = @alt_names
+    [alt_names]
+    DNS.1 = traefik.local
+    DNS.2 = opensearch.local
+    DNS.3 = grafana.local
+EOF
+
+    ##openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /data/traefik/certs/server.key -out /data/traefik/certs/server.crt -subj "/C=US/ST=NY/L=IT/O=security/CN=logger" 
+    #openssl genrsa -out /data/traefik/certs/traefik_rsa.key 2048
+    #openssl pkcs8 -inform PEM -outform PEM -in /data/traefik/certs/traefik_rsa.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out /data/traefik/certs/traefik.key
+    #openssl req -new -subj "/C=US/ST=NY/L=IT/O=security/CN=traefik" -key /data/traefik/certs/traefik.key -out /data/traefik/certs/traefik.csr
+    #openssl x509 -req -days 3650 -in /data/traefik/certs/traefik.csr -CA /data/certificates/certs/opensearch-ca.pem -CAkey /data/certificates/certs/opensearch-ca.key -CAcreateserial -sha256 -out /data/traefik/certs/traefik.pem
+    #openssl verify -CAfile /data/certificates/certs/opensearch-ca.pem /data/traefik/certs/traefik.pem
+    #openssl x509 -noout -subject -in /data/traefik/certs/traefik.pem
+
+    openssl genrsa -out /data/traefik/certs/server_rsa.key 2048
+    openssl pkcs8 -inform PEM -outform PEM -in /data/traefik/certs/server_rsa.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out /data/traefik/certs/server.key
+    openssl req -new -config /tmp/request.conf -key /data/traefik/certs/server.key -out /data/traefik/certs/server.csr
+    openssl x509 -req -days 3650 -extfile /tmp/request.conf -extensions v3_req -in /data/traefik/certs/server.csr -CA /data/certificates/certs/opensearch-ca.pem -CAkey /data/certificates/certs/opensearch-ca.key -CAcreateserial -sha256 -out /data/traefik/certs/server.pem
+        
+    #openssl verify -CAfile /data/traefik/certs/server.pem /data/traefik/certs/server.pem
+    #openssl x509 -text -in /data/traefik/certs/server.pem
+fi
+
+sleep 2
--- a/data/setup/build/03_configure_opensearch.sh
+++ b/data/setup/build/03_configure_opensearch.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+## run security_admin in each node
+#for NODE_NAME in "node1" "node2"
+#do
+#
+#    COMMAND=(docker exec -it opensearch-$NODE_NAME /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh --clustername opensearch-cluster --configdir /usr/share/opensearch/config/opensearch-security -cacert /usr/share/opensearch/config/certs/opensearch-ca.pem -key /usr/share/opensearch/config/certs/opensearch-admin.key -cert /usr/share/opensearch/config/certs/opensearch-admin.pem -h opensearch-$NODE_NAME)
+#
+#    until "${COMMAND[@]}" ; do
+#        echo "opensearch not up yet. retrying in 10 seconds..."
+#        sleep 10
+#    done
+#done
+
+# use opensearch-dashboards api to create index pattern logstash-* for global tennant until it succeeds. (this will not create it for you personal tenant)
+cat > /tmp/opensearch_create_index_pattern.sh << EOF
+    curl -k \
+         -X POST "http://opensearch-dashboards:5601/api/saved_objects/index-pattern/logstash-*" \
+         -u 'admin:vagrant' \
+         -H "securitytenant:global" \
+         -H "osd-xsrf:true" \
+         -H "content-type:application/json" \
+         -d "{ \"attributes\": { \"title\": \"logstash-*\", \"timeFieldName\": \"@timestamp\" } }"     
+EOF
+
+cat > /tmp/opensearch_check_index_pattern.sh << EOF
+    curl -k \
+         -X GET "http://opensearch-dashboards:5601/api/saved_objects/index-pattern/logstash-*" \
+         -u 'admin:vagrant' \
+         -H "securitytenant:global" \
+         -H "osd-xsrf:true" \
+         -H "content-type:application/json" \
+    | grep "namespace"
+EOF
+chmod +x /tmp/opensearch_*.sh
+
+until "/tmp/opensearch_check_index_pattern.sh" ; do
+    echo "opensearch index-pattern does not exist; trying to create logstash-*" 
+    /tmp/opensearch_create_index_pattern.sh
+    sleep 10
+done
+echo "opensearch index-pattern created"
--- a/data/setup/build/04_configure_grafana.sh
+++ b/data/setup/build/04_configure_grafana.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+cat > /tmp/grafana_check.sh << EOF
+curl -k \
+     -X GET "http://grafana:3000/api/datasources" \
+     -u 'admin:vagrant' \
+     -H "content-type:application/json" \
+| grep '"name":"OpenSearch"'
+EOF
+
+cat > /tmp/grafana_initial_setup.sh << EOF
+curl -k \
+     -X POST "http://grafana:3000/api/datasources" \
+     -u 'admin:vagrant' \
+     -H "content-type:application/json" \
+     -d '
+        {
+          "orgId": 1,
+          "name": "OpenSearch",
+          "type": "grafana-opensearch-datasource",
+          "typeName": "OpenSearch",
+          "typeLogoUrl": "public/plugins/grafana-opensearch-datasource/img/logo.svg",
+          "access": "proxy",
+          "url": "https://opensearch-node1:9200",
+          "basicAuth": true,
+          "basicAuthUser": "admin",
+          "isDefault": true,
+          "secureJsonData": {
+            "basicAuthPassword": "vagrant"
+          },
+          "jsonData": {
+            "database": "logstash-*",
+            "esVersion": "8.0.0",
+            "flavor": "opensearch",
+            "logLevelField": "fields.level",
+            "logMessageField": "message",
+            "maxConcurrentShardRequests": 5,
+            "pplEnabled": true,
+            "timeField": "@timestamp",
+            "tlsAuthWithCACert": false,
+            "tlsSkipVerify": true,
+            "version": "1.0.0"
+          },
+          "readOnly": false
+        }
+     '
+EOF
+
+chmod +x /tmp/grafana*.sh
+
+until "/tmp/grafana_check.sh" ; do
+    echo "Grafana settings not applied; retrying" 
+    /tmp/grafana_initial_setup.sh
+    sleep 10
+done
+echo "Grafana settings applied"
--- a/data/setup/build/Dockerfile
+++ b/data/setup/build/Dockerfile
@@ -0,0 +1,9 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && apt-get -y upgrade 
+RUN apt-get -y install openssl docker.io curl dos2unix
+
+COPY *.sh /opt/
+RUN chmod +x /opt/*.sh ; dos2unix /opt/*.sh
+
+CMD ["bash", "/opt/entrypoint.sh"]
--- a/data/setup/build/entrypoint.sh
+++ b/data/setup/build/entrypoint.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+/opt/01_precreate_folders.sh
+/opt/02_generate_certificates.sh
+echo "initial setup done. tag setup container as healthy to start other containers" 
+touch /tmp/healthcheck.txt
+
+/opt/03_configure_opensearch.sh
+/opt/04_configure_grafana.sh
+
+sleep infinity