#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved. import argparse import errno import getpass import glob import json import os import platform import re import shutil import stat import sys from malcolm_common import * from collections import defaultdict, namedtuple from subprocess import (PIPE, STDOUT, Popen, check_call, CalledProcessError) try: from contextlib import nullcontext except ImportError: class nullcontext(object): def __init__(self, enter_result=None): self.enter_result = enter_result def __enter__(self): return self.enter_result def __exit__(self, *args): pass ################################################################################################### ScriptName = os.path.basename(__file__) pyPlatform = platform.system() args = None dockerBin = None dockerComposeBin = None opensslBin = None ################################################################################################### try: from colorama import init as ColoramaInit, Fore, Back, Style ColoramaInit() coloramaImported = True except: coloramaImported = False ################################################################################################### # perform a service-keystore operation in a Docker container # # service - the service in the docker-compose YML file # keystore_args - arguments to pass to the service-keystore binary in the container # run_process_kwargs - keyword arguments to pass to run_process # # returns True (success) or False (failure) # def keystore_op(service, dropPriv=False, *keystore_args, **run_process_kwargs): global args global dockerBin global dockerComposeBin err = -1 results = [] # the elastic containers all follow the same naming pattern for these executables keystoreBinProc = f"/usr/share/{service}/bin/{service}-keystore" # if we're using docker-uid-gid-setup.sh to drop privileges as we spin up a container dockerUidGuidSetup = "/usr/local/bin/docker-uid-gid-setup.sh" # docker-compose use local temporary path osEnv = os.environ.copy() osEnv['TMPDIR'] = MalcolmTmpPath # open up the docker-compose file and "grep" for the line where the keystore file # is bind-mounted into the service container (once and only once). the bind # mount needs to exist in the YML file and the local directory containing the # keystore file needs to exist (although the file itself might not yet). # also get PUID and PGID variables from the docker-compose file. localKeystore = None localKeystoreDir = None localKeystorePreExists = False volumeKeystore = None volumeKeystoreDir = None uidGidDict = defaultdict(str) try: composeFileLines = list() uidGidDict['PUID'] = f'{os.getuid()}' if (pyPlatform != PLATFORM_WINDOWS) else '1000' uidGidDict['PGID'] = f'{os.getgid()}' if (pyPlatform != PLATFORM_WINDOWS) else '1000' with open(args.composeFile, 'r') as f: allLines = f.readlines() composeFileLines = [x for x in allLines if re.search(fr'-.*?{service}.keystore\s*:.*{service}.keystore', x)] uidGidDict.update(dict(x.split(':') for x in [''.join(x.split()) for x in allLines if re.search(fr'^\s*P[UG]ID\s*:\s*\d+\s*$', x)])) if (len(composeFileLines) == 1) and (len(composeFileLines[0]) > 0): matches = re.search(fr'-\s*(?P.*?{service}.keystore)\s*:\s*(?P.*?{service}.keystore)', composeFileLines[0]) if matches: localKeystore = os.path.realpath(matches.group('localKeystore')) localKeystoreDir = os.path.dirname(localKeystore) volumeKeystore = matches.group('volumeKeystore') volumeKeystoreDir = os.path.dirname(volumeKeystore) if (localKeystore is not None) and (volumeKeystore is not None) and os.path.isdir(localKeystoreDir): localKeystorePreExists = os.path.isfile(localKeystore) dockerCmd = None # determine if Malcolm is running; if so, we'll use docker-compose exec, other wise we'll use docker run err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'ps', '-q', service], env=osEnv, debug=args.debug) out[:] = [x for x in out if x] if (err == 0) and (len(out) > 0): # Malcolm is running, we can use an existing container # assemble the service-keystore command dockerCmd = [dockerComposeBin, 'exec', # if using stdin, indicate the container is "interactive", else noop (duplicate --rm) '-T' if ('stdin' in run_process_kwargs and run_process_kwargs['stdin']) else '', # execute as UID:GID in docker-compose.yml file '-u', f'{uidGidDict["PUID"]}:{uidGidDict["PGID"]}' # the work directory in the container is the directory to contain the keystore file '-w', volumeKeystoreDir, # the service name service, # the executable filespec keystoreBinProc] else: # Malcolm isn't running, do 'docker run' to spin up a temporary container to run the ocmmand # "grep" the docker image out of the service's image: value from the docker-compose YML file serviceImage = None composeFileLines = list() with open(args.composeFile, 'r') as f: composeFileLines = [x for x in f.readlines() if f'image: malcolmnetsec/{service}' in x] if (len(composeFileLines) > 0) and (len(composeFileLines[0]) > 0): imageLineValues = composeFileLines[0].split() if (len(imageLineValues) > 1): serviceImage = imageLineValues[1] if serviceImage is not None: # assemble the service-keystore command dockerCmd = [dockerBin, 'run', # remove the container when complete '--rm', # if using stdin, indicate the container is "interactive", else noop '-i' if ('stdin' in run_process_kwargs and run_process_kwargs['stdin']) else '', # if dropPriv, dockerUidGuidSetup will take care of dropping privileges for the correct UID/GID # if NOT dropPriv, enter with the keystore executable directly '--entrypoint', dockerUidGuidSetup if dropPriv else keystoreBinProc, '--env', f'DEFAULT_UID={uidGidDict["PUID"]}', '--env', f'DEFAULT_GID={uidGidDict["PGID"]}', '--env', f'PUSER_CHOWN={volumeKeystoreDir}', # rw bind mount the local directory to contain the keystore file to the container directory '-v', f'{localKeystoreDir}:{volumeKeystoreDir}:rw', # the work directory in the container is the directory to contain the keystore file '-w', volumeKeystoreDir, # if dropPriv, execute as root, as docker-uid-gid-setup.sh will drop privileges for us # if NOT dropPriv, execute as UID:GID in docker-compose.yml file '-u', 'root' if dropPriv else f'{uidGidDict["PUID"]}:{uidGidDict["PGID"]}', # the service image name grepped from the YML file serviceImage] if dropPriv: # the keystore executable filespec (as we used dockerUidGuidSetup as the entrypoint) dockerCmd.append(keystoreBinProc) else: raise Exception(f'Unable to identify docker image for {service} in {args.composeFile}') if (dockerCmd is not None): # append whatever other arguments to pass to the executable filespec if keystore_args: dockerCmd.extend(list(keystore_args)) dockerCmd[:] = [x for x in dockerCmd if x] # execute the command, passing through run_process_kwargs to run_process as expanded keyword arguments err, results = run_process(dockerCmd, env=osEnv, debug=args.debug, **run_process_kwargs) if (err != 0) or (not os.path.isfile(localKeystore)): raise Exception(f'Error processing command {service} keystore: {results}') else: raise Exception(f'Unable formulate keystore command for {service} in {args.composeFile}') else: raise Exception(f'Unable to identify a unique keystore file bind mount for {service} in {args.composeFile}') except Exception as e: if (err == 0): err = -1 # don't be so whiny if the "create" failed just because it already existed or a 'remove' failed on a nonexistant item if ((not args.debug) and list(keystore_args) and (len(list(keystore_args)) > 0) and (list(keystore_args)[0].lower() in ('create', 'remove')) and localKeystorePreExists): pass else: eprint(e) # success = (error == 0) return (err == 0), results ################################################################################################### def status(): global args global dockerComposeBin # docker-compose use local temporary path osEnv = os.environ.copy() osEnv['TMPDIR'] = MalcolmTmpPath err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'ps'], env=osEnv, debug=args.debug) if (err == 0): print("\n".join(out)) else: eprint("Failed to display Malcolm status\n") eprint("\n".join(out)) exit(err) ################################################################################################### def logs(): global args global dockerBin global dockerComposeBin # noisy logs (a lot of it is NGINX logs from health checks) ignoreRegEx = re.compile(r""" .+( deprecated | DEPRECATION | eshealth | remov(ed|ing)\s+(old\s+file|dead\s+symlink|empty\s+directory) | update_mapping | throttling\s+index | executing\s+attempt_(transition|set_replica_count)\s+for | but\s+there\s+are\s+no\s+living\s+connections | saved_objects | /kibana/(api/ui_metric/report|internal/search/es) | retry\.go.+(send\s+unwait|done$) | scheduling\s+job\s*id.+opendistro-ism | descheduling\s+job\s*id | updating\s+number_of_replicas | running\s+full\s+sweep | (async|output)\.go.+(reset\s+by\s+peer|Connecting\s+to\s+backoff|backoff.+established$) | \b(d|es)?stats\.json | /_ns_/nstest\.html | esindices/list | _cat/indices | use_field_mapping | reaped\s+unknown\s+pid | Successfully\s+handled\s+GET\s+request\s+for\s+'/' | GET\s+/(_cat/health|api/status|sessions2-).+HTTP/[\d\.].+\b200\b | POST\s+/(d?stats/(d?stat|_doc|_search)|_bulk|fields/(field/)?_search).+HTTP/[\d\.].+\b20[01]\b | POST\s+HTTP/[\d\.].+\b200\b | POST\s+/server/php/\s+HTTP/\d+\.\d+"\s+\d+\s+\d+.*:8443/ | curl.+localhost.+GET\s+/api/status\s+200 | "GET\s+/\s+HTTP/1\.\d+"\s+200\s+- | \b1.+GET\s+/\s+.+401.+curl ) """, re.VERBOSE | re.IGNORECASE) serviceRegEx = re.compile(r'^(?P.+?\|)\s*(?P.*)$') # increase COMPOSE_HTTP_TIMEOUT to be ridiculously large so docker-compose never times out the TTY doing debug output osEnv = os.environ.copy() osEnv['COMPOSE_HTTP_TIMEOUT'] = '100000000' # docker-compose use local temporary path osEnv['TMPDIR'] = MalcolmTmpPath err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'ps'], env=osEnv, debug=args.debug) print("\n".join(out)) process = Popen([dockerComposeBin, '-f', args.composeFile, 'logs', '-f'], env=osEnv, stdout=PIPE) while True: output = process.stdout.readline() if (len(output) == 0) and (process.poll() is not None): break if output: outputStr = output.decode().strip() outputStrEscaped = EscapeAnsi(outputStr) if ignoreRegEx.match(outputStrEscaped): pass ### print(f'!!!!!!!: {outputStr}') else: serviceMatch = serviceRegEx.search(outputStrEscaped) serviceMatchFmt = serviceRegEx.search(outputStr) if coloramaImported else serviceMatch serviceStr = serviceMatchFmt.group('service') if (serviceMatchFmt is not None) else '' messageStr = serviceMatch.group('message') if (serviceMatch is not None) else '' outputJson = LoadStrIfJson(messageStr) if (outputJson is not None): # if there's a timestamp in the JSON, move it outside of the JSON to the beginning of the log string timeKey = None if 'time' in outputJson: timeKey = 'time' elif 'timestamp' in outputJson: timeKey = 'timestamp' elif '@timestamp' in outputJson: timeKey = '@timestamp' timeStr = '' if timeKey is not None: timeStr = f"{outputJson[timeKey]} " outputJson.pop(timeKey, None) if ('job.schedule' in outputJson) and ('job.position' in outputJson) and ('job.command' in outputJson): # this is a status output line from supercronic, let's format and clean it up so it fits in better with the rest of the logs # remove some clutter for the display for noisyKey in ['level', 'channel', 'iteration', 'job.position', 'job.schedule']: outputJson.pop(noisyKey, None) # if it's just command and message, format those NOT as JSON jobCmd = outputJson['job.command'] jobStatus = outputJson['msg'] if (len(outputJson.keys()) == 2) and ('job.command' in outputJson) and ('msg' in outputJson): # if it's the most common status (starting or job succeeded) then don't print unless debug mode if args.debug or ((jobStatus != 'starting') and (jobStatus != 'job succeeded')): print(f"{serviceStr}{Style.RESET_ALL if coloramaImported else ''} {timeStr} {jobCmd}: {jobStatus}") else: pass else: # standardize and print the JSON output print(f"{serviceStr}{Style.RESET_ALL if coloramaImported else ''} {timeStr}{json.dumps(outputJson)}") elif ('kibana' in serviceStr): # this is an output line from kibana, let's clean it up a bit: remove some clutter for the display for noisyKey in ['type', 'tags', 'pid', 'method', 'prevState', 'prevMsg']: outputJson.pop(noisyKey, None) # standardize and print the JSON output print(f"{serviceStr}{Style.RESET_ALL if coloramaImported else ''} {timeStr}{json.dumps(outputJson)}") else: # standardize and print the JSON output print(f"{serviceStr}{Style.RESET_ALL if coloramaImported else ''} {timeStr}{json.dumps(outputJson)}") else: # just a regular non-JSON string, print as-is print(outputStr if coloramaImported else outputStrEscaped) else: time.sleep(0.5) process.poll() ################################################################################################### def stop(wipe=False): global args global dockerBin global dockerComposeBin # docker-compose use local temporary path osEnv = os.environ.copy() osEnv['TMPDIR'] = MalcolmTmpPath if wipe: # attempt to DELETE _template/zeek_template in Elasticsearch err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'exec', 'arkime', 'bash', '-c', 'curl -fs --output /dev/null -H"Content-Type: application/json" -XDELETE "http://$ES_HOST:$ES_PORT/_template/zeek_template"'], env=osEnv, debug=args.debug) # if stop.sh is being called with wipe.sh (after the docker-compose file) # then also remove named and anonymous volumes (not external volumes, of course) err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'down', '--volumes'][:5 if wipe else -1], env=osEnv, debug=args.debug) if (err == 0): eprint("Stopped Malcolm\n") else: eprint("Malcolm failed to stop\n") eprint("\n".join(out)) exit(err) if wipe: # delete elasticsearch database shutil.rmtree(os.path.join(MalcolmPath, 'elasticsearch/nodes'), ignore_errors=True) # delete data files (backups, zeek logs, arkime logs, PCAP files, captured PCAP files) for dataDir in ['elasticsearch-backup', 'zeek-logs', 'moloch-logs', 'pcap', 'moloch-raw']: for root, dirnames, filenames in os.walk(os.path.join(MalcolmPath, dataDir), topdown=True, onerror=None): for file in filenames: fileSpec = os.path.join(root, file) if (os.path.isfile(fileSpec) or os.path.islink(fileSpec)) and (not file.startswith('.git')): try: os.remove(fileSpec) except: pass # clean up empty directories for dataDir in [os.path.join('elasticsearch-backup', 'logs'), os.path.join('zeek-logs', 'processed'), os.path.join('zeek-logs', 'current')]: RemoveEmptyFolders(dataDir, removeRoot=False) eprint("Malcolm has been stopped and its data cleared\n") ################################################################################################### def start(): global args global dockerBin global dockerComposeBin # make sure the auth files exist. if we are in an interactive shell and we're # missing any of the auth files, prompt to create them now if sys.__stdin__.isatty() and (not MalcolmAuthFilesExist()): authSetup() # still missing? sorry charlie if (not MalcolmAuthFilesExist()): raise Exception('Malcolm administrator account authentication files are missing, please run ./scripts/auth_setup to generate them') # touch the metadata file open(os.path.join(MalcolmPath, os.path.join('htadmin', 'metadata')), 'a').close() # if the elasticsearch and logstash keystore don't exist exist, create empty ones if not os.path.isfile(os.path.join(MalcolmPath, os.path.join('elasticsearch', 'elasticsearch.keystore'))): keystore_op('elasticsearch', True, 'create') if not os.path.isfile(os.path.join(MalcolmPath, os.path.join('logstash', os.path.join('certs', 'logstash.keystore')))): keystore_op('logstash', True, 'create') # make sure permissions are set correctly for the nginx worker processes for authFile in [os.path.join(MalcolmPath, os.path.join('nginx', 'htpasswd')), os.path.join(MalcolmPath, os.path.join('htadmin', 'config.ini')), os.path.join(MalcolmPath, os.path.join('htadmin', 'metadata'))]: # chmod 644 authFile os.chmod(authFile, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) for authFile in [os.path.join(MalcolmPath, os.path.join('nginx', 'nginx_ldap.conf'))]: # chmod 600 authFile os.chmod(authFile, stat.S_IRUSR | stat.S_IWUSR) # make sure some directories exist before we start for path in [os.path.join(MalcolmPath, 'elasticsearch'), os.path.join(MalcolmPath, 'elasticsearch-backup'), os.path.join(MalcolmPath, os.path.join('nginx', 'ca-trust')), os.path.join(MalcolmPath, os.path.join('pcap', 'upload')), os.path.join(MalcolmPath, os.path.join('pcap', 'processed')), os.path.join(MalcolmPath, os.path.join('zeek-logs', 'current')), os.path.join(MalcolmPath, os.path.join('zeek-logs', 'upload')), os.path.join(MalcolmPath, os.path.join('zeek-logs', 'processed')), os.path.join(MalcolmPath, os.path.join('zeek-logs', 'extract_files'))]: try: os.makedirs(path) except OSError as exc: if (exc.errno == errno.EEXIST) and os.path.isdir(path): pass else: raise # increase COMPOSE_HTTP_TIMEOUT to be ridiculously large so docker-compose never times out the TTY doing debug output osEnv = os.environ.copy() osEnv['COMPOSE_HTTP_TIMEOUT'] = '100000000' # docker-compose use local temporary path osEnv['TMPDIR'] = MalcolmTmpPath # start docker err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'up', '--detach'], env=osEnv, debug=args.debug) if (err == 0): eprint("Started Malcolm\n\n") eprint("In a few minutes, Malcolm services will be accessible via the following URLs:") eprint("------------------------------------------------------------------------------") eprint(" - Arkime: https://localhost/") eprint(" - Kibana: https://localhost/kibana/") eprint(" - PCAP upload (web): https://localhost/upload/") eprint(" - PCAP upload (sftp): sftp://username@127.0.0.1:8022/files/") eprint(" - Host and subnet name mapping editor: https://localhost/name-map-ui/\n") eprint(" - Account management: https://localhost:488/\n") else: eprint("Malcolm failed to start\n") eprint("\n".join(out)) exit(err) ################################################################################################### def authSetup(wipe=False): global args global dockerBin global dockerComposeBin global opensslBin if YesOrNo('Store administrator username/password for local Malcolm access?', default=True): # prompt username and password usernamePrevious = None password = None passwordConfirm = None passwordEncrypted = '' username = AskForString("Administrator username") while True: password = AskForPassword(f"{username} password: ") passwordConfirm = AskForPassword(f"{username} password (again): ") if (password == passwordConfirm): break eprint("Passwords do not match") # get previous admin username to remove from htpasswd file if it's changed authEnvFile = os.path.join(MalcolmPath, 'auth.env') if os.path.isfile(authEnvFile): prevAuthInfo = defaultdict(str) with open(authEnvFile, 'r') as f: for line in f: try: k, v = line.rstrip().split("=") prevAuthInfo[k] = v.strip('"') except: pass if (len(prevAuthInfo['MALCOLM_USERNAME']) > 0): usernamePrevious = prevAuthInfo['MALCOLM_USERNAME'] # get openssl hash of password err, out = run_process([opensslBin, 'passwd', '-1', '-stdin'], stdin=password, stderr=False, debug=args.debug) if (err == 0) and (len(out) > 0) and (len(out[0]) > 0): passwordEncrypted = out[0] else: raise Exception('Unable to generate password hash with openssl') # write auth.env (used by htadmin and file-upload containers) with open(authEnvFile, 'w') as f: f.write("# Malcolm Administrator username and encrypted password for nginx reverse proxy (and upload server's SFTP access)\n") f.write(f'MALCOLM_USERNAME={username}\n') f.write(f'MALCOLM_PASSWORD={passwordEncrypted}\n') os.chmod(authEnvFile, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) # create or update the htpasswd file htpasswdFile = os.path.join(MalcolmPath, os.path.join('nginx', 'htpasswd')) htpasswdCmd = ['htpasswd', '-i', '-B', htpasswdFile, username] if not os.path.isfile(htpasswdFile): htpasswdCmd.insert(1, '-c') err, out = run_process(htpasswdCmd, stdin=password, stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate htpasswd file: {out}') # if the admininstrator username has changed, remove the previous administrator username from htpasswd if (usernamePrevious is not None) and (usernamePrevious != username): htpasswdLines = list() with open(htpasswdFile, 'r') as f: htpasswdLines = f.readlines() with open(htpasswdFile, 'w') as f: for line in htpasswdLines: if not line.startswith(f"{usernamePrevious}:"): f.write(line) # configure default LDAP stuff (they'll have to edit it by hand later) ldapConfFile = os.path.join(MalcolmPath, os.path.join('nginx', 'nginx_ldap.conf')) if not os.path.isfile(ldapConfFile): ldapDefaults = defaultdict(str) if os.path.isfile(os.path.join(MalcolmPath, '.ldap_config_defaults')): ldapDefaults = defaultdict(str) with open(os.path.join(MalcolmPath, '.ldap_config_defaults'), 'r') as f: for line in f: try: k, v = line.rstrip().split("=") ldapDefaults[k] = v.strip('"').strip("'") except: pass ldapProto = ldapDefaults.get("LDAP_PROTO", "ldap://") ldapHost = ldapDefaults.get("LDAP_HOST", "ds.example.com") ldapPort = ldapDefaults.get("LDAP_PORT", "3268") ldapType = ldapDefaults.get("LDAP_SERVER_TYPE", "winldap") if (ldapType == "openldap"): ldapUri = 'DC=example,DC=com?uid?sub?(objectClass=posixAccount)' ldapGroupAttr = "memberUid" ldapGroupAttrIsDN = "off" else: ldapUri = 'DC=example,DC=com?sAMAccountName?sub?(objectClass=person)' ldapGroupAttr = "member" ldapGroupAttrIsDN = "on" with open(ldapConfFile, 'w') as f: f.write('# This is a sample configuration for the ldap_server section of nginx.conf.\n') f.write('# Yours will vary depending on how your Active Directory/LDAP server is configured.\n') f.write('# See https://github.com/kvspb/nginx-auth-ldap#available-config-parameters for options.\n\n') f.write('ldap_server ad_server {\n') f.write(f' url "{ldapProto}{ldapHost}:{ldapPort}/{ldapUri}";\n\n') f.write(' binddn "bind_dn";\n') f.write(' binddn_passwd "bind_dn_password";\n\n') f.write(f' group_attribute {ldapGroupAttr};\n') f.write(f' group_attribute_is_dn {ldapGroupAttrIsDN};\n') f.write(' require group "CN=malcolm,OU=groups,DC=example,DC=com";\n') f.write(' require valid_user;\n') f.write(' satisfy all;\n') f.write('}\n\n') f.write('auth_ldap_cache_enabled on;\n') f.write('auth_ldap_cache_expiration_time 10000;\n') f.write('auth_ldap_cache_size 1000;\n') os.chmod(ldapConfFile, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) # populate htadmin config file with open(os.path.join(MalcolmPath, os.path.join('htadmin', 'config.ini')), 'w') as f: f.write('; HTAdmin config file.\n\n') f.write('[application]\n') f.write('; Change this to customize your title:\n') f.write('app_title = Malcolm User Management\n\n') f.write('; htpasswd file\n') f.write('secure_path = ./config/htpasswd\n') f.write('; metadata file\n') f.write('metadata_path = ./config/metadata\n\n') f.write('; administrator user/password (htpasswd -b -c -B ...)\n') f.write(f'admin_user = {username}\n\n') f.write('; username field quality checks\n') f.write(';\n') f.write('min_username_len = 4\n') f.write('max_username_len = 12\n\n') f.write('; Password field quality checks\n') f.write(';\n') f.write('min_password_len = 6\n') f.write('max_password_len = 20\n\n') # touch the metadata file open(os.path.join(MalcolmPath, os.path.join('htadmin', 'metadata')), 'a').close() # generate HTTPS self-signed certificates if YesOrNo('(Re)generate self-signed certificates for HTTPS access', default=True): with pushd(os.path.join(MalcolmPath, os.path.join('nginx', 'certs'))): # remove previous files for oldfile in glob.glob("*.pem"): os.remove(oldfile) # generate dhparam ------------------------------- err, out = run_process([opensslBin, 'dhparam', '-out', 'dhparam.pem', '2048'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate dhparam.pem file: {out}') # generate key/cert ------------------------------- err, out = run_process([opensslBin, 'req', '-subj', '/CN=localhost', '-x509', '-newkey', 'rsa:4096', '-nodes', '-keyout', 'key.pem', '-out', 'cert.pem', '-days', '3650'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate key.pem/cert.pem file(s): {out}') # generate beats/logstash self-signed certificates logstashPath = os.path.join(MalcolmPath, os.path.join('logstash', 'certs')) filebeatPath = os.path.join(MalcolmPath, os.path.join('filebeat', 'certs')) if YesOrNo('(Re)generate self-signed certificates for a remote log forwarder', default=True): with pushd(logstashPath): # make clean to clean previous files for pat in ['*.srl', '*.csr', '*.key', '*.crt', '*.pem']: for oldfile in glob.glob(pat): os.remove(oldfile) # ----------------------------------------------- # generate new ca/server/client certificates/keys # ca ------------------------------- err, out = run_process([opensslBin, 'genrsa', '-out', 'ca.key', '2048'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate ca.key: {out}') err, out = run_process([opensslBin, 'req', '-x509', '-new', '-nodes', '-key', 'ca.key', '-sha256', '-days', '9999', '-subj', '/C=US/ST=ID/O=sensor/OU=ca', '-out', 'ca.crt'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate ca.crt: {out}') # server ------------------------------- err, out = run_process([opensslBin, 'genrsa', '-out', 'server.key', '2048'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate server.key: {out}') err, out = run_process([opensslBin, 'req', '-sha512', '-new', '-key', 'server.key', '-out', 'server.csr', '-config', 'server.conf'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate server.csr: {out}') err, out = run_process([opensslBin, 'x509', '-days', '3650', '-req', '-sha512', '-in', 'server.csr', '-CAcreateserial', '-CA', 'ca.crt', '-CAkey', 'ca.key', '-out', 'server.crt', '-extensions', 'v3_req', '-extfile', 'server.conf'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate server.crt: {out}') shutil.move("server.key", "server.key.pem") err, out = run_process([opensslBin, 'pkcs8', '-in', 'server.key.pem', '-topk8', '-nocrypt', '-out', 'server.key'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate server.key: {out}') # client ------------------------------- err, out = run_process([opensslBin, 'genrsa', '-out', 'client.key', '2048'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate client.key: {out}') err, out = run_process([opensslBin, 'req', '-sha512', '-new', '-key', 'client.key', '-out', 'client.csr', '-config', 'client.conf'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate client.csr: {out}') err, out = run_process([opensslBin, 'x509', '-days', '3650', '-req', '-sha512', '-in', 'client.csr', '-CAcreateserial', '-CA', 'ca.crt', '-CAkey', 'ca.key', '-out', 'client.crt', '-extensions', 'v3_req', '-extensions', 'usr_cert', '-extfile', 'client.conf'], stderr=True, debug=args.debug) if (err != 0): raise Exception(f'Unable to generate client.crt: {out}') # ----------------------------------------------- # mkdir filebeat/certs if it doesn't exist try: os.makedirs(filebeatPath) except OSError as exc: if (exc.errno == errno.EEXIST) and os.path.isdir(filebeatPath): pass else: raise # remove previous files in filebeat/certs for oldfile in glob.glob(os.path.join(filebeatPath, "*")): os.remove(oldfile) # copy the ca so logstasn and filebeat both have it shutil.copy2(os.path.join(logstashPath, "ca.crt"), filebeatPath) # move the client certs for filebeat for f in ['client.key', 'client.crt']: shutil.move(os.path.join(logstashPath, f), filebeatPath) # remove leftovers for pat in ['*.srl', '*.csr', '*.pem']: for oldfile in glob.glob(pat): os.remove(oldfile) # create and populate keystore for remote if YesOrNo('Store username/password for forwarding Logstash events to a secondary, external Elasticsearch instance', default=False): # prompt username and password esPassword = None esPasswordConfirm = None esUsername = AskForString("External Elasticsearch username") while True: esPassword = AskForPassword(f"{esUsername} password: ") esPasswordConfirm = AskForPassword(f"{esUsername} password (again): ") if (esPassword == esPasswordConfirm): break eprint("Passwords do not match") # create logstash keystore file, don't complain if it already exists, and set the keystore items keystore_op('logstash', False, 'create', stdin='N') keystore_op('logstash', False, 'remove', 'ES_EXTERNAL_USER', '--force') keystore_op('logstash', False, 'add', 'ES_EXTERNAL_USER', '--stdin', '--force', stdin=esUsername) keystore_op('logstash', False, 'remove', 'ES_EXTERNAL_PASSWORD', '--force') keystore_op('logstash', False, 'add', 'ES_EXTERNAL_PASSWORD', '--stdin', '--force', stdin=esPassword) success, results = keystore_op('logstash', False, 'list') results = [x.upper() for x in results if x and (not x.upper().startswith('WARNING')) and (not x.upper().startswith('KEYSTORE')) and (not x.upper().startswith('USING BUNDLED JDK'))] if success and ('ES_EXTERNAL_USER' in results) and ('ES_EXTERNAL_PASSWORD' in results): eprint(f"External Elasticsearch instance variables stored: {', '.join(results)}") else: eprint("Failed to store external Elasticsearch instance variables:\n") eprint("\n".join(results)) # Open Distro for Elasticsearch authenticate sender account credentials # https://opendistro.github.io/for-elasticsearch-docs/docs/alerting/monitors/#authenticate-sender-account if YesOrNo('Store username/password for email alert sender account (see https://opendistro.github.io/for-elasticsearch-docs/docs/alerting/monitors/#authenticate-sender-account)', default=False): # prompt username and password emailPassword = None emailPasswordConfirm = None emailSender = AskForString("Open Distro alerting email sender name") emailUsername = AskForString("Email account username") while True: emailPassword = AskForPassword(f"{emailUsername} password: ") emailPasswordConfirm = AskForPassword(f"{emailUsername} password (again): ") if (emailPassword == emailPasswordConfirm): break eprint("Passwords do not match") # create elasticsearch keystore file, don't complain if it already exists, and set the keystore items usernameKey = f'opendistro.alerting.destination.email.{emailSender}.username' passwordKey = f'opendistro.alerting.destination.email.{emailSender}.password' keystore_op('elasticsearch', True, 'create', stdin='N') keystore_op('elasticsearch', True, 'remove', usernameKey) keystore_op('elasticsearch', True, 'add', usernameKey, '--stdin', stdin=emailUsername) keystore_op('elasticsearch', True, 'remove', passwordKey) keystore_op('elasticsearch', True, 'add', passwordKey, '--stdin', stdin=emailPassword) success, results = keystore_op('elasticsearch', True, 'list') results = [x for x in results if x and (not x.upper().startswith('WARNING')) and (not x.upper().startswith('KEYSTORE'))] if success and (usernameKey in results) and (passwordKey in results): eprint(f"Email alert sender account variables stored: {', '.join(results)}") else: eprint("Failed to store email alert sender account variables:\n") eprint("\n".join(results)) ################################################################################################### # main def main(): global args global dockerBin global dockerComposeBin global opensslBin # extract arguments from the command line # print (sys.argv[1:]); parser = argparse.ArgumentParser(description='Malcolm control script', add_help=False, usage=f'{ScriptName} ') parser.add_argument('-v', '--verbose', dest='debug', type=str2bool, nargs='?', const=True, default=False, help="Verbose output") parser.add_argument('-f', '--file', required=False, dest='composeFile', metavar='', type=str, default='docker-compose.yml', help='docker-compose YML file') parser.add_argument('-l', '--logs', dest='cmdLogs', type=str2bool, nargs='?', const=True, default=False, help="Tail Malcolm logs") parser.add_argument('--start', dest='cmdStart', type=str2bool, nargs='?', const=True, default=False, help="Start Malcolm") parser.add_argument('--restart', dest='cmdRestart', type=str2bool, nargs='?', const=True, default=False, help="Stop and restart Malcolm") parser.add_argument('--stop', dest='cmdStop', type=str2bool, nargs='?', const=True, default=False, help="Stop Malcolm") parser.add_argument('--wipe', dest='cmdWipe', type=str2bool, nargs='?', const=True, default=False, help="Stop Malcolm and delete all data") parser.add_argument('--auth', dest='cmdAuthSetup', type=str2bool, nargs='?', const=True, default=False, help="Configure Malcolm authentication") parser.add_argument('--status', dest='cmdStatus', type=str2bool, nargs='?', const=True, default=False, help="Display status of Malcolm components") try: parser.error = parser.exit args = parser.parse_args() except SystemExit: parser.print_help() exit(2) if args.debug: eprint(os.path.join(ScriptPath, ScriptName)) eprint(f"Arguments: {sys.argv[1:]}") eprint(f"Arguments: {args}") eprint("Malcolm path:", MalcolmPath) else: sys.tracebacklimit = 0 with pushd(MalcolmPath): # don't run this as root if (pyPlatform != PLATFORM_WINDOWS) and ((os.getuid() == 0) or (os.geteuid() == 0) or (getpass.getuser() == 'root')): raise Exception(f'{ScriptName} should not be run as root') # create local temporary directory for docker-compose because we may have noexec on /tmp try: os.makedirs(MalcolmTmpPath) except OSError as exc: if (exc.errno == errno.EEXIST) and os.path.isdir(MalcolmTmpPath): pass else: raise # docker-compose use local temporary path osEnv = os.environ.copy() osEnv['TMPDIR'] = MalcolmTmpPath # make sure docker/docker-compose is available dockerBin = 'docker.exe' if ((pyPlatform == PLATFORM_WINDOWS) and Which('docker.exe')) else 'docker' dockerComposeBin = 'docker-compose.exe' if ((pyPlatform == PLATFORM_WINDOWS) and Which('docker-compose.exe')) else 'docker-compose' err, out = run_process([dockerBin, 'info'], debug=args.debug) if (err != 0): raise Exception(f'{ScriptName} requires docker, please run install.py') err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'version'], env=osEnv, debug=args.debug) if (err != 0): raise Exception(f'{ScriptName} requires docker-compose, please run install.py') # identify openssl binary opensslBin = 'openssl.exe' if ((pyPlatform == PLATFORM_WINDOWS) and Which('openssl.exe')) else 'openssl' # if executed via a symlink, figure out what was intended via the symlink name if os.path.islink(os.path.join(ScriptPath, ScriptName)): if (ScriptName == "logs"): args.cmdLogs = True elif (ScriptName == "status"): args.cmdStatus = True elif (ScriptName == "start"): args.cmdStart = True elif (ScriptName == "restart"): args.cmdRestart = True elif (ScriptName == "stop"): args.cmdStop = True elif (ScriptName == "wipe"): args.cmdWipe = True elif (ScriptName.startswith("auth")): args.cmdAuthSetup = True # stop Malcolm (and wipe data if requestsed) if args.cmdRestart or args.cmdStop or args.cmdWipe: stop(wipe=args.cmdWipe) # configure Malcolm authentication if args.cmdAuthSetup: authSetup() # start Malcolm if args.cmdStart or args.cmdRestart: start() # tail Malcolm logs if args.cmdStart or args.cmdRestart or args.cmdLogs: logs() # display Malcolm status if args.cmdStatus: status() if __name__ == '__main__': main() if coloramaImported: print(Style.RESET_ALL)