#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved. # script for configuring sensor capture and forwarding parameters import locale import os import re import shutil import sys import fileinput from collections import defaultdict from dialog import Dialog from zeek_carve_utils import * from sensorcommon import * class Constants: CONFIG_CAP = 'Capture Configuration' DEV_IDENTIFIER_FILE = '/etc/installer' DEV_UNKNOWN = 'unknown' DEV_AGGREGATOR = 'aggregator' DEV_SENSOR = 'sensor' DEV_VALID = {DEV_AGGREGATOR, DEV_SENSOR} MSG_ERR_DEV_INVALID = f'Could not determine installation type (not one of {DEV_VALID})' MSG_ERR_DEV_INCORRECT = 'This tool is not suitable for configuring {}s' SENSOR_CAPTURE_CONFIG = '/opt/sensor/sensor_ctl/control_vars.conf' PCAP_CAPTURE_AUTOSTART_ENTRIES = {'AUTOSTART_ARKIME', 'AUTOSTART_NETSNIFF', 'AUTOSTART_TCPDUMP'} ZEEK_FILE_CARVING_NONE = 'none' ZEEK_FILE_CARVING_ALL = 'all' ZEEK_FILE_CARVING_KNOWN = 'known' ZEEK_FILE_CARVING_MAPPED = 'mapped' ZEEK_FILE_CARVING_MAPPED_MINUS_TEXT = 'mapped (except common plain text files)' ZEEK_FILE_CARVING_INTERESTING = 'interesting' ZEEK_FILE_CARVING_CUSTOM = 'custom' ZEEK_FILE_CARVING_CUSTOM_MIME = 'custom (mime-sorted)' ZEEK_FILE_CARVING_CUSTOM_EXT = 'custom (extension-sorted)' ZEEK_FILE_CARVING_DEFAULTS = '/opt/zeek/share/zeek/site/extractor_params.zeek' ZEEK_FILE_CARVING_OVERRIDE_FILE = '/opt/sensor/sensor_ctl/extractor_override.zeek' ZEEK_FILE_CARVING_OVERRIDE_INTERESTING_FILE = '/opt/sensor/sensor_ctl/extractor_override.interesting.zeek' ZEEK_FILE_CARVING_OVERRIDE_FILE_MAP_NAME = 'extractor_mime_to_ext_map' ZEEK_FILE_CARVING_PLAIN_TEXT_MIMES = { "application/json", "application/x-x509-ca-cert", "application/xml", "text/plain", "text/xml" } FILEBEAT='filebeat' METRICBEAT='metricbeat' AUDITBEAT='auditbeat' HEATBEAT='heatbeat' # protologbeat to log temperature and other misc. stuff SYSLOGBEAT='filebeat-syslog' # another filebeat instance for syslog ARKIMECAP='moloch-capture' BEAT_DIR = { FILEBEAT : f'/opt/sensor/sensor_ctl/{FILEBEAT}', METRICBEAT : f'/opt/sensor/sensor_ctl/{METRICBEAT}', AUDITBEAT : f'/opt/sensor/sensor_ctl/{AUDITBEAT}', SYSLOGBEAT : f'/opt/sensor/sensor_ctl/{SYSLOGBEAT}', HEATBEAT : f'/opt/sensor/sensor_ctl/{HEATBEAT}' } BEAT_KIBANA_DIR = { FILEBEAT : f'/usr/share/{FILEBEAT}/kibana', METRICBEAT : f'/usr/share/{METRICBEAT}/kibana', AUDITBEAT : f'/usr/share/{AUDITBEAT}/kibana', SYSLOGBEAT : f'/usr/share/{FILEBEAT}/kibana', HEATBEAT : f'/usr/share/protologbeat/kibana/' } BEAT_CMD = { FILEBEAT : f'{FILEBEAT} --path.home "{BEAT_DIR[FILEBEAT]}" --path.config "{BEAT_DIR[FILEBEAT]}" --path.data "{BEAT_DIR[FILEBEAT]}/data" --path.logs "{BEAT_DIR[FILEBEAT]}/logs" -c "{BEAT_DIR[FILEBEAT]}/{FILEBEAT}.yml"', METRICBEAT : f'{METRICBEAT} --path.home "{BEAT_DIR[METRICBEAT]}" --path.config "{BEAT_DIR[METRICBEAT]}" --path.data "{BEAT_DIR[METRICBEAT]}/data" --path.logs "{BEAT_DIR[METRICBEAT]}/logs" -c "{BEAT_DIR[METRICBEAT]}/{METRICBEAT}.yml"', AUDITBEAT : f'{AUDITBEAT} --path.home "{BEAT_DIR[AUDITBEAT]}" --path.config "{BEAT_DIR[AUDITBEAT]}" --path.data "{BEAT_DIR[AUDITBEAT]}/data" --path.logs "{BEAT_DIR[AUDITBEAT]}/logs" -c "{BEAT_DIR[AUDITBEAT]}/{AUDITBEAT}.yml"', SYSLOGBEAT : f'{FILEBEAT} --path.home "{BEAT_DIR[SYSLOGBEAT]}" --path.config "{BEAT_DIR[SYSLOGBEAT]}" --path.data "{BEAT_DIR[SYSLOGBEAT]}/data" --path.logs "{BEAT_DIR[SYSLOGBEAT]}/logs" -c "{BEAT_DIR[SYSLOGBEAT]}/{SYSLOGBEAT}.yml"', HEATBEAT : f'protologbeat --path.home "{BEAT_DIR[HEATBEAT]}" --path.config "{BEAT_DIR[HEATBEAT]}" --path.data "{BEAT_DIR[HEATBEAT]}/data" --path.logs "{BEAT_DIR[HEATBEAT]}/logs" -c "{BEAT_DIR[HEATBEAT]}/protologbeat.yml"' } # specific to beats forwarded to logstash (eg., filebeat) BEAT_LS_HOST = 'BEAT_LS_HOST' BEAT_LS_PORT = 'BEAT_LS_PORT' BEAT_LS_SSL = 'BEAT_LS_SSL' BEAT_LS_SSL_CA_CRT = 'BEAT_LS_SSL_CA_CRT' BEAT_LS_SSL_CLIENT_CRT = 'BEAT_LS_SSL_CLIENT_CRT' BEAT_LS_SSL_CLIENT_KEY = 'BEAT_LS_SSL_CLIENT_KEY' BEAT_LS_SSL_VERIFY = 'BEAT_LS_SSL_VERIFY' # specific to beats forwarded to elasticsearch (eg., metricbeat, auditbeat, filebeat-syslog) BEAT_ES_HOST = "BEAT_ES_HOST" BEAT_ES_PORT = "BEAT_ES_PORT" BEAT_ES_PROTOCOL = "BEAT_ES_PROTOCOL" BEAT_ES_SSL_VERIFY = "BEAT_ES_SSL_VERIFY" BEAT_HTTP_PASSWORD = "BEAT_HTTP_PASSWORD" BEAT_HTTP_USERNAME = "BEAT_HTTP_USERNAME" BEAT_KIBANA_DASHBOARDS_ENABLED = "BEAT_KIBANA_DASHBOARDS_ENABLED" BEAT_KIBANA_DASHBOARDS_PATH = "BEAT_KIBANA_DASHBOARDS_PATH" BEAT_KIBANA_HOST = "BEAT_KIBANA_HOST" BEAT_KIBANA_PORT = "BEAT_KIBANA_PORT" BEAT_KIBANA_PROTOCOL = "BEAT_KIBANA_PROTOCOL" BEAT_KIBANA_SSL_VERIFY = "BEAT_KIBANA_SSL_VERIFY" # specific to filebeat BEAT_LOG_PATH_SUBDIR = os.path.join('logs', 'current') BEAT_LOG_PATTERN_KEY = 'BEAT_LOG_PATTERN' BEAT_LOG_PATTERN_VAL = '*.log' # specific to metricbeat BEAT_INTERVAL = "BEAT_INTERVAL" # specific to moloch ARKIME_PACKET_ACL = "ARKIME_PACKET_ACL" MSG_CONFIG_MODE = 'Configuration Mode' MSG_CONFIG_MODE_CAPTURE = 'Configure Capture' MSG_CONFIG_MODE_FORWARD = 'Configure Forwarding' MSG_CONFIG_MODE_AUTOSTART = 'Configure Autostart Services' MSG_CONFIG_GENERIC = 'Configure {}' MSG_CONFIG_ARKIME = (f'{ARKIMECAP}', f'Configure Arkime session forwarding via {ARKIMECAP}') MSG_CONFIG_FILEBEAT = (f'{FILEBEAT}', f'Configure Zeek log forwarding via {FILEBEAT}') MSG_CONFIG_METRICBEAT = (f'{METRICBEAT}', f'Configure resource metrics forwarding via {METRICBEAT}') MSG_CONFIG_AUDITBEAT = (f'{AUDITBEAT}', f'Configure audit log forwarding via {AUDITBEAT}') MSG_CONFIG_SYSLOGBEAT = (f'{SYSLOGBEAT}', f'Configure syslog forwarding via {FILEBEAT}') MSG_CONFIG_HEATBEAT = (f'{HEATBEAT}', f'Configure hardware metrics (temperature, etc.) forwarding via protologbeat') MSG_OVERWRITE_CONFIG = '{} is already configured, overwrite current settings?' MSG_IDENTIFY_NICS = 'Do you need help identifying network interfaces?' MSG_BACKGROUND_TITLE = 'Sensor Configuration' MSG_CONFIG_AUTOSTARTS = 'Specify autostart processes' MSG_CONFIG_ZEEK_CARVED_SCANNERS = 'Specify scanners for Zeek-carved files' MSG_CONFIG_ZEEK_CARVING = 'Specify Zeek file carving mode' MSG_CONFIG_ZEEK_CARVING_MIMES = 'Specify file types to carve' MSG_CONFIG_CARVED_FILE_PRESERVATION = 'Specify which carved files to preserve' MSG_CONFIG_CAP_CONFIRM = 'Sensor will capture traffic with the following parameters:\n\n{}' MSG_CONFIG_AUTOSTART_CONFIRM = 'Sensor autostart the following services:\n\n{}' MSG_CONFIG_FORWARDING_CONFIRM = '{} will forward with the following parameters:\n\n{}' MSG_CONFIG_CAP_PATHS = 'Provide paths for captured PCAPs and Zeek logs' MSG_CONFIG_CAPTURE_SUCCESS = 'Capture interface set to {} in {}.\n\nReboot to apply changes.' MSG_CONFIG_AUTOSTART_SUCCESS = 'Autostart services configured.\n\nReboot to apply changes.' MSG_CONFIG_FORWARDING_SUCCESS = '{} forwarding configured:\n\n{}\n\nRestart forwarding services or reboot to apply changes.' MSG_CONFIG_ARKIME_PCAP_ACL = 'Specify IP addresses for PCAP retrieval ACL (one per line)' MSG_ERR_PLEBE_REQUIRED = 'this utility should be be run as non-privileged user' MSG_ERROR_DIR_NOT_FOUND = 'One or more of the paths specified does not exist' MSG_ERROR_FILE_NOT_FOUND = 'One or more of the files specified does not exist' MSG_ERROR_BAD_HOST = 'Invalid host or port' MSG_ERROR_FWD_DIR_NOT_FOUND = 'The path {} does not exist, {} cannot be configured' MSG_ERROR_MISSING_CAP_CONFIG = f'Capture configuration file {SENSOR_CAPTURE_CONFIG} does not exist' MSG_ERROR_KEYSTORE = 'There was an error creating the keystore for {}:\n\n{}' MSG_ERROR_FILTER_VALIDATION = "Warning: capture filter failed validation ({}). Adjust filter, or resubmit unchanged to ignore warning." MSG_MESSAGE_ERROR = 'Error: {}\n\nPlease try again.' MSG_CANCEL_ERROR = 'Operation cancelled, goodbye!' MSG_EMPTY_CONFIG_ERROR = "No configuration values were supplied" MSG_SELECT_INTERFACE = 'Select capture interface(s)' MSG_SELECT_BLINK_INTERFACE = 'Select capture interface to identify' MSG_BLINK_INTERFACE = '{} will blink for {} seconds' MSG_WELCOME_TITLE = 'Welcome to the sensor capture and forwarding configuration utility!' MSG_TESTING_CONNECTION = 'Testing {} connection...' MSG_TESTING_CONNECTION_SUCCESS = '{} connection succeeded! ({} {})' MSG_TESTING_CONNECTION_FAILURE = "{} connection error: {} {}:\n\n {}" MSG_TESTING_CONNECTION_FAILURE_LOGSTASH = "{} connection error: could not connect to {}:{}" MSG_WARNING_MULTIPLE_PCAP = "Warning: multiple PCAP processes are enabled ({}). Using a single PCAP process is recommended." # the main dialog window used for the duration of this tool d = Dialog(dialog='dialog', autowidgetsize=True) d.set_background_title(Constants.MSG_BACKGROUND_TITLE) ################################################################################################### def mime_to_extension_mappings(mapfile): # get all mime-to-extension mappings from our mapping zeek file into a dictionary mime_maps = defaultdict(str) if os.path.exists(mapfile): maps_list = [] with open(mapfile) as f: maps_list = [x.replace(' ', '') for x in re.findall(r'(\[\s*"[A-Za-z0-9/\.\+_-]+"\s*\]\s*=\s*"[A-Za-z0-9\.\+_-]+")', f.read(), re.MULTILINE)] mime_map_re = re.compile(r'\[\s*"([A-Za-z0-9/\.\+_-]+)"\s*\]\s*=\s*"([A-Za-z0-9\.\+_-]+)"') for mime_map in maps_list: match = mime_map_re.search(mime_map) if match: mime_maps[match.group(1)] = match.group(2) return mime_maps ################################################################################################### def input_elasticsearch_connection_info(forwarder, default_es_host=None, default_es_port=None, default_kibana_host=None, default_kibana_port=None, default_username=None, default_password=None): return_dict = defaultdict(str) # Elasticsearch configuration # elasticsearch protocol and SSL verification mode elastic_protocol = "http" elastic_ssl_verify = "none" if (d.yesno("Elasticsearch connection protocol", yes_label="HTTPS", no_label="HTTP") == Dialog.OK): elastic_protocol = "https" if (d.yesno("Elasticsearch SSL verification", yes_label="None", no_label="Full") != Dialog.OK): elastic_ssl_verify = "full" return_dict[Constants.BEAT_ES_PROTOCOL] = elastic_protocol return_dict[Constants.BEAT_ES_SSL_VERIFY] = elastic_ssl_verify while True: # host/port for Elasticsearch code, values = d.form(Constants.MSG_CONFIG_GENERIC.format(forwarder), [ ('Elasticsearch Host', 1, 1, default_es_host or "", 1, 25, 30, 255), ('Elasticsearch Port', 2, 1, default_es_port or "9200", 2, 25, 6, 5) ]) values = [x.strip() for x in values] if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError elif (len(values[0]) <= 0) or (len(values[1]) <= 0) or (not values[1].isnumeric()): code = d.msgbox(text=Constants.MSG_ERROR_BAD_HOST) else: return_dict[Constants.BEAT_ES_HOST] = values[0] return_dict[Constants.BEAT_ES_PORT] = values[1] break # Kibana configuration (if supported by forwarder) if (forwarder in Constants.BEAT_KIBANA_DIR.keys()) and (d.yesno(f"Configure {forwarder} Kibana connectivity?") == Dialog.OK): # elasticsearch protocol and SSL verification mode kibana_protocol = "http" kibana_ssl_verify = "none" if (d.yesno("Kibana connection protocol", yes_label="HTTPS", no_label="HTTP") == Dialog.OK): kibana_protocol = "https" if (d.yesno("Kibana SSL verification", yes_label="None", no_label="Full") != Dialog.OK): kibana_ssl_verify = "full" return_dict[Constants.BEAT_KIBANA_PROTOCOL] = kibana_protocol return_dict[Constants.BEAT_KIBANA_SSL_VERIFY] = kibana_ssl_verify while True: # host/port for Kibana code, values = d.form(Constants.MSG_CONFIG_GENERIC.format(forwarder), [ ('Kibana Host', 1, 1, default_kibana_host or "", 1, 20, 30, 255), ('Kibana Port', 2, 1, default_kibana_port or "5601", 2, 20, 6, 5) ]) values = [x.strip() for x in values] if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError elif (len(values[0]) <= 0) or (len(values[1]) <= 0) or (not values[1].isnumeric()): code = d.msgbox(text=Constants.MSG_ERROR_BAD_HOST) else: return_dict[Constants.BEAT_KIBANA_HOST] = values[0] return_dict[Constants.BEAT_KIBANA_PORT] = values[1] break if (d.yesno(f"Configure {forwarder} Kibana dashboards?") == Dialog.OK): kibana_dashboards = "true" else: kibana_dashboards = "false" return_dict[Constants.BEAT_KIBANA_DASHBOARDS_ENABLED] = kibana_dashboards if kibana_dashboards == "true": while True: code, values = d.form(Constants.MSG_CONFIG_GENERIC.format(forwarder), [ ('Kibana Dashboards Path', 1, 1, Constants.BEAT_KIBANA_DIR[forwarder], 1, 30, 30, 255) ]) values = [x.strip() for x in values] if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError elif (len(values[0]) <= 0) or (not os.path.isdir(values[0])): code = d.msgbox(text=Constants.MSG_ERROR_DIR_NOT_FOUND) else: return_dict[Constants.BEAT_KIBANA_DASHBOARDS_PATH] = values[0] break server_display_name = "Elasticsearch/Kibana" if Constants.BEAT_KIBANA_HOST in return_dict.keys() else "Elasticsearch" # HTTP/HTTPS authentication code, http_username = d.inputbox(f"{server_display_name} HTTP/HTTPS server username", init=default_username) if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError return_dict[Constants.BEAT_HTTP_USERNAME] = http_username.strip() # make them enter the password twice while True: code, http_password = d.passwordbox(f"{server_display_name} HTTP/HTTPS server password", insecure=True, init=default_password) if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError code, http_password2 = d.passwordbox(f"{server_display_name} HTTP/HTTPS server password (again)", insecure=True, init=default_password if (http_password == default_password) else "") if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError if (http_password == http_password2): return_dict[Constants.BEAT_HTTP_PASSWORD] = http_password.strip() break else: code = d.msgbox(text=Constants.MSG_MESSAGE_ERROR.format("Passwords did not match")) # test Elasticsearch connection code = d.infobox(Constants.MSG_TESTING_CONNECTION.format("Elasticsearch")) retcode, message, output = test_connection(protocol=return_dict[Constants.BEAT_ES_PROTOCOL], host=return_dict[Constants.BEAT_ES_HOST], port=return_dict[Constants.BEAT_ES_PORT], username=return_dict[Constants.BEAT_HTTP_USERNAME] if (len(return_dict[Constants.BEAT_HTTP_USERNAME]) > 0) else None, password=return_dict[Constants.BEAT_HTTP_PASSWORD] if (len(return_dict[Constants.BEAT_HTTP_PASSWORD]) > 0) else None, ssl_verify=return_dict[Constants.BEAT_ES_SSL_VERIFY]) if (retcode == 200): code = d.msgbox(text=Constants.MSG_TESTING_CONNECTION_SUCCESS.format("Elasticsearch", retcode, message)) else: code = d.yesno(text=Constants.MSG_TESTING_CONNECTION_FAILURE.format("Elasticsearch", retcode, message, "\n".join(output)), yes_label="Ignore Error", no_label="Start Over") if code != Dialog.OK: raise CancelledError # test Kibana connection if Constants.BEAT_KIBANA_HOST in return_dict.keys(): code = d.infobox(Constants.MSG_TESTING_CONNECTION.format("Kibana")) retcode, message, output = test_connection(protocol=return_dict[Constants.BEAT_KIBANA_PROTOCOL], host=return_dict[Constants.BEAT_KIBANA_HOST], port=return_dict[Constants.BEAT_KIBANA_PORT], uri="api/status", username=return_dict[Constants.BEAT_HTTP_USERNAME] if (len(return_dict[Constants.BEAT_HTTP_USERNAME]) > 0) else None, password=return_dict[Constants.BEAT_HTTP_PASSWORD] if (len(return_dict[Constants.BEAT_HTTP_PASSWORD]) > 0) else None, ssl_verify=return_dict[Constants.BEAT_KIBANA_SSL_VERIFY]) if (retcode == 200): code = d.msgbox(text=Constants.MSG_TESTING_CONNECTION_SUCCESS.format("Kibana", retcode, message)) else: code = d.yesno(text=Constants.MSG_TESTING_CONNECTION_FAILURE.format("Kibana", retcode, message, "\n".join(output)), yes_label="Ignore Error", no_label="Start Over") if code != Dialog.OK: raise CancelledError return return_dict ################################################################################################### ################################################################################################### def main(): locale.setlocale(locale.LC_ALL, '') # make sure we are NOT being run as root if os.getuid() == 0: print(Constants.MSG_ERR_PLEBE_REQUIRED) sys.exit(1) # what are we (sensor vs. aggregator) installation = Constants.DEV_UNKNOWN modeChoices = [] try: with open(Constants.DEV_IDENTIFIER_FILE, 'r') as f: installation = f.readline().strip() except: pass if (installation not in Constants.DEV_VALID): print(Constants.MSG_ERR_DEV_INVALID) sys.exit(1) elif (installation == Constants.DEV_SENSOR): modeChoices = [(Constants.MSG_CONFIG_MODE_CAPTURE, ""), (Constants.MSG_CONFIG_MODE_FORWARD, ""), (Constants.MSG_CONFIG_MODE_AUTOSTART, "")] else: print(Constants.MSG_ERR_DEV_INCORRECT.format(installation)) sys.exit(1) start_dir = os.getcwd() quit_flag = False # store previously-entered elasticsearch values in case they are going through the loop # mulitple times to prevent them from having to enter them over and over previous_config_values = defaultdict(str) while not quit_flag: try: os.chdir(start_dir) if not os.path.isfile(Constants.SENSOR_CAPTURE_CONFIG): # SENSOR_CAPTURE_CONFIG file doesn't exist, can't continue raise Exception(Constants.MSG_ERROR_MISSING_CAP_CONFIG) # read existing configuration from SENSOR_CAPTURE_CONFIG into a dictionary file (not written back out as such, just used # as a basis for default values) capture_config_dict = defaultdict(str) with open(Constants.SENSOR_CAPTURE_CONFIG) as file: for line in file: if len(line.strip()) > 0: name, var = remove_prefix(line, "export").partition("=")[::2] capture_config_dict[name.strip()] = var.strip().strip("'").strip('"') if (Constants.BEAT_ES_HOST not in previous_config_values.keys()) and ("ES_HOST" in capture_config_dict.keys()): previous_config_values[Constants.BEAT_ES_HOST] = capture_config_dict["ES_HOST"] previous_config_values[Constants.BEAT_KIBANA_HOST] = capture_config_dict["ES_HOST"] if (Constants.BEAT_ES_PORT not in previous_config_values.keys()) and ("ES_PORT" in capture_config_dict.keys()): previous_config_values[Constants.BEAT_ES_PORT] = capture_config_dict["ES_PORT"] if (Constants.BEAT_HTTP_USERNAME not in previous_config_values.keys()) and ("ES_USERNAME" in capture_config_dict.keys()): previous_config_values[Constants.BEAT_HTTP_USERNAME] = capture_config_dict["ES_USERNAME"] if (Constants.ARKIME_PACKET_ACL not in previous_config_values.keys()) and ("ARKIME_PACKET_ACL" in capture_config_dict.keys()): previous_config_values[Constants.ARKIME_PACKET_ACL] = capture_config_dict[Constants.ARKIME_PACKET_ACL] code = d.yesno(Constants.MSG_WELCOME_TITLE, yes_label="Continue", no_label="Quit") if (code == Dialog.CANCEL or code == Dialog.ESC): quit_flag = True raise CancelledError code, mode = d.menu(Constants.MSG_CONFIG_MODE, choices=modeChoices) if code != Dialog.OK: quit_flag = True raise CancelledError if mode == Constants.MSG_CONFIG_MODE_AUTOSTART: ##### sensor autostart services configuration ####################################################################################### while True: # select processes for autostart (except for the file scan ones, handle those with the file scanning stuff) autostart_choices = [] for k, v in sorted(capture_config_dict.items()): if k.startswith("AUTOSTART_"): autostart_choices.append((k, '', v.lower() == "true")) code, autostart_tags = d.checklist(Constants.MSG_CONFIG_AUTOSTARTS, choices=autostart_choices) if (code == Dialog.CANCEL or code == Dialog.ESC): raise CancelledError for tag in [x[0] for x in autostart_choices]: capture_config_dict[tag] = "false" for tag in autostart_tags: capture_config_dict[tag] = "true" # warn them if we're doing mulitple PCAP capture processes pcap_procs_enabled = [x for x in autostart_tags if x in Constants.PCAP_CAPTURE_AUTOSTART_ENTRIES] if ((len(pcap_procs_enabled) <= 1) or (d.yesno(text=Constants.MSG_WARNING_MULTIPLE_PCAP.format(", ".join(pcap_procs_enabled)), yes_label="Continue Anyway", no_label="Adjust Selections") == Dialog.OK)): break # get confirmation from user that we really want to do this code = d.yesno(Constants.MSG_CONFIG_AUTOSTART_CONFIRM.format("\n".join(sorted([f"{k}={v}" for k, v in capture_config_dict.items() if "AUTOSTART" in k]))), yes_label="OK", no_label="Cancel") if code == Dialog.OK: # modify specified values in-place in SENSOR_CAPTURE_CONFIG file autostart_re = re.compile(r"(\bAUTOSTART_\w+)\s*=\s*.+?$") with fileinput.FileInput(Constants.SENSOR_CAPTURE_CONFIG, inplace=True, backup='.bak') as file: for line in file: line = line.rstrip("\n") autostart_match = autostart_re.search(line) if autostart_match is not None: print(autostart_re.sub(r"\1=%s" % capture_config_dict[autostart_match.group(1)], line)) else: print(line) # hooray code = d.msgbox(text=Constants.MSG_CONFIG_AUTOSTART_SUCCESS) elif mode == Constants.MSG_CONFIG_MODE_CAPTURE: ##### sensor capture configuration ################################################################################################## # determine a list of available (non-virtual) adapters available_adapters = get_available_adapters() # previously used capture interfaces preselected_ifaces = set([x.strip() for x in capture_config_dict["CAPTURE_INTERFACE"].split(',')]) while (len(available_adapters) > 0) and (d.yesno(Constants.MSG_IDENTIFY_NICS) == Dialog.OK): code, blinky_iface = d.radiolist(Constants.MSG_SELECT_BLINK_INTERFACE, choices=[(adapter.name, adapter.description, False) for adapter in available_adapters]) if (code == Dialog.OK) and (len(blinky_iface) > 0): if (d.yesno(Constants.MSG_BLINK_INTERFACE.format(blinky_iface, NIC_BLINK_SECONDS), yes_label="Ready", no_label="Cancel") == Dialog.OK): identify_adapter(adapter=blinky_iface, duration=NIC_BLINK_SECONDS, background=True) code = d.pause(f"Identifying {blinky_iface}", seconds=NIC_BLINK_SECONDS, width=60, height=15) elif (code != Dialog.OK): break # user selects interface(s) for capture code, tag = d.checklist(Constants.MSG_SELECT_INTERFACE, choices=[(adapter.name, adapter.description, adapter.name in preselected_ifaces) for adapter in available_adapters]) if code != Dialog.OK: raise CancelledError selected_ifaces = tag if (len(selected_ifaces) > 0): # user specifies capture filter (and we validate it with tcpdump) prev_capture_filter = capture_config_dict["CAPTURE_FILTER"] while True: code, capture_filter = d.inputbox("PCAP capture filter (tcpdump-like filter expression; leave blank to capture all traffic)", init=prev_capture_filter) if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError capture_filter = capture_filter.strip() if (len(capture_filter) > 0): # test out the capture filter to see if there's a syntax error ecode, filter_test_results = run_process(f'tcpdump -i {selected_ifaces[0]} -d "{capture_filter}"', stdout=False, stderr=True) else: # nothing to validate ecode = 0 filter_test_results = [""] if (prev_capture_filter == capture_filter) or ((ecode == 0) and (not any(x.lower().startswith("tcpdump: warning") for x in filter_test_results)) and (not any(x.lower().startswith("tcpdump: error") for x in filter_test_results)) and (not any("syntax error" in x.lower() for x in filter_test_results))): break else: code = d.msgbox(text=Constants.MSG_ERROR_FILTER_VALIDATION.format(" ".join([x.strip() for x in filter_test_results]))) prev_capture_filter = capture_filter # regular expressions for selected name=value pairs to update in configuration file capture_interface_re = re.compile(r"(\bCAPTURE_INTERFACE)\s*=\s*.+?$") capture_filter_re = re.compile(r"(\bCAPTURE_FILTER)\s*=\s*.*?$") pcap_path_re = re.compile(r"(\bPCAP_PATH)\s*=\s*.+?$") zeek_path_re = re.compile(r"(\bZEEK_LOG_PATH)\s*=\s*.+?$") zeek_carve_re = re.compile(r"(\bZEEK_EXTRACTOR_MODE)\s*=\s*.+?$") zeek_file_preservation_re = re.compile(r"(\bEXTRACTED_FILE_PRESERVATION)\s*=\s*.+?$") zeek_carve_override_re = re.compile(r"(\bZEEK_EXTRACTOR_OVERRIDE_FILE)\s*=\s*.*?$") zeek_file_watch_re = re.compile(r"(\bZEEK_FILE_WATCH)\s*=\s*.+?$") zeek_file_scanner_re = re.compile(r"(\bZEEK_FILE_SCAN_\w+)\s*=\s*.+?$") # get paths for captured PCAP and Zeek files while True: code, path_values = d.form(Constants.MSG_CONFIG_CAP_PATHS, [ ('PCAP Path', 1, 1, capture_config_dict.get("PCAP_PATH", ""), 1, 20, 30, 255), ('Zeek Log Path', 2, 1, capture_config_dict.get("ZEEK_LOG_PATH", ""), 2, 20, 30, 255), ]) path_values = [x.strip() for x in path_values] if (code == Dialog.CANCEL or code == Dialog.ESC): raise CancelledError # paths must be specified, and must already exist if ((len(path_values[0]) > 0) and os.path.isdir(path_values[0]) and (len(path_values[1]) > 0) and os.path.isdir(path_values[1])): break else: code = d.msgbox(text=Constants.MSG_ERROR_DIR_NOT_FOUND) # configure file carving code, zeek_carve_mode = d.radiolist(Constants.MSG_CONFIG_ZEEK_CARVING, choices=[(Constants.ZEEK_FILE_CARVING_NONE, 'Disable file carving', (capture_config_dict["ZEEK_EXTRACTOR_MODE"] == Constants.ZEEK_FILE_CARVING_NONE)), (Constants.ZEEK_FILE_CARVING_MAPPED, 'Carve files with recognized mime types', ((capture_config_dict["ZEEK_EXTRACTOR_MODE"] == Constants.ZEEK_FILE_CARVING_MAPPED) and (len(capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"]) == 0))), (Constants.ZEEK_FILE_CARVING_MAPPED_MINUS_TEXT, 'Carve files with recognized mime types (except common plain text files)', False), (Constants.ZEEK_FILE_CARVING_KNOWN, 'Carve files for which any mime type can be determined', (capture_config_dict["ZEEK_EXTRACTOR_MODE"] == Constants.ZEEK_FILE_CARVING_KNOWN)), (Constants.ZEEK_FILE_CARVING_INTERESTING, 'Carve files with mime types of common attack vectors', False), (Constants.ZEEK_FILE_CARVING_CUSTOM_MIME, 'Use a custom selection of mime types (sorted by mime type)', ((capture_config_dict["ZEEK_EXTRACTOR_MODE"] == Constants.ZEEK_FILE_CARVING_MAPPED) and (len(capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"]) > 0))), (Constants.ZEEK_FILE_CARVING_CUSTOM_EXT, 'Use a custom selection of mime types (sorted by file extension)', False), (Constants.ZEEK_FILE_CARVING_ALL, 'Carve all files', (capture_config_dict["ZEEK_EXTRACTOR_MODE"] == Constants.ZEEK_FILE_CARVING_ALL))]) if (code == Dialog.CANCEL or code == Dialog.ESC): raise CancelledError mime_tags = [] capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"] = "" zeek_carved_file_preservation = PRESERVE_NONE if zeek_carve_mode.startswith(Constants.ZEEK_FILE_CARVING_CUSTOM) or zeek_carve_mode.startswith(Constants.ZEEK_FILE_CARVING_MAPPED_MINUS_TEXT): # get all known mime-to-extension mappings into a dictionary all_mime_maps = mime_to_extension_mappings(Constants.ZEEK_FILE_CARVING_DEFAULTS) if (zeek_carve_mode == Constants.ZEEK_FILE_CARVING_MAPPED_MINUS_TEXT): # all mime types minus common text mime types mime_tags.extend([mime for mime in all_mime_maps.keys() if mime not in Constants.ZEEK_FILE_CARVING_PLAIN_TEXT_MIMES]) else: # select mimes to carve (pre-selecting items previously in the override file) if (zeek_carve_mode == Constants.ZEEK_FILE_CARVING_CUSTOM_EXT): mime_choices = [(pair[0], pair[1], pair[0] in mime_to_extension_mappings(Constants.ZEEK_FILE_CARVING_OVERRIDE_FILE)) for pair in sorted(all_mime_maps.items(), key=lambda x: x[1].lower())] else: mime_choices = [(pair[0], pair[1], pair[0] in mime_to_extension_mappings(Constants.ZEEK_FILE_CARVING_OVERRIDE_FILE)) for pair in sorted(all_mime_maps.items(), key=lambda x: x[0].lower())] code, mime_tags = d.checklist(Constants.MSG_CONFIG_ZEEK_CARVING_MIMES, choices=mime_choices) if (code == Dialog.CANCEL or code == Dialog.ESC): raise CancelledError mime_tags.sort() if (len(mime_tags) == 0): zeek_carve_mode = Constants.ZEEK_FILE_CARVING_NONE elif (len(mime_tags) >= len(all_mime_maps)): zeek_carve_mode = Constants.ZEEK_FILE_CARVING_MAPPED elif len(mime_tags) > 0: zeek_carve_mode = Constants.ZEEK_FILE_CARVING_MAPPED capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"] = Constants.ZEEK_FILE_CARVING_OVERRIDE_FILE else: zeek_carve_mode = Constants.ZEEK_FILE_CARVING_MAPPED elif zeek_carve_mode.startswith(Constants.ZEEK_FILE_CARVING_INTERESTING): shutil.copy(Constants.ZEEK_FILE_CARVING_OVERRIDE_INTERESTING_FILE, Constants.ZEEK_FILE_CARVING_OVERRIDE_FILE) zeek_carve_mode = Constants.ZEEK_FILE_CARVING_MAPPED capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"] = Constants.ZEEK_FILE_CARVING_OVERRIDE_FILE # what to do with carved files if (zeek_carve_mode != Constants.ZEEK_FILE_CARVING_NONE): # select engines for file scanning scanner_choices = [] for k, v in sorted(capture_config_dict.items()): if k.startswith("ZEEK_FILE_SCAN_"): scanner_choices.append((k, '', v.lower() == "true")) code, scanner_tags = d.checklist(Constants.MSG_CONFIG_ZEEK_CARVED_SCANNERS, choices=scanner_choices) if (code == Dialog.CANCEL or code == Dialog.ESC): raise CancelledError for tag in [x[0] for x in scanner_choices]: capture_config_dict[tag] = "false" for tag in scanner_tags: capture_config_dict[tag] = "true" capture_config_dict["ZEEK_FILE_WATCH"] = "true" if (len(scanner_tags) > 0) else "false" # specify what to do with files that triggered the scanner engine(s) code, zeek_carved_file_preservation = d.radiolist(Constants.MSG_CONFIG_CARVED_FILE_PRESERVATION, choices=[(PRESERVE_QUARANTINED, 'Preserve only quarantined files', (capture_config_dict["EXTRACTED_FILE_PRESERVATION"] == PRESERVE_QUARANTINED)), (PRESERVE_ALL, 'Preserve all files', (capture_config_dict["EXTRACTED_FILE_PRESERVATION"] == PRESERVE_ALL)), (PRESERVE_NONE, 'Preserve no files', (capture_config_dict["EXTRACTED_FILE_PRESERVATION"] == PRESERVE_NONE))]) if (code == Dialog.CANCEL or code == Dialog.ESC): raise CancelledError else: # file carving disabled, so disable file scanning as well for key in ["ZEEK_FILE_WATCH", "ZEEK_FILE_SCAN_CLAMAV", "ZEEK_FILE_SCAN_VTOT", "ZEEK_FILE_SCAN_MALASS", "ZEEK_FILE_SCAN_YARA"]: capture_config_dict[key] = "false" # reconstitute dictionary with user-specified values capture_config_dict["CAPTURE_INTERFACE"] = ",".join(selected_ifaces) capture_config_dict["CAPTURE_FILTER"] = capture_filter capture_config_dict["PCAP_PATH"] = path_values[0] capture_config_dict["ZEEK_LOG_PATH"] = path_values[1] capture_config_dict["ZEEK_EXTRACTOR_MODE"] = zeek_carve_mode capture_config_dict["EXTRACTED_FILE_PRESERVATION"] = zeek_carved_file_preservation # get confirmation from user that we really want to do this code = d.yesno(Constants.MSG_CONFIG_CAP_CONFIRM.format("\n".join(sorted([f"{k}={v}" for k, v in capture_config_dict.items() if (not k.startswith("#")) and ("AUTOSTART" not in k) and ("PASSWORD" not in k)]))), yes_label="OK", no_label="Cancel") if code == Dialog.OK: # modify specified values in-place in SENSOR_CAPTURE_CONFIG file with fileinput.FileInput(Constants.SENSOR_CAPTURE_CONFIG, inplace=True, backup='.bak') as file: for line in file: line = line.rstrip("\n") if capture_interface_re.search(line) is not None: print(capture_interface_re.sub(r"\1=%s" % ",".join(selected_ifaces), line)) elif zeek_carve_override_re.search(line) is not None: print(zeek_carve_override_re.sub(r'\1="%s"' % capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"], line)) elif zeek_carve_re.search(line) is not None: print(zeek_carve_re.sub(r"\1=%s" % zeek_carve_mode, line)) elif zeek_file_preservation_re.search(line) is not None: print(zeek_file_preservation_re.sub(r"\1=%s" % zeek_carved_file_preservation, line)) elif capture_filter_re.search(line) is not None: print(capture_filter_re.sub(r'\1="%s"' % capture_filter, line)) elif pcap_path_re.search(line) is not None: print(pcap_path_re.sub(r'\1="%s"' % capture_config_dict["PCAP_PATH"], line)) elif zeek_path_re.search(line) is not None: print(zeek_path_re.sub(r'\1="%s"' % capture_config_dict["ZEEK_LOG_PATH"], line)) elif zeek_file_watch_re.search(line) is not None: print(zeek_file_watch_re.sub(r"\1=%s" % capture_config_dict["ZEEK_FILE_WATCH"], line)) else: zeek_file_scanner_match = zeek_file_scanner_re.search(line) if zeek_file_scanner_match is not None: print(zeek_file_scanner_re.sub(r"\1=%s" % capture_config_dict[zeek_file_scanner_match.group(1)], line)) else: print(line) # write out file carving overrides if specified if (len(mime_tags) > 0) and (len(capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"]) > 0): with open(capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"], "w+") as f: f.write('#!/usr/bin/env zeek\n') f.write('\n') f.write('export {\n') f.write(f' redef {Constants.ZEEK_FILE_CARVING_OVERRIDE_FILE_MAP_NAME} : table[string] of string = {{\n') f.write(",\n".join([f' ["{m}"] = "{all_mime_maps[m]}"' for m in mime_tags])) f.write('\n } &default="bin";\n') f.write('}\n') # hooray code = d.msgbox(text=Constants.MSG_CONFIG_CAPTURE_SUCCESS.format(",".join(selected_ifaces), Constants.SENSOR_CAPTURE_CONFIG)) elif mode == Constants.MSG_CONFIG_MODE_FORWARD: ##### sensor forwarding (beats) configuration ######################################################################### code, fwd_mode = d.menu(Constants.MSG_CONFIG_MODE, choices=[Constants.MSG_CONFIG_FILEBEAT, Constants.MSG_CONFIG_ARKIME, Constants.MSG_CONFIG_METRICBEAT, Constants.MSG_CONFIG_AUDITBEAT, Constants.MSG_CONFIG_SYSLOGBEAT, Constants.MSG_CONFIG_HEATBEAT]) if code != Dialog.OK: raise CancelledError if (fwd_mode == Constants.ARKIMECAP): # forwarding configuration for moloch-capture # get elasticsearch/kibana connection information from user elastic_config_dict = input_elasticsearch_connection_info(forwarder=fwd_mode, default_es_host=previous_config_values[Constants.BEAT_ES_HOST], default_es_port=previous_config_values[Constants.BEAT_ES_PORT], default_username=previous_config_values[Constants.BEAT_HTTP_USERNAME], default_password=previous_config_values[Constants.BEAT_HTTP_PASSWORD]) moloch_elastic_config_dict = elastic_config_dict.copy() # massage the data a bit for how moloch's going to want it in the control_vars.conf file if Constants.BEAT_HTTP_USERNAME in moloch_elastic_config_dict.keys(): moloch_elastic_config_dict["ES_USERNAME"] = moloch_elastic_config_dict.pop(Constants.BEAT_HTTP_USERNAME) if Constants.BEAT_HTTP_PASSWORD in moloch_elastic_config_dict.keys(): moloch_elastic_config_dict["ES_PASSWORD"] = aggressive_url_encode(moloch_elastic_config_dict.pop(Constants.BEAT_HTTP_PASSWORD)) moloch_elastic_config_dict = { k.replace('BEAT_', ''): v for k, v in moloch_elastic_config_dict.items() } # get list of IP addresses allowed for packet payload retrieval lines = previous_config_values[Constants.ARKIME_PACKET_ACL].split(",") lines.append(elastic_config_dict[Constants.BEAT_ES_HOST]) code, lines = d.editbox_str("\n".join(list(filter(None, list(set(lines))))), title=Constants.MSG_CONFIG_ARKIME_PCAP_ACL) if code != Dialog.OK: raise CancelledError moloch_elastic_config_dict[Constants.ARKIME_PACKET_ACL] = ','.join([ip for ip in list(set(filter(None, [x.strip() for x in lines.split('\n')]))) if isipaddress(ip)]) list_results = sorted([f"{k}={v}" for k, v in moloch_elastic_config_dict.items() if ("PASSWORD" not in k) and (not k.startswith("#"))]) code = d.yesno(Constants.MSG_CONFIG_FORWARDING_CONFIRM.format(fwd_mode, "\n".join(list_results)), yes_label="OK", no_label="Cancel") if code != Dialog.OK: raise CancelledError previous_config_values = elastic_config_dict.copy() # modify specified values in-place in SENSOR_CAPTURE_CONFIG file elastic_values_re = re.compile(r"\b(" + '|'.join(list(moloch_elastic_config_dict.keys())) + r")\s*=\s*.*?$") with fileinput.FileInput(Constants.SENSOR_CAPTURE_CONFIG, inplace=True, backup='.bak') as file: for line in file: line = line.rstrip("\n") elastic_key_match = elastic_values_re.search(line) if elastic_key_match is not None: print(elastic_values_re.sub(r"\1=%s" % moloch_elastic_config_dict[elastic_key_match.group(1)], line)) else: print(line) # hooray code = d.msgbox(text=Constants.MSG_CONFIG_FORWARDING_SUCCESS.format(fwd_mode, "\n".join(list_results))) elif (fwd_mode == Constants.FILEBEAT) or (fwd_mode == Constants.METRICBEAT) or (fwd_mode == Constants.AUDITBEAT) or (fwd_mode == Constants.SYSLOGBEAT) or (fwd_mode == Constants.HEATBEAT): # forwarder configuration for beats if not os.path.isdir(Constants.BEAT_DIR[fwd_mode]): # beat dir not found, give up raise Exception(Constants.MSG_ERROR_FWD_DIR_NOT_FOUND.format(Constants.BEAT_DIR[fwd_mode], fwd_mode)) # chdir to the beat directory os.chdir(Constants.BEAT_DIR[fwd_mode]) # check to see if a keystore has already been created for the forwarder ecode, list_results = run_process(f"{Constants.BEAT_CMD[fwd_mode]} keystore list") if (ecode == 0) and (len(list_results) > 0): # it has, do they wish to overwrite it? if (d.yesno(Constants.MSG_OVERWRITE_CONFIG.format(fwd_mode)) != Dialog.OK): raise CancelledError ecode, create_results = run_process(f"{Constants.BEAT_CMD[fwd_mode]} keystore create --force", stderr=True) if (ecode != 0): # keystore creation failed raise Exception(Constants.MSG_ERROR_KEYSTORE.format(fwd_mode, "\n".join(create_results))) forwarder_dict = defaultdict(str) if (fwd_mode == Constants.METRICBEAT) or (fwd_mode == Constants.AUDITBEAT) or (fwd_mode == Constants.SYSLOGBEAT) or (fwd_mode == Constants.HEATBEAT): #### auditbeat/metricbeat/filebeat-syslog ################################################################### # enter beat configuration (in a few steps) if (fwd_mode == Constants.METRICBEAT): # interval is metricbeat only, the rest is used by both code, beat_interval = d.rangebox(f"{Constants.MSG_CONFIG_GENERIC.format(fwd_mode)} interval (seconds)", width=60, min=1, max=60, init=30) if (code == Dialog.CANCEL or code == Dialog.ESC): raise CancelledError forwarder_dict[Constants.BEAT_INTERVAL] = f"{beat_interval}s" # get elasticsearch/kibana connection information from user forwarder_dict.update(input_elasticsearch_connection_info(forwarder=fwd_mode, default_es_host=previous_config_values[Constants.BEAT_ES_HOST], default_es_port=previous_config_values[Constants.BEAT_ES_PORT], default_kibana_host=previous_config_values[Constants.BEAT_KIBANA_HOST], default_kibana_port=previous_config_values[Constants.BEAT_KIBANA_PORT], default_username=previous_config_values[Constants.BEAT_HTTP_USERNAME], default_password=previous_config_values[Constants.BEAT_HTTP_PASSWORD])) elif (fwd_mode == Constants.FILEBEAT): #### filebeat ############################################################################################# while True: forwarder_dict = defaultdict(str) # enter main filebeat configuration code, values = d.form(Constants.MSG_CONFIG_GENERIC.format(fwd_mode), [ ('Log Path', 1, 1, capture_config_dict["ZEEK_LOG_PATH"], 1, 20, 30, 255), ('Destination Host', 2, 1, "", 2, 20, 30, 255), ('Destination Port', 3, 1, "5044", 3, 20, 6, 5) ]) values = [x.strip() for x in values] if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError elif (len(values[0]) <= 0) or (not os.path.isdir(values[0])): code = d.msgbox(text=Constants.MSG_ERROR_DIR_NOT_FOUND) elif (len(values[1]) <= 0) or (len(values[2]) <= 0) or (not values[2].isnumeric()): code = d.msgbox(text=Constants.MSG_ERROR_BAD_HOST) else: forwarder_dict[Constants.BEAT_LOG_PATTERN_KEY] = os.path.join(os.path.join(values[0], Constants.BEAT_LOG_PATH_SUBDIR), Constants.BEAT_LOG_PATTERN_VAL) forwarder_dict[Constants.BEAT_LS_HOST] = values[1] forwarder_dict[Constants.BEAT_LS_PORT] = values[2] break # optionally, filebeat can use SSL if Logstash is configured for it logstash_ssl = "false" logstash_ssl_verify = "none" if (d.yesno("Forward Zeek logs over SSL? (Note: This requires the destination to be similarly configured and a corresponding copy of the client SSL files.)", yes_label="SSL", no_label="Unencrypted") == Dialog.OK): logstash_ssl = "true" if (d.yesno("Logstash SSL verification", yes_label="None", no_label="Force Peer") != Dialog.OK): logstash_ssl_verify = "force_peer" forwarder_dict[Constants.BEAT_LS_SSL] = logstash_ssl forwarder_dict[Constants.BEAT_LS_SSL_VERIFY] = logstash_ssl_verify if (forwarder_dict[Constants.BEAT_LS_SSL] == "true"): while True: code, values = d.form(Constants.MSG_CONFIG_GENERIC.format(fwd_mode), [ ('SSL Certificate Authorities File', 1, 1, "", 1, 35, 30, 255), ('SSL Certificate File', 2, 1, "", 2, 35, 30, 255), ('SSL Key File', 3, 1, "", 3, 35, 30, 255), ]) values = [x.strip() for x in values] if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError elif ((len(values[0]) <= 0) or (not os.path.isfile(values[0])) or (len(values[1]) <= 0) or (not os.path.isfile(values[1])) or (len(values[2]) <= 0) or (not os.path.isfile(values[2]))): code = d.msgbox(text=Constants.MSG_ERROR_FILE_NOT_FOUND) else: forwarder_dict[Constants.BEAT_LS_SSL_CA_CRT] = values[0] forwarder_dict[Constants.BEAT_LS_SSL_CLIENT_CRT] = values[1] forwarder_dict[Constants.BEAT_LS_SSL_CLIENT_KEY] = values[2] break else: forwarder_dict[Constants.BEAT_LS_SSL_CA_CRT] = "" forwarder_dict[Constants.BEAT_LS_SSL_CLIENT_CRT] = "" forwarder_dict[Constants.BEAT_LS_SSL_CLIENT_KEY] = "" # see if logstash port is open (not a great connection test, but better than nothing!) code = d.infobox(Constants.MSG_TESTING_CONNECTION.format("Logstash")) if not check_socket(forwarder_dict[Constants.BEAT_LS_HOST], int(forwarder_dict[Constants.BEAT_LS_PORT])): code = d.yesno(text=Constants.MSG_TESTING_CONNECTION_FAILURE_LOGSTASH.format("Logstash", forwarder_dict[Constants.BEAT_LS_HOST], forwarder_dict[Constants.BEAT_LS_PORT]), yes_label="Ignore Error", no_label="Start Over") if code != Dialog.OK: raise CancelledError # outside of filebeat/metricbeat if/else, get confirmation and write out the values to the keystore if forwarder_dict: # get confirmation of parameters before we pull the trigger code = d.yesno(Constants.MSG_CONFIG_FORWARDING_CONFIRM.format(fwd_mode, "\n".join(sorted([f"{k}={v}" for k, v in forwarder_dict.items() if "PASSWORD" not in k]))), yes_label="OK", no_label="Cancel") if code != Dialog.OK: raise CancelledError previous_config_values = forwarder_dict.copy() # it's go time, call keystore add for each item for k, v in sorted(forwarder_dict.items()): ecode, add_results = run_process(f"{Constants.BEAT_CMD[fwd_mode]} keystore add {k} --stdin --force", stdin=v, stderr=True) if (ecode != 0): # keystore creation failed raise Exception(Constants.MSG_ERROR_KEYSTORE.format(fwd_mode, "\n".join(add_results))) # get a final list of parameters that were set to show the user that stuff happened ecode, list_results = run_process(f"{Constants.BEAT_CMD[fwd_mode]} keystore list") if (ecode == 0): code = d.msgbox(text=Constants.MSG_CONFIG_FORWARDING_SUCCESS.format(fwd_mode, "\n".join(list_results))) else: # keystore list failed raise Exception(Constants.MSG_ERROR_KEYSTORE.format(fwd_mode, "\n".join(add_results))) else: # we got through the config but ended up with no values for configuration! raise Exception(Constants.MSG_MESSAGE_ERROR.format(Constants.MSG_EMPTY_CONFIG_ERROR)) except CancelledError as c: # d.msgbox(text=Constants.MSG_CANCEL_ERROR) # just start over continue except Exception as e: d.msgbox(text=Constants.MSG_MESSAGE_ERROR.format(e)) raise if __name__ == '__main__': main() clearquit()