1094 lines
43 KiB
Python
1094 lines
43 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
|
|
|
|
import clamd
|
|
import hashlib
|
|
import json
|
|
import malass_client
|
|
import os
|
|
import re
|
|
import requests
|
|
import sys
|
|
import time
|
|
import yara
|
|
import zmq
|
|
|
|
from abc import ABC, abstractmethod
|
|
from bs4 import BeautifulSoup
|
|
from collections import Counter
|
|
from collections import deque
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
from multiprocessing import RawValue
|
|
from subprocess import (PIPE, Popen)
|
|
from threading import get_ident
|
|
from threading import Lock
|
|
|
|
###################################################################################################
|
|
VENTILATOR_PORT = 5987
|
|
SINK_PORT = 5988
|
|
TOPIC_FILE_SCAN = "file"
|
|
|
|
###################################################################################################
|
|
# modes for file preservation settings
|
|
PRESERVE_QUARANTINED = "quarantined"
|
|
PRESERVE_ALL = "all"
|
|
PRESERVE_NONE = "none"
|
|
|
|
PRESERVE_QUARANTINED_DIR_NAME = "quarantine"
|
|
PRESERVE_PRESERVED_DIR_NAME = "preserved"
|
|
|
|
###################################################################################################
|
|
FILE_SCAN_RESULT_SCANNER = "scanner"
|
|
FILE_SCAN_RESULT_FILE = "file"
|
|
FILE_SCAN_RESULT_FILE_SIZE = "size"
|
|
FILE_SCAN_RESULT_FILE_TYPE = "type"
|
|
FILE_SCAN_RESULT_ENGINES = "engines"
|
|
FILE_SCAN_RESULT_HITS = "hits"
|
|
FILE_SCAN_RESULT_MESSAGE = "message"
|
|
FILE_SCAN_RESULT_DESCRIPTION = "description"
|
|
|
|
###################################################################################################
|
|
# the notice field for the signature.log we're writing out mimicing Zeek
|
|
ZEEK_SIGNATURE_NOTICE = "Signatures::Sensitive_Signature"
|
|
|
|
###################################################################################################
|
|
# VirusTotal public API
|
|
VTOT_MAX_REQS = 4 # maximum 4 public API requests (default)
|
|
VTOT_MAX_SEC = 60 # in 60 seconds (default)
|
|
VTOT_CHECK_INTERVAL = 0.05
|
|
VTOT_URL = 'https://www.virustotal.com/vtapi/v2/file/report'
|
|
VTOT_RESP_NOT_FOUND = 0
|
|
VTOT_RESP_FOUND = 1
|
|
VTOT_RESP_QUEUED = -2
|
|
|
|
###################################################################################################
|
|
# Malass web API
|
|
MAL_MAX_REQS = 20 # maximum scanning requests concurrently
|
|
MAL_END_OF_TRANSACTION = 'End_of_Transaction'
|
|
MAL_SUBMIT_TIMEOUT_SEC = 60
|
|
MAL_CHECK_INTERVAL = 1
|
|
MAL_RESP_NOT_FOUND = 0
|
|
MAL_RESP_FOUND = 1
|
|
MAL_RESP_QUEUED = -2
|
|
|
|
###################################################################################################
|
|
# ClamAV Interface
|
|
CLAM_MAX_REQS = 8 # maximum scanning requests concurrently, should be <= clamd.conf MaxThreads
|
|
CLAM_SUBMIT_TIMEOUT_SEC = 10
|
|
CLAM_CHECK_INTERVAL = 0.1
|
|
CLAM_ENGINE_ID = 'ClamAV'
|
|
CLAM_FOUND_KEY = 'FOUND'
|
|
|
|
###################################################################################################
|
|
# Yara Interface
|
|
YARA_RULES_DIR = os.path.join(os.getenv('YARA_RULES_DIR', "/yara-rules"), '')
|
|
YARA_CUSTOM_RULES_DIR = os.path.join(YARA_RULES_DIR, "custom")
|
|
YARA_SUBMIT_TIMEOUT_SEC = 60
|
|
YARA_ENGINE_ID = 'Yara'
|
|
YARA_MAX_REQS = 8 # maximum scanning threads concurrently
|
|
YARA_CHECK_INTERVAL = 0.1
|
|
YARA_RUN_TIMEOUT_SEC = 300
|
|
|
|
###################################################################################################
|
|
# Capa
|
|
CAPA_MAX_REQS = 4 # maximum scanning threads concurrently
|
|
CAPA_SUBMIT_TIMEOUT_SEC = 60
|
|
CAPA_ENGINE_ID = 'Capa'
|
|
CAPA_CHECK_INTERVAL = 0.1
|
|
CAPA_MIMES_TO_SCAN = ('application/bat', 'application/ecmascript', 'application/javascript', 'application/PowerShell', 'application/vnd.microsoft.portable-executable', 'application/x-bat', 'application/x-dosexec', 'application/x-executable', 'application/x-msdos-program', 'application/x-msdownload', 'application/x-pe-app-32bit-i386', 'application/x-sh', 'text/jscript', 'text/vbscript', 'text/x-python', 'text/x-shellscript')
|
|
CAPA_VIV_SUFFIX = '.viv'
|
|
CAPA_VIV_MIME = 'data'
|
|
CAPA_ATTACK_KEY = 'att&ck'
|
|
CAPA_RUN_TIMEOUT_SEC = 300
|
|
|
|
###################################################################################################
|
|
|
|
|
|
# a structure representing the fields of a line of Zeek's signatures.log, and the corresponding string formatting and type definitions
|
|
class BroSignatureLine:
|
|
__slots__ = ('ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'note', 'signature_id', 'event_message', 'sub_message', 'signature_count', 'host_count')
|
|
def __init__(self, ts='-', uid='-', orig_h='-', orig_p='-', resp_h='-', resp_p='-', note='-', signature_id='-', event_message='-', sub_message='-', signature_count='-', host_count='-'):
|
|
self.ts = ts
|
|
self.uid = uid
|
|
self.orig_h = orig_h
|
|
self.orig_p = orig_p
|
|
self.resp_h = resp_h
|
|
self.resp_p = resp_p
|
|
self.note = note
|
|
self.signature_id = signature_id
|
|
self.event_message = event_message
|
|
self.sub_message = sub_message
|
|
self.signature_count = signature_count
|
|
self.host_count = host_count
|
|
|
|
def __str__(self):
|
|
return "\t".join(map(str, [self.ts, self.uid, self.orig_h, self.orig_p, self.resp_h, self.resp_p, self.note, self.signature_id, self.event_message, self.sub_message, self.signature_count, self.host_count]))
|
|
|
|
@classmethod
|
|
def signature_format_line(cls):
|
|
return "\t".join(['{'+x+'}' for x in cls.__slots__])
|
|
|
|
@classmethod
|
|
def signature_types_line(cls):
|
|
return "\t".join(['time', 'string', 'addr', 'port', 'addr', 'port', 'enum', 'string', 'string', 'string', 'count', 'count'])
|
|
|
|
# AnalyzerScan
|
|
# .provider - a FileScanProvider subclass doing the scan/lookup
|
|
# .name - the filename to be scanned
|
|
# .size - the size (in bytes) of the file
|
|
# .fileType - the file's mime type
|
|
# .submissionResponse - a unique identifier to be returned by the provider with which to check status
|
|
class AnalyzerScan:
|
|
__slots__ = ('provider', 'name', 'size', 'fileType', 'submissionResponse')
|
|
def __init__(self, provider=None, name=None, size=None, fileType=None, submissionResponse=None):
|
|
self.provider = provider
|
|
self.name = name
|
|
self.size = size
|
|
self.fileType = fileType
|
|
self.submissionResponse = submissionResponse
|
|
|
|
# AnalyzerResult
|
|
# .finished - the scan/lookup is no longer executing (whether or not it was successful or returned a "match")
|
|
# .success - requesting the status was done successfully (whether or not it was finished)
|
|
# .result - the "result" of the scan/lookup, in whatever format is native to the provider
|
|
class AnalyzerResult:
|
|
__slots__ = ('finished', 'success', 'verbose', 'result')
|
|
def __init__(self, finished=False, success=False, verbose=False, result=None):
|
|
self.finished = finished
|
|
self.success = success
|
|
self.verbose = verbose
|
|
self.result = result
|
|
|
|
# the filename parts used by our Zeek instance for extracted files:
|
|
# source-fuid-uid-time.ext, eg., SSL-FTnzwn4hEPJi7BfzRk-CsRaviydrGyYROuX3-20190402105425.crt
|
|
class ExtractedFileNameParts:
|
|
__slots__ = ('source', 'fid', 'uid', 'time', 'ext')
|
|
def __init__(self, source=None, fid=None, uid=None, time=None, ext=None):
|
|
self.source = source
|
|
self.fid = fid
|
|
self.uid = uid
|
|
self.time = time
|
|
self.ext = ext
|
|
|
|
###################################################################################################
|
|
# convenient boolean argument parsing
|
|
def str2bool(v):
|
|
if v.lower() in ('yes', 'true', 't', 'y', '1'):
|
|
return True
|
|
elif v.lower() in ('no', 'false', 'f', 'n', '0'):
|
|
return False
|
|
else:
|
|
raise argparse.ArgumentTypeError('Boolean value expected.')
|
|
|
|
###################################################################################################
|
|
# print to stderr
|
|
def eprint(*args, **kwargs):
|
|
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), *args, file=sys.stderr, **kwargs)
|
|
|
|
###################################################################################################
|
|
# calculate a sha256 hash of a file
|
|
def sha256sum(filename):
|
|
h = hashlib.sha256()
|
|
b = bytearray(64 * 1024)
|
|
mv = memoryview(b)
|
|
with open(filename, 'rb', buffering=0) as f:
|
|
for n in iter(lambda : f.readinto(mv), 0):
|
|
h.update(mv[:n])
|
|
return h.hexdigest()
|
|
|
|
###################################################################################################
|
|
# recursive dictionary key search
|
|
def dictsearch(d, target):
|
|
val = filter(None, [[b] if a == target else dictsearch(b, target) if isinstance(b, dict) else None for a, b in d.items()])
|
|
return [i for b in val for i in b]
|
|
|
|
###################################################################################################
|
|
# filespec to various fields as per the extractor zeek script (/opt/zeek/share/zeek/site/extractor.zeek)
|
|
# source-fuid-uid-time.ext
|
|
# eg.
|
|
# SSL-FTnzwn4hEPJi7BfzRk-CsRaviydrGyYROuX3-20190402105425.crt
|
|
#
|
|
# there are other extracted files that come from the mitre-attack/bzar scripts, they are formatted like this:
|
|
# local fname = fmt("%s_%s%s", c$uid, f$id, subst_string(smb_name, "\\", "_"));
|
|
#
|
|
# CR7X4q2hmcXKqP0vVj_F3jZ2VjYttqhKaGfh__172.16.1.8_C$_WINDOWS_sny4u_un1zbd94ytwj99hcymmsad7j54gr4wdskwnqs0ki252jdsrf763zsm531b.exe
|
|
# └----------------┘ └---------------┘└------------------------------------------------------------------------------------------┘
|
|
# UID FID subst_string(smb_name, "\\", "_"))
|
|
#
|
|
# (see https://github.com/mitre-attack/bzar/blob/master/scripts/bzar_files.zeek#L50)
|
|
def extracted_filespec_to_fields(filespec):
|
|
baseFileSpec = os.path.basename(filespec)
|
|
match = re.search(r'^(?P<source>.*)-(?P<fid>.*)-(?P<uid>.*)-(?P<time>\d+)\.(?P<ext>.*?)$', baseFileSpec)
|
|
if match is not None:
|
|
result = ExtractedFileNameParts(match.group('source'), match.group('fid'), match.group('uid'),
|
|
time.mktime(datetime.strptime(match.group('time'), "%Y%m%d%H%M%S").timetuple()),
|
|
match.group('ext'))
|
|
else:
|
|
match = re.search(r'^(?P<uid>[0-9a-zA-Z]+)_(?P<fid>[0-9a-zA-Z]+).+\.(?P<ext>.*?)$', baseFileSpec)
|
|
if match is not None:
|
|
result = ExtractedFileNameParts('MITRE', match.group('fid'), match.group('uid'), time.time(), match.group('ext'))
|
|
else:
|
|
result = ExtractedFileNameParts(None, None, None, time.time(), None)
|
|
|
|
return result
|
|
|
|
###################################################################################################
|
|
# open a file and close it, updating its access time
|
|
def touch(filename):
|
|
open(filename, 'a').close()
|
|
os.utime(filename, None)
|
|
|
|
###################################################################################################
|
|
# run command with arguments and return its exit code, stdout, and stderr
|
|
def check_output_input(*popenargs, **kwargs):
|
|
|
|
if 'stdout' in kwargs:
|
|
raise ValueError('stdout argument not allowed, it will be overridden')
|
|
|
|
if 'stderr' in kwargs:
|
|
raise ValueError('stderr argument not allowed, it will be overridden')
|
|
|
|
if 'input' in kwargs and kwargs['input']:
|
|
if 'stdin' in kwargs:
|
|
raise ValueError('stdin and input arguments may not both be used')
|
|
inputdata = kwargs['input']
|
|
kwargs['stdin'] = PIPE
|
|
else:
|
|
inputdata = None
|
|
kwargs.pop('input', None)
|
|
|
|
process = Popen(*popenargs, stdout=PIPE, stderr=PIPE, **kwargs)
|
|
try:
|
|
output, errput = process.communicate(input=inputdata)
|
|
except:
|
|
process.kill()
|
|
process.wait()
|
|
raise
|
|
|
|
retcode = process.poll()
|
|
|
|
return retcode, output, errput
|
|
|
|
###################################################################################################
|
|
# run command with arguments and return its exit code and output
|
|
def run_process(command, stdout=True, stderr=True, stdin=None, cwd=None, env=None, debug=False):
|
|
|
|
retcode = -1
|
|
output = []
|
|
|
|
try:
|
|
# run the command
|
|
retcode, cmdout, cmderr = check_output_input(command, input=stdin.encode() if stdin else None, cwd=cwd, env=env)
|
|
|
|
# split the output on newlines to return a list
|
|
if stderr and (len(cmderr) > 0): output.extend(cmderr.decode(sys.getdefaultencoding()).split('\n'))
|
|
if stdout and (len(cmdout) > 0): output.extend(cmdout.decode(sys.getdefaultencoding()).split('\n'))
|
|
|
|
except (FileNotFoundError, OSError, IOError) as e:
|
|
if stderr:
|
|
output.append("Command {} not found or unable to execute".format(command))
|
|
|
|
if debug:
|
|
eprint("{}{} returned {}: {}".format(command, "({})".format(stdin[:80] + bool(stdin[80:]) * '...' if stdin else ""), retcode, output))
|
|
|
|
return retcode, output
|
|
|
|
###################################################################################################
|
|
class AtomicInt:
|
|
def __init__(self, value=0):
|
|
self.val = RawValue('i', value)
|
|
self.lock = Lock()
|
|
|
|
def increment(self):
|
|
with self.lock:
|
|
self.val.value += 1
|
|
return self.val.value
|
|
|
|
def decrement(self):
|
|
with self.lock:
|
|
self.val.value -= 1
|
|
return self.val.value
|
|
|
|
def value(self):
|
|
with self.lock:
|
|
return self.val.value
|
|
|
|
###################################################################################################
|
|
class CarvedFileSubscriberThreaded:
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# constructor
|
|
def __init__(self, debug=False, verboseDebug=False, host="localhost", port=VENTILATOR_PORT, context=None, topic='', rcvTimeout=5000, scriptName=''):
|
|
self.debug = debug
|
|
self.verboseDebug = verboseDebug
|
|
self.scriptName = scriptName
|
|
|
|
self.lock = Lock()
|
|
|
|
# initialize ZeroMQ context and socket(s) to receive filenames and send scan results
|
|
self.context = context if (context is not None) else zmq.Context()
|
|
|
|
# Socket to receive messages on
|
|
self.newFilesSocket = self.context.socket(zmq.SUB)
|
|
self.newFilesSocket.connect(f"tcp://{host}:{port}")
|
|
self.newFilesSocket.setsockopt(zmq.SUBSCRIBE, bytes(topic, encoding='ascii'))
|
|
self.newFilesSocket.RCVTIMEO = rcvTimeout
|
|
if self.debug: eprint(f"{self.scriptName}:\tbound to ventilator at {port}")
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
def Pull(self, scanWorkerId=0):
|
|
|
|
fileinfo = defaultdict(str)
|
|
|
|
with self.lock:
|
|
# accept a fileinfo dict from newFilesSocket
|
|
try:
|
|
fileinfo.update(json.loads(self.newFilesSocket.recv_string()))
|
|
except zmq.Again as timeout:
|
|
# no file received due to timeout, return empty dict. which means no file available
|
|
pass
|
|
|
|
if self.verboseDebug:
|
|
eprint(f"{self.scriptName}[{scanWorkerId}]:\t{'📨' if (FILE_SCAN_RESULT_FILE in fileinfo) else '🕑'}\t{fileinfo[FILE_SCAN_RESULT_FILE] if (FILE_SCAN_RESULT_FILE in fileinfo) else '(recv)'}")
|
|
|
|
return fileinfo
|
|
|
|
###################################################################################################
|
|
class FileScanProvider(ABC):
|
|
|
|
@staticmethod
|
|
@abstractmethod
|
|
def scanner_name(cls):
|
|
# returns this scanner name
|
|
pass
|
|
|
|
@abstractmethod
|
|
def max_requests(self):
|
|
# returns the maximum number of concurrently open requests this type of provider can handle
|
|
pass
|
|
|
|
@staticmethod
|
|
@abstractmethod
|
|
def check_interval(cls):
|
|
# returns the amount of seconds you should sleep between checking for results
|
|
pass
|
|
|
|
@abstractmethod
|
|
def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=0):
|
|
# returns something that can be passed into check_result for checking the scan status
|
|
pass
|
|
|
|
@abstractmethod
|
|
def check_result(self, submissionResponse):
|
|
# returns AnalyzerResult based on submissionResponse
|
|
pass
|
|
|
|
@staticmethod
|
|
@abstractmethod
|
|
def format(cls, fileName, response):
|
|
# returns result dict based on response (see FILE_SCAN_RESULT_* above)
|
|
pass
|
|
|
|
###################################################################################################
|
|
# class for searching for a hash with a VirusTotal public API, handling rate limiting
|
|
class VirusTotalSearch(FileScanProvider):
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# constructor
|
|
def __init__(self, apiKey, reqLimit=None, reqLimitSec=None):
|
|
self.apiKey = apiKey
|
|
self.lock = Lock()
|
|
self.history = deque()
|
|
self.reqLimit = reqLimit if reqLimit else VTOT_MAX_REQS
|
|
self.reqLimitSec = reqLimitSec if reqLimitSec else VTOT_MAX_SEC
|
|
|
|
@staticmethod
|
|
def scanner_name():
|
|
return 'virustotal'
|
|
|
|
def max_requests(self):
|
|
return self.reqLimit
|
|
|
|
@staticmethod
|
|
def check_interval():
|
|
return VTOT_CHECK_INTERVAL
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# do a hash lookup against VirusTotal, respecting rate limiting
|
|
# VirusTotalSearch does the request and gets the response immediately;
|
|
# the subsequent call to check_result (using submit's response as input)
|
|
# will always return "True" since the work has already been done
|
|
def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=0):
|
|
if timeout is None:
|
|
timeout = self.reqLimitSec+5
|
|
|
|
allowed = False
|
|
response = None
|
|
|
|
# timeout only applies if block=True
|
|
timeoutTime = int(time.time()) + timeout
|
|
|
|
# while limit only repeats if block=True
|
|
while (not allowed) and (response is None):
|
|
|
|
with self.lock:
|
|
# first make sure we haven't exceeded rate limits
|
|
nowTime = int(time.time())
|
|
|
|
if (len(self.history) < self.reqLimit):
|
|
# we've done fewer than the allowed requests, so assume we're good to go
|
|
self.history.append(nowTime + self.reqLimitSec)
|
|
allowed = True
|
|
|
|
elif (self.history[0] < nowTime):
|
|
# we've done more than the allowed requests, but the oldest one is older than the window
|
|
_ = self.history.popleft()
|
|
self.history.append(nowTime + self.reqLimitSec)
|
|
allowed = True
|
|
|
|
if allowed:
|
|
try:
|
|
response = requests.get(VTOT_URL, params={ 'apikey': self.apiKey, 'resource': sha256sum(fileName) })
|
|
except requests.exceptions.RequestException as e:
|
|
# things are bad
|
|
return None
|
|
|
|
elif block and (nowTime < timeoutTime):
|
|
# rate limited, wait for a bit and come around and try again
|
|
time.sleep(1)
|
|
|
|
else:
|
|
break
|
|
|
|
return response
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# see comment for VirusTotalSearch.submit, the work has already been done
|
|
def check_result(self, submissionResponse):
|
|
result = AnalyzerResult(finished=True)
|
|
|
|
if submissionResponse is not None:
|
|
try:
|
|
result.success = submissionResponse.ok
|
|
except:
|
|
pass
|
|
|
|
try:
|
|
result.result = submissionResponse.json()
|
|
except (ValueError, TypeError):
|
|
result.success = False
|
|
|
|
return result
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# static method for formatting the response JSON (from requests.get) as a dict
|
|
@staticmethod
|
|
def format(fileName, response):
|
|
result = {FILE_SCAN_RESULT_SCANNER : VirusTotalSearch.scanner_name(),
|
|
FILE_SCAN_RESULT_FILE : fileName,
|
|
FILE_SCAN_RESULT_ENGINES : 0,
|
|
FILE_SCAN_RESULT_HITS : 0,
|
|
FILE_SCAN_RESULT_MESSAGE : None,
|
|
FILE_SCAN_RESULT_DESCRIPTION : None}
|
|
|
|
if isinstance(response, AnalyzerResult):
|
|
resp = response.result
|
|
else:
|
|
resp = response
|
|
|
|
if isinstance(resp, str):
|
|
try:
|
|
resp = json.loads(resp)
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
# see https://www.virustotal.com/en/documentation/public-api/
|
|
if isinstance(resp, dict):
|
|
if 'response_code' in resp:
|
|
if (resp['response_code'] == VTOT_RESP_FOUND) and ('positives' in resp) and (resp['positives'] >0):
|
|
result[FILE_SCAN_RESULT_HITS] = resp['positives']
|
|
if ('scans' in resp):
|
|
result[FILE_SCAN_RESULT_ENGINES] = len(resp['scans'])
|
|
scans = {engine:resp['scans'][engine] for engine in resp['scans'] if ('detected' in resp['scans'][engine]) and (resp['scans'][engine]['detected'] == True)}
|
|
hits = defaultdict(list)
|
|
for k, v in scans.items():
|
|
hits[v['result'] if 'result' in v else 'unknown'].append(k)
|
|
if (len(hits) > 0):
|
|
# short result is most common signature name
|
|
result[FILE_SCAN_RESULT_MESSAGE] = max(hits, key= lambda x: len(set(hits[x])))
|
|
# long result is list of the signature names and the engines which generated them
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = ";".join([f"{k}<{','.join(v)}>" for k, v in hits.items()])
|
|
else:
|
|
# we were reported positives, but no no details
|
|
result[FILE_SCAN_RESULT_MESSAGE] = "VirusTotal reported signature matches"
|
|
if 'permalink' in resp:
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = resp['permalink']
|
|
else:
|
|
# this shouldn't have happened after our checking above, so I guess just return the string
|
|
# and let the caller deal with it
|
|
result[FILE_SCAN_RESULT_MESSAGE] = "Invalid response"
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}"
|
|
|
|
return result
|
|
|
|
###################################################################################################
|
|
# class for scanning a file with Malass
|
|
class MalassScan(FileScanProvider):
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# constructor
|
|
def __init__(self, host, port, reqLimit=None):
|
|
self.host = host
|
|
self.port = port
|
|
self.reqLimit = reqLimit if reqLimit else MAL_MAX_REQS
|
|
self.transactionIdToFilenameDict = defaultdict(str)
|
|
self.scanningFilesCount = AtomicInt(value=0)
|
|
|
|
@staticmethod
|
|
def scanner_name():
|
|
return 'malass'
|
|
|
|
def max_requests(self):
|
|
return self.reqLimit
|
|
|
|
@staticmethod
|
|
def check_interval():
|
|
return MAL_CHECK_INTERVAL
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# upload a file to scan with Malass, respecting rate limiting. return submitted transaction ID
|
|
def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=MAL_SUBMIT_TIMEOUT_SEC):
|
|
submittedTransactionId = None
|
|
allowed = False
|
|
|
|
# timeout only applies if block=True
|
|
timeoutTime = int(time.time()) + timeout
|
|
|
|
# while limit only repeats if block=True
|
|
while (not allowed) and (submittedTransactionId is None):
|
|
|
|
# first make sure we haven't exceeded rate limits
|
|
if (self.scanningFilesCount.increment() <= self.reqLimit):
|
|
# we've got fewer than the allowed requests open, so we're good to go!
|
|
allowed = True
|
|
else:
|
|
self.scanningFilesCount.decrement()
|
|
|
|
if allowed:
|
|
# submit the sample for scanning
|
|
success, transactionId, httpResponsePage = malass_client.upload_file_to_malass(fileName, self.host, self.port)
|
|
if success:
|
|
submittedTransactionId = transactionId
|
|
self.transactionIdToFilenameDict[submittedTransactionId] = os.path.basename(fileName)
|
|
|
|
elif block and (nowTime < timeoutTime):
|
|
# rate limited, wait for a bit and come around and try again
|
|
time.sleep(1)
|
|
|
|
else:
|
|
break
|
|
|
|
return submittedTransactionId
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# check the status of a previously submitted file
|
|
def check_result(self, transactionId):
|
|
|
|
result = AnalyzerResult()
|
|
|
|
# make a nice dictionary of this AV report
|
|
summaryDict = dict()
|
|
finishedAvsDict = dict()
|
|
summaryDict['complete'] = False
|
|
summaryDict['error'] = ""
|
|
|
|
filename = self.transactionIdToFilenameDict[transactionId]
|
|
|
|
try:
|
|
success, errMsg, avSummaryStr = malass_client.query_av_summary_rpt(transactionId, filename, self.host, self.port)
|
|
if success:
|
|
|
|
# get body text
|
|
body = BeautifulSoup(avSummaryStr, "html.parser").find("body")
|
|
if body is not None:
|
|
result.success = True
|
|
|
|
lines = body.text.split('\n')
|
|
|
|
# see if analysis is complete (look for termination string, then truncate the list starting at MAL_END_OF_TRANSACTION, inclusive)
|
|
eotIndices = [i for i, s in enumerate(lines) if MAL_END_OF_TRANSACTION in s]
|
|
summaryDict['complete'] = (len(eotIndices) > 0)
|
|
if summaryDict['complete']:
|
|
del lines[eotIndices[0]:]
|
|
|
|
# take report name/value pairs (minus AV results) and insert them into summaryDict
|
|
try:
|
|
summaryDict.update(dict(map(str, x[1:].split('=')) for x in lines if x.startswith('#')))
|
|
except (ValueError, TypeError) as e:
|
|
summaryDict['error'] = f"Report parse error: {str(e)}"
|
|
summaryDict['complete'] = True # avoid future lookups, abandon submission
|
|
|
|
# take AV results in this report and merge them into summaryDict
|
|
summaryDict['av'] = {}
|
|
for vmLine in [x for x in lines if x.startswith('av_vm_')]:
|
|
avDict = dict(map(str, x.split('=')) for x in vmLine.split(","))
|
|
if ('av_vm_name' in avDict) and (len(avDict['av_vm_name']) > 0):
|
|
summaryDict['av'][avDict['av_vm_name']] = avDict
|
|
|
|
# merge any new av results in this response into the final finishedAvsDict
|
|
for avName, avEntry in summaryDict['av'].items():
|
|
if ('av_vm_num' in avEntry) and (avEntry['av_vm_num'].isnumeric()) and (not (int(avEntry['av_vm_num']) in finishedAvsDict)):
|
|
finishedAvsDict[int(avEntry['av_vm_num'])] = avName
|
|
|
|
# are we done?
|
|
if summaryDict['complete']:
|
|
|
|
# yes, we are done! let's make sure at least one AV scanned, and report an error if not
|
|
if (len(finishedAvsDict) == 0) and (len(summaryDict['error']) == 0):
|
|
summaryDict['error'] = f"No AVs scanned file sample ({transactionId}/{filename})"
|
|
|
|
else:
|
|
summaryDict['error'] = f"Summary report contained no body ({transactionId}/{filename})"
|
|
summaryDict['complete'] = True # avoid future lookups, abandon submission
|
|
|
|
else:
|
|
summaryDict['error'] = f"Summary report was not generated: {errMsg} ({transactionId}/{filename})"
|
|
summaryDict['complete'] = True # avoid future lookups, abandon submission
|
|
|
|
finally:
|
|
if (transactionId is not None) and summaryDict['complete']:
|
|
# decrement scanning counter and remove trans->filename mapping if this scan is complete
|
|
self.scanningFilesCount.decrement()
|
|
self.transactionIdToFilenameDict.pop(transactionId, None)
|
|
|
|
result.finished = summaryDict['complete']
|
|
result.result = summaryDict
|
|
|
|
return result
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# static method for formatting the response summaryDict (from check_result)
|
|
@staticmethod
|
|
def format(fileName, response):
|
|
result = {FILE_SCAN_RESULT_SCANNER : MalassScan.scanner_name(),
|
|
FILE_SCAN_RESULT_FILE : fileName,
|
|
FILE_SCAN_RESULT_ENGINES : 0,
|
|
FILE_SCAN_RESULT_HITS : 0,
|
|
FILE_SCAN_RESULT_MESSAGE : None,
|
|
FILE_SCAN_RESULT_DESCRIPTION : None}
|
|
|
|
if isinstance(response, AnalyzerResult):
|
|
resp = response.result
|
|
else:
|
|
resp = response
|
|
|
|
if isinstance(resp, dict) and ('av' in resp) and (len(resp['av']) > 0):
|
|
hitAvs = {k : v for k, v in resp['av'].items() if ('contains_a_virus' in resp['av'][k]) and (resp['av'][k]['contains_a_virus'].lower() == "yes")}
|
|
result[FILE_SCAN_RESULT_HITS] = len(hitAvs)
|
|
result[FILE_SCAN_RESULT_ENGINES] = len(resp['av'])
|
|
hits = defaultdict(list)
|
|
for k, v in hitAvs.items():
|
|
hits[v['virus_name'] if 'virus_name' in v else 'unknown'].append(k)
|
|
if (len(hits) > 0):
|
|
# short result is most common signature name
|
|
result[FILE_SCAN_RESULT_MESSAGE] = max(hits, key= lambda x: len(set(hits[x])))
|
|
# long result is list of the signature names and the engines which generated them
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = ";".join([f"{k}<{','.join(v)}>" for k, v in hits.items()])
|
|
|
|
else:
|
|
result[FILE_SCAN_RESULT_MESSAGE] = "Error or invalid response"
|
|
if isinstance(resp, dict) and ('error' in resp):
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp['error']}"
|
|
else:
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}"
|
|
|
|
return result
|
|
|
|
###################################################################################################
|
|
# class for scanning a file with ClamAV
|
|
class ClamAVScan(FileScanProvider):
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# constructor
|
|
def __init__(self, debug=False, verboseDebug=False, socketFileName=None, reqLimit=None):
|
|
self.scanningFilesCount = AtomicInt(value=0)
|
|
self.debug = debug
|
|
self.verboseDebug = verboseDebug
|
|
self.socketFileName = socketFileName
|
|
self.reqLimit = reqLimit if reqLimit else CLAM_MAX_REQS
|
|
|
|
@staticmethod
|
|
def scanner_name():
|
|
return 'clamav'
|
|
|
|
def max_requests(self):
|
|
return self.reqLimit
|
|
|
|
@staticmethod
|
|
def check_interval():
|
|
return CLAM_CHECK_INTERVAL
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# submit a file to scan with ClamAV, respecting rate limiting. return scan result
|
|
def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=CLAM_SUBMIT_TIMEOUT_SEC):
|
|
clamavResult = AnalyzerResult()
|
|
allowed = False
|
|
connected = False
|
|
|
|
# timeout only applies if block=True
|
|
timeoutTime = int(time.time()) + timeout
|
|
|
|
# while limit only repeats if block=True
|
|
while (not allowed) and (not clamavResult.finished):
|
|
|
|
if not connected:
|
|
if self.verboseDebug: eprint(f"{get_ident()}: ClamAV attempting connection")
|
|
clamAv = clamd.ClamdUnixSocket(path=self.socketFileName) if self.socketFileName is not None else clamd.ClamdUnixSocket()
|
|
try:
|
|
clamAv.ping()
|
|
connected = True
|
|
if self.verboseDebug: eprint(f"{get_ident()}: ClamAV connected!")
|
|
except Exception as e:
|
|
connected = False
|
|
if self.debug: eprint(f"{get_ident()}: ClamAV connection failed: {str(e)}")
|
|
|
|
if connected:
|
|
# first make sure we haven't exceeded rate limits
|
|
if (self.scanningFilesCount.increment() <= self.reqLimit):
|
|
# we've got fewer than the allowed requests open, so we're good to go!
|
|
allowed = True
|
|
else:
|
|
self.scanningFilesCount.decrement()
|
|
|
|
if connected and allowed:
|
|
try:
|
|
if self.verboseDebug: eprint(f'{get_ident()} ClamAV scanning: {fileName}')
|
|
clamavResult.result = clamAv.scan(fileName)
|
|
if self.verboseDebug: eprint(f'{get_ident()} ClamAV scan result: {clamavResult.result}')
|
|
clamavResult.success = (clamavResult.result is not None)
|
|
clamavResult.finished = True
|
|
except Exception as e:
|
|
if clamavResult.result is None:
|
|
clamavResult.result = str(e)
|
|
if self.debug: eprint(f'{get_ident()} ClamAV scan error: {clamavResult.result}')
|
|
finally:
|
|
self.scanningFilesCount.decrement()
|
|
|
|
elif block and (nowTime < timeoutTime):
|
|
# rate limited, wait for a bit and come around and try again
|
|
time.sleep(1)
|
|
|
|
else:
|
|
break
|
|
|
|
return clamavResult
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# return the result of the previously scanned file
|
|
def check_result(self, clamavResult):
|
|
return clamavResult if isinstance(clamavResult, AnalyzerResult) else AnalyzerResult(finished=True, success=False, result=None)
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# static method for formatting the response summaryDict (from check_result)
|
|
@staticmethod
|
|
def format(fileName, response):
|
|
result = {FILE_SCAN_RESULT_SCANNER : ClamAVScan.scanner_name(),
|
|
FILE_SCAN_RESULT_FILE : fileName,
|
|
FILE_SCAN_RESULT_ENGINES : 1,
|
|
FILE_SCAN_RESULT_HITS : 0,
|
|
FILE_SCAN_RESULT_MESSAGE : None,
|
|
FILE_SCAN_RESULT_DESCRIPTION : None}
|
|
|
|
if isinstance(response, AnalyzerResult):
|
|
resp = response.result
|
|
else:
|
|
resp = response
|
|
|
|
if isinstance(resp, dict):
|
|
hits = []
|
|
for filename, resultTuple in resp.items():
|
|
if (len(resultTuple) == 2) and (resultTuple[0] == CLAM_FOUND_KEY):
|
|
hits.append(resultTuple[1])
|
|
result[FILE_SCAN_RESULT_HITS] = len(hits)
|
|
if (len(hits) > 0):
|
|
cnt = Counter(hits)
|
|
# short message is most common signature name
|
|
result[FILE_SCAN_RESULT_MESSAGE] = cnt.most_common(1)[0][0]
|
|
# long description is list of the signature names and the engines which generated them
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = ";".join([f"{x}<{CLAM_ENGINE_ID}>" for x in hits])
|
|
|
|
else:
|
|
result[FILE_SCAN_RESULT_MESSAGE] = "Error or invalid response"
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}"
|
|
|
|
return result
|
|
|
|
###################################################################################################
|
|
# class for scanning a file with Yara
|
|
class YaraScan(FileScanProvider):
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# constructor
|
|
def __init__(self, debug=False, verboseDebug=False, rulesDirs=[], reqLimit=None):
|
|
self.scanningFilesCount = AtomicInt(value=0)
|
|
self.debug = debug
|
|
self.verboseDebug = verboseDebug
|
|
self.reqLimit = reqLimit if reqLimit else YARA_MAX_REQS
|
|
self.ruleFilespecs = {}
|
|
for yaraDir in rulesDirs:
|
|
for root, dirs, files in os.walk(yaraDir):
|
|
for file in files:
|
|
# skip hidden, backup or system related files
|
|
if file.startswith(".") or file.startswith("~") or file.startswith("_"):
|
|
continue
|
|
filename = os.path.join(root, file)
|
|
extension = os.path.splitext(file)[1].lower()
|
|
try:
|
|
testCompile = yara.compile(filename)
|
|
self.ruleFilespecs[filename] = filename
|
|
except yara.SyntaxError as e:
|
|
if self.debug: eprint(f'{get_ident()} Ignored Yara compile error in {filename}: {e}')
|
|
if self.verboseDebug:
|
|
eprint(f"{get_ident()}: Initializing Yara with {len(self.ruleFilespecs)} rules files: {self.ruleFilespecs}")
|
|
elif self.debug:
|
|
eprint(f"{get_ident()}: Initializing Yara with {len(self.ruleFilespecs)} rules files")
|
|
self.compiledRules = yara.compile(filepaths = self.ruleFilespecs)
|
|
|
|
@staticmethod
|
|
def scanner_name():
|
|
return 'yara'
|
|
|
|
def max_requests(self):
|
|
return self.reqLimit
|
|
|
|
@staticmethod
|
|
def check_interval():
|
|
return YARA_CHECK_INTERVAL
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# submit a file to scan with Yara, respecting rate limiting. return scan result
|
|
def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=YARA_SUBMIT_TIMEOUT_SEC):
|
|
yaraResult = AnalyzerResult()
|
|
allowed = False
|
|
matches = []
|
|
|
|
# timeout only applies if block=True
|
|
timeoutTime = int(time.time()) + timeout
|
|
|
|
# while limit only repeats if block=True
|
|
while (not allowed) and (not yaraResult.finished):
|
|
|
|
# first make sure we haven't exceeded rate limits
|
|
if (self.scanningFilesCount.increment() <= self.reqLimit):
|
|
# we've got fewer than the allowed requests open, so we're good to go!
|
|
allowed = True
|
|
else:
|
|
self.scanningFilesCount.decrement()
|
|
|
|
if allowed:
|
|
try:
|
|
if self.verboseDebug: eprint(f'{get_ident()} Yara scanning: {fileName}')
|
|
yaraResult.result = self.compiledRules.match(fileName, timeout=YARA_RUN_TIMEOUT_SEC)
|
|
if self.verboseDebug: eprint(f'{get_ident()} Yara scan result: {yaraResult.result}')
|
|
yaraResult.success = (yaraResult.result is not None)
|
|
yaraResult.finished = True
|
|
except Exception as e:
|
|
if yaraResult.result is None:
|
|
yaraResult.result = {"error" : str(e)}
|
|
yaraResult.success = False
|
|
yaraResult.finished = True
|
|
if self.debug: eprint(f'{get_ident()} Yara scan error: {yaraResult.result}')
|
|
finally:
|
|
self.scanningFilesCount.decrement()
|
|
|
|
elif block and (nowTime < timeoutTime):
|
|
# rate limited, wait for a bit and come around and try again
|
|
time.sleep(1)
|
|
|
|
else:
|
|
break
|
|
|
|
return yaraResult
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# return the result of the previously scanned file
|
|
def check_result(self, yaraResult):
|
|
return yaraResult if isinstance(yaraResult, AnalyzerResult) else AnalyzerResult(finished=True, success=False, result=None)
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# static method for formatting the response summaryDict (from check_result)
|
|
@staticmethod
|
|
def format(fileName, response):
|
|
result = {FILE_SCAN_RESULT_SCANNER : YaraScan.scanner_name(),
|
|
FILE_SCAN_RESULT_FILE : fileName,
|
|
FILE_SCAN_RESULT_ENGINES : 1,
|
|
FILE_SCAN_RESULT_HITS : 0,
|
|
FILE_SCAN_RESULT_MESSAGE : None,
|
|
FILE_SCAN_RESULT_DESCRIPTION : None}
|
|
|
|
if isinstance(response, AnalyzerResult):
|
|
resp = response.result
|
|
else:
|
|
resp = response
|
|
|
|
if isinstance(resp, list):
|
|
hits = [match.rule for match in resp if isinstance(match, yara.Match)]
|
|
result[FILE_SCAN_RESULT_HITS] = len(hits)
|
|
if (len(hits) > 0):
|
|
cnt = Counter(hits)
|
|
# short message is most common signature name (todo: they won't have duplicate names, so I guess this is just going to take the first...)
|
|
result[FILE_SCAN_RESULT_MESSAGE] = cnt.most_common(1)[0][0]
|
|
# long description is list of the signature names and the engines which generated them
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = ";".join([f"{x}<{YARA_ENGINE_ID}>" for x in hits])
|
|
|
|
else:
|
|
result[FILE_SCAN_RESULT_MESSAGE] = "Error or invalid response"
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}"
|
|
|
|
return result
|
|
|
|
###################################################################################################
|
|
# class for scanning a file with Capa
|
|
class CapaScan(FileScanProvider):
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# constructor
|
|
def __init__(self, debug=False, verboseDebug=False, rulesDir=None, verboseHits=False, reqLimit=None):
|
|
self.scanningFilesCount = AtomicInt(value=0)
|
|
self.rulesDir = rulesDir
|
|
self.debug = debug
|
|
self.verboseDebug = verboseDebug
|
|
self.verboseHits = verboseHits
|
|
self.reqLimit = reqLimit if reqLimit else CAPA_MAX_REQS
|
|
|
|
@staticmethod
|
|
def scanner_name():
|
|
return 'capa'
|
|
|
|
def max_requests(self):
|
|
return self.reqLimit
|
|
|
|
@staticmethod
|
|
def check_interval():
|
|
return CAPA_CHECK_INTERVAL
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# submit a file to scan with Capa, respecting rate limiting. return scan result
|
|
def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=CAPA_SUBMIT_TIMEOUT_SEC):
|
|
capaResult = AnalyzerResult(verbose=self.verboseHits)
|
|
|
|
if (fileType is not None) and (fileType in CAPA_MIMES_TO_SCAN):
|
|
allowed = False
|
|
|
|
# timeout only applies if block=True
|
|
timeoutTime = int(time.time()) + timeout
|
|
|
|
# while limit only repeats if block=True
|
|
while (not allowed) and (not capaResult.finished):
|
|
|
|
# first make sure we haven't exceeded rate limits
|
|
if (self.scanningFilesCount.increment() <= self.reqLimit):
|
|
# we've got fewer than the allowed requests open, so we're good to go!
|
|
allowed = True
|
|
else:
|
|
self.scanningFilesCount.decrement()
|
|
|
|
if allowed:
|
|
try:
|
|
if self.verboseDebug: eprint(f'{get_ident()} Capa scanning: {fileName}')
|
|
|
|
if (self.rulesDir is not None):
|
|
cmd = ['timeout', '-k', '10', '-s', 'TERM', str(CAPA_RUN_TIMEOUT_SEC), 'capa', '--quiet', '-r', self.rulesDir, '--json', '--color', 'never', fileName]
|
|
else:
|
|
cmd = ['timeout', '-k', '10', '-s', 'TERM', str(CAPA_RUN_TIMEOUT_SEC), 'capa', '--quiet', '--json', '--color', 'never', fileName]
|
|
capaErr, capaOut = run_process(cmd, stderr=False, debug=self.debug)
|
|
if (capaErr == 0) and (len(capaOut) > 0) and (len(capaOut[0]) > 0):
|
|
# load the JSON output from capa into the .result
|
|
try:
|
|
capaResult.result = json.loads(capaOut[0])
|
|
except (ValueError, TypeError):
|
|
capaResult.result = {"error" : f"Invalid response: {'; '.join(capaOut)}"}
|
|
|
|
else:
|
|
# probably failed because it's not an executable, ignore it
|
|
capaResult.result = {"error" : str(capaErr)}
|
|
|
|
if self.verboseDebug: eprint(f'{get_ident()} Capa scan result: {capaResult.result}')
|
|
capaResult.success = (capaResult.result is not None)
|
|
capaResult.finished = True
|
|
|
|
except Exception as e:
|
|
if capaResult.result is None:
|
|
capaResult.result = str(e)
|
|
if self.debug: eprint(f'{get_ident()} Capa scan error: {capaResult.result}')
|
|
|
|
finally:
|
|
self.scanningFilesCount.decrement()
|
|
try:
|
|
if os.path.isfile(fileName + CAPA_VIV_SUFFIX):
|
|
os.remove(fileName + CAPA_VIV_SUFFIX)
|
|
except Exception as fe:
|
|
pass
|
|
|
|
elif block and (nowTime < timeoutTime):
|
|
# rate limited, wait for a bit and come around and try again
|
|
time.sleep(1)
|
|
|
|
else:
|
|
break
|
|
|
|
else:
|
|
# not an executable, don't need to scan it
|
|
capaResult.result = {}
|
|
capaResult.success = True
|
|
capaResult.finished = True
|
|
|
|
return capaResult
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# return the result of the previously scanned file
|
|
def check_result(self, capaResult):
|
|
return capaResult if isinstance(capaResult, AnalyzerResult) else AnalyzerResult(finished=True, success=False, result=None)
|
|
|
|
# ---------------------------------------------------------------------------------
|
|
# static method for formatting the response summaryDict (from check_result)
|
|
@staticmethod
|
|
def format(fileName, response):
|
|
result = {FILE_SCAN_RESULT_SCANNER : CapaScan.scanner_name(),
|
|
FILE_SCAN_RESULT_FILE : fileName,
|
|
FILE_SCAN_RESULT_ENGINES : 1,
|
|
FILE_SCAN_RESULT_HITS : 0,
|
|
FILE_SCAN_RESULT_MESSAGE : None,
|
|
FILE_SCAN_RESULT_DESCRIPTION : None}
|
|
|
|
if isinstance(response, AnalyzerResult):
|
|
resp = response.result
|
|
verboseHits = response.verbose
|
|
else:
|
|
resp = response
|
|
verboseHits = False
|
|
|
|
if isinstance(resp, dict):
|
|
hits = []
|
|
if 'rules' in resp and isinstance(resp['rules'], dict):
|
|
hits.extend([item.replace('[', '[ATT&CK ') for sublist in dictsearch(resp['rules'], CAPA_ATTACK_KEY) for item in sublist])
|
|
if verboseHits:
|
|
hits.extend(list(resp['rules'].keys()))
|
|
|
|
result[FILE_SCAN_RESULT_HITS] = len(hits)
|
|
if (len(hits) > 0):
|
|
hits = list(set(hits))
|
|
cnt = Counter(hits)
|
|
# short message is most common signature name (todo: they won't have duplicate names, so I guess this is just going to take the first...)
|
|
result[FILE_SCAN_RESULT_MESSAGE] = cnt.most_common(1)[0][0]
|
|
# long description is list of the signature names and the engines which generated them
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = ";".join([f"{x}<{CAPA_ENGINE_ID}>" for x in hits])
|
|
|
|
else:
|
|
result[FILE_SCAN_RESULT_MESSAGE] = "Error or invalid response"
|
|
result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}"
|
|
|
|
return result |