Skip to content

Commit

Permalink
Updates to nmap importer, add script to build huge bogus nmap XML to …
Browse files Browse the repository at this point in the history
…test with
  • Loading branch information
sscottgvit committed Nov 21, 2023
1 parent df4f0d7 commit 83a253f
Show file tree
Hide file tree
Showing 10 changed files with 216 additions and 33 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -128,3 +128,4 @@ docker/cleanupExited.sh
log/*.log
fixname.sh
ghostdriver.log
huge_nmap.xml
2 changes: 1 addition & 1 deletion app/ApplicationInfo.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
applicationInfo = {
"name": "LEGION",
"version": "0.4.2",
"build": '1700506312',
"build": '1700525714',
"author": "Gotham Security",
"copyright": "2023",
"links": ["http://github.com/GoVanguard/legion/issues", "https://gotham-security.com/legion"],
Expand Down
4 changes: 2 additions & 2 deletions app/actions/updateProgress/UpdateProgressObservable.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,6 @@ def start(self):
for observer in self._observers:
observer.onStart()

def updateProgress(self, progress):
def updateProgress(self, progress, title):
for observer in self._observers:
observer.onProgressUpdate(progress)
observer.onProgressUpdate(progress, title)
11 changes: 8 additions & 3 deletions app/auxiliary.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,14 @@ def sortArrayWithArray(array, arrayToSort):

# converts an IP address to an integer (for the sort function)
def IP2Int(ip):
ip = ip.split("/")[0] # bug fix: remove slash if it's a range
o = list(map(int, ip.split('.')))
res = (16777216 * o[0]) + (65536 * o[1]) + (256 * o[2]) + o[3]
try:
res = 0
ip = ip.split("/")[0] # bug fix: remove slash if it's a range
o = list(map(int, ip.split('.')))
res = (16777216 * o[0]) + (65536 * o[1]) + (256 * o[2]) + o[3]
except:
log.error("Input IP {0} is not valid. Passing for now.".format(str(ip)))
pass
return res


Expand Down
63 changes: 44 additions & 19 deletions app/importers/NmapImporter.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,18 +85,20 @@ def run(self):
n = nmapSessionObj(self.filename, s.startTime, s.finish_time, s.nmapVersion, s.scanArgs, s.totalHosts,
s.upHosts, s.downHosts)
session.add(n)
session.commit()

allHosts = nmapReport.getAllHosts()
hostCount = len(allHosts)
if hostCount == 0: # to fix a division by zero if we ran nmap on one host
hostCount = 1
totalprogress = 0

self.updateProgressObservable.updateProgress(totalprogress)

createProgress = 0
createOsNodesProgress = 0
createPortsProgress = 0
createDbScriptsProgress = 0
updateObjectsRunScriptsProgress = 0

self.updateProgressObservable.updateProgress(int(createProgress), 'Adding hosts...')

for h in allHosts: # create all the hosts that need to be created
db_host = self.hostRepository.getHostInformation(h.ip)
Expand All @@ -107,16 +109,17 @@ def run(self):
lastboot=h.lastboot, distance=h.distance, state=h.state, count=h.count)
self.tsLog("Adding db_host")
session.add(hid)
session.commit()
t_note = note(h.ip, 'Added by nmap')
session.add(t_note)
session.commit()
else:
self.tsLog("Found db_host already in db")

createProgress = createProgress + ((100.0 / hostCount) / 5)
totalprogress = totalprogress + createProgress
self.updateProgressObservable.updateProgress(int(totalprogress))
createProgress = createProgress + (100.0 / hostCount)
self.updateProgressObservable.updateProgress(int(createProgress), 'Adding hosts...')

session.commit()
self.updateProgressObservable.updateProgress(int(createOsNodesProgress), 'Creating Service, Port and OS children...')

for h in allHosts: # create all OS, service and port objects that need to be created
self.tsLog("Processing h {ip}".format(ip=h.ip))
Expand All @@ -125,7 +128,10 @@ def run(self):
if db_host:
self.tsLog("Found db_host during os/ports/service processing")
else:
self.log("Did not find db_host during os/ports/service processing")
self.tsLog("Did not find db_host during os/ports/service processing")
self.tsLog("A host that should have been found was not. Something is wrong. Save your session and report a bug.")
self.tsLog("Include your nmap file, sanitized if needed.")
continue

os_nodes = h.getOs() # parse and store all the OS nodes
self.tsLog(" 'os_nodes' to process: {os_nodes}".format(os_nodes=str(len(os_nodes))))
Expand All @@ -139,14 +145,15 @@ def run(self):
t_osObj = osObj(os.name, os.family, os.generation, os.osType, os.vendor, os.accuracy,
db_host.id)
session.add(t_osObj)
session.commit()

createOsNodesProgress = createOsNodesProgress + ((100.0 / hostCount) / 5)
totalprogress = totalprogress + createOsNodesProgress
self.updateProgressObservable.updateProgress(int(totalprogress))
createOsNodesProgress = createOsNodesProgress + (100.0 / hostCount)
self.updateProgressObservable.updateProgress(int(createOsNodesProgress), 'Creating Service, Port and OS children...')

session.commit()
self.updateProgressObservable.updateProgress(int(createPortsProgress), 'Processing ports...')

all_ports = h.all_ports()
portCount = len(all_ports)
self.tsLog(" 'ports' to process: {all_ports}".format(all_ports=str(len(all_ports))))
for p in all_ports: # parse the ports
self.tsLog(" Processing port obj {port}".format(port=str(p.portId)))
Expand All @@ -164,6 +171,7 @@ def run(self):
db_service = serviceObj(s.name, db_host.id, s.product, s.version, s.extrainfo,
s.fingerprint)
session.add(db_service)
session.commit()
else: # else, there is no service info to parse
db_service = None
# fetch the port
Expand All @@ -177,11 +185,11 @@ def run(self):
else:
db_port = portObj(p.portId, p.protocol, p.state, db_host.id, '')
session.add(db_port)
createPortsProgress = createPortsProgress + ((100.0 / hostCount) / 5)
totalprogress = totalprogress + createPortsProgress
self.updateProgressObservable.updateProgress(totalprogress)
session.commit()
createPortsProgress = createPortsProgress + (100.0 / hostCount)
self.updateProgressObservable.updateProgress(createPortsProgress, 'Processing ports...')

session.commit()
self.updateProgressObservable.updateProgress(createDbScriptsProgress, 'Creating script objects...')

for h in allHosts: # create all script objects that need to be created
db_host = self.hostRepository.getHostInformation(h.ip)
Expand All @@ -202,19 +210,26 @@ def run(self):
t_l1ScriptObj = l1ScriptObj(scr.scriptId, scr.output, db_port.id, db_host.id)
self.tsLog(" Adding l1ScriptObj obj {script}".format(script=scr.scriptId))
session.add(t_l1ScriptObj)

session.commit()
for hs in h.getHostScripts():
db_script = session.query(l1ScriptObj).filter_by(scriptId=hs.scriptId) \
.filter_by(hostId=db_host.id).first()
if not db_script:
t_l1ScriptObj = l1ScriptObj(hs.scriptId, hs.output, None, db_host.id)
session.add(t_l1ScriptObj)
session.commit()

session.commit()
createDbScriptsProgress = createDbScriptsProgress + (100.0 / hostCount)
self.updateProgressObservable.updateProgress(createDbScriptsProgress, 'Creating script objects...')

self.updateProgressObservable.updateProgress(updateObjectsRunScriptsProgress, 'Update objects and run scripts...')

for h in allHosts: # update everything

db_host = self.hostRepository.getHostInformation(h.ip)
if not db_host:
self.tsLog(" A host that should have been found was not. Something is wrong. Save your session and report a bug.")
self.tsLog(" Include your nmap file, sanitized if needed.")

if db_host.ipv4 == '' and not h.ipv4 == '':
db_host.ipv4 = h.ipv4
Expand All @@ -240,6 +255,7 @@ def run(self):
db_host.count = h.count

session.add(db_host)
session.commit()

tmp_name = ''
tmp_accuracy = '0' # TODO: check if better to convert to int for comparison
Expand All @@ -265,20 +281,23 @@ def run(self):
db_host.osAccuracy = tmp_accuracy

session.add(db_host)
session.commit()

for scr in h.getHostScripts():
self.tsLog("-----------------------Host SCR: {0}".format(scr.scriptId))
db_host = self.hostRepository.getHostInformation(h.ip)
scrProcessorResults = scr.scriptSelector(db_host)
for scrProcessorResult in scrProcessorResults:
session.add(scrProcessorResult)
session.commit()

for scr in h.getScripts():
self.tsLog("-----------------------SCR: {0}".format(scr.scriptId))
db_host = self.hostRepository.getHostInformation(h.ip)
scrProcessorResults = scr.scriptSelector(db_host)
for scrProcessorResult in scrProcessorResults:
session.add(scrProcessorResult)
session.commit()

for p in h.all_ports():
s = p.getService()
Expand All @@ -296,10 +315,12 @@ def run(self):
if db_port.state != p.state:
db_port.state = p.state
session.add(db_port)
session.commit()
# if there is some new service information, update it -- might be causing issue 164
if not (db_service is None) and db_port.serviceId != db_service.id:
db_port.serviceId = db_service.id
session.add(db_port)
session.commit()
# store the script results (note that existing script outputs are also kept)
for scr in p.getScripts():
db_script = session.query(l1ScriptObj).filter_by(scriptId=scr.scriptId) \
Expand All @@ -309,8 +330,12 @@ def run(self):
db_script.output = scr.output

session.add(db_script)
session.commit()

updateObjectsRunScriptsProgress = updateObjectsRunScriptsProgress + (100.0 / hostCount)
self.updateProgressObservable.updateProgress(updateObjectsRunScriptsProgress, 'Update objects and run scripts...')

self.updateProgressObservable.updateProgress(100)
self.updateProgressObservable.updateProgress(100, 'Almost done...')

session.commit()
self.db.dbsemaphore.release() # we are done with the DB
Expand Down
2 changes: 1 addition & 1 deletion app/logging/legionLog.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def getOrCreateCachedLogger(logName: str, logPath: str, console: bool, cachedLog
from rich.logging import RichHandler

logging.basicConfig(
level="WARN",
level="INFO",
format="%(message)s",
datefmt="[%X]",
handlers=[RichHandler(rich_tracebacks=True)]
Expand Down
148 changes: 148 additions & 0 deletions buildHugeNmapTest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
import xml.etree.ElementTree as ET
from random import randint, choice, sample
import datetime

def generate_nmap_xml(num_hosts=1000, base_subnet="172.16"):
"""
Generate a full sample nmap XML file with session information included.
Parameters:
num_hosts (int): Number of hosts to generate.
base_subnet (str): Base subnet for the hosts.
Returns:
str: XML content as a string.
"""
# XML header
xml_header = '''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE nmaprun>
<?xml-stylesheet href="file:///C:/Program Files (x86)/Nmap/nmap.xsl" type="text/xsl"?>
<!-- Nmap 7.94 scan initiated as: nmap -sV -oX -->'''

# Create root element
nmaprun = ET.Element("nmaprun", {
"scanner": "nmap",
"args": "nmap -sV -oX",
"start": "123456789",
"startstr": "Mon Nov 20 13:33:54 2023",
"version": "7.94",
"xmloutputversion": "1.05"
})

# Create scaninfo, verbose, and debugging elements
scaninfo = ET.SubElement(nmaprun, "scaninfo", {
"type": "syn",
"protocol": "tcp",
"numservices": "1000",
"services": "1-65535"
})
ET.SubElement(nmaprun, "verbose", {"level": "0"})
ET.SubElement(nmaprun, "debugging", {"level": "0"})

# OS, services, and port mapping
os_services = {
"Linux": {"http": 80, "ssh": 22, "smtp": 25, "ftp": 21, "telnet": 23},
"Windows": {"http": 80, "msrpc": 135, "netbios-ssn": 139, "rdp": 3389, "smb": 445},
"Solaris": {"http": 80, "ssh": 22, "telnet": 23, "netbios-ssn": 139, "ftp": 21},
"Darwin": {"http": 80, "netbios-ssn": 139, "ipp": 631, "afp": 548, "ssh": 22}
}

# Service products and versions
service_info = {
"http": {"product": "Apache httpd", "version": "2.4.41"},
"ssh": {"product": "OpenSSH", "version": "8.0"},
"smtp": {"product": "Postfix SMTP", "version": "3.4.8"},
"ftp": {"product": "vsftpd", "version": "3.0.3"},
"telnet": {"product": "Telnet Server", "version": "1.2"},
"msrpc": {"product": "Microsoft RPC", "version": "5.0"},
"netbios-ssn": {"product": "Samba smbd", "version": "4.10.10"},
"rdp": {"product": "Microsoft Terminal Service", "version": "10.0"},
"smb": {"product": "Microsoft SMB", "version": "3.1.1"},
"ipp": {"product": "CUPS", "version": "2.3.0"},
"afp": {"product": "Netatalk AFP", "version": "3.1.12"}
}

# Function to create a random IP address within the extended subnet range
def random_ip(base_subnet, host_number):
subnet_third_octet = host_number // 254
host_fourth_octet = host_number % 254 + 1
return f"{base_subnet}.{subnet_third_octet}.{host_fourth_octet}"

# Generating hosts with updated IP address method
for i in range(num_hosts):
host_os = choice(list(os_services.keys()))

host = ET.Element("host")
ET.SubElement(host, "status", {"state": "up", "reason": "arp-response", "reason_ttl": "0"})
ET.SubElement(host, "address", {"addr": random_ip(base_subnet, i), "addrtype": "ipv4"})
ET.SubElement(host, "hostnames")

# Ports
ports = ET.SubElement(host, "ports")
open_ports_count = randint(1, 5) # Random number of open ports (1 to 5)
services = sample(list(os_services[host_os].items()), open_ports_count)
for service, port in services:
port_element = ET.SubElement(ports, "port", {"protocol": "tcp", "portid": str(port)})
ET.SubElement(port_element, "state", {"state": "open", "reason": "syn-ack", "reason_ttl": "64"})
service_element = ET.SubElement(port_element, "service", {
"name": service,
"product": service_info[service]["product"],
"version": service_info[service]["version"],
"ostype": host_os
})

# OS element with osmatch
os = ET.SubElement(host, "os")
osclass = ET.SubElement(os, "osclass", {
"type": "general purpose",
"vendor": host_os,
"osfamily": host_os,
"osgen": "Unknown", # OS generation can be set accordingly
"accuracy": "98"
})
ET.SubElement(osclass, "cpe", text=f"cpe:/o:{host_os.lower()}:{host_os.lower()}")
osmatch = ET.SubElement(os, "osmatch", {
"name": f"{host_os} Generic Match",
"accuracy": str(randint(90, 100)),
"line": str(randint(1000, 2000))
})
ET.SubElement(osmatch, "osclass", {
"type": "general purpose",
"vendor": host_os,
"osfamily": host_os,
"osgen": "Unknown",
"accuracy": "98"
})

nmaprun.append(host)

# Runstats
runstats = ET.SubElement(nmaprun, "runstats")
ET.SubElement(runstats, "finished", {
"time": "123456790",
"timestr": "Mon Nov 20 13:34:10 2023",
"summary": f"Nmap done; {num_hosts} IP addresses ({num_hosts} hosts up) scanned in 16.42 seconds",
"elapsed": "16.42",
"exit": "success"
})
ET.SubElement(runstats, "hosts", {
"up": str(num_hosts),
"down": "0",
"total": str(num_hosts)
})

# Convert the XML to string
xml_str = xml_header + '\n' + ET.tostring(nmaprun, encoding='unicode', method='xml')
return xml_str

def save_nmap_xml(filename, num_hosts=1000, base_subnet="172.16"):
# Generate the XML content
xml_content = generate_nmap_xml(num_hosts, base_subnet)

# Save the content to the specified file
with open(filename, "w") as file:
file.write(xml_content)

# Specify the filename and call the function to save the file
filename = "huge_nmap.xml"
save_nmap_xml(filename)
Loading

0 comments on commit 83a253f

Please sign in to comment.