Generate HTTP URLs from Nmap XML (and optionally use VirtualHosts)

nmap-http-url.py

#!/usr/bin/env python3
#
# inputs: nmap.xml (nmap scan xml output), subdomains.csv (optional virtualhost info, hostname + ip address csv file)
# output: url listing (useful for tools like EyeWitness)
#
# sample usage: ./nmap-http-url.py nmap.xml subdomains.csv | sort -u | gowitness file -f -
#

description = '''
Generate HTTP URLs from Nmap XML (and optionally additional VirtualHost listing, taken from e.g. subdomain enumeration).
Useful for tools taking screenshots of websites like EyeWitness.
Although these tools usually support Nmap XML input, guessing what service is HTTP is sometimes weak and
populating the URLs with VirtualHost information could be a missing feature.
Generating the HTTP URLs from the Nmap XML follows the Nmap NSE shortport.http logic here.
'''

import xml.etree.ElementTree as ET
import argparse

parser = argparse.ArgumentParser(description=description)
parser.add_argument('nmap.xml', nargs=1, help='Nmap XML output file')
parser.add_argument('vhosts.csv', nargs='?', help='VirtualHost CSV file (COL1: virtualhost domain; COL2: ip address, separated by semicolon)')
args = parser.parse_args()

# nmap version 7.92 shortport.http logic
LIKELY_HTTP_PORTS = { 80, 443, 631, 7080, 8080, 8443, 8088, 5800, 3872, 8180, 8000 }
LIKELY_HTTP_SERVICES = { "http", "https", "ipp", "http-alt", "https-alt", "vnc-http", "oem-agent", "soap", "http-proxy", "caldav", "carddav", "webdav"}
LIKELY_SSL_PORTS = { 261, 271, 324, 443, 465, 563, 585, 636, 853, 989, 990, 992, 993, 994, 995, 2221, 2252, 2376, 3269, 3389, 4433, 4911, 5061, 5986, 6679, 6697, 8443, 9001, 8883 }
LIKELY_SSL_SERVICES = { "ftps", "ftps-data", "ftps-control", "https", "https-alt", "imaps", "ircs", "ldapssl", "ms-wbt-server", "pop3s", "sip-tls", "smtps", "telnets", "tor-orport" }

# vhosts csv format: vhost_domain;ip_address
VHOSTS_DELIMITER = ";"
vhosts = {}
if getattr(args, 'vhosts.csv') is not None:
    with open(getattr(args, 'vhosts.csv'), "r") as f:
        for s in f:
            x = s.strip().split(VHOSTS_DELIMITER)
            if x[1] in vhosts:
                vhosts[x[1]].append(x[0])
            else:
                vhosts[x[1]] = [x[0]]

tree = ET.parse(getattr(args, 'nmap.xml')[0])
root = tree.getroot()
for host in root.findall('./host'):
    ip = host.find('address').attrib['addr']
    hostnames = { ip }
    for hostname in host.findall('./hostnames/hostname'):
        hostnames.add(hostname.attrib['name'])
    if ip in vhosts:
        for hostname in vhosts[ip]:
            hostnames.add(hostname)
    for port in host.findall('./ports/port'):
        service = port.find('service')
        if service is not None:
            if port.attrib['protocol'] == 'tcp' and port.find('state').attrib['state'] == 'open' and (port.attrib['portid'] in LIKELY_HTTP_PORTS or service.attrib['name'] in LIKELY_HTTP_SERVICES):
                proto = "https" if ('tunnel' in service.attrib and service.attrib['tunnel'] == 'ssl') or port.attrib['portid'] in LIKELY_SSL_PORTS or service.attrib['name'] in LIKELY_SSL_SERVICES else "http"
                for hostname in hostnames:
                    print("{}://{}:{}".format(proto, hostname, port.attrib['portid']))