replace requests with iohttp

core/module_protocols/http.py

@@ -2,17 +2,41 @@
 # -*- coding: utf-8 -*-
 
 import re
-import requests
+import aiohttp
+import asyncio
 import copy
 import random
+import time
 from core.utility import reverse_and_regex_condition
 from core.utility import process_conditions
 from core.utility import get_dependent_results_from_database
 from core.utility import replace_dependent_values
 from core.utility import replace_dependent_response
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
+async def perform_request_action(action, request_options):
+    start_time = time.time()
+    async with action(**request_options) as response:
+        return {
+            "reason": response.reason,
+            "status_code": str(response.status),
+            "content": await response.content.read(),
+            "headers": dict(response.headers),
+            "responsetime": time.time() - start_time
+        }
+
+
+async def send_request(request_options, method):
+    async with aiohttp.ClientSession() as session:
+        action = getattr(session, method, None)
+        response = await asyncio.gather(
+            *[
+                asyncio.ensure_future(
+                    perform_request_action(action, request_options)
+                )
+            ]
+        )
+        return response[0]
 
 
 def response_conditions_matched(sub_step, response):
@@ -71,18 +95,18 @@ def response_conditions_matched(sub_step, response):
         ) or (
                 'headers' in condition_results and
                 (
-                        
-                    len(list(condition_results.values())) +
-                    len(list(condition_results['headers'].values())) -
-                    list(condition_results.values()).count([]) -
-                    list(condition_results['headers'].values()).count([]) -
-                    1 != 0
+
+                        len(list(condition_results.values())) +
+                        len(list(condition_results['headers'].values())) -
+                        list(condition_results.values()).count([]) -
+                        list(condition_results['headers'].values()).count([]) -
+                        1 != 0
                 )
-            ):
-            if sub_step['response'].get('log',False):
-                condition_results['log']=sub_step['response']['log']
+        ):
+            if sub_step['response'].get('log', False):
+                condition_results['log'] = sub_step['response']['log']
                 if 'response_dependent' in condition_results['log']:
-                    condition_results['log'] = replace_dependent_response(condition_results['log'],condition_results)
+                    condition_results['log'] = replace_dependent_response(condition_results['log'], condition_results)
             return condition_results
         else:
             return {}
@@ -91,10 +115,10 @@ def response_conditions_matched(sub_step, response):
                 ('headers' in condition_results and [] in condition_results['headers'].values()):
             return {}
         else:
-            if sub_step['response'].get('log',False):
-                condition_results['log']=sub_step['response']['log']
+            if sub_step['response'].get('log', False):
+                condition_results['log'] = sub_step['response']['log']
                 if 'response_dependent' in condition_results['log']:
-                    condition_results['log'] = replace_dependent_response(condition_results['log'],condition_results)
+                    condition_results['log'] = replace_dependent_response(condition_results['log'], condition_results)
             return condition_results
     return {}
 
@@ -114,8 +138,8 @@ def run(
     ):
         backup_method = copy.deepcopy(sub_step['method'])
         backup_response = copy.deepcopy(sub_step['response'])
-        backup_iterative_response_match = copy.deepcopy(sub_step['response']['conditions'].get('iterative_response_match',None))
-        action = getattr(requests, backup_method, None)
+        backup_iterative_response_match = copy.deepcopy(
+            sub_step['response']['conditions'].get('iterative_response_match', None))
         if options['user_agent'] == 'random_user_agent':
             sub_step['headers']['User-Agent'] = random.choice(options['user_agents'])
         del sub_step['method']
@@ -134,33 +158,29 @@ def run(
         del sub_step['response']
         for _ in range(options['retries']):
             try:
-                response = action(**sub_step)
-                response = {
-                    "reason": response.reason,
-                    "status_code": str(response.status_code),
-                    "content": response.content.decode(errors="ignore"),
-                    "headers": dict(response.headers),
-                    "responsetime": response.elapsed.total_seconds()
-                }
+                response = asyncio.run(send_request(sub_step, backup_method))
+                response['content'] = response['content'].decode(errors="ignore")
                 break
             except Exception:
                 response = []
         sub_step['method'] = backup_method
         sub_step['response'] = backup_response
-        
+
         if backup_iterative_response_match != None:
-            backup_iterative_response_match = copy.deepcopy(sub_step['response']['conditions'].get('iterative_response_match'))
+            backup_iterative_response_match = copy.deepcopy(
+                sub_step['response']['conditions'].get('iterative_response_match'))
             del sub_step['response']['conditions']['iterative_response_match']
 
         sub_step['response']['conditions_results'] = response_conditions_matched(sub_step, response)
 
-        if backup_iterative_response_match != None and (sub_step['response']['conditions_results'] or sub_step['response']['condition_type']=='or') :
+        if backup_iterative_response_match != None and (
+                sub_step['response']['conditions_results'] or sub_step['response']['condition_type'] == 'or'):
             sub_step['response']['conditions']['iterative_response_match'] = backup_iterative_response_match
             for key in sub_step['response']['conditions']['iterative_response_match']:
                 result = response_conditions_matched(
-                    sub_step['response']['conditions']['iterative_response_match'][key],response)
+                    sub_step['response']['conditions']['iterative_response_match'][key], response)
                 if result:
-                    sub_step['response']['conditions_results'][key]=result
+                    sub_step['response']['conditions_results'][key] = result
 
         return process_conditions(
             sub_step,

modules/scan/admin.yaml

@@ -14,11 +14,7 @@ payloads:
   - library: http
     steps:
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         allow_redirects: false

modules/scan/drupal_modules.yaml

@@ -15,11 +15,7 @@ payloads:
   - library: http
     steps:
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         allow_redirects: false

modules/scan/drupal_theme.yaml

@@ -15,11 +15,7 @@ payloads:
   - library: http
     steps:
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         allow_redirects: false

modules/scan/drupal_version.yaml

@@ -15,11 +15,7 @@ payloads:
   - library: http
     steps:
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         allow_redirects: false

modules/scan/joomla_template.yaml

@@ -15,11 +15,7 @@ payloads:
   - library: http
     steps:
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         allow_redirects: false

modules/scan/joomla_user_enum.yaml

@@ -15,11 +15,7 @@ payloads:
   - library: http
     steps:
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         allow_redirects: false

modules/scan/joomla_version.yaml

@@ -15,11 +15,7 @@ payloads:
   - library: http
     steps:
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         allow_redirects: false

modules/scan/pma.yaml

@@ -14,11 +14,7 @@ payloads:
   - library: http
     steps:
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         allow_redirects: false

modules/scan/subdomain.yaml

@@ -15,11 +15,7 @@ payloads:
   - library: http
     steps:
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://jldc.me/anubis/subdomains/{target}"
@@ -33,11 +29,7 @@ payloads:
               regex: "\"([a-zA-Z0-9\\-\\_.\\s]+)\""
               reverse: false
       - method: post
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://api.certspotter.com/v1/issuances?domain={target}&expand=dns_names&expand=issuer"
@@ -51,11 +43,7 @@ payloads:
               regex: "\"([a-zA-Z0-9\\-\\_.\\s]+.{target})\""
               reverse: false
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={target}"
@@ -70,11 +58,7 @@ payloads:
               reverse: false
 
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://urlscan.io/api/v1/search/?q=domain:{target}"
@@ -89,11 +73,7 @@ payloads:
               reverse: false
 
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://dns.bufferover.run/dns?q={target}"
@@ -108,11 +88,7 @@ payloads:
               reverse: false
 
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://otx.alienvault.com/api/v1/indicator/domain/{target}/passive_dns"
@@ -127,11 +103,7 @@ payloads:
               reverse: false
 
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://api.threatminer.org/v2/domain.php?q={target}&api=True&rt=5"
@@ -146,11 +118,7 @@ payloads:
               reverse: false
 
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://crt.sh/?q=%.{target}"
@@ -164,11 +132,7 @@ payloads:
               regex: "[a-zA-Z0-9\\-\\_\\s]+[\\.]+[a-zA-Z0-9\\-\\_\\s]+\\.{target}"
               reverse: false
       - method: post
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://hackertarget.com/find-dns-host-records/"
@@ -188,11 +152,7 @@ payloads:
               reverse: false
 
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://dnsdumpster.com/"
@@ -212,11 +172,7 @@ payloads:
                 reverse: false
 
       - method: post
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
           Referer: "https://dnsdumpster.com/"
@@ -238,11 +194,7 @@ payloads:
               reverse: false
 
       - method: get
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
         url: "https://toolbox.googleapps.com/apps/dig/#ANY/"
@@ -262,11 +214,7 @@ payloads:
                 reverse: false
 
       - method: post
-        verify: false
         timeout: 3
-        cert: ""
-        stream: false
-        proxies: ""
         headers:
           User-Agent: "{user_agent}"
           Referer: "https://toolbox.googleapps.com/apps/dig/"