Merge branch 'ossf:main' into new-ci-api-test

scripts/oss-fuzz-gen-e2e/web_run_all.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -eux
+# Copyright 2024 Fuzz Introspector Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+
+ROOT_FI=$PWD/../../
+BASE_DIR=$PWD/workdir
+BENCHMARK_HEURISTICS="${VARIABLE:-far-reach-low-coverage,low-cov-with-fuzz-keyword}"
+OSS_FUZZ_GEN_MODEL=${MODEL}
+VAR_HARNESSES_PER_ORACLE="${HARNESS_PER_ORACLE:-10}"
+VAR_LLM_FIX_LIMIT="${LLM_FIX_LIMIT:-1}"
+PROJECT=${@}
+
+comma_separated=""
+for proj in ${PROJECT}; do
+  echo ${proj}
+  comma_separated="${comma_separated}${proj},"
+done
+comma_separated=${comma_separated::-1}
+
+# Launch virtualenv
+cd ${BASE_DIR}
+. .venv/bin/activate
+
+# Create webserver DB
+echo "[+] Creating the webapp DB"
+cd $ROOT_FI/tools/web-fuzzing-introspection/app/static/assets/db/
+python3 ./web_db_creator_from_summary.py \
+    --since-date=20-04-2023 \
+    --output-dir=$PWD \
+    --input-dir=$PWD \
+    --includes=${comma_separated}
+
+# Start webserver DB                                                            
+echo "Shutting down server in case it's running"
+curl --silent http://localhost:8080/api/shutdown || true
+
+echo "[+] Launching FI webapp"
+cd $ROOT_FI/tools/web-fuzzing-introspection/app/
+FUZZ_INTROSPECTOR_SHUTDOWN=1 python3 ./main.py >> /dev/null &
+
+SECONDS=5
+while true
+do
+  # Checking if exists
+  MSG=$(curl -v --silent 127.0.0.1:8080 2>&1 | grep "Fuzzing" | wc -l)
+  if [[ $MSG > 0 ]]; then
+    echo "Found it"
+    break
+  fi
+  echo "- Waiting for webapp to load. Sleeping ${SECONDS} seconds."
+  sleep ${SECONDS}
+done
+
+# Deactivate
+echo "[+] Running OSS-Fuzz-gen experiment"
+export LLM_FIX_LIMIT=${VAR_LLM_FIX_LIMIT}
+cd $BASE_DIR/oss-fuzz-gen
+./run_all_experiments.py \
+    --model=$OSS_FUZZ_GEN_MODEL \
+    -g ${BENCHMARK_HEURISTICS} \
+    -gp ${comma_separated} \
+    -gm ${VAR_HARNESSES_PER_ORACLE} \
+    -e http://127.0.0.1:8080/api
+
+echo "Shutting down started webserver"
+curl --silent http://localhost:8080/api/shutdown || true

tools/web-fuzzing-introspection/app/main.py

@@ -56,4 +56,4 @@ def create_app():
 if __name__ == "__main__":
     create_app().run(debug=False,
                      host="0.0.0.0",
-                     port=os.environ.get("WEBAPP_PORT", 8080))
+                     port=int(os.environ.get("WEBAPP_PORT", '8080')))

tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py

@@ -193,7 +193,7 @@ def extract_new_introspector_constructors(project_name, date_str):
     # Read the introspector artifact
     try:
         raw_introspector_json_request = requests.get(
-            introspector_functions_url, timeout=10)
+            introspector_constructor_url, timeout=10)
         introspector_constructors = json.loads(
             raw_introspector_json_request.text)
     except:
@@ -331,3 +331,26 @@ def try_to_get_project_language(project_name):
         project_yaml = yaml.safe_load(r.text)
         return project_yaml['language']
     return "N/A"
+
+
+def try_to_get_project_repository(project_name):
+    if os.path.isdir(constants.OSS_FUZZ_CLONE):
+        local_project_path = os.path.join(constants.OSS_FUZZ_CLONE, "projects",
+                                          project_name)
+        if os.path.isdir(local_project_path):
+            project_yaml_path = os.path.join(local_project_path,
+                                             "project.yaml")
+            if os.path.isfile(project_yaml_path):
+                with open(project_yaml_path, "r") as f:
+                    project_yaml = yaml.safe_load(f.read())
+                    return project_yaml['main_repo']
+    else:
+        proj_yaml_url = 'https://raw.githubusercontent.com/google/oss-fuzz/master/projects/%s/project.yaml' % (
+            project_name)
+        try:
+            r = requests.get(proj_yaml_url, timeout=10)
+        except:
+            return "N/A"
+        project_yaml = yaml.safe_load(r.text)
+        return project_yaml['main_repo']
+    return "N/A"

tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py

@@ -336,6 +336,11 @@ def extract_local_project_data(project_name, oss_fuzz_path,
 
     # Dump things we dont want to accummulate.
     #save_branch_blockers(branch_pairs, project_name)
+    try:
+        project_repository = oss_fuzz.try_to_get_project_repository(
+            project_name)
+    except:
+        project_repository = 'N/A'
 
     introspector_data_dict = {
         "introspector_report_url": 'introspector_url',
@@ -365,6 +370,7 @@ def extract_local_project_data(project_name, oss_fuzz_path,
         'coverage-data': code_coverage_data_dict,
         'introspector-data': introspector_data_dict,
         'fuzzer-count': amount_of_fuzzers,
+        'project_repository': project_repository,
     }
 
     dictionary_key = '%s###%s' % (project_name, '')
@@ -408,6 +414,12 @@ def extract_project_data(project_name, date_str, should_include_details,
         # Default set to c++ as this is OSS-Fuzz's default.
         project_language = 'c++'
 
+    try:
+        project_repository = oss_fuzz.try_to_get_project_repository(
+            project_name)
+    except:
+        project_repository = 'N/A'
+
     collect_debug_info = project_language in {'c', 'c++'}
 
     # Extract code coverage and introspector reports.
@@ -587,6 +599,7 @@ def extract_project_data(project_name, date_str, should_include_details,
         'coverage-data': code_coverage_data_dict,
         'introspector-data': introspector_data_dict,
         'fuzzer-count': amount_of_fuzzers,
+        'project_repository': project_repository,
     }
 
     dictionary_key = '%s###%s' % (project_name, date_str)

tools/web-fuzzing-introspection/app/webapp/__init__.py

@@ -5,19 +5,9 @@
 from . import models
 
 
-def is_db_valid():
-    db_timestamps_file = os.path.join(
-        os.path.dirname(__file__), "../static/assets/db/db-timestamps.json")
-    if not os.path.isfile(db_timestamps_file):
-        return False
-    return True
-
-
 def load_db():
     """Loads the database"""
     print("Loading db")
-    if not is_db_valid():
-        update_db()
 
     db_timestamps_file = os.path.join(
         os.path.dirname(__file__), "../static/assets/db/db-timestamps.json")
@@ -92,7 +82,8 @@ def load_db():
                 date=project_timestamp['date'],
                 coverage_data=project_timestamp['coverage-data'],
                 introspector_data=project_timestamp['introspector-data'],
-                fuzzer_count=project_timestamp['fuzzer-count']))
+                fuzzer_count=project_timestamp['fuzzer-count'],
+                project_repository=project_timestamp['project_repository']))
 
         introspector_data = project_timestamp.get('introspector-data', None)
         if introspector_data is None:

tools/web-fuzzing-introspection/app/webapp/data_storage.py

@@ -1,30 +1,30 @@
 # Auto-generated
 #from app.site.models import *
 
-from typing import List
+from typing import List, Dict, Any
 
 import os
 import json
 
 from .models import *
 
-PROJECT_TIMESTAMPS = []
+PROJECT_TIMESTAMPS: List[ProjectTimestamp] = []
 
-DB_TIMESTAMPS = []
+DB_TIMESTAMPS: List[DBTimestamp] = []
 
-PROJECTS = []
+PROJECTS: List[Project] = []
 
-FUNCTIONS = []
+FUNCTIONS: List[Function] = []
 
-CONSTRUCTORS = []
+CONSTRUCTORS: List[Function] = []
 
-BLOCKERS = []
+BLOCKERS: List[BranchBlocker] = []
 
 BUILD_STATUS: List[BuildStatus] = []
 
-PROJECT_DEBUG_DATA = []
+PROJECT_DEBUG_DATA: List[DebugStatus] = []
 
-ALL_HEADER_FILES = []
+ALL_HEADER_FILES: List[Dict[str, Any]] = []
 
 
 def get_projects():

tools/web-fuzzing-introspection/app/webapp/models.py

@@ -66,13 +66,14 @@ def has_introspector(self) -> bool:
 class Project:
 
     def __init__(self, name, language, date, coverage_data, introspector_data,
-                 fuzzer_count):
+                 fuzzer_count, project_repository):
         self.name = name
         self.language = language
         self.date = date
         self.coverage_data = coverage_data
         self.introspector_data = introspector_data
         self.fuzzer_count = fuzzer_count
+        self.project_repository = project_repository
 
     def has_introspector(self) -> bool:
         return self.introspector_data != None
@@ -128,7 +129,7 @@ def __init__(self,
         self.is_jvm_library = is_jvm_library
         self.is_enum_class = is_enum_class
 
-    def __dict__(self):
+    def to_dict(self):
         return {
             'function_name': self.name,
             'function_arguments': self.function_arguments,