Add light analysis for other language (#1837)

* Add other language for light analysisy

Signed-off-by: Arthur Chan <arthur.chan@adalogics.com>

* Fix formatting

Signed-off-by: Arthur Chan <arthur.chan@adalogics.com>

---------

Signed-off-by: Arthur Chan <arthur.chan@adalogics.com>

oss_fuzz_integration/oss-fuzz-patches.diff

@@ -45,15 +45,14 @@ index d9077510f..5baa138a6 100755
 
  if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
    if [ "$FUZZING_ENGINE" != "libfuzzer" ] && [ "$FUZZING_ENGINE" != "wycheproof" ]; then
-@@ -66,6 +67,14 @@ if [ "$FUZZING_LANGUAGE" = "python" ]; then
+@@ -66,6 +67,13 @@ if [ "$FUZZING_LANGUAGE" = "python" ]; then
    fi
  fi
 
 +if [ "$FUZZING_LANGUAGE" = "rust" ]; then
 +  if [ "$SANITIZER" = "introspector" ]; then
 +    # introspector sanitizer flag will cause cargo build failed, remove it temporary and store it in separate variable
 +    export SANITIZER=address
-+    pip3 install rust-demangler
 +  fi
 +fi
 +
@@ -80,6 +79,28 @@ index d9077510f..5baa138a6 100755
    export AR=llvm-ar
    export NM=llvm-nm
    export RANLIB=llvm-ranlib
+@@ -210,10 +220,19 @@ if [ "$SANITIZER" = "introspector" ]; then
+
+   apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev
+   python3 -m pip install --upgrade pip setuptools
+-  python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve
++  python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve rust-demangler
+   python3 -m pip install --prefer-binary matplotlib
+
+-  python3 /fuzz-introspector/src/main.py light
++  if [ "$FUZZING_LANGUAGE" = "python" ]; then
++    python3 /fuzz-introspector/src/main.py light --language=python
++  elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then
++    python3 /fuzz-introspector/src/main.py light --language=jvm
++  elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
++    python3 /fuzz-introspector/src/main.py light --language=rust
++  else
++    python3 /fuzz-introspector/src/main.py light
++  fi
++
+   rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
+ fi
+
 @@ -280,7 +288,7 @@ else
    fi
  fi

src/fuzz_introspector/analysis.py

@@ -1073,10 +1073,10 @@ def _extract_test_information_cpp(report_dict):
         if path.startswith('/usr/'):
             continue
         directories.add('/'.join(path.split('/')[:-1]))
-    return extract_tests_from_directories(directories)
+    return extract_tests_from_directories(directories, 'c-cpp')
 
 
-def extract_tests_from_directories(directories) -> Set[str]:
+def extract_tests_from_directories(directories, language) -> Set[str]:
     """Extracts test files from a given collection of directory paths and also
     copies them to the `constants.SAVED_SOURCE_FOLDER` folder with the same
     absolute path appended."""
@@ -1101,8 +1101,19 @@ def extract_tests_from_directories(directories) -> Set[str]:
         if any(ins in directory for ins in inspirations):
             all_inspiration_dirs.add(directory)
 
-    # Get all .c files
-    test_extensions = ['.cc', '.cpp', '.cxx', '.c++', '.c']
+    if language == 'jvm':
+        # Get all jvm source files
+        test_extensions = ['.java', '.scala', '.sc', '.groovy', '.kt', '.kts']
+    elif language == 'python':
+        # Get all python source files
+        test_extensions = ['.py']
+    elif language == 'rust':
+        # Get all rust source files
+        test_extensions = ['.rs']
+    else:
+        # Get all c/cpp source files
+        test_extensions = ['.cc', '.cpp', '.cxx', '.c++', '.c']
+
     all_test_files = set()
     to_avoid = [
         'fuzztest', 'aflplusplus', 'libfuzzer', 'googletest', 'thirdparty',
@@ -1126,6 +1137,15 @@ def extract_tests_from_directories(directories) -> Set[str]:
                     with open(absolute_path, 'r') as file_fp:
                         if 'LLVMFuzzerTestOneInput' in file_fp.read():
                             continue
+                        # For rust projects
+                        if 'fuzz_target' in file_fp.read():
+                            continue
+                        # For python projects
+                        if '.Fuzz()' in file_fp.read():
+                            continue
+                        # For jvm projects
+                        if 'fuzzerTestOneInput' in file_fp.read():
+                            continue
                 except UnicodeDecodeError:
                     continue
                 all_test_files.add(absolute_path)
@@ -1225,8 +1245,12 @@ def _extract_test_information_jvm():
     return all_test_files
 
 
-def light_correlate_source_to_executable():
+def light_correlate_source_to_executable(language):
     """Extracts pairs of harness source/executable"""
+    if language == 'jvm' or language == 'python':
+        # Skip this step for jvm or python projects
+        return []
+
     out_dir = os.getenv('OUT', '/out/')
     textcov_dir = os.path.join(out_dir, 'textcov_reports')
 
@@ -1240,7 +1264,7 @@ def light_correlate_source_to_executable():
     for cov_report in cov_reports:
         print('- cov report: %s' % (cov_report))
 
-    all_source_files = extract_all_sources('cpp')
+    all_source_files = extract_all_sources(language)
     pairs = []
     # Match based on file names. This should be the most primitive but
     # will catch a large number of targets

src/fuzz_introspector/commands.py

@@ -81,16 +81,17 @@ def light_analysis(args) -> int:
     if not os.path.isdir(light_dir):
         os.makedirs(light_dir, exist_ok=True)
 
-    all_tests = analysis.extract_tests_from_directories({src_dir})
+    all_tests = analysis.extract_tests_from_directories({src_dir},
+                                                        args.language)
 
     with open(os.path.join(light_dir, 'all_tests.json'), 'w') as f:
         f.write(json.dumps(list(all_tests)))
 
-    pairs = analysis.light_correlate_source_to_executable()
+    pairs = analysis.light_correlate_source_to_executable(args.language)
     with open(os.path.join(light_dir, 'all_pairs.json'), 'w') as f:
         f.write(json.dumps(list(pairs)))
 
-    all_source_files = analysis.extract_all_sources('cpp')
+    all_source_files = analysis.extract_all_sources(args.language)
     light_out_src = os.path.join(light_dir, 'source_files')
 
     for source_file in all_source_files:

src/main.py

@@ -31,10 +31,14 @@ def get_cmdline_parser() -> argparse.ArgumentParser:
 
     subparsers = parser.add_subparsers(dest='command')
 
-    subparsers.add_parser(
+    light_parser = subparsers.add_parser(
         "light",
         help="Perform light analysis of project. This involves no compilaiton.",
     )
+    light_parser.add_argument("--language",
+                              type=str,
+                              default="c-cpp",
+                              help="Language of project")
 
     # Report generation command
     report_parser = subparsers.add_parser(