Merge pull request #463 from owasp-noir/dev

Release v0.18.3

.github/ISSUE_TEMPLATE/bug_report.md

@@ -23,7 +23,7 @@ If applicable, add screenshots to help explain your problem.
 
 **Versions**
  - OS: [e.g. macos, linux]
- - Version [e.g. v0.18.2]
+ - Version [e.g. v0.18.3]
 
 **Additional context**
 Add any other context about the problem here.

.github/workflows/deadlinks.yml

@@ -1,30 +1,45 @@
 ---
-    name: DeadLink
-    # Controls when the workflow will run
-    on:
-      # Allows you to run this workflow manually from the Actions tab
-      workflow_dispatch:
-    # A workflow run is made up of one or more jobs that can run sequentially or in parallel
-    jobs:
-      # This workflow contains a single job called "build"
-      build:
-        # The type of runner that the job will run on
-        runs-on: ubuntu-latest
-
-        # Steps represent a sequence of tasks that will be executed as part of the job
-        steps:
-          - name: Find Broken Link
-            uses: hahwul/deadfinder@1.4.4
-            id: broken-link
-            with:
-              command: sitemap
-              target: https://owasp-noir.github.io/noir/sitemap.xml
-          - name: Create an issue
-            uses: dacbd/create-issue-action@main
-            with:
-              token: ${{ github.token }}
-              title: DeadLink Issue
-              body: |
-                ```json
-                ${{ steps.broken-link.outputs.output }}
-                ```
+  name: DeadLink
+  # Controls when the workflow will run
+  on:
+    # Allows you to run this workflow manually from the Actions tab
+    workflow_dispatch:
+  # A workflow run is made up of one or more jobs that can run sequentially or in parallel
+  jobs:
+    # This workflow contains a single job called "build"
+    build:
+      # The type of runner that the job will run on
+      runs-on: ubuntu-latest
+
+      # Steps represent a sequence of tasks that will be executed as part of the job
+      steps:
+        - name: Find Broken Link
+          uses: hahwul/deadfinder@1.5.0
+          id: broken-link
+          with:
+            command: sitemap
+            target: https://owasp-noir.github.io/noir/sitemap.xml
+
+        - name: Create Markdown Table from JSON
+          id: create-markdown-table
+          run: |
+            echo "## DeadLink Report" > deadlink_report.md
+            echo "" >> deadlink_report.md
+            echo "| Target URL | Deadlink   |" >> deadlink_report.md
+            echo "|------------|------------|" >> deadlink_report.md
+            echo '${{ steps.broken-link.outputs.output }}' | jq -r 'to_entries[] | .key as $k | .value[] | "| \($k) | \(.) |"' >> deadlink_report.md
+  
+        - name: Read Markdown Table from File
+          id: read-markdown-table
+          run: |
+            table_content=$(cat deadlink_report.md)
+            echo "TABLE_CONTENT<<EOF" >> $GITHUB_ENV
+            echo "$table_content" >> $GITHUB_ENV
+            echo "EOF" >> $GITHUB_ENV
+  
+        - name: Create an issue
+          uses: dacbd/create-issue-action@main
+          with:
+            token: ${{ secrets.GITHUB_TOKEN }}
+            title: DeadLink Issue
+            body: ${{ env.TABLE_CONTENT }}

Rakefile

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 namespace :docs do
-  desc "Serve the documentation site"
+  desc 'Serve the documentation site'
   task :serve do
     within_docs_directory do
       unless system('bundle check')
@@ -11,44 +13,74 @@ namespace :docs do
     end
   end
 
-  desc "Install dependencies for the documentation site"
+  desc 'Install dependencies for the documentation site'
   task :install do
     within_docs_directory do
       sh 'bundle install'
     end
   end
 
-  desc "Generate usage documentation"
+  desc 'Generate usage documentation'
   task :generate_usage do
     output = `./bin/noir -h`
     cleaned_output = output.gsub(/\e\[[0-9;]*m/, '') # Remove ANSI color codes
     File.write('docs/_includes/usage.md', cleaned_output)
   end
 
-  def within_docs_directory
-    Dir.chdir('docs') do
-      yield
-    end
+  def within_docs_directory(&block)
+    Dir.chdir('docs', &block)
   rescue Errno::ENOENT => e
     puts "Directory 'docs' not found: #{e.message}"
     exit 1
-  rescue => e
+  rescue StandardError => e
     puts "An error occurred: #{e.message}"
     exit 1
   end
 end
 
 namespace :lint do
-  desc "Format the code using crystal tool format"
+  desc 'Format the code using crystal tool format'
   task :format do
     sh 'crystal tool format'
   end
 
-  desc "Lint the code using ameba"
+  desc 'Lint the code using ameba'
   task :ameba do
     sh 'ameba --fix'
   end
 
-  desc "Run all linting tasks"
-  task :all => [:format, :ameba]
+  desc 'Run all linting tasks'
+  task all: %i[format ameba]
+end
+
+namespace :completion do
+  desc 'Check for missing flags in completion scripts'
+  task :check do
+    # Extract flags from ./bin/noir -h
+    noir_help_output = `./bin/noir -h`
+    noir_flags = noir_help_output.scan(/^\s+(-\w|--\w[\w-]*)/).flatten.uniq
+
+    # Generate completion scripts
+    zsh_completion = `./bin/noir --generate-completion=zsh`
+    bash_completion = `./bin/noir --generate-completion=bash`
+    fish_completion = `./bin/noir --generate-completion=fish`
+
+    # Extract flags from generated completion scripts
+    completion_scripts = {
+      'zsh' => zsh_completion,
+      'bash' => bash_completion,
+      'fish' => fish_completion
+    }
+
+    completion_scripts.each do |shell, content|
+      missing_flags = noir_flags.reject { |flag| content.include?(flag) }
+
+      if missing_flags.empty?
+        puts "All flags are present in the #{shell} completion script."
+      else
+        puts "Missing flags in #{shell} completion script:"
+        missing_flags.each { |flag| puts flag }
+      end
+    end
+  end
 end

docs/_advanced/tips/community-articles.md

@@ -8,4 +8,5 @@ layout: page
 
 * [Hello Noir 👋🏼 by HAHWUL](https://www.hahwul.com/2023/08/03/hello-noir/)
 * [API Attack Surface Detection using Noir by DANA EPP](https://danaepp.com/api-attack-surface-detection-using-noir)
-* [Exploring OWASP Noir's PassiveScan by HAHWUL](https://www.hahwul.com/2024/11/03/passivescan-in-owasp-noir/)
+* [Exploring OWASP Noir's PassiveScan by HAHWUL](https://www.hahwul.com/2024/11/03/passivescan-in-owasp-noir/)
+* [Powering Up DAST with ZAP and Noir by ZAP Blog](https://www.zaproxy.org/blog/2024-11-11-powering-up-dast-with-zap-and-noir/)

docs/_advanced/tips/pipeline-for-dast.md

@@ -0,0 +1,33 @@
+---
+title: Pipeline for DAST
+parent: Tips
+has_children: false
+nav_order: 3
+layout: page
+---
+
+## Proxy Tool Integration
+
+This command automates the use of a proxy tool in a security testing pipeline. noir is used with the `-b` option to specify the base directory (.) and -u to target a local application (http://localhost.hahwul.com:3000). The --send-proxy parameter directs traffic to a proxy server running on http://localhost:8090. This setup allows for monitoring and intercepting HTTP requests through tools like ZAP, Caido, or Burp Suite during the testing process.
+
+```bash
+noir -b . -u http://localhost.hahwul.com:3000 --send-proxy "http://localhost:8090"
+```
+
+## ZAP Integration
+
+The process begins with endpoint discovery using noir, which scans the application source code in the specified directory (~/app_source), generates an OpenAPI specification (doc.json), and saves it in JSON format.
+
+Next, the doc.json file is used in an automated ZAP scan. The zap.sh script, with the `-openapifile` option, loads the generated endpoints and uses `-openapitargeturl` to specify the target URL for testing. The `-cmd` and `-autorun` options allow for automated execution of ZAP commands based on zap.yaml, along with any additional configuration parameters. This setup enables comprehensive vulnerability assessment across discovered endpoints in the target application.
+
+```bash
+# Discovering endpoints
+noir -b ~/app_source -f oas3 --no-log -o doc.json
+
+# Automation scan with endpoints
+./zap.sh -openapifile ./doc.json \
+    -openapitargeturl <TARGET> \
+    -cmd -autorun zap.yaml <any other ZAP options>
+```
+
+For further details on integrating Noir and ZAP for enhanced DAST capabilities, refer to the [Powering Up DAST with ZAP and Noir](https://www.zaproxy.org/blog/2024-11-11-powering-up-dast-with-zap-and-noir/) ZAP blog post.

docs/_advanced/tips/shell-completion.md

@@ -6,7 +6,7 @@ nav_order: 2
 layout: page
 ---
 
-## Zsh completion
+## Zsh Completion
 
 To enable auto-completion for Zsh, run the following command to generate the completion script:
 
@@ -23,7 +23,7 @@ Then, move the generated script to your Zsh completions directory, typically `~/
 noir --generate-completion zsh > ~/.zsh/completion/_noir
 ```
 
-## Bash completion
+## Bash Completion
 
 For Bash, generate the completion script by running:
 
@@ -40,7 +40,7 @@ After generating the script, move it to the appropriate directory for Bash compl
 noir --generate-completion bash > ~/.local/share/bash-completion/completions/noir
 ```
 
-## Fish completion
+## Fish Completion
 
 For Fish, generate the completion script by running:
 
@@ -54,4 +54,8 @@ After generating the script, move it to the Fish completions directory, typicall
 
 ```bash
 noir --generate-completion fish > ~/.config/fish/completions/noir.fish
-```
+```
+
+## Shell Completion for Homebrew Users
+
+For Homebrew users, shell completion (e.g., for Zsh, Bash, etc.) is installed automatically, and no additional configuration is needed. The completions are ready to use immediately after installation.

docs/_get_started/basic.md

@@ -24,9 +24,10 @@ noir -b <BASE_PATH>
 
 ## Outputs
 
-The output will display endpoints (such as paths, methods, parameters, headers, etc.), and you can specify the output format using flags `-f` or `--format`. If you're curious about the supported formats, please refer to [this](/get_started/output/) document.
+The output will display endpoints (such as paths, methods, parameters, headers, etc.), and you can specify the output format using flags `-f` or `--format`. If you're curious about the supported formats, please refer to [this](/noir/get_started/output/) document.
 
 ![](../../images/get_started/basic.png)
+*Example of plain output*
 
 ## Usage
 

shard.yml

@@ -1,6 +1,6 @@
 # Project Metadata
 name: noir
-version: 0.18.2
+version: 0.18.3
 authors:
   - hahwul <hahwul@gmail.com>
   - ksg97031 <ksg97031@gmail.com>

snap/snapcraft.yaml

@@ -1,6 +1,6 @@
 name: noir
 base: core20
-version: 0.18.2
+version: 0.18.3
 summary: Attack surface detector that identifies endpoints by static analysis.
 description: |
   Noir is an open-source project specializing in identifying attack surfaces for enhanced whitebox security testing and security pipeline. 

spec/functional_test/fixtures/java/spring/src/RequestMethodClass.java

@@ -0,0 +1,13 @@
+package com.test;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RequestMapping(value = "/empty") // Comment
+
+public class RequestMethodClass {
+
+	@GetMapping("")
+	public void getData() {
+	}
+}

spec/functional_test/testers/java/spring_spec.cr

@@ -34,6 +34,8 @@ extected_endpoints = [
     Param.new("b", "", "query"),
     Param.new("name", "", "query"),
   ]),
+  # EmptyController.java
+  Endpoint.new("/empty", "GET"),
 ]
 
 FunctionalTester.new("fixtures/java/spring/", {

src/analyzer/analyzers/java/spring.cr

@@ -143,11 +143,13 @@ module Analyzer::Java
               class_annotation = class_model.annotations["RequestMapping"]?
               if !class_annotation.nil?
                 next if class_annotation.params.size == 0
-                class_path_token = class_annotation.params[0][-1]
-                if class_path_token.type == :STRING_LITERAL
-                  url = class_path_token.value[1..-2]
-                  if url.ends_with? "*"
-                    url = url[0..-2]
+                if class_annotation.params[0].size > 0
+                  class_path_token = class_annotation.params[0][-1]
+                  if class_path_token.type == :STRING_LITERAL
+                    url = class_path_token.value[1..-2]
+                    if url.ends_with? "*"
+                      url = url[0..-2]
+                    end
                   end
                 end
               end

src/analyzer/analyzers/python/django.cr

@@ -6,6 +6,7 @@ module Analyzer::Python
   class Django < Python
     # Base path for the Django project
     @django_base_path : ::String = ""
+    @visited_url_paths = Hash(String, Bool).new
 
     # Regular expressions for extracting Django URL configurations
     REGEX_ROOT_URLCONF  = /\s*ROOT_URLCONF\s*=\s*r?['"]([^'"\\]*)['"]/
@@ -98,6 +99,7 @@ module Analyzer::Python
       logger.debug "Extracting endpoints from #{django_urls.filepath}"
       endpoints = [] of Endpoint
       url_base_path = File.dirname(django_urls.filepath)
+      @visited_url_paths[django_urls.filepath] = true
 
       file = File.open(django_urls.filepath, encoding: "utf-8", invalid: :skip)
       content = file.gets_to_end
@@ -129,7 +131,7 @@ module Analyzer::Python
           if File.exists?(new_route_path)
             new_django_urls = DjangoUrls.new("#{django_urls.prefix}#{route}", new_route_path, django_urls.basepath)
             details = Details.new(PathInfo.new(new_route_path))
-            if new_django_urls.filepath != django_urls.filepath
+            unless @visited_url_paths.has_key? new_django_urls.filepath
               extract_endpoints(new_django_urls).each do |endpoint|
                 endpoint.details = details
                 endpoints << endpoint
@@ -159,7 +161,7 @@ module Analyzer::Python
             end
           end
 
-          if filepath != ""
+          if filepath != "" && /^[a-zA-Z_][a-zA-Z0-9_]*$/.match(function_or_class_name)
             extract_endpoints_from_file(url, filepath, function_or_class_name).each do |endpoint|
               endpoint.details = details
               endpoints << endpoint

src/minilexers/java.cr

@@ -217,7 +217,6 @@ class JavaLexer < MiniLexer
         @position += 2
         while @position < @input.size
           if @input[@position] == '\n'
-            @position += 1
             break
           end
           @position += 1