From c918430288fa00cbf002ef307bab10e118ec73af Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Fri, 9 Feb 2024 11:06:09 -0300
Subject: [PATCH 1/7] Summary - Audit Status

Implementation of a system for auditing stage status.

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 .../20240207070831_create_audit_status.rb     | 21 ++++++
 db/schema.rb                                  | 12 +++-
 lib/github/build/action.rb                    |  2 +-
 lib/github/build/summary.rb                   | 37 +++++++----
 lib/github/build/unavailable_jobs.rb          |  2 +-
 lib/github/check.rb                           | 11 ++++
 lib/github/update_status.rb                   |  6 +-
 lib/models/audit_status.rb                    | 17 +++++
 lib/models/ci_job.rb                          | 24 ++++---
 lib/models/stage.rb                           | 38 +++++++++--
 spec/lib/github/build/summary_spec.rb         | 65 ++++++++++++++++++-
 spec/lib/github/update_status_spec.rb         |  4 +-
 workers/watch_dog.rb                          |  8 +--
 13 files changed, 208 insertions(+), 39 deletions(-)
 create mode 100644 db/migrate/20240207070831_create_audit_status.rb
 create mode 100644 lib/models/audit_status.rb

diff --git a/db/migrate/20240207070831_create_audit_status.rb b/db/migrate/20240207070831_create_audit_status.rb
new file mode 100644
index 0000000..a6686c9
--- /dev/null
+++ b/db/migrate/20240207070831_create_audit_status.rb
@@ -0,0 +1,21 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  20231214093515_create_stages.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2023 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class CreateAuditStatus < ActiveRecord::Migration[6.0]
+  def change
+    create_table :audit_statuses do |t|
+      t.integer :status, null: false, default: 0
+      t.string :agent
+      t.datetime :created_at, null: false, precision: 6
+
+      t.references :auditable, polymorphic: true, index: true, null: false
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 9019ce1..779ce4e 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,10 +10,20 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_12_14_094534) do
+ActiveRecord::Schema[7.0].define(version: 2024_02_07_070831) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "audit_statuses", force: :cascade do |t|
+    t.integer "status", default: 0, null: false
+    t.string "agent"
+    t.datetime "created_at", null: false
+    t.string "auditable_type", null: false
+    t.bigint "auditable_id", null: false
+
+    t.index ["auditable_type", "auditable_id"], name: "index_audit_statuses_on_auditable_type_and_auditable_id"
+  end
+
   create_table "check_suites", force: :cascade do |t|
     t.string "author", null: false
     t.string "commit_sha_ref", null: false
diff --git a/lib/github/build/action.rb b/lib/github/build/action.rb
index 03c6415..8b0e496 100644
--- a/lib/github/build/action.rb
+++ b/lib/github/build/action.rb
@@ -59,7 +59,7 @@ def stage_with_start_in_progress(ci_job)
         return unless !ci_job.stage.nil? and ci_job.stage.configuration.start_in_progress?
 
         ci_job.in_progress(@github)
-        ci_job.stage.in_progress(@github, output: {}, job: ci_job)
+        ci_job.stage.in_progress(@github, output: {})
       end
 
       def create_ci_job(job)
diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb
index ff7a4db..02e52a9 100644
--- a/lib/github/build/summary.rb
+++ b/lib/github/build/summary.rb
@@ -16,11 +16,12 @@
 module Github
   module Build
     class Summary
-      def initialize(job, logger_level: Logger::INFO)
+      def initialize(job, logger_level: Logger::INFO, agent: 'Github')
         @job = job.reload
         @check_suite = @job.check_suite
         @github = Github::Check.new(@check_suite)
         @loggers = []
+        @agent = agent
 
         %w[github_app.log github_build_summary.log].each do |filename|
           @loggers << GithubLogger.instance.create(filename, logger_level)
@@ -30,6 +31,7 @@ def initialize(job, logger_level: Logger::INFO)
       def build_summary
         current_stage = @job.stage
         current_stage = fetch_parent_stage if current_stage.nil?
+        check_and_update_github_ref(current_stage)
 
         logger(Logger::INFO, "build_summary: #{current_stage.inspect}")
 
@@ -48,6 +50,22 @@ def build_summary
 
       private
 
+      def check_and_update_github_ref(current_stage)
+        current_refs = @github.fetch_check_runs
+
+        logger(Logger::INFO,
+               'check_and_update_github_ref - current_stage: ' \
+               "#{current_stage.inspect} current_refs: #{current_refs.inspect}")
+
+        return if current_refs.include? current_stage.check_ref.to_i
+
+        logger(Logger::INFO,
+               'check_and_update_github_ref - current_stage: ' \
+               "#{current_stage.inspect} current_refs: #{current_refs.inspect} - Refreshing reference.")
+
+        current_stage.refresh_reference(@github)
+      end
+
       def must_update_previous_stage(current_stage)
         previous_stage = current_stage.previous_stage
 
@@ -87,14 +105,6 @@ def must_continue_next_stage(current_stage)
         update_summary(next_stage)
       end
 
-      def bamboo_stage_check_positions(pending_stage, stage)
-        pending_stage_position = pending_stage.configuration.position
-        stage_position = stage.configuration.position
-
-        pending_stage_position <= stage_position or
-          pending_stage_position + 1 != stage_position
-      end
-
       def cancelling_next_stage(pending_stage)
         url = "https://ci1.netdef.org/browse/#{pending_stage.check_suite.bamboo_ci_ref}"
         output = {
@@ -130,7 +140,11 @@ def finished_stage_summary(stage)
 
         logger(Logger::INFO, "finished_stage_summary: #{stage.inspect} #{output.inspect}")
 
-        stage.jobs.failure.empty? ? stage.success(@github, output: output) : stage.failure(@github, output: output)
+        if stage.jobs.failure.empty?
+          stage.success(@github, output: output, agent: @agent)
+        else
+          stage.failure(@github, output: output, agent: @agent)
+        end
       end
 
       def update_summary(stage)
@@ -144,7 +158,8 @@ def update_summary(stage)
 
         logger(Logger::INFO, "update_summary: #{stage.inspect} #{output.inspect}")
 
-        stage.in_progress(@github, output: output, job: @job)
+        stage.in_progress(@github, output: output)
+        stage.update_output(@github, output: output)
       end
 
       def summary_basic_output(stage)
diff --git a/lib/github/build/unavailable_jobs.rb b/lib/github/build/unavailable_jobs.rb
index 82fa13a..5c30fbd 100644
--- a/lib/github/build/unavailable_jobs.rb
+++ b/lib/github/build/unavailable_jobs.rb
@@ -31,7 +31,7 @@ def update(new_check_suite: nil)
           BambooCi::RunningPlan.fetch(@check_suite.bamboo_ci_ref).map { |entry| entry[:job_ref] }
 
         @check_suite.ci_jobs.where.not(job_ref: running_jobs).each do |unavailable_job|
-          unavailable_job.skipped(@github, output(unavailable_job))
+          unavailable_job.skipped(@github, output: output(unavailable_job))
           unavailable_job.update(check_suite: new_check_suite) unless new_check_suite.nil?
         end
       end
diff --git a/lib/github/check.rb b/lib/github/check.rb
index 3e914ed..9c9fdd0 100644
--- a/lib/github/check.rb
+++ b/lib/github/check.rb
@@ -89,6 +89,17 @@ def get_check_run(check_ref)
       @app.check_run(@check_suite.pull_request.repository, check_ref).to_h
     end
 
+    def fetch_check_runs
+      return [] if @check_suite.nil?
+
+      @app
+        .check_runs_for_ref(@check_suite.pull_request.repository, @check_suite.pull_request.branch_name)
+        .to_h[:check_runs]
+        .map do |check_run|
+        check_run[:id]
+      end
+    end
+
     def installation_id
       @authenticate_app.find_app_installations.first['id'].to_i
     end
diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb
index 9949392..2e29449 100644
--- a/lib/github/update_status.rb
+++ b/lib/github/update_status.rb
@@ -60,9 +60,9 @@ def failures_stats
     def update_status
       case @status
       when 'in_progress'
-        @job.in_progress(@github_check, @output)
+        @job.in_progress(@github_check, output: @output)
       when 'success'
-        @job.success(@github_check, @output)
+        @job.success(@github_check, output: @output)
         slack_notify_success
       else
         failure
@@ -101,7 +101,7 @@ def failure
       unable2find = "There was some test that failed, but I couldn't find the log."
       fetch_and_update_failures(unable2find) if !@output.empty? and @output[:summary].match?(unable2find)
 
-      @job.failure(@github_check, @output)
+      @job.failure(@github_check, output: @output)
       failures_stats if @job.name.downcase.match? 'topotest' and @failures.is_a? Array
     end
 
diff --git a/lib/models/audit_status.rb b/lib/models/audit_status.rb
new file mode 100644
index 0000000..5ec16f8
--- /dev/null
+++ b/lib/models/audit_status.rb
@@ -0,0 +1,17 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  audit_status.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+require 'otr-activerecord'
+
+class AuditStatus < ActiveRecord::Base
+  enum status: { queued: 0, in_progress: 1, success: 2, refresh: 3, cancelled: -1, failure: -2, skipped: -3 }
+
+  belongs_to :auditable, polymorphic: true
+end
diff --git a/lib/models/ci_job.rb b/lib/models/ci_job.rb
index d74d931..c4665dc 100644
--- a/lib/models/ci_job.rb
+++ b/lib/models/ci_job.rb
@@ -16,9 +16,10 @@ class CiJob < ActiveRecord::Base
   validates :name, presence: true
   validates :job_ref, presence: true
 
-  belongs_to :check_suite
   has_many :topotest_failures, dependent: :delete_all
+  has_many :audit_statuses, as: :auditable
   belongs_to :stage
+  belongs_to :check_suite
 
   scope :sha256, ->(sha) { joins(:check_suite).where(check_suite: { commit_sha_ref: sha }) }
   scope :filter_by, ->(filter) { where('name ~ ?', filter) }
@@ -32,56 +33,63 @@ def finished?
     !%w[queued in_progress].include?(status)
   end
 
-  def create_check_run
+  def create_check_run(agent: 'Github')
+    AuditStatus.create(auditable: self, status: :queued, agent: agent, created_at: Time.now)
     update(status: :queued)
   end
 
-  def enqueue(_github, _output = {})
+  def enqueue(_github, _output = {}, agent: 'Github')
+    AuditStatus.create(auditable: self, status: :queued, agent: agent, created_at: Time.now)
     update(status: :queued)
   end
 
-  def in_progress(github, output = {})
+  def in_progress(github, output: {}, agent: 'Github')
     unless check_ref.nil?
       create_github_check(github)
       github.in_progress(check_ref, output)
     end
 
+    AuditStatus.create(auditable: self, status: :in_progress, agent: agent, created_at: Time.now)
     update(status: :in_progress)
   end
 
-  def cancelled(github, output = {})
+  def cancelled(github, output: {}, agent: 'Github')
     unless check_ref.nil?
       create_github_check(github)
       github.cancelled(check_ref, output)
     end
 
+    AuditStatus.create(auditable: self, status: :cancelled, agent: agent, created_at: Time.now)
     update(status: :cancelled)
   end
 
-  def failure(github, output = {})
+  def failure(github, output: {}, agent: 'Github')
     unless check_ref.nil?
       create_github_check(github)
       github.failure(check_ref, output)
     end
 
+    AuditStatus.create(auditable: self, status: :failure, agent: agent, created_at: Time.now)
     update(status: :failure)
   end
 
-  def success(github, output = {})
+  def success(github, output: {}, agent: 'Github')
     unless check_ref.nil?
       create_github_check(github)
       github.success(check_ref, output)
     end
 
+    AuditStatus.create(auditable: self, status: :success, agent: agent, created_at: Time.now)
     update(status: :success)
   end
 
-  def skipped(github, output = {})
+  def skipped(github, output: {}, agent: 'Github')
     unless check_ref.nil?
       create_github_check(github)
       github.skipped(check_ref, output)
     end
 
+    AuditStatus.create(auditable: self, status: :skipped, agent: agent, created_at: Time.now)
     update(status: :skipped)
   end
 
diff --git a/lib/models/stage.rb b/lib/models/stage.rb
index 455156f..26d6b14 100644
--- a/lib/models/stage.rb
+++ b/lib/models/stage.rb
@@ -12,6 +12,7 @@ class Stage < ActiveRecord::Base
   enum status: { queued: 0, in_progress: 1, success: 2, cancelled: -1, failure: -2, skipped: -3 }
 
   has_many :jobs, class_name: 'CiJob'
+  has_many :audit_statuses, as: :auditable
   belongs_to :configuration, class_name: 'StageConfiguration', foreign_key: 'stage_configuration_id'
   belongs_to :check_suite
 
@@ -28,42 +29,65 @@ def finished?
     jobs.where(status: %w[queued in_progress]).empty?
   end
 
-  def enqueue(github, output: {})
+  def enqueue(github, output: {}, agent: 'Github')
     check_run = github.create(github_stage_full_name(name))
     github.queued(check_run.id, output)
+    AuditStatus.create(auditable: self, status: :queued, agent: agent, created_at: Time.now)
     update(check_ref: check_run.id, status: :queued)
   end
 
-  def in_progress(github, output: output_in_progress, job: nil)
+  def in_progress(github, output: output_in_progress, agent: 'Github')
+    return if in_progress?
+
     create_github_check(github)
     github.in_progress(check_ref, output)
 
-    in_progress_notification if !job.nil? and first_job == job
-
+    AuditStatus.create(auditable: self, status: :in_progress, agent: agent, created_at: Time.now)
     update(status: :in_progress)
+
+    in_progress_notification
   end
 
-  def cancelled(github, output: {})
+  def update_output(github, output: output_in_progress)
+    github.in_progress(check_ref, output)
+  end
+
+  def cancelled(github, output: {}, agent: 'Github')
+    return if cancelled?
+
     create_github_check(github)
     github.cancelled(check_ref, output)
     update(status: :cancelled)
+    AuditStatus.create(auditable: self, status: :cancelled, agent: agent, created_at: Time.now)
     notification
   end
 
-  def failure(github, output: {})
+  def failure(github, output: {}, agent: 'Github')
+    return if failure?
+
     create_github_check(github)
     github.failure(check_ref, output)
     update(status: :failure)
+    AuditStatus.create(auditable: self, status: :failure, agent: agent, created_at: Time.now)
     notification
   end
 
-  def success(github, output: {})
+  def success(github, output: {}, agent: 'Github')
+    return if success?
+
     create_github_check(github)
     github.success(check_ref, output)
     update(status: :success)
+    AuditStatus.create(auditable: self, status: :success, agent: agent, created_at: Time.now)
     notification
   end
 
+  def refresh_reference(github, agent: 'Github')
+    check_run = github.create(github_stage_full_name(name))
+    update(check_ref: check_run.id)
+    AuditStatus.create(auditable: self, status: :refresh, agent: agent, created_at: Time.now)
+  end
+
   private
 
   def in_progress_notification
diff --git a/spec/lib/github/build/summary_spec.rb b/spec/lib/github/build/summary_spec.rb
index 6af4513..ab95dc6 100644
--- a/spec/lib/github/build/summary_spec.rb
+++ b/spec/lib/github/build/summary_spec.rb
@@ -12,7 +12,8 @@
   let(:summary) { described_class.new(ci_job) }
   let(:fake_client) { Octokit::Client.new }
   let(:fake_github_check) { Github::Check.new(nil) }
-  let(:check_suite) { create(:check_suite) }
+  let(:pull_request) { create(:pull_request) }
+  let(:check_suite) { create(:check_suite, pull_request: pull_request) }
   let(:position1) { BambooStageTranslation.find_by_position(1) }
   let(:position2) { BambooStageTranslation.find_by_position(2) }
   let(:parent_stage1) { create(:stage, check_suite: check_suite, name: position1.github_check_run_name) }
@@ -138,6 +139,68 @@
     end
   end
 
+  context 'when the tests stage finished unsuccessfully and build_message returns null' do
+    let(:first_stage_config) { create(:stage_configuration, position: 1) }
+    let(:second_stage_config) { create(:stage_configuration, position: 2) }
+    let(:first_stage) { create(:stage, configuration: first_stage_config, check_suite: check_suite) }
+    let(:second_stage) { create(:stage, name: 'Build', configuration: second_stage_config, check_suite: check_suite) }
+    let(:ci_job2) { create(:ci_job, :success, check_suite: check_suite, stage: first_stage) }
+    let(:ci_job) { create(:ci_job, :failure, name: 'Ubuntu Build', check_suite: check_suite, stage: second_stage) }
+
+    before do
+      ci_job
+      ci_job2
+
+      allow(BambooCi::Result).to receive(:fetch).and_return({})
+    end
+
+    it 'must update stage' do
+      summary.build_summary
+      expect(ci_job.stage.reload.status).to eq('failure')
+      expect(ci_job2.stage.reload.status).to eq('success')
+    end
+  end
+
+  context 'when the tests stage finished unsuccessfully and build_message returns errorlog' do
+    let(:first_stage_config) { create(:stage_configuration, position: 1) }
+    let(:second_stage_config) { create(:stage_configuration, position: 2) }
+    let(:first_stage) { create(:stage, configuration: first_stage_config, check_suite: check_suite) }
+    let(:second_stage) { create(:stage, name: 'Build', configuration: second_stage_config, check_suite: check_suite) }
+    let(:ci_job2) { create(:ci_job, :success, check_suite: check_suite, stage: first_stage) }
+    let(:ci_job) { create(:ci_job, :failure, name: 'Ubuntu Build', check_suite: check_suite, stage: second_stage) }
+
+    let(:bamboo_result) do
+      {
+        'artifacts' =>
+          {
+            'artifact' =>
+              [
+                {
+                  'name' => 'ErrorLog',
+                  'link' => {
+                    'href' => 'https://ci1.netdef.org/browse/UBUNTU-BUILD-1/artifact/shared/ErrorLog/ErrorLog'
+                  }
+                }
+              ]
+          }
+      }
+    end
+
+    before do
+      ci_job
+      ci_job2
+
+      allow(BambooCi::Result).to receive(:fetch).and_return(bamboo_result)
+      allow(BambooCi::Download).to receive(:build_log).and_return('ErrorLog')
+    end
+
+    it 'must update stage' do
+      summary.build_summary
+      expect(ci_job.stage.reload.status).to eq('failure')
+      expect(ci_job2.stage.reload.status).to eq('success')
+    end
+  end
+
   context 'when the tests stage still running' do
     let(:first_stage_config) { create(:stage_configuration, position: 1) }
     let(:first_stage) { create(:stage, configuration: first_stage_config, check_suite: check_suite) }
diff --git a/spec/lib/github/update_status_spec.rb b/spec/lib/github/update_status_spec.rb
index e6b158c..551558c 100644
--- a/spec/lib/github/update_status_spec.rb
+++ b/spec/lib/github/update_status_spec.rb
@@ -271,7 +271,7 @@
         end
 
         it 'must update the output' do
-          expect(ci_job).to have_received(:failure).with(fake_github_check, expected_output)
+          expect(ci_job).to have_received(:failure).with(fake_github_check, output: expected_output)
         end
 
         it 'must create TopoTestFailure' do
@@ -318,7 +318,7 @@
         end
 
         it 'must maintain the same output' do
-          expect(ci_job).to have_received(:failure).with(fake_github_check, expected_output)
+          expect(ci_job).to have_received(:failure).with(fake_github_check, output: expected_output)
         end
 
         it 'must not create a TopoTestFailure' do
diff --git a/workers/watch_dog.rb b/workers/watch_dog.rb
index d8c77aa..ed8bda7 100644
--- a/workers/watch_dog.rb
+++ b/workers/watch_dog.rb
@@ -114,13 +114,13 @@ def update_ci_job_status(github_check, ci_job, state)
     @logger.info ">>> CiJob: #{ci_job.inspect} updating status"
     case state
     when 'Unknown'
-      ci_job.cancelled(github_check, output)
+      ci_job.cancelled(github_check, output: output, agent: 'WatchDog')
       slack_notify_cancelled(ci_job)
     when 'Failed'
-      ci_job.failure(github_check, output)
+      ci_job.failure(github_check, output: output, agent: 'WatchDog')
       slack_notify_failure(ci_job)
     when 'Successful'
-      ci_job.success(github_check, output)
+      ci_job.success(github_check, output: output, agent: 'WatchDog')
       slack_notify_success(ci_job)
     else
       puts 'Ignored'
@@ -139,7 +139,7 @@ def create_output_message(ci_job)
   end
 
   def build_summary(ci_job)
-    summary = Github::Build::Summary.new(ci_job)
+    summary = Github::Build::Summary.new(ci_job, agent: 'WatchDog')
     summary.build_summary
 
     finished_execution?(ci_job.check_suite)

From e89b29e97a6e3b1301f701262ece49d01e0db874 Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Fri, 9 Feb 2024 16:16:54 -0300
Subject: [PATCH 2/7] Summary - Audit Status

Fixing schema

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 db/schema.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/db/schema.rb b/db/schema.rb
index 779ce4e..d6fb9a9 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -20,7 +20,6 @@
     t.datetime "created_at", null: false
     t.string "auditable_type", null: false
     t.bigint "auditable_id", null: false
-
     t.index ["auditable_type", "auditable_id"], name: "index_audit_statuses_on_auditable_type_and_auditable_id"
   end
 

From 44cebbbf63bfbb5d8b09a0bc0fd3a887ccf2404d Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Wed, 14 Feb 2024 10:00:57 -0300
Subject: [PATCH 3/7] Summary - Audit Status

Fixing unit tests

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 spec/lib/models/stage_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/lib/models/stage_spec.rb b/spec/lib/models/stage_spec.rb
index ae07fd6..4f44173 100644
--- a/spec/lib/models/stage_spec.rb
+++ b/spec/lib/models/stage_spec.rb
@@ -81,7 +81,7 @@
       let(:stage) { create(:stage, :with_check_suite, :with_job, check_ref: nil) }
 
       it 'must update status' do
-        stage.in_progress(github, job: stage.jobs.first)
+        stage.in_progress(github)
         expect(stage.reload.status).to eq('in_progress')
       end
     end

From 0d8dfa25ac96adfa7ed1449bc106c6ff1099140b Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Wed, 14 Feb 2024 10:14:52 -0300
Subject: [PATCH 4/7] Summary - Audit Status

Updating unit tests and removing useless code

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 lib/models/stage.rb           |  8 --------
 spec/lib/models/stage_spec.rb | 12 ++++++++++++
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/lib/models/stage.rb b/lib/models/stage.rb
index 26d6b14..7b23dac 100644
--- a/lib/models/stage.rb
+++ b/lib/models/stage.rb
@@ -94,14 +94,6 @@ def in_progress_notification
     SlackBot.instance.stage_in_progress_notification(self)
   end
 
-  def first_job
-    jobs
-      .reload
-      .where.not(status: %i[success failure cancelled skipped])
-      .order('ci_jobs.id')
-      .first
-  end
-
   def notification
     SlackBot.instance.stage_finished_notification(self)
   end
diff --git a/spec/lib/models/stage_spec.rb b/spec/lib/models/stage_spec.rb
index 4f44173..a53ec5a 100644
--- a/spec/lib/models/stage_spec.rb
+++ b/spec/lib/models/stage_spec.rb
@@ -94,6 +94,12 @@
       stage.failure(github)
       expect(stage.reload.status).to eq('failure')
     end
+
+    it 'must only update 1 time' do
+      stage.failure(github)
+      stage.failure(github)
+      expect(stage.reload.status).to eq('failure')
+    end
   end
 
   describe '#success' do
@@ -103,5 +109,11 @@
       stage.success(github)
       expect(stage.reload.status).to eq('success')
     end
+
+    it 'must only update 1 time' do
+      stage.success(github)
+      stage.success(github)
+      expect(stage.reload.status).to eq('success')
+    end
   end
 end

From e0c666076804e9e8dc144ab26d3580db63971a9a Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Fri, 16 Feb 2024 17:54:43 -0300
Subject: [PATCH 5/7] Summary - Audit Status

Adding extra logs and PR log

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 lib/github/build/action.rb  |  3 ++-
 lib/github/build/retry.rb   |  2 ++
 lib/github/build/summary.rb | 13 ++++++++----
 lib/github/update_status.rb | 40 ++++++++++++++++++++++++++++++-------
 4 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/lib/github/build/action.rb b/lib/github/build/action.rb
index 8b0e496..2db2e7c 100644
--- a/lib/github/build/action.rb
+++ b/lib/github/build/action.rb
@@ -21,8 +21,9 @@ def initialize(check_suite, github, jobs, logger_level: Logger::INFO)
         %w[github_app.log github_build_action.log].each do |filename|
           @loggers << GithubLogger.instance.create(filename, logger_level)
         end
+        @loggers << GithubLogger.instance.create("pr#{@check_suite.pull_request.github_pr_id}.log", logger_level)
 
-        logger(Logger::INFO, "Building action to CheckSuite @#{@check_suite.inspect}")
+        logger(Logger::WARN, ">>>> Building action to CheckSuite: #{@check_suite.inspect}")
       end
 
       def create_summary(rerun: false)
diff --git a/lib/github/build/retry.rb b/lib/github/build/retry.rb
index ef2f1d0..09590d9 100644
--- a/lib/github/build/retry.rb
+++ b/lib/github/build/retry.rb
@@ -22,6 +22,8 @@ def initialize(check_suite, github, logger_level: Logger::INFO)
         %w[github_app.log github_build_retry.log].each do |filename|
           @loggers << GithubLogger.instance.create(filename, logger_level)
         end
+        @loggers << GithubLogger.instance.create("pr#{@check_suite.pull_request.github_pr_id}.log", logger_level)
+        logger(Logger::WARN, ">>>> Retrying check_suite: #{@check_suite.inspect}")
       end
 
       def enqueued_stages
diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb
index 02e52a9..c3f2b45 100644
--- a/lib/github/build/summary.rb
+++ b/lib/github/build/summary.rb
@@ -26,6 +26,8 @@ def initialize(job, logger_level: Logger::INFO, agent: 'Github')
         %w[github_app.log github_build_summary.log].each do |filename|
           @loggers << GithubLogger.instance.create(filename, logger_level)
         end
+
+        @loggers << GithubLogger.instance.create("pr#{@check_suite.pull_request.github_pr_id}.log", logger_level)
       end
 
       def build_summary
@@ -122,7 +124,7 @@ def cancelling_next_stage(pending_stage)
 
       def finished_summary(stage)
         logger(Logger::INFO, "Finished stage: #{stage.inspect}, CiJob status: #{@job.status}")
-        logger(Logger::INFO, "Finished stage: #{stage.inspect}, jobs: #{stage.reload.jobs.inspect}")
+        logger(Logger::INFO, "Finished stage: #{stage.inspect}, running? #{stage.reload.running?}")
 
         return if @job.in_progress? or stage.running?
 
@@ -130,7 +132,7 @@ def finished_summary(stage)
       end
 
       def finished_stage_summary(stage)
-        logger(Logger::INFO, "finished_build_summary: #{stage.inspect}. Reason Job: #{@job.inspect}")
+        logger(Logger::INFO, "finished_stage_summary: #{stage.inspect}. Reason Job: #{@job.inspect}")
 
         url = "https://ci1.netdef.org/browse/#{stage.check_suite.bamboo_ci_ref}"
         output = {
@@ -138,13 +140,15 @@ def finished_stage_summary(stage)
           summary: "#{summary_basic_output(stage)}\nDetails at [#{url}](#{url}).".force_encoding('utf-8')
         }
 
-        logger(Logger::INFO, "finished_stage_summary: #{stage.inspect} #{output.inspect}")
-
         if stage.jobs.failure.empty?
+          logger(Logger::WARN, "Stage: #{stage.name} finished - failure")
           stage.success(@github, output: output, agent: @agent)
         else
+          logger(Logger::WARN, "Stage: #{stage.name} finished - success")
           stage.failure(@github, output: output, agent: @agent)
         end
+
+        logger(Logger::INFO, "finished_stage_summary: #{stage.inspect} #{output.inspect}")
       end
 
       def update_summary(stage)
@@ -158,6 +162,7 @@ def update_summary(stage)
 
         logger(Logger::INFO, "update_summary: #{stage.inspect} #{output.inspect}")
 
+        logger(Logger::WARN, "Updating stage: #{stage.name} to in_progress")
         stage.in_progress(@github, output: output)
         stage.update_output(@github, output: output)
       end
diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb
index 2e29449..0bb137a 100644
--- a/lib/github/update_status.rb
+++ b/lib/github/update_status.rb
@@ -19,7 +19,6 @@ module Github
   class UpdateStatus
     def initialize(payload)
       @status = payload['status']
-      @logger = GithubLogger.instance.create('github_update_status.log', Logger::INFO)
 
       @output =
         if payload.dig('output', 'title').nil? and payload.dig('output', 'summary').nil?
@@ -28,13 +27,18 @@ def initialize(payload)
           { title: payload.dig('output', 'title'), summary: payload.dig('output', 'summary') }
         end
 
+      @reference = payload['bamboo_ref'] || 'invalid_reference'
       @job = CiJob.find_by(job_ref: payload['bamboo_ref'])
       @check_suite = @job&.check_suite
       @failures = payload['failures']
+
+      logger_initializer
+      logger(Logger::WARN, "UpdateStatus: #{@reference} #{@status} (Output in info log)")
+      logger(Logger::INFO, "UpdateStatus: #{@reference} #{@status} #{@output}")
     end
 
     def update
-      return [404, 'CI JOB not found'] if @job.nil?
+      return job_not_found if @job.nil?
       return [304, 'Not Modified'] if @job.queued? and @status != 'in_progress' and @job.name != 'Checkout Code'
       return [304, 'Not Modified'] if @job.in_progress? and !%w[success failure].include? @status
 
@@ -47,6 +51,12 @@ def update
 
     private
 
+    def job_not_found
+      logger(Logger::ERROR, "CI JOB not found: '#{@reference}'")
+
+      [404, 'CI JOB not found']
+    end
+
     def failures_stats
       @failures.each do |failure|
         TopotestFailure.create(ci_job: @job,
@@ -79,8 +89,8 @@ def finished_execution?
       return false unless current_execution?
       return false unless @check_suite.finished?
 
-      @logger.info ">>> @check_suite#{@check_suite.inspect} -> finished? #{@check_suite.finished?}"
-      @logger.info @check_suite.ci_jobs.last.inspect
+      logger Logger::INFO, ">>> @check_suite#{@check_suite.inspect} -> finished? #{@check_suite.finished?}"
+      logger Logger::INFO, @check_suite.ci_jobs.last.inspect
 
       SlackBot.instance.execution_finished_notification(@check_suite)
     end
@@ -89,8 +99,8 @@ def current_execution?
       pull_request = @check_suite.pull_request
       last_check_suite = pull_request.check_suites.reload.all.order(:created_at).last
 
-      @logger.info "last_check_suite: #{last_check_suite.inspect}"
-      @logger.info "@check_suite: #{@check_suite.inspect}"
+      logger Logger::INFO, "last_check_suite: #{last_check_suite.inspect}"
+      logger Logger::INFO, "@check_suite: #{@check_suite.inspect}"
 
       @check_suite.id == last_check_suite.id
     end
@@ -113,7 +123,7 @@ def fetch_and_update_failures(to_be_replaced)
 
         @output[:summary] = @output[:summary].sub(to_be_replaced, fetch_failures(output))[0..65_535]
       rescue NoMethodError => e
-        @logger.error "#{e.class} #{e.message}"
+        logger Logger::ERROR, "#{e.class} #{e.message}"
         count += 1
         sleep 5
         retry if count <= 10
@@ -151,5 +161,21 @@ def slack_notify_failure
 
       SlackBot.instance.notify_errors(@job)
     end
+
+    def logger(severity, message)
+      @loggers.each do |logger_object|
+        logger_object.add(severity, message)
+      end
+    end
+
+    def logger_initializer
+      @loggers = []
+      @loggers << GithubLogger.instance.create('github_update_status.log', Logger::INFO)
+      @loggers << if @job.nil?
+                    GithubLogger.instance.create(@reference, Logger::INFO)
+                  else
+                    GithubLogger.instance.create("pr#{@job.check_suite.pull_request.github_pr_id}.log", Logger::INFO)
+                  end
+    end
   end
 end

From 89e3da10bb79131a04fb8686be40906eee00fc01 Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Fri, 16 Feb 2024 17:56:29 -0300
Subject: [PATCH 6/7] Summary - Audit Status

Fixing rubocop issue

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 lib/github/build/summary.rb | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb
index c3f2b45..00883f6 100644
--- a/lib/github/build/summary.rb
+++ b/lib/github/build/summary.rb
@@ -140,6 +140,12 @@ def finished_stage_summary(stage)
           summary: "#{summary_basic_output(stage)}\nDetails at [#{url}](#{url}).".force_encoding('utf-8')
         }
 
+        finished_stage_update(stage)
+
+        logger(Logger::INFO, "finished_stage_summary: #{stage.inspect} #{output.inspect}")
+      end
+
+      def finished_stage_update(stage)
         if stage.jobs.failure.empty?
           logger(Logger::WARN, "Stage: #{stage.name} finished - failure")
           stage.success(@github, output: output, agent: @agent)
@@ -147,8 +153,6 @@ def finished_stage_summary(stage)
           logger(Logger::WARN, "Stage: #{stage.name} finished - success")
           stage.failure(@github, output: output, agent: @agent)
         end
-
-        logger(Logger::INFO, "finished_stage_summary: #{stage.inspect} #{output.inspect}")
       end
 
       def update_summary(stage)

From 7838be83e241bdc59dbc40fb4988d63e38068004 Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Fri, 16 Feb 2024 18:03:20 -0300
Subject: [PATCH 7/7] Summary - Audit Status

Fixing method parameter

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 lib/github/build/summary.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb
index 00883f6..15cb9d2 100644
--- a/lib/github/build/summary.rb
+++ b/lib/github/build/summary.rb
@@ -140,12 +140,12 @@ def finished_stage_summary(stage)
           summary: "#{summary_basic_output(stage)}\nDetails at [#{url}](#{url}).".force_encoding('utf-8')
         }
 
-        finished_stage_update(stage)
+        finished_stage_update(stage, output)
 
         logger(Logger::INFO, "finished_stage_summary: #{stage.inspect} #{output.inspect}")
       end
 
-      def finished_stage_update(stage)
+      def finished_stage_update(stage, output)
         if stage.jobs.failure.empty?
           logger(Logger::WARN, "Stage: #{stage.name} finished - failure")
           stage.success(@github, output: output, agent: @agent)