Notify when back to normal

db/migrate/030_add_notify_back_to_normal.rb

@@ -0,0 +1,5 @@
+class AddNotifyBackToNormal < ActiveRecord::Migration[5.0]
+  def change
+    add_column :job_instances, :retrying, :boolean, default: false, null: false
+  end
+end

lib/autoload/kuroko2/workflow/engine.rb

@@ -25,7 +25,7 @@ def retry(token)
           node = extract_node(token)
 
           message = "(token #{token.uuid}) Retry current node: '#{node.type}: #{node.option}'"
-          token.job_instance.update_column(:error_at, nil)
+          token.job_instance.update_columns(error_at: nil, retrying: true)
           token.job_instance.logs.info(message)
 
           token.mark_as_working
@@ -119,7 +119,11 @@ def process_next(node, token)
           token.mark_as_finished
           unless token.parent
             token.job_instance.touch(:finished_at)
-            Notifier.notify(:finished, token.job_instance)
+            if token.job_instance.retrying?
+              Notifier.notify(:back_to_normal, token.job_instance)
+            else
+              Notifier.notify(:finished, token.job_instance)
+            end
             token.destroy!
           end
         end

lib/autoload/kuroko2/workflow/notifier/concerns/chat_message_builder.rb

@@ -20,6 +20,10 @@ def launched_text
             "Launched '#{@definition.name}'"
           end
 
+          def back_to_normal_text
+            "Backed to normal '#{@definition.name}'"
+          end
+
           def retrying_text
             "Retrying the current task in '#{@definition.name}'"
           end

lib/autoload/kuroko2/workflow/notifier/hipchat.rb

@@ -79,6 +79,11 @@ def notify_finished
           end
         end
 
+        def notify_back_to_normal
+          message = build_message(level: 'SUCCESS', text: message_builder.back_to_normal_text)
+          send_to_hipchat(message)
+        end
+
         def notify_long_elapsed_time
           message = build_message(level: 'WARNING', text: message_builder.long_elapsed_time_text)
           send_to_hipchat(message, color: 'red')

lib/autoload/kuroko2/workflow/notifier/mail.rb

@@ -37,6 +37,10 @@ def notify_finished
           # do nothing
         end
 
+        def notify_back_to_normal
+          # do nothing
+        end
+
         def notify_long_elapsed_time
           Kuroko2::Notifications.notify_long_elapsed_time(@job_instance).deliver_now
         end

lib/autoload/kuroko2/workflow/notifier/slack.rb

@@ -88,6 +88,13 @@ def notify_finished
           end
         end
 
+        def notify_back_to_normal
+          send_attachment_message_to_slack(
+            level: 'SUCCESS',
+            text: message_builder.back_to_normal_text,
+          )
+        end
+
         def notify_long_elapsed_time
           send_attachment_message_to_slack(
             level: 'WARNING',

lib/autoload/kuroko2/workflow/notifier/webhook.rb

@@ -100,6 +100,16 @@ def notify_finished
           end
         end
 
+        def notify_back_to_normal
+          request(
+            build_payload(
+              action: 'notify_back_to_normal',
+              level: 'SUCCESS',
+              subject: message_builder.back_to_normal_text,
+            )
+          )
+        end
+
         def notify_long_elapsed_time
           request(
             build_payload(

spec/workflow/engine_spec.rb

@@ -306,7 +306,9 @@ module Kuroko2::Workflow
 
           FileUtils.touch(tmpfile)
 
+          expect(token.job_instance.retrying?).to be_falsy
           subject.retry(token)
+          expect(token.job_instance.retrying?).to be_truthy
 
           subject.process(token)
           shell.execute

spec/workflow/notifier/hipchat_spec.rb

@@ -125,6 +125,17 @@ module Kuroko2::Workflow
       end
     end
 
+    describe '#notify_back_to_normal' do
+      it 'sends back_to_normal message' do
+        expect(hipchat_room_object).to receive(:send) do |_, message, option|
+          expect(message).to include('SUCCESS')
+          expect(option[:color]).to eq('green')
+        end
+
+        notifier.notify_back_to_normal
+      end
+    end
+
     describe '#notify_retrying' do
       context 'with notify_finished' do
         before do

spec/workflow/notifier/mail_spec.rb

@@ -67,6 +67,14 @@ module Kuroko2::Workflow
       end
     end
 
+    describe '#notify_back_to_normal' do
+      it 'does not send mail' do
+        expect { notifier.notify_back_to_normal }.not_to change {
+          ActionMailer::Base.deliveries.size
+        }
+      end
+    end
+
     describe '#notify_long_elapsed_time' do
       it 'sends warning mesasge' do
         expect { notifier.notify_long_elapsed_time }.to change {

spec/workflow/notifier/slack_spec.rb

@@ -90,6 +90,15 @@ module Kuroko2::Workflow
       end
     end
 
+    describe '#notify_back_to_normal' do
+      it 'sends back_to_normal mesasge' do
+        expect(notifier).to receive(:send_to_slack).
+          with(hash_including(channel: slack_channel)).and_call_original
+
+        notifier.notify_back_to_normal
+      end
+    end
+
     describe '#notify_retrying' do
       context 'with notify_finished' do
         before do

spec/workflow/notifier/webhook_spec.rb

@@ -72,6 +72,17 @@ module Kuroko2::Workflow
       end
     end
 
+    describe '#notify_back_to_normal' do
+      it 'sends back_to_normal message' do
+        stub.with { |req|
+          expect(JSON.parse(req.body)).to include("action"=>"notify_back_to_normal")
+        }
+
+        notifier.notify_back_to_normal
+        expect(stub).to have_been_requested
+      end
+    end
+
     describe '#notify_retrying' do
       context 'with notify_finished' do
         before do