From 7d9444dd64bf7aecbfe3cd896b536135b695fd6a Mon Sep 17 00:00:00 2001 From: Kristen Brown Date: Thu, 30 Mar 2023 09:03:42 -0400 Subject: [PATCH] API-24392: Add Feature Flag Monitoring Job for `VBADocuments` (#12256) * API-24392: Add feature flag monitoring job for VBADocuments * API-24392: Simplify since only DefaultNotification exists for now * API-24392: Rename to --- config/sidekiq_scheduler.yml | 5 + .../appeals_api/flipper_status_alert.rb | 3 +- .../vba_documents/slack/hash_notification.rb | 34 ++++ .../services/vba_documents/slack/messenger.rb | 37 +++++ .../vba_documents/flipper_status_alert.rb | 85 ++++++++++ .../config/flipper/enabled_features.yml | 6 + .../slack/hash_notification_spec.rb | 34 ++++ .../vba_documents/slack/messenger_spec.rb | 34 ++++ .../spec/workers/flipper_status_alert_spec.rb | 146 ++++++++++++++++++ 9 files changed, 382 insertions(+), 2 deletions(-) create mode 100644 modules/vba_documents/app/services/vba_documents/slack/hash_notification.rb create mode 100644 modules/vba_documents/app/services/vba_documents/slack/messenger.rb create mode 100644 modules/vba_documents/app/workers/vba_documents/flipper_status_alert.rb create mode 100644 modules/vba_documents/config/flipper/enabled_features.yml create mode 100644 modules/vba_documents/spec/services/vba_documents/slack/hash_notification_spec.rb create mode 100644 modules/vba_documents/spec/services/vba_documents/slack/messenger_spec.rb create mode 100644 modules/vba_documents/spec/workers/flipper_status_alert_spec.rb diff --git a/config/sidekiq_scheduler.yml b/config/sidekiq_scheduler.yml index 04982453233..ccd8c479d26 100644 --- a/config/sidekiq_scheduler.yml +++ b/config/sidekiq_scheduler.yml @@ -256,6 +256,11 @@ VANotify::InProgressForms: class: VANotify::InProgressForms description: "In progress form reminder emails (sent via VaNotify)" +VBADocuments::FlipperStatusAlert: + cron: '0 2,9,16 * * MON-FRI America/New_York' + class: VBADocuments::FlipperStatusAlert + description: "Checks status of Flipper features expected to be enabled and alerts to Slack if any are not enabled" + VBADocuments::ReportMonthlySubmissions: cron: "4 3 2 1 * * America/New_York" class: VBADocuments::ReportMonthlySubmissions diff --git a/modules/appeals_api/app/workers/appeals_api/flipper_status_alert.rb b/modules/appeals_api/app/workers/appeals_api/flipper_status_alert.rb index 94c8d78227b..5ac16174b34 100644 --- a/modules/appeals_api/app/workers/appeals_api/flipper_status_alert.rb +++ b/modules/appeals_api/app/workers/appeals_api/flipper_status_alert.rb @@ -11,8 +11,7 @@ class FlipperStatusAlert DISABLED_FLAG_EMOJI = ':vertical_traffic_light:' MISSING_FLAG_EMOJI = ':no_entry_sign:' - # No need to retry since the schedule will run this every hour - sidekiq_options retry: false, unique_for: 30.minutes + sidekiq_options retry: 5, unique_for: 30.minutes def perform features_to_check = load_features_from_config diff --git a/modules/vba_documents/app/services/vba_documents/slack/hash_notification.rb b/modules/vba_documents/app/services/vba_documents/slack/hash_notification.rb new file mode 100644 index 00000000000..85d6fb33155 --- /dev/null +++ b/modules/vba_documents/app/services/vba_documents/slack/hash_notification.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module VBADocuments + module Slack + class HashNotification + def initialize(params) + # Params are expected to be in the Sidekiq Job Format (https://github.com/mperham/sidekiq/wiki/Job-Format), + @params = params + end + + def message_text + msg = "ENVIRONMENT: #{environment}".dup + + params.each do |k, v| + msg << "\n#{k.to_s.upcase} : #{v}" + end + + msg + end + + private + + attr_accessor :params + + def environment + env = Settings.vsp_environment + + env_emoji = Messenger::ENVIRONMENT_EMOJIS[env.to_sym] + + ":#{env_emoji}: #{env} :#{env_emoji}:" + end + end + end +end diff --git a/modules/vba_documents/app/services/vba_documents/slack/messenger.rb b/modules/vba_documents/app/services/vba_documents/slack/messenger.rb new file mode 100644 index 00000000000..f27455b665d --- /dev/null +++ b/modules/vba_documents/app/services/vba_documents/slack/messenger.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'common/client/base' + +module VBADocuments + module Slack + class Messenger + ALERT_URL = Settings.vba_documents.slack.default_alert_url + ENVIRONMENT_EMOJIS = { production: 'rotating_light', sandbox: 'rocket', staging: 'construction', + development: 'brain' }.freeze + + def initialize(params) + @params = params + end + + def notify! + Faraday.post(ALERT_URL, request_body, request_headers) + end + + private + + attr_reader :params + + def notification + VBADocuments::Slack::HashNotification.new(params) + end + + def request_body + { text: notification.message_text }.to_json + end + + def request_headers + { 'Content-type' => 'application/json; charset=utf-8' } + end + end + end +end diff --git a/modules/vba_documents/app/workers/vba_documents/flipper_status_alert.rb b/modules/vba_documents/app/workers/vba_documents/flipper_status_alert.rb new file mode 100644 index 00000000000..8f0eb9a6692 --- /dev/null +++ b/modules/vba_documents/app/workers/vba_documents/flipper_status_alert.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require 'sidekiq' +require 'flipper/utilities/bulk_feature_checker' + +module VBADocuments + class FlipperStatusAlert + include Sidekiq::Worker + + WARNING_EMOJI = ':warning:' + DISABLED_FLAG_EMOJI = ':vertical_traffic_light:' + MISSING_FLAG_EMOJI = ':no_entry_sign:' + + sidekiq_options retry: 5, unique_for: 30.minutes + + def perform + features_to_check = load_features_from_config + if features_to_check.present? + @feature_statuses = Flipper::Utilities::BulkFeatureChecker.enabled_status(features_to_check) + + slack_notify unless all_features_enabled? + end + end + + private + + def load_features_from_config + file_path = VBADocuments::Engine.root.join('config', 'flipper', 'enabled_features.yml') + feature_hash = read_config_file(file_path) + return [] if feature_hash.nil? + + env_hash = feature_hash.fetch(Settings.vsp_environment.to_s, []) || [] + features = (env_hash + (feature_hash['common'] || [])).uniq.sort + + if features.empty? + VBADocuments::Slack::Messenger.new( + { + warning: "#{WARNING_EMOJI} #{self.class.name} has no configured enabled features", + file_path: file_path.to_s + } + ).notify! + end + + features + end + + def read_config_file(path) + if File.exist?(path) + YAML.load_file(path) || {} + else + VBADocuments::Slack::Messenger.new( + { + warning: "#{WARNING_EMOJI} #{self.class.name} features file does not exist", + file_path: path.to_s + } + ).notify! + nil + end + end + + def all_features_enabled? + @feature_statuses[:missing].empty? && @feature_statuses[:disabled].empty? + end + + def slack_notify + slack_details = { + class: self.class.name, + warning: "#{WARNING_EMOJI} One or more features expected to be enabled were found disabled or missing", + disabled_flags: slack_message(:disabled), + missing_flags: slack_message(:missing) + } + + VBADocuments::Slack::Messenger.new(slack_details).notify! + end + + def slack_message(flag_category) + if @feature_statuses[flag_category].present? + emoji = self.class.const_get("#{flag_category.upcase}_FLAG_EMOJI") + "#{emoji} #{@feature_statuses[flag_category].join(', ')} #{emoji}" + else + 'None' + end + end + end +end diff --git a/modules/vba_documents/config/flipper/enabled_features.yml b/modules/vba_documents/config/flipper/enabled_features.yml new file mode 100644 index 00000000000..20ea1052b38 --- /dev/null +++ b/modules/vba_documents/config/flipper/enabled_features.yml @@ -0,0 +1,6 @@ +common: + - test_feature_flag_alert_temp +development: +staging: +sandbox: +production: diff --git a/modules/vba_documents/spec/services/vba_documents/slack/hash_notification_spec.rb b/modules/vba_documents/spec/services/vba_documents/slack/hash_notification_spec.rb new file mode 100644 index 00000000000..44164f92a35 --- /dev/null +++ b/modules/vba_documents/spec/services/vba_documents/slack/hash_notification_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe VBADocuments::Slack::HashNotification do + describe '#message_text' do + let(:params) do + { + 'test_key' => 'test_value', + 'args' => %w[1234 5678], + 'gibberish' => 2, + 'indeed' => 'indeed gibberish', + 'message' => 'Something happened here' + } + end + + it 'returns the VSP environment' do + with_settings(Settings, vsp_environment: 'staging') do + expect( + described_class.new(params).message_text + ).to include('ENVIRONMENT: :construction: staging :construction:') + end + end + + it 'displays all the keys capitalized and formatted' do + with_settings(Settings, vsp_environment: 'staging') do + expect(described_class.new(params).message_text).to include( + "\nTEST_KEY : test_value\nARGS : [\"1234\", \"5678\"] +GIBBERISH : 2\nINDEED : indeed gibberish\nMESSAGE : Something happened here" + ) + end + end + end +end diff --git a/modules/vba_documents/spec/services/vba_documents/slack/messenger_spec.rb b/modules/vba_documents/spec/services/vba_documents/slack/messenger_spec.rb new file mode 100644 index 00000000000..dcbd7827a2e --- /dev/null +++ b/modules/vba_documents/spec/services/vba_documents/slack/messenger_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe VBADocuments::Slack::Messenger do + describe '.notify!' do + let(:params) do + { + 'class' => 'SomeClass', + 'args' => %w[1234 5678], + 'retry_count' => 2, + 'error_class' => 'RuntimeError', + 'error_message' => 'Here there be dragons', + 'failed_at' => 1_613_670_737.966083, + 'retried_at' => 1_613_680_062.5507782 + } + end + + it 'sends a network request' do + with_settings(Settings, vsp_environment: 'production') do + with_settings(Settings.vba_documents.slack, default_alert_url: 'default alert url') do + body = { text: VBADocuments::Slack::HashNotification.new(params).message_text }.to_json + headers = { 'Content-type' => 'application/json; charset=utf-8' } + + allow(Faraday).to receive(:post).with(VBADocuments::Slack::Messenger::ALERT_URL, body, headers) + + VBADocuments::Slack::Messenger.new(params).notify! + + expect(Faraday).to have_received(:post).with(VBADocuments::Slack::Messenger::ALERT_URL, body, headers) + end + end + end + end +end diff --git a/modules/vba_documents/spec/workers/flipper_status_alert_spec.rb b/modules/vba_documents/spec/workers/flipper_status_alert_spec.rb new file mode 100644 index 00000000000..74841e71580 --- /dev/null +++ b/modules/vba_documents/spec/workers/flipper_status_alert_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'flipper/utilities/bulk_feature_checker' + +describe VBADocuments::FlipperStatusAlert, type: :job do + include FixtureHelpers + + before { Sidekiq::Worker.clear_all } + + describe '#perform' do + let(:messenger_instance) { instance_double(VBADocuments::Slack::Messenger) } + let(:config_file_path) { VBADocuments::Engine.root.join('config', 'flipper', 'enabled_features.yml') } + + let(:warning_emoji) { described_class::WARNING_EMOJI } + let(:disabled_flag_emoji) { described_class::DISABLED_FLAG_EMOJI } + let(:missing_flag_emoji) { described_class::MISSING_FLAG_EMOJI } + + let(:missing_file_message) { "#{warning_emoji} #{described_class} features file does not exist" } + let(:no_features_message) { "#{warning_emoji} #{described_class} has no configured enabled features" } + let(:flag_status_message) do + "#{warning_emoji} One or more features expected to be enabled were found disabled or missing" + end + + it 'notifies Slack of missing config file when no config file found' do + allow(File).to receive(:exist?).and_return(false) + expected_notify = { warning: missing_file_message, file_path: config_file_path.to_s } + expect(VBADocuments::Slack::Messenger).to receive(:new).with(expected_notify).and_return(messenger_instance) + expect(messenger_instance).to receive(:notify!).once + + described_class.new.perform + end + + it 'notifies Slack that no features were found when config file contains no features' do + allow(YAML).to receive(:load_file).and_return({ 'common' => nil, 'production' => nil }) + expected_notify = { warning: no_features_message, file_path: config_file_path.to_s } + expect(VBADocuments::Slack::Messenger).to receive(:new).with(expected_notify).and_return(messenger_instance) + expect(messenger_instance).to receive(:notify!).once + + with_settings(Settings, vsp_environment: 'production') do + described_class.new.perform + end + end + + it 'notifies Slack that no features were found when config file is empty (no keys)' do + allow(YAML).to receive(:load_file).and_return(nil) + expected_notify = { warning: no_features_message, file_path: config_file_path.to_s } + expect(VBADocuments::Slack::Messenger).to receive(:new).with(expected_notify).and_return(messenger_instance) + expect(messenger_instance).to receive(:notify!).once + + described_class.new.perform + end + + it 'fetches enabled status of common and current env features when config file contains both' do + allow(YAML).to receive(:load_file).and_return({ 'common' => %w[feature1], 'production' => %w[feature2] }) + bulk_checker_result = { enabled: [], disabled: [], missing: %w[feature1 feature2] } + expect(Flipper::Utilities::BulkFeatureChecker) + .to receive(:enabled_status).with(%w[feature1 feature2]).and_return(bulk_checker_result) + allow(VBADocuments::Slack::Messenger).to receive(:new).and_return(messenger_instance) + allow(messenger_instance).to receive(:notify!) + + with_settings(Settings, vsp_environment: 'production') do + described_class.new.perform + end + end + + it 'fetches enabled status of common features only when config file contains no current env features' do + allow(YAML).to receive(:load_file).and_return({ 'common' => %w[feature1], 'development' => nil }) + bulk_checker_result = { enabled: [], disabled: [], missing: %w[feature1] } + expect(Flipper::Utilities::BulkFeatureChecker) + .to receive(:enabled_status).with(%w[feature1]).and_return(bulk_checker_result) + allow(VBADocuments::Slack::Messenger).to receive(:new).and_return(messenger_instance) + allow(messenger_instance).to receive(:notify!) + + with_settings(Settings, vsp_environment: 'development') do + described_class.new.perform + end + end + + it 'fetches enabled status of current env features only when config file contains no common features' do + allow(YAML).to receive(:load_file).and_return({ 'common' => nil, 'staging' => %w[feature1] }) + bulk_checker_result = { enabled: [], disabled: [], missing: %w[feature1] } + expect(Flipper::Utilities::BulkFeatureChecker) + .to receive(:enabled_status).with(%w[feature1]).and_return(bulk_checker_result) + allow(VBADocuments::Slack::Messenger).to receive(:new).and_return(messenger_instance) + allow(messenger_instance).to receive(:notify!) + + with_settings(Settings, vsp_environment: 'staging') do + described_class.new.perform + end + end + + it 'does not notify Slack when all features are enabled' do + bulk_checker_result = { enabled: %w[feature1 feature2], disabled: [], missing: [] } + allow(Flipper::Utilities::BulkFeatureChecker).to receive(:enabled_status).and_return(bulk_checker_result) + expect(VBADocuments::Slack::Messenger).not_to receive(:new) + + described_class.new.perform + end + + it 'notifies Slack when some features are disabled' do + bulk_checker_result = { enabled: %w[feature1], disabled: %w[feature2 feature3], missing: [] } + allow(Flipper::Utilities::BulkFeatureChecker).to receive(:enabled_status).and_return(bulk_checker_result) + expected_notify = { + class: described_class.name, + warning: flag_status_message, + disabled_flags: "#{disabled_flag_emoji} feature2, feature3 #{disabled_flag_emoji}", + missing_flags: 'None' + } + expect(VBADocuments::Slack::Messenger).to receive(:new).with(expected_notify).and_return(messenger_instance) + expect(messenger_instance).to receive(:notify!).once + + described_class.new.perform + end + + it 'notifies Slack when some features are missing' do + bulk_checker_result = { enabled: %w[feature1], disabled: [], missing: %w[feature2 feature3] } + allow(Flipper::Utilities::BulkFeatureChecker).to receive(:enabled_status).and_return(bulk_checker_result) + expected_notify = { + class: described_class.name, + warning: flag_status_message, + disabled_flags: 'None', + missing_flags: "#{missing_flag_emoji} feature2, feature3 #{missing_flag_emoji}" + } + expect(VBADocuments::Slack::Messenger).to receive(:new).with(expected_notify).and_return(messenger_instance) + expect(messenger_instance).to receive(:notify!).once + + described_class.new.perform + end + + it 'notifies slack when there are disabled and missing features' do + bulk_checker_result = { enabled: %w[feature1], disabled: %w[feature2 feature3], missing: %w[feature4 feature5] } + allow(Flipper::Utilities::BulkFeatureChecker).to receive(:enabled_status).and_return(bulk_checker_result) + expected_notify = { + class: described_class.name, + warning: flag_status_message, + disabled_flags: "#{disabled_flag_emoji} feature2, feature3 #{disabled_flag_emoji}", + missing_flags: "#{missing_flag_emoji} feature4, feature5 #{missing_flag_emoji}" + } + expect(VBADocuments::Slack::Messenger).to receive(:new).with(expected_notify).and_return(messenger_instance) + expect(messenger_instance).to receive(:notify!).once + + described_class.new.perform + end + end +end