From f9021af83728d1c58a4c028d5b2c5aa7cc5a86b5 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 18 Jul 2024 14:42:59 -0400
Subject: [PATCH 01/83] create execute.rb

---
 lib/inferno/apps/cli/execute.rb | 10 ++++++++++
 lib/inferno/apps/cli/main.rb    | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 lib/inferno/apps/cli/execute.rb

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
new file mode 100644
index 000000000..f0fde886c
--- /dev/null
+++ b/lib/inferno/apps/cli/execute.rb
@@ -0,0 +1,10 @@
+module Inferno
+  module CLI
+    class Execute
+      # TODO use Thor Groups? TTY-Markdown + colorize?
+      def run(options)
+        puts "Work in progress..."
+      end
+    end
+  end
+end
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index 808e772aa..5ab86801c 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -5,6 +5,7 @@
 require_relative 'suites'
 require_relative 'new'
 require_relative '../../version'
+require_relative 'execute'
 
 module Inferno
   module CLI
@@ -65,6 +66,37 @@ def version
         puts "Inferno Core v#{Inferno::VERSION}"
       end
 
+      desc 'execute', 'Run Inferno tests in command line'
+      option :docker_compose,
+             default: true,
+             type: :boolean,
+             desc: 'Run within Docker Compose'
+      option :suite,
+             aliases: ['-s'],
+             required: true,
+             type: :string,
+             desc: 'Test Suite ID to run',
+             banner: 'ID'
+      option :inputs,
+             aliases: ['-i'],
+             optional: true,
+             type: :hash,
+             desc: 'Inputs (i.e: --inputs=foo:bar goo:baz)'
+      def execute
+        # TODO: fold this into Execute
+        if options[:docker_compose]
+          rebuilt_options = options
+                              .merge({docker_compose: false})
+                              .transform_values{|value| value.is_a?(Hash) ? value.to_a.map{|arr| arr.join(':')} : value}
+                              .transform_values{|value| value.is_a?(Array) ? value.join(' ') : value}
+                              .to_a
+                              .join('=')
+          `docker compose run inferno bundle exec inferno execute #{rebuilt_options}`
+        else
+          Execute.new.run(options)
+        end
+      end
+
       private
 
       # https://github.com/rubocop/rubocop/issues/12571 - still affects Ruby 3.1 upto Rubocop 1.63

From 7d8b8d56cb706b28c596773b929ed2cd5473281f Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 19 Jul 2024 11:47:18 -0400
Subject: [PATCH 02/83] move docker compose option into cli/execute.rb

---
 lib/inferno/apps/cli/execute.rb | 12 +++++++++++-
 lib/inferno/apps/cli/main.rb    | 13 +------------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index f0fde886c..65e5352de 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -3,7 +3,17 @@ module CLI
     class Execute
       # TODO use Thor Groups? TTY-Markdown + colorize?
       def run(options)
-        puts "Work in progress..."
+        if options[:docker_compose]
+          rebuilt_options = options
+                              .merge({docker_compose: false})
+                              .transform_values{|value| value.is_a?(Hash) ? value.to_a.map{|arr| arr.join(':')} : value}
+                              .transform_values{|value| value.is_a?(Array) ? value.join(' ') : value}
+                              .to_a
+                              .join('=')
+          `docker compose run inferno bundle exec inferno execute #{rebuilt_options}`
+        else
+          puts "TODO: heavy lifting here"
+        end
       end
     end
   end
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index 5ab86801c..139c1ba0a 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -83,18 +83,7 @@ def version
              type: :hash,
              desc: 'Inputs (i.e: --inputs=foo:bar goo:baz)'
       def execute
-        # TODO: fold this into Execute
-        if options[:docker_compose]
-          rebuilt_options = options
-                              .merge({docker_compose: false})
-                              .transform_values{|value| value.is_a?(Hash) ? value.to_a.map{|arr| arr.join(':')} : value}
-                              .transform_values{|value| value.is_a?(Array) ? value.join(' ') : value}
-                              .to_a
-                              .join('=')
-          `docker compose run inferno bundle exec inferno execute #{rebuilt_options}`
-        else
-          Execute.new.run(options)
-        end
+        Execute.new.run(options)
       end
 
       private

From 5a725145052bf181c57bd33f1da90f19a173b6d3 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 26 Jul 2024 17:52:03 -0400
Subject: [PATCH 03/83] wip

---
 Gemfile.lock                    |   1 +
 inferno_core.gemspec            |   1 +
 lib/inferno/apps/cli/execute.rb | 187 ++++++++++++++++++++++++++++++--
 lib/inferno/apps/cli/main.rb    |   4 -
 4 files changed, 178 insertions(+), 15 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 17972c736..6ec64f2ff 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -18,6 +18,7 @@ PATH
       hanami-controller (= 2.0.0)
       hanami-router (= 2.0.0)
       oj (= 3.11.0)
+      pastel (~> 0.8.0)
       pry
       pry-byebug
       puma (~> 5.6.7)
diff --git a/inferno_core.gemspec b/inferno_core.gemspec
index 4a16513ec..4e3757dc3 100644
--- a/inferno_core.gemspec
+++ b/inferno_core.gemspec
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'hanami-controller', '2.0.0'
   spec.add_runtime_dependency 'hanami-router', '2.0.0'
   spec.add_runtime_dependency 'oj', '3.11.0'
+  spec.add_runtime_dependency 'pastel', '~> 0.8.0'
   spec.add_runtime_dependency 'pry'
   spec.add_runtime_dependency 'pry-byebug'
   spec.add_runtime_dependency 'puma', '~> 5.6.7'
diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 65e5352de..45cd51931 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -1,20 +1,185 @@
+# frozen_string_literal: true
+
+require 'pastel'
+require_relative '../../../inferno'
+
 module Inferno
   module CLI
     class Execute
-      # TODO use Thor Groups? TTY-Markdown + colorize?
+
+      COLOR = Pastel.new
+
+      include Import[
+                test_sessions_repo: 'inferno.repositories.test_sessions',
+                session_data_repo: 'inferno.repositories.session_data',
+                test_runs_repo: 'inferno.repositories.test_runs'
+              ]
+
+      def verify_runnable(runnable, inputs, selected_suite_options)
+        missing_inputs = runnable&.missing_inputs(inputs, selected_suite_options)
+        user_runnable = runnable&.user_runnable?
+        raise Inferno::Exceptions::RequiredInputsNotFound, missing_inputs if missing_inputs&.any?
+        raise Inferno::Exceptions::NotUserRunnableException unless user_runnable
+      end
+
+      def persist_inputs(params, test_run)
+        available_inputs = test_run.runnable.available_inputs
+        params[:inputs]&.each do |input_params|
+          input =
+            available_inputs
+              .find { |_, runnable_input| runnable_input.name == input_params[:name] }
+              &.last
+
+          if input.nil?
+            Inferno::Application['logger'].warn(
+              "Unknown input `#{input_params[:name]}` for #{test_run.runnable.id}: #{test_run.runnable.title}"
+            )
+            next
+          end
+
+          session_data_repo.save(
+            test_session_id: test_run.test_session_id,
+            name: input.name,
+            value: input_params[:value],
+            type: input.type
+          )
+        end
+      end
+
+=begin
+      def handle(req, res)
+        test_session = test_sessions_repo.find(req.params[:test_session_id])
+
+        # if testsession.nil?
+        if test_runs_repo.active_test_run_for_session?(test_session.id)
+          halt 409, { error: 'Cannot run new test while another test run is in progress' }.to_json
+        end
+
+        verify_runnable(
+          repo.build_entity(create_params(req.params)).runnable,
+          req.params[:inputs],
+          test_session.suite_options
+        )
+
+        test_run = repo.create(create_params(req.params).merge(status: 'queued'))
+
+        res.body = serialize(test_run, suite_options: test_session.suite_options)
+
+        persist_inputs(req.params, test_run)
+
+        Jobs.perform(Jobs::ExecuteTestRun, test_run.id)
+      rescue Sequel::ValidationFailed, Sequel::ForeignKeyConstraintViolation,
+             Inferno::Exceptions::RequiredInputsNotFound,
+             Inferno::Exceptions::NotUserRunnableException => e
+        halt 422, { errors: e.message }.to_json
+      rescue StandardError => e
+        Application['logger'].error(e.full_message)
+        halt 500, { errors: e.message }.to_json
+      end
+=end
+
       def run(options)
-        if options[:docker_compose]
-          rebuilt_options = options
-                              .merge({docker_compose: false})
-                              .transform_values{|value| value.is_a?(Hash) ? value.to_a.map{|arr| arr.join(':')} : value}
-                              .transform_values{|value| value.is_a?(Array) ? value.join(' ') : value}
-                              .to_a
-                              .join('=')
-          `docker compose run inferno bundle exec inferno execute #{rebuilt_options}`
-        else
-          puts "TODO: heavy lifting here"
+        puts ''
+        puts '=========================================='
+        puts " Testing #{options[:suite]} Suite"
+        puts '=========================================='
+
+        Inferno::Application.start(:suites)        
+
+        suite = Inferno::Repositories::TestSuites.new.find(options[:suite])
+        raise StandardError, "Suite #{options[:suite]} not found" if suite.nil?
+
+        test_session = test_sessions_repo.create({test_suite_id: suite.id}) # TODO add suite options
+
+        # skip active test run check since new test session is minted
+
+        verify_runnable(
+          test_runs_repo.build_entity({
+            test_session_id: test_session.id,
+            test_suite_id: suite.id,
+            inputs: thor_hash_to_inputs_array(options[:inputs])
+          }).runnable,
+          thor_hash_to_inputs_array(options[:inputs]),
+          test_session.suite_options
+        )
+
+        test_run = test_runs_repo.create({
+          test_session_id: test_session.id,
+          test_suite_id: suite.id,
+          inputs: thor_hash_to_inputs_array(options[:inputs]),
+          status: 'queued'
+        })
+
+        persist_inputs({test_session_id: test_session.id, test_suite_id: suite.id, inputs: thor_hash_to_inputs_array(options[:inputs])}, test_run)
+
+        # TODO to job or not to job that is the question
+        Jobs.perform(Jobs::ExecuteTestRun, test_run.id)
+
+        # how to properly wait for jobs to finish; stall? poll?
+        sleep(20) # seconds 
+
+        results = test_runs_repo.results_for_test_run(test_run.id)
+        #puts serialize(results)
+        puts results
+        puts results.class
+        puts results.to_json
+
+        puts COLOR.yellow "WIP: implementing"
+
+      rescue Sequel::ValidationFailed, Sequel::ForeignKeyConstraintViolation,
+             Inferno::Exceptions::RequiredInputsNotFound,
+             Inferno::Exceptions::NotUserRunnableException => e
+        $stderr.puts COLOR.red "Error: #{e.full_message}"
+        # TODO: use Application Logger?
+      rescue StandardError => e
+        $stderr.puts COLOR.red "Error: #{e.full_message}"
+      end
+
+      def thor_hash_to_inputs_array(hash)
+        hash.to_a.map{|pair| {name: pair[0], value: pair[1]}}
+      end
+=begin
+      def suppress_output
+        begin
+          original_stderr = $stderr.clone
+          original_stdout = $stdout.clone
+          $stderr.reopen(File.new('/dev/null', 'w'))
+          $stdout.reopen(File.new('/dev/null', 'w'))
+          retval = yield
+        rescue StandardError => e
+          $stdout.reopen(original_stdout)
+          $stderr.reopen(original_stderr)
+          raise e
+        ensure
+          $stdout.reopen(original_stdout)
+          $stderr.reopen(original_stderr)
+        end
+        retval
+      end
+
+      def unindent_markdown(markdown)
+        return nil if markdown.nil?
+
+        natural_indent = markdown.lines.collect { |l| l.index(/[^ ]/) }.select { |l| !l.nil? && l.positive? }.min || 0
+        markdown.lines.map { |l| l[natural_indent..-1] || "\n" }.join.lstrip
+      end
+
+      def print_requests(result)
+        result.request_responses.map do |req_res|
+          req_res.response_code.to_s + ' ' + req_res.request_method.upcase + ' ' + req_res.request_url
+        end
+      end
+
+      # TODO better name
+      def _run(runnable, test_session, inputs = {})
+        test_run_params = { test_session_id: test_session.id }.merge(runnable.reference_hash)
+        test_run = Inferno::Repositories::TestRuns.new.create(test_run_params)
+        inputs.each do |name, value|
+          session_data_repo.save(test_session_id: test_session.id, name: name, value: value, type: 'text')
         end
+        Inferno::TestRunner.new(test_session: test_session, test_run: test_run).run(runnable)
       end
+=end
     end
   end
 end
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index 139c1ba0a..bccf11d9f 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -67,10 +67,6 @@ def version
       end
 
       desc 'execute', 'Run Inferno tests in command line'
-      option :docker_compose,
-             default: true,
-             type: :boolean,
-             desc: 'Run within Docker Compose'
       option :suite,
              aliases: ['-s'],
              required: true,

From 625400ddebde0dac889dd43d22fcc7c1469f4cf5 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 31 Jul 2024 10:39:19 -0400
Subject: [PATCH 04/83] IT WORKS

---
 lib/inferno/apps/cli/execute.rb               | 80 +++++++++----------
 lib/inferno/apps/cli/main.rb                  |  5 ++
 .../apps/web/serializers/serializer.rb        |  3 +
 3 files changed, 47 insertions(+), 41 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 45cd51931..0b43b25a9 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 require 'pastel'
+require 'active_support'
 require_relative '../../../inferno'
+require_relative '../web/serializers/test_run'
+require_relative '../web/serializers/result'
 
 module Inferno
   module CLI
@@ -15,6 +18,8 @@ class Execute
                 test_runs_repo: 'inferno.repositories.test_runs'
               ]
 
+      attr_accessor :options
+
       def verify_runnable(runnable, inputs, selected_suite_options)
         missing_inputs = runnable&.missing_inputs(inputs, selected_suite_options)
         user_runnable = runnable&.user_runnable?
@@ -46,44 +51,17 @@ def persist_inputs(params, test_run)
         end
       end
 
-=begin
-      def handle(req, res)
-        test_session = test_sessions_repo.find(req.params[:test_session_id])
-
-        # if testsession.nil?
-        if test_runs_repo.active_test_run_for_session?(test_session.id)
-          halt 409, { error: 'Cannot run new test while another test run is in progress' }.to_json
-        end
-
-        verify_runnable(
-          repo.build_entity(create_params(req.params)).runnable,
-          req.params[:inputs],
-          test_session.suite_options
-        )
-
-        test_run = repo.create(create_params(req.params).merge(status: 'queued'))
-
-        res.body = serialize(test_run, suite_options: test_session.suite_options)
-
-        persist_inputs(req.params, test_run)
-
-        Jobs.perform(Jobs::ExecuteTestRun, test_run.id)
-      rescue Sequel::ValidationFailed, Sequel::ForeignKeyConstraintViolation,
-             Inferno::Exceptions::RequiredInputsNotFound,
-             Inferno::Exceptions::NotUserRunnableException => e
-        halt 422, { errors: e.message }.to_json
-      rescue StandardError => e
-        Application['logger'].error(e.full_message)
-        halt 500, { errors: e.message }.to_json
-      end
-=end
-
+      ## MAIN
       def run(options)
         puts ''
         puts '=========================================='
         puts " Testing #{options[:suite]} Suite"
         puts '=========================================='
 
+        self.options = options
+        verbose_puts "In verbose mode"
+
+        # TODO: hijack Application logger?
         Inferno::Application.start(:suites)        
 
         suite = Inferno::Repositories::TestSuites.new.find(options[:suite])
@@ -91,8 +69,6 @@ def run(options)
 
         test_session = test_sessions_repo.create({test_suite_id: suite.id}) # TODO add suite options
 
-        # skip active test run check since new test session is minted
-
         verify_runnable(
           test_runs_repo.build_entity({
             test_session_id: test_session.id,
@@ -112,17 +88,18 @@ def run(options)
 
         persist_inputs({test_session_id: test_session.id, test_suite_id: suite.id, inputs: thor_hash_to_inputs_array(options[:inputs])}, test_run)
 
-        # TODO to job or not to job that is the question
         Jobs.perform(Jobs::ExecuteTestRun, test_run.id)
 
-        # how to properly wait for jobs to finish; stall? poll?
-        sleep(20) # seconds 
+        # TODO how to properly wait for jobs to finish; stall? poll?
+        sleep(10) # seconds 
 
         results = test_runs_repo.results_for_test_run(test_run.id)
-        #puts serialize(results)
-        puts results
-        puts results.class
-        puts results.to_json
+        verbose_puts "Results:"
+        verbose_puts serialize(results)
+
+        results.each do |result|
+          puts serialize(result)
+        end
 
         puts COLOR.yellow "WIP: implementing"
 
@@ -138,6 +115,27 @@ def run(options)
       def thor_hash_to_inputs_array(hash)
         hash.to_a.map{|pair| {name: pair[0], value: pair[1]}}
       end
+
+      def serialize(entity)
+        case entity.class.to_s
+        when 'Array'
+          entity.map{ |item| serialize(item) }
+        when ->(x) { defined?(x.constantize) && defined?("Inferno::Web::Serializers::#{x.split('::').last}".constantize) }
+          "Inferno::Web::Serializers::#{entity.class.to_s.split('::').last}".constantize.render(entity)
+        else
+          raise StandardError, "CLI does not know how to serialize #{entity.class}"
+        end
+      end
+
+      def verbose_print(*args)
+        print(COLOR.dim(*args)) if self.options[:verbose]
+      end
+
+      def verbose_puts(*args)
+        args.push("\n")
+        verbose_print(*args)
+      end
+
 =begin
       def suppress_output
         begin
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index bccf11d9f..a3bf153ed 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -78,6 +78,11 @@ def version
              optional: true,
              type: :hash,
              desc: 'Inputs (i.e: --inputs=foo:bar goo:baz)'
+      option :verbose,
+             aliases: ['-v'],
+             type: :boolean,
+             default: false,
+             desc: 'Output additional information for debugging'
       def execute
         Execute.new.run(options)
       end
diff --git a/lib/inferno/apps/web/serializers/serializer.rb b/lib/inferno/apps/web/serializers/serializer.rb
index eec20a2c6..221109f5e 100644
--- a/lib/inferno/apps/web/serializers/serializer.rb
+++ b/lib/inferno/apps/web/serializers/serializer.rb
@@ -1,3 +1,6 @@
+require 'oj'
+require 'blueprinter'
+
 module Inferno
   module Web
     module Serializers

From 72fb726f2c6c45dd0b9dccb9cd6c870dfa7496f9 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 1 Aug 2024 16:22:57 -0400
Subject: [PATCH 05/83] add force_synchronous kwarg to Jobs

---
 lib/inferno/apps/cli/execute.rb | 2 +-
 lib/inferno/jobs.rb             | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 0b43b25a9..739dffab4 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -88,7 +88,7 @@ def run(options)
 
         persist_inputs({test_session_id: test_session.id, test_suite_id: suite.id, inputs: thor_hash_to_inputs_array(options[:inputs])}, test_run)
 
-        Jobs.perform(Jobs::ExecuteTestRun, test_run.id)
+        Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
 
         # TODO how to properly wait for jobs to finish; stall? poll?
         sleep(10) # seconds 
diff --git a/lib/inferno/jobs.rb b/lib/inferno/jobs.rb
index ece844a51..fe564ffdf 100644
--- a/lib/inferno/jobs.rb
+++ b/lib/inferno/jobs.rb
@@ -6,11 +6,11 @@
 
 module Inferno
   module Jobs
-    def self.perform(job_klass, *params)
-      if Application['async_jobs']
-        job_klass.perform_async(*params)
-      else
+    def self.perform(job_klass, *params, force_synchronous: false)
+      if force_synchronous || (Application['async_jobs'] === false)
         job_klass.new.perform(*params)
+      else
+        job_klass.perform_async(*params)
       end
     end
   end

From 4588c3f5cd47bfd5d0c1890ae492418a4d5399de Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 2 Aug 2024 11:43:47 -0400
Subject: [PATCH 06/83] add colorized ascii output

---
 lib/inferno/apps/cli/execute.rb | 70 +++++++++++++++++++++------------
 1 file changed, 45 insertions(+), 25 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 739dffab4..62cb3fa3a 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -11,6 +11,7 @@ module CLI
     class Execute
 
       COLOR = Pastel.new
+      CHECKMARK = "\u2713"
 
       include Import[
                 test_sessions_repo: 'inferno.repositories.test_sessions',
@@ -20,6 +21,7 @@ class Execute
 
       attr_accessor :options
 
+      # TODO factorize into some helper/util with hanami controller
       def verify_runnable(runnable, inputs, selected_suite_options)
         missing_inputs = runnable&.missing_inputs(inputs, selected_suite_options)
         user_runnable = runnable&.user_runnable?
@@ -27,6 +29,7 @@ def verify_runnable(runnable, inputs, selected_suite_options)
         raise Inferno::Exceptions::NotUserRunnableException unless user_runnable
       end
 
+      # TODO likewise
       def persist_inputs(params, test_run)
         available_inputs = test_run.runnable.available_inputs
         params[:inputs]&.each do |input_params|
@@ -51,7 +54,6 @@ def persist_inputs(params, test_run)
         end
       end
 
-      ## MAIN
       def run(options)
         puts ''
         puts '=========================================='
@@ -59,7 +61,7 @@ def run(options)
         puts '=========================================='
 
         self.options = options
-        verbose_puts "In verbose mode"
+        verbose_puts "options:", self.options
 
         # TODO: hijack Application logger?
         Inferno::Application.start(:suites)        
@@ -88,28 +90,53 @@ def run(options)
 
         persist_inputs({test_session_id: test_session.id, test_suite_id: suite.id, inputs: thor_hash_to_inputs_array(options[:inputs])}, test_run)
 
+        puts "Running tests. This may take a while..."
         Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
 
-        # TODO how to properly wait for jobs to finish; stall? poll?
-        sleep(10) # seconds 
-
-        results = test_runs_repo.results_for_test_run(test_run.id)
-        verbose_puts "Results:"
+        results = test_runs_repo.results_for_test_run(test_run.id)&.reverse
+        verbose_puts '=========================================='
+        verbose_puts "JSON Results:"
+        verbose_puts '=========================================='
         verbose_puts serialize(results)
+        verbose_puts '=========================================='
 
+        puts '=========================================='
+        puts "Color Results:"
+        puts '=========================================='
         results.each do |result|
-          puts serialize(result)
+          print fetch_test_id(result), ":"
+          case result.result
+          when 'pass'
+            print COLOR.green(CHECKMARK, ' pass')
+          when 'fail'
+            print COLOR.red 'X fail'
+          when 'skip'
+            print COLOR.yellow '* skip'
+          when 'omit'
+            print COLOR.blue '* omit'
+          when 'error'
+            print COLOR.magenta 'X error'
+          else
+            # TODO strict behavior or no?
+            #raise StandardError.new, "Unrecognized result #{result.result}" # strict
+            print '- unknown'                                               # unstrict
+          end
+          puts ''
+          verbose_puts "\tmessage: ", result.result_message
+          verbose_puts "\trequests: ", print_requests(result)
         end
+        puts '=========================================='
 
-        puts COLOR.yellow "WIP: implementing"
 
       rescue Sequel::ValidationFailed, Sequel::ForeignKeyConstraintViolation,
              Inferno::Exceptions::RequiredInputsNotFound,
              Inferno::Exceptions::NotUserRunnableException => e
         $stderr.puts COLOR.red "Error: #{e.full_message}"
         # TODO: use Application Logger?
+        exit(1)
       rescue StandardError => e
         $stderr.puts COLOR.red "Error: #{e.full_message}"
+        exit(1)
       end
 
       def thor_hash_to_inputs_array(hash)
@@ -119,7 +146,7 @@ def thor_hash_to_inputs_array(hash)
       def serialize(entity)
         case entity.class.to_s
         when 'Array'
-          entity.map{ |item| serialize(item) }
+          JSON.pretty_generate entity.map{ |item| JSON.parse serialize(item) }
         when ->(x) { defined?(x.constantize) && defined?("Inferno::Web::Serializers::#{x.split('::').last}".constantize) }
           "Inferno::Web::Serializers::#{entity.class.to_s.split('::').last}".constantize.render(entity)
         else
@@ -136,7 +163,6 @@ def verbose_puts(*args)
         verbose_print(*args)
       end
 
-=begin
       def suppress_output
         begin
           original_stderr = $stderr.clone
@@ -155,29 +181,23 @@ def suppress_output
         retval
       end
 
+      def fetch_test_id(result)
+        [result.test_id, result.test_group_id, result.test_suite_id].find { |x| x.presence }
+      end
+
+=begin
       def unindent_markdown(markdown)
         return nil if markdown.nil?
 
         natural_indent = markdown.lines.collect { |l| l.index(/[^ ]/) }.select { |l| !l.nil? && l.positive? }.min || 0
         markdown.lines.map { |l| l[natural_indent..-1] || "\n" }.join.lstrip
       end
-
+=end
       def print_requests(result)
-        result.request_responses.map do |req_res|
-          req_res.response_code.to_s + ' ' + req_res.request_method.upcase + ' ' + req_res.request_url
+        result.requests.map do |req_res|
+          req_res.status.to_s + ' ' + req_res.verb.upcase + ' ' + req_res.url
         end
       end
-
-      # TODO better name
-      def _run(runnable, test_session, inputs = {})
-        test_run_params = { test_session_id: test_session.id }.merge(runnable.reference_hash)
-        test_run = Inferno::Repositories::TestRuns.new.create(test_run_params)
-        inputs.each do |name, value|
-          session_data_repo.save(test_session_id: test_session.id, name: name, value: value, type: 'text')
-        end
-        Inferno::TestRunner.new(test_session: test_session, test_run: test_run).run(runnable)
-      end
-=end
     end
   end
 end

From de3849d76b8850a15caf624a0911c990b056a6c3 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 2 Aug 2024 11:52:39 -0400
Subject: [PATCH 07/83] suppress migration output

---
 lib/inferno/apps/cli/execute.rb | 55 +++++++++++++++------------------
 1 file changed, 25 insertions(+), 30 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 62cb3fa3a..752136409 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -2,7 +2,6 @@
 
 require 'pastel'
 require 'active_support'
-require_relative '../../../inferno'
 require_relative '../web/serializers/test_run'
 require_relative '../web/serializers/result'
 
@@ -10,6 +9,28 @@ module Inferno
   module CLI
     class Execute
 
+      def self.suppress_output
+        begin
+          original_stderr = $stderr.clone
+          original_stdout = $stdout.clone
+          $stderr.reopen(File.new('/dev/null', 'w'))
+          $stdout.reopen(File.new('/dev/null', 'w'))
+          retval = yield
+        rescue StandardError => e
+          $stdout.reopen(original_stdout)
+          $stderr.reopen(original_stderr)
+          raise e
+        ensure
+          $stdout.reopen(original_stdout)
+          $stderr.reopen(original_stderr)
+        end
+        retval
+      end
+
+      # Inferno boot flow triggers migration and logger outputs it
+      # I would be allow this in verbose mode but definitely not for JSON output
+      suppress_output{ require_relative '../../../inferno' }
+
       COLOR = Pastel.new
       CHECKMARK = "\u2713"
 
@@ -57,14 +78,14 @@ def persist_inputs(params, test_run)
       def run(options)
         puts ''
         puts '=========================================='
-        puts " Testing #{options[:suite]} Suite"
+        puts "Testing #{options[:suite]} Suite"
         puts '=========================================='
 
         self.options = options
         verbose_puts "options:", self.options
 
-        # TODO: hijack Application logger?
-        Inferno::Application.start(:suites)        
+        # TODO: hijack Application logger so it displays in verbose mode?
+        Inferno::Application.start(:suites)
 
         suite = Inferno::Repositories::TestSuites.new.find(options[:suite])
         raise StandardError, "Suite #{options[:suite]} not found" if suite.nil?
@@ -163,36 +184,10 @@ def verbose_puts(*args)
         verbose_print(*args)
       end
 
-      def suppress_output
-        begin
-          original_stderr = $stderr.clone
-          original_stdout = $stdout.clone
-          $stderr.reopen(File.new('/dev/null', 'w'))
-          $stdout.reopen(File.new('/dev/null', 'w'))
-          retval = yield
-        rescue StandardError => e
-          $stdout.reopen(original_stdout)
-          $stderr.reopen(original_stderr)
-          raise e
-        ensure
-          $stdout.reopen(original_stdout)
-          $stderr.reopen(original_stderr)
-        end
-        retval
-      end
-
       def fetch_test_id(result)
         [result.test_id, result.test_group_id, result.test_suite_id].find { |x| x.presence }
       end
 
-=begin
-      def unindent_markdown(markdown)
-        return nil if markdown.nil?
-
-        natural_indent = markdown.lines.collect { |l| l.index(/[^ ]/) }.select { |l| !l.nil? && l.positive? }.min || 0
-        markdown.lines.map { |l| l[natural_indent..-1] || "\n" }.join.lstrip
-      end
-=end
       def print_requests(result)
         result.requests.map do |req_res|
           req_res.status.to_s + ' ' + req_res.verb.upcase + ' ' + req_res.url

From 6a917f6777527535871eadee20879ac962ae455e Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Mon, 5 Aug 2024 17:33:19 -0400
Subject: [PATCH 08/83] create dev_validator_suite

---
 .../dev_validator_suite/validator_suite.rb    | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 dev_suites/dev_validator_suite/validator_suite.rb

diff --git a/dev_suites/dev_validator_suite/validator_suite.rb b/dev_suites/dev_validator_suite/validator_suite.rb
new file mode 100644
index 000000000..d562ef16f
--- /dev/null
+++ b/dev_suites/dev_validator_suite/validator_suite.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module DevValidatorSuite # rubocop:disable Naming/ClassAndModuleCamelCase
+  class ValidatorSuite < Inferno::TestSuite
+    title 'Validator Suite'
+    id :dev_validator
+    description "Inferno Core Developer Suite that makes calls to the HL7 Validator."  
+
+    input :url,
+          title: 'FHIR Server Base Url'
+
+    input :access_token,
+          title: 'Bearer/Access Token',
+          optional: true
+
+    fhir_client do
+      url :url
+      bearer_token :access_token
+    end
+
+    fhir_resource_validator do
+      url "http://localhost/hl7validatorapi"
+    end
+
+    group do
+      title 'Patient Test Group'
+      id :patient_group
+
+      input :patient_id,
+            title: 'Patient ID'
+
+      test do
+        title 'Patient Read Test'
+        id :patient_read_test
+
+        makes_request :patient_read
+
+        run do
+          fhir_read(:patient, patient_id, name: :patient_read)
+
+          assert_response_status 200
+        end
+      end
+
+      test do
+        title 'Patient Validate Test'
+        id :patient_validate_test
+
+        uses_request :patient_read
+
+        run do
+          assert_resource_type(:patient)
+          assert_valid_resource
+        end
+      end
+    end
+
+  end
+end

From d5fbe40789b7a8f577c0f3f6181cb02a332fe9c3 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 6 Aug 2024 17:11:44 -0400
Subject: [PATCH 09/83] create_params and print_messages functions

---
 lib/inferno/apps/cli/execute.rb | 54 ++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 21 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 752136409..cbe31c39c 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -84,7 +84,6 @@ def run(options)
         self.options = options
         verbose_puts "options:", self.options
 
-        # TODO: hijack Application logger so it displays in verbose mode?
         Inferno::Application.start(:suites)
 
         suite = Inferno::Repositories::TestSuites.new.find(options[:suite])
@@ -93,57 +92,56 @@ def run(options)
         test_session = test_sessions_repo.create({test_suite_id: suite.id}) # TODO add suite options
 
         verify_runnable(
-          test_runs_repo.build_entity({
-            test_session_id: test_session.id,
-            test_suite_id: suite.id,
-            inputs: thor_hash_to_inputs_array(options[:inputs])
-          }).runnable,
+          test_runs_repo.build_entity(create_params(test_session,suite)).runnable,
           thor_hash_to_inputs_array(options[:inputs]),
           test_session.suite_options
         )
 
-        test_run = test_runs_repo.create({
-          test_session_id: test_session.id,
-          test_suite_id: suite.id,
-          inputs: thor_hash_to_inputs_array(options[:inputs]),
-          status: 'queued'
-        })
+        test_run = test_runs_repo.create(
+          create_params(test_session,suite).merge({status: 'queued'})
+        )
 
-        persist_inputs({test_session_id: test_session.id, test_suite_id: suite.id, inputs: thor_hash_to_inputs_array(options[:inputs])}, test_run)
+        persist_inputs(create_params(test_session, suite), test_run)
 
         puts "Running tests. This may take a while..."
         Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
 
         results = test_runs_repo.results_for_test_run(test_run.id)&.reverse
         verbose_puts '=========================================='
-        verbose_puts "JSON Results:"
+        verbose_puts "JSON Test Results:"
         verbose_puts '=========================================='
         verbose_puts serialize(results)
         verbose_puts '=========================================='
 
         puts '=========================================='
-        puts "Color Results:"
+        puts "Colored Test Results:"
         puts '=========================================='
         results.each do |result|
-          print fetch_test_id(result), ":"
+          print fetch_test_id(result), ": "
           case result.result
           when 'pass'
-            print COLOR.green(CHECKMARK, ' pass')
+            print COLOR.bold.green(CHECKMARK, ' pass')
           when 'fail'
-            print COLOR.red 'X fail'
+            print COLOR.bold.red 'X fail'
           when 'skip'
             print COLOR.yellow '* skip'
           when 'omit'
             print COLOR.blue '* omit'
           when 'error'
             print COLOR.magenta 'X error'
+          when 'wait'
+            # This may be dead code with synchronous test execution
+            print '. wait'
+          when 'cancel'
+            print COLOR.red 'X cancel'
           else
             # TODO strict behavior or no?
             #raise StandardError.new, "Unrecognized result #{result.result}" # strict
-            print '- unknown'                                               # unstrict
+            print '- unknown'                                                # unstrict
           end
           puts ''
-          verbose_puts "\tmessage: ", result.result_message
+          verbose_puts "\tsummary: ",  result.result_message
+          verbose_puts "\tmessages: ", print_messages(result)
           verbose_puts "\trequests: ", print_requests(result)
         end
         puts '=========================================='
@@ -164,6 +162,14 @@ def thor_hash_to_inputs_array(hash)
         hash.to_a.map{|pair| {name: pair[0], value: pair[1]}}
       end
 
+      def create_params(test_session, suite)
+        {
+          test_session_id: test_session.id,
+          test_suite_id: suite.id,
+          inputs: thor_hash_to_inputs_array(self.options[:inputs])
+        }
+      end
+
       def serialize(entity)
         case entity.class.to_s
         when 'Array'
@@ -188,9 +194,15 @@ def fetch_test_id(result)
         [result.test_id, result.test_group_id, result.test_suite_id].find { |x| x.presence }
       end
 
+      def print_messages(result)
+        result.messages.map do |message|
+          "\n\t\t" + message.type + ": " + message.message
+        end
+      end
+
       def print_requests(result)
         result.requests.map do |req_res|
-          req_res.status.to_s + ' ' + req_res.verb.upcase + ' ' + req_res.url
+          "\n\t\t" + req_res.status.to_s + ' ' + req_res.verb.upcase + ' ' + req_res.url
         end
       end
     end

From ea9e5920e2f420baf91a145726e6521d80718d05 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 6 Aug 2024 17:26:31 -0400
Subject: [PATCH 10/83] factorize persist_inputs and verify_runnable functions

---
 lib/inferno/apps/cli/execute.rb               | 39 ++++---------------
 .../apps/web/controllers/test_runs/create.rb  | 38 ++++--------------
 lib/inferno/utils/persist_inputs.rb           | 31 +++++++++++++++
 lib/inferno/utils/verify_runnable.rb          | 16 ++++++++
 4 files changed, 61 insertions(+), 63 deletions(-)
 create mode 100644 lib/inferno/utils/persist_inputs.rb
 create mode 100644 lib/inferno/utils/verify_runnable.rb

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index cbe31c39c..b01b1ee1d 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -4,11 +4,16 @@
 require 'active_support'
 require_relative '../web/serializers/test_run'
 require_relative '../web/serializers/result'
+require_relative '../../utils/verify_runnable'
+require_relative '../../utils/persist_inputs'
 
 module Inferno
   module CLI
     class Execute
 
+      include ::Inferno::Utils::VerifyRunnable
+      include ::Inferno::Utils::PersistInputs
+
       def self.suppress_output
         begin
           original_stderr = $stderr.clone
@@ -31,6 +36,8 @@ def self.suppress_output
       # I would be allow this in verbose mode but definitely not for JSON output
       suppress_output{ require_relative '../../../inferno' }
 
+      # TODO hijack logger and suppress or redirect its output
+
       COLOR = Pastel.new
       CHECKMARK = "\u2713"
 
@@ -42,38 +49,6 @@ def self.suppress_output
 
       attr_accessor :options
 
-      # TODO factorize into some helper/util with hanami controller
-      def verify_runnable(runnable, inputs, selected_suite_options)
-        missing_inputs = runnable&.missing_inputs(inputs, selected_suite_options)
-        user_runnable = runnable&.user_runnable?
-        raise Inferno::Exceptions::RequiredInputsNotFound, missing_inputs if missing_inputs&.any?
-        raise Inferno::Exceptions::NotUserRunnableException unless user_runnable
-      end
-
-      # TODO likewise
-      def persist_inputs(params, test_run)
-        available_inputs = test_run.runnable.available_inputs
-        params[:inputs]&.each do |input_params|
-          input =
-            available_inputs
-              .find { |_, runnable_input| runnable_input.name == input_params[:name] }
-              &.last
-
-          if input.nil?
-            Inferno::Application['logger'].warn(
-              "Unknown input `#{input_params[:name]}` for #{test_run.runnable.id}: #{test_run.runnable.title}"
-            )
-            next
-          end
-
-          session_data_repo.save(
-            test_session_id: test_run.test_session_id,
-            name: input.name,
-            value: input_params[:value],
-            type: input.type
-          )
-        end
-      end
 
       def run(options)
         puts ''
diff --git a/lib/inferno/apps/web/controllers/test_runs/create.rb b/lib/inferno/apps/web/controllers/test_runs/create.rb
index 113fec3d0..5913c5742 100644
--- a/lib/inferno/apps/web/controllers/test_runs/create.rb
+++ b/lib/inferno/apps/web/controllers/test_runs/create.rb
@@ -1,8 +1,15 @@
+require_relative '../../../../utils/verify_runnable'
+require_relative '../../../../utils/persist_inputs'
+
 module Inferno
   module Web
     module Controllers
       module TestRuns
         class Create < Controller
+
+          include ::Inferno::Utils::VerifyRunnable
+          include ::Inferno::Utils::PersistInputs
+
           include Import[
                     test_sessions_repo: 'inferno.repositories.test_sessions',
                     session_data_repo: 'inferno.repositories.session_data',
@@ -11,37 +18,6 @@ class Create < Controller
 
           PARAMS = [:test_session_id, :test_suite_id, :test_group_id, :test_id].freeze
 
-          def verify_runnable(runnable, inputs, selected_suite_options)
-            missing_inputs = runnable&.missing_inputs(inputs, selected_suite_options)
-            user_runnable = runnable&.user_runnable?
-            raise Inferno::Exceptions::RequiredInputsNotFound, missing_inputs if missing_inputs&.any?
-            raise Inferno::Exceptions::NotUserRunnableException unless user_runnable
-          end
-
-          def persist_inputs(params, test_run)
-            available_inputs = test_run.runnable.available_inputs
-            params[:inputs]&.each do |input_params|
-              input =
-                available_inputs
-                  .find { |_, runnable_input| runnable_input.name == input_params[:name] }
-                  &.last
-
-              if input.nil?
-                Inferno::Application['logger'].warn(
-                  "Unknown input `#{input_params[:name]}` for #{test_run.runnable.id}: #{test_run.runnable.title}"
-                )
-                next
-              end
-
-              session_data_repo.save(
-                test_session_id: test_run.test_session_id,
-                name: input.name,
-                value: input_params[:value],
-                type: input.type
-              )
-            end
-          end
-
           def handle(req, res)
             test_session = test_sessions_repo.find(req.params[:test_session_id])
 
diff --git a/lib/inferno/utils/persist_inputs.rb b/lib/inferno/utils/persist_inputs.rb
new file mode 100644
index 000000000..74177f602
--- /dev/null
+++ b/lib/inferno/utils/persist_inputs.rb
@@ -0,0 +1,31 @@
+module Inferno
+  module Utils
+    module PersistInputs
+
+      def persist_inputs(params, test_run)
+        available_inputs = test_run.runnable.available_inputs
+        params[:inputs]&.each do |input_params|
+          input =
+            available_inputs
+              .find { |_, runnable_input| runnable_input.name == input_params[:name] }
+              &.last
+
+          if input.nil?
+            Inferno::Application['logger'].warn(
+              "Unknown input `#{input_params[:name]}` for #{test_run.runnable.id}: #{test_run.runnable.title}"
+            )
+            next
+          end
+
+          session_data_repo.save(
+            test_session_id: test_run.test_session_id,
+            name: input.name,
+            value: input_params[:value],
+            type: input.type
+          )
+        end
+      end
+
+    end
+  end
+end
diff --git a/lib/inferno/utils/verify_runnable.rb b/lib/inferno/utils/verify_runnable.rb
new file mode 100644
index 000000000..ab217e87c
--- /dev/null
+++ b/lib/inferno/utils/verify_runnable.rb
@@ -0,0 +1,16 @@
+require_relative '../exceptions'
+
+module Inferno
+  module Utils
+    module VerifyRunnable
+
+      def verify_runnable(runnable, inputs, selected_suite_options)
+        missing_inputs = runnable&.missing_inputs(inputs, selected_suite_options)
+        user_runnable = runnable&.user_runnable?
+        raise Inferno::Exceptions::RequiredInputsNotFound, missing_inputs if missing_inputs&.any?
+        raise Inferno::Exceptions::NotUserRunnableException unless user_runnable
+      end
+
+    end
+  end
+end

From 71cf9afceb35e0d9085390e7b5228972ce7fe8c0 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 7 Aug 2024 10:35:05 -0400
Subject: [PATCH 11/83] add exit codes

---
 lib/inferno/apps/cli/execute.rb | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index b01b1ee1d..23d1866d6 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -81,7 +81,7 @@ def run(options)
         puts "Running tests. This may take a while..."
         Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
 
-        results = test_runs_repo.results_for_test_run(test_run.id)&.reverse
+        results = test_runs_repo.results_for_test_run(test_run.id).reverse
         verbose_puts '=========================================='
         verbose_puts "JSON Test Results:"
         verbose_puts '=========================================='
@@ -116,21 +116,24 @@ def run(options)
           end
           puts ''
           verbose_puts "\tsummary: ",  result.result_message
-          verbose_puts "\tmessages: ", print_messages(result)
-          verbose_puts "\trequests: ", print_requests(result)
+          verbose_puts "\tmessages: ", format_messages(result)
+          verbose_puts "\trequests: ", format_requests(result)
         end
         puts '=========================================='
 
+        exit(0) if results.find{|result| result.test_suite_id == options[:suite_id]}.result == 'pass'
 
-      rescue Sequel::ValidationFailed, Sequel::ForeignKeyConstraintViolation,
-             Inferno::Exceptions::RequiredInputsNotFound,
-             Inferno::Exceptions::NotUserRunnableException => e
-        $stderr.puts COLOR.red "Error: #{e.full_message}"
-        # TODO: use Application Logger?
         exit(1)
+      rescue Sequel::ValidationFailed => e
+        print_error_and_exit(e, 3)
+      rescue Sequel::ForeignKeyConstraintViolation => e
+        print_error_and_exit(e, 4)
+      rescue Inferno::Exceptions::RequiredInputsNotFound => e
+        print_error_and_exit(e, 5)
+      rescue Inferno::Exceptions::NotUserRunnableException => e
+        print_error_and_exit(e, 6)
       rescue StandardError => e
-        $stderr.puts COLOR.red "Error: #{e.full_message}"
-        exit(1)
+        print_error_and_exit(e, 2)
       end
 
       def thor_hash_to_inputs_array(hash)
@@ -169,17 +172,23 @@ def fetch_test_id(result)
         [result.test_id, result.test_group_id, result.test_suite_id].find { |x| x.presence }
       end
 
-      def print_messages(result)
+      def format_messages(result)
         result.messages.map do |message|
           "\n\t\t" + message.type + ": " + message.message
         end
       end
 
-      def print_requests(result)
+      def format_requests(result)
         result.requests.map do |req_res|
           "\n\t\t" + req_res.status.to_s + ' ' + req_res.verb.upcase + ' ' + req_res.url
         end
       end
+
+      def print_error_and_exit(e, code)
+        # TODO: use Application Logger for stderr?
+        $stderr.puts COLOR.red "Error: #{e.full_message}"
+        exit(code)
+      end
     end
   end
 end

From ff68b3bb1c3d90b8f673f9c7f12103701dd6e00a Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 7 Aug 2024 11:04:14 -0400
Subject: [PATCH 12/83] add inputs and outputs to verbose

---
 lib/inferno/apps/cli/execute.rb | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 23d1866d6..6334316e8 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -118,6 +118,8 @@ def run(options)
           verbose_puts "\tsummary: ",  result.result_message
           verbose_puts "\tmessages: ", format_messages(result)
           verbose_puts "\trequests: ", format_requests(result)
+          verbose_puts "\tinputs: ",   format_inputs(result)
+          verbose_puts "\toutputs: ",  format_outputs(result)
         end
         puts '=========================================='
 
@@ -184,6 +186,19 @@ def format_requests(result)
         end
       end
 
+      def format_inputs(result, attr = :input_json)
+        input_json = result.send(attr)
+        return '' if input_json.nil?
+
+        JSON.parse(input_json).map do |input|
+          "\n\t\t#{input['name']}: #{input['value']}"
+        end
+      end
+
+      def format_outputs(result)
+        format_inputs(result, :output_json)
+      end
+
       def print_error_and_exit(e, code)
         # TODO: use Application Logger for stderr?
         $stderr.puts COLOR.red "Error: #{e.full_message}"

From df1a988850998130f5ca8d4e2fc5351763623db8 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 7 Aug 2024 13:49:22 -0400
Subject: [PATCH 13/83] add long help and fix thor depreciation notice

---
 lib/inferno/apps/cli/main.rb | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index a3bf153ed..d682c23cc 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -66,7 +66,20 @@ def version
         puts "Inferno Core v#{Inferno::VERSION}"
       end
 
+      EXECUTE_HELP = <<~EOT
+        Run Inferno tests in the command line. Exits with 0 only if test suite passes. Must be run from test kit as working directory.
+
+        You must have background services running: `bundle exec inferno services start`
+
+        You can view suite ids with: `bundle exec inferno suites`
+
+        Examples:
+
+            `bundle exec inferno execute --suite dev_validator --inputs "url:https://hapi.fhir.org/baseR4" patient_id:1234321`
+            => Outputs test results
+      EOT
       desc 'execute', 'Run Inferno tests in command line'
+      long_desc EXECUTE_HELP, wrap: false
       option :suite,
              aliases: ['-s'],
              required: true,
@@ -87,6 +100,11 @@ def execute
         Execute.new.run(options)
       end
 
+      # https://github.com/rails/thor/issues/244 - Make Thor exit(1) on Errors/Exceptions
+      def self.exit_on_failure?
+        true
+      end
+
       private
 
       # https://github.com/rubocop/rubocop/issues/12571 - still affects Ruby 3.1 upto Rubocop 1.63

From c397a2cb23ba54fc5e01b7095d8bf57d6fe35a38 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 7 Aug 2024 17:30:22 -0400
Subject: [PATCH 14/83] wip rspec

---
 spec/inferno/apps/cli/execute_spec.rb      | 86 ++++++++++++++++++++++
 spec/inferno/utils/persist_inputs_spec.rb  | 12 +++
 spec/inferno/utils/verify_runnable_spec.rb |  9 +++
 3 files changed, 107 insertions(+)
 create mode 100644 spec/inferno/apps/cli/execute_spec.rb
 create mode 100644 spec/inferno/utils/persist_inputs_spec.rb
 create mode 100644 spec/inferno/utils/verify_runnable_spec.rb

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
new file mode 100644
index 000000000..b9aa86710
--- /dev/null
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -0,0 +1,86 @@
+require_relative '../../../../lib/inferno/apps/cli/execute.rb'
+
+RSpec.describe Inferno::CLI::Execute do
+  
+  describe '#thor_hash_to_inputs_array' do
+    let(:hash) { {url: 'https://example.com'} }
+
+    it 'converts hash to array' do
+      result = described_class.new.thor_hash_to_inputs_array(hash)
+      expect(result.class).to eq(Array)
+    end
+
+    it 'returns proper inputs array' do
+      result = described_class.new.thor_hash_to_inputs_array(hash)
+      expect(result).to eq([{name: :url, value: 'https://example.com'}])
+    end
+  end
+
+  describe '#create_params' do
+    let(:test_suite) { BasicTestSuite::Suite }
+    let(:test_sessions_repo) { Inferno::Repositories::TestSessions.new }
+
+    it 'returns test run params' do
+      instance = described_class.new
+      allow(instance).to receive(:options).and_return({inputs: {url: 'https://example.com'}})
+      test_session = test_sessions_repo.create(test_suite_id: test_suite.id)
+
+      result = instance.create_params(test_session, test_suite)
+      expect(result).to eq({test_session_id: test_session.id, test_suite_id: 'basic', inputs: [{name: :url, value: 'https://example.com'}]})
+    end
+  end
+
+  describe '#serialize' do
+    it 'handles an array of test results' do
+      pending 'TODO'
+    end
+  end
+
+  describe '#verbose_print' do
+    it 'works' do
+      pending 'TODO'
+    end
+  end
+
+  describe '#verbose_puts' do
+    it 'works' do
+      pending 'TODO'
+    end
+  end
+
+  describe '#fetch_test_id' do
+    it 'works' do
+      pending 'TODO'
+    end
+  end
+
+  describe '#format_messages' do
+    it 'works' do
+      pending 'TODO'
+    end
+  end
+
+  describe '#format_requests' do
+    it 'works' do
+      pending 'TODO'
+    end
+  end
+
+  describe '#format_inputs' do
+    it 'works' do
+      pending 'TODO'
+    end
+  end
+
+  describe '#format_outputs' do
+    it 'works' do
+      pending 'TODO'
+    end
+  end
+
+  describe '#print_error_and_exit' do
+    it 'works' do
+      pending 'TODO'
+    end
+  end
+end
diff --git a/spec/inferno/utils/persist_inputs_spec.rb b/spec/inferno/utils/persist_inputs_spec.rb
new file mode 100644
index 000000000..31d8089cb
--- /dev/null
+++ b/spec/inferno/utils/persist_inputs_spec.rb
@@ -0,0 +1,12 @@
+require_relative '../../../lib/inferno/utils/persist_inputs'
+
+RSpec.describe Inferno::Utils::PersistInputs do
+  
+  describe '#persist_inputs' do
+    it 'is defined' do
+      expect(described_class.method_defined?(:persist_inputs)).to eq(true)
+    end
+  end
+
+end
+
diff --git a/spec/inferno/utils/verify_runnable_spec.rb b/spec/inferno/utils/verify_runnable_spec.rb
new file mode 100644
index 000000000..68a03c0d4
--- /dev/null
+++ b/spec/inferno/utils/verify_runnable_spec.rb
@@ -0,0 +1,9 @@
+require_relative '../../../lib/inferno/utils/verify_runnable.rb'
+
+RSpec.describe Inferno::Utils::VerifyRunnable do
+  describe '#verify_runnable' do
+    it 'is defined' do
+      expect(described_class.method_defined?(:verify_runnable)).to eq(true)
+    end
+  end
+end

From aad9453dbf917cfa05d4096f13d9081b039afb3f Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 7 Aug 2024 21:31:45 -0400
Subject: [PATCH 15/83] wip rspec

---
 lib/inferno/apps/cli/execute.rb       |   3 +-
 spec/inferno/apps/cli/execute_spec.rb | 136 ++++++++++++++++++++++----
 2 files changed, 120 insertions(+), 19 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 6334316e8..d048d07d4 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -78,7 +78,7 @@ def run(options)
 
         persist_inputs(create_params(test_session, suite), test_run)
 
-        puts "Running tests. This may take a while..."
+        puts "Running tests. This may take a while..." # TODO spinner/progress bar
         Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
 
         results = test_runs_repo.results_for_test_run(test_run.id).reverse
@@ -170,6 +170,7 @@ def verbose_puts(*args)
         verbose_print(*args)
       end
 
+      # TODO - try to replace this with `result.runnable.id`
       def fetch_test_id(result)
         [result.test_id, result.test_group_id, result.test_suite_id].find { |x| x.presence }
       end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index b9aa86710..5a937f829 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -1,17 +1,20 @@
 require_relative '../../../../lib/inferno/apps/cli/execute.rb'
 
-RSpec.describe Inferno::CLI::Execute do
-  
+## TODO REFACTOR ALL THESE TESTS WITH FACTORY BOT
+
+RSpec.describe Inferno::CLI::Execute do  
+  let(:instance) { described_class.new }
+
   describe '#thor_hash_to_inputs_array' do
     let(:hash) { {url: 'https://example.com'} }
 
     it 'converts hash to array' do
-      result = described_class.new.thor_hash_to_inputs_array(hash)
+      result = instance.thor_hash_to_inputs_array(hash)
       expect(result.class).to eq(Array)
     end
 
     it 'returns proper inputs array' do
-      result = described_class.new.thor_hash_to_inputs_array(hash)
+      result = instance.thor_hash_to_inputs_array(hash)
       expect(result).to eq([{name: :url, value: 'https://example.com'}])
     end
   end
@@ -21,42 +24,137 @@
     let(:test_sessions_repo) { Inferno::Repositories::TestSessions.new }
 
     it 'returns test run params' do
-      instance = described_class.new
-      allow(instance).to receive(:options).and_return({inputs: {url: 'https://example.com'}})
+      stubbed_instance = instance()
+      allow(stubbed_instance).to receive(:options).and_return({inputs: {url: 'https://example.com'}})
       test_session = test_sessions_repo.create(test_suite_id: test_suite.id)
 
-      result = instance.create_params(test_session, test_suite)
+      result = stubbed_instance.create_params(test_session, test_suite)
       expect(result).to eq({test_session_id: test_session.id, test_suite_id: 'basic', inputs: [{name: :url, value: 'https://example.com'}]})
     end
   end
 
   describe '#serialize' do
-    it 'handles an array of test results' do
-      pending 'TODO'
+    let(:test_suite) { BasicTestSuite::Suite }
+    let(:test_session) { Inferno::Repositories::TestSessions.new.create(test_suite_id: test_suite.id) }
+    let(:test_run) { Inferno::Repositories::TestRun.new.create({test_session:, test_suite:, status: 'done'}) }
+    let(:test_result) do
+      Inferno::Repositories::Result.new.create({
+        test_suite_id: test_suite.id,
+        test_session_id: test_session.id,
+        test_run_id: test_run.id,
+        result: 'pass',
+        result_message: 'This is a mock result'
+      })
+    end
+    let(:test_results) { [test_result, test_result] }
+
+    it 'handles an array of test results without raising exception' do
+      expect { instance.serialize(test_results) }.not_to raise_error(StandardError)
+    end
+
+    it 'returns legit JSON' do
+      expect { JSON.parse(instance.serialize(test_results)) }.not_to raise_error(JSON::ParserError)
+      expect { JSON.parse(instance.serialize(test_results)) }.not_to raise_error(JSON::NestingError)
+      expect { JSON.parse(instance.serialize(test_results)) }.not_to raise_error(TypeError)
     end
   end
 
   describe '#verbose_print' do
-    it 'works' do
-      pending 'TODO'
+    it 'outputs when verbose is true' do
+      stubbed_instance = instance()
+      allow(stubbed_instance).to receive(:options).and_return({verbose: true})
+
+      expect { stubbed_instance.verbose_print('Lorem') }.to output(/Lorem/).to_stdout
+    end
+
+    it 'does not output when verbose is false' do
+      stubbed_instance = instance()
+      allow(stubbed_instance).to receive(:options).and_return({verbose: false})
+
+      expect { stubbed_instance.verbose_print('Lorem') }.not_to output(/.+/).to_stdout
     end
   end
 
   describe '#verbose_puts' do
-    it 'works' do
-      pending 'TODO'
+    it 'has output ending with \n with when verbose is true' do
+      stubbed_instance = instance()
+      allow(stubbed_instance).to receive(:options).and_return({verbose: true})
+
+      expect { stubbed_instance.verbose_puts('Lorem') }.to output(/Lorem\n$/).to_stdout
     end
   end
 
+  # TODO: see if I can replace fetch_test_id with .runnable.id
   describe '#fetch_test_id' do
-    it 'works' do
-      pending 'TODO'
+    let(:test_suite) { BasicTestSuite::Suite }
+    let(:test_group) { BasicTestSuite::AbcGroup }
+    let(:test) { test_group.tests.first }
+    let(:test_session) { Inferno::Repositories::TestSessions.new.create(test_suite_id: test_suite.id) }
+    let(:test_run) { Inferno::Repositories::TestRun.new.create({test_session:, test_suite:, status: 'done'}) }
+
+    it 'returns suite id if test result belongs to suite' do
+      let(:test_result) do
+        Inferno::Repositories::Result.new.create({
+          test_suite_id: test_suite.id,
+          test_session_id: test_session.id,
+          test_run_id: test_run.id,
+          result: 'pass',
+          result_message: 'This is a mock result'
+        })
+      end
+
+      expect( instance.fetch_test_id(test_result) ).to eq( test_suite.id )
+    end
+
+    it 'returns group id if test result belongs to group' do
+      let(:test_result) do
+        Inferno::Repositories::Result.new.create({
+          test_group_id: test_group.id,
+          test_session_id: test_session.id,
+          test_run_id: test_run.id,
+          result: 'pass',
+          result_message: 'This is a mock result'
+        })
+      end
+
+      expect( instance.fetch_test_id(test_result) ).to eq( test_group.id )
+    end
+
+    it 'returns test id if test result belongs to test' do
+      let(:test_result) do
+        Inferno::Repositories::Result.new.create({
+          test_id: test.id,
+          test_session_id: test_session.id,
+          test_run_id: test_run.id,
+          result: 'pass',
+          result_message: 'This is a mock result'
+        })
+      end
+
+      expect( instance.fetch_test_id(test_result) ).to eq( test.id )
     end
   end
 
   describe '#format_messages' do
-    it 'works' do
-      pending 'TODO'
+    let(:test_suite) { BasicTestSuite::Suite }
+    let(:test_session) { Inferno::Repositories::TestSessions.new.create(test_suite_id: test_suite.id) }
+    let(:test_run) { Inferno::Repositories::TestRun.new.create({test_session:, test_suite:, status: 'done'}) }
+    let(:message) { Inferno::Entities::Messages.new }
+    let(:test_result) do
+      Inferno::Repositories::Result.new.create({
+        test_suite_id: test_suite.id,
+        test_session_id: test_session.id,
+        test_run_id: test_run.id,
+        result: 'pass',
+        result_message: 'This is a mock result',
+        messages: [
+          
+        ]
+      })
+    end
+
+    it 'does not omit any data' do
+      
     end
   end
 
@@ -80,7 +178,9 @@
 
   describe '#print_error_and_exit' do
     it 'works' do
-      pending 'TODO'
+      let(:mock_error_class) { Class.new(StandardError) }
+      let(:mock_error) { mock_error_class.new('mock message') }
+
     end
   end
 end

From 095117b27adfc9e49aa44d59868e67d70861f9e7 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 8 Aug 2024 16:03:37 -0400
Subject: [PATCH 16/83] debugging

---
 spec/inferno/apps/cli/execute_spec.rb | 138 +++++++++++---------------
 1 file changed, 60 insertions(+), 78 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 5a937f829..05ace6c8f 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -2,7 +2,7 @@
 
 ## TODO REFACTOR ALL THESE TESTS WITH FACTORY BOT
 
-RSpec.describe Inferno::CLI::Execute do  
+RSpec.describe Inferno::CLI::Execute do # rubocop:disable RSpec/FilePath
   let(:instance) { described_class.new }
 
   describe '#thor_hash_to_inputs_array' do
@@ -21,32 +21,22 @@
 
   describe '#create_params' do
     let(:test_suite) { BasicTestSuite::Suite }
-    let(:test_sessions_repo) { Inferno::Repositories::TestSessions.new }
+    let(:test_session) { create(:test_session) }
+    let(:inputs_hash) { {url: 'https://example.com'} }
+    let(:inputs_array) { [{name: :url, value: 'https://example.com'}] }
 
     it 'returns test run params' do
       stubbed_instance = instance()
-      allow(stubbed_instance).to receive(:options).and_return({inputs: {url: 'https://example.com'}})
-      test_session = test_sessions_repo.create(test_suite_id: test_suite.id)
+      allow(stubbed_instance).to receive(:options).and_return({inputs: inputs_hash})
+      test_session_inst = test_session()
 
-      result = stubbed_instance.create_params(test_session, test_suite)
-      expect(result).to eq({test_session_id: test_session.id, test_suite_id: 'basic', inputs: [{name: :url, value: 'https://example.com'}]})
+      result = stubbed_instance.create_params(test_session_inst, test_suite)
+      expect(result).to eq({test_session_id: test_session.id, test_suite_id: test_suite.id, inputs: inputs_array})
     end
   end
 
   describe '#serialize' do
-    let(:test_suite) { BasicTestSuite::Suite }
-    let(:test_session) { Inferno::Repositories::TestSessions.new.create(test_suite_id: test_suite.id) }
-    let(:test_run) { Inferno::Repositories::TestRun.new.create({test_session:, test_suite:, status: 'done'}) }
-    let(:test_result) do
-      Inferno::Repositories::Result.new.create({
-        test_suite_id: test_suite.id,
-        test_session_id: test_session.id,
-        test_run_id: test_run.id,
-        result: 'pass',
-        result_message: 'This is a mock result'
-      })
-    end
-    let(:test_results) { [test_result, test_result] }
+    let(:test_results) { create_list(:result, 2) }
 
     it 'handles an array of test results without raising exception' do
       expect { instance.serialize(test_results) }.not_to raise_error(StandardError)
@@ -80,7 +70,7 @@
       stubbed_instance = instance()
       allow(stubbed_instance).to receive(:options).and_return({verbose: true})
 
-      expect { stubbed_instance.verbose_puts('Lorem') }.to output(/Lorem\n$/).to_stdout
+      expect { stubbed_instance.verbose_puts('Lorem') }.to output(/Lorem\n/).to_stdout
     end
   end
 
@@ -89,98 +79,90 @@
     let(:test_suite) { BasicTestSuite::Suite }
     let(:test_group) { BasicTestSuite::AbcGroup }
     let(:test) { test_group.tests.first }
-    let(:test_session) { Inferno::Repositories::TestSessions.new.create(test_suite_id: test_suite.id) }
-    let(:test_run) { Inferno::Repositories::TestRun.new.create({test_session:, test_suite:, status: 'done'}) }
+    let(:instance) { described_class.new }
 
     it 'returns suite id if test result belongs to suite' do
-      let(:test_result) do
-        Inferno::Repositories::Result.new.create({
-          test_suite_id: test_suite.id,
-          test_session_id: test_session.id,
-          test_run_id: test_run.id,
-          result: 'pass',
-          result_message: 'This is a mock result'
-        })
-      end
+      test_result = create(:result, runnable: {test_suite_id: test_suite.id})
 
       expect( instance.fetch_test_id(test_result) ).to eq( test_suite.id )
     end
 
     it 'returns group id if test result belongs to group' do
-      let(:test_result) do
-        Inferno::Repositories::Result.new.create({
-          test_group_id: test_group.id,
-          test_session_id: test_session.id,
-          test_run_id: test_run.id,
-          result: 'pass',
-          result_message: 'This is a mock result'
-        })
-      end
+      test_result = create(:result, runnable: {test_group_id: test_group.id})
 
       expect( instance.fetch_test_id(test_result) ).to eq( test_group.id )
     end
 
     it 'returns test id if test result belongs to test' do
-      let(:test_result) do
-        Inferno::Repositories::Result.new.create({
-          test_id: test.id,
-          test_session_id: test_session.id,
-          test_run_id: test_run.id,
-          result: 'pass',
-          result_message: 'This is a mock result'
-        })
-      end
+      test_result = create(:result, runnable: {test_id: test.id});
 
       expect( instance.fetch_test_id(test_result) ).to eq( test.id )
     end
   end
 
   describe '#format_messages' do
-    let(:test_suite) { BasicTestSuite::Suite }
-    let(:test_session) { Inferno::Repositories::TestSessions.new.create(test_suite_id: test_suite.id) }
-    let(:test_run) { Inferno::Repositories::TestRun.new.create({test_session:, test_suite:, status: 'done'}) }
-    let(:message) { Inferno::Entities::Messages.new }
-    let(:test_result) do
-      Inferno::Repositories::Result.new.create({
-        test_suite_id: test_suite.id,
-        test_session_id: test_session.id,
-        test_run_id: test_run.id,
-        result: 'pass',
-        result_message: 'This is a mock result',
-        messages: [
-          
-        ]
-      })
-    end
-
-    it 'does not omit any data' do
-      
+    let(:test_result) { create(:result, message_count: 10) }
+    let(:instance) { described_class.new }
+
+    it 'includes all characters case-insensitive' do
+      messages = test_result.messages
+      formatted_string = instance.format_messages(messages)
+
+      messages.each do |message|
+        expect(formatted_string.upcase).to include message.upcase
+      end
     end
   end
 
   describe '#format_requests' do
-    it 'works' do
-      pending 'TODO'
+    let(:test_result) { create(:result, request_count: 10) }
+    let(:instance) { described_class.new }
+
+    it 'includes all status codes' do
+      requests = test_result.requests
+      formatted_string = instance.format_messages(messages)
+
+      requests.each do |request|
+        expect(formatted_string.upcase).to include request.status.to_s.upcase
+      end
     end
   end
 
   describe '#format_inputs' do
-    it 'works' do
-      pending 'TODO'
+    let(:inputs) {[{name: :url, value: 'https://example.com'}]}
+    let(:test_result) { create(:result, input_json: JSON.generate(inputs)) }
+    let(:instance) { described_class.new }
+
+    it 'includes all values' do
+      formatted_string = instance.format_inputs(test_result).join
+      inputs.each do |input_element|
+        expect(formatted_string).to include  input_element[:value]
+      end
     end
   end
 
   describe '#format_outputs' do
-    it 'works' do
-      pending 'TODO'
+    let(:outputs) {[{name: :token, value: 'SAMPLE_OUTPUT'}]}
+    let(:test_result) { create(:result, output_json: JSON.generate(outputs)) }
+    let(:instance) { described_class.new }
+
+    it 'includes all values' do
+      formatted_string = instance.format_outputs(test_result).join
+      outputs.each do |output_element|
+        expect(formatted_string).to include output_element[:value]
+      end
     end
   end
 
   describe '#print_error_and_exit' do
-    it 'works' do
-      let(:mock_error_class) { Class.new(StandardError) }
-      let(:mock_error) { mock_error_class.new('mock message') }
-
+    let(:mock_error_class) { Class.new(StandardError) }
+    let(:mock_error) { mock_error_class.new('mock message') }
+    let(:instance) { described_class.new }
+
+    it 'outputs to stderr and exits' do
+      expect do
+        expect { instance.print_error_and_exit(mock_error, 2) }.to output(/Error/).to_stderr
+      end.to raise_error(SystemExit)
     end
   end
 end

From 35334b96a706c01ec117f53420ddb5f3e5198020 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 8 Aug 2024 16:22:13 -0400
Subject: [PATCH 17/83] debug

---
 lib/inferno/apps/cli/execute.rb       |  6 +++---
 spec/inferno/apps/cli/execute_spec.rb | 14 +++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index d048d07d4..76ca272ad 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -178,13 +178,13 @@ def fetch_test_id(result)
       def format_messages(result)
         result.messages.map do |message|
           "\n\t\t" + message.type + ": " + message.message
-        end
+        end.join('')
       end
 
       def format_requests(result)
         result.requests.map do |req_res|
           "\n\t\t" + req_res.status.to_s + ' ' + req_res.verb.upcase + ' ' + req_res.url
-        end
+        end.join('')
       end
 
       def format_inputs(result, attr = :input_json)
@@ -193,7 +193,7 @@ def format_inputs(result, attr = :input_json)
 
         JSON.parse(input_json).map do |input|
           "\n\t\t#{input['name']}: #{input['value']}"
-        end
+        end.join('')
       end
 
       def format_outputs(result)
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 05ace6c8f..9941b99b1 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -101,26 +101,26 @@
   end
 
   describe '#format_messages' do
-    let(:test_result) { create(:result, message_count: 10) }
+    let(:test_result) { repo_create(:result, message_count: 10) }
     let(:instance) { described_class.new }
 
     it 'includes all characters case-insensitive' do
       messages = test_result.messages
-      formatted_string = instance.format_messages(messages)
+      formatted_string = instance.format_messages(test_result)
 
       messages.each do |message|
-        expect(formatted_string.upcase).to include message.upcase
+        expect(formatted_string.upcase).to include message.message.upcase
       end
     end
   end
 
   describe '#format_requests' do
-    let(:test_result) { create(:result, request_count: 10) }
+    let(:test_result) { repo_create(:result, request_count: 10) }
     let(:instance) { described_class.new }
 
     it 'includes all status codes' do
       requests = test_result.requests
-      formatted_string = instance.format_messages(messages)
+      formatted_string = instance.format_requests(test_result)
 
       requests.each do |request|
         expect(formatted_string.upcase).to include request.status.to_s.upcase
@@ -134,7 +134,7 @@
     let(:instance) { described_class.new }
 
     it 'includes all values' do
-      formatted_string = instance.format_inputs(test_result).join
+      formatted_string = instance.format_inputs(test_result)
       inputs.each do |input_element|
         expect(formatted_string).to include  input_element[:value]
       end
@@ -147,7 +147,7 @@
     let(:instance) { described_class.new }
 
     it 'includes all values' do
-      formatted_string = instance.format_outputs(test_result).join
+      formatted_string = instance.format_outputs(test_result)
       outputs.each do |output_element|
         expect(formatted_string).to include output_element[:value]
       end

From 7ef1523e9e06326b4e359cef0a3721e2262201fd Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 8 Aug 2024 16:24:45 -0400
Subject: [PATCH 18/83] rename fetch_test_id to format_id

---
 lib/inferno/apps/cli/execute.rb       | 7 +++----
 spec/inferno/apps/cli/execute_spec.rb | 9 ++++-----
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 76ca272ad..b4df9c878 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -92,7 +92,7 @@ def run(options)
         puts "Colored Test Results:"
         puts '=========================================='
         results.each do |result|
-          print fetch_test_id(result), ": "
+          print format_id(result), ": "
           case result.result
           when 'pass'
             print COLOR.bold.green(CHECKMARK, ' pass')
@@ -170,9 +170,8 @@ def verbose_puts(*args)
         verbose_print(*args)
       end
 
-      # TODO - try to replace this with `result.runnable.id`
-      def fetch_test_id(result)
-        [result.test_id, result.test_group_id, result.test_suite_id].find { |x| x.presence }
+      def format_id(result)
+        result.runnable.id
       end
 
       def format_messages(result)
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 9941b99b1..246e7f036 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -74,8 +74,7 @@
     end
   end
 
-  # TODO: see if I can replace fetch_test_id with .runnable.id
-  describe '#fetch_test_id' do
+  describe '#format_id' do
     let(:test_suite) { BasicTestSuite::Suite }
     let(:test_group) { BasicTestSuite::AbcGroup }
     let(:test) { test_group.tests.first }
@@ -84,19 +83,19 @@
     it 'returns suite id if test result belongs to suite' do
       test_result = create(:result, runnable: {test_suite_id: test_suite.id})
 
-      expect( instance.fetch_test_id(test_result) ).to eq( test_suite.id )
+      expect( instance.format_id(test_result) ).to eq( test_suite.id )
     end
 
     it 'returns group id if test result belongs to group' do
       test_result = create(:result, runnable: {test_group_id: test_group.id})
 
-      expect( instance.fetch_test_id(test_result) ).to eq( test_group.id )
+      expect( instance.format_id(test_result) ).to eq( test_group.id )
     end
 
     it 'returns test id if test result belongs to test' do
       test_result = create(:result, runnable: {test_id: test.id});
 
-      expect( instance.fetch_test_id(test_result) ).to eq( test.id )
+      expect( instance.format_id(test_result) ).to eq( test.id )
     end
   end
 

From 2df72f334a61927b0385b5230cda17d7649ae01d Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 8 Aug 2024 16:25:46 -0400
Subject: [PATCH 19/83] cleaning

---
 spec/inferno/apps/cli/execute_spec.rb | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 246e7f036..895deb129 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -78,7 +78,6 @@
     let(:test_suite) { BasicTestSuite::Suite }
     let(:test_group) { BasicTestSuite::AbcGroup }
     let(:test) { test_group.tests.first }
-    let(:instance) { described_class.new }
 
     it 'returns suite id if test result belongs to suite' do
       test_result = create(:result, runnable: {test_suite_id: test_suite.id})
@@ -101,7 +100,6 @@
 
   describe '#format_messages' do
     let(:test_result) { repo_create(:result, message_count: 10) }
-    let(:instance) { described_class.new }
 
     it 'includes all characters case-insensitive' do
       messages = test_result.messages
@@ -130,7 +128,6 @@
   describe '#format_inputs' do
     let(:inputs) {[{name: :url, value: 'https://example.com'}]}
     let(:test_result) { create(:result, input_json: JSON.generate(inputs)) }
-    let(:instance) { described_class.new }
 
     it 'includes all values' do
       formatted_string = instance.format_inputs(test_result)
@@ -143,7 +140,6 @@
   describe '#format_outputs' do
     let(:outputs) {[{name: :token, value: 'SAMPLE_OUTPUT'}]}
     let(:test_result) { create(:result, output_json: JSON.generate(outputs)) }
-    let(:instance) { described_class.new }
 
     it 'includes all values' do
       formatted_string = instance.format_outputs(test_result)
@@ -156,7 +152,6 @@
   describe '#print_error_and_exit' do
     let(:mock_error_class) { Class.new(StandardError) }
     let(:mock_error) { mock_error_class.new('mock message') }
-    let(:instance) { described_class.new }
 
     it 'outputs to stderr and exits' do
       expect do

From d44d4085aacae065f980f4b59b91e9b8c0597a7f Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 8 Aug 2024 16:48:38 -0400
Subject: [PATCH 20/83] cop

---
 .../dev_validator_suite/validator_suite.rb    |  7 +--
 lib/inferno/apps/cli/execute.rb               | 46 +++++++--------
 .../apps/web/controllers/test_runs/create.rb  |  1 -
 lib/inferno/utils/persist_inputs.rb           |  2 -
 lib/inferno/utils/verify_runnable.rb          |  2 -
 spec/inferno/apps/cli/execute_spec.rb         | 58 +++++++++----------
 spec/inferno/utils/persist_inputs_spec.rb     |  3 -
 spec/inferno/utils/verify_runnable_spec.rb    |  2 +-
 8 files changed, 56 insertions(+), 65 deletions(-)

diff --git a/dev_suites/dev_validator_suite/validator_suite.rb b/dev_suites/dev_validator_suite/validator_suite.rb
index d562ef16f..edc848b00 100644
--- a/dev_suites/dev_validator_suite/validator_suite.rb
+++ b/dev_suites/dev_validator_suite/validator_suite.rb
@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-module DevValidatorSuite # rubocop:disable Naming/ClassAndModuleCamelCase
+module DevValidatorSuite
   class ValidatorSuite < Inferno::TestSuite
     title 'Validator Suite'
     id :dev_validator
-    description "Inferno Core Developer Suite that makes calls to the HL7 Validator."  
+    description 'Inferno Core Developer Suite that makes calls to the HL7 Validator.'
 
     input :url,
           title: 'FHIR Server Base Url'
@@ -19,7 +19,7 @@ class ValidatorSuite < Inferno::TestSuite
     end
 
     fhir_resource_validator do
-      url "http://localhost/hl7validatorapi"
+      url 'http://localhost/hl7validatorapi'
     end
 
     group do
@@ -54,6 +54,5 @@ class ValidatorSuite < Inferno::TestSuite
         end
       end
     end
-
   end
 end
diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index b4df9c878..f03aaaa73 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -10,7 +10,6 @@
 module Inferno
   module CLI
     class Execute
-
       include ::Inferno::Utils::VerifyRunnable
       include ::Inferno::Utils::PersistInputs
 
@@ -34,9 +33,9 @@ def self.suppress_output
 
       # Inferno boot flow triggers migration and logger outputs it
       # I would be allow this in verbose mode but definitely not for JSON output
-      suppress_output{ require_relative '../../../inferno' }
+      suppress_output { require_relative '../../../inferno' }
 
-      # TODO hijack logger and suppress or redirect its output
+      # TODO: hijack logger and suppress or redirect its output
 
       COLOR = Pastel.new
       CHECKMARK = "\u2713"
@@ -49,7 +48,6 @@ def self.suppress_output
 
       attr_accessor :options
 
-
       def run(options)
         puts ''
         puts '=========================================='
@@ -57,42 +55,42 @@ def run(options)
         puts '=========================================='
 
         self.options = options
-        verbose_puts "options:", self.options
+        verbose_puts 'options:', self.options
 
         Inferno::Application.start(:suites)
 
         suite = Inferno::Repositories::TestSuites.new.find(options[:suite])
         raise StandardError, "Suite #{options[:suite]} not found" if suite.nil?
 
-        test_session = test_sessions_repo.create({test_suite_id: suite.id}) # TODO add suite options
+        test_session = test_sessions_repo.create({ test_suite_id: suite.id }) # TODO: add suite options
 
         verify_runnable(
-          test_runs_repo.build_entity(create_params(test_session,suite)).runnable,
+          test_runs_repo.build_entity(create_params(test_session, suite)).runnable,
           thor_hash_to_inputs_array(options[:inputs]),
           test_session.suite_options
         )
 
         test_run = test_runs_repo.create(
-          create_params(test_session,suite).merge({status: 'queued'})
+          create_params(test_session, suite).merge({ status: 'queued' })
         )
 
         persist_inputs(create_params(test_session, suite), test_run)
 
-        puts "Running tests. This may take a while..." # TODO spinner/progress bar
+        puts 'Running tests. This may take a while...' # TODO: spinner/progress bar
         Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
 
         results = test_runs_repo.results_for_test_run(test_run.id).reverse
         verbose_puts '=========================================='
-        verbose_puts "JSON Test Results:"
+        verbose_puts 'JSON Test Results:'
         verbose_puts '=========================================='
         verbose_puts serialize(results)
         verbose_puts '=========================================='
 
         puts '=========================================='
-        puts "Colored Test Results:"
+        puts 'Colored Test Results:'
         puts '=========================================='
         results.each do |result|
-          print format_id(result), ": "
+          print format_id(result), ': '
           case result.result
           when 'pass'
             print COLOR.bold.green(CHECKMARK, ' pass')
@@ -110,9 +108,9 @@ def run(options)
           when 'cancel'
             print COLOR.red 'X cancel'
           else
-            # TODO strict behavior or no?
-            #raise StandardError.new, "Unrecognized result #{result.result}" # strict
-            print '- unknown'                                                # unstrict
+            # TODO: strict behavior or no?
+            # raise StandardError.new, "Unrecognized result #{result.result}" # strict
+            print '- unknown' # unstrict
           end
           puts ''
           verbose_puts "\tsummary: ",  result.result_message
@@ -123,7 +121,7 @@ def run(options)
         end
         puts '=========================================='
 
-        exit(0) if results.find{|result| result.test_suite_id == options[:suite_id]}.result == 'pass'
+        exit(0) if results.find { |result| result.test_suite_id == options[:suite_id] }.result == 'pass'
 
         exit(1)
       rescue Sequel::ValidationFailed => e
@@ -139,22 +137,24 @@ def run(options)
       end
 
       def thor_hash_to_inputs_array(hash)
-        hash.to_a.map{|pair| {name: pair[0], value: pair[1]}}
+        hash.to_a.map { |pair| { name: pair[0], value: pair[1] } }
       end
 
       def create_params(test_session, suite)
         {
           test_session_id: test_session.id,
           test_suite_id: suite.id,
-          inputs: thor_hash_to_inputs_array(self.options[:inputs])
+          inputs: thor_hash_to_inputs_array(options[:inputs])
         }
       end
 
       def serialize(entity)
         case entity.class.to_s
         when 'Array'
-          JSON.pretty_generate entity.map{ |item| JSON.parse serialize(item) }
-        when ->(x) { defined?(x.constantize) && defined?("Inferno::Web::Serializers::#{x.split('::').last}".constantize) }
+          JSON.pretty_generate entity.map { |item| JSON.parse serialize(item) }
+        when lambda { |x|
+               defined?(x.constantize) && defined?("Inferno::Web::Serializers::#{x.split('::').last}".constantize)
+             }
           "Inferno::Web::Serializers::#{entity.class.to_s.split('::').last}".constantize.render(entity)
         else
           raise StandardError, "CLI does not know how to serialize #{entity.class}"
@@ -162,7 +162,7 @@ def serialize(entity)
       end
 
       def verbose_print(*args)
-        print(COLOR.dim(*args)) if self.options[:verbose]
+        print(COLOR.dim(*args)) if options[:verbose]
       end
 
       def verbose_puts(*args)
@@ -176,7 +176,7 @@ def format_id(result)
 
       def format_messages(result)
         result.messages.map do |message|
-          "\n\t\t" + message.type + ": " + message.message
+          "\n\t\t" + message.type + ': ' + message.message
         end.join('')
       end
 
@@ -201,7 +201,7 @@ def format_outputs(result)
 
       def print_error_and_exit(e, code)
         # TODO: use Application Logger for stderr?
-        $stderr.puts COLOR.red "Error: #{e.full_message}"
+        warn COLOR.red "Error: #{e.full_message}"
         exit(code)
       end
     end
diff --git a/lib/inferno/apps/web/controllers/test_runs/create.rb b/lib/inferno/apps/web/controllers/test_runs/create.rb
index 5913c5742..c72f40a57 100644
--- a/lib/inferno/apps/web/controllers/test_runs/create.rb
+++ b/lib/inferno/apps/web/controllers/test_runs/create.rb
@@ -6,7 +6,6 @@ module Web
     module Controllers
       module TestRuns
         class Create < Controller
-
           include ::Inferno::Utils::VerifyRunnable
           include ::Inferno::Utils::PersistInputs
 
diff --git a/lib/inferno/utils/persist_inputs.rb b/lib/inferno/utils/persist_inputs.rb
index 74177f602..dbfe345c5 100644
--- a/lib/inferno/utils/persist_inputs.rb
+++ b/lib/inferno/utils/persist_inputs.rb
@@ -1,7 +1,6 @@
 module Inferno
   module Utils
     module PersistInputs
-
       def persist_inputs(params, test_run)
         available_inputs = test_run.runnable.available_inputs
         params[:inputs]&.each do |input_params|
@@ -25,7 +24,6 @@ def persist_inputs(params, test_run)
           )
         end
       end
-
     end
   end
 end
diff --git a/lib/inferno/utils/verify_runnable.rb b/lib/inferno/utils/verify_runnable.rb
index ab217e87c..df8622a60 100644
--- a/lib/inferno/utils/verify_runnable.rb
+++ b/lib/inferno/utils/verify_runnable.rb
@@ -3,14 +3,12 @@
 module Inferno
   module Utils
     module VerifyRunnable
-
       def verify_runnable(runnable, inputs, selected_suite_options)
         missing_inputs = runnable&.missing_inputs(inputs, selected_suite_options)
         user_runnable = runnable&.user_runnable?
         raise Inferno::Exceptions::RequiredInputsNotFound, missing_inputs if missing_inputs&.any?
         raise Inferno::Exceptions::NotUserRunnableException unless user_runnable
       end
-
     end
   end
 end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 895deb129..b68a93834 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -1,4 +1,4 @@
-require_relative '../../../../lib/inferno/apps/cli/execute.rb'
+require_relative '../../../../lib/inferno/apps/cli/execute'
 
 ## TODO REFACTOR ALL THESE TESTS WITH FACTORY BOT
 
@@ -6,7 +6,7 @@
   let(:instance) { described_class.new }
 
   describe '#thor_hash_to_inputs_array' do
-    let(:hash) { {url: 'https://example.com'} }
+    let(:hash) { { url: 'https://example.com' } }
 
     it 'converts hash to array' do
       result = instance.thor_hash_to_inputs_array(hash)
@@ -15,23 +15,23 @@
 
     it 'returns proper inputs array' do
       result = instance.thor_hash_to_inputs_array(hash)
-      expect(result).to eq([{name: :url, value: 'https://example.com'}])
+      expect(result).to eq([{ name: :url, value: 'https://example.com' }])
     end
   end
 
   describe '#create_params' do
     let(:test_suite) { BasicTestSuite::Suite }
     let(:test_session) { create(:test_session) }
-    let(:inputs_hash) { {url: 'https://example.com'} }
-    let(:inputs_array) { [{name: :url, value: 'https://example.com'}] }
+    let(:inputs_hash) { { url: 'https://example.com' } }
+    let(:inputs_array) { [{ name: :url, value: 'https://example.com' }] }
 
     it 'returns test run params' do
-      stubbed_instance = instance()
-      allow(stubbed_instance).to receive(:options).and_return({inputs: inputs_hash})
-      test_session_inst = test_session()
+      stubbed_instance = instance
+      allow(stubbed_instance).to receive(:options).and_return({ inputs: inputs_hash })
+      test_session_inst = test_session
 
       result = stubbed_instance.create_params(test_session_inst, test_suite)
-      expect(result).to eq({test_session_id: test_session.id, test_suite_id: test_suite.id, inputs: inputs_array})
+      expect(result).to eq({ test_session_id: test_session.id, test_suite_id: test_suite.id, inputs: inputs_array })
     end
   end
 
@@ -39,36 +39,36 @@
     let(:test_results) { create_list(:result, 2) }
 
     it 'handles an array of test results without raising exception' do
-      expect { instance.serialize(test_results) }.not_to raise_error(StandardError)
+      expect { instance.serialize(test_results) }.to_not raise_error(StandardError)
     end
 
     it 'returns legit JSON' do
-      expect { JSON.parse(instance.serialize(test_results)) }.not_to raise_error(JSON::ParserError)
-      expect { JSON.parse(instance.serialize(test_results)) }.not_to raise_error(JSON::NestingError)
-      expect { JSON.parse(instance.serialize(test_results)) }.not_to raise_error(TypeError)
+      expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(JSON::ParserError)
+      expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(JSON::NestingError)
+      expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(TypeError)
     end
   end
 
   describe '#verbose_print' do
     it 'outputs when verbose is true' do
-      stubbed_instance = instance()
-      allow(stubbed_instance).to receive(:options).and_return({verbose: true})
+      stubbed_instance = instance
+      allow(stubbed_instance).to receive(:options).and_return({ verbose: true })
 
       expect { stubbed_instance.verbose_print('Lorem') }.to output(/Lorem/).to_stdout
     end
 
     it 'does not output when verbose is false' do
-      stubbed_instance = instance()
-      allow(stubbed_instance).to receive(:options).and_return({verbose: false})
+      stubbed_instance = instance
+      allow(stubbed_instance).to receive(:options).and_return({ verbose: false })
 
-      expect { stubbed_instance.verbose_print('Lorem') }.not_to output(/.+/).to_stdout
+      expect { stubbed_instance.verbose_print('Lorem') }.to_not output(/.+/).to_stdout
     end
   end
 
   describe '#verbose_puts' do
     it 'has output ending with \n with when verbose is true' do
-      stubbed_instance = instance()
-      allow(stubbed_instance).to receive(:options).and_return({verbose: true})
+      stubbed_instance = instance
+      allow(stubbed_instance).to receive(:options).and_return({ verbose: true })
 
       expect { stubbed_instance.verbose_puts('Lorem') }.to output(/Lorem\n/).to_stdout
     end
@@ -80,21 +80,21 @@
     let(:test) { test_group.tests.first }
 
     it 'returns suite id if test result belongs to suite' do
-      test_result = create(:result, runnable: {test_suite_id: test_suite.id})
+      test_result = create(:result, runnable: { test_suite_id: test_suite.id })
 
-      expect( instance.format_id(test_result) ).to eq( test_suite.id )
+      expect(instance.format_id(test_result)).to eq(test_suite.id)
     end
 
     it 'returns group id if test result belongs to group' do
-      test_result = create(:result, runnable: {test_group_id: test_group.id})
+      test_result = create(:result, runnable: { test_group_id: test_group.id })
 
-      expect( instance.format_id(test_result) ).to eq( test_group.id )
+      expect(instance.format_id(test_result)).to eq(test_group.id)
     end
 
     it 'returns test id if test result belongs to test' do
-      test_result = create(:result, runnable: {test_id: test.id});
+      test_result = create(:result, runnable: { test_id: test.id })
 
-      expect( instance.format_id(test_result) ).to eq( test.id )
+      expect(instance.format_id(test_result)).to eq(test.id)
     end
   end
 
@@ -126,19 +126,19 @@
   end
 
   describe '#format_inputs' do
-    let(:inputs) {[{name: :url, value: 'https://example.com'}]}
+    let(:inputs) { [{ name: :url, value: 'https://example.com' }] }
     let(:test_result) { create(:result, input_json: JSON.generate(inputs)) }
 
     it 'includes all values' do
       formatted_string = instance.format_inputs(test_result)
       inputs.each do |input_element|
-        expect(formatted_string).to include  input_element[:value]
+        expect(formatted_string).to include input_element[:value]
       end
     end
   end
 
   describe '#format_outputs' do
-    let(:outputs) {[{name: :token, value: 'SAMPLE_OUTPUT'}]}
+    let(:outputs) { [{ name: :token, value: 'SAMPLE_OUTPUT' }] }
     let(:test_result) { create(:result, output_json: JSON.generate(outputs)) }
 
     it 'includes all values' do
diff --git a/spec/inferno/utils/persist_inputs_spec.rb b/spec/inferno/utils/persist_inputs_spec.rb
index 31d8089cb..9c8b3fc36 100644
--- a/spec/inferno/utils/persist_inputs_spec.rb
+++ b/spec/inferno/utils/persist_inputs_spec.rb
@@ -1,12 +1,9 @@
 require_relative '../../../lib/inferno/utils/persist_inputs'
 
 RSpec.describe Inferno::Utils::PersistInputs do
-  
   describe '#persist_inputs' do
     it 'is defined' do
       expect(described_class.method_defined?(:persist_inputs)).to eq(true)
     end
   end
-
 end
-
diff --git a/spec/inferno/utils/verify_runnable_spec.rb b/spec/inferno/utils/verify_runnable_spec.rb
index 68a03c0d4..e201c94f3 100644
--- a/spec/inferno/utils/verify_runnable_spec.rb
+++ b/spec/inferno/utils/verify_runnable_spec.rb
@@ -1,4 +1,4 @@
-require_relative '../../../lib/inferno/utils/verify_runnable.rb'
+require_relative '../../../lib/inferno/utils/verify_runnable'
 
 RSpec.describe Inferno::Utils::VerifyRunnable do
   describe '#verify_runnable' do

From e5670e44e99b7e08d57cc2490edc745eefe53908 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 9 Aug 2024 10:52:20 -0400
Subject: [PATCH 21/83] more cop

---
 lib/inferno/apps/cli/execute.rb | 102 +++++++++++++++++---------------
 lib/inferno/apps/cli/main.rb    |   4 +-
 lib/inferno/jobs.rb             |   2 +-
 3 files changed, 58 insertions(+), 50 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index f03aaaa73..acba3ed97 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -80,46 +80,9 @@ def run(options)
         Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
 
         results = test_runs_repo.results_for_test_run(test_run.id).reverse
-        verbose_puts '=========================================='
-        verbose_puts 'JSON Test Results:'
-        verbose_puts '=========================================='
-        verbose_puts serialize(results)
-        verbose_puts '=========================================='
 
-        puts '=========================================='
-        puts 'Colored Test Results:'
-        puts '=========================================='
-        results.each do |result|
-          print format_id(result), ': '
-          case result.result
-          when 'pass'
-            print COLOR.bold.green(CHECKMARK, ' pass')
-          when 'fail'
-            print COLOR.bold.red 'X fail'
-          when 'skip'
-            print COLOR.yellow '* skip'
-          when 'omit'
-            print COLOR.blue '* omit'
-          when 'error'
-            print COLOR.magenta 'X error'
-          when 'wait'
-            # This may be dead code with synchronous test execution
-            print '. wait'
-          when 'cancel'
-            print COLOR.red 'X cancel'
-          else
-            # TODO: strict behavior or no?
-            # raise StandardError.new, "Unrecognized result #{result.result}" # strict
-            print '- unknown' # unstrict
-          end
-          puts ''
-          verbose_puts "\tsummary: ",  result.result_message
-          verbose_puts "\tmessages: ", format_messages(result)
-          verbose_puts "\trequests: ", format_requests(result)
-          verbose_puts "\tinputs: ",   format_inputs(result)
-          verbose_puts "\toutputs: ",  format_outputs(result)
-        end
-        puts '=========================================='
+        verbose_print_json_results(results)
+        print_color_results(results)
 
         exit(0) if results.find { |result| result.test_suite_id == options[:suite_id] }.result == 'pass'
 
@@ -151,7 +114,7 @@ def create_params(test_session, suite)
       def serialize(entity)
         case entity.class.to_s
         when 'Array'
-          JSON.pretty_generate entity.map { |item| JSON.parse serialize(item) }
+          JSON.pretty_generate(entity.map { |item| JSON.parse serialize(item) })
         when lambda { |x|
                defined?(x.constantize) && defined?("Inferno::Web::Serializers::#{x.split('::').last}".constantize)
              }
@@ -176,14 +139,14 @@ def format_id(result)
 
       def format_messages(result)
         result.messages.map do |message|
-          "\n\t\t" + message.type + ': ' + message.message
-        end.join('')
+          "\n\t\t#{message.type}: #{message.message}"
+        end.join
       end
 
       def format_requests(result)
         result.requests.map do |req_res|
-          "\n\t\t" + req_res.status.to_s + ' ' + req_res.verb.upcase + ' ' + req_res.url
-        end.join('')
+          "\n\t\t#{req_res.status} #{req_res.verb.upcase} #{req_res.url}"
+        end.join
       end
 
       def format_inputs(result, attr = :input_json)
@@ -192,18 +155,63 @@ def format_inputs(result, attr = :input_json)
 
         JSON.parse(input_json).map do |input|
           "\n\t\t#{input['name']}: #{input['value']}"
-        end.join('')
+        end.join
       end
 
       def format_outputs(result)
         format_inputs(result, :output_json)
       end
 
-      def print_error_and_exit(e, code)
+      def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
+        case result.result
+        when 'pass'
+          COLOR.bold.green(CHECKMARK, ' pass')
+        when 'fail'
+          COLOR.bold.red 'X fail'
+        when 'skip'
+          COLOR.yellow '* skip'
+        when 'omit'
+          COLOR.blue '* omit'
+        when 'error'
+          COLOR.magenta 'X error'
+        when 'wait'
+          # This may be dead code with synchronous test execution
+          '. wait'
+        when 'cancel'
+          COLOR.red 'X cancel'
+        else
+          raise StandardError.new, "Unrecognized result #{result.result}"
+        end
+      end
+
+      def print_error_and_exit(err, code)
         # TODO: use Application Logger for stderr?
-        warn COLOR.red "Error: #{e.full_message}"
+        warn COLOR.red "Error: #{err.full_message}"
         exit(code)
       end
+
+      def verbose_print_json_results(results)
+        verbose_puts '=========================================='
+        verbose_puts 'JSON Test Results:'
+        verbose_puts '=========================================='
+        verbose_puts serialize(results)
+        verbose_puts '=========================================='
+      end
+
+      def print_color_results(results)
+        puts '=========================================='
+        puts 'Colored Test Results:'
+        puts '=========================================='
+        results.each do |result|
+          print format_id(result), ': ', format_result(result), "\n"
+          verbose_puts "\tsummary: ",   result.result_message
+          verbose_puts "\tmessages: ",  format_messages(result)
+          verbose_puts "\trequests: ",  format_requests(result)
+          verbose_puts "\tinputs: ",    format_inputs(result)
+          verbose_puts "\toutputs: ",   format_outputs(result)
+        end
+        puts '=========================================='
+      end
     end
   end
 end
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index d682c23cc..4de8254a5 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -66,7 +66,7 @@ def version
         puts "Inferno Core v#{Inferno::VERSION}"
       end
 
-      EXECUTE_HELP = <<~EOT
+      EXECUTE_HELP = <<~END_OF_HELP.freeze
         Run Inferno tests in the command line. Exits with 0 only if test suite passes. Must be run from test kit as working directory.
 
         You must have background services running: `bundle exec inferno services start`
@@ -77,7 +77,7 @@ def version
 
             `bundle exec inferno execute --suite dev_validator --inputs "url:https://hapi.fhir.org/baseR4" patient_id:1234321`
             => Outputs test results
-      EOT
+      END_OF_HELP
       desc 'execute', 'Run Inferno tests in command line'
       long_desc EXECUTE_HELP, wrap: false
       option :suite,
diff --git a/lib/inferno/jobs.rb b/lib/inferno/jobs.rb
index fe564ffdf..6fceb3038 100644
--- a/lib/inferno/jobs.rb
+++ b/lib/inferno/jobs.rb
@@ -7,7 +7,7 @@
 module Inferno
   module Jobs
     def self.perform(job_klass, *params, force_synchronous: false)
-      if force_synchronous || (Application['async_jobs'] === false)
+      if force_synchronous || (Application['async_jobs'] == false)
         job_klass.new.perform(*params)
       else
         job_klass.perform_async(*params)

From 3ee2b50574f58fa53e4f8d173081127027f2d16f Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 9 Aug 2024 11:56:28 -0400
Subject: [PATCH 22/83] verify runnable spec

---
 spec/fixtures/run_as_group_test_group.rb   | 11 +++++++++++
 spec/inferno/utils/verify_runnable_spec.rb | 20 ++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 spec/fixtures/run_as_group_test_group.rb

diff --git a/spec/fixtures/run_as_group_test_group.rb b/spec/fixtures/run_as_group_test_group.rb
new file mode 100644
index 000000000..5a067de1d
--- /dev/null
+++ b/spec/fixtures/run_as_group_test_group.rb
@@ -0,0 +1,11 @@
+module BasicTestSuite
+  class DefGroup < Inferno::Entities::TestGroup
+    title 'DEF Group'
+
+    run_as_group
+
+    test 'this_test_cannot_run_alone' do
+      run { 2 + 2 }
+    end
+  end
+end
diff --git a/spec/inferno/utils/verify_runnable_spec.rb b/spec/inferno/utils/verify_runnable_spec.rb
index e201c94f3..66a8c8d8b 100644
--- a/spec/inferno/utils/verify_runnable_spec.rb
+++ b/spec/inferno/utils/verify_runnable_spec.rb
@@ -2,8 +2,28 @@
 
 RSpec.describe Inferno::Utils::VerifyRunnable do
   describe '#verify_runnable' do
+    let(:dummy_class) { Class.new { include Inferno::Utils::VerifyRunnable } }
+    let(:dummy) { dummy_class.new }
+    let(:suite) { BasicTestSuite::Suite }
+    let(:group) { BasicTestSuite::AbcGroup }
+    let(:good_inputs) { [{name: 'input1', value: 'baz'}, {name: 'input2', value: 'foo'}] }
+    let(:bad_inputs) { [{name: :input2, value: :foo}] }
+    let(:unrunnable) { BasicTestSuite::DefGroup.tests.first }
+
     it 'is defined' do
       expect(described_class.method_defined?(:verify_runnable)).to eq(true)
     end
+
+    it 'allows legit runnables' do
+      expect { dummy.verify_runnable(suite, good_inputs, {}) }.not_to raise_error
+    end
+
+    it 'rejects bad inputs' do
+      expect { dummy.verify_runnable(group, bad_inputs, {}) }.to raise_error(Inferno::Exceptions::RequiredInputsNotFound)
+    end
+
+    it 'rejects tests that are part of run_as_group' do
+      expect { dummy.verify_runnable(unrunnable, [], {}) }.to raise_error(Inferno::Exceptions::NotUserRunnableException)
+    end
   end
 end

From 3bac5e1e110ebd87d13005b4690a62accab587ab Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 9 Aug 2024 17:00:30 -0400
Subject: [PATCH 23/83] fix persist_inputs spec

---
 lib/inferno/apps/cli/execute.rb               |  2 +-
 .../apps/web/controllers/test_runs/create.rb  |  2 +-
 lib/inferno/utils/persist_inputs.rb           |  2 +-
 spec/inferno/utils/persist_inputs_spec.rb     | 37 +++++++++++++++++++
 spec/inferno/utils/verify_runnable_spec.rb    | 10 +++--
 5 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index acba3ed97..bbf2f08a3 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -74,7 +74,7 @@ def run(options)
           create_params(test_session, suite).merge({ status: 'queued' })
         )
 
-        persist_inputs(create_params(test_session, suite), test_run)
+        persist_inputs(session_data_repo, create_params(test_session, suite), test_run)
 
         puts 'Running tests. This may take a while...' # TODO: spinner/progress bar
         Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
diff --git a/lib/inferno/apps/web/controllers/test_runs/create.rb b/lib/inferno/apps/web/controllers/test_runs/create.rb
index c72f40a57..20cdc2853 100644
--- a/lib/inferno/apps/web/controllers/test_runs/create.rb
+++ b/lib/inferno/apps/web/controllers/test_runs/create.rb
@@ -35,7 +35,7 @@ def handle(req, res)
 
             res.body = serialize(test_run, suite_options: test_session.suite_options)
 
-            persist_inputs(req.params, test_run)
+            persist_inputs(session_data_repo, req.params, test_run)
 
             Jobs.perform(Jobs::ExecuteTestRun, test_run.id)
           rescue Sequel::ValidationFailed, Sequel::ForeignKeyConstraintViolation,
diff --git a/lib/inferno/utils/persist_inputs.rb b/lib/inferno/utils/persist_inputs.rb
index dbfe345c5..6fd679a76 100644
--- a/lib/inferno/utils/persist_inputs.rb
+++ b/lib/inferno/utils/persist_inputs.rb
@@ -1,7 +1,7 @@
 module Inferno
   module Utils
     module PersistInputs
-      def persist_inputs(params, test_run)
+      def persist_inputs(session_data_repo, params, test_run)
         available_inputs = test_run.runnable.available_inputs
         params[:inputs]&.each do |input_params|
           input =
diff --git a/spec/inferno/utils/persist_inputs_spec.rb b/spec/inferno/utils/persist_inputs_spec.rb
index 9c8b3fc36..2f0ffe64e 100644
--- a/spec/inferno/utils/persist_inputs_spec.rb
+++ b/spec/inferno/utils/persist_inputs_spec.rb
@@ -2,8 +2,45 @@
 
 RSpec.describe Inferno::Utils::PersistInputs do
   describe '#persist_inputs' do
+    let(:dummy_class) do
+      Class.new do
+        include Inferno::Utils::PersistInputs
+      end
+    end
+    let(:dummy) { dummy_class.new }
+    let(:suite) { BasicTestSuite::Suite }
+    let(:test_sessions_repo) { Inferno::Repositories::TestSessions.new }
+    let(:session_data_repo) { Inferno::Repositories::SessionData.new }
+
     it 'is defined' do
       expect(described_class.method_defined?(:persist_inputs)).to eq(true)
     end
+
+    it 'saves inputs to db' do
+      test_session = test_sessions_repo.create(test_suite_id: suite.id)
+
+      test_run = create(:test_run, test_session:)
+      test_run.test_session_id = test_session.id
+
+      print "\ntest_run: ", test_run
+      print "\ntest_session: ", test_session
+      print "\ntest_run.test_session: ", test_run.test_session
+      print "\ntest_session.id: ", test_session.id
+      print "\ntest_run.test_session_id: ", test_run.test_session_id
+      print "\n"
+
+      params = {
+        test_session_id: test_session.id,
+        test_suite_id: suite.id,
+        inputs: [
+          { name: 'input1', value: 'persist me' }
+        ]
+      }
+
+      dummy.persist_inputs(session_data_repo, params, test_run)
+      persisted_data = session_data_repo.load(test_session_id: test_run.test_session_id, name: 'input1')
+
+      expect(persisted_data).to eq('persist me')
+    end
   end
 end
diff --git a/spec/inferno/utils/verify_runnable_spec.rb b/spec/inferno/utils/verify_runnable_spec.rb
index 66a8c8d8b..2d0595b25 100644
--- a/spec/inferno/utils/verify_runnable_spec.rb
+++ b/spec/inferno/utils/verify_runnable_spec.rb
@@ -6,8 +6,8 @@
     let(:dummy) { dummy_class.new }
     let(:suite) { BasicTestSuite::Suite }
     let(:group) { BasicTestSuite::AbcGroup }
-    let(:good_inputs) { [{name: 'input1', value: 'baz'}, {name: 'input2', value: 'foo'}] }
-    let(:bad_inputs) { [{name: :input2, value: :foo}] }
+    let(:good_inputs) { [{ name: 'input1', value: 'baz' }, { name: 'input2', value: 'foo' }] }
+    let(:bad_inputs) { [{ name: :input2, value: :foo }] }
     let(:unrunnable) { BasicTestSuite::DefGroup.tests.first }
 
     it 'is defined' do
@@ -15,11 +15,13 @@
     end
 
     it 'allows legit runnables' do
-      expect { dummy.verify_runnable(suite, good_inputs, {}) }.not_to raise_error
+      expect { dummy.verify_runnable(suite, good_inputs, {}) }.to_not raise_error
     end
 
     it 'rejects bad inputs' do
-      expect { dummy.verify_runnable(group, bad_inputs, {}) }.to raise_error(Inferno::Exceptions::RequiredInputsNotFound)
+      expect do
+        dummy.verify_runnable(group, bad_inputs, {})
+      end.to raise_error(Inferno::Exceptions::RequiredInputsNotFound)
     end
 
     it 'rejects tests that are part of run_as_group' do

From 8038d406e0ed9ea9d1f1894a0ee98c28384805e7 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 9 Aug 2024 17:05:02 -0400
Subject: [PATCH 24/83] clean

---
 spec/inferno/utils/persist_inputs_spec.rb  | 7 -------
 spec/inferno/utils/verify_runnable_spec.rb | 8 ++++++--
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/spec/inferno/utils/persist_inputs_spec.rb b/spec/inferno/utils/persist_inputs_spec.rb
index 2f0ffe64e..dcaa94b03 100644
--- a/spec/inferno/utils/persist_inputs_spec.rb
+++ b/spec/inferno/utils/persist_inputs_spec.rb
@@ -22,13 +22,6 @@
       test_run = create(:test_run, test_session:)
       test_run.test_session_id = test_session.id
 
-      print "\ntest_run: ", test_run
-      print "\ntest_session: ", test_session
-      print "\ntest_run.test_session: ", test_run.test_session
-      print "\ntest_session.id: ", test_session.id
-      print "\ntest_run.test_session_id: ", test_run.test_session_id
-      print "\n"
-
       params = {
         test_session_id: test_session.id,
         test_suite_id: suite.id,
diff --git a/spec/inferno/utils/verify_runnable_spec.rb b/spec/inferno/utils/verify_runnable_spec.rb
index 2d0595b25..6513003cb 100644
--- a/spec/inferno/utils/verify_runnable_spec.rb
+++ b/spec/inferno/utils/verify_runnable_spec.rb
@@ -6,7 +6,9 @@
     let(:dummy) { dummy_class.new }
     let(:suite) { BasicTestSuite::Suite }
     let(:group) { BasicTestSuite::AbcGroup }
-    let(:good_inputs) { [{ name: 'input1', value: 'baz' }, { name: 'input2', value: 'foo' }] }
+    let(:good_inputs) do
+      [{ name: 'input1', value: 'baz' }, { name: 'input2', value: 'foo' }]
+    end
     let(:bad_inputs) { [{ name: :input2, value: :foo }] }
     let(:unrunnable) { BasicTestSuite::DefGroup.tests.first }
 
@@ -25,7 +27,9 @@
     end
 
     it 'rejects tests that are part of run_as_group' do
-      expect { dummy.verify_runnable(unrunnable, [], {}) }.to raise_error(Inferno::Exceptions::NotUserRunnableException)
+      expect do
+        dummy.verify_runnable(unrunnable, [], {})
+      end.to raise_error(Inferno::Exceptions::NotUserRunnableException)
     end
   end
 end

From d1c5a11027bab1952caaea9ecc56868cea90e394 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 9 Aug 2024 17:44:47 -0400
Subject: [PATCH 25/83] rspec format_result and print_color_results

---
 lib/inferno/apps/cli/execute.rb       |  6 +++--
 spec/factories/result.rb              |  4 ++++
 spec/inferno/apps/cli/execute_spec.rb | 33 +++++++++++++++++++++++++--
 3 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index bbf2f08a3..6596176b9 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -107,7 +107,7 @@ def create_params(test_session, suite)
         {
           test_session_id: test_session.id,
           test_suite_id: suite.id,
-          inputs: thor_hash_to_inputs_array(options[:inputs])
+          inputs: thor_hash_to_inputs_array(self.options[:inputs])
         }
       end
 
@@ -125,7 +125,7 @@ def serialize(entity)
       end
 
       def verbose_print(*args)
-        print(COLOR.dim(*args)) if options[:verbose]
+        print(COLOR.dim(*args)) if self.options[:verbose]
       end
 
       def verbose_puts(*args)
@@ -179,6 +179,8 @@ def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
           '. wait'
         when 'cancel'
           COLOR.red 'X cancel'
+        when 'running'
+          '- running'
         else
           raise StandardError.new, "Unrecognized result #{result.result}"
         end
diff --git a/spec/factories/result.rb b/spec/factories/result.rb
index eae89c94b..8cbd6dd21 100644
--- a/spec/factories/result.rb
+++ b/spec/factories/result.rb
@@ -42,5 +42,9 @@
         repo_create_list(:request, evaluator.request_count, result_id: instance.id)
       )
     end
+
+    factory :random_result do
+      result { Inferno::Entities::Result::RESULT_OPTIONS.sample }
+    end
   end
 end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index b68a93834..83ffe2a64 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -1,7 +1,5 @@
 require_relative '../../../../lib/inferno/apps/cli/execute'
 
-## TODO REFACTOR ALL THESE TESTS WITH FACTORY BOT
-
 RSpec.describe Inferno::CLI::Execute do # rubocop:disable RSpec/FilePath
   let(:instance) { described_class.new }
 
@@ -159,4 +157,35 @@
       end.to raise_error(SystemExit)
     end
   end
+
+  describe '#format_result' do
+    Inferno::Entities::Result::RESULT_OPTIONS.each do |result_option|
+      it "can format #{result_option} result type" do
+        result = create(:result, result: result_option)
+        expect{ instance.format_result(result) }.not_to raise_error
+      end
+
+      it "includes result type in return value" do
+        result = create(:result, result: result_option)
+        expect( instance.format_result(result).upcase ).to include result_option.upcase
+      end
+    end
+
+  end
+
+  describe '#print_color_results' do
+    let(:results) { create_list(:random_result, 10) }
+
+    it "outputs something with 10 random results" do
+      stubbed_instance = instance
+      allow(stubbed_instance).to receive(:options).and_return({ verbose: false })
+      expect{ stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout
+    end
+
+    it "outputs something with verbose true" do
+      stubbed_instance = instance
+      allow(stubbed_instance).to receive(:options).and_return({ verbose: true })
+      expect{ stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout
+    end
+  end
 end

From b3b3ac65220f51b468d7e9b2c6c416a10db4220f Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 9 Aug 2024 17:46:14 -0400
Subject: [PATCH 26/83] cop

---
 lib/inferno/apps/cli/execute.rb       |  4 ++--
 spec/inferno/apps/cli/execute_spec.rb | 15 +++++++--------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 6596176b9..816500f2a 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -107,7 +107,7 @@ def create_params(test_session, suite)
         {
           test_session_id: test_session.id,
           test_suite_id: suite.id,
-          inputs: thor_hash_to_inputs_array(self.options[:inputs])
+          inputs: thor_hash_to_inputs_array(options[:inputs])
         }
       end
 
@@ -125,7 +125,7 @@ def serialize(entity)
       end
 
       def verbose_print(*args)
-        print(COLOR.dim(*args)) if self.options[:verbose]
+        print(COLOR.dim(*args)) if options[:verbose]
       end
 
       def verbose_puts(*args)
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 83ffe2a64..04fb66d9e 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -162,30 +162,29 @@
     Inferno::Entities::Result::RESULT_OPTIONS.each do |result_option|
       it "can format #{result_option} result type" do
         result = create(:result, result: result_option)
-        expect{ instance.format_result(result) }.not_to raise_error
+        expect { instance.format_result(result) }.to_not raise_error
       end
 
-      it "includes result type in return value" do
+      it 'includes result type in return value' do
         result = create(:result, result: result_option)
-        expect( instance.format_result(result).upcase ).to include result_option.upcase
+        expect(instance.format_result(result).upcase).to include result_option.upcase
       end
     end
-
   end
 
   describe '#print_color_results' do
     let(:results) { create_list(:random_result, 10) }
 
-    it "outputs something with 10 random results" do
+    it 'outputs something with 10 random results' do
       stubbed_instance = instance
       allow(stubbed_instance).to receive(:options).and_return({ verbose: false })
-      expect{ stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout
+      expect { stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout
     end
 
-    it "outputs something with verbose true" do
+    it 'outputs something with verbose true' do
       stubbed_instance = instance
       allow(stubbed_instance).to receive(:options).and_return({ verbose: true })
-      expect{ stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout
+      expect { stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout
     end
   end
 end

From 2d6ae5a3b6f109165fc7d74395aeee63e886e9f4 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 9 Aug 2024 18:11:35 -0400
Subject: [PATCH 27/83] increase codecov

---
 spec/inferno/utils/persist_inputs_spec.rb | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/spec/inferno/utils/persist_inputs_spec.rb b/spec/inferno/utils/persist_inputs_spec.rb
index dcaa94b03..0cae96b0b 100644
--- a/spec/inferno/utils/persist_inputs_spec.rb
+++ b/spec/inferno/utils/persist_inputs_spec.rb
@@ -35,5 +35,26 @@
 
       expect(persisted_data).to eq('persist me')
     end
+
+    it 'can handle unregistered inputs' do
+      test_session = test_sessions_repo.create(test_suite_id: suite.id)
+
+      test_run = create(:test_run, test_session:)
+      test_run.test_session_id = test_session.id
+
+      params = {
+        test_session_id: test_session.id,
+        test_suite_id: suite.id,
+        inputs: [
+          { name: 'unregistered', value: 'omit me' },
+          { name: 'input1', value: 'persist me' }
+        ]
+      }
+
+      expect { dummy.persist_inputs(session_data_repo, params, test_run) }.to_not raise_error
+
+      persisted_data = session_data_repo.load(test_session_id: test_run.test_session_id, name: 'input1')
+      expect(persisted_data).to eq('persist me')
+    end
   end
 end

From 845f43bbf0bc6f4fabfad2a52950403a895bfcb4 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 9 Aug 2024 19:23:09 -0400
Subject: [PATCH 28/83] add execute run test

---
 lib/inferno/apps/cli/execute.rb       |  2 +-
 spec/inferno/apps/cli/execute_spec.rb | 34 +++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 816500f2a..2081d9793 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -188,7 +188,7 @@ def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
 
       def print_error_and_exit(err, code)
         # TODO: use Application Logger for stderr?
-        warn COLOR.red "Error: #{err.full_message}"
+        $stderr.puts COLOR.red "Error: #{err.full_message}"
         exit(code)
       end
 
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 04fb66d9e..7ba0b83eb 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -187,4 +187,38 @@
       expect { stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout
     end
   end
+
+  describe '#run' do
+    let(:suite) { 'dev_validator' }
+    let(:session_data_repo) { Inferno::Repositories::SessionData.new }
+    let(:test_session) { repo_create(:test_session, test_suite_id: suite.id) }
+    #let(:url) { 'http://example.com/fhir' }
+    #let(:group) { suite.groups.first.groups.first }
+
+    let(:success_outcome) do
+    {
+      outcomes: [{
+        issues: []
+      }],
+      sessionId: ''
+    }
+    end
+
+    let(:inputs) {{ 'url' => 'https://example.com', 'patient_id' => '1' }}
+
+    it 'works on dev_validator suite' do
+      stub_request(:post, "#{ENV.fetch('FHIR_RESOURCE_VALIDATOR_URL')}/validate")
+        .with(query: hash_including({}))
+        .to_return(status: 200, body: success_outcome.to_json)
+
+      stub_request(:get, "https://example.com/Patient/1")
+        .to_return(status: 200, body: FHIR::Patient.new({name: {given: 'Smith'}}).to_json)
+
+      expect do
+        expect { instance.run({ suite:, inputs:, verbose: true }) }
+          .to raise_error(an_instance_of(SystemExit).and having_attributes(status: 0))
+      end.to output(/.+/).to_stdout
+    end
+
+  end
 end

From 726f1dd49cd55e6a45166f72a744aa37d2bde70d Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 9 Aug 2024 19:28:36 -0400
Subject: [PATCH 29/83] debug

---
 lib/inferno/apps/cli/execute.rb       |  2 +-
 spec/inferno/apps/cli/execute_spec.rb | 23 ++++++++++-------------
 2 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 2081d9793..bac71e36c 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -188,7 +188,7 @@ def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
 
       def print_error_and_exit(err, code)
         # TODO: use Application Logger for stderr?
-        $stderr.puts COLOR.red "Error: #{err.full_message}"
+        $stderr.puts COLOR.red "Error: #{err.full_message}" # rubocop:disable Style/StderrPuts # always print this error instead of using `warn`
         exit(code)
       end
 
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 7ba0b83eb..b0409a150 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -192,33 +192,30 @@
     let(:suite) { 'dev_validator' }
     let(:session_data_repo) { Inferno::Repositories::SessionData.new }
     let(:test_session) { repo_create(:test_session, test_suite_id: suite.id) }
-    #let(:url) { 'http://example.com/fhir' }
-    #let(:group) { suite.groups.first.groups.first }
 
     let(:success_outcome) do
-    {
-      outcomes: [{
-        issues: []
-      }],
-      sessionId: ''
-    }
+      {
+        outcomes: [{
+          issues: []
+        }],
+        sessionId: ''
+      }
     end
 
-    let(:inputs) {{ 'url' => 'https://example.com', 'patient_id' => '1' }}
+    let(:inputs) { { 'url' => 'https://example.com', 'patient_id' => '1' } }
 
     it 'works on dev_validator suite' do
       stub_request(:post, "#{ENV.fetch('FHIR_RESOURCE_VALIDATOR_URL')}/validate")
         .with(query: hash_including({}))
         .to_return(status: 200, body: success_outcome.to_json)
 
-      stub_request(:get, "https://example.com/Patient/1")
-        .to_return(status: 200, body: FHIR::Patient.new({name: {given: 'Smith'}}).to_json)
+      stub_request(:get, 'https://example.com/Patient/1')
+        .to_return(status: 200, body: FHIR::Patient.new({ name: { given: 'Smith' } }).to_json)
 
       expect do
         expect { instance.run({ suite:, inputs:, verbose: true }) }
-          .to raise_error(an_instance_of(SystemExit).and having_attributes(status: 0))
+          .to raise_error(an_instance_of(SystemExit).and(having_attributes(status: 0)))
       end.to output(/.+/).to_stdout
     end
-
   end
 end

From 5914d3155bcf49f9c73faeba4dcfc8d83eb4fce4 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 13 Aug 2024 16:09:34 -0400
Subject: [PATCH 30/83] add --help to inferno execute; adjust exit codes

---
 lib/inferno/apps/cli/execute.rb | 19 +++++++++++++------
 lib/inferno/apps/cli/main.rb    |  7 +++++--
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index bac71e36c..9f2edde7a 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -49,6 +49,11 @@ def self.suppress_output
       attr_accessor :options
 
       def run(options)
+        if options[:help]
+          puts `bundle exec inferno help execute`
+          exit(3)
+        end
+
         puts ''
         puts '=========================================='
         puts "Testing #{options[:suite]} Suite"
@@ -86,17 +91,19 @@ def run(options)
 
         exit(0) if results.find { |result| result.test_suite_id == options[:suite_id] }.result == 'pass'
 
-        exit(1)
+        # exit(1) is for Thor failures
+        # exit(2) is for shell builtin failures
+        exit(3)
       rescue Sequel::ValidationFailed => e
-        print_error_and_exit(e, 3)
-      rescue Sequel::ForeignKeyConstraintViolation => e
         print_error_and_exit(e, 4)
-      rescue Inferno::Exceptions::RequiredInputsNotFound => e
+      rescue Sequel::ForeignKeyConstraintViolation => e
         print_error_and_exit(e, 5)
-      rescue Inferno::Exceptions::NotUserRunnableException => e
+      rescue Inferno::Exceptions::RequiredInputsNotFound => e
         print_error_and_exit(e, 6)
+      rescue Inferno::Exceptions::NotUserRunnableException => e
+        print_error_and_exit(e, 7)
       rescue StandardError => e
-        print_error_and_exit(e, 2)
+        print_error_and_exit(e, 8)
       end
 
       def thor_hash_to_inputs_array(hash)
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index 4de8254a5..d30cee366 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -82,13 +82,11 @@ def version
       long_desc EXECUTE_HELP, wrap: false
       option :suite,
              aliases: ['-s'],
-             required: true,
              type: :string,
              desc: 'Test Suite ID to run',
              banner: 'ID'
       option :inputs,
              aliases: ['-i'],
-             optional: true,
              type: :hash,
              desc: 'Inputs (i.e: --inputs=foo:bar goo:baz)'
       option :verbose,
@@ -96,6 +94,11 @@ def version
              type: :boolean,
              default: false,
              desc: 'Output additional information for debugging'
+      option :help,
+             aliases: ['-h'],
+             type: :boolean,
+             default: false,
+             desc: 'Display this message'
       def execute
         Execute.new.run(options)
       end

From 25ee150a6bf392df2b595aa56f3d5b1d1f4dfd72 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 13 Aug 2024 16:24:22 -0400
Subject: [PATCH 31/83] suppress fhir client logger output

---
 lib/inferno/apps/cli/execute.rb | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 9f2edde7a..697824f76 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -35,8 +35,6 @@ def self.suppress_output
       # I would be allow this in verbose mode but definitely not for JSON output
       suppress_output { require_relative '../../../inferno' }
 
-      # TODO: hijack logger and suppress or redirect its output
-
       COLOR = Pastel.new
       CHECKMARK = "\u2713"
 
@@ -82,7 +80,15 @@ def run(options)
         persist_inputs(session_data_repo, create_params(test_session, suite), test_run)
 
         puts 'Running tests. This may take a while...' # TODO: spinner/progress bar
-        Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+
+        # TODO: hijack logger instead of using this if-case
+        if options[:verbose]
+          Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+        else
+          Inferno::CLI::Execute.suppress_output do
+            Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+          end
+        end
 
         results = test_runs_repo.results_for_test_run(test_run.id).reverse
 

From a0d2bcbcdf9d7cc1decf5e896fa1a480831ce044 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 14 Aug 2024 11:42:49 -0400
Subject: [PATCH 32/83] impl suite options

---
 lib/inferno/apps/cli/execute.rb | 11 +++++++++--
 lib/inferno/apps/cli/main.rb    |  6 +++++-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 697824f76..2094fa831 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -65,7 +65,10 @@ def run(options)
         suite = Inferno::Repositories::TestSuites.new.find(options[:suite])
         raise StandardError, "Suite #{options[:suite]} not found" if suite.nil?
 
-        test_session = test_sessions_repo.create({ test_suite_id: suite.id }) # TODO: add suite options
+        test_session = test_sessions_repo.create({
+          test_suite_id: suite.id,
+          suite_options: thor_hash_to_suite_options_array(options[:suite_options])
+        })
 
         verify_runnable(
           test_runs_repo.build_entity(create_params(test_session, suite)).runnable,
@@ -112,7 +115,11 @@ def run(options)
         print_error_and_exit(e, 8)
       end
 
-      def thor_hash_to_inputs_array(hash)
+      def thor_hash_to_suite_options_array(hash = {})
+        hash.to_a.map { |pair| Inferno::DSL::SuiteOption.new({id: pair[0], value: pair[1]}) }
+      end
+
+      def thor_hash_to_inputs_array(hash = {})
         hash.to_a.map { |pair| { name: pair[0], value: pair[1] } }
       end
 
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index d30cee366..91a7b8168 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -84,7 +84,11 @@ def version
              aliases: ['-s'],
              type: :string,
              desc: 'Test Suite ID to run',
-             banner: 'ID'
+             banner: 'id'
+      option :suite_options,
+             aliases: ['-o'],
+             type: :hash,
+             desc: 'Suite options'
       option :inputs,
              aliases: ['-i'],
              type: :hash,

From bcc109bd0bbd131dba829464884ea3e894302514 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 14 Aug 2024 15:13:21 -0400
Subject: [PATCH 33/83] impl group/test execution

---
 lib/inferno/apps/cli/execute.rb | 48 ++++++++++++++++++++++++++-------
 lib/inferno/apps/cli/main.rb    | 10 ++++++-
 2 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 2094fa831..9ccbee00d 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -45,6 +45,7 @@ def self.suppress_output
               ]
 
       attr_accessor :options
+      attr_accessor :runnable_type
 
       def run(options)
         if options[:help]
@@ -54,7 +55,7 @@ def run(options)
 
         puts ''
         puts '=========================================='
-        puts "Testing #{options[:suite]} Suite"
+        puts "Testing #{options[:suite] || options[:group] || options[:test]}"
         puts '=========================================='
 
         self.options = options
@@ -62,25 +63,39 @@ def run(options)
 
         Inferno::Application.start(:suites)
 
-        suite = Inferno::Repositories::TestSuites.new.find(options[:suite])
-        raise StandardError, "Suite #{options[:suite]} not found" if suite.nil?
+        if options[:suite]
+          self.runnable_type = 'suite'
+          runnable = Inferno::Repositories::TestSuites.new.find(options[:suite])
+          raise StandardError, "Suite #{options[:suite]} not found" if runnable.nil?
+        elsif options[:group]
+          self.runnable_type = 'group'
+          runnable = Inferno::Repositories::TestGroups.new.find(options[:group])
+          raise StandardError, "Group #{options[:group]} not found" if runnable.nil?
+        elsif options[:test]
+          self.runnable_type = 'test'
+          runnable = Inferno::Repositories::Tests.new.find(options[:test])
+          raise StandardError, "Group #{options[:test]} not found" if runnable.nil?
+        else
+          raise StandardError, "No suite or group id provided"
+        end
 
         test_session = test_sessions_repo.create({
-          test_suite_id: suite.id,
+          test_suite_id: runnable.suite.id,
           suite_options: thor_hash_to_suite_options_array(options[:suite_options])
         })
 
         verify_runnable(
-          test_runs_repo.build_entity(create_params(test_session, suite)).runnable,
+          # test_runs_repo.build_entity(create_params(test_session, suite)).runnable,
+          runnable,
           thor_hash_to_inputs_array(options[:inputs]),
           test_session.suite_options
         )
 
         test_run = test_runs_repo.create(
-          create_params(test_session, suite).merge({ status: 'queued' })
+          create_params(test_session, runnable).merge({ status: 'queued' })
         )
 
-        persist_inputs(session_data_repo, create_params(test_session, suite), test_run)
+        persist_inputs(session_data_repo, create_params(test_session, runnable), test_run)
 
         puts 'Running tests. This may take a while...' # TODO: spinner/progress bar
 
@@ -98,7 +113,7 @@ def run(options)
         verbose_print_json_results(results)
         print_color_results(results)
 
-        exit(0) if results.find { |result| result.test_suite_id == options[:suite_id] }.result == 'pass'
+        exit(0) if results.find { |result| result.send(runnable_id_key) == options[runnable_type.to_sym] }.result == 'pass'
 
         # exit(1) is for Thor failures
         # exit(2) is for shell builtin failures
@@ -115,6 +130,19 @@ def run(options)
         print_error_and_exit(e, 8)
       end
 
+      def runnable_id_key
+        case self.runnable_type
+        when 'suite'
+          :test_suite_id
+        when 'group'
+          :test_group_id
+        when 'test'
+          :test_id
+        else
+          raise StandardError, "Unrecognized runnable type #{self.runnable_type}"
+        end
+      end
+
       def thor_hash_to_suite_options_array(hash = {})
         hash.to_a.map { |pair| Inferno::DSL::SuiteOption.new({id: pair[0], value: pair[1]}) }
       end
@@ -123,10 +151,10 @@ def thor_hash_to_inputs_array(hash = {})
         hash.to_a.map { |pair| { name: pair[0], value: pair[1] } }
       end
 
-      def create_params(test_session, suite)
+      def create_params(test_session, runnable)
         {
           test_session_id: test_session.id,
-          test_suite_id: suite.id,
+          runnable_id_key => runnable.id,
           inputs: thor_hash_to_inputs_array(options[:inputs])
         }
       end
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index 91a7b8168..b1c8f2ba0 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -83,12 +83,20 @@ def version
       option :suite,
              aliases: ['-s'],
              type: :string,
-             desc: 'Test Suite ID to run',
+             desc: 'Test suite id to run, mutually exclusive with --group and --test',
              banner: 'id'
       option :suite_options,
              aliases: ['-o'],
              type: :hash,
              desc: 'Suite options'
+      option :group,
+             aliases: ['-g'],
+             type: :string,
+             desc: 'Test group id to run, mutually exclusive with --suite and --test' # TODO make them work together
+      option :test,
+             aliases: ['-t'],
+             type: :string,
+             desc: 'Test id to run, mutually exclusive with --suite and --group'
       option :inputs,
              aliases: ['-i'],
              type: :hash,

From d0e3b4c89da5eee86e203ae48fdbbab23e1f1e9f Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 15 Aug 2024 10:01:42 -0400
Subject: [PATCH 34/83] factoring

---
 lib/inferno/apps/cli/execute.rb       | 60 ++++++++++++++-------------
 spec/inferno/apps/cli/execute_spec.rb |  1 +
 2 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 9ccbee00d..6c5e922a3 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -32,7 +32,6 @@ def self.suppress_output
       end
 
       # Inferno boot flow triggers migration and logger outputs it
-      # I would be allow this in verbose mode but definitely not for JSON output
       suppress_output { require_relative '../../../inferno' }
 
       COLOR = Pastel.new
@@ -45,13 +44,11 @@ def self.suppress_output
               ]
 
       attr_accessor :options
+      attr_accessor :runnable
       attr_accessor :runnable_type
 
       def run(options)
-        if options[:help]
-          puts `bundle exec inferno help execute`
-          exit(3)
-        end
+        print_help_and_exit if options[:help]
 
         puts ''
         puts '=========================================='
@@ -63,21 +60,7 @@ def run(options)
 
         Inferno::Application.start(:suites)
 
-        if options[:suite]
-          self.runnable_type = 'suite'
-          runnable = Inferno::Repositories::TestSuites.new.find(options[:suite])
-          raise StandardError, "Suite #{options[:suite]} not found" if runnable.nil?
-        elsif options[:group]
-          self.runnable_type = 'group'
-          runnable = Inferno::Repositories::TestGroups.new.find(options[:group])
-          raise StandardError, "Group #{options[:group]} not found" if runnable.nil?
-        elsif options[:test]
-          self.runnable_type = 'test'
-          runnable = Inferno::Repositories::Tests.new.find(options[:test])
-          raise StandardError, "Group #{options[:test]} not found" if runnable.nil?
-        else
-          raise StandardError, "No suite or group id provided"
-        end
+        set_runnable!
 
         test_session = test_sessions_repo.create({
           test_suite_id: runnable.suite.id,
@@ -85,7 +68,6 @@ def run(options)
         })
 
         verify_runnable(
-          # test_runs_repo.build_entity(create_params(test_session, suite)).runnable,
           runnable,
           thor_hash_to_inputs_array(options[:inputs]),
           test_session.suite_options
@@ -130,13 +112,36 @@ def run(options)
         print_error_and_exit(e, 8)
       end
 
+      def print_help_and_exit
+        puts `bundle exec inferno help execute`
+        exit(3)
+      end
+
+      def set_runnable!
+        if self.options[:suite]
+          self.runnable_type = 'suite'
+          self.runnable = Inferno::Repositories::TestSuites.new.find(self.options[:suite])
+          raise StandardError, "Suite #{self.options[:suite]} not found" if self.runnable.nil?
+        elsif self.options[:group]
+          self.runnable_type = 'group'
+          self.runnable = Inferno::Repositories::TestGroups.new.find(self.options[:group])
+          raise StandardError, "Group #{self.options[:group]} not found" if self.runnable.nil?
+        elsif self.options[:test]
+          self.runnable_type = 'test'
+          self.runnable = Inferno::Repositories::Tests.new.find(self.options[:test])
+          raise StandardError, "Group #{self.options[:test]} not found" if self.runnable.nil?
+        else
+          raise StandardError, "No suite or group id provided"
+        end
+      end
+
       def runnable_id_key
-        case self.runnable_type
-        when 'suite'
+        case self.runnable_type&.to_sym
+        when :suite
           :test_suite_id
-        when 'group'
+        when :group
           :test_group_id
-        when 'test'
+        when :test
           :test_id
         else
           raise StandardError, "Unrecognized runnable type #{self.runnable_type}"
@@ -223,12 +228,11 @@ def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
         when 'error'
           COLOR.magenta 'X error'
         when 'wait'
-          # This may be dead code with synchronous test execution
-          '. wait'
+          COLOR.bold '. wait'
         when 'cancel'
           COLOR.red 'X cancel'
         when 'running'
-          '- running'
+          COLOR.bold '- running'
         else
           raise StandardError.new, "Unrecognized result #{result.result}"
         end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index b0409a150..b690eba61 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -26,6 +26,7 @@
     it 'returns test run params' do
       stubbed_instance = instance
       allow(stubbed_instance).to receive(:options).and_return({ inputs: inputs_hash })
+      allow(stubbed_instance).to receive(:runnable_type).and_return('suite')
       test_session_inst = test_session
 
       result = stubbed_instance.create_params(test_session_inst, test_suite)

From e361b8d1c89258e9a393ab125d5576a4a52e1c1e Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 15 Aug 2024 13:23:00 -0400
Subject: [PATCH 35/83] debugging rspec

---
 lib/inferno/apps/cli/execute.rb       | 14 +++---
 spec/fixtures/basic_test_group.rb     |  3 +-
 spec/inferno/apps/cli/execute_spec.rb | 70 ++++++++++++++++++++++-----
 3 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 6c5e922a3..0199f8228 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -129,7 +129,7 @@ def set_runnable!
         elsif self.options[:test]
           self.runnable_type = 'test'
           self.runnable = Inferno::Repositories::Tests.new.find(self.options[:test])
-          raise StandardError, "Group #{self.options[:test]} not found" if self.runnable.nil?
+          raise StandardError, "Test #{self.options[:test]} not found" if self.runnable.nil?
         else
           raise StandardError, "No suite or group id provided"
         end
@@ -238,12 +238,6 @@ def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
         end
       end
 
-      def print_error_and_exit(err, code)
-        # TODO: use Application Logger for stderr?
-        $stderr.puts COLOR.red "Error: #{err.full_message}" # rubocop:disable Style/StderrPuts # always print this error instead of using `warn`
-        exit(code)
-      end
-
       def verbose_print_json_results(results)
         verbose_puts '=========================================='
         verbose_puts 'JSON Test Results:'
@@ -266,6 +260,12 @@ def print_color_results(results)
         end
         puts '=========================================='
       end
+
+      def print_error_and_exit(err, code)
+        # TODO: use Application Logger for stderr?
+        $stderr.puts COLOR.red "Error: #{err.full_message}" # rubocop:disable Style/StderrPuts # always print this error instead of using `warn`
+        exit(code)
+      end
     end
   end
 end
diff --git a/spec/fixtures/basic_test_group.rb b/spec/fixtures/basic_test_group.rb
index 5328b512d..85a4dd691 100644
--- a/spec/fixtures/basic_test_group.rb
+++ b/spec/fixtures/basic_test_group.rb
@@ -4,7 +4,8 @@ class AbcGroup < Inferno::Entities::TestGroup
 
     input :input1, :input2
     test 'demo_test' do
-      1 + 1
+      id :demo_test
+      run { 1 + 1 }
     end
   end
 end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index b690eba61..39008c342 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -3,6 +3,52 @@
 RSpec.describe Inferno::CLI::Execute do # rubocop:disable RSpec/FilePath
   let(:instance) { described_class.new }
 
+  # TODO: test print_help_and_exit, set_runnable, runnable_id_key, thor_hash_to_suite_options_array
+  describe '#print_help_and_exit' do
+    it 'outputs something and exits' do
+      expect do
+        expect { instance.print_help_and_exit }.to output(/.+/).to_stdout
+      end.to raise_error(SystemExit)
+    end
+  end
+
+  describe '#set_runnable!' do
+    [{suite: 'basic'}, {group: 'BasicTestSuite::AbcGroup'}, {test: 'BasicTestSuite::AbcGroup-demo_test'}].each do |given_options|
+      context "with #{given_options.keys.first} option" do
+        it 'does not raise error' do
+          stubbed_instance = instance
+          allow(stubbed_instance).to receive(:options).and_return(given_options)
+
+          expect { stubbed_instance.set_runnable! }.not_to raise_error(StandardError)
+        end
+
+        it 'sets runnable' do
+          stubbed_instance = instance
+          allow(stubbed_instance).to receive(:options).and_return(given_options)
+
+          stubbed_instance.set_runnable!
+          klass = case given_options.keys.first
+                  when :suite
+                    Inferno::TestSuite
+                  when :group
+                    Inferno::TestGroup
+                  else
+                    Inferno::Test
+                  end
+          expect(stubbed_instance.runnable).to be < klass
+        end
+
+        it 'sets runnable_type' do
+          stubbed_instance = instance
+          allow(stubbed_instance).to receive(:options).and_return(given_options)
+
+          stubbed_instance.set_runnable!
+          expect(stubbed_instance.runnable_type).to eq(given_options.keys.first.to_s)
+        end
+      end
+    end
+  end
+
   describe '#thor_hash_to_inputs_array' do
     let(:hash) { { url: 'https://example.com' } }
 
@@ -23,6 +69,7 @@
     let(:inputs_hash) { { url: 'https://example.com' } }
     let(:inputs_array) { [{ name: :url, value: 'https://example.com' }] }
 
+    # TODO test all cases [{suite: }]
     it 'returns test run params' do
       stubbed_instance = instance
       allow(stubbed_instance).to receive(:options).and_return({ inputs: inputs_hash })
@@ -112,7 +159,6 @@
 
   describe '#format_requests' do
     let(:test_result) { repo_create(:result, request_count: 10) }
-    let(:instance) { described_class.new }
 
     it 'includes all status codes' do
       requests = test_result.requests
@@ -148,17 +194,6 @@
     end
   end
 
-  describe '#print_error_and_exit' do
-    let(:mock_error_class) { Class.new(StandardError) }
-    let(:mock_error) { mock_error_class.new('mock message') }
-
-    it 'outputs to stderr and exits' do
-      expect do
-        expect { instance.print_error_and_exit(mock_error, 2) }.to output(/Error/).to_stderr
-      end.to raise_error(SystemExit)
-    end
-  end
-
   describe '#format_result' do
     Inferno::Entities::Result::RESULT_OPTIONS.each do |result_option|
       it "can format #{result_option} result type" do
@@ -189,6 +224,17 @@
     end
   end
 
+  describe '#print_error_and_exit' do
+    let(:mock_error_class) { Class.new(StandardError) }
+    let(:mock_error) { mock_error_class.new('mock message') }
+
+    it 'outputs to stderr and exits' do
+      expect do
+        expect { instance.print_error_and_exit(mock_error, 2) }.to output(/Error/).to_stderr
+      end.to raise_error(SystemExit)
+    end
+  end
+
   describe '#run' do
     let(:suite) { 'dev_validator' }
     let(:session_data_repo) { Inferno::Repositories::SessionData.new }

From 51f6cf227f1b610d7e1badb06a498919fe9d9565 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 15 Aug 2024 14:10:22 -0400
Subject: [PATCH 36/83] rubocopping and cleaning

---
 lib/inferno/apps/cli/execute.rb       | 68 +++++++++++++++------------
 lib/inferno/apps/cli/main.rb          |  2 +-
 spec/inferno/apps/cli/execute_spec.rb |  7 +--
 3 files changed, 42 insertions(+), 35 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 0199f8228..96a568091 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -36,6 +36,7 @@ def self.suppress_output
 
       COLOR = Pastel.new
       CHECKMARK = "\u2713"
+      BAR = '=========================================='
 
       include Import[
                 test_sessions_repo: 'inferno.repositories.test_sessions',
@@ -43,19 +44,13 @@ def self.suppress_output
                 test_runs_repo: 'inferno.repositories.test_runs'
               ]
 
-      attr_accessor :options
-      attr_accessor :runnable
-      attr_accessor :runnable_type
+      attr_accessor :options, :runnable, :runnable_type
 
       def run(options)
         print_help_and_exit if options[:help]
 
-        puts ''
-        puts '=========================================='
-        puts "Testing #{options[:suite] || options[:group] || options[:test]}"
-        puts '=========================================='
-
         self.options = options
+        print_start_message
         verbose_puts 'options:', self.options
 
         Inferno::Application.start(:suites)
@@ -63,9 +58,11 @@ def run(options)
         set_runnable!
 
         test_session = test_sessions_repo.create({
-          test_suite_id: runnable.suite.id,
-          suite_options: thor_hash_to_suite_options_array(options[:suite_options])
-        })
+                                                   test_suite_id: runnable.suite.id,
+                                                   suite_options: thor_hash_to_suite_options_array(
+                                                     options[:suite_options]
+                                                   )
+                                                 })
 
         verify_runnable(
           runnable,
@@ -95,7 +92,9 @@ def run(options)
         verbose_print_json_results(results)
         print_color_results(results)
 
-        exit(0) if results.find { |result| result.send(runnable_id_key) == options[runnable_type.to_sym] }.result == 'pass'
+        exit(0) if results.find do |result|
+                     result.send(runnable_id_key) == options[runnable_type.to_sym]
+                   end.result == 'pass'
 
         # exit(1) is for Thor failures
         # exit(2) is for shell builtin failures
@@ -117,26 +116,33 @@ def print_help_and_exit
         exit(3)
       end
 
+      def print_start_message
+        puts ''
+        puts BAR
+        puts "Testing #{options[:suite] || options[:group] || options[:test]}"
+        puts BAR
+      end
+
       def set_runnable!
-        if self.options[:suite]
+        if options[:suite]
           self.runnable_type = 'suite'
-          self.runnable = Inferno::Repositories::TestSuites.new.find(self.options[:suite])
-          raise StandardError, "Suite #{self.options[:suite]} not found" if self.runnable.nil?
-        elsif self.options[:group]
+          self.runnable = Inferno::Repositories::TestSuites.new.find(options[:suite])
+          raise StandardError, "Suite #{options[:suite]} not found" if runnable.nil?
+        elsif options[:group]
           self.runnable_type = 'group'
-          self.runnable = Inferno::Repositories::TestGroups.new.find(self.options[:group])
-          raise StandardError, "Group #{self.options[:group]} not found" if self.runnable.nil?
-        elsif self.options[:test]
+          self.runnable = Inferno::Repositories::TestGroups.new.find(options[:group])
+          raise StandardError, "Group #{options[:group]} not found" if runnable.nil?
+        elsif options[:test]
           self.runnable_type = 'test'
-          self.runnable = Inferno::Repositories::Tests.new.find(self.options[:test])
-          raise StandardError, "Test #{self.options[:test]} not found" if self.runnable.nil?
+          self.runnable = Inferno::Repositories::Tests.new.find(options[:test])
+          raise StandardError, "Test #{options[:test]} not found" if runnable.nil?
         else
-          raise StandardError, "No suite or group id provided"
+          raise StandardError, 'No suite or group id provided'
         end
       end
 
       def runnable_id_key
-        case self.runnable_type&.to_sym
+        case runnable_type&.to_sym
         when :suite
           :test_suite_id
         when :group
@@ -144,12 +150,12 @@ def runnable_id_key
         when :test
           :test_id
         else
-          raise StandardError, "Unrecognized runnable type #{self.runnable_type}"
+          raise StandardError, "Unrecognized runnable type #{runnable_type}"
         end
       end
 
       def thor_hash_to_suite_options_array(hash = {})
-        hash.to_a.map { |pair| Inferno::DSL::SuiteOption.new({id: pair[0], value: pair[1]}) }
+        hash.to_a.map { |pair| Inferno::DSL::SuiteOption.new({ id: pair[0], value: pair[1] }) }
       end
 
       def thor_hash_to_inputs_array(hash = {})
@@ -239,17 +245,17 @@ def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
       end
 
       def verbose_print_json_results(results)
-        verbose_puts '=========================================='
+        verbose_puts BAR
         verbose_puts 'JSON Test Results:'
-        verbose_puts '=========================================='
+        verbose_puts BAR
         verbose_puts serialize(results)
-        verbose_puts '=========================================='
+        verbose_puts BAR
       end
 
       def print_color_results(results)
-        puts '=========================================='
+        puts BAR
         puts 'Colored Test Results:'
-        puts '=========================================='
+        puts BAR
         results.each do |result|
           print format_id(result), ': ', format_result(result), "\n"
           verbose_puts "\tsummary: ",   result.result_message
@@ -258,7 +264,7 @@ def print_color_results(results)
           verbose_puts "\tinputs: ",    format_inputs(result)
           verbose_puts "\toutputs: ",   format_outputs(result)
         end
-        puts '=========================================='
+        puts BAR
       end
 
       def print_error_and_exit(err, code)
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index b1c8f2ba0..aecc72088 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -92,7 +92,7 @@ def version
       option :group,
              aliases: ['-g'],
              type: :string,
-             desc: 'Test group id to run, mutually exclusive with --suite and --test' # TODO make them work together
+             desc: 'Test group id to run, mutually exclusive with --suite and --test' # TODO: make them work together
       option :test,
              aliases: ['-t'],
              type: :string,
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 39008c342..615be846a 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -13,13 +13,14 @@
   end
 
   describe '#set_runnable!' do
-    [{suite: 'basic'}, {group: 'BasicTestSuite::AbcGroup'}, {test: 'BasicTestSuite::AbcGroup-demo_test'}].each do |given_options|
+    [{ suite: 'basic' }, { group: 'BasicTestSuite::AbcGroup' },
+     { test: 'BasicTestSuite::AbcGroup-demo_test' }].each do |given_options|
       context "with #{given_options.keys.first} option" do
         it 'does not raise error' do
           stubbed_instance = instance
           allow(stubbed_instance).to receive(:options).and_return(given_options)
 
-          expect { stubbed_instance.set_runnable! }.not_to raise_error(StandardError)
+          expect { stubbed_instance.set_runnable! }.to_not raise_error(StandardError)
         end
 
         it 'sets runnable' do
@@ -69,7 +70,7 @@
     let(:inputs_hash) { { url: 'https://example.com' } }
     let(:inputs_array) { [{ name: :url, value: 'https://example.com' }] }
 
-    # TODO test all cases [{suite: }]
+    # TODO: test all cases [{suite: }]
     it 'returns test run params' do
       stubbed_instance = instance
       allow(stubbed_instance).to receive(:options).and_return({ inputs: inputs_hash })

From 88c2412fe6854afca9d3117dcbb0b51c5d765d85 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Mon, 19 Aug 2024 17:26:22 -0400
Subject: [PATCH 37/83] create cli boot via dry-rb system

---
 lib/inferno/apps/cli.rb               |  1 -
 lib/inferno/apps/cli/execute.rb       |  5 +++--
 lib/inferno/config/boot/cli.rb        | 21 +++++++++++++++++++++
 spec/inferno/apps/cli/execute_spec.rb |  4 +++-
 4 files changed, 27 insertions(+), 4 deletions(-)
 create mode 100644 lib/inferno/config/boot/cli.rb

diff --git a/lib/inferno/apps/cli.rb b/lib/inferno/apps/cli.rb
index 7018b444e..b5db5f430 100644
--- a/lib/inferno/apps/cli.rb
+++ b/lib/inferno/apps/cli.rb
@@ -1,5 +1,4 @@
 require 'thor'
-
 require_relative 'cli/main'
 
 module Inferno
diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 96a568091..f0a57a42d 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -31,6 +31,7 @@ def self.suppress_output
         retval
       end
 
+      ENV['NO_DB'] = 'true'
       # Inferno boot flow triggers migration and logger outputs it
       suppress_output { require_relative '../../../inferno' }
 
@@ -53,7 +54,7 @@ def run(options)
         print_start_message
         verbose_puts 'options:', self.options
 
-        Inferno::Application.start(:suites)
+        Inferno::Application.start(:cli)
 
         set_runnable!
 
@@ -112,7 +113,7 @@ def run(options)
       end
 
       def print_help_and_exit
-        puts `bundle exec inferno help execute`
+        puts `NO_DB=true bundle exec inferno help execute`
         exit(3)
       end
 
diff --git a/lib/inferno/config/boot/cli.rb b/lib/inferno/config/boot/cli.rb
new file mode 100644
index 000000000..cb95d7d1b
--- /dev/null
+++ b/lib/inferno/config/boot/cli.rb
@@ -0,0 +1,21 @@
+Inferno::Application.register_provider(:cli) do
+  prepare do
+    target_container.start :logging
+
+    require 'oj'
+    require 'blueprinter'
+
+    Blueprinter.configure do |config|
+      config.generator = Oj
+    end
+
+    target_container.start :suites
+
+    # This line is required to bypass the NO_DB env variable and load all repositories
+    # but the NO_DB env variable itself is required to bypass specific Inferno boot bugs
+    Dir.glob('../../../repositories/*.rb').each do |repository|
+      require_relative repository
+      puts "Require'd #{repository}"
+    end
+  end
+end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 615be846a..1e0483f71 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -3,7 +3,9 @@
 RSpec.describe Inferno::CLI::Execute do # rubocop:disable RSpec/FilePath
   let(:instance) { described_class.new }
 
-  # TODO: test print_help_and_exit, set_runnable, runnable_id_key, thor_hash_to_suite_options_array
+  # TODO: runnable_id_key, thor_hash_to_suite_options_array
+
+  # TODO this test catches a weird inferno glitch
   describe '#print_help_and_exit' do
     it 'outputs something and exits' do
       expect do

From 46be4f786135c3877cd8d9f4365da9ba3664745a Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Sun, 25 Aug 2024 17:57:27 -0400
Subject: [PATCH 38/83] debug validator sessions loading error

---
 lib/inferno/apps/cli/execute.rb | 2 +-
 lib/inferno/config/boot/cli.rb  | 8 +-------
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index f0a57a42d..c825e7c04 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -31,7 +31,7 @@ def self.suppress_output
         retval
       end
 
-      ENV['NO_DB'] = 'true'
+      ENV['NO_DB'] = 'false'
       # Inferno boot flow triggers migration and logger outputs it
       suppress_output { require_relative '../../../inferno' }
 
diff --git a/lib/inferno/config/boot/cli.rb b/lib/inferno/config/boot/cli.rb
index cb95d7d1b..5dcce122c 100644
--- a/lib/inferno/config/boot/cli.rb
+++ b/lib/inferno/config/boot/cli.rb
@@ -10,12 +10,6 @@
     end
 
     target_container.start :suites
-
-    # This line is required to bypass the NO_DB env variable and load all repositories
-    # but the NO_DB env variable itself is required to bypass specific Inferno boot bugs
-    Dir.glob('../../../repositories/*.rb').each do |repository|
-      require_relative repository
-      puts "Require'd #{repository}"
-    end
+    target_container.start :validator
   end
 end

From f3605bef397509c7a746e62df0aa39439c132921 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Mon, 26 Aug 2024 10:59:01 -0400
Subject: [PATCH 39/83] cop

---
 spec/inferno/apps/cli/execute_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 1e0483f71..677d8eae3 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -5,7 +5,7 @@
 
   # TODO: runnable_id_key, thor_hash_to_suite_options_array
 
-  # TODO this test catches a weird inferno glitch
+  # TODO: this test catches a weird inferno glitch
   describe '#print_help_and_exit' do
     it 'outputs something and exits' do
       expect do

From 2ff92e64924933548a5d03107929f69646599508 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 27 Aug 2024 16:57:01 -0400
Subject: [PATCH 40/83] the big brain play

---
 lib/inferno/apps/cli/execute.rb | 46 ++++++++++++++++++++-------------
 lib/inferno/apps/cli/main.rb    |  1 +
 2 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index c825e7c04..347482f84 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -13,16 +13,20 @@ class Execute
       include ::Inferno::Utils::VerifyRunnable
       include ::Inferno::Utils::PersistInputs
 
+      COLOR = Pastel.new
+      CHECKMARK = "\u2713"
+      BAR = '=========================================='
+
+      attr_accessor :options, :runnable, :runnable_type
+
       def self.suppress_output
         begin
           original_stderr = $stderr.clone
           original_stdout = $stdout.clone
-          $stderr.reopen(File.new('/dev/null', 'w'))
-          $stdout.reopen(File.new('/dev/null', 'w'))
+          $stderr.reopen(File.new(File::NULL, 'w+'))
+          $stdout.reopen(File.new(File::NULL, 'w+'))
           retval = yield
-        rescue StandardError => e
-          $stdout.reopen(original_stdout)
-          $stderr.reopen(original_stderr)
+        rescue Exception => e
           raise e
         ensure
           $stdout.reopen(original_stdout)
@@ -31,21 +35,25 @@ def self.suppress_output
         retval
       end
 
-      ENV['NO_DB'] = 'false'
-      # Inferno boot flow triggers migration and logger outputs it
-      suppress_output { require_relative '../../../inferno' }
+      def self.boot_full_inferno
+        ENV['NO_DB'] = 'false'
 
-      COLOR = Pastel.new
-      CHECKMARK = "\u2713"
-      BAR = '=========================================='
+        # Inferno boot flow triggers migration and logger outputs it
+        Inferno::CLI::Execute.suppress_output { require_relative '../../../inferno' }
 
-      include Import[
-                test_sessions_repo: 'inferno.repositories.test_sessions',
-                session_data_repo: 'inferno.repositories.session_data',
-                test_runs_repo: 'inferno.repositories.test_runs'
-              ]
+        Inferno::Application.start(:cli)
 
-      attr_accessor :options, :runnable, :runnable_type
+=begin
+        Inferno::CLI::Execute.class_eval do
+          include Import[
+                    test_sessions_repo: 'inferno.repositories.test_sessions',
+                    session_data_repo: 'inferno.repositories.session_data',
+                    test_runs_repo: 'inferno.repositories.test_runs'
+                  ]
+        end
+=end
+        
+      end
 
       def run(options)
         print_help_and_exit if options[:help]
@@ -54,7 +62,9 @@ def run(options)
         print_start_message
         verbose_puts 'options:', self.options
 
-        Inferno::Application.start(:cli)
+        test_sessions_repo = Inferno::Repositories::TestSessions.new
+        session_data_repo = Inferno::Repositories::SessionData.new
+        test_runs_repo = Inferno::Repositories::TestRuns.new
 
         set_runnable!
 
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index aecc72088..4b0d42389 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -112,6 +112,7 @@ def version
              default: false,
              desc: 'Display this message'
       def execute
+        Execute.boot_full_inferno
         Execute.new.run(options)
       end
 

From ef4d8eb8252505d7069e18c1245bb87336065428 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 28 Aug 2024 11:34:23 -0400
Subject: [PATCH 41/83] rename cli booter to executor

---
 lib/inferno/apps/cli/execute.rb                 | 13 +------------
 lib/inferno/config/boot/{cli.rb => executor.rb} |  2 +-
 2 files changed, 2 insertions(+), 13 deletions(-)
 rename lib/inferno/config/boot/{cli.rb => executor.rb} (82%)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 347482f84..12dcd484d 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -41,18 +41,7 @@ def self.boot_full_inferno
         # Inferno boot flow triggers migration and logger outputs it
         Inferno::CLI::Execute.suppress_output { require_relative '../../../inferno' }
 
-        Inferno::Application.start(:cli)
-
-=begin
-        Inferno::CLI::Execute.class_eval do
-          include Import[
-                    test_sessions_repo: 'inferno.repositories.test_sessions',
-                    session_data_repo: 'inferno.repositories.session_data',
-                    test_runs_repo: 'inferno.repositories.test_runs'
-                  ]
-        end
-=end
-        
+        Inferno::Application.start(:executor)
       end
 
       def run(options)
diff --git a/lib/inferno/config/boot/cli.rb b/lib/inferno/config/boot/executor.rb
similarity index 82%
rename from lib/inferno/config/boot/cli.rb
rename to lib/inferno/config/boot/executor.rb
index 5dcce122c..f670e71a9 100644
--- a/lib/inferno/config/boot/cli.rb
+++ b/lib/inferno/config/boot/executor.rb
@@ -1,4 +1,4 @@
-Inferno::Application.register_provider(:cli) do
+Inferno::Application.register_provider(:executor) do
   prepare do
     target_container.start :logging
 

From 747d484939827b609bdfc104eea0323d5aee1fe3 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 28 Aug 2024 11:55:54 -0400
Subject: [PATCH 42/83] lint

---
 lib/inferno/apps/cli/execute.rb | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 12dcd484d..2ba971cb7 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -26,8 +26,6 @@ def self.suppress_output
           $stderr.reopen(File.new(File::NULL, 'w+'))
           $stdout.reopen(File.new(File::NULL, 'w+'))
           retval = yield
-        rescue Exception => e
-          raise e
         ensure
           $stdout.reopen(original_stdout)
           $stderr.reopen(original_stderr)

From 150464bd79b49c4daaa97c1cfec8e0e30872ae78 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 28 Aug 2024 12:38:59 -0400
Subject: [PATCH 43/83] more rspec

---
 spec/inferno/apps/cli/execute_spec.rb | 47 +++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 3 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 677d8eae3..92dda347c 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -3,9 +3,26 @@
 RSpec.describe Inferno::CLI::Execute do # rubocop:disable RSpec/FilePath
   let(:instance) { described_class.new }
 
-  # TODO: runnable_id_key, thor_hash_to_suite_options_array
+  describe '.suppress_output' do
+    it 'disables stdout' do
+      expect do
+        described_class.suppress_output { puts 'Hide me' }
+      end.to_not output(/.+/).to_stdout_from_any_process
+    end
+
+    it 'disables stderr' do
+      expect do
+        described_class.suppress_output { warn 'Hide me' }
+      end.to_not output(/.+/).to_stderr_from_any_process
+    end
+  end
+
+  describe '.boot_full_inferno' do
+    it 'does not raise error' do
+      expect { described_class.boot_full_inferno }.to_not raise_error(StandardError)
+    end
+  end
 
-  # TODO: this test catches a weird inferno glitch
   describe '#print_help_and_exit' do
     it 'outputs something and exits' do
       expect do
@@ -52,6 +69,31 @@
     end
   end
 
+  describe '#runnable_id_key' do
+    { suite: :test_suite_id, group: :test_group_id, test: :test_id }.each do |runnable_type, id_key|
+      it "returns proper id for runnable type #{runnable_type}" do
+        stubbed_instance = instance
+        allow(stubbed_instance).to receive(:runnable_type).and_return(runnable_type)
+
+        expect(stubbed_instance.runnable_id_key).to eq(id_key)
+      end
+    end
+  end
+
+  describe '#thor_hash_to_suite_options_array' do
+    let(:hash) { { us_core: 'us_core_v311' } }
+
+    it 'converts hash to array' do
+      result = instance.thor_hash_to_suite_options_array(hash)
+      expect(result.class).to eq(Array)
+    end
+
+    it 'returns proper inputs array' do
+      result = instance.thor_hash_to_inputs_array(hash)
+      expect(result).to eq([{ name: :us_core, value: 'us_core_v311' }])
+    end
+  end
+
   describe '#thor_hash_to_inputs_array' do
     let(:hash) { { url: 'https://example.com' } }
 
@@ -72,7 +114,6 @@
     let(:inputs_hash) { { url: 'https://example.com' } }
     let(:inputs_array) { [{ name: :url, value: 'https://example.com' }] }
 
-    # TODO: test all cases [{suite: }]
     it 'returns test run params' do
       stubbed_instance = instance
       allow(stubbed_instance).to receive(:options).and_return({ inputs: inputs_hash })

From 5c85015c2b81833327c7625f933a3e5a0f361df6 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 13:27:49 -0400
Subject: [PATCH 44/83] In execute_spec.rb rename legit JSON to valid JSON

Co-authored-by: Stephen MacVicar <Jammjammjamm@users.noreply.github.com>
---
 spec/inferno/apps/cli/execute_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 92dda347c..d21a9abd6 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -132,7 +132,7 @@
       expect { instance.serialize(test_results) }.to_not raise_error(StandardError)
     end
 
-    it 'returns legit JSON' do
+    it 'returns valid JSON' do
       expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(JSON::ParserError)
       expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(JSON::NestingError)
       expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(TypeError)

From d132650f27e67e6ce05a2160b96b4826d58bb66f Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 13:35:26 -0400
Subject: [PATCH 45/83] address util spec, reword test titles, remove useless
 defined test

---
 spec/inferno/utils/persist_inputs_spec.rb  | 8 ++------
 spec/inferno/utils/verify_runnable_spec.rb | 6 +-----
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/spec/inferno/utils/persist_inputs_spec.rb b/spec/inferno/utils/persist_inputs_spec.rb
index 0cae96b0b..7cded414a 100644
--- a/spec/inferno/utils/persist_inputs_spec.rb
+++ b/spec/inferno/utils/persist_inputs_spec.rb
@@ -12,10 +12,6 @@
     let(:test_sessions_repo) { Inferno::Repositories::TestSessions.new }
     let(:session_data_repo) { Inferno::Repositories::SessionData.new }
 
-    it 'is defined' do
-      expect(described_class.method_defined?(:persist_inputs)).to eq(true)
-    end
-
     it 'saves inputs to db' do
       test_session = test_sessions_repo.create(test_suite_id: suite.id)
 
@@ -36,7 +32,7 @@
       expect(persisted_data).to eq('persist me')
     end
 
-    it 'can handle unregistered inputs' do
+    it 'saves known inputs when given unknown extraneous inputs' do
       test_session = test_sessions_repo.create(test_suite_id: suite.id)
 
       test_run = create(:test_run, test_session:)
@@ -46,7 +42,7 @@
         test_session_id: test_session.id,
         test_suite_id: suite.id,
         inputs: [
-          { name: 'unregistered', value: 'omit me' },
+          { name: 'extraneous', value: 'omit me' },
           { name: 'input1', value: 'persist me' }
         ]
       }
diff --git a/spec/inferno/utils/verify_runnable_spec.rb b/spec/inferno/utils/verify_runnable_spec.rb
index 6513003cb..c5e3574b2 100644
--- a/spec/inferno/utils/verify_runnable_spec.rb
+++ b/spec/inferno/utils/verify_runnable_spec.rb
@@ -12,11 +12,7 @@
     let(:bad_inputs) { [{ name: :input2, value: :foo }] }
     let(:unrunnable) { BasicTestSuite::DefGroup.tests.first }
 
-    it 'is defined' do
-      expect(described_class.method_defined?(:verify_runnable)).to eq(true)
-    end
-
-    it 'allows legit runnables' do
+    it 'allows runnables with good inputs' do
       expect { dummy.verify_runnable(suite, good_inputs, {}) }.to_not raise_error
     end
 

From 23b448e5c53a659a16595a3e601b53211b7703fa Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 13:55:48 -0400
Subject: [PATCH 46/83] rename execute_spec#print_color_results test

---
 spec/inferno/apps/cli/execute_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index d21a9abd6..6cde24940 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -255,7 +255,7 @@
   describe '#print_color_results' do
     let(:results) { create_list(:random_result, 10) }
 
-    it 'outputs something with 10 random results' do
+    it 'outputs something with verbose false' do
       stubbed_instance = instance
       allow(stubbed_instance).to receive(:options).and_return({ verbose: false })
       expect { stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout

From d19b8a83442220edc35c5010d90361a99b2ec613 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 14:10:20 -0400
Subject: [PATCH 47/83] fix execute.rb multi-line reverse condition

---
 lib/inferno/apps/cli/execute.rb | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 2ba971cb7..b5e61a216 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -90,9 +90,11 @@ def run(options)
         verbose_print_json_results(results)
         print_color_results(results)
 
-        exit(0) if results.find do |result|
-                     result.send(runnable_id_key) == options[runnable_type.to_sym]
-                   end.result == 'pass'
+        if results.find do |result|
+             result.send(runnable_id_key) == options[runnable_type.to_sym]
+           end.result == 'pass'
+          exit(0)
+        end
 
         # exit(1) is for Thor failures
         # exit(2) is for shell builtin failures

From f772642aac69a70641a4a00544a45b9c421cd80d Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 14:19:18 -0400
Subject: [PATCH 48/83] execute_spec make #format_messages tests case sensitive

---
 spec/inferno/apps/cli/execute_spec.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 6cde24940..e0d57cd44 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -191,12 +191,12 @@
   describe '#format_messages' do
     let(:test_result) { repo_create(:result, message_count: 10) }
 
-    it 'includes all characters case-insensitive' do
+    it 'includes all characters' do
       messages = test_result.messages
       formatted_string = instance.format_messages(test_result)
 
       messages.each do |message|
-        expect(formatted_string.upcase).to include message.message.upcase
+        expect(formatted_string).to include message.message
       end
     end
   end
@@ -209,7 +209,7 @@
       formatted_string = instance.format_requests(test_result)
 
       requests.each do |request|
-        expect(formatted_string.upcase).to include request.status.to_s.upcase
+        expect(formatted_string).to include request.status.to_s
       end
     end
   end

From 45353d428e641286a220abc356f4e701e1e8fbae Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 15:03:54 -0400
Subject: [PATCH 49/83] wrap execute help message

---
 lib/inferno/apps/cli/main.rb | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index 4b0d42389..f6241d40e 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -67,7 +67,8 @@ def version
       end
 
       EXECUTE_HELP = <<~END_OF_HELP.freeze
-        Run Inferno tests in the command line. Exits with 0 only if test suite passes. Must be run from test kit as working directory.
+        Run Inferno tests in the command line. Exits with 0 only if test entity passes.
+        Must be run with test kit as working directory.
 
         You must have background services running: `bundle exec inferno services start`
 
@@ -75,7 +76,9 @@ def version
 
         Examples:
 
-            `bundle exec inferno execute --suite dev_validator --inputs "url:https://hapi.fhir.org/baseR4" patient_id:1234321`
+            `bundle exec inferno execute --suite dev_validator \
+                                        --inputs "url:https://hapi.fhir.org/baseR4" \
+                                                 patient_id:1234321`
             => Outputs test results
       END_OF_HELP
       desc 'execute', 'Run Inferno tests in command line'

From 729ae9df71303fb1c60e3a07e84e230307a21aec Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 15:34:58 -0400
Subject: [PATCH 50/83] factorize format_input and format_output into
 format_session_data and rspec

---
 lib/inferno/apps/cli/execute.rb       | 16 +++++++++------
 spec/inferno/apps/cli/execute_spec.rb | 29 +++++++++++----------------
 2 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index b5e61a216..0dd047e5c 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -208,17 +208,21 @@ def format_requests(result)
         end.join
       end
 
-      def format_inputs(result, attr = :input_json)
-        input_json = result.send(attr)
-        return '' if input_json.nil?
+      def format_session_data(result, attr)
+        json = result.send(attr)
+        return '' if json.nil?
 
-        JSON.parse(input_json).map do |input|
-          "\n\t\t#{input['name']}: #{input['value']}"
+        JSON.parse(json).map do |hash|
+          "\n\t\t#{hash['name']}: #{hash['value']}"
         end.join
       end
 
+      def format_inputs(result)
+        format_session_data(result, :input_json)
+      end
+
       def format_outputs(result)
-        format_inputs(result, :output_json)
+        format_session_data(result, :output_json)
       end
 
       def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index e0d57cd44..2fbe6fa6f 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -214,26 +214,21 @@
     end
   end
 
-  describe '#format_inputs' do
-    let(:inputs) { [{ name: :url, value: 'https://example.com' }] }
-    let(:test_result) { create(:result, input_json: JSON.generate(inputs)) }
-
-    it 'includes all values' do
-      formatted_string = instance.format_inputs(test_result)
-      inputs.each do |input_element|
-        expect(formatted_string).to include input_element[:value]
+  describe '#format_session_data' do
+    let(:data) { [{ name: :url, value: 'https://example.com' }, { name: :token, value: 'SAMPLE_OUTPUT' }] }
+    let(:test_result) { create(:result, input_json: JSON.generate(data), output_json: JSON.generate(data)) }
+
+    it 'includes all values for input_json' do
+      formatted_string = instance.format_session_data(test_result, :input_json)
+      data.each do |data_element|
+        expect(formatted_string).to include data_element[:value]
       end
     end
-  end
-
-  describe '#format_outputs' do
-    let(:outputs) { [{ name: :token, value: 'SAMPLE_OUTPUT' }] }
-    let(:test_result) { create(:result, output_json: JSON.generate(outputs)) }
 
-    it 'includes all values' do
-      formatted_string = instance.format_outputs(test_result)
-      outputs.each do |output_element|
-        expect(formatted_string).to include output_element[:value]
+    it 'includes all values for output_json' do
+      formatted_string = instance.format_session_data(test_result, :output_json)
+      data.each do |data_element|
+        expect(formatted_string).to include data_element[:value]
       end
     end
   end

From d6fef8be6aaba987835cabf00c1f9712b6cbd50f Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 15:40:36 -0400
Subject: [PATCH 51/83] execute_spec JSON parse validation

---
 spec/inferno/apps/cli/execute_spec.rb | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 2fbe6fa6f..430e52d57 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -133,9 +133,7 @@
     end
 
     it 'returns valid JSON' do
-      expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(JSON::ParserError)
-      expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(JSON::NestingError)
-      expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(TypeError)
+      expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(StandardError)
     end
   end
 

From 88709a561670f5bb5f8214b1ae68f7744622c6d7 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 15:46:26 -0400
Subject: [PATCH 52/83] execute_spec remove all stubbed_instance variables

---
 spec/inferno/apps/cli/execute_spec.rb | 56 +++++++++++----------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 430e52d57..7f092ed86 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -36,17 +36,15 @@
      { test: 'BasicTestSuite::AbcGroup-demo_test' }].each do |given_options|
       context "with #{given_options.keys.first} option" do
         it 'does not raise error' do
-          stubbed_instance = instance
-          allow(stubbed_instance).to receive(:options).and_return(given_options)
+          allow(instance).to receive(:options).and_return(given_options)
 
-          expect { stubbed_instance.set_runnable! }.to_not raise_error(StandardError)
+          expect { instance.set_runnable! }.to_not raise_error(StandardError)
         end
 
         it 'sets runnable' do
-          stubbed_instance = instance
-          allow(stubbed_instance).to receive(:options).and_return(given_options)
+          allow(instance).to receive(:options).and_return(given_options)
 
-          stubbed_instance.set_runnable!
+          instance.set_runnable!
           klass = case given_options.keys.first
                   when :suite
                     Inferno::TestSuite
@@ -55,15 +53,14 @@
                   else
                     Inferno::Test
                   end
-          expect(stubbed_instance.runnable).to be < klass
+          expect(instance.runnable).to be < klass
         end
 
         it 'sets runnable_type' do
-          stubbed_instance = instance
-          allow(stubbed_instance).to receive(:options).and_return(given_options)
+          allow(instance).to receive(:options).and_return(given_options)
 
-          stubbed_instance.set_runnable!
-          expect(stubbed_instance.runnable_type).to eq(given_options.keys.first.to_s)
+          instance.set_runnable!
+          expect(instance.runnable_type).to eq(given_options.keys.first.to_s)
         end
       end
     end
@@ -72,10 +69,9 @@
   describe '#runnable_id_key' do
     { suite: :test_suite_id, group: :test_group_id, test: :test_id }.each do |runnable_type, id_key|
       it "returns proper id for runnable type #{runnable_type}" do
-        stubbed_instance = instance
-        allow(stubbed_instance).to receive(:runnable_type).and_return(runnable_type)
+        allow(instance).to receive(:runnable_type).and_return(runnable_type)
 
-        expect(stubbed_instance.runnable_id_key).to eq(id_key)
+        expect(instance.runnable_id_key).to eq(id_key)
       end
     end
   end
@@ -115,12 +111,11 @@
     let(:inputs_array) { [{ name: :url, value: 'https://example.com' }] }
 
     it 'returns test run params' do
-      stubbed_instance = instance
-      allow(stubbed_instance).to receive(:options).and_return({ inputs: inputs_hash })
-      allow(stubbed_instance).to receive(:runnable_type).and_return('suite')
+      allow(instance).to receive(:options).and_return({ inputs: inputs_hash })
+      allow(instance).to receive(:runnable_type).and_return('suite')
       test_session_inst = test_session
 
-      result = stubbed_instance.create_params(test_session_inst, test_suite)
+      result = instance.create_params(test_session_inst, test_suite)
       expect(result).to eq({ test_session_id: test_session.id, test_suite_id: test_suite.id, inputs: inputs_array })
     end
   end
@@ -139,26 +134,23 @@
 
   describe '#verbose_print' do
     it 'outputs when verbose is true' do
-      stubbed_instance = instance
-      allow(stubbed_instance).to receive(:options).and_return({ verbose: true })
+      allow(instance).to receive(:options).and_return({ verbose: true })
 
-      expect { stubbed_instance.verbose_print('Lorem') }.to output(/Lorem/).to_stdout
+      expect { instance.verbose_print('Lorem') }.to output(/Lorem/).to_stdout
     end
 
     it 'does not output when verbose is false' do
-      stubbed_instance = instance
-      allow(stubbed_instance).to receive(:options).and_return({ verbose: false })
+      allow(instance).to receive(:options).and_return({ verbose: false })
 
-      expect { stubbed_instance.verbose_print('Lorem') }.to_not output(/.+/).to_stdout
+      expect { instance.verbose_print('Lorem') }.to_not output(/.+/).to_stdout
     end
   end
 
   describe '#verbose_puts' do
     it 'has output ending with \n with when verbose is true' do
-      stubbed_instance = instance
-      allow(stubbed_instance).to receive(:options).and_return({ verbose: true })
+      allow(instance).to receive(:options).and_return({ verbose: true })
 
-      expect { stubbed_instance.verbose_puts('Lorem') }.to output(/Lorem\n/).to_stdout
+      expect { instance.verbose_puts('Lorem') }.to output(/Lorem\n/).to_stdout
     end
   end
 
@@ -249,15 +241,13 @@
     let(:results) { create_list(:random_result, 10) }
 
     it 'outputs something with verbose false' do
-      stubbed_instance = instance
-      allow(stubbed_instance).to receive(:options).and_return({ verbose: false })
-      expect { stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout
+      allow(instance).to receive(:options).and_return({ verbose: false })
+      expect { instance.print_color_results(results) }.to output(/.+/).to_stdout
     end
 
     it 'outputs something with verbose true' do
-      stubbed_instance = instance
-      allow(stubbed_instance).to receive(:options).and_return({ verbose: true })
-      expect { stubbed_instance.print_color_results(results) }.to output(/.+/).to_stdout
+      allow(instance).to receive(:options).and_return({ verbose: true })
+      expect { instance.print_color_results(results) }.to output(/.+/).to_stdout
     end
   end
 

From a1f96220a7d0e2e36da6c0f77b18b13681ecd72f Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 15:48:58 -0400
Subject: [PATCH 53/83] execute_spec#set_runnable remove useless test

---
 spec/inferno/apps/cli/execute_spec.rb | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 7f092ed86..a583414ff 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -35,12 +35,6 @@
     [{ suite: 'basic' }, { group: 'BasicTestSuite::AbcGroup' },
      { test: 'BasicTestSuite::AbcGroup-demo_test' }].each do |given_options|
       context "with #{given_options.keys.first} option" do
-        it 'does not raise error' do
-          allow(instance).to receive(:options).and_return(given_options)
-
-          expect { instance.set_runnable! }.to_not raise_error(StandardError)
-        end
-
         it 'sets runnable' do
           allow(instance).to receive(:options).and_return(given_options)
 
@@ -56,6 +50,7 @@
           expect(instance.runnable).to be < klass
         end
 
+        # TODO: change with new custom getter
         it 'sets runnable_type' do
           allow(instance).to receive(:options).and_return(given_options)
 

From 4162814e3714112efbbaa49c52b7018d9dd6246c Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 16:02:24 -0400
Subject: [PATCH 54/83] fix rubocop but on stderr test

---
 lib/inferno/apps/cli/execute.rb       | 6 +++---
 spec/inferno/apps/cli/execute_spec.rb | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 0dd047e5c..d29eb158b 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -21,14 +21,14 @@ class Execute
 
       def self.suppress_output
         begin
-          original_stderr = $stderr.clone
+          # original_stderr = $stderr.clone
           original_stdout = $stdout.clone
-          $stderr.reopen(File.new(File::NULL, 'w+'))
+          # $stderr.reopen(File.new(File::NULL, 'w+'))
           $stdout.reopen(File.new(File::NULL, 'w+'))
           retval = yield
         ensure
           $stdout.reopen(original_stdout)
-          $stderr.reopen(original_stderr)
+          # $stderr.reopen(original_stderr)
         end
         retval
       end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index a583414ff..42c706077 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -12,7 +12,7 @@
 
     it 'disables stderr' do
       expect do
-        described_class.suppress_output { warn 'Hide me' }
+        described_class.suppress_output { $stderr.puts 'Hide me' } # rubocop:disable Style/StderrPuts
       end.to_not output(/.+/).to_stderr_from_any_process
     end
   end

From e54444a45fb379e7fa03986b3623184637e9b47b Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 16:05:29 -0400
Subject: [PATCH 55/83] remove stderr suppression from Execute.suppress_output

---
 lib/inferno/apps/cli/execute.rb       | 3 ---
 spec/inferno/apps/cli/execute_spec.rb | 6 ------
 2 files changed, 9 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index d29eb158b..d373a0c34 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -21,14 +21,11 @@ class Execute
 
       def self.suppress_output
         begin
-          # original_stderr = $stderr.clone
           original_stdout = $stdout.clone
-          # $stderr.reopen(File.new(File::NULL, 'w+'))
           $stdout.reopen(File.new(File::NULL, 'w+'))
           retval = yield
         ensure
           $stdout.reopen(original_stdout)
-          # $stderr.reopen(original_stderr)
         end
         retval
       end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 42c706077..efb36d08c 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -9,12 +9,6 @@
         described_class.suppress_output { puts 'Hide me' }
       end.to_not output(/.+/).to_stdout_from_any_process
     end
-
-    it 'disables stderr' do
-      expect do
-        described_class.suppress_output { $stderr.puts 'Hide me' } # rubocop:disable Style/StderrPuts
-      end.to_not output(/.+/).to_stderr_from_any_process
-    end
   end
 
   describe '.boot_full_inferno' do

From 209ef9968397b4ffc7d7ed0b12871ebafaf86d09 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 29 Aug 2024 16:51:15 -0400
Subject: [PATCH 56/83] replace set_runnable with better runnable and
 runnable_type getters

---
 lib/inferno/apps/cli/execute.rb       | 47 ++++++++++++++----------
 spec/inferno/apps/cli/execute_spec.rb | 52 +++++++++++++--------------
 2 files changed, 54 insertions(+), 45 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index d373a0c34..c75701d94 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -17,7 +17,7 @@ class Execute
       CHECKMARK = "\u2713"
       BAR = '=========================================='
 
-      attr_accessor :options, :runnable, :runnable_type
+      attr_accessor :options
 
       def self.suppress_output
         begin
@@ -50,8 +50,6 @@ def run(options)
         session_data_repo = Inferno::Repositories::SessionData.new
         test_runs_repo = Inferno::Repositories::TestRuns.new
 
-        set_runnable!
-
         test_session = test_sessions_repo.create({
                                                    test_suite_id: runnable.suite.id,
                                                    suite_options: thor_hash_to_suite_options_array(
@@ -120,22 +118,33 @@ def print_start_message
         puts BAR
       end
 
-      def set_runnable!
-        if options[:suite]
-          self.runnable_type = 'suite'
-          self.runnable = Inferno::Repositories::TestSuites.new.find(options[:suite])
-          raise StandardError, "Suite #{options[:suite]} not found" if runnable.nil?
-        elsif options[:group]
-          self.runnable_type = 'group'
-          self.runnable = Inferno::Repositories::TestGroups.new.find(options[:group])
-          raise StandardError, "Group #{options[:group]} not found" if runnable.nil?
-        elsif options[:test]
-          self.runnable_type = 'test'
-          self.runnable = Inferno::Repositories::Tests.new.find(options[:test])
-          raise StandardError, "Test #{options[:test]} not found" if runnable.nil?
-        else
-          raise StandardError, 'No suite or group id provided'
-        end
+      def runnable
+        @runnable ||=
+          case runnable_type
+          when 'suite'
+            Inferno::Repositories::TestSuites.new.find(options[:suite])
+          when 'group'
+            Inferno::Repositories::TestGroups.new.find(options[:group])
+          when 'test'
+            Inferno::Repositories::Tests.new.find(options[:test])
+          end
+
+        raise StandardError, "#{runnable_type.capitalize} #{options[runnable_type.to_sym]} not found" if @runnable.nil?
+
+        @runnable
+      end
+
+      def runnable_type
+        @runnable_type ||=
+          if options[:suite]
+            'suite'
+          elsif options[:group]
+            'group'
+          elsif options[:test]
+            'test'
+          else
+            raise StandardError, 'No suite, group, or test id provided'
+          end
       end
 
       def runnable_id_key
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index efb36d08c..8790c27f0 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -25,32 +25,32 @@
     end
   end
 
-  describe '#set_runnable!' do
-    [{ suite: 'basic' }, { group: 'BasicTestSuite::AbcGroup' },
-     { test: 'BasicTestSuite::AbcGroup-demo_test' }].each do |given_options|
-      context "with #{given_options.keys.first} option" do
-        it 'sets runnable' do
-          allow(instance).to receive(:options).and_return(given_options)
-
-          instance.set_runnable!
-          klass = case given_options.keys.first
-                  when :suite
-                    Inferno::TestSuite
-                  when :group
-                    Inferno::TestGroup
-                  else
-                    Inferno::Test
-                  end
-          expect(instance.runnable).to be < klass
-        end
-
-        # TODO: change with new custom getter
-        it 'sets runnable_type' do
-          allow(instance).to receive(:options).and_return(given_options)
-
-          instance.set_runnable!
-          expect(instance.runnable_type).to eq(given_options.keys.first.to_s)
-        end
+  [
+    { suite: 'basic' },
+    { group: 'BasicTestSuite::AbcGroup' },
+    { test: 'BasicTestSuite::AbcGroup-demo_test' }
+  ].each do |given_options|
+    describe '#runnable' do
+      it "sets runnable to #{given_options.keys.first} entity" do
+        allow(instance).to receive(:options).and_return(given_options)
+        klass = case given_options.keys.first
+                when :suite
+                  Inferno::TestSuite
+                when :group
+                  Inferno::TestGroup
+                else
+                  Inferno::Test
+                end
+
+        expect(instance.runnable).to be < klass
+      end
+    end
+
+    describe '#runnable_type' do
+      it "sets runnable_type to #{given_options.keys.first}" do
+        allow(instance).to receive(:options).and_return(given_options)
+
+        expect(instance.runnable_type).to eq(given_options.keys.first.to_s)
       end
     end
   end

From c3ec5f60c9bffa2439c013f8051095828b24dc90 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Fri, 30 Aug 2024 10:25:17 -0400
Subject: [PATCH 57/83] rm redundant rspec test_session_inst

---
 spec/inferno/apps/cli/execute_spec.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 8790c27f0..d91f4df60 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -102,9 +102,8 @@
     it 'returns test run params' do
       allow(instance).to receive(:options).and_return({ inputs: inputs_hash })
       allow(instance).to receive(:runnable_type).and_return('suite')
-      test_session_inst = test_session
 
-      result = instance.create_params(test_session_inst, test_suite)
+      result = instance.create_params(test_session, test_suite)
       expect(result).to eq({ test_session_id: test_session.id, test_suite_id: test_suite.id, inputs: inputs_array })
     end
   end

From efb99833ee40e914e5ce72c3609e0ed320c314dd Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 3 Sep 2024 10:56:01 -0400
Subject: [PATCH 58/83] execute.rb fix results title

Co-authored-by: Stephen MacVicar <Jammjammjamm@users.noreply.github.com>
---
 lib/inferno/apps/cli/execute.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index c75701d94..b13f5b73c 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -264,7 +264,7 @@ def verbose_print_json_results(results)
 
       def print_color_results(results)
         puts BAR
-        puts 'Colored Test Results:'
+        puts 'Test Results:'
         puts BAR
         results.each do |result|
           print format_id(result), ': ', format_result(result), "\n"

From 43b4f533456d1094a5601885b75fd0cf0dcce24a Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 3 Sep 2024 11:07:53 -0400
Subject: [PATCH 59/83] make --help exit with 0

---
 lib/inferno/apps/cli/execute.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index b13f5b73c..9d2dfa71c 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -108,7 +108,7 @@ def run(options)
 
       def print_help_and_exit
         puts `NO_DB=true bundle exec inferno help execute`
-        exit(3)
+        exit(0)
       end
 
       def print_start_message

From da7acd4472cce19f51db3705351cfd605711a468 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 3 Sep 2024 11:45:20 -0400
Subject: [PATCH 60/83] cop

---
 lib/inferno/apps/cli/execute.rb | 42 +++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 9d2dfa71c..c72b0fc35 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -46,27 +46,12 @@ def run(options)
         print_start_message
         verbose_puts 'options:', self.options
 
-        test_sessions_repo = Inferno::Repositories::TestSessions.new
-        session_data_repo = Inferno::Repositories::SessionData.new
-        test_runs_repo = Inferno::Repositories::TestRuns.new
-
-        test_session = test_sessions_repo.create({
-                                                   test_suite_id: runnable.suite.id,
-                                                   suite_options: thor_hash_to_suite_options_array(
-                                                     options[:suite_options]
-                                                   )
-                                                 })
-
         verify_runnable(
           runnable,
           thor_hash_to_inputs_array(options[:inputs]),
           test_session.suite_options
         )
 
-        test_run = test_runs_repo.create(
-          create_params(test_session, runnable).merge({ status: 'queued' })
-        )
-
         persist_inputs(session_data_repo, create_params(test_session, runnable), test_run)
 
         puts 'Running tests. This may take a while...' # TODO: spinner/progress bar
@@ -118,6 +103,33 @@ def print_start_message
         puts BAR
       end
 
+      def test_sessions_repo
+        @test_sessions_repo ||= Inferno::Repositories::TestSessions.new
+      end
+
+      def session_data_repo
+        @session_data_repo ||= Inferno::Repositories::SessionData.new
+      end
+
+      def test_runs_repo
+        @test_runs_repo ||= Inferno::Repositories::TestRuns.new
+      end
+
+      def test_session
+        @test_session ||= test_sessions_repo.create({
+                                                      test_suite_id: runnable.suite.id,
+                                                      suite_options: thor_hash_to_suite_options_array(
+                                                        options[:suite_options]
+                                                      )
+                                                    })
+      end
+
+      def test_run
+        @test_run ||= test_runs_repo.create(
+          create_params(test_session, runnable).merge({ status: 'queued' })
+        )
+      end
+
       def runnable
         @runnable ||=
           case runnable_type

From 7c36858943982013a7ee65ac11bd2474a6c648aa Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 3 Sep 2024 14:45:05 -0400
Subject: [PATCH 61/83] replace format_id with format_tag

---
 lib/inferno/apps/cli/execute.rb       | 16 +++++++++++++---
 spec/inferno/apps/cli/execute_spec.rb | 18 ++++++++++--------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index c72b0fc35..52337a9f5 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -210,8 +210,18 @@ def verbose_puts(*args)
         verbose_print(*args)
       end
 
-      def format_id(result)
-        result.runnable.id
+      def format_tag(result)
+        if result.runnable.respond_to?(:short_id) && result.runnable.short_title.presence
+          "#{result.runnable.short_id} #{result.runnable.short_title}"
+        elsif result.runnable.respond_to?(:short_id) && result.runnable.title.presence
+          "#{result.runnable.short_id} #{result.runnable.title}"
+        elsif result.runnable.short_title.presence
+          result.runnable.short_title
+        elsif result.runnable.title.presence
+          result.runnable.title
+        else
+          result.runnable.id
+        end
       end
 
       def format_messages(result)
@@ -279,7 +289,7 @@ def print_color_results(results)
         puts 'Test Results:'
         puts BAR
         results.each do |result|
-          print format_id(result), ': ', format_result(result), "\n"
+          print format_tag(result), ': ', format_result(result), "\n"
           verbose_puts "\tsummary: ",   result.result_message
           verbose_puts "\tmessages: ",  format_messages(result)
           verbose_puts "\trequests: ",  format_requests(result)
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index d91f4df60..c44f4f9b2 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -142,27 +142,29 @@
     end
   end
 
-  describe '#format_id' do
+  describe '#format_tag' do
     let(:test_suite) { BasicTestSuite::Suite }
+    let(:test_suite_result) { create(:result, runnable: { test_suite_id: test_suite.id }) }
     let(:test_group) { BasicTestSuite::AbcGroup }
     let(:test) { test_group.tests.first }
 
-    it 'returns suite id if test result belongs to suite' do
-      test_result = create(:result, runnable: { test_suite_id: test_suite.id })
+    it "includes a suite's title if short_title not found" do
+      # test_result = create(:result, runnable: { test_suite_id: test_suite.id })
+      test_result = test_suite_result
 
-      expect(instance.format_id(test_result)).to eq(test_suite.id)
+      expect(instance.format_tag(test_result)).to match(test_suite.short_title)
     end
 
-    it 'returns group id if test result belongs to group' do
+    it "includes a group's short id" do
       test_result = create(:result, runnable: { test_group_id: test_group.id })
 
-      expect(instance.format_id(test_result)).to eq(test_group.id)
+      expect(instance.format_tag(test_result)).to match(test_group.short_id)
     end
 
-    it 'returns test id if test result belongs to test' do
+    it "includes a test's short it" do
       test_result = create(:result, runnable: { test_id: test.id })
 
-      expect(instance.format_id(test_result)).to eq(test.id)
+      expect(instance.format_tag(test_result)).to match(test.short_id)
     end
   end
 

From 424d2e265ca56dcdc666583bf93364d573868c92 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 3 Sep 2024 16:39:36 -0400
Subject: [PATCH 62/83] rspec and cop format_tag

---
 lib/inferno/apps/cli/execute.rb       | 16 ++---
 spec/inferno/apps/cli/execute_spec.rb | 95 +++++++++++++++++++++++----
 2 files changed, 88 insertions(+), 23 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 52337a9f5..a36ea0db3 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -211,19 +211,17 @@ def verbose_puts(*args)
       end
 
       def format_tag(result)
-        if result.runnable.respond_to?(:short_id) && result.runnable.short_title.presence
-          "#{result.runnable.short_id} #{result.runnable.short_title}"
-        elsif result.runnable.respond_to?(:short_id) && result.runnable.title.presence
-          "#{result.runnable.short_id} #{result.runnable.title}"
-        elsif result.runnable.short_title.presence
-          result.runnable.short_title
-        elsif result.runnable.title.presence
-          result.runnable.title
+        if result.runnable.respond_to?(:short_id)
+          "#{result.runnable.short_id} #{format_tag_suffix(result)}"
         else
-          result.runnable.id
+          format_tag_suffix(result)
         end
       end
 
+      def format_tag_suffix(result)
+        result.runnable.short_title.presence || result.runnable.title.presence || result.runnable.id
+      end
+
       def format_messages(result)
         result.messages.map do |message|
           "\n\t\t#{message.type}: #{message.message}"
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index c44f4f9b2..34ea22ff9 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -143,28 +143,95 @@
   end
 
   describe '#format_tag' do
-    let(:test_suite) { BasicTestSuite::Suite }
-    let(:test_suite_result) { create(:result, runnable: { test_suite_id: test_suite.id }) }
-    let(:test_group) { BasicTestSuite::AbcGroup }
-    let(:test) { test_group.tests.first }
+    let(:suites_repo) { Inferno::Repositories::TestSuites.new }
+
+    let(:suite_all) do
+      Class.new(Inferno::TestSuite) do
+        id 'mock_suite_id_1'
+        short_title 'short'
+        title 'title'
+      end
+    end
+
+    let(:suite_no_short_title) do
+      Class.new(Inferno::TestSuite) do
+        id 'mock_suite_id_2'
+        title 'title'
+      end
+    end
+
+    let(:suite_id_only) do
+      Class.new(Inferno::TestSuite) do
+        id 'mock_suite_id_3'
+      end
+    end
+
+    let(:groups_repo) { Inferno::Repositories::TestGroups.new }
+
+    let(:group_all) do
+      Class.new(Inferno::TestGroup) do
+        id 'mock_group_id_1'
+        short_title 'short'
+        title 'title'
+      end
+    end
+
+    let(:group_no_short_title) do
+      Class.new(Inferno::TestGroup) do
+        id 'mock_group_id_2'
+        title 'title'
+      end
+    end
+
+    let(:group_no_titles) do
+      Class.new(Inferno::TestGroup) do
+        id 'mock_group_id_3'
+      end
+    end
+
+    it "includes a runnable's short_id and short_title if possible" do
+      groups_repo.insert(group_all)
+      test_result = create(:result, runnable: { test_group_id: group_all.id })
+
+      expect(instance.format_tag(test_result)).to match(group_all.short_id)
+      expect(instance.format_tag(test_result)).to match(group_all.short_title)
+    end
+
+    it "includes a runnable's short_id and title if no short_title found" do
+      groups_repo.insert(group_no_short_title)
+      test_result = create(:result, runnable: { test_group_id: group_no_short_title.id })
+
+      expect(instance.format_tag(test_result)).to match(group_no_short_title.short_id)
+      expect(instance.format_tag(test_result)).to match(group_no_short_title.title)
+    end
+
+    it "includes a runnable's short_id and id if no title/short_title found" do
+      groups_repo.insert(group_no_titles)
+      test_result = create(:result, runnable: { test_group_id: group_no_titles.id })
+
+      expect(instance.format_tag(test_result)).to match(group_no_titles.short_id)
+      expect(instance.format_tag(test_result)).to match(group_no_titles.id)
+    end
 
-    it "includes a suite's title if short_title not found" do
-      # test_result = create(:result, runnable: { test_suite_id: test_suite.id })
-      test_result = test_suite_result
+    it "include's a runnable's short_title if no short_id found" do
+      suites_repo.insert(suite_all)
+      test_result = create(:result, runnable: { test_suite_id: suite_all.id })
 
-      expect(instance.format_tag(test_result)).to match(test_suite.short_title)
+      expect(instance.format_tag(test_result)).to match(suite_all.short_title)
     end
 
-    it "includes a group's short id" do
-      test_result = create(:result, runnable: { test_group_id: test_group.id })
+    it "include's a runnable's title if no short_id/short_title found" do
+      suites_repo.insert(suite_no_short_title)
+      test_result = create(:result, runnable: { test_suite_id: suite_no_short_title.id })
 
-      expect(instance.format_tag(test_result)).to match(test_group.short_id)
+      expect(instance.format_tag(test_result)).to match(suite_no_short_title.title)
     end
 
-    it "includes a test's short it" do
-      test_result = create(:result, runnable: { test_id: test.id })
+    it "include's a runnable's id if no short_id/short_title/title found" do
+      suites_repo.insert(suite_id_only)
+      test_result = create(:result, runnable: { test_suite_id: suite_id_only.id })
 
-      expect(instance.format_tag(test_result)).to match(test.short_id)
+      expect(instance.format_tag(test_result)).to match(suite_id_only.id)
     end
   end
 

From 7cab803934d4590c424ff1dd8385bd296b7bc364 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 4 Sep 2024 11:01:31 -0400
Subject: [PATCH 63/83] add yard private tag to new utils

---
 lib/inferno/utils/persist_inputs.rb  | 1 +
 lib/inferno/utils/verify_runnable.rb | 1 +
 2 files changed, 2 insertions(+)

diff --git a/lib/inferno/utils/persist_inputs.rb b/lib/inferno/utils/persist_inputs.rb
index 6fd679a76..2703a0b71 100644
--- a/lib/inferno/utils/persist_inputs.rb
+++ b/lib/inferno/utils/persist_inputs.rb
@@ -1,5 +1,6 @@
 module Inferno
   module Utils
+    # @private
     module PersistInputs
       def persist_inputs(session_data_repo, params, test_run)
         available_inputs = test_run.runnable.available_inputs
diff --git a/lib/inferno/utils/verify_runnable.rb b/lib/inferno/utils/verify_runnable.rb
index df8622a60..c6c19d07e 100644
--- a/lib/inferno/utils/verify_runnable.rb
+++ b/lib/inferno/utils/verify_runnable.rb
@@ -2,6 +2,7 @@
 
 module Inferno
   module Utils
+    # @private
     module VerifyRunnable
       def verify_runnable(runnable, inputs, selected_suite_options)
         missing_inputs = runnable&.missing_inputs(inputs, selected_suite_options)

From c47714ac98ca7d5ea4a9c2b7acc4cfecbd95ae83 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 4 Sep 2024 11:07:45 -0400
Subject: [PATCH 64/83] impl console outputter class

---
 lib/inferno/apps/cli/execute.rb               | 139 +++--------------
 .../apps/cli/execute/abstract_outputter.rb    |  35 +++++
 .../apps/cli/execute/console_outputter.rb     | 141 ++++++++++++++++++
 3 files changed, 195 insertions(+), 120 deletions(-)
 create mode 100644 lib/inferno/apps/cli/execute/abstract_outputter.rb
 create mode 100644 lib/inferno/apps/cli/execute/console_outputter.rb

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index a36ea0db3..975e8fd15 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -6,6 +6,7 @@
 require_relative '../web/serializers/result'
 require_relative '../../utils/verify_runnable'
 require_relative '../../utils/persist_inputs'
+require_relative 'execute/console_outputter'
 
 module Inferno
   module CLI
@@ -13,10 +14,6 @@ class Execute
       include ::Inferno::Utils::VerifyRunnable
       include ::Inferno::Utils::PersistInputs
 
-      COLOR = Pastel.new
-      CHECKMARK = "\u2713"
-      BAR = '=========================================='
-
       attr_accessor :options
 
       def self.suppress_output
@@ -43,8 +40,8 @@ def run(options)
         print_help_and_exit if options[:help]
 
         self.options = options
-        print_start_message
-        verbose_puts 'options:', self.options
+
+        outputter.print_start_message(options)
 
         verify_runnable(
           runnable,
@@ -54,21 +51,22 @@ def run(options)
 
         persist_inputs(session_data_repo, create_params(test_session, runnable), test_run)
 
-        puts 'Running tests. This may take a while...' # TODO: spinner/progress bar
-
-        # TODO: hijack logger instead of using this if-case
-        if options[:verbose]
-          Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
-        else
-          Inferno::CLI::Execute.suppress_output do
+        outputter.print_around_run(options) do
+          # TODO: move suppression into outputter?
+          if options[:verbose]
             Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+          else
+            Inferno::CLI::Execute.suppress_output do
+              Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+            end
           end
         end
 
         results = test_runs_repo.results_for_test_run(test_run.id).reverse
 
-        verbose_print_json_results(results)
-        print_color_results(results)
+        outputter.print_results(options, results)
+
+        outputter.print_end_message(options)
 
         if results.find do |result|
              result.send(runnable_id_key) == options[runnable_type.to_sym]
@@ -96,13 +94,6 @@ def print_help_and_exit
         exit(0)
       end
 
-      def print_start_message
-        puts ''
-        puts BAR
-        puts "Testing #{options[:suite] || options[:group] || options[:test]}"
-        puts BAR
-      end
-
       def test_sessions_repo
         @test_sessions_repo ||= Inferno::Repositories::TestSessions.new
       end
@@ -201,107 +192,15 @@ def serialize(entity)
         end
       end
 
-      def verbose_print(*args)
-        print(COLOR.dim(*args)) if options[:verbose]
-      end
-
-      def verbose_puts(*args)
-        args.push("\n")
-        verbose_print(*args)
-      end
-
-      def format_tag(result)
-        if result.runnable.respond_to?(:short_id)
-          "#{result.runnable.short_id} #{format_tag_suffix(result)}"
-        else
-          format_tag_suffix(result)
-        end
-      end
-
-      def format_tag_suffix(result)
-        result.runnable.short_title.presence || result.runnable.title.presence || result.runnable.id
-      end
-
-      def format_messages(result)
-        result.messages.map do |message|
-          "\n\t\t#{message.type}: #{message.message}"
-        end.join
-      end
-
-      def format_requests(result)
-        result.requests.map do |req_res|
-          "\n\t\t#{req_res.status} #{req_res.verb.upcase} #{req_res.url}"
-        end.join
-      end
-
-      def format_session_data(result, attr)
-        json = result.send(attr)
-        return '' if json.nil?
-
-        JSON.parse(json).map do |hash|
-          "\n\t\t#{hash['name']}: #{hash['value']}"
-        end.join
-      end
-
-      def format_inputs(result)
-        format_session_data(result, :input_json)
-      end
-
-      def format_outputs(result)
-        format_session_data(result, :output_json)
-      end
-
-      def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
-        case result.result
-        when 'pass'
-          COLOR.bold.green(CHECKMARK, ' pass')
-        when 'fail'
-          COLOR.bold.red 'X fail'
-        when 'skip'
-          COLOR.yellow '* skip'
-        when 'omit'
-          COLOR.blue '* omit'
-        when 'error'
-          COLOR.magenta 'X error'
-        when 'wait'
-          COLOR.bold '. wait'
-        when 'cancel'
-          COLOR.red 'X cancel'
-        when 'running'
-          COLOR.bold '- running'
-        else
-          raise StandardError.new, "Unrecognized result #{result.result}"
-        end
-      end
-
-      def verbose_print_json_results(results)
-        verbose_puts BAR
-        verbose_puts 'JSON Test Results:'
-        verbose_puts BAR
-        verbose_puts serialize(results)
-        verbose_puts BAR
-      end
-
-      def print_color_results(results)
-        puts BAR
-        puts 'Test Results:'
-        puts BAR
-        results.each do |result|
-          print format_tag(result), ': ', format_result(result), "\n"
-          verbose_puts "\tsummary: ",   result.result_message
-          verbose_puts "\tmessages: ",  format_messages(result)
-          verbose_puts "\trequests: ",  format_requests(result)
-          verbose_puts "\tinputs: ",    format_inputs(result)
-          verbose_puts "\toutputs: ",   format_outputs(result)
-        end
-        puts BAR
-      end
-
       def print_error_and_exit(err, code)
-        # TODO: use Application Logger for stderr?
-        $stderr.puts COLOR.red "Error: #{err.full_message}" # rubocop:disable Style/StderrPuts # always print this error instead of using `warn`
+        outputter.print_error(err)
         exit(code)
       end
+
+      def outputter
+        # TODO: swap outputter based on options
+        @outputter ||= Inferno::CLI::Execute::ConsoleOutputter.new
+      end
     end
   end
 end
diff --git a/lib/inferno/apps/cli/execute/abstract_outputter.rb b/lib/inferno/apps/cli/execute/abstract_outputter.rb
new file mode 100644
index 000000000..e06af384f
--- /dev/null
+++ b/lib/inferno/apps/cli/execute/abstract_outputter.rb
@@ -0,0 +1,35 @@
+
+module Inferno
+  module CLI
+    class Execute
+
+      # Subclass AbstractOutputter to implement your own outputter
+      class AbstractOutputter
+
+        # @see Inferno::CLI::Main#execute for options
+        def print_start_message(options)
+        end
+
+        # Implementation MUST call `yield`
+        # @see Inferno::CLI::Main#execute for options
+        def print_around_run(options, &block)
+        end
+
+        # @param results [Array<Inferno::Entity::Result>]
+        # @see Inferno::CLI::Main#execute for options
+        def print_results(options, results)
+        end
+
+        # @see Inferno::CLI::Main#execute for options
+        def print_end_message(options)
+        end
+
+        # Implementation MUST NOT re-raise exception or exit
+        # @param exception [Exception]
+        def print_error(exception)
+        end
+
+      end
+    end
+  end
+end
diff --git a/lib/inferno/apps/cli/execute/console_outputter.rb b/lib/inferno/apps/cli/execute/console_outputter.rb
new file mode 100644
index 000000000..4c3674a01
--- /dev/null
+++ b/lib/inferno/apps/cli/execute/console_outputter.rb
@@ -0,0 +1,141 @@
+require 'pastel'
+require_relative 'abstract_outputter'
+
+module Inferno
+  module CLI
+    class Execute
+      # @private
+      class ConsoleOutputter  < AbstractOutputter
+
+        COLOR = Pastel.new
+        CHECKMARK = "\u2713".freeze
+        BAR = ('=' * 80).freeze
+
+        attr_accessor :verbose
+
+        def print_start_message(options)
+          self.verbose = options[:verbose]
+
+          puts ''
+          puts BAR
+          puts "Testing #{options[:suite] || options[:group] || options[:test]}"
+          puts BAR
+        end
+
+        def print_around_run(options, &block)
+          puts "Running tests. This may take a while..."
+          # TODO: spinner/progress bar
+          yield
+        end
+
+        def print_results(options, results)
+          verbose_print_json_results
+
+          puts BAR
+          puts 'Test Results:'
+          puts BAR
+          results.each do |result|
+            print format_tag(result), ': ', format_result(result), "\n"
+            verbose_puts "\tsummary: ",   result.result_message
+            verbose_puts "\tmessages: ",  format_messages(result)
+            verbose_puts "\trequests: ",  format_requests(result)
+            verbose_puts "\tinputs: ",    format_inputs(result)
+            verbose_puts "\toutputs: ",   format_outputs(result)
+          end
+          puts BAR
+        end
+
+        def print_end_message(options)
+        end
+
+        def print_error(exception)
+          puts COLOR.red "Error: #{exception.full_message}"
+          verbose_print(exception.backtrace.join('\n'))
+        end
+
+        private
+
+        def verbose_print(*args)
+          print(COLOR.dim(*args)) if self.verbose
+        end
+
+        def verbose_puts(*args)
+          args.push("\n")
+          verbose_print(*args)
+        end
+
+        def format_tag(result)
+          if result.runnable.respond_to?(:short_id)
+            "#{result.runnable.short_id} #{format_tag_suffix(result)}"
+          else
+            format_tag_suffix(result)
+          end
+        end
+
+        def format_tag_suffix(result)
+          result.runnable.short_title.presence || result.runnable.title.presence || result.runnable.id
+        end
+
+        def format_messages(result)
+          result.messages.map do |message|
+            "\n\t\t#{message.type}: #{message.message}"
+          end.join
+        end
+
+        def format_requests(result)
+          result.requests.map do |req_res|
+            "\n\t\t#{req_res.status} #{req_res.verb.upcase} #{req_res.url}"
+          end.join
+        end
+
+        def format_session_data(result, attr)
+          json = result.send(attr)
+          return '' if json.nil?
+
+          JSON.parse(json).map do |hash|
+            "\n\t\t#{hash['name']}: #{hash['value']}"
+          end.join
+        end
+
+        def format_inputs(result)
+          format_session_data(result, :input_json)
+        end
+
+        def format_outputs(result)
+          format_session_data(result, :output_json)
+        end
+
+        def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
+          case result.result
+          when 'pass'
+            COLOR.bold.green(CHECKMARK, ' pass')
+          when 'fail'
+            COLOR.bold.red 'X fail'
+          when 'skip'
+            COLOR.yellow '* skip'
+          when 'omit'
+            COLOR.blue '* omit'
+          when 'error'
+            COLOR.magenta 'X error'
+          when 'wait'
+            COLOR.bold '. wait'
+          when 'cancel'
+            COLOR.red 'X cancel'
+          when 'running'
+            COLOR.bold '- running'
+          else
+            raise StandardError.new, "Unrecognized result #{result.result}"
+          end
+        end
+
+        def verbose_print_json_results(results)
+          verbose_puts BAR
+          verbose_puts 'JSON Test Results:'
+          verbose_puts BAR
+          verbose_puts serialize(results)
+          verbose_puts BAR
+        end
+      end
+    end
+  end
+end

From 86db812054f1461c4725e95da84336cbb942ead0 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 4 Sep 2024 11:19:02 -0400
Subject: [PATCH 65/83] debugging

---
 lib/inferno/apps/cli/execute.rb                | 15 ---------------
 .../apps/cli/execute/console_outputter.rb      | 18 +++++++++++++++++-
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 975e8fd15..ea1a4d485 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -2,8 +2,6 @@
 
 require 'pastel'
 require 'active_support'
-require_relative '../web/serializers/test_run'
-require_relative '../web/serializers/result'
 require_relative '../../utils/verify_runnable'
 require_relative '../../utils/persist_inputs'
 require_relative 'execute/console_outputter'
@@ -179,19 +177,6 @@ def create_params(test_session, runnable)
         }
       end
 
-      def serialize(entity)
-        case entity.class.to_s
-        when 'Array'
-          JSON.pretty_generate(entity.map { |item| JSON.parse serialize(item) })
-        when lambda { |x|
-               defined?(x.constantize) && defined?("Inferno::Web::Serializers::#{x.split('::').last}".constantize)
-             }
-          "Inferno::Web::Serializers::#{entity.class.to_s.split('::').last}".constantize.render(entity)
-        else
-          raise StandardError, "CLI does not know how to serialize #{entity.class}"
-        end
-      end
-
       def print_error_and_exit(err, code)
         outputter.print_error(err)
         exit(code)
diff --git a/lib/inferno/apps/cli/execute/console_outputter.rb b/lib/inferno/apps/cli/execute/console_outputter.rb
index 4c3674a01..ba6edbafa 100644
--- a/lib/inferno/apps/cli/execute/console_outputter.rb
+++ b/lib/inferno/apps/cli/execute/console_outputter.rb
@@ -1,5 +1,7 @@
 require 'pastel'
 require_relative 'abstract_outputter'
+require_relative '../../web/serializers/test_run'
+require_relative '../../web/serializers/result'
 
 module Inferno
   module CLI
@@ -29,7 +31,7 @@ def print_around_run(options, &block)
         end
 
         def print_results(options, results)
-          verbose_print_json_results
+          verbose_print_json_results(results)
 
           puts BAR
           puts 'Test Results:'
@@ -135,6 +137,20 @@ def verbose_print_json_results(results)
           verbose_puts serialize(results)
           verbose_puts BAR
         end
+
+        def serialize(entity)
+          case entity.class.to_s
+          when 'Array'
+            JSON.pretty_generate(entity.map { |item| JSON.parse serialize(item) })
+          when lambda { |x|
+                 defined?(x.constantize) && defined?("Inferno::Web::Serializers::#{x.split('::').last}".constantize)
+               }
+            "Inferno::Web::Serializers::#{entity.class.to_s.split('::').last}".constantize.render(entity)
+          else
+            raise StandardError, "CLI does not know how to serialize #{entity.class}"
+          end
+        end
+  
       end
     end
   end

From 94d6a795b985010021f807dd152a60e2ebf6850e Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 4 Sep 2024 17:16:56 -0400
Subject: [PATCH 66/83] debug

---
 lib/inferno/apps/cli/execute.rb               |   2 +-
 .../apps/cli/execute/abstract_outputter.rb    |  10 +-
 .../apps/cli/execute/console_outputter.rb     |  44 ++--
 .../cli/execute/console_outputter_spec.rb     | 199 +++++++++++++++++
 spec/inferno/apps/cli/execute_spec.rb         | 211 ------------------
 5 files changed, 228 insertions(+), 238 deletions(-)
 create mode 100644 spec/inferno/apps/cli/execute/console_outputter_spec.rb

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index ea1a4d485..e86b3c8c6 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -178,7 +178,7 @@ def create_params(test_session, runnable)
       end
 
       def print_error_and_exit(err, code)
-        outputter.print_error(err)
+        outputter.print_error(self.options || {}, err)
         exit(code)
       end
 
diff --git a/lib/inferno/apps/cli/execute/abstract_outputter.rb b/lib/inferno/apps/cli/execute/abstract_outputter.rb
index e06af384f..0774e710e 100644
--- a/lib/inferno/apps/cli/execute/abstract_outputter.rb
+++ b/lib/inferno/apps/cli/execute/abstract_outputter.rb
@@ -8,25 +8,31 @@ class AbstractOutputter
 
         # @see Inferno::CLI::Main#execute for options
         def print_start_message(options)
+          raise StandardError, 'not implemented'
         end
 
         # Implementation MUST call `yield`
         # @see Inferno::CLI::Main#execute for options
         def print_around_run(options, &block)
+          raise StandardError, 'not implemented'
         end
 
-        # @param results [Array<Inferno::Entity::Result>]
         # @see Inferno::CLI::Main#execute for options
+        # @param results [Array<Inferno::Entity::Result>]
         def print_results(options, results)
+          raise StandardError, 'not implemented'
         end
 
         # @see Inferno::CLI::Main#execute for options
         def print_end_message(options)
+          raise StandardError, 'not implemented'
         end
 
         # Implementation MUST NOT re-raise exception or exit
+        # @see Inferno::CLI::Main#execute for options
         # @param exception [Exception]
-        def print_error(exception)
+        def print_error(options, exception)
+          raise StandardError, 'not implemented'
         end
 
       end
diff --git a/lib/inferno/apps/cli/execute/console_outputter.rb b/lib/inferno/apps/cli/execute/console_outputter.rb
index ba6edbafa..d68f94396 100644
--- a/lib/inferno/apps/cli/execute/console_outputter.rb
+++ b/lib/inferno/apps/cli/execute/console_outputter.rb
@@ -7,17 +7,13 @@ module Inferno
   module CLI
     class Execute
       # @private
-      class ConsoleOutputter  < AbstractOutputter
+      class ConsoleOutputter < AbstractOutputter
 
         COLOR = Pastel.new
         CHECKMARK = "\u2713".freeze
         BAR = ('=' * 80).freeze
 
-        attr_accessor :verbose
-
         def print_start_message(options)
-          self.verbose = options[:verbose]
-
           puts ''
           puts BAR
           puts "Testing #{options[:suite] || options[:group] || options[:test]}"
@@ -31,18 +27,18 @@ def print_around_run(options, &block)
         end
 
         def print_results(options, results)
-          verbose_print_json_results(results)
+          verbose_print_json_results(options, results)
 
           puts BAR
           puts 'Test Results:'
           puts BAR
           results.each do |result|
             print format_tag(result), ': ', format_result(result), "\n"
-            verbose_puts "\tsummary: ",   result.result_message
-            verbose_puts "\tmessages: ",  format_messages(result)
-            verbose_puts "\trequests: ",  format_requests(result)
-            verbose_puts "\tinputs: ",    format_inputs(result)
-            verbose_puts "\toutputs: ",   format_outputs(result)
+            verbose_puts(options, "\tsummary: ",     result.result_message)
+            verbose_puts(options, "\tmessages: ",    format_messages(result))
+            verbose_puts(options, "\trequests: ",    format_requests(result))
+            verbose_puts(options, "\tinputs: ",      format_inputs(result))
+            verbose_puts(options, "\toutputs: ",     format_outputs(result))
           end
           puts BAR
         end
@@ -50,20 +46,20 @@ def print_results(options, results)
         def print_end_message(options)
         end
 
-        def print_error(exception)
+        def print_error(options, exception)
           puts COLOR.red "Error: #{exception.full_message}"
-          verbose_print(exception.backtrace.join('\n'))
+          verbose_print(exception.backtrace&.join('\n'))
         end
 
-        private
+        # private
 
-        def verbose_print(*args)
-          print(COLOR.dim(*args)) if self.verbose
+        def verbose_print(options, *args)
+          print(COLOR.dim(*args)) if options[:verbose]
         end
 
-        def verbose_puts(*args)
+        def verbose_puts(options, *args)
           args.push("\n")
-          verbose_print(*args)
+          verbose_print(options, *args)
         end
 
         def format_tag(result)
@@ -130,12 +126,12 @@ def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity
           end
         end
 
-        def verbose_print_json_results(results)
-          verbose_puts BAR
-          verbose_puts 'JSON Test Results:'
-          verbose_puts BAR
-          verbose_puts serialize(results)
-          verbose_puts BAR
+        def verbose_print_json_results(options, results)
+          verbose_puts(options, BAR)
+          verbose_puts(options, 'JSON Test Results:')
+          verbose_puts(options, BAR)
+          verbose_puts(options, serialize(results))
+          verbose_puts(options, BAR)
         end
 
         def serialize(entity)
diff --git a/spec/inferno/apps/cli/execute/console_outputter_spec.rb b/spec/inferno/apps/cli/execute/console_outputter_spec.rb
new file mode 100644
index 000000000..5cb42e21a
--- /dev/null
+++ b/spec/inferno/apps/cli/execute/console_outputter_spec.rb
@@ -0,0 +1,199 @@
+require_relative '../../../../../lib/inferno/apps/cli/execute/console_outputter'
+
+RSpec.describe Inferno::CLI::Execute::ConsoleOutputter do # rubocop:disable RSpec/FilePath
+  let(:instance) { described_class.new }
+  let(:options) { {verbose: true} }
+
+  describe '#serialize' do
+    let(:test_results) { create_list(:result, 2) }
+
+    it 'handles an array of test results without raising exception' do
+      expect { instance.serialize(test_results) }.to_not raise_error(StandardError)
+    end
+
+    it 'returns valid JSON' do
+      expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(StandardError)
+    end
+  end
+
+  describe '#verbose_print' do
+    it 'outputs when verbose is true' do
+      expect { instance.verbose_print(options, 'Lorem') }.to output(/Lorem/).to_stdout
+    end
+
+    it 'does not output when verbose is false' do
+      expect { instance.verbose_print({verbose: false}, 'Lorem') }.to_not output(/.+/).to_stdout
+    end
+  end
+
+  describe '#verbose_puts' do
+    it 'has output ending with \n with when verbose is true' do
+      expect { instance.verbose_puts(options, 'Lorem') }.to output(/Lorem\n/).to_stdout
+    end
+  end
+
+  describe '#format_tag' do
+    let(:suites_repo) { Inferno::Repositories::TestSuites.new }
+
+    let(:suite_all) do
+      Class.new(Inferno::TestSuite) do
+        id 'mock_suite_id_1'
+        short_title 'short'
+        title 'title'
+      end
+    end
+
+    let(:suite_no_short_title) do
+      Class.new(Inferno::TestSuite) do
+        id 'mock_suite_id_2'
+        title 'title'
+      end
+    end
+
+    let(:suite_id_only) do
+      Class.new(Inferno::TestSuite) do
+        id 'mock_suite_id_3'
+      end
+    end
+
+    let(:groups_repo) { Inferno::Repositories::TestGroups.new }
+
+    let(:group_all) do
+      Class.new(Inferno::TestGroup) do
+        id 'mock_group_id_1'
+        short_title 'short'
+        title 'title'
+      end
+    end
+
+    let(:group_no_short_title) do
+      Class.new(Inferno::TestGroup) do
+        id 'mock_group_id_2'
+        title 'title'
+      end
+    end
+
+    let(:group_no_titles) do
+      Class.new(Inferno::TestGroup) do
+        id 'mock_group_id_3'
+      end
+    end
+
+    it "includes a runnable's short_id and short_title if possible" do
+      groups_repo.insert(group_all)
+      test_result = create(:result, runnable: { test_group_id: group_all.id })
+
+      expect(instance.format_tag(test_result)).to match(group_all.short_id)
+      expect(instance.format_tag(test_result)).to match(group_all.short_title)
+    end
+
+    it "includes a runnable's short_id and title if no short_title found" do
+      groups_repo.insert(group_no_short_title)
+      test_result = create(:result, runnable: { test_group_id: group_no_short_title.id })
+
+      expect(instance.format_tag(test_result)).to match(group_no_short_title.short_id)
+      expect(instance.format_tag(test_result)).to match(group_no_short_title.title)
+    end
+
+    it "includes a runnable's short_id and id if no title/short_title found" do
+      groups_repo.insert(group_no_titles)
+      test_result = create(:result, runnable: { test_group_id: group_no_titles.id })
+
+      expect(instance.format_tag(test_result)).to match(group_no_titles.short_id)
+      expect(instance.format_tag(test_result)).to match(group_no_titles.id)
+    end
+
+    it "include's a runnable's short_title if no short_id found" do
+      suites_repo.insert(suite_all)
+      test_result = create(:result, runnable: { test_suite_id: suite_all.id })
+
+      expect(instance.format_tag(test_result)).to match(suite_all.short_title)
+    end
+
+    it "include's a runnable's title if no short_id/short_title found" do
+      suites_repo.insert(suite_no_short_title)
+      test_result = create(:result, runnable: { test_suite_id: suite_no_short_title.id })
+
+      expect(instance.format_tag(test_result)).to match(suite_no_short_title.title)
+    end
+
+    it "include's a runnable's id if no short_id/short_title/title found" do
+      suites_repo.insert(suite_id_only)
+      test_result = create(:result, runnable: { test_suite_id: suite_id_only.id })
+
+      expect(instance.format_tag(test_result)).to match(suite_id_only.id)
+    end
+  end
+
+  describe '#format_messages' do
+    let(:test_result) { repo_create(:result, message_count: 10) }
+
+    it 'includes all characters' do
+      messages = test_result.messages
+      formatted_string = instance.format_messages(test_result)
+
+      messages.each do |message|
+        expect(formatted_string).to include message.message
+      end
+    end
+  end
+
+  describe '#format_requests' do
+    let(:test_result) { repo_create(:result, request_count: 10) }
+
+    it 'includes all status codes' do
+      requests = test_result.requests
+      formatted_string = instance.format_requests(test_result)
+
+      requests.each do |request|
+        expect(formatted_string).to include request.status.to_s
+      end
+    end
+  end
+
+  describe '#format_session_data' do
+    let(:data) { [{ name: :url, value: 'https://example.com' }, { name: :token, value: 'SAMPLE_OUTPUT' }] }
+    let(:test_result) { create(:result, input_json: JSON.generate(data), output_json: JSON.generate(data)) }
+
+    it 'includes all values for input_json' do
+      formatted_string = instance.format_session_data(test_result, :input_json)
+      data.each do |data_element|
+        expect(formatted_string).to include data_element[:value]
+      end
+    end
+
+    it 'includes all values for output_json' do
+      formatted_string = instance.format_session_data(test_result, :output_json)
+      data.each do |data_element|
+        expect(formatted_string).to include data_element[:value]
+      end
+    end
+  end
+
+  describe '#format_result' do
+    Inferno::Entities::Result::RESULT_OPTIONS.each do |result_option|
+      it "can format #{result_option} result type" do
+        result = create(:result, result: result_option)
+        expect { instance.format_result(result) }.to_not raise_error
+      end
+
+      it 'includes result type in return value' do
+        result = create(:result, result: result_option)
+        expect(instance.format_result(result).upcase).to include result_option.upcase
+      end
+    end
+  end
+
+  describe '#print_color_results' do
+    let(:results) { create_list(:random_result, 10) }
+
+    it 'outputs something with verbose false' do
+      expect { instance.print_results({verbose: false}, results) }.to output(/.+/).to_stdout
+    end
+
+    it 'outputs something with verbose true' do
+      expect { instance.print_results(options, results) }.to output(/.+/).to_stdout
+    end
+  end
+end
+
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 34ea22ff9..85a1032ea 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -108,217 +108,6 @@
     end
   end
 
-  describe '#serialize' do
-    let(:test_results) { create_list(:result, 2) }
-
-    it 'handles an array of test results without raising exception' do
-      expect { instance.serialize(test_results) }.to_not raise_error(StandardError)
-    end
-
-    it 'returns valid JSON' do
-      expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(StandardError)
-    end
-  end
-
-  describe '#verbose_print' do
-    it 'outputs when verbose is true' do
-      allow(instance).to receive(:options).and_return({ verbose: true })
-
-      expect { instance.verbose_print('Lorem') }.to output(/Lorem/).to_stdout
-    end
-
-    it 'does not output when verbose is false' do
-      allow(instance).to receive(:options).and_return({ verbose: false })
-
-      expect { instance.verbose_print('Lorem') }.to_not output(/.+/).to_stdout
-    end
-  end
-
-  describe '#verbose_puts' do
-    it 'has output ending with \n with when verbose is true' do
-      allow(instance).to receive(:options).and_return({ verbose: true })
-
-      expect { instance.verbose_puts('Lorem') }.to output(/Lorem\n/).to_stdout
-    end
-  end
-
-  describe '#format_tag' do
-    let(:suites_repo) { Inferno::Repositories::TestSuites.new }
-
-    let(:suite_all) do
-      Class.new(Inferno::TestSuite) do
-        id 'mock_suite_id_1'
-        short_title 'short'
-        title 'title'
-      end
-    end
-
-    let(:suite_no_short_title) do
-      Class.new(Inferno::TestSuite) do
-        id 'mock_suite_id_2'
-        title 'title'
-      end
-    end
-
-    let(:suite_id_only) do
-      Class.new(Inferno::TestSuite) do
-        id 'mock_suite_id_3'
-      end
-    end
-
-    let(:groups_repo) { Inferno::Repositories::TestGroups.new }
-
-    let(:group_all) do
-      Class.new(Inferno::TestGroup) do
-        id 'mock_group_id_1'
-        short_title 'short'
-        title 'title'
-      end
-    end
-
-    let(:group_no_short_title) do
-      Class.new(Inferno::TestGroup) do
-        id 'mock_group_id_2'
-        title 'title'
-      end
-    end
-
-    let(:group_no_titles) do
-      Class.new(Inferno::TestGroup) do
-        id 'mock_group_id_3'
-      end
-    end
-
-    it "includes a runnable's short_id and short_title if possible" do
-      groups_repo.insert(group_all)
-      test_result = create(:result, runnable: { test_group_id: group_all.id })
-
-      expect(instance.format_tag(test_result)).to match(group_all.short_id)
-      expect(instance.format_tag(test_result)).to match(group_all.short_title)
-    end
-
-    it "includes a runnable's short_id and title if no short_title found" do
-      groups_repo.insert(group_no_short_title)
-      test_result = create(:result, runnable: { test_group_id: group_no_short_title.id })
-
-      expect(instance.format_tag(test_result)).to match(group_no_short_title.short_id)
-      expect(instance.format_tag(test_result)).to match(group_no_short_title.title)
-    end
-
-    it "includes a runnable's short_id and id if no title/short_title found" do
-      groups_repo.insert(group_no_titles)
-      test_result = create(:result, runnable: { test_group_id: group_no_titles.id })
-
-      expect(instance.format_tag(test_result)).to match(group_no_titles.short_id)
-      expect(instance.format_tag(test_result)).to match(group_no_titles.id)
-    end
-
-    it "include's a runnable's short_title if no short_id found" do
-      suites_repo.insert(suite_all)
-      test_result = create(:result, runnable: { test_suite_id: suite_all.id })
-
-      expect(instance.format_tag(test_result)).to match(suite_all.short_title)
-    end
-
-    it "include's a runnable's title if no short_id/short_title found" do
-      suites_repo.insert(suite_no_short_title)
-      test_result = create(:result, runnable: { test_suite_id: suite_no_short_title.id })
-
-      expect(instance.format_tag(test_result)).to match(suite_no_short_title.title)
-    end
-
-    it "include's a runnable's id if no short_id/short_title/title found" do
-      suites_repo.insert(suite_id_only)
-      test_result = create(:result, runnable: { test_suite_id: suite_id_only.id })
-
-      expect(instance.format_tag(test_result)).to match(suite_id_only.id)
-    end
-  end
-
-  describe '#format_messages' do
-    let(:test_result) { repo_create(:result, message_count: 10) }
-
-    it 'includes all characters' do
-      messages = test_result.messages
-      formatted_string = instance.format_messages(test_result)
-
-      messages.each do |message|
-        expect(formatted_string).to include message.message
-      end
-    end
-  end
-
-  describe '#format_requests' do
-    let(:test_result) { repo_create(:result, request_count: 10) }
-
-    it 'includes all status codes' do
-      requests = test_result.requests
-      formatted_string = instance.format_requests(test_result)
-
-      requests.each do |request|
-        expect(formatted_string).to include request.status.to_s
-      end
-    end
-  end
-
-  describe '#format_session_data' do
-    let(:data) { [{ name: :url, value: 'https://example.com' }, { name: :token, value: 'SAMPLE_OUTPUT' }] }
-    let(:test_result) { create(:result, input_json: JSON.generate(data), output_json: JSON.generate(data)) }
-
-    it 'includes all values for input_json' do
-      formatted_string = instance.format_session_data(test_result, :input_json)
-      data.each do |data_element|
-        expect(formatted_string).to include data_element[:value]
-      end
-    end
-
-    it 'includes all values for output_json' do
-      formatted_string = instance.format_session_data(test_result, :output_json)
-      data.each do |data_element|
-        expect(formatted_string).to include data_element[:value]
-      end
-    end
-  end
-
-  describe '#format_result' do
-    Inferno::Entities::Result::RESULT_OPTIONS.each do |result_option|
-      it "can format #{result_option} result type" do
-        result = create(:result, result: result_option)
-        expect { instance.format_result(result) }.to_not raise_error
-      end
-
-      it 'includes result type in return value' do
-        result = create(:result, result: result_option)
-        expect(instance.format_result(result).upcase).to include result_option.upcase
-      end
-    end
-  end
-
-  describe '#print_color_results' do
-    let(:results) { create_list(:random_result, 10) }
-
-    it 'outputs something with verbose false' do
-      allow(instance).to receive(:options).and_return({ verbose: false })
-      expect { instance.print_color_results(results) }.to output(/.+/).to_stdout
-    end
-
-    it 'outputs something with verbose true' do
-      allow(instance).to receive(:options).and_return({ verbose: true })
-      expect { instance.print_color_results(results) }.to output(/.+/).to_stdout
-    end
-  end
-
-  describe '#print_error_and_exit' do
-    let(:mock_error_class) { Class.new(StandardError) }
-    let(:mock_error) { mock_error_class.new('mock message') }
-
-    it 'outputs to stderr and exits' do
-      expect do
-        expect { instance.print_error_and_exit(mock_error, 2) }.to output(/Error/).to_stderr
-      end.to raise_error(SystemExit)
-    end
-  end
-
   describe '#run' do
     let(:suite) { 'dev_validator' }
     let(:session_data_repo) { Inferno::Repositories::SessionData.new }

From 641496c5b62cc14d21266f8b999c449f13bc07e8 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 4 Sep 2024 17:25:57 -0400
Subject: [PATCH 67/83] cop

---
 lib/inferno/apps/cli/execute.rb                       |  2 +-
 lib/inferno/apps/cli/execute/abstract_outputter.rb    |  8 +++-----
 lib/inferno/apps/cli/execute/console_outputter.rb     | 11 ++++-------
 .../apps/cli/execute/console_outputter_spec.rb        |  7 +++----
 4 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index e86b3c8c6..eacc4356c 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -178,7 +178,7 @@ def create_params(test_session, runnable)
       end
 
       def print_error_and_exit(err, code)
-        outputter.print_error(self.options || {}, err)
+        outputter.print_error(options || {}, err)
         exit(code)
       end
 
diff --git a/lib/inferno/apps/cli/execute/abstract_outputter.rb b/lib/inferno/apps/cli/execute/abstract_outputter.rb
index 0774e710e..47e23870f 100644
--- a/lib/inferno/apps/cli/execute/abstract_outputter.rb
+++ b/lib/inferno/apps/cli/execute/abstract_outputter.rb
@@ -1,11 +1,9 @@
-
 module Inferno
   module CLI
     class Execute
-
       # Subclass AbstractOutputter to implement your own outputter
+      # rubocop:disable Lint/UnusedMethodArgument
       class AbstractOutputter
-
         # @see Inferno::CLI::Main#execute for options
         def print_start_message(options)
           raise StandardError, 'not implemented'
@@ -13,7 +11,7 @@ def print_start_message(options)
 
         # Implementation MUST call `yield`
         # @see Inferno::CLI::Main#execute for options
-        def print_around_run(options, &block)
+        def print_around_run(options, &)
           raise StandardError, 'not implemented'
         end
 
@@ -34,8 +32,8 @@ def print_end_message(options)
         def print_error(options, exception)
           raise StandardError, 'not implemented'
         end
-
       end
+      # rubocop:enable Lint/UnusedMethodArgument
     end
   end
 end
diff --git a/lib/inferno/apps/cli/execute/console_outputter.rb b/lib/inferno/apps/cli/execute/console_outputter.rb
index d68f94396..b5b3e0ab0 100644
--- a/lib/inferno/apps/cli/execute/console_outputter.rb
+++ b/lib/inferno/apps/cli/execute/console_outputter.rb
@@ -8,7 +8,6 @@ module CLI
     class Execute
       # @private
       class ConsoleOutputter < AbstractOutputter
-
         COLOR = Pastel.new
         CHECKMARK = "\u2713".freeze
         BAR = ('=' * 80).freeze
@@ -20,8 +19,8 @@ def print_start_message(options)
           puts BAR
         end
 
-        def print_around_run(options, &block)
-          puts "Running tests. This may take a while..."
+        def print_around_run(_options)
+          puts 'Running tests. This may take a while...'
           # TODO: spinner/progress bar
           yield
         end
@@ -43,10 +42,9 @@ def print_results(options, results)
           puts BAR
         end
 
-        def print_end_message(options)
-        end
+        def print_end_message(options); end
 
-        def print_error(options, exception)
+        def print_error(_options, exception)
           puts COLOR.red "Error: #{exception.full_message}"
           verbose_print(exception.backtrace&.join('\n'))
         end
@@ -146,7 +144,6 @@ def serialize(entity)
             raise StandardError, "CLI does not know how to serialize #{entity.class}"
           end
         end
-  
       end
     end
   end
diff --git a/spec/inferno/apps/cli/execute/console_outputter_spec.rb b/spec/inferno/apps/cli/execute/console_outputter_spec.rb
index 5cb42e21a..4b9092863 100644
--- a/spec/inferno/apps/cli/execute/console_outputter_spec.rb
+++ b/spec/inferno/apps/cli/execute/console_outputter_spec.rb
@@ -2,7 +2,7 @@
 
 RSpec.describe Inferno::CLI::Execute::ConsoleOutputter do # rubocop:disable RSpec/FilePath
   let(:instance) { described_class.new }
-  let(:options) { {verbose: true} }
+  let(:options) { { verbose: true } }
 
   describe '#serialize' do
     let(:test_results) { create_list(:result, 2) }
@@ -22,7 +22,7 @@
     end
 
     it 'does not output when verbose is false' do
-      expect { instance.verbose_print({verbose: false}, 'Lorem') }.to_not output(/.+/).to_stdout
+      expect { instance.verbose_print({ verbose: false }, 'Lorem') }.to_not output(/.+/).to_stdout
     end
   end
 
@@ -188,7 +188,7 @@
     let(:results) { create_list(:random_result, 10) }
 
     it 'outputs something with verbose false' do
-      expect { instance.print_results({verbose: false}, results) }.to output(/.+/).to_stdout
+      expect { instance.print_results({ verbose: false }, results) }.to output(/.+/).to_stdout
     end
 
     it 'outputs something with verbose true' do
@@ -196,4 +196,3 @@
     end
   end
 end
-

From 7fbd56471f6ec78cacbf57445005b76e7753189c Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 10 Sep 2024 13:37:29 -0400
Subject: [PATCH 68/83] refactor code to take short ids

---
 lib/inferno/apps/cli/execute.rb | 145 ++++++++++++++++++++++++--------
 lib/inferno/apps/cli/main.rb    |  14 +--
 2 files changed, 118 insertions(+), 41 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index eacc4356c..7ebdf9edd 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'pastel'
 require 'active_support'
 require_relative '../../utils/verify_runnable'
@@ -41,36 +39,42 @@ def run(options)
 
         outputter.print_start_message(options)
 
-        verify_runnable(
-          runnable,
-          thor_hash_to_inputs_array(options[:inputs]),
-          test_session.suite_options
-        )
+        results = []
+        outputter.print_around_run(options) do
+          if runnables.empty? # TODO better cond?
+            # TODO factorize:
+            verify_runnable(
+              suite,
+              thor_hash_to_inputs_array(options[:inputs]),
+              test_session.suite_options
+            )
 
-        persist_inputs(session_data_repo, create_params(test_session, runnable), test_run)
+            persist_inputs(session_data_repo, create_params(test_session, suite), test_run(suite)) # NOTE: diff
 
-        outputter.print_around_run(options) do
-          # TODO: move suppression into outputter?
-          if options[:verbose]
-            Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+            dispatch_job(test_run(suite))
+
+            results = test_runs_repo.results_for_test_run(test_run(suite).id).reverse
           else
-            Inferno::CLI::Execute.suppress_output do
-              Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+            runnables.each do |runnable|
+                verify_runnable(
+                  suite,
+                  thor_hash_to_inputs_array(options[:inputs]),
+                  test_session.suite_options
+                )
+
+                persist_inputs(session_data_repo, create_params(test_session, suite), test_run(runnable))
+
+                dispatch_job(test_run(runnable))
+                results += test_runs_repo.results_for_test_run(test_run(runnable).id).reverse # NOTE: diff
             end
           end
         end
 
-        results = test_runs_repo.results_for_test_run(test_run.id).reverse
-
         outputter.print_results(options, results)
 
         outputter.print_end_message(options)
 
-        if results.find do |result|
-             result.send(runnable_id_key) == options[runnable_type.to_sym]
-           end.result == 'pass'
-          exit(0)
-        end
+        exit(0) if results.all? { |result| result.result == 'pass' }
 
         # exit(1) is for Thor failures
         # exit(2) is for shell builtin failures
@@ -92,6 +96,17 @@ def print_help_and_exit
         exit(0)
       end
 
+      def dispatch_job(test_run)
+        # TODO: move suppression to outputter? better suppresion?
+        if options[:verbose]
+          Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+        else
+          Inferno::CLI::Execute.suppress_output do
+            Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+          end
+        end
+      end
+
       def test_sessions_repo
         @test_sessions_repo ||= Inferno::Repositories::TestSessions.new
       end
@@ -106,19 +121,81 @@ def test_runs_repo
 
       def test_session
         @test_session ||= test_sessions_repo.create({
-                                                      test_suite_id: runnable.suite.id,
+                                                      test_suite_id: suite.id,
                                                       suite_options: thor_hash_to_suite_options_array(
                                                         options[:suite_options]
                                                       )
                                                     })
       end
 
-      def test_run
-        @test_run ||= test_runs_repo.create(
-          create_params(test_session, runnable).merge({ status: 'queued' })
+      def test_run(runnable_param)
+        @test_runs ||= {}
+
+        @test_runs[runnable_param] ||= test_runs_repo.create(
+          create_params(test_session, runnable_param).merge({ status: 'queued' })
         )
+
+        @test_runs[runnable_param]
       end
 
+      # TODO run per
+      def create_params(test_session, runnable)
+        {
+          test_session_id: test_session.id,
+          runnable_id_key(runnable) => runnable.id,
+          inputs: thor_hash_to_inputs_array(options[:inputs])
+        }
+      end
+
+      def suite
+        @suite ||= Inferno::Repositories::TestSuites.new.find(options[:suite])
+
+        raise StandardError, "Test suite #{options[:suite]} not found" if @suite.nil?
+
+        @suite
+      end
+
+      def groups
+        @groups ||= Inferno::Repositories::TestGroups.new.all.select do |group|
+          options[:groups]&.include?(group.short_id) && group.suite.id == suite.id
+        end
+      end
+
+      def tests
+        @tests ||= Inferno::Repositories::Tests.new.all.select do |test|
+          options[:tests]&.include?(test.short_id) && test.suite.id == suite.id
+        end
+      end
+
+      def runnables
+        groups + tests
+      end
+
+      def runnable_type(runnable)
+        if Inferno::TestSuite.subclasses.include? runnable
+          :suite
+        elsif Inferno::TestGroup.subclasses.include? runnable
+          :group
+        elsif Inferno::Test.subclasses.include? runnable
+          :test
+        else
+          raise StandardError, "Unidentified runnable #{runnable}"
+        end
+      end
+
+      def runnable_id_key(runnable)
+        case runnable_type(runnable)
+        when :suite
+          :test_suite_id
+        when :group
+          :test_group_id
+        else
+          :test_id
+        end
+      end
+
+=begin
+      # TODO: redo/rm this for short ids in groups and tests
       def runnable
         @runnable ||=
           case runnable_type
@@ -160,6 +237,7 @@ def runnable_id_key
           raise StandardError, "Unrecognized runnable type #{runnable_type}"
         end
       end
+=end
 
       def thor_hash_to_suite_options_array(hash = {})
         hash.to_a.map { |pair| Inferno::DSL::SuiteOption.new({ id: pair[0], value: pair[1] }) }
@@ -169,17 +247,16 @@ def thor_hash_to_inputs_array(hash = {})
         hash.to_a.map { |pair| { name: pair[0], value: pair[1] } }
       end
 
-      def create_params(test_session, runnable)
-        {
-          test_session_id: test_session.id,
-          runnable_id_key => runnable.id,
-          inputs: thor_hash_to_inputs_array(options[:inputs])
-        }
-      end
 
       def print_error_and_exit(err, code)
-        outputter.print_error(options || {}, err)
-        exit(code)
+        # TODO FIXME
+        begin
+          outputter.print_error(options || {}, err)
+        rescue Exception => outputter_err
+          $stderr.puts "Caught exception #{outputter_err} while printing exception #{err}. Exiting."
+        ensure
+          exit(code)
+        end
       end
 
       def outputter
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index f6241d40e..4597e7019 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -86,20 +86,20 @@ def version
       option :suite,
              aliases: ['-s'],
              type: :string,
-             desc: 'Test suite id to run, mutually exclusive with --group and --test',
+             desc: 'Test suite id to run or to select groups and tests from',
              banner: 'id'
       option :suite_options,
              aliases: ['-o'],
              type: :hash,
              desc: 'Suite options'
-      option :group,
+      option :groups,
              aliases: ['-g'],
-             type: :string,
-             desc: 'Test group id to run, mutually exclusive with --suite and --test' # TODO: make them work together
-      option :test,
+             type: :array,
+             desc: 'Comma delimited test group short ids (AKA sequence number) to run, requires suite'
+      option :tests,
              aliases: ['-t'],
-             type: :string,
-             desc: 'Test id to run, mutually exclusive with --suite and --group'
+             type: :array,
+             desc: 'Comma delimited test short ids (AKA sequence number) to run, requires suite'
       option :inputs,
              aliases: ['-i'],
              type: :hash,

From e481fca7cd7a87b459a670a61eb64a7d13ef1750 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 10 Sep 2024 13:58:19 -0400
Subject: [PATCH 69/83] cleaning

---
 lib/inferno/apps/cli/execute.rb | 145 ++++++++++----------------------
 1 file changed, 45 insertions(+), 100 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 7ebdf9edd..56d8c8269 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -41,31 +41,13 @@ def run(options)
 
         results = []
         outputter.print_around_run(options) do
-          if runnables.empty? # TODO better cond?
-            # TODO factorize:
-            verify_runnable(
-              suite,
-              thor_hash_to_inputs_array(options[:inputs]),
-              test_session.suite_options
-            )
-
-            persist_inputs(session_data_repo, create_params(test_session, suite), test_run(suite)) # NOTE: diff
-
-            dispatch_job(test_run(suite))
-
+          if selected_runnables.empty?
+            run_one(suite)
             results = test_runs_repo.results_for_test_run(test_run(suite).id).reverse
           else
-            runnables.each do |runnable|
-                verify_runnable(
-                  suite,
-                  thor_hash_to_inputs_array(options[:inputs]),
-                  test_session.suite_options
-                )
-
-                persist_inputs(session_data_repo, create_params(test_session, suite), test_run(runnable))
-
-                dispatch_job(test_run(runnable))
-                results += test_runs_repo.results_for_test_run(test_run(runnable).id).reverse # NOTE: diff
+            selected_runnables.each do |runnable|
+              run_one(runnable)
+              results += test_runs_repo.results_for_test_run(test_run(runnable).id).reverse
             end
           end
         end
@@ -96,6 +78,18 @@ def print_help_and_exit
         exit(0)
       end
 
+      def run_one(runnable)
+        verify_runnable(
+          suite,
+          thor_hash_to_inputs_array(options[:inputs]),
+          test_session.suite_options
+        )
+
+        persist_inputs(session_data_repo, create_params(test_session, suite), test_run(runnable))
+
+        dispatch_job(test_run(runnable))
+      end
+
       def dispatch_job(test_run)
         # TODO: move suppression to outputter? better suppresion?
         if options[:verbose]
@@ -107,27 +101,6 @@ def dispatch_job(test_run)
         end
       end
 
-      def test_sessions_repo
-        @test_sessions_repo ||= Inferno::Repositories::TestSessions.new
-      end
-
-      def session_data_repo
-        @session_data_repo ||= Inferno::Repositories::SessionData.new
-      end
-
-      def test_runs_repo
-        @test_runs_repo ||= Inferno::Repositories::TestRuns.new
-      end
-
-      def test_session
-        @test_session ||= test_sessions_repo.create({
-                                                      test_suite_id: suite.id,
-                                                      suite_options: thor_hash_to_suite_options_array(
-                                                        options[:suite_options]
-                                                      )
-                                                    })
-      end
-
       def test_run(runnable_param)
         @test_runs ||= {}
 
@@ -138,7 +111,6 @@ def test_run(runnable_param)
         @test_runs[runnable_param]
       end
 
-      # TODO run per
       def create_params(test_session, runnable)
         {
           test_session_id: test_session.id,
@@ -147,6 +119,16 @@ def create_params(test_session, runnable)
         }
       end
 
+      def test_session
+        @test_session ||= test_sessions_repo.create({
+                                                      test_suite_id: suite.id,
+                                                      suite_options: thor_hash_to_suite_options_array(
+                                                        options[:suite_options]
+                                                      )
+                                                    })
+      end
+
+
       def suite
         @suite ||= Inferno::Repositories::TestSuites.new.find(options[:suite])
 
@@ -167,7 +149,19 @@ def tests
         end
       end
 
-      def runnables
+      def test_sessions_repo
+        @test_sessions_repo ||= Inferno::Repositories::TestSessions.new
+      end
+
+      def session_data_repo
+        @session_data_repo ||= Inferno::Repositories::SessionData.new
+      end
+
+      def test_runs_repo
+        @test_runs_repo ||= Inferno::Repositories::TestRuns.new
+      end
+
+      def selected_runnables
         groups + tests
       end
 
@@ -194,51 +188,6 @@ def runnable_id_key(runnable)
         end
       end
 
-=begin
-      # TODO: redo/rm this for short ids in groups and tests
-      def runnable
-        @runnable ||=
-          case runnable_type
-          when 'suite'
-            Inferno::Repositories::TestSuites.new.find(options[:suite])
-          when 'group'
-            Inferno::Repositories::TestGroups.new.find(options[:group])
-          when 'test'
-            Inferno::Repositories::Tests.new.find(options[:test])
-          end
-
-        raise StandardError, "#{runnable_type.capitalize} #{options[runnable_type.to_sym]} not found" if @runnable.nil?
-
-        @runnable
-      end
-
-      def runnable_type
-        @runnable_type ||=
-          if options[:suite]
-            'suite'
-          elsif options[:group]
-            'group'
-          elsif options[:test]
-            'test'
-          else
-            raise StandardError, 'No suite, group, or test id provided'
-          end
-      end
-
-      def runnable_id_key
-        case runnable_type&.to_sym
-        when :suite
-          :test_suite_id
-        when :group
-          :test_group_id
-        when :test
-          :test_id
-        else
-          raise StandardError, "Unrecognized runnable type #{runnable_type}"
-        end
-      end
-=end
-
       def thor_hash_to_suite_options_array(hash = {})
         hash.to_a.map { |pair| Inferno::DSL::SuiteOption.new({ id: pair[0], value: pair[1] }) }
       end
@@ -247,16 +196,12 @@ def thor_hash_to_inputs_array(hash = {})
         hash.to_a.map { |pair| { name: pair[0], value: pair[1] } }
       end
 
-
       def print_error_and_exit(err, code)
-        # TODO FIXME
-        begin
-          outputter.print_error(options || {}, err)
-        rescue Exception => outputter_err
-          $stderr.puts "Caught exception #{outputter_err} while printing exception #{err}. Exiting."
-        ensure
-          exit(code)
-        end
+        outputter.print_error(options || {}, err)
+      rescue StandardError => outputter_err
+        puts "Caught exception #{outputter_err} while printing exception #{err}. Exiting."
+      ensure
+        exit(code)
       end
 
       def outputter

From 40073f882d6c7aa4bdb0da29b39b54add9811bc3 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 10 Sep 2024 15:54:22 -0400
Subject: [PATCH 70/83] debug

---
 lib/inferno/apps/cli/execute.rb               | 24 +++++++++++++++----
 .../apps/cli/execute/abstract_outputter.rb    |  2 ++
 lib/inferno/apps/cli/main.rb                  | 10 ++++++--
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 56d8c8269..f8dacfb3f 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -138,15 +138,20 @@ def suite
       end
 
       def groups
-        @groups ||= Inferno::Repositories::TestGroups.new.all.select do |group|
-          options[:groups]&.include?(group.short_id) && group.suite.id == suite.id
-        end
+        return [] if options[:groups].blank?
+        @groups ||= options[:groups]&.map { |short_id| find_by_short_id!(test_groups_repo, short_id) }
       end
 
       def tests
-        @tests ||= Inferno::Repositories::Tests.new.all.select do |test|
-          options[:tests]&.include?(test.short_id) && test.suite.id == suite.id
+        return [] if options[:tests].blank?
+        @tests ||= options[:tests]&.map { |short_id| find_by_short_id!(tests_repo, short_id) }
+      end
+
+      def find_by_short_id!(repo, short_id)
+        repo.all.each do |entity|
+          return entity if short_id == entity.short_id && suite.id == entity.suite.id
         end
+        raise StandardError, "Group or test #{short_id} not found"
       end
 
       def test_sessions_repo
@@ -161,6 +166,14 @@ def test_runs_repo
         @test_runs_repo ||= Inferno::Repositories::TestRuns.new
       end
 
+      def test_groups_repo
+        @test_groups_repo ||= Inferno::Repositories::TestGroups.new
+      end
+
+      def tests_repo
+        @tests_repo ||= Inferno::Repositories::Tests.new
+      end
+
       def selected_runnables
         groups + tests
       end
@@ -208,6 +221,7 @@ def outputter
         # TODO: swap outputter based on options
         @outputter ||= Inferno::CLI::Execute::ConsoleOutputter.new
       end
+
     end
   end
 end
diff --git a/lib/inferno/apps/cli/execute/abstract_outputter.rb b/lib/inferno/apps/cli/execute/abstract_outputter.rb
index 47e23870f..daa067c7c 100644
--- a/lib/inferno/apps/cli/execute/abstract_outputter.rb
+++ b/lib/inferno/apps/cli/execute/abstract_outputter.rb
@@ -2,6 +2,8 @@ module Inferno
   module CLI
     class Execute
       # Subclass AbstractOutputter to implement your own outputter
+      #
+      # @abstract
       # rubocop:disable Lint/UnusedMethodArgument
       class AbstractOutputter
         # @see Inferno::CLI::Main#execute for options
diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index 4597e7019..fa605d2db 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -80,6 +80,12 @@ def version
                                         --inputs "url:https://hapi.fhir.org/baseR4" \
                                                  patient_id:1234321`
             => Outputs test results
+
+            `bundle exec inferno execute --suite dev_validator \
+                                         --inputs "url:https://hapi.fhir.org/baseR4" \
+                                                  patient_id:1234321 \
+                                         --tests 1.01 1.02`
+            => Run specific tests from suite
       END_OF_HELP
       desc 'execute', 'Run Inferno tests in command line'
       long_desc EXECUTE_HELP, wrap: false
@@ -95,11 +101,11 @@ def version
       option :groups,
              aliases: ['-g'],
              type: :array,
-             desc: 'Comma delimited test group short ids (AKA sequence number) to run, requires suite'
+             desc: 'Series of test group short ids (AKA sequence number) to run, requires suite'
       option :tests,
              aliases: ['-t'],
              type: :array,
-             desc: 'Comma delimited test short ids (AKA sequence number) to run, requires suite'
+             desc: 'Series of test short ids (AKA sequence number) to run, requires suite'
       option :inputs,
              aliases: ['-i'],
              type: :hash,

From 3609646268b2b485ccfe9de411d882b7119a448d Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 10 Sep 2024 16:14:28 -0400
Subject: [PATCH 71/83] move repo methods to bottom

---
 lib/inferno/apps/cli/execute.rb | 40 ++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index f8dacfb3f..ff47ce51d 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -154,26 +154,6 @@ def find_by_short_id!(repo, short_id)
         raise StandardError, "Group or test #{short_id} not found"
       end
 
-      def test_sessions_repo
-        @test_sessions_repo ||= Inferno::Repositories::TestSessions.new
-      end
-
-      def session_data_repo
-        @session_data_repo ||= Inferno::Repositories::SessionData.new
-      end
-
-      def test_runs_repo
-        @test_runs_repo ||= Inferno::Repositories::TestRuns.new
-      end
-
-      def test_groups_repo
-        @test_groups_repo ||= Inferno::Repositories::TestGroups.new
-      end
-
-      def tests_repo
-        @tests_repo ||= Inferno::Repositories::Tests.new
-      end
-
       def selected_runnables
         groups + tests
       end
@@ -222,6 +202,26 @@ def outputter
         @outputter ||= Inferno::CLI::Execute::ConsoleOutputter.new
       end
 
+      def test_sessions_repo
+        @test_sessions_repo ||= Inferno::Repositories::TestSessions.new
+      end
+
+      def session_data_repo
+        @session_data_repo ||= Inferno::Repositories::SessionData.new
+      end
+
+      def test_runs_repo
+        @test_runs_repo ||= Inferno::Repositories::TestRuns.new
+      end
+
+      def test_groups_repo
+        @test_groups_repo ||= Inferno::Repositories::TestGroups.new
+      end
+
+      def tests_repo
+        @tests_repo ||= Inferno::Repositories::Tests.new
+      end
+
     end
   end
 end

From 8dbe38539349a773490a01b692d2815774ea9bb9 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 10 Sep 2024 16:17:32 -0400
Subject: [PATCH 72/83] resort execute methods in call order

---
 lib/inferno/apps/cli/execute.rb | 128 ++++++++++++++++----------------
 1 file changed, 63 insertions(+), 65 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index ff47ce51d..b934db7b3 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -78,6 +78,15 @@ def print_help_and_exit
         exit(0)
       end
 
+      def outputter
+        # TODO: swap outputter based on options
+        @outputter ||= Inferno::CLI::Execute::ConsoleOutputter.new
+      end
+
+      def selected_runnables
+        groups + tests
+      end
+
       def run_one(runnable)
         verify_runnable(
           suite,
@@ -90,15 +99,16 @@ def run_one(runnable)
         dispatch_job(test_run(runnable))
       end
 
-      def dispatch_job(test_run)
-        # TODO: move suppression to outputter? better suppresion?
-        if options[:verbose]
-          Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
-        else
-          Inferno::CLI::Execute.suppress_output do
-            Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
-          end
-        end
+      def suite
+        @suite ||= Inferno::Repositories::TestSuites.new.find(options[:suite])
+
+        raise StandardError, "Test suite #{options[:suite]} not found" if @suite.nil?
+
+        @suite
+      end
+
+      def test_runs_repo
+        @test_runs_repo ||= Inferno::Repositories::TestRuns.new
       end
 
       def test_run(runnable_param)
@@ -111,12 +121,20 @@ def test_run(runnable_param)
         @test_runs[runnable_param]
       end
 
-      def create_params(test_session, runnable)
-        {
-          test_session_id: test_session.id,
-          runnable_id_key(runnable) => runnable.id,
-          inputs: thor_hash_to_inputs_array(options[:inputs])
-        }
+      def test_groups_repo
+        @test_groups_repo ||= Inferno::Repositories::TestGroups.new
+      end
+
+      def tests_repo
+        @tests_repo ||= Inferno::Repositories::Tests.new
+      end
+
+      def test_sessions_repo
+        @test_sessions_repo ||= Inferno::Repositories::TestSessions.new
+      end
+
+      def session_data_repo
+        @session_data_repo ||= Inferno::Repositories::SessionData.new
       end
 
       def test_session
@@ -128,13 +146,23 @@ def test_session
                                                     })
       end
 
+      def create_params(test_session, runnable)
+        {
+          test_session_id: test_session.id,
+          runnable_id_key(runnable) => runnable.id,
+          inputs: thor_hash_to_inputs_array(options[:inputs])
+        }
+      end
 
-      def suite
-        @suite ||= Inferno::Repositories::TestSuites.new.find(options[:suite])
-
-        raise StandardError, "Test suite #{options[:suite]} not found" if @suite.nil?
-
-        @suite
+      def dispatch_job(test_run)
+        # TODO: move suppression to outputter? better suppression?
+        if options[:verbose]
+          Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+        else
+          Inferno::CLI::Execute.suppress_output do
+            Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true)
+          end
+        end
       end
 
       def groups
@@ -154,8 +182,20 @@ def find_by_short_id!(repo, short_id)
         raise StandardError, "Group or test #{short_id} not found"
       end
 
-      def selected_runnables
-        groups + tests
+      def thor_hash_to_suite_options_array(hash = {})
+        hash.to_a.map { |pair| Inferno::DSL::SuiteOption.new({ id: pair[0], value: pair[1] }) }
+      end
+
+      def thor_hash_to_inputs_array(hash = {})
+        hash.to_a.map { |pair| { name: pair[0], value: pair[1] } }
+      end
+
+      def print_error_and_exit(err, code)
+        outputter.print_error(options || {}, err)
+      rescue StandardError => outputter_err
+        puts "Caught exception #{outputter_err} while printing exception #{err}. Exiting."
+      ensure
+        exit(code)
       end
 
       def runnable_type(runnable)
@@ -180,48 +220,6 @@ def runnable_id_key(runnable)
           :test_id
         end
       end
-
-      def thor_hash_to_suite_options_array(hash = {})
-        hash.to_a.map { |pair| Inferno::DSL::SuiteOption.new({ id: pair[0], value: pair[1] }) }
-      end
-
-      def thor_hash_to_inputs_array(hash = {})
-        hash.to_a.map { |pair| { name: pair[0], value: pair[1] } }
-      end
-
-      def print_error_and_exit(err, code)
-        outputter.print_error(options || {}, err)
-      rescue StandardError => outputter_err
-        puts "Caught exception #{outputter_err} while printing exception #{err}. Exiting."
-      ensure
-        exit(code)
-      end
-
-      def outputter
-        # TODO: swap outputter based on options
-        @outputter ||= Inferno::CLI::Execute::ConsoleOutputter.new
-      end
-
-      def test_sessions_repo
-        @test_sessions_repo ||= Inferno::Repositories::TestSessions.new
-      end
-
-      def session_data_repo
-        @session_data_repo ||= Inferno::Repositories::SessionData.new
-      end
-
-      def test_runs_repo
-        @test_runs_repo ||= Inferno::Repositories::TestRuns.new
-      end
-
-      def test_groups_repo
-        @test_groups_repo ||= Inferno::Repositories::TestGroups.new
-      end
-
-      def tests_repo
-        @tests_repo ||= Inferno::Repositories::Tests.new
-      end
-
     end
   end
 end

From 9619e7421b2251eaccc6db89d48e326c5e6e118a Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 10 Sep 2024 17:47:40 -0400
Subject: [PATCH 73/83] wip rspec

---
 spec/inferno/apps/cli/execute_spec.rb | 59 ++++++++++++++++-----------
 1 file changed, 35 insertions(+), 24 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 85a1032ea..e59549e1e 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -25,36 +25,47 @@
     end
   end
 
-  [
-    { suite: 'basic' },
-    { group: 'BasicTestSuite::AbcGroup' },
-    { test: 'BasicTestSuite::AbcGroup-demo_test' }
-  ].each do |given_options|
-    describe '#runnable' do
-      it "sets runnable to #{given_options.keys.first} entity" do
-        allow(instance).to receive(:options).and_return(given_options)
-        klass = case given_options.keys.first
-                when :suite
-                  Inferno::TestSuite
-                when :group
-                  Inferno::TestGroup
-                else
-                  Inferno::Test
-                end
-
-        expect(instance.runnable).to be < klass
-      end
+  describe '#outputter' do
+    it 'returns an outputter instance' do
+      expect(instance.outputter).to be_an_instance_of(Inferno::CLI::Execute::AbstractOutputter)
     end
+  end
 
-    describe '#runnable_type' do
-      it "sets runnable_type to #{given_options.keys.first}" do
-        allow(instance).to receive(:options).and_return(given_options)
+  describe '#selected_runnables' do
+    it 'returns empty array when no short ids given' do
+      allow(instance).to receive(:options).and_return({suite: 'basic'})
+      expect(instance.selected_runnables).to eq([])
+    end
 
-        expect(instance.runnable_type).to eq(given_options.keys.first.to_s)
-      end
+    it 'returns both groups and tests when short ids for both are given' do
+      allow(instance).to receive(:options).and_return({suite: 'basic', groups: '1', tests: '1.01'})
+      expect(instance.selected_runnables.length).to eq(2)
+    end
+  end
+
+  # TODO: run_one spec
+
+  describe '#suite' do
+    it 'returns the correct Inferno TestSuite entity' do
+      allow(instance).to recieve(:options).and_return({suite: 'basic'})
+      expect(instance.suite).to eq(BasicTestSuite::Suite)
+    end
+
+    it 'raises standard error if no suite provided' do
+      expect{ instance.suite }.to raise_error(StandardError)
     end
   end
 
+  describe '#test_run' do
+    it 'creates a test run entity' do
+      runnable = BasicTestSuite::Suite
+      expect{ test_run(runnable) }.not_to raise_error
+      expect(Inferno::Repositories::TestRuns.new.all.length).to eq(1) # TODO better?
+    end
+  end
+
+  # TODO continue here
+
   describe '#runnable_id_key' do
     { suite: :test_suite_id, group: :test_group_id, test: :test_id }.each do |runnable_type, id_key|
       it "returns proper id for runnable type #{runnable_type}" do

From 6aaf29ce2ed7ed3afb299683300cf599dc964c5b Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Tue, 10 Sep 2024 23:41:47 -0400
Subject: [PATCH 74/83] wip rspec

---
 spec/inferno/apps/cli/execute_spec.rb | 100 ++++++++++++++++++++------
 1 file changed, 79 insertions(+), 21 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index e59549e1e..d31cfa5ac 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -57,22 +57,75 @@
   end
 
   describe '#test_run' do
-    it 'creates a test run entity' do
-      runnable = BasicTestSuite::Suite
-      expect{ test_run(runnable) }.not_to raise_error
-      expect(Inferno::Repositories::TestRuns.new.all.length).to eq(1) # TODO better?
+    {suite: BasicTestSuite::Suite, group: BasicTestSuite::AbcGroup, test: BasicTestSuite::Suite.tests.first}.each do |type, runnable|
+      it "returns a test run for #{type}" do
+        expect(test_run(runnable)).to be_instance_of Inferno::Entities::TestRun
+      end
     end
   end
 
-  # TODO continue here
+  describe '#test_session' do
+    it 'returns test session given suite options' do
+      allow(instance).to receive(:options).and_return({suite: 'basic', suite_options: {option: 'a'}})
+      allow(instance).to receive(:suite).and_return(BasicTestSuite::Suite)
+      expect(instance.test_session).to be_instance_of Inferno::Entities::TestSession
+    end
+  end
 
-  describe '#runnable_id_key' do
-    { suite: :test_suite_id, group: :test_group_id, test: :test_id }.each do |runnable_type, id_key|
-      it "returns proper id for runnable type #{runnable_type}" do
-        allow(instance).to receive(:runnable_type).and_return(runnable_type)
 
-        expect(instance.runnable_id_key).to eq(id_key)
-      end
+  describe '#create_params' do
+    let(:test_suite) { BasicTestSuite::Suite }
+    let(:test_session) { create(:test_session) }
+    let(:inputs_hash) { { url: 'https://example.com' } }
+    let(:inputs_array) { [{ name: :url, value: 'https://example.com' }] }
+
+    it 'returns test run params' do
+      allow(instance).to receive(:options).and_return({ inputs: inputs_hash })
+      allow(instance).to receive(:runnable_type).and_return('suite')
+
+      result = instance.create_params(test_session, test_suite)
+      expect(result).to eq({ test_session_id: test_session.id, test_suite_id: test_suite.id, inputs: inputs_array })
+    end
+  end
+
+  describe '#dispatch_job' do
+    it 'supresses output if verbose is false' do
+      allow(instance).to receive(:options).and_return({verbose: false})
+      allow(instance).to receive(:test_run).and_return(create(:test_run))
+
+      expect{ instance.dispatch_job }.to_not output(/.+/)
+    end
+  end
+
+  describe '#groups' do
+    it 'parses single group by short id' do
+      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1']})
+      expect{ instance.groups }.to eq([BasicTestSuite::AbcGroup])
+    end
+
+    it 'parses multiple groups by short id' do
+      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1', '2']})
+      expect{ instance.groups }.to eq([BasicTestSuite::AbcGroup, BasicTestSuite::DefGroup])
+    end
+  end
+
+  describe '#tests' do
+    it 'parses single test by short id' do
+      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1.01']})
+      expect{ instance.groups }.to eq([BasicTestSuite::Suite.tests.first])
+    end
+
+=begin # TODO: change to fixture with multiple tests
+    it 'parses multiple tests by short id' do
+      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1.01', '2.01']})
+      expect{ instance.groups }.to eq([BasicTestSuite::Suite.tests.first, BasicTestSuite::DefGroup])
+    end
+=end
+  end
+
+  describe '#find_by_short_id!' do
+    it 'raises standard error when entity not found by short id' do
+      expect { instance.find_by_short_id!(Inferno::Repositories::Tests.new, 'does_not_exist') }.to raise_error(StandardError)
     end
   end
 
@@ -90,6 +143,8 @@
     end
   end
 
+  # TODO print_error_and_exit test?
+
   describe '#thor_hash_to_inputs_array' do
     let(:hash) { { url: 'https://example.com' } }
 
@@ -104,18 +159,21 @@
     end
   end
 
-  describe '#create_params' do
-    let(:test_suite) { BasicTestSuite::Suite }
-    let(:test_session) { create(:test_session) }
-    let(:inputs_hash) { { url: 'https://example.com' } }
-    let(:inputs_array) { [{ name: :url, value: 'https://example.com' }] }
+  describe '#runnable_type' do
+    {BasicTestSuite::Suite => :suite, BasicTestSuite::AbcGroup => :group, BasicTestSuite::Suite.tests.first => :test}.each do |runnable, type|
+      it "can return #{type} type" do
+        expect( instance.runnable_type(runnable) ).to eq(type)
+      end
+    end
+  end
 
-    it 'returns test run params' do
-      allow(instance).to receive(:options).and_return({ inputs: inputs_hash })
-      allow(instance).to receive(:runnable_type).and_return('suite')
+  describe '#runnable_id_key' do
+    { suite: :test_suite_id, group: :test_group_id, test: :test_id }.each do |runnable_type, id_key|
+      it "returns proper id for runnable type #{runnable_type}" do
+        allow(instance).to receive(:runnable_type).and_return(runnable_type)
 
-      result = instance.create_params(test_session, test_suite)
-      expect(result).to eq({ test_session_id: test_session.id, test_suite_id: test_suite.id, inputs: inputs_array })
+        expect(instance.runnable_id_key).to eq(id_key)
+      end
     end
   end
 

From df0d205122ce38ed54acc0555e3d8b7d9a9bf56d Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 11 Sep 2024 10:02:11 -0400
Subject: [PATCH 75/83] wip rspec

---
 spec/inferno/apps/cli/execute_spec.rb | 31 ++++++++++++++++-----------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index d31cfa5ac..71f292e9d 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -38,7 +38,7 @@
     end
 
     it 'returns both groups and tests when short ids for both are given' do
-      allow(instance).to receive(:options).and_return({suite: 'basic', groups: '1', tests: '1.01'})
+      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1'], tests: ['1.01']})
       expect(instance.selected_runnables.length).to eq(2)
     end
   end
@@ -47,7 +47,7 @@
 
   describe '#suite' do
     it 'returns the correct Inferno TestSuite entity' do
-      allow(instance).to recieve(:options).and_return({suite: 'basic'})
+      allow(instance).to receive(:options).and_return({suite: 'basic'})
       expect(instance.suite).to eq(BasicTestSuite::Suite)
     end
 
@@ -57,7 +57,7 @@
   end
 
   describe '#test_run' do
-    {suite: BasicTestSuite::Suite, group: BasicTestSuite::AbcGroup, test: BasicTestSuite::Suite.tests.first}.each do |type, runnable|
+    {suite: BasicTestSuite::Suite, group: BasicTestSuite::AbcGroup, test: BasicTestSuite::AbcGroup.tests.first}.each do |type, runnable|
       it "returns a test run for #{type}" do
         expect(test_run(runnable)).to be_instance_of Inferno::Entities::TestRun
       end
@@ -93,32 +93,32 @@
       allow(instance).to receive(:options).and_return({verbose: false})
       allow(instance).to receive(:test_run).and_return(create(:test_run))
 
-      expect{ instance.dispatch_job }.to_not output(/.+/)
+      expect{ instance.dispatch_job }.to_not output(/.+/).to_stdout
     end
   end
 
   describe '#groups' do
     it 'parses single group by short id' do
       allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1']})
-      expect{ instance.groups }.to eq([BasicTestSuite::AbcGroup])
+      expect(instance.groups).to eq([BasicTestSuite::AbcGroup])
     end
 
     it 'parses multiple groups by short id' do
       allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1', '2']})
-      expect{ instance.groups }.to eq([BasicTestSuite::AbcGroup, BasicTestSuite::DefGroup])
+      expect(instance.groups).to eq([BasicTestSuite::AbcGroup, BasicTestSuite::DefGroup])
     end
   end
 
   describe '#tests' do
     it 'parses single test by short id' do
       allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1.01']})
-      expect{ instance.groups }.to eq([BasicTestSuite::Suite.tests.first])
+      expect(instance.groups).to eq([BasicTestSuite::AbcGroup.tests.first])
     end
 
 =begin # TODO: change to fixture with multiple tests
     it 'parses multiple tests by short id' do
       allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1.01', '2.01']})
-      expect{ instance.groups }.to eq([BasicTestSuite::Suite.tests.first, BasicTestSuite::DefGroup])
+      expect{ instance.groups }.to eq([BasicTestSuite::AbcGroup.tests.first, BasicTestSuite::DefGroup])
     end
 =end
   end
@@ -160,7 +160,7 @@
   end
 
   describe '#runnable_type' do
-    {BasicTestSuite::Suite => :suite, BasicTestSuite::AbcGroup => :group, BasicTestSuite::Suite.tests.first => :test}.each do |runnable, type|
+    {BasicTestSuite::Suite => :suite, BasicTestSuite::AbcGroup => :group, BasicTestSuite::AbcGroup.tests.first => :test}.each do |runnable, type|
       it "can return #{type} type" do
         expect( instance.runnable_type(runnable) ).to eq(type)
       end
@@ -169,10 +169,17 @@
 
   describe '#runnable_id_key' do
     { suite: :test_suite_id, group: :test_group_id, test: :test_id }.each do |runnable_type, id_key|
-      it "returns proper id for runnable type #{runnable_type}" do
+      it "returns proper id key for runnable type #{runnable_type}" do
         allow(instance).to receive(:runnable_type).and_return(runnable_type)
-
-        expect(instance.runnable_id_key).to eq(id_key)
+        runnable = case runnable_type
+                   when :suite
+                     BasicTestSuite::Suite
+                   when :group
+                     BasicTestSuite::AbcGroup
+                   else
+                     BasicTestSuite::AbcGroup.tests.first
+                   end
+        expect(instance.runnable_id_key(runnable)).to eq(id_key)
       end
     end
   end

From 18e4bbe724039b5677de823a3a1c50f0ed9455fe Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 11 Sep 2024 11:58:36 -0400
Subject: [PATCH 76/83] wip rspec

---
 spec/inferno/apps/cli/execute_spec.rb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 71f292e9d..957ce4efa 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -27,7 +27,7 @@
 
   describe '#outputter' do
     it 'returns an outputter instance' do
-      expect(instance.outputter).to be_an_instance_of(Inferno::CLI::Execute::AbstractOutputter)
+      expect(instance.outputter).to be_a_kind_of(Inferno::CLI::Execute::AbstractOutputter)
     end
   end
 
@@ -59,7 +59,8 @@
   describe '#test_run' do
     {suite: BasicTestSuite::Suite, group: BasicTestSuite::AbcGroup, test: BasicTestSuite::AbcGroup.tests.first}.each do |type, runnable|
       it "returns a test run for #{type}" do
-        expect(test_run(runnable)).to be_instance_of Inferno::Entities::TestRun
+        allow(instance).to receive(:options).and_return({suite: 'basic'})
+        expect(instance.test_run(runnable)).to be_instance_of Inferno::Entities::TestRun
       end
     end
   end
@@ -72,7 +73,6 @@
     end
   end
 
-
   describe '#create_params' do
     let(:test_suite) { BasicTestSuite::Suite }
     let(:test_session) { create(:test_session) }
@@ -80,8 +80,8 @@
     let(:inputs_array) { [{ name: :url, value: 'https://example.com' }] }
 
     it 'returns test run params' do
-      allow(instance).to receive(:options).and_return({ inputs: inputs_hash })
-      allow(instance).to receive(:runnable_type).and_return('suite')
+      allow(instance).to receive(:options).and_return({ suite: test_suite.id, inputs: inputs_hash })
+      allow(instance).to receive(:runnable_type).and_return(:suite)
 
       result = instance.create_params(test_session, test_suite)
       expect(result).to eq({ test_session_id: test_session.id, test_suite_id: test_suite.id, inputs: inputs_array })

From 098212c62f3e0caa7c616a585e252a50825eaf54 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 11 Sep 2024 16:42:34 -0400
Subject: [PATCH 77/83] wip rspec

---
 spec/inferno/apps/cli/execute_spec.rb | 34 ++++++++-------------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 957ce4efa..9287c2249 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -43,8 +43,6 @@
     end
   end
 
-  # TODO: run_one spec
-
   describe '#suite' do
     it 'returns the correct Inferno TestSuite entity' do
       allow(instance).to receive(:options).and_return({suite: 'basic'})
@@ -90,37 +88,27 @@
 
   describe '#dispatch_job' do
     it 'supresses output if verbose is false' do
-      allow(instance).to receive(:options).and_return({verbose: false})
-      allow(instance).to receive(:test_run).and_return(create(:test_run))
+      allow(instance).to receive(:options).and_return({suite: 'basic', verbose: false})
+      test_session = create(:test_session)
+      test_session.id = SecureRandom.base36
+      test_run = create(:test_run, test_session:, test_session_id: test_session.id)
 
-      expect{ instance.dispatch_job }.to_not output(/.+/).to_stdout
+      expect{ instance.dispatch_job(test_run) }.to_not output(/.+/).to_stdout_from_any_process
     end
   end
 
   describe '#groups' do
-    it 'parses single group by short id' do
+    it 'parses group by short id' do
       allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1']})
-      expect(instance.groups).to eq([BasicTestSuite::AbcGroup])
-    end
-
-    it 'parses multiple groups by short id' do
-      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1', '2']})
-      expect(instance.groups).to eq([BasicTestSuite::AbcGroup, BasicTestSuite::DefGroup])
+      expect(instance.groups).to eq([BasicTestSuite::Suite.groups.first])
     end
   end
 
   describe '#tests' do
-    it 'parses single test by short id' do
-      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1.01']})
-      expect(instance.groups).to eq([BasicTestSuite::AbcGroup.tests.first])
+    it 'parses test by short id' do
+      allow(instance).to receive(:options).and_return({suite: 'basic', tests: ['1.01']})
+      expect(instance.tests).to eq([BasicTestSuite::Suite.groups.first.tests.first])
     end
-
-=begin # TODO: change to fixture with multiple tests
-    it 'parses multiple tests by short id' do
-      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1.01', '2.01']})
-      expect{ instance.groups }.to eq([BasicTestSuite::AbcGroup.tests.first, BasicTestSuite::DefGroup])
-    end
-=end
   end
 
   describe '#find_by_short_id!' do
@@ -143,8 +131,6 @@
     end
   end
 
-  # TODO print_error_and_exit test?
-
   describe '#thor_hash_to_inputs_array' do
     let(:hash) { { url: 'https://example.com' } }
 

From dcc046303aa1818a158f9eaa856462ac3dca61e4 Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 12 Sep 2024 11:10:43 -0400
Subject: [PATCH 78/83] fix rspec

---
 lib/inferno/apps/cli/execute.rb       |  6 +++--
 spec/inferno/apps/cli/execute_spec.rb | 39 +++++++++++++++------------
 2 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index b934db7b3..7bca723d4 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -167,11 +167,13 @@ def dispatch_job(test_run)
 
       def groups
         return [] if options[:groups].blank?
+
         @groups ||= options[:groups]&.map { |short_id| find_by_short_id!(test_groups_repo, short_id) }
       end
 
       def tests
         return [] if options[:tests].blank?
+
         @tests ||= options[:tests]&.map { |short_id| find_by_short_id!(tests_repo, short_id) }
       end
 
@@ -192,8 +194,8 @@ def thor_hash_to_inputs_array(hash = {})
 
       def print_error_and_exit(err, code)
         outputter.print_error(options || {}, err)
-      rescue StandardError => outputter_err
-        puts "Caught exception #{outputter_err} while printing exception #{err}. Exiting."
+      rescue StandardError => e
+        puts "Caught exception #{e} while printing exception #{err}. Exiting."
       ensure
         exit(code)
       end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index 9287c2249..deaa946d0 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -33,31 +33,32 @@
 
   describe '#selected_runnables' do
     it 'returns empty array when no short ids given' do
-      allow(instance).to receive(:options).and_return({suite: 'basic'})
+      allow(instance).to receive(:options).and_return({ suite: 'basic' })
       expect(instance.selected_runnables).to eq([])
     end
 
     it 'returns both groups and tests when short ids for both are given' do
-      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1'], tests: ['1.01']})
+      allow(instance).to receive(:options).and_return({ suite: 'basic', groups: ['1'], tests: ['1.01'] })
       expect(instance.selected_runnables.length).to eq(2)
     end
   end
 
   describe '#suite' do
     it 'returns the correct Inferno TestSuite entity' do
-      allow(instance).to receive(:options).and_return({suite: 'basic'})
+      allow(instance).to receive(:options).and_return({ suite: 'basic' })
       expect(instance.suite).to eq(BasicTestSuite::Suite)
     end
 
     it 'raises standard error if no suite provided' do
-      expect{ instance.suite }.to raise_error(StandardError)
+      expect { instance.suite }.to raise_error(StandardError)
     end
   end
 
   describe '#test_run' do
-    {suite: BasicTestSuite::Suite, group: BasicTestSuite::AbcGroup, test: BasicTestSuite::AbcGroup.tests.first}.each do |type, runnable|
+    { suite: BasicTestSuite::Suite, group: BasicTestSuite::AbcGroup,
+      test: BasicTestSuite::AbcGroup.tests.first }.each do |type, runnable|
       it "returns a test run for #{type}" do
-        allow(instance).to receive(:options).and_return({suite: 'basic'})
+        allow(instance).to receive(:options).and_return({ suite: 'basic' })
         expect(instance.test_run(runnable)).to be_instance_of Inferno::Entities::TestRun
       end
     end
@@ -65,7 +66,7 @@
 
   describe '#test_session' do
     it 'returns test session given suite options' do
-      allow(instance).to receive(:options).and_return({suite: 'basic', suite_options: {option: 'a'}})
+      allow(instance).to receive(:options).and_return({ suite: 'basic', suite_options: { option: 'a' } })
       allow(instance).to receive(:suite).and_return(BasicTestSuite::Suite)
       expect(instance.test_session).to be_instance_of Inferno::Entities::TestSession
     end
@@ -87,33 +88,36 @@
   end
 
   describe '#dispatch_job' do
+    let(:test_session) { test_run.test_session }
+    let(:test_run) { repo_create(:test_run, test_suite_id: 'basic') }
+
     it 'supresses output if verbose is false' do
-      allow(instance).to receive(:options).and_return({suite: 'basic', verbose: false})
-      test_session = create(:test_session)
-      test_session.id = SecureRandom.base36
-      test_run = create(:test_run, test_session:, test_session_id: test_session.id)
+      allow(instance).to receive(:test_session).and_return(test_session)
+      allow(instance).to receive(:options).and_return({ suite: 'basic', verbose: false })
 
-      expect{ instance.dispatch_job(test_run) }.to_not output(/.+/).to_stdout_from_any_process
+      expect { instance.dispatch_job(test_run) }.to_not output(/.+/).to_stdout_from_any_process
     end
   end
 
   describe '#groups' do
     it 'parses group by short id' do
-      allow(instance).to receive(:options).and_return({suite: 'basic', groups: ['1']})
+      allow(instance).to receive(:options).and_return({ suite: 'basic', groups: ['1'] })
       expect(instance.groups).to eq([BasicTestSuite::Suite.groups.first])
     end
   end
 
   describe '#tests' do
     it 'parses test by short id' do
-      allow(instance).to receive(:options).and_return({suite: 'basic', tests: ['1.01']})
+      allow(instance).to receive(:options).and_return({ suite: 'basic', tests: ['1.01'] })
       expect(instance.tests).to eq([BasicTestSuite::Suite.groups.first.tests.first])
     end
   end
 
   describe '#find_by_short_id!' do
     it 'raises standard error when entity not found by short id' do
-      expect { instance.find_by_short_id!(Inferno::Repositories::Tests.new, 'does_not_exist') }.to raise_error(StandardError)
+      expect do
+        instance.find_by_short_id!(Inferno::Repositories::Tests.new, 'does_not_exist')
+      end.to raise_error(StandardError)
     end
   end
 
@@ -146,9 +150,10 @@
   end
 
   describe '#runnable_type' do
-    {BasicTestSuite::Suite => :suite, BasicTestSuite::AbcGroup => :group, BasicTestSuite::AbcGroup.tests.first => :test}.each do |runnable, type|
+    { BasicTestSuite::Suite => :suite, BasicTestSuite::AbcGroup => :group,
+      BasicTestSuite::AbcGroup.tests.first => :test }.each do |runnable, type|
       it "can return #{type} type" do
-        expect( instance.runnable_type(runnable) ).to eq(type)
+        expect(instance.runnable_type(runnable)).to eq(type)
       end
     end
   end

From 1969015897a0701b7b0e51b7b1d63e0e666456fb Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 12 Sep 2024 11:17:11 -0400
Subject: [PATCH 79/83] remove validator boot from executor boot

---
 lib/inferno/config/boot/executor.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/inferno/config/boot/executor.rb b/lib/inferno/config/boot/executor.rb
index f670e71a9..84b28a2c8 100644
--- a/lib/inferno/config/boot/executor.rb
+++ b/lib/inferno/config/boot/executor.rb
@@ -10,6 +10,5 @@
     end
 
     target_container.start :suites
-    target_container.start :validator
   end
 end

From 9ac3c19deb70f286740a49e235493b861140907a Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Thu, 12 Sep 2024 11:26:22 -0400
Subject: [PATCH 80/83] replace -o alias with -u for suite options

---
 lib/inferno/apps/cli/main.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb
index fa605d2db..2a0d25a47 100644
--- a/lib/inferno/apps/cli/main.rb
+++ b/lib/inferno/apps/cli/main.rb
@@ -95,7 +95,7 @@ def version
              desc: 'Test suite id to run or to select groups and tests from',
              banner: 'id'
       option :suite_options,
-             aliases: ['-o'],
+             aliases: ['-u'], # NOTE: -o will be for outputter
              type: :hash,
              desc: 'Suite options'
       option :groups,

From e690d77932bcd6e633bee6eadc0266a245519bca Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 25 Sep 2024 11:41:21 -0400
Subject: [PATCH 81/83] remove bang from find_by_short_id name

---
 lib/inferno/apps/cli/execute.rb       | 6 +++---
 spec/inferno/apps/cli/execute_spec.rb | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb
index 7bca723d4..9284ce975 100644
--- a/lib/inferno/apps/cli/execute.rb
+++ b/lib/inferno/apps/cli/execute.rb
@@ -168,16 +168,16 @@ def dispatch_job(test_run)
       def groups
         return [] if options[:groups].blank?
 
-        @groups ||= options[:groups]&.map { |short_id| find_by_short_id!(test_groups_repo, short_id) }
+        @groups ||= options[:groups]&.map { |short_id| find_by_short_id(test_groups_repo, short_id) }
       end
 
       def tests
         return [] if options[:tests].blank?
 
-        @tests ||= options[:tests]&.map { |short_id| find_by_short_id!(tests_repo, short_id) }
+        @tests ||= options[:tests]&.map { |short_id| find_by_short_id(tests_repo, short_id) }
       end
 
-      def find_by_short_id!(repo, short_id)
+      def find_by_short_id(repo, short_id)
         repo.all.each do |entity|
           return entity if short_id == entity.short_id && suite.id == entity.suite.id
         end
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index deaa946d0..c3cd42b0d 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -113,10 +113,10 @@
     end
   end
 
-  describe '#find_by_short_id!' do
+  describe '#find_by_short_id' do
     it 'raises standard error when entity not found by short id' do
       expect do
-        instance.find_by_short_id!(Inferno::Repositories::Tests.new, 'does_not_exist')
+        instance.find_by_short_id(Inferno::Repositories::Tests.new, 'does_not_exist')
       end.to raise_error(StandardError)
     end
   end

From cd674c0e446dfe000d10a10472fbd8cff48926af Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 25 Sep 2024 11:41:50 -0400
Subject: [PATCH 82/83] remove exception raises from abstract methods in
 abstract outputter

---
 lib/inferno/apps/cli/execute/abstract_outputter.rb | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lib/inferno/apps/cli/execute/abstract_outputter.rb b/lib/inferno/apps/cli/execute/abstract_outputter.rb
index daa067c7c..ef36a2d85 100644
--- a/lib/inferno/apps/cli/execute/abstract_outputter.rb
+++ b/lib/inferno/apps/cli/execute/abstract_outputter.rb
@@ -8,31 +8,26 @@ class Execute
       class AbstractOutputter
         # @see Inferno::CLI::Main#execute for options
         def print_start_message(options)
-          raise StandardError, 'not implemented'
         end
 
         # Implementation MUST call `yield`
         # @see Inferno::CLI::Main#execute for options
         def print_around_run(options, &)
-          raise StandardError, 'not implemented'
         end
 
         # @see Inferno::CLI::Main#execute for options
         # @param results [Array<Inferno::Entity::Result>]
         def print_results(options, results)
-          raise StandardError, 'not implemented'
         end
 
         # @see Inferno::CLI::Main#execute for options
         def print_end_message(options)
-          raise StandardError, 'not implemented'
         end
 
         # Implementation MUST NOT re-raise exception or exit
         # @see Inferno::CLI::Main#execute for options
         # @param exception [Exception]
         def print_error(options, exception)
-          raise StandardError, 'not implemented'
         end
       end
       # rubocop:enable Lint/UnusedMethodArgument

From 8833d975ca1e027c5524ab6b3477e5e141c4b20e Mon Sep 17 00:00:00 2001
From: Shaumik-Ashraf <shaumikashraf@mitre.org>
Date: Wed, 25 Sep 2024 12:07:10 -0400
Subject: [PATCH 83/83] yank abstract outputter

---
 .../apps/cli/execute/abstract_outputter.rb    | 36 -------------------
 .../apps/cli/execute/console_outputter.rb     |  7 ++--
 spec/inferno/apps/cli/execute_spec.rb         | 33 +++++++++++++++--
 3 files changed, 34 insertions(+), 42 deletions(-)
 delete mode 100644 lib/inferno/apps/cli/execute/abstract_outputter.rb

diff --git a/lib/inferno/apps/cli/execute/abstract_outputter.rb b/lib/inferno/apps/cli/execute/abstract_outputter.rb
deleted file mode 100644
index ef36a2d85..000000000
--- a/lib/inferno/apps/cli/execute/abstract_outputter.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-module Inferno
-  module CLI
-    class Execute
-      # Subclass AbstractOutputter to implement your own outputter
-      #
-      # @abstract
-      # rubocop:disable Lint/UnusedMethodArgument
-      class AbstractOutputter
-        # @see Inferno::CLI::Main#execute for options
-        def print_start_message(options)
-        end
-
-        # Implementation MUST call `yield`
-        # @see Inferno::CLI::Main#execute for options
-        def print_around_run(options, &)
-        end
-
-        # @see Inferno::CLI::Main#execute for options
-        # @param results [Array<Inferno::Entity::Result>]
-        def print_results(options, results)
-        end
-
-        # @see Inferno::CLI::Main#execute for options
-        def print_end_message(options)
-        end
-
-        # Implementation MUST NOT re-raise exception or exit
-        # @see Inferno::CLI::Main#execute for options
-        # @param exception [Exception]
-        def print_error(options, exception)
-        end
-      end
-      # rubocop:enable Lint/UnusedMethodArgument
-    end
-  end
-end
diff --git a/lib/inferno/apps/cli/execute/console_outputter.rb b/lib/inferno/apps/cli/execute/console_outputter.rb
index b5b3e0ab0..d7cf17cb7 100644
--- a/lib/inferno/apps/cli/execute/console_outputter.rb
+++ b/lib/inferno/apps/cli/execute/console_outputter.rb
@@ -1,5 +1,4 @@
 require 'pastel'
-require_relative 'abstract_outputter'
 require_relative '../../web/serializers/test_run'
 require_relative '../../web/serializers/result'
 
@@ -7,7 +6,7 @@ module Inferno
   module CLI
     class Execute
       # @private
-      class ConsoleOutputter < AbstractOutputter
+      class ConsoleOutputter
         COLOR = Pastel.new
         CHECKMARK = "\u2713".freeze
         BAR = ('=' * 80).freeze
@@ -44,9 +43,9 @@ def print_results(options, results)
 
         def print_end_message(options); end
 
-        def print_error(_options, exception)
+        def print_error(options, exception)
           puts COLOR.red "Error: #{exception.full_message}"
-          verbose_print(exception.backtrace&.join('\n'))
+          verbose_print(options, exception.backtrace&.join('\n'))
         end
 
         # private
diff --git a/spec/inferno/apps/cli/execute_spec.rb b/spec/inferno/apps/cli/execute_spec.rb
index c3cd42b0d..700b1ad14 100644
--- a/spec/inferno/apps/cli/execute_spec.rb
+++ b/spec/inferno/apps/cli/execute_spec.rb
@@ -26,8 +26,37 @@
   end
 
   describe '#outputter' do
-    it 'returns an outputter instance' do
-      expect(instance.outputter).to be_a_kind_of(Inferno::CLI::Execute::AbstractOutputter)
+    it 'returns an object that responds to print_start_message' do
+      expect(instance.outputter).to respond_to(:print_start_message)
+    end
+
+    it 'returns an object that responds to print_around_run' do
+      expect(instance.outputter).to respond_to(:print_around_run)
+    end
+
+    it 'returns an object whose print_around_run yields' do
+      expect do
+        expect { |b| instance.outputter.print_around_run({}, &b) }.to yield_control
+      end.to output(/.?/).to_stdout # required to prevent output in rspec
+    end
+
+    it 'returns an object that responds to print_results' do
+      expect(instance.outputter).to respond_to(:print_results)
+    end
+
+    it 'returns an object that responds to print_end_message' do
+      expect(instance.outputter).to respond_to(:print_end_message)
+    end
+
+    it 'returns an object that responds to print_error' do
+      expect(instance.outputter).to respond_to(:print_error)
+    end
+
+    it 'returns an object whose print_error does not raise exception nor exit' do
+      allow(instance).to receive(:options).and_return({})
+      expect do
+        expect { instance.outputter.print_error({}, StandardError.new('my error')) }.to_not raise_error
+      end.to output(/.?/).to_stdout # required to prevent output in rspec
     end
   end