Refactor Helper Module

README.md

@@ -33,8 +33,6 @@ This InSpec resource pack uses the Azure REST API and provides the required reso
   - [Direnv](#direnv)
   - [Rake Commands](#rake-commands)
   - [Optional Components](#optional-components)
-    - [Graph API](#graph-api)
-  - [Network Watcher](#network-watcher)
 
 ## Prerequisites
 
@@ -486,30 +484,51 @@ To run integration tests:
 ```shell
 rake test:integration
 ```
+Please note that Graph API resource requires specific privileges granted to your service principal.
+Please refer to the [Microsoft Documentation](https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-integrating-applications#updating-an-application) for information on how to grant these permissions to your application.
+
+To run a control called `azure_virtual_machine` only:
+```shell
+rake test:integration[azurerm_virtual_machine]
+```
+Note that in zsh you need to escape the `[`, `]` characters.
+
+You may run selected multiple controls only:
+```shell
+rake test:integration[azure_aks_cluster,azure_virtual_machine]
+```
 To run lint and unit tests:
 ```shell
 rake
 ```
-### Optional Components
 
-By default, rake tasks will only use core components. Optional components have associated integrations that will be skipped unless you enable these. We have the following optional pieces that may be managed with Terraform.
+### Optional Components
 
-#### Graph API
+The creation of the following resources can be skipped if there is any resource constraints.
 
-Graph API support may be enabled to test with `azure_graph` related resources.
-Each resource requires specific privileges granted to your service principal.
-Please refer to the [Microsoft Documentation](https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-integrating-applications#updating-an-application) for information on how to grant these permissions to your application.
-If your account does not have access, leave this disabled.
+- Network Watcher
 
-Note: An Azure Administrator must grant your application these permissions.
+```shell
+rake tf:apply[network_watcher]
 ```
-rake options[graph]
-direnv allow # or source .envrc
-rake tf:apply
+- HDinsight Interactive Query Cluster
+```shell
+rake tf:apply[hdinsight_cluster]
+```
+- Public IP
+```shell
+rake tf:apply[public_ip]
+```
+- API Management
+```shell
+rake tf:apply[api_management]
+```
+- Management Group
+```shell
+rake tf:apply[management_group]
 ```
-### Network Watcher
 
-Network Watcher may be enabled to run integration tests related to the Network Watcher. 
-We recommend leaving this disabled unless you are specifically working on related resources. 
-You may only have one Network Watcher enabled per an Azure subscription at a time. 
-To enable Network Watcher, update the `default` value of `network_watcher` variable to `1` in the [`terraform/variables.tf`](terraform/variables.tf) file.
+A combination of the above can be provided.
+```shell
+rake tf:apply[management_group,public_ip,network_watcher]
+```

Rakefile

@@ -16,6 +16,9 @@ FIXTURE_DIR   = "#{Dir.pwd}/test/fixtures"
 TERRAFORM_DIR = 'terraform'
 REQUIRED_ENVS = %w{AZURE_CLIENT_ID AZURE_CLIENT_SECRET AZURE_TENANT_ID AZURE_SUBSCRIPTION_ID}.freeze
 INTEGRATION_DIR = 'test/integration/verify'
+TF_PLAN_FILE_NAME = 'inspec-azure.plan'
+TF_PLAN_FILE = File.join(TERRAFORM_DIR, TF_PLAN_FILE_NAME)
+ATTRIBUTES_FILE_NAME = ''
 
 task default: :test
 desc 'Testing tasks'
@@ -30,7 +33,6 @@ task :setup_env do
   ENV['TF_VAR_tenant_id']       = ENV['AZURE_TENANT_ID']
   ENV['TF_VAR_client_id']       = ENV['AZURE_CLIENT_ID']
   ENV['TF_VAR_client_secret']   = ENV['AZURE_CLIENT_SECRET']
-  ENV['TF_VAR_public_vm_count'] = '1' if ENV.key?('MSI')
 
   puts '-> Ensuring required Environment Variables are set'
   missing = REQUIRED_ENVS.reject { |var| ENV.key?(var) }
@@ -109,7 +111,7 @@ namespace :test do
     t.test_files = FileList['test/unit/**/*_test.rb']
   end
 
-  task :integration, [:controls] => ['attributes:write', :setup_env] do |_t, args|
+  task :integration, [:controls] => ['tf:write_tf_output_to_file', :setup_env] do |_t, args|
     cmd = %W( bundle exec inspec exec #{INTEGRATION_DIR}
               --input-file terraform/#{ENV['ATTRIBUTES_FILE']}
               --reporter cli
@@ -125,7 +127,7 @@ namespace :test do
   end
 end
 
-namespace :tf do
+namespace :tf do    # rubocop:disable Metrics/BlockLength
   workspace = ENV['WORKSPACE']
 
   task init: [:'azure:login'] do
@@ -146,28 +148,34 @@ namespace :tf do
   end
 
   desc 'Creates a Terraform execution plan from the plan file'
-  task plan: [:workspace] do
+  task :plan, [:optionals] => [:workspace] do |_t, args|
+    if args[:optionals]
+      ignore_list = Array(args[:optionals]) + args.extras
+      ignore_list.each do |component|
+        ENV["TF_VAR_#{component}_count"] = '0'
+      end
+    end
     Dir.chdir(TERRAFORM_DIR) do
       sh('terraform', 'get')
       sh('terraform', 'plan', '-out', 'inspec-azure.plan')
     end
+
+    Rake::Task['tf:write_tf_output_to_file'].invoke
   end
 
   desc 'Executes the Terraform plan'
-  task apply: [:plan] do
-    Dir.chdir(TERRAFORM_DIR) do
-      sh('terraform', 'apply', 'inspec-azure.plan')
+  task :apply, [:optionals] do |_t, args|
+    if File.exist?(TF_PLAN_FILE)
+      puts "-> Applying an existing terraform plan: #{TF_PLAN_FILE}"
+      unless args[:optionals].nil?
+        puts "These arguments are ignored: #{Array(args[:optionals]) + args.extras}."
+      end
+    else
+      Rake::Task['tf:plan'].invoke(args[:optionals])
     end
-
-    Rake::Task['attributes:write'].invoke
-  end
-
-  task :apply_only do
     Dir.chdir(TERRAFORM_DIR) do
       sh('terraform', 'apply', 'inspec-azure.plan')
     end
-
-    Rake::Task['attributes:write'].invoke
   end
 
   desc 'Destroys the Terraform environment'
@@ -182,6 +190,8 @@ namespace :tf do
       stdout, stderr, status = Open3.capture3('terraform output -json')
 
       abort(stderr) unless status.success?
+      abort('$ATTRIBUTES_FILE not set. Please source .envrc.') if ENV['ATTRIBUTES_FILE'].nil?
+      abort('$ATTRIBUTES_FILE has no content. Check .envrc.') if ENV['ATTRIBUTES_FILE'].empty?
 
       AttributeFileWriter.write_yaml(ENV['ATTRIBUTES_FILE'], stdout)
     end
@@ -206,32 +216,5 @@ namespace :attributes do
     abort('$ATTRIBUTES_FILE not set. Please source .envrc.') if ENV['ATTRIBUTES_FILE'].nil?
     abort('$ATTRIBUTES_FILE has no content. Check .envrc.') if ENV['ATTRIBUTES_FILE'].empty?
     Rake::Task['tf:write_tf_output_to_file'].invoke
-    Rake::Task['attributes:write_guest_presence_to_file'].invoke
-  end
-
-  task :write_guest_presence_to_file do
-    if ENV.key?('GRAPH')
-      Dir.chdir(TERRAFORM_DIR) do
-        stdout, stderr, status = Open3.capture3("az ad user list --query=\"length([?userType == 'Guest'])\"")
-
-        abort(stderr) unless status.success?
-
-        AttributeFileWriter.append(ENV['ATTRIBUTES_FILE'], "guest_accounts: #{stdout.to_i}")
-      end
-    end
-  end
-end
-
-desc 'Enables given optional components. See README for details.'
-task :options, :component do |_t_, args|
-  components = []
-  components << args[:component] if args[:component]
-  components += args.extras unless args.extras.nil?
-
-  begin
-    env_file = EnvironmentFile.new('.envrc')
-    env_file.synchronize(components)
-  rescue RuntimeError => e
-    puts e.message
   end
 end

libraries/azure_backend.rb

@@ -84,7 +84,7 @@ def initialize(opts = {})
   #     result can be tailored by passing parameters as `?$select=objectId,displayName,givenName`
   #
   def resource_from_graph_api(opts)
-    Helpers.validate_parameters(resource_name: @__resource_name__, allow: %i(api_version query_parameters),
+    Validators.validate_parameters(resource_name: @__resource_name__, allow: %i(api_version query_parameters),
                                 required: %i(resource), opts: opts)
     api_version = opts[:api_version] || @azure.graph_api_endpoint_api_version
     if api_version.size > 10 || api_version.include?('/')
@@ -242,7 +242,7 @@ def construct_resource_id
   #       `user_provided`, `latest`, `default`.
   #
   def get_resource(opts = {})
-    Helpers.validate_parameters(resource_name: @__resource_name__,
+    Validators.validate_parameters(resource_name: @__resource_name__,
                                 required: %i(resource_uri),
                                 allow: %i(query_parameters headers method req_body is_uri_a_url audience),
                                 opts: opts)
@@ -394,7 +394,7 @@ def validate_short_desc(resource_list, filter, singular = true) # rubocop:disabl
   def validate_resource_uri(opts = @opts)
     return true if opts[:is_uri_a_url]
     opts[:resource_uri].prepend('/') unless opts[:resource_uri].start_with?('/')
-    Helpers.validate_params_required(%i(add_subscription_id), opts)
+    Validators.validate_params_required(%i(add_subscription_id), opts)
     if opts[:add_subscription_id] == true
       opts[:resource_uri] = "/subscriptions/#{@azure.credentials[:subscription_id]}/#{opts[:resource_uri]}"
                             .gsub('//', '/')
@@ -486,7 +486,7 @@ def validate_parameters(allow: [], required: nil, require_any_of: nil)
     opts = @opts
     allow += %i(azure_retry_limit azure_retry_backoff azure_retry_backoff_factor
                 endpoint api_version required_parameters allowed_parameters display_name)
-    Helpers.validate_parameters(resource_name: @__resource_name__,
+    Validators.validate_parameters(resource_name: @__resource_name__,
                                 allow: allow, required: required,
                                 require_any_of: require_any_of, opts: opts)
     true

libraries/azure_generic_resource.rb

@@ -36,7 +36,7 @@ def initialize(opts = {}, static_resource = false) # rubocop:disable Style/Optio
     else
       resource_fail('There is not enough input to create an Azure resource ID.') if @resource_id.empty?
       # This is the last check on resource_id before talking to resource manager endpoint to get the detailed information.
-      Helpers.validate_resource_uri(@resource_id)
+      Validators.validate_resource_uri(@resource_id)
       query_params = { resource_uri: @resource_id }
     end
     query_params[:query_parameters] = {}
@@ -102,7 +102,7 @@ def failed_resource?
   #     Microsoft.Sql/servers/{serverName}/firewallRules'.
   #   api_version [string] The api version of the endpoint (default - latest).
   def additional_resource_properties(opts = {})
-    Helpers.validate_parameters(resource_name: @__resource_name__,
+    Validators.validate_parameters(resource_name: @__resource_name__,
                                 required: %i(property_name property_endpoint),
                                 allow: %i(api_version filter_free_text add_subscription_id method req_body headers),
                                 opts: opts)
@@ -150,7 +150,7 @@ def validate_static_resource
       # The `name` parameter should have been required in the static resource.
       # Since it is a mandatory field, it is better to make sure that it is in the required list before validations.
       @opts[:resource_identifiers] << :name unless @opts[:resource_identifiers].include?(:name)
-      provided = Helpers.validate_params_only_one_of(@__resource_name__, @opts[:resource_identifiers], @opts)
+      provided = Validators.validate_params_only_one_of(@__resource_name__, @opts[:resource_identifiers], @opts)
       # Remove resource identifiers other than `:name`.
       unless provided == :name
         @opts[:name] = @opts[provided]

libraries/azure_graph_generic_resource.rb

@@ -32,7 +32,7 @@ def initialize(opts = {}, static_resource = false) # rubocop:disable Style/Optio
     #
     if static_resource
       raise ArgumentError, '`:resource_identifiers` have to be provided within a list' unless @opts[:resource_identifiers]
-      provided = Helpers.validate_params_only_one_of(@__resource_name__, @opts[:resource_identifiers], @opts)
+      provided = Validators.validate_params_only_one_of(@__resource_name__, @opts[:resource_identifiers], @opts)
       # We should remove resource identifiers other than `:id`.
       unless provided == :id
         @opts[:id] = @opts[provided]

libraries/azure_network_security_group.rb

@@ -80,7 +80,7 @@ def deny_rules
   #   it { should allow(destination_service_tag: 'VirtualNetwork', direction: 'outbound', protocol: 'TCP') }
   #   it { should allow(source_ip_range: '0:0:0:0:0:ffff:a05:0', direction: 'inbound') }
   def allow?(criteria = {})
-    Helpers.validate_params_required(@__resource_name__, %i(direction), criteria)
+    Validators.validate_params_required(@__resource_name__, %i(direction), criteria)
     criteria[:access] = 'allow'
     rules = criteria[:direction] == 'inbound' ? inbound_rules : outbound_rules
     normalized_security_rules.go_compare(rules, criteria)

libraries/backend/azure_connection.rb

@@ -101,7 +101,7 @@ def provider_details
   #   audience: The audience for the authentication. Optional, it will be extracted frm the URL unless provided.
   #
   def rest_api_call(opts)
-    Helpers.validate_parameters(resource_name: @__resource_name__,
+    Validators.validate_parameters(resource_name: @__resource_name__,
                                 required: %i(url),
                                 allow: %i(params headers method req_body audience),
                                 opts: opts, skip_length: true)

libraries/backend/azure_security_rules_helpers.rb

@@ -122,7 +122,7 @@ def compliant?(criteria) # rubocop:disable Metrics/CyclomaticComplexity, Metrics
     allowed = %i(source_port destination_port protocol)
     required = %i(access direction)
     require_any = %i(destination_ip_range source_ip_range destination_service_tag source_service_tag)
-    Helpers.validate_parameters(allow: allowed, required: required, require_any_of: require_any, opts: criteria)
+    Validators.validate_parameters(allow: allowed, required: required, require_any_of: require_any, opts: criteria)
 
     # This will be updated by the relevant checks.
     compliant = false

libraries/backend/helpers.rb

@@ -54,7 +54,7 @@ def suggested_api_version(wrong_api_version = nil)
         #
         # There are cases where the stable api_versions are too old and don't return JSON response.
         # If the latest stable is too old (based on the age_criteria), then return the preview versions as well.
-        # This is a quick fix until TODO finding a more stable solution.
+        # This is a quick fix until TODO: finding a more stable solution.
         stable_api_versions = message.scan(/\d{4}-\d{2}-\d{2}[,']/).map(&:chop).sort.reverse
         preview_api_versions = message.scan(/\d{4}-\d{2}-\d{2}-preview/).sort.reverse
         if wrong_api_version
@@ -193,7 +193,7 @@ def self.recursive
   end
 end
 
-module Helpers
+module Validators
   # @see https://github.com/inspec/inspec-aws/blob/master/libraries/aws_backend.rb#L209
   #
   # @param opts [Hash] The parameters to be validated.
@@ -204,20 +204,20 @@ module Helpers
   def self.validate_parameters(resource_name: nil, allow: [], required: nil, require_any_of: nil, opts: {}, skip_length: false) # rubocop:disable Metrics/ParameterLists
     raise ArgumentError, "Parameters must be provided with as a Hash object. Provided #{opts.class}" unless opts.is_a?(Hash)
     if required
-      allow += Helpers.validate_params_required(resource_name, required, opts)
+      allow += Validators.validate_params_required(resource_name, required, opts)
     end
     if require_any_of
-      allow += Helpers.validate_params_require_any_of(resource_name, require_any_of, opts)
+      allow += Validators.validate_params_require_any_of(resource_name, require_any_of, opts)
     end
-    Helpers.validate_params_allow(allow, opts, skip_length)
+    Validators.validate_params_allow(allow, opts, skip_length)
     true
   end
 
   # @return [String] Provided parameter within require only one of parameters.
   # @param require_only_one_of [Array]
   def self.validate_params_only_one_of(resource_name = nil, require_only_one_of, opts)
     # At least one of them has to exist.
-    Helpers.validate_params_require_any_of(resource_name, require_only_one_of, opts)
+    Validators.validate_params_require_any_of(resource_name, require_only_one_of, opts)
     provided = require_only_one_of.select { |i| opts.key?(i) }
     if provided.size > 1
       raise ArgumentError, "Either one of #{require_only_one_of} is required. Provided: #{provided}."
@@ -255,6 +255,15 @@ def self.validate_params_allow(allow, opts, skip_length = false) # rubocop:disab
     end
   end
 
+  def self.validate_resource_uri(resource_uri)
+    resource_uri_format = '/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/'\
+      'Microsoft.Compute/virtualMachines/{resource_name}'
+    raise ArgumentError, "Resource URI should be in the format of #{resource_uri_format}. Found: #{resource_uri}" \
+      unless resource_uri.start_with?('/subscriptions/') || resource_uri.include?('providers')
+  end
+end
+
+module Helpers
   # Convert provided data into Odata query format.
   # @see
   #   https://www.odata.org/getting-started/basic-tutorial/
@@ -295,13 +304,6 @@ def self.odata_query(data)
     query
   end
 
-  def self.validate_resource_uri(resource_uri)
-    resource_uri_format = '/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/'\
-      'Microsoft.Compute/virtualMachines/{resource_name}'
-    raise ArgumentError, "Resource URI should be in the format of #{resource_uri_format}. Found: #{resource_uri}" \
-      unless resource_uri.start_with?('/subscriptions/') || resource_uri.include?('providers')
-  end
-
   # Disassemble resource_id and extract the resource_group, provider and resource_provider.
   #
   # This is the one and only method where the `resource_provider` is defined differently from the rest.
@@ -319,7 +321,7 @@ def self.validate_resource_uri(resource_uri)
   #   /subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/
   #   Microsoft.Compute/virtualMachines/{resource_name}
   def self.res_group_provider_type_from_uri(resource_uri)
-    Helpers.validate_resource_uri(resource_uri)
+    Validators.validate_resource_uri(resource_uri)
     subscription_resource_group, provider_resource_type = resource_uri.split('/providers/')
     resource_group = subscription_resource_group.split('/').last
     interim_array = provider_resource_type.split('/')