From a9819402192676b446e29956c73bd1999980669c Mon Sep 17 00:00:00 2001 From: Wilson Carey Date: Sun, 6 May 2018 11:21:41 -0400 Subject: [PATCH 1/7] Updated support for nested complex inputs and shell interpolations --- CHANGELOG.md | 7 +++ lib/covalence/core/entities/input.rb | 65 +++++++++++--------- lib/covalence/core/entities/stack.rb | 5 +- lib/covalence/helpers/shell_interpolation.rb | 28 +++++++++ spec/core/entities/input_spec.rb | 46 +++++++++++++- 5 files changed, 119 insertions(+), 32 deletions(-) create mode 100644 lib/covalence/helpers/shell_interpolation.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index feacee0..5eed56d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ * Add support for Terraform environments * Add ability to toggle primary state store +## 0.7.7 +IMPROVEMENTS: +- Extended shell interpolation for input values to support nesting and escaping + +FIXES: +- Updated input processing to support nested complex types properly. + ## 0.7.6 (December 6, 2017) IMPROVEMENTS: - Terraform init failures were not being reported. Update spec tests to catch errors with 'terraform init'. diff --git a/lib/covalence/core/entities/input.rb b/lib/covalence/core/entities/input.rb index f02af88..882b813 100644 --- a/lib/covalence/core/entities/input.rb +++ b/lib/covalence/core/entities/input.rb @@ -2,7 +2,8 @@ require 'active_support/core_ext/string/inflections' require 'active_support/core_ext/hash' require 'active_model' -require 'open3' + +require_relative '../../helpers/shell_interpolation' module Covalence class Input @@ -24,33 +25,7 @@ def value end def to_command_option - parsed_value = value() - - if parsed_value.nil? - "#{name} = \"\"" - - elsif parsed_value.is_a?(Hash) - config = "#{name} = {\n" - parsed_value.each do |k,v| - config += " \"#{k}\" = \"#{v}\"\n" - end - config += "}" - - elsif parsed_value.is_a?(Array) - config = "#{name} = [\n" - parsed_value.each do |v| - config += " \"#{v}\",\n" - end - config += "]" - - elsif parsed_value.start_with?("$(") - Covalence::LOGGER.info "Evaluating interpolated value: \"#{parsed_value}\"" - interpolated_value = Open3.capture2e(ENV, "echo \"#{parsed_value}\"")[0].chomp - "#{name} = \"#{interpolated_value}\"" - - else - "#{name} = \"#{parsed_value}\"" - end + "#{name} = #{parse_input(value())}" end private @@ -72,6 +47,40 @@ def get_value(input) end end + def parse_array(input) + config = "[\n" + input.each do |v| + config += " #{parse_input(v)},\n" + end + config += "]" + end + + def parse_hash(input) + config = "{\n" + input.each do |k,v| + config += " \"#{k}\" = #{parse_input(v)}\n" + end + config += "}" + end + + def parse_input(input) + if input.nil? + "\"\"" + + elsif input.is_a?(Hash) + parse_hash(input) + + elsif input.is_a?(Array) + parse_array(input) + + elsif input.include?("$(") + "\"#{Covalence::Helpers::ShellInterpolation.parse_shell(input)}\"" + + else + "\"#{input}\"" + end + end + # :reek:FeatureEnvy def parse_type(input) if input.stringify_keys.has_key?('type') diff --git a/lib/covalence/core/entities/stack.rb b/lib/covalence/core/entities/stack.rb index fb31c69..0ecddfe 100644 --- a/lib/covalence/core/entities/stack.rb +++ b/lib/covalence/core/entities/stack.rb @@ -53,8 +53,9 @@ def materialize_cmd_inputs inputs.each do |name, input| config[name] = input.value end - logger.info "\nStack inputs:\n\n#{JSON.generate(config)}" - File.open('covalence-inputs.json','w') {|f| f.write(JSON.generate(config))} + config_json = JSON.generate(config) + logger.info "\nStack inputs:\n\n#{config_json}" + File.open('covalence-inputs.json','w') {|f| f.write(config_json)} end end diff --git a/lib/covalence/helpers/shell_interpolation.rb b/lib/covalence/helpers/shell_interpolation.rb new file mode 100644 index 0000000..ba3e9ff --- /dev/null +++ b/lib/covalence/helpers/shell_interpolation.rb @@ -0,0 +1,28 @@ +require 'open3' + +module Covalence + module Helpers + class ShellInterpolation + + def self.parse_shell(input) + Covalence::LOGGER.info "Evaluating requested interpolation: \"#{input}\"" + matches = input.scan(/.?\$\([^)]*\)/) + + Covalence::LOGGER.debug "matches: #{matches}" + matches.each do |cmd| + if cmd[0] != "\\" + cmd = cmd[1..-1] unless cmd[0] == "$" + interpolated_value = Open3.capture2e(ENV, "echo \"#{cmd}\"")[0].chomp + input = input.gsub(cmd, interpolated_value) + Covalence::LOGGER.debug "updated value: #{input}" + else + input = input.gsub(cmd, cmd[1..-1]) + end + end + Covalence::LOGGER.info "Interpolated value: \"#{input}\"" + return input + end + + end + end +end diff --git a/spec/core/entities/input_spec.rb b/spec/core/entities/input_spec.rb index d0a266e..f874591 100644 --- a/spec/core/entities/input_spec.rb +++ b/spec/core/entities/input_spec.rb @@ -26,6 +26,18 @@ module Covalence it { expect(input.value).to eq(raw_value) } end + context "simple list with nested complex types" do + let(:raw_value) { ["foo", ["bar"], {"foo"=>"bar"}] } + + it { expect(input.value).to eq(raw_value) } + end + + context "simple map with nested complex types" do + let(:raw_value) { {"foo"=>["bar"], "bar"=>{"foo"=>"bar"}} } + + it { expect(input.value).to eq(raw_value) } + end + context "complex string" do let(:raw_value) { {"type"=>"string","value"=>"test"} } @@ -33,16 +45,28 @@ module Covalence end context "complex list" do - let(:raw_value) { {"type"=>"string","value"=>["test"]} } + let(:raw_value) { {"type"=>"list","value"=>["test"]} } it { expect(input.value).to eq(["test"]) } end + context "complex list with nested complex types" do + let(:raw_value) { {"type"=>"list", "value"=>["foo", ["bar"], {"foo"=>"bar"}]} } + + it { expect(input.value).to eq(["foo", ["bar"], {"foo"=>"bar"}]) } + end + context "complex map" do - let(:raw_value) { {"type"=>"string","value"=>{"foo"=>"bar"}} } + let(:raw_value) { {"type"=>"map","value"=>{"foo"=>"bar"}} } it { expect(input.value).to eq({"foo"=>"bar"}) } end + + context "complex map with nested complex types" do + let(:raw_value) { {"type"=>"map", "value"=>{"foo"=>["bar"], "bar"=>{"foo"=>"bar"}}} } + + it { expect(input.value).to eq({"foo"=>["bar"], "bar"=>{"foo"=>"bar"}}) } + end end context "with remote input" do @@ -112,6 +136,24 @@ module Covalence it { expect(input.to_command_option).to eq("input = \"#{`pwd`.chomp}\"") } end + context "with nested interpolated shell value" do + let(:raw_value) { "this-is-a-test-$(pwd)" } + + it { expect(input.to_command_option).to eq("input = \"this-is-a-test-#{`pwd`.chomp}\"") } + end + + context "with nested interpolated shell values" do + let(:raw_value) { "this-is-a-test-$(pwd)-and-$(date)" } + + it { expect(input.to_command_option).to eq("input = \"this-is-a-test-#{`pwd`.chomp}-and-#{`date`.chomp}\"") } + end + + context "with nested interpolated shell values with escapes" do + let(:raw_value) { "this-is-a-test-\\$(pwd)-and-$(date)" } + + it { expect(input.to_command_option).to eq("input = \"this-is-a-test-$(pwd)-and-#{`date`.chomp}\"") } + end + context "all other values" do it { expect(input.to_command_option).to eq("input = \"test\"") } end From a7ce1cb076aae81a16d25e4a57a0d1e4d6fbadd3 Mon Sep 17 00:00:00 2001 From: Wilson Carey Date: Sun, 6 May 2018 11:22:04 -0400 Subject: [PATCH 2/7] Added support for shell interpolation of state store inputs --- lib/covalence/core/state_stores/consul.rb | 3 +++ lib/covalence/core/state_stores/s3.rb | 16 +++++++++++++++- spec/core/entities/state_store_spec.rb | 12 ++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/covalence/core/state_stores/consul.rb b/lib/covalence/core/state_stores/consul.rb index cb81b00..0550fed 100644 --- a/lib/covalence/core/state_stores/consul.rb +++ b/lib/covalence/core/state_stores/consul.rb @@ -2,6 +2,8 @@ require 'rest-client' require 'base64' +require_relative '../../helpers/shell_interpolation' + module Covalence module Consul @@ -99,6 +101,7 @@ def self.get_state_store(params) params.delete('name') params.each do |k,v| + v = Covalence::Helpers::ShellInterpolation.parse_shell(v) if v.include?("$(") config += " #{k} = \"#{v}\"\n" end diff --git a/lib/covalence/core/state_stores/s3.rb b/lib/covalence/core/state_stores/s3.rb index 5f1ce19..697ae34 100644 --- a/lib/covalence/core/state_stores/s3.rb +++ b/lib/covalence/core/state_stores/s3.rb @@ -1,6 +1,8 @@ require 'json' require 'aws-sdk' +require_relative '../../helpers/shell_interpolation' + module Covalence module S3 @@ -70,14 +72,26 @@ def self.get_state_store(params) raise "Missing '#{param}' store parameter" unless params.has_key?(param) end - config = <<-CONF + if params.has_key?('key') + config = <<-CONF +terraform { + backend "s3" { +CONF + + else + config = <<-CONF terraform { backend "s3" { key = "#{params['name']}/terraform.tfstate" CONF + Covalence::LOGGER.debug "'key' parameter not specified. Inferring value from 'name' parameter." + end + params.delete('name') + params.each do |k,v| + v = Covalence::Helpers::ShellInterpolation.parse_shell(v) if v.include?("$(") config += " #{k} = \"#{v}\"\n" end diff --git a/spec/core/entities/state_store_spec.rb b/spec/core/entities/state_store_spec.rb index da8a4cb..06d7cba 100644 --- a/spec/core/entities/state_store_spec.rb +++ b/spec/core/entities/state_store_spec.rb @@ -35,6 +35,18 @@ module Covalence expect(S3).to receive(:get_state_store).with({"name" => "example/state_store"}) state_store.get_config end + + it "does return a state configuration" do + state_store = Fabricate(:state_store, params: { bucket: "test", name: "example/state_store", foo: "bar" }) + + expect(state_store.get_config).to eq("terraform {\n backend \"s3\" {\n key = \"example/state_store/terraform.tfstate\"\n bucket = \"test\"\n foo = \"bar\"\n }\n}\n") + end + + it "does process shell interpolations" do + state_store = Fabricate(:state_store, params: { bucket: "test", name: "example/state_store", path: "$(pwd)" }) + + expect(state_store.get_config).to eq("terraform {\n backend \"s3\" {\n key = \"example/state_store/terraform.tfstate\"\n bucket = \"test\"\n path = \"#{`pwd`.chomp}\"\n }\n}\n") + end end describe '#params' do From f9cf63a3b4da5616b900a01cbf841c59e893de87 Mon Sep 17 00:00:00 2001 From: Wilson Carey Date: Sun, 6 May 2018 12:29:22 -0400 Subject: [PATCH 3/7] Added support for plugin caching to Terraform init --- CHANGELOG.md | 1 + lib/covalence.rb | 3 ++- lib/covalence/core/cli_wrappers/terraform_cli.rb | 2 +- spec/core/cli_wrappers/terraform_cli_spec.rb | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eed56d..95fc1ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## 0.7.7 IMPROVEMENTS: - Extended shell interpolation for input values to support nesting and escaping +- Improved support for Terraform plugin caching FIXES: - Updated input processing to support nested complex types properly. diff --git a/lib/covalence.rb b/lib/covalence.rb index ceb9cae..956a079 100644 --- a/lib/covalence.rb +++ b/lib/covalence.rb @@ -11,7 +11,7 @@ module Covalence # Configurable constants #TODO: look into how WORKSPACE is being used, maybe this can just be an internal ROOT and make CONFIG not depend on WORKSPACE - WORKSPACE = File.absolute_path((ENV['COVALENCE_WORKSPACE'] || '.')) + WORKSPACE = File.absolute_path(ENV['COVALENCE_WORKSPACE'] || '.') CONFIG = File.join(WORKSPACE, (ENV['COVALENCE_CONFIG'] || 'covalence.yaml')) # TODO: could use better naming PACKER = File.absolute_path(File.join(WORKSPACE, (ENV['COVALENCE_PACKER_DIR'] || '.'))) @@ -22,6 +22,7 @@ module Covalence TERRAFORM_CMD = ENV['TERRAFORM_CMD'] || "terraform" TERRAFORM_VERSION = ENV['TERRAFORM_VERSION'] || `#{TERRAFORM_CMD} version`.split("\n", 2)[0].gsub('Terraform v','') + TERRAFORM_PLUGIN_CACHE = File.absolute_path(ENV['TF_PLUGIN_CACHE_DIR'] || "#{ENV['HOME']}/.terraform.d/plugin-cache") PACKER_CMD = ENV['PACKER_CMD'] || "packer" diff --git a/lib/covalence/core/cli_wrappers/terraform_cli.rb b/lib/covalence/core/cli_wrappers/terraform_cli.rb index 7fc6583..4a175a6 100644 --- a/lib/covalence/core/cli_wrappers/terraform_cli.rb +++ b/lib/covalence/core/cli_wrappers/terraform_cli.rb @@ -42,7 +42,7 @@ def self.terraform_check_style(path) def self.terraform_init(path='', args: '', ignore_exitcode: false) output = PopenWrapper.run([ - Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false"], + Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false", "-plugin-dir=#{Covalence::TERRAFORM_PLUGIN_CACHE}"], path, args, ignore_exitcode: ignore_exitcode) diff --git a/spec/core/cli_wrappers/terraform_cli_spec.rb b/spec/core/cli_wrappers/terraform_cli_spec.rb index 60d84e8..b376a95 100644 --- a/spec/core/cli_wrappers/terraform_cli_spec.rb +++ b/spec/core/cli_wrappers/terraform_cli_spec.rb @@ -65,7 +65,7 @@ module Covalence end it "#terraform_init" do - expected_args = [ENV, "terraform init -get=false -input=false", anything] + expected_args = [ENV, "terraform init -get=false -input=false -plugin-dir=#{Covalence::TERRAFORM_PLUGIN_CACHE}", anything] expect(PopenWrapper).to receive(:spawn_subprocess).with(*expected_args).and_return(0) expect(described_class.terraform_init).to be true end From 5771249b6b5a8a8eac83d58ada1dbbf87de02238 Mon Sep 17 00:00:00 2001 From: Wilson Carey Date: Sun, 6 May 2018 12:48:37 -0400 Subject: [PATCH 4/7] Replaced deprecated force option for Terraform destroy task --- CHANGELOG.md | 8 ++++++-- lib/covalence/core/services/terraform_stack_tasks.rb | 2 +- spec/rake/environment_tasks_spec.rb | 6 +++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95fc1ec..f4261cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,15 @@ ## Unreleased -* Add support for Terraform environments +* Add support for Terraform workspaces * Add ability to toggle primary state store -## 0.7.7 +## 0.7.7 (May 6, 2018) +BACKWARDS INCOMPATIBILITIES: +- Versions of Terraform prior to v0.11.4 are no longer supported. + IMPROVEMENTS: - Extended shell interpolation for input values to support nesting and escaping - Improved support for Terraform plugin caching +- Replaced deprecated `-force` option for Terraform `destroy` task with `-auto-approve=true`. FIXES: - Updated input processing to support nested complex types properly. diff --git a/lib/covalence/core/services/terraform_stack_tasks.rb b/lib/covalence/core/services/terraform_stack_tasks.rb index a8c157e..d41458d 100644 --- a/lib/covalence/core/services/terraform_stack_tasks.rb +++ b/lib/covalence/core/services/terraform_stack_tasks.rb @@ -163,7 +163,7 @@ def context_destroy(*additional_args) stack.materialize_cmd_inputs args = collect_args("-input=false", - "-force", + "-auto-approve=true", stack.args, additional_args, "-var-file=covalence-inputs.tfvars") diff --git a/spec/rake/environment_tasks_spec.rb b/spec/rake/environment_tasks_spec.rb index b86f073..62deab2 100644 --- a/spec/rake/environment_tasks_spec.rb +++ b/spec/rake/environment_tasks_spec.rb @@ -387,7 +387,7 @@ module Covalence "-input=false", "-no-color", "-target=\"module.az0\"", - "-force", + "-auto-approve=true", "-var-file=covalence-inputs.tfvars" ))) subject.invoke @@ -639,7 +639,7 @@ module Covalence it "executes a destroy" do expect(TerraformCli).to receive(:terraform_destroy).with(hash_including(args: [ "-input=false", - "-force", + "-auto-approve=true", "-no-color", "-target=\"module.az1\"", "-target=\"module.common.aws_eip.myapp\"", @@ -898,7 +898,7 @@ module Covalence it "executes a destroy" do expect(TerraformCli).to receive(:terraform_destroy).with(hash_including(args: [ "-input=false", - "-force", + "-auto-approve=true", "-var-file=covalence-inputs.tfvars" ])) subject.invoke From 554a233010d798d02f5b476da3b1dc3bb89f58d4 Mon Sep 17 00:00:00 2001 From: Wilson Carey Date: Sun, 6 May 2018 12:48:51 -0400 Subject: [PATCH 5/7] Updated version for release --- .circleci/config.yml | 4 ++-- Gemfile.lock | 2 +- lib/covalence/version.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f76f9cd..70945e4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,8 +8,8 @@ jobs: - image: circleci/ruby:2.3.6 environment: - COVALENCE_VERSION: 0.7.6 - TERRAFORM_VERSION: 0.11.3 + COVALENCE_VERSION: 0.7.7 + TERRAFORM_VERSION: 0.11.7 steps: - checkout diff --git a/Gemfile.lock b/Gemfile.lock index f136052..484aaea 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - covalence (0.7.6) + covalence (0.7.7) activemodel (~> 4.2.6) activesupport (~> 4.2.6) aws-sdk (~> 2.9.5) diff --git a/lib/covalence/version.rb b/lib/covalence/version.rb index 865055a..6399c3c 100644 --- a/lib/covalence/version.rb +++ b/lib/covalence/version.rb @@ -1,3 +1,3 @@ module Covalence - VERSION = "0.7.6" + VERSION = "0.7.7" end From 48a7d2aac9d00f653a5ec9988d089e841aac62de Mon Sep 17 00:00:00 2001 From: Wilson Carey Date: Sun, 6 May 2018 14:42:16 -0400 Subject: [PATCH 6/7] Added flag for initializing Terraform from cache only --- lib/covalence.rb | 2 +- lib/covalence/core/cli_wrappers/terraform_cli.rb | 10 ++++++++-- spec/core/cli_wrappers/terraform_cli_spec.rb | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/covalence.rb b/lib/covalence.rb index 956a079..6593145 100644 --- a/lib/covalence.rb +++ b/lib/covalence.rb @@ -22,7 +22,7 @@ module Covalence TERRAFORM_CMD = ENV['TERRAFORM_CMD'] || "terraform" TERRAFORM_VERSION = ENV['TERRAFORM_VERSION'] || `#{TERRAFORM_CMD} version`.split("\n", 2)[0].gsub('Terraform v','') - TERRAFORM_PLUGIN_CACHE = File.absolute_path(ENV['TF_PLUGIN_CACHE_DIR'] || "#{ENV['HOME']}/.terraform.d/plugin-cache") + TERRAFORM_PLUGIN_CACHE = File.absolute_path("#{ENV['TF_PLUGIN_CACHE_DIR']}/linux_amd64" || "#{ENV['HOME']}/.terraform.d/plugin-cache/linus_amd64") PACKER_CMD = ENV['PACKER_CMD'] || "packer" diff --git a/lib/covalence/core/cli_wrappers/terraform_cli.rb b/lib/covalence/core/cli_wrappers/terraform_cli.rb index 4a175a6..c0f080c 100644 --- a/lib/covalence/core/cli_wrappers/terraform_cli.rb +++ b/lib/covalence/core/cli_wrappers/terraform_cli.rb @@ -41,8 +41,14 @@ def self.terraform_check_style(path) end def self.terraform_init(path='', args: '', ignore_exitcode: false) - output = PopenWrapper.run([ - Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false", "-plugin-dir=#{Covalence::TERRAFORM_PLUGIN_CACHE}"], + if ENV['TF_PLUGIN_LOCAL'] == 'true' + cmd = [Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false", "-plugin-dir=#{Covalence::TERRAFORM_PLUGIN_CACHE}"] + else + cmd = [Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false"] + end + + output = PopenWrapper.run( + cmd, path, args, ignore_exitcode: ignore_exitcode) diff --git a/spec/core/cli_wrappers/terraform_cli_spec.rb b/spec/core/cli_wrappers/terraform_cli_spec.rb index b376a95..60d84e8 100644 --- a/spec/core/cli_wrappers/terraform_cli_spec.rb +++ b/spec/core/cli_wrappers/terraform_cli_spec.rb @@ -65,7 +65,7 @@ module Covalence end it "#terraform_init" do - expected_args = [ENV, "terraform init -get=false -input=false -plugin-dir=#{Covalence::TERRAFORM_PLUGIN_CACHE}", anything] + expected_args = [ENV, "terraform init -get=false -input=false", anything] expect(PopenWrapper).to receive(:spawn_subprocess).with(*expected_args).and_return(0) expect(described_class.terraform_init).to be true end From 891a962651b5c3fec19d8e5a98e083d798d036cb Mon Sep 17 00:00:00 2001 From: Wilson Carey Date: Mon, 7 May 2018 20:24:39 -0400 Subject: [PATCH 7/7] Updated shell interpolation check to handle non-string values --- lib/covalence/core/entities/input.rb | 2 +- lib/covalence/core/state_stores/s3.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/covalence/core/entities/input.rb b/lib/covalence/core/entities/input.rb index 882b813..28405bb 100644 --- a/lib/covalence/core/entities/input.rb +++ b/lib/covalence/core/entities/input.rb @@ -73,7 +73,7 @@ def parse_input(input) elsif input.is_a?(Array) parse_array(input) - elsif input.include?("$(") + elsif input.to_s.include?("$(") "\"#{Covalence::Helpers::ShellInterpolation.parse_shell(input)}\"" else diff --git a/lib/covalence/core/state_stores/s3.rb b/lib/covalence/core/state_stores/s3.rb index 697ae34..6bed8a3 100644 --- a/lib/covalence/core/state_stores/s3.rb +++ b/lib/covalence/core/state_stores/s3.rb @@ -91,7 +91,7 @@ def self.get_state_store(params) params.delete('name') params.each do |k,v| - v = Covalence::Helpers::ShellInterpolation.parse_shell(v) if v.include?("$(") + v = Covalence::Helpers::ShellInterpolation.parse_shell(v) if v.to_s.include?("$(") config += " #{k} = \"#{v}\"\n" end