Merge branch 'master' of https://github.com/imperva/dsfkit

deploy/examples/se_demo/main.tf

@@ -73,6 +73,9 @@ module "vpc" {
 
   cidr = "10.0.0.0/16"
 
+  enable_nat_gateway = true
+  single_nat_gateway = true
+
   azs             = ["${local.region}a", "${local.region}b", "${local.region}c"]
   private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
   public_subnets  = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"]

deploy/examples/se_demo/outputs.tf

@@ -13,11 +13,11 @@ output "dsf_hubs" {
 }
 
 output "dsf_hub_web_console_url" {
-    value     = can(module.hub.public_address) ? join("", ["https://", module.hub.public_address, ":8443/" ]) : null
+    value     = module.hub.public_address != null ? join("", ["https://", module.hub.public_address, ":8443/" ]) : null
 }
 
 output "primary_hub_ssh_command" {
-    value     = join("", ["ssh -i ${resource.local_sensitive_file.dsf_ssh_key_file.filename} ec2-user@", module.hub.public_address])
+    value     = module.hub.public_address != null ? join("", ["ssh -i ${resource.local_sensitive_file.dsf_ssh_key_file.filename} ec2-user@", module.hub.public_address]) : null
 }
 
 output "admin_password" {

deploy/examples/se_demo_hadr/main.tf

@@ -11,7 +11,7 @@ resource "null_resource" "myip" {
     always_run = "${timestamp()}"
   }
   provisioner "local-exec" {
-    command         = "mkdir -p tmp && curl http://ipv4.icanhazip.com > tmp/myip"
+    command         = "curl http://ipv4.icanhazip.com > myip-${terraform.workspace}"
     interpreter     = ["/bin/bash", "-c"]
   }
 }
@@ -22,7 +22,7 @@ resource "null_resource" "myip" {
 # }
 
 data "local_file" "myip_file" { # data "http" doesn't work as expected on Terraform cloud platform
-    filename = "tmp/myip"
+    filename = "myip-${terraform.workspace}"
     depends_on = [
       resource.null_resource.myip
     ]
@@ -61,7 +61,7 @@ module "key_pair" {
 resource "local_sensitive_file" "dsf_ssh_key_file" {
   content         = module.key_pair.private_key_pem
   file_permission = 400
-  filename        = "ssh_keys/dsf_hub_ssh_key"
+  filename        = "ssh_keys/dsf_hub_ssh_key-${terraform.workspace}"
 }
 
 ##############################
@@ -73,12 +73,12 @@ module "vpc" {
 
   cidr = "10.0.0.0/16"
 
+  enable_nat_gateway = true
+  single_nat_gateway = true
+
   azs             = ["${local.region}a", "${local.region}b", "${local.region}c"]
   private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
   public_subnets  = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"]
-
-  enable_nat_gateway = false
-  single_nat_gateway = true
 }
 
 ##############################
@@ -98,7 +98,7 @@ module "hub" {
 module "hub_secondary" {
   source            = "../../modules/hub"
   name              = join("-", [local.deployment_name, "secondary"])
-  subnet_id         = module.vpc.public_subnets[0]
+  subnet_id         = module.vpc.public_subnets[1]
   key_pair          = module.key_pair.key_pair_name
   web_console_sg_ingress_cidr = var.web_console_cidr
   sg_ingress_cidr   = concat(local.workstation_cidr, ["${module.hub.private_address}/32"])
@@ -111,7 +111,7 @@ module "agentless_gw" {
   count             = var.gw_count
   source            = "../../modules/gw"
   name              = local.deployment_name
-  subnet_id         = module.vpc.public_subnets[0]
+  subnet_id         = module.vpc.private_subnets[0]
   key_pair          = module.key_pair.key_pair_name
   sg_ingress_cidr   = concat(local.workstation_cidr, ["${module.hub.private_address}/32", "${module.hub_secondary.private_address}/32"])
   tarball_bucket_name = local.tarball_location.s3_bucket
@@ -182,3 +182,16 @@ module "gw_attachments" {
     module.hadr
   ]
 }
+
+# module "db_onboarding" {
+#   count = 1
+#   source = "../../modules/db_onboarding"
+#   hub_address = module.hub.public_address
+#   hub_ssh_key_path = resource.local_sensitive_file.dsf_ssh_key_file.filename
+#   assignee_gw = module.hub_install["primary_hub"].jsonar_uid
+# }
+
+# output "db_details" {
+#   value = module.db_onboarding
+#   sensitive = true
+# }

deploy/examples/se_demo_hadr/outputs.tf

@@ -18,11 +18,11 @@ output "dsf_hubs" {
 }
 
 output "dsf_hub_web_console_url" {
-    value     = can(module.hub.public_address) ? join("", ["https://", module.hub.public_address, ":8443/" ]) : null
+    value     = module.hub.public_address != null ? join("", ["https://", module.hub.public_address, ":8443/" ]) : null
 }
 
 output "primary_hub_ssh_command" {
-    value     = join("", ["ssh -i ${resource.local_sensitive_file.dsf_ssh_key_file.filename} ec2-user@", module.hub.public_address])
+    value     = module.hub.public_address != null ? join("", ["ssh -i ${resource.local_sensitive_file.dsf_ssh_key_file.filename} ec2-user@", module.hub.public_address]) : null
 }
 
 output "admin_password" {

deploy/examples/se_demo_hadr/variables.tf

@@ -8,9 +8,14 @@ variable "tarball_s3_bucket" {
   default = "1ef8de27-ed95-40ff-8c08-7969fc1b7901"
 }
 
+variable "sonar_version" {
+  type    = string
+  default = "4.10"
+}
+
 variable "tarball_s3_key" {
   type    = string
-  default = "jsonar-4.10.0.0.0-dev_20221006123138.tar.gz"
+  default = "jsonar-4.10.0.0.0-rc1_20221019194459.tar.gz"
 }
 
 variable "gw_count" {

deploy/modules/db_onboarding/artifacts/generate_token.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+client_id="terraform-automation"
+reason="Token autogenerated by terraform"
+
+# Generate access token to hub
+sudo curl -w '\n' \
+    --cacert $JSONAR_LOCALDIR/ssl/ca/ca.cert.pem \
+    --cert $JSONAR_LOCALDIR/ssl/client/admin/cert.pem \
+    --key $JSONAR_LOCALDIR/ssl/client/admin/key.pem \
+    -X POST 'https://localhost:27920/tokens' \
+    -H 'Content-type: application/json' \
+    -d '{"client_id":"'$client_id'","user":"admin","reason":"'"$reason"'","grants":["usc:access"]}' | cut -d\" -f4

deploy/modules/db_onboarding/artifacts/s3get.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+function s3get {
+	#helper functions
+	function fail {	echo "$1" > /dev/stderr; exit 1; }
+	#dependency check
+	if ! hash openssl 2>/dev/null; then fail "openssl not installed"; fi
+	if ! hash curl 2>/dev/null; then fail "curl not installed"; fi
+	#params
+	path="${1}"
+	bucket=$(cut -d '/' -f 1 <<< "$path")
+	key=$(cut -d '/' -f 2- <<< "$path")
+	#load creds
+	access="$AWS_ACCESS_KEY_ID"
+	secret="$AWS_SECRET_ACCESS_KEY"
+	#validate
+	if [[ "$bucket" = "" ]]; then fail "missing bucket (arg 1)"; fi;
+	if [[ "$key" = ""    ]]; then fail "missing key (arg 1)"; fi;
+	if [[ "$access" = "" ]]; then fail "missing AWS_ACCESS_KEY (env var)"; fi;
+	if [[ "$secret" = "" ]]; then fail "missing AWS_SECRET_KEY (env var)"; fi;
+	#compute signature
+	contentType="text/html; charset=UTF-8" 
+	date="`date -u +'%a, %d %b %Y %H:%M:%S GMT'`"
+	resource="/${bucket}/${key}"
+	string="GET\n\n${contentType}\n\nx-amz-date:${date}\n${resource}"
+	signature=`echo -en $string | openssl sha1 -hmac "${secret}" -binary | base64` 
+	#get!
+	curl -H "x-amz-date: ${date}" \
+		-H "Content-Type: ${contentType}" \
+		-H "Authorization: AWS ${access}:${signature}" \
+		"https://s3.amazonaws.com${resource}"
+}
+
+#example usage
+#s3get bucket/path/to/file > /tmp/file

deploy/modules/db_onboarding/onboarder.tpl

@@ -1,24 +1,28 @@
 #!/bin/bash -x
 set -e
 
-client_id=terraform-automation
-reason="Token autogenerated by terraform"
-
-# Generate access token to hub
-cat << EOF > generate_token.sh
-sudo curl -w '\n' \
-    --cacert \$JSONAR_LOCALDIR/ssl/ca/ca.cert.pem \
-    --cert \$JSONAR_LOCALDIR/ssl/client/admin/cert.pem \
-    --key \$JSONAR_LOCALDIR/ssl/client/admin/key.pem \
-    -X POST 'https://localhost:27920/tokens' \
-    -H 'Content-type: application/json' \
-    -d '{"client_id":"'$client_id'","user":"admin","reason":"'"$reason"'","grants":["usc:access"]}' | cut -d\" -f4
-EOF
-
-scp -o StrictHostKeyChecking="no" -i ${ssh_key_path} generate_token.sh ec2-user@${dsf_hub_address}:generate_token.sh
+scp -o StrictHostKeyChecking="no" -i ${ssh_key_path} ${module_path}/artifacts/generate_token.sh ec2-user@${dsf_hub_address}:generate_token.sh
 ssh -o StrictHostKeyChecking="no" -i ${ssh_key_path} ec2-user@${dsf_hub_address} -C "chmod +x ./generate_token.sh && ./generate_token.sh" > hub_token
 hub_token=$(cat hub_token)
 echo token: $hub_token
 
 # Run oboarder jar
-java -jar ${module_path}/artifacts/sonar_onboarder-1.4-SNAPSHOT-all.jar ${db_arn} ${dsf_hub_address} $hub_token ${assignee_gw} ${db_user} ${db_password}
+JAR=${module_path}/artifacts/sonar_onboarder-1.4.1-SNAPSHOT-all.jar
+JDK=jdk-16.0.2_linux-x64_bin.tar.gz
+JDK_BUCKET=1ef8de27-ed95-40ff-8c08-7969fc1b7901
+
+if command -v java &> /dev/null; then
+    java -jar $JAR ${db_arn} ${dsf_hub_address} $hub_token ${assignee_gw} ${db_user} ${db_password}
+else
+    echo "jave is not installed on the workstation. Copying jar to hub and run it from there"
+    if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+        echo "For overcming the lack of java problem, we need the have AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY defined"
+        exit 1
+    else
+        set -x
+        . ${module_path}/artifacts/s3get.sh
+        s3get $JDK_BUCKET/$JDK > $JDK
+        tar zxvf $JAR
+        ./jdk-16.0.2/bin/java -jar $JAR ${db_arn} ${dsf_hub_address} $hub_token ${assignee_gw} ${db_user} ${db_password}
+    fi
+fi

deploy/modules/gw/variables.tf

@@ -25,7 +25,7 @@ variable "public_ip" {
 
 variable "instance_type" {
   type = string
-  default = "c5.9xlarge"
+  default = "r6i.xlarge"
   description = "Ec2 instance type for the DSF agentless gw"
 }
 

deploy/modules/gw_attachment/federate_hub.tpl

@@ -3,4 +3,4 @@ set -e
 
 echo 'Running federation hub[${dsf_hub_ip}]->gw[${dsf_gw_ip}]'
 # ssh -o ConnectionAttempts=10 -o StrictHostKeyChecking=no -i ${ssh_key_path} ec2-user@${dsf_hub_ip} 'sudo "$JSONAR_BASEDIR"/bin/federated warehouse ${dsf_hub_ip} ${dsf_gw_ip}'
-ssh -o ConnectionAttempts=10 -o StrictHostKeyChecking=no -i ${ssh_key_path} ec2-user@${dsf_hub_ip} 'sudo "$JSONAR_BASEDIR"/bin/federated warehouse --new-remote ${dsf_hub_ip} ${dsf_gw_ip} || sudo bash -c "cat $JSONAR_LOGDIR/sonarw/replication.log; cat $JSONAR_LOGDIR/sonarw/sonarw.log; cat $JSONAR_LOGDIR/sonarg/federated.log"'
+ssh -o ConnectionAttempts=10 -o StrictHostKeyChecking=no -i ${ssh_key_path} ec2-user@${dsf_hub_ip} 'sudo "$JSONAR_BASEDIR"/bin/federated warehouse --new-remote ${dsf_hub_ip} ${dsf_gw_ip} || sudo bash -c "set -x; cat $JSONAR_LOGDIR/sonarw/replication.log; cat $JSONAR_LOGDIR/sonarw/sonarw.log; cat $JSONAR_LOGDIR/sonarg/federated.log"'

deploy/modules/hub/main.tf

@@ -9,7 +9,10 @@ resource "tls_private_key" "dsf_hub_ssh_federation_key" {
 
 locals {
   dsf_hub_ssh_federation_key = "${chomp(resource.tls_private_key.dsf_hub_ssh_federation_key.public_key_openssh)} produced-by-terraform"
-  secret_aws_arn = ! var.hadr_secondary_node ? resource.aws_secretsmanager_secret.dsf_hub_federation_private_key[0].arn : var.hadr_main_hub_sonarw_secret.arn
+  created_secret_aws_arn  = length(resource.aws_secretsmanager_secret.dsf_hub_federation_private_key) > 0 ? resource.aws_secretsmanager_secret.dsf_hub_federation_private_key[0].arn : ""
+  created_secret_aws_name = length(resource.aws_secretsmanager_secret.dsf_hub_federation_private_key) > 0 ? resource.aws_secretsmanager_secret.dsf_hub_federation_private_key[0].name : ""
+  secret_aws_arn  = ! var.hadr_secondary_node ? local.created_secret_aws_arn  : var.hadr_main_hub_sonarw_secret.arn
+  secret_aws_name = ! var.hadr_secondary_node ? local.created_secret_aws_name : var.hadr_main_hub_sonarw_secret.name
 }
 
 resource "aws_secretsmanager_secret" "dsf_hub_federation_public_key" {

deploy/modules/hub/outputs.tf

@@ -16,7 +16,7 @@ output "sonarw_public_key" {
 
 output "sonarw_secret" {
   value = {
-    name = ! var.hadr_secondary_node ? resource.aws_secretsmanager_secret.dsf_hub_federation_private_key[0].name : var.hadr_main_hub_sonarw_secret.name
-    arn = ! var.hadr_secondary_node ? resource.aws_secretsmanager_secret.dsf_hub_federation_private_key[0].arn : var.hadr_main_hub_sonarw_secret.arn
+    name = local.secret_aws_name
+    arn  = local.secret_aws_arn
   }
 }

deploy/modules/hub/variables.tf

@@ -19,7 +19,7 @@ variable "subnet_id" {
 
 variable "instance_type" {
   type = string
-  default = "c5.9xlarge"
+  default = "r6i.2xlarge"
   description = "Ec2 instance type for the DSF hub"
 }
 

deploy/modules/install/install.tpl

@@ -30,6 +30,7 @@ function setup() {
         --jsonar-datadir=$STATE_DIR/data \
         --jsonar-localdir=$STATE_DIR/local \
         --jsonar-logdir=$STATE_DIR/logs \
+        --instance-IP-or-DNS=${instance_fqdn} \
         $(test "${dsf_type}" == "gw" && echo "--remote-machine")
 }
 

deploy/modules/install/main.tf

@@ -27,6 +27,7 @@ data "template_file" "install" {
     ssh_key_pair_path   = var.ssh_key_pair_path
     sonarw_public_key   = var.sonarw_public_key
     sonarw_secret_name  = var.sonarw_secret_name
+    instance_fqdn       = var.instance_address
   }
 }
 

deploy/modules/sonar_base_instance/prepare_machine.tpl

@@ -54,6 +54,6 @@ function attach_disk() {
     mount -a
 }
 
-# wait_for_network
+wait_for_network
 install_deps
 attach_disk