Merge pull request #351 from FINRAOS/develop

OSS 0.50.0
FINRAOS · Aug 28, 2017 · 755430c · 755430c
2 parents 16803c9 + 57f8b93
commit 755430c
Show file tree

Hide file tree

Showing 113 changed files with 4,553 additions and 952 deletions.
diff --git a/herd-build/herd-build-tools/pom.xml b/herd-build/herd-build-tools/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-build</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd build tools</name>

diff --git a/herd-build/herd-docs/herd-docs-javadoc/pom.xml b/herd-build/herd-docs/herd-docs-javadoc/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-docs</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd docs javadoc</name>

diff --git a/herd-build/herd-docs/herd-docs-swagger/pom.xml b/herd-build/herd-docs/herd-docs-swagger/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-docs</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd docs swagger</name>

diff --git a/herd-build/herd-docs/pom.xml b/herd-build/herd-docs/pom.xml
@@ -22,7 +22,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-build</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd docs</name>

diff --git a/herd-build/herd-scripts/herd-scripts-bootstrap/pom.xml b/herd-build/herd-scripts/herd-scripts-bootstrap/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-scripts</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <description>This project contains the core herd EMR bootstrap scripts.</description>

diff --git a/herd-build/herd-scripts/herd-scripts-cloud-formation/pom.xml b/herd-build/herd-scripts/herd-scripts-cloud-formation/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-scripts</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd scripts cloud formation</name>

diff --git a/...ild/herd-scripts/herd-scripts-cloud-formation/src/main/resources/herd.localdb.template.vm b/...ild/herd-scripts/herd-scripts-cloud-formation/src/main/resources/herd.localdb.template.vm
@@ -8,7 +8,7 @@
     "InstanceType" : {
       "Description" : "Herd Application and Database Server EC2 instance type",
       "Type" : "String",
-      "Default" : "t1.micro",
+      "Default" : "m1.large",
       "AllowedValues" : [ "t1.micro", "t2.micro", "t2.small", "t2.medium",
                           "m1.small", "m1.medium", "m1.large", "m1.xlarge",
                           "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium",
@@ -146,6 +146,10 @@
       "Type" : "AWS::SQS::Queue"
     },
 
+    "searchIndexUpdateSqsQueue" : {
+      "Type" : "AWS::SQS::Queue"
+    },
+
     "herdS3Bucket" : {
       "Type" : "AWS::S3::Bucket",
       "Properties" : {
@@ -158,12 +162,40 @@
       "Metadata" : {
         "Comment" : "Install Herd",
         "AWS::CloudFormation::Init" : {
-          "config": {
+          "configSets" : {
+            "PreInstallAndInstall" : [ "PreInstall", "Install" ]
+          },
+          "PreInstall": {
+            "files": {
+              "/etc/yum.repos.d/elasticsearch.repo": {
+                "content": {
+                  "Fn::Join": [
+                    "\n", [
+                      "[elasticsearch-5.x]",
+                      "name=Elasticsearch repository for 5.x packages",
+                      "baseurl=https://artifacts.elastic.co/packages/5.x/yum",
+                      "gpgcheck=1",
+                      "gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch",
+                      "enabled=1",
+                      "autorefresh=1",
+                      "type=rpm-md\n"
+                    ]
+                  ]
+                },
+                "mode": "000644",
+                "owner": "root",
+                "group": "root"
+              }
+            }
+          },
+          "Install": {
             "packages": {
               "yum": {
                 "java-1.8.0-openjdk": [],
                 "postgresql9*": [],
-                "tomcat8": []
+                "tomcat8": [],
+                "elasticsearch": [],
+                "jq": []
               }
             },
             "commands": {
@@ -196,6 +228,7 @@
                       "action=/opt/aws/bin/cfn-init -v ",
                       "         --stack ", {"Ref": "AWS::StackName"},
                       "         --resource herdApplicationServer ",
+                      "         --configsets PreInstallAndInstall ",
                       "         --region ", {"Ref": "AWS::Region"}, "\n",
                       "runas=root\n"
                     ]
@@ -435,6 +468,7 @@
             },
             "services": {
               "sysvinit": {
+                "elasticsearch"  : { "enabled" : "true", "ensureRunning" : "true" },
                 "cfn-hup": {
                   "enabled": "true", "ensureRunning": "true",
                   "files": ["/etc/cfn/cfn-hup.conf", "/etc/cfn/hooks.d/cfn-auto-reloader.conf"]
@@ -459,11 +493,15 @@
           "/opt/aws/bin/cfn-init -v ",
           "         --stack ", { "Ref" : "AWS::StackName" },
           "         --resource herdApplicationServer ",
+          "         --configsets PreInstallAndInstall ",
           "         --region ", { "Ref" : "AWS::Region" }, "\n",
 
           "set -x\n",
           "exec > >(tee /var/log/user-data.log|logger -t user-data ) 2>&1\n",
 
+          "# Ensure Elasticsearch is running\n",
+          "/usr/bin/curl -s -XGET http://localhost:9200\n",
+
           "# Configure Postgresql\n",
           "service postgresql initdb\n",
           "service postgresql start\n",
@@ -495,15 +533,19 @@
           "psql -c \"INSERT INTO cnfgn VALUES ('s3.managed.bucket.name','",{"Ref": "herdS3Bucket" },"', NULL);\"\n",
           "psql -c \"DELETE FROM cnfgn WHERE cnfgn_key_nm = 'herd.notification.sqs.incoming.queue.name';\"\n",
           "psql -c \"INSERT INTO cnfgn VALUES ('herd.notification.sqs.incoming.queue.name','",{"Fn::GetAtt": ["herdQueueIn", "QueueName"]},"', NULL);\"\n",
+          "psql -c \"DELETE FROM cnfgn WHERE cnfgn_key_nm = 'search.index.update.sqs.queue.name';\"\n",
+          "psql -c \"INSERT INTO cnfgn VALUES ('search.index.update.sqs.queue.name','",{"Fn::GetAtt": ["searchIndexUpdateSqsQueue", "QueueName"]},"', NULL);\"\n",
 
           "service tomcat8 start\n",
           "\nsleep 15\n",
 
           "/usr/bin/wget http://localhost:8080/herd-app/rest/buildInfo -t12 --retry-connrefused -qO-\n",
           "\n",
+
           "/usr/bin/curl -s -X POST --header 'Content-Type: application/json' -d '{",
           "\"namespaceCode\": \"Demo_App_A\"",
           "}' http://localhost:8080/herd-app/rest/namespaces\n",
+
 #set($H = '#')
           "/usr/bin/curl -s -X POST --header 'Content-Type: application/json' -d '{",
           "\"attributes\": [",
@@ -537,7 +579,23 @@
           "\"namespace\":\"Demo_App_A\",\"businessObjectDefinitionName\":\"Data_Object_A\",\"businessObjectFormatUsage\":\"Source\",\"businessObjectFormatFileType\":\"BZ\",\"businessObjectFormatVersion\":0,\"partitionKey\":\"Transaction_Date\",\"partitionValue\":\"2015-06-01\",\"storageUnits\":[",
           "{\"storageName\":\"Demo_Storage\",\"storageFiles\":[",
           "{\"filePath\":\"demo-app-a\/exchange\/source\/bz\/data-object-a\/frmt-v0\/data-v0\/transaction-date=2015-06-01\/object_a.gz\",\"fileSizeBytes\":\"9511\",\"rowCount\":\"1000\"}","]}","],\"attributes\":[","{\"name\":\"Processor_Id\",\"value\":\"8801\"}","]",
-          "}' http://localhost:8080/herd-app/rest/businessObjectData\n ",
+          "}' http://localhost:8080/herd-app/rest/businessObjectData\n",
+
+          "bdef_index_name=$(/usr/bin/curl -s -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{",
+          "\"searchIndexType\": \"BUS_OBJCT_DFNTN\"",
+          "}' http://localhost:8080/herd-app/rest/searchIndexes | jq -r '.searchIndexKey.searchIndexName')\n",
+
+          "/usr/bin/curl -s -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{",
+          "\"searchIndexKey\": { \"searchIndexName\": \"${bdef_index_name}\" }",
+          "}' http://localhost:8080/herd-app/rest/searchIndexActivations\n",
+
+          "tag_index_name=$(/usr/bin/curl -s -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{",
+          "\"searchIndexType\": \"TAG\"",
+          "}' http://localhost:8080/herd-app/rest/searchIndexes | jq -r '.searchIndexKey.searchIndexName')\n",
+
+          "/usr/bin/curl -s -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{",
+          "\"searchIndexKey\": { \"searchIndexName\": \"${tag_index_name}\" }",
+          "}' http://localhost:8080/herd-app/rest/searchIndexActivations\n",
 
           "# If all went well, signal success\n",
           "/opt/aws/bin/cfn-signal -e $? -r 'Herd Server configuration' '", { "Ref" : "herdServerWaitHandle" }, "'\n"
@@ -583,6 +641,10 @@
     "SQSQueueName" : {
       "Value" : {"Fn::GetAtt": ["herdQueueIn", "QueueName"]},
       "Description" : "Name of Herd SQS in queue"
+    },
+    "SearchIndexUpdateSqsQueueName" : {
+      "Value" : {"Fn::GetAtt": ["searchIndexUpdateSqsQueue", "QueueName"]},
+      "Description" : "Name of Herd SQS in queue"
     }
   }
 }
diff --git a/herd-build/herd-scripts/herd-scripts-sql/pom.xml b/herd-build/herd-scripts/herd-scripts-sql/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-scripts</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd scripts sql</name>

diff --git a/...rd-scripts/herd-scripts-sql/src/main/resources/herd.postgres.0.49.0-to-0.50.0.upgrade.sql b/...rd-scripts/herd-scripts-sql/src/main/resources/herd.postgres.0.49.0-to-0.50.0.upgrade.sql
@@ -0,0 +1,25 @@
+/*
+* Copyright 2015 herd contributors
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+-----------------------------------------------------------------
+
+alter table srch_idx add column actv_fl char(1) NULL;
+
+insert into scrty_fn_lk (scrty_fn_cd, creat_ts, creat_user_id, updt_ts, updt_user_id)
+values ('FN_BUSINESS_OBJECT_DATA_ATTRIBUTES_ALL_PUT', current_timestamp, 'SYSTEM', current_timestamp, 'SYSTEM');
+
+insert into scrty_fn_lk (scrty_fn_cd, creat_ts, creat_user_id, updt_ts, updt_user_id)
+values ('FN_SEARCH_INDEXES_ACTIVATION_POST', current_timestamp, 'SYSTEM', current_timestamp, 'SYSTEM');
+
diff --git a/...rd-scripts/herd-scripts-sql/src/main/resources/herd.postgres.0.50.0-to-0.51.0.upgrade.sql b/...rd-scripts/herd-scripts-sql/src/main/resources/herd.postgres.0.50.0-to-0.51.0.upgrade.sql
@@ -0,0 +1,37 @@
+/*
+* Copyright 2015 herd contributors
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+-----------------------------------------------------------------
+
+create table bus_objct_dfntn_chg
+(
+  bus_objct_dfntn_chg_id     int8            NOT NULL,
+  bus_objct_dfntn_id         int8            NOT NULL,
+  desc_tx                    varchar(24000),
+  dsply_name_tx              varchar(100),
+  creat_ts                   timestamp       NOT NULL DEFAULT ('now'::text)::timestamp without time zone,
+  creat_user_id              varchar(100)    NOT NULL,
+  updt_ts                    timestamp       NOT NULL DEFAULT ('now'::text)::timestamp without time zone,
+  updt_user_id               varchar(100)    NOT NULL,
+  frmt_usage_cd              varchar(50),
+  frmt_file_type_cd          varchar(20)
+);
+
+alter table bus_objct_dfntn_chg add constraint bus_objct_dfntn_chg_pk  primary key (bus_objct_dfntn_chg_id); 
+alter table bus_objct_dfntn_chg add constraint bus_objct_dfntn_chg_fk1 foreign key (bus_objct_dfntn_id) references bus_objct_dfntn(bus_objct_dfntn_id);
+
+create index bus_objct_dfntn_chg_ix1 ON bus_objct_dfntn_chg(bus_objct_dfntn_id);
+
+create sequence bus_objct_dfntn_chg_seq;
diff --git a/herd-build/herd-scripts/pom.xml b/herd-build/herd-scripts/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-build</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd scripts</name>

diff --git a/herd-build/herd-swaggergen-maven-plugin/pom.xml b/herd-build/herd-swaggergen-maven-plugin/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-build</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd swaggergen maven plugin</name>

diff --git a/herd-build/pom.xml b/herd-build/pom.xml
@@ -22,7 +22,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd build</name>

diff --git a/herd-code/herd-app/pom.xml b/herd-code/herd-app/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-code</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd app</name>

diff --git a/herd-code/herd-core/pom.xml b/herd-code/herd-core/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-code</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd core</name>

diff --git a/herd-code/herd-dao/pom.xml b/herd-code/herd-dao/pom.xml
@@ -21,7 +21,7 @@
    <parent>
       <groupId>org.finra.herd</groupId>
       <artifactId>herd-code</artifactId>
-      <version>0.49.0-SNAPSHOT</version>
+      <version>0.51.0-SNAPSHOT</version>
    </parent>
 
    <name>herd dao</name>

diff --git a/herd-code/herd-dao/src/main/java/org/finra/herd/dao/IndexFunctionsDao.java b/herd-code/herd-dao/src/main/java/org/finra/herd/dao/IndexFunctionsDao.java
@@ -91,8 +91,9 @@ public interface IndexFunctionsDao extends BaseJpaDao
      * @param documentType document type
      * @param mapping mapping
      * @param settings setting
+      * @param alias alias
      */
-    public void createIndex(String indexName, String documentType, String mapping, String settings);
+     public void createIndex(String indexName, String documentType, String mapping, String settings, String alias);
 
     /**
      * The delete document by id function will delete a document in the index by the document id.

diff --git a/herd-code/herd-dao/src/main/java/org/finra/herd/dao/IndexSearchDao.java b/herd-code/herd-dao/src/main/java/org/finra/herd/dao/IndexSearchDao.java
@@ -34,5 +34,5 @@ public interface IndexSearchDao
      *
      * @return the index search response containing the search results
      */
-    IndexSearchResponse indexSearch(final IndexSearchRequest request, final Set<String> fields);
+    IndexSearchResponse indexSearch(final IndexSearchRequest request, final Set<String> fields, String bdefActiveIndex, String tagActiveIndex);
 }