Merge branch 'main' of github.com:opensearch-project/security into ci…

…rcuit-breaker
derek-ho · Oct 4, 2024 · c81ee08 · c81ee08
2 parents 3bc92dd + 199662e
commit c81ee08
Show file tree

Hide file tree

Showing 130 changed files with 2,167 additions and 6,760 deletions.
diff --git a/build.gradle b/build.gradle
@@ -65,7 +65,7 @@ plugins {
     id 'com.diffplug.spotless' version '6.25.0'
     id 'checkstyle'
     id 'com.netflix.nebula.ospackage' version "11.10.0"
-    id "org.gradle.test-retry" version "1.5.10"
+    id "org.gradle.test-retry" version "1.6.0"
     id 'eclipse'
     id "com.github.spotbugs" version "5.2.5"
     id "com.google.osdetector" version "1.7.3"
@@ -471,7 +471,7 @@ configurations {
         resolutionStrategy {
             force 'commons-codec:commons-codec:1.17.1'
             force 'org.slf4j:slf4j-api:1.7.36'
-            force 'org.scala-lang:scala-library:2.13.14'
+            force 'org.scala-lang:scala-library:2.13.15'
             force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
             force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
             force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
@@ -482,7 +482,7 @@ configurations {
             force "io.netty:netty-transport:${versions.netty}"
             force "io.netty:netty-transport-native-unix-common:${versions.netty}"
             force "com.github.luben:zstd-jni:${versions.zstd}"
-            force "org.xerial.snappy:snappy-java:1.1.10.6"
+            force "org.xerial.snappy:snappy-java:1.1.10.7"
             force "com.google.guava:guava:${guava_version}"
 
             // for spotbugs dependency conflict
@@ -495,9 +495,9 @@ configurations {
             // For integrationTest
             force "org.apache.httpcomponents:httpclient:4.5.14"
             force "org.apache.httpcomponents:httpcore:4.4.16"
-            force "com.google.errorprone:error_prone_annotations:2.31.0"
-            force "org.checkerframework:checker-qual:3.46.0"
-            force "ch.qos.logback:logback-classic:1.5.7"
+            force "com.google.errorprone:error_prone_annotations:2.32.0"
+            force "org.checkerframework:checker-qual:3.47.0"
+            force "ch.qos.logback:logback-classic:1.5.8"
         }
     }
 
@@ -580,7 +580,7 @@ dependencies {
     implementation 'commons-cli:commons-cli:1.9.0'
     implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
     implementation 'org.ldaptive:ldaptive:1.2.3'
-    implementation 'com.nimbusds:nimbus-jose-jwt:9.40'
+    implementation 'com.nimbusds:nimbus-jose-jwt:9.41.1'
     implementation 'com.rfksystems:blake2b:2.0.0'
     implementation 'com.password4j:password4j:1.8.2'
     //JWT
@@ -602,7 +602,7 @@ dependencies {
     runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5'
     runtimeOnly 'commons-codec:commons-codec:1.17.1'
     runtimeOnly 'org.cryptacular:cryptacular:1.2.7'
-    compileOnly 'com.google.errorprone:error_prone_annotations:2.31.0'
+    compileOnly 'com.google.errorprone:error_prone_annotations:2.32.0'
     runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0'
     runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
     runtimeOnly 'org.ow2.asm:asm:9.7'
@@ -639,14 +639,14 @@ dependencies {
     runtimeOnly 'org.lz4:lz4-java:1.8.0'
     runtimeOnly 'org.slf4j:slf4j-api:1.7.36'
     runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
-    runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.6'
+    runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.7'
     runtimeOnly 'org.codehaus.woodstox:stax2-api:4.2.2'
     runtimeOnly "org.glassfish.jaxb:txw2:${jaxb_version}"
     runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.7.0'
     runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1'
     runtimeOnly 'org.apache.santuario:xmlsec:2.3.4'
     runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}"
-    runtimeOnly 'org.checkerframework:checker-qual:3.46.0'
+    runtimeOnly 'org.checkerframework:checker-qual:3.47.0'
     runtimeOnly "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
     runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2'
 
@@ -679,8 +679,8 @@ dependencies {
     testImplementation 'commons-validator:commons-validator:1.9.0'
     testImplementation 'org.springframework.kafka:spring-kafka-test:2.9.13'
     testImplementation "org.springframework:spring-beans:${spring_version}"
-    testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0'
-    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.11.0'
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.11.1'
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.11.1'
     testImplementation('org.awaitility:awaitility:4.2.2') {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }
@@ -696,7 +696,7 @@ dependencies {
     testRuntimeOnly ("org.springframework:spring-core:${spring_version}") {
         exclude(group:'org.springframework', module: 'spring-jcl' )
     }
-    testRuntimeOnly 'org.scala-lang:scala-library:2.13.14'
+    testRuntimeOnly 'org.scala-lang:scala-library:2.13.15'
     testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5'
     testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.2') {
         exclude(group:'ch.qos.logback', module: 'logback-classic' )
@@ -718,7 +718,7 @@ dependencies {
     integrationTestImplementation 'junit:junit:4.13.2'
     integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
     integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
-    integrationTestImplementation 'commons-io:commons-io:2.16.1'
+    integrationTestImplementation 'commons-io:commons-io:2.17.0'
     integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
     integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
     integrationTestImplementation 'org.hamcrest:hamcrest:2.2'

diff --git a/config/roles.yml b/config/roles.yml
@@ -32,6 +32,7 @@ alerting_read_access:
     - 'cluster:admin/opendistro/alerting/destination/get'
     - 'cluster:admin/opendistro/alerting/monitor/get'
     - 'cluster:admin/opendistro/alerting/monitor/search'
+    - 'cluster:admin/opensearch/alerting/comments/search'
     - 'cluster:admin/opensearch/alerting/findings/get'
     - 'cluster:admin/opensearch/alerting/remote/indexes/get'
     - 'cluster:admin/opensearch/alerting/workflow/get'
@@ -44,6 +45,7 @@ alerting_ack_alerts:
     - 'cluster:admin/opendistro/alerting/alerts/*'
     - 'cluster:admin/opendistro/alerting/chained_alerts/*'
     - 'cluster:admin/opendistro/alerting/workflow_alerts/*'
+    - 'cluster:admin/opensearch/alerting/comments/*'
 
 # Allows users to use all alerting functionality
 alerting_full_access:
@@ -84,7 +86,12 @@ anomaly_full_access:
         - '*'
       allowed_actions:
         - 'indices:admin/aliases/get'
+        - 'indices:admin/mappings/fields/get'
+        - 'indices:admin/mappings/fields/get*'
         - 'indices:admin/mappings/get'
+        - 'indices:admin/resolve/index'
+        - 'indices:data/read/field_caps*'
+        - 'indices:data/read/search'
         - 'indices_monitor'
 
 # Allow users to execute read only k-NN actions
@@ -409,7 +416,7 @@ security_analytics_ack_alerts:
   reserved: true
   cluster_permissions:
     - 'cluster:admin/opensearch/securityanalytics/alerts/*'
-    - 'cluster:admin/opensearch/securityanalytics/correlationAlerts/ack'
+    - 'cluster:admin/opensearch/securityanalytics/correlationAlerts/*'
     - 'cluster:admin/opensearch/securityanalytics/threatintel/alerts/*'
 
 # Allows users to use all Flow Framework functionality

diff --git a/release-notes/opensearch-security.release-notes-2.17.0.0.md b/release-notes/opensearch-security.release-notes-2.17.0.0.md
@@ -0,0 +1,39 @@
+## Version 2.17.0 Release Notes
+
+Compatible with OpenSearch and OpenSearch Dashboards version 2.17.0
+
+### Enhancements
+* Add `ignore_hosts` config option for auth failure listener ([#4538](https://github.com/opensearch-project/security/pull/4538))
+* added API roles for correlationAlerts ([#4689](https://github.com/opensearch-project/security/pull/4689))
+* Allow multiple signing keys to be provided ([#4666](https://github.com/opensearch-project/security/pull/4666))
+* adding alerting comments security actions to roles.yml ([#4700](https://github.com/opensearch-project/security/pull/4700))
+* Permission changes for correlationAlerts ([#4704](https://github.com/opensearch-project/security/pull/4704))
+
+### Bug Fixes
+* Addresses a bug with `plugins.security.allow_unsafe_democertificates` setting ([#4603](https://github.com/opensearch-project/security/pull/4603))
+* Fix covereage-report workflow ([#4684](https://github.com/opensearch-project/security/pull/4684), [#4683](https://github.com/opensearch-project/security/pull/4683))
+* Handle the audit config being null ([#4664](https://github.com/opensearch-project/security/pull/4664))
+* Fixes authtoken endpoint ([#4631](https://github.com/opensearch-project/security/pull/4631))
+* Fixed READ_ACTIONS required by TermsAggregationEvaluator ([#4607](https://github.com/opensearch-project/security/pull/4607))
+* Sort the DNS Names in the SANs ([#4640](https://github.com/opensearch-project/security/pull/4640))
+
+### Maintenance
+* Bump com.google.errorprone:error_prone_annotations from 2.30.0 to 2.31.0 ([#4696](https://github.com/opensearch-project/security/pull/4696))
+* Bump org.passay:passay from 1.6.4 to 1.6.5 ([#4682](https://github.com/opensearch-project/security/pull/4682))
+* Bump spring_version from 5.3.37 to 5.3.39 ([#4661](https://github.com/opensearch-project/security/pull/4661))
+* Bump commons-cli:commons-cli from 1.8.0 to 1.9.0 ([#4659](https://github.com/opensearch-project/security/pull/4659))
+* Bump org.junit.jupiter:junit-jupiter from 5.10.3 to 5.11.0 ([#4657](https://github.com/opensearch-project/security/pull/4657))
+* Bump org.cryptacular:cryptacular from 1.2.6 to 1.2.7 ([#4656](https://github.com/opensearch-project/security/pull/4656))
+* Update Gradle to 8.10 ([#4646](https://github.com/opensearch-project/security/pull/4646))
+* Bump org.xerial.snappy:snappy-java from 1.1.10.5 to 1.1.10.6 ([#4639](https://github.com/opensearch-project/security/pull/4639))
+* Bump com.google.googlejavaformat:google-java-format from 1.22.0 to 1.23.0 ([#4622](https://github.com/opensearch-project/security/pull/4622))
+* Increment version to 2.17.0-SNAPSHOT ([#4615](https://github.com/opensearch-project/security/pull/4615))
+* Backports PRs with `backport-failed` labels that weren't actually backported ([#4610](https://github.com/opensearch-project/security/pull/4610))
+* Bump io.dropwizard.metrics:metrics-core from 4.2.26 to 4.2.27 ([#4660](https://github.com/opensearch-project/security/pull/4660))
+* Bump com.netflix.nebula.ospackage from 11.9.1 to 11.10.0 ([#4681](https://github.com/opensearch-project/security/pull/4681))
+* Interim build fix for PluginSubject related changes ([#4694](https://github.com/opensearch-project/security/pull/4694))
+* Add Nils Bandener (Github: nibix) as a maintainer ([#4673](https://github.com/opensearch-project/security/pull/4673))
+* Remove usages of org.apache.logging.log4j.util.Strings ([#4653](https://github.com/opensearch-project/security/pull/4653))
+* Update backport section of PR template ([#4625](https://github.com/opensearch-project/security/pull/4625))
+* Bump org.checkerframework:checker-qual from 3.45.0 to 3.46.0 ([#4623](https://github.com/opensearch-project/security/pull/4623))
+* Refactor security provider instantiation ([#4611](https://github.com/opensearch-project/security/pull/4611))
diff --git a/src/integrationTest/java/org/opensearch/security/FlsAndFieldMaskingTests.java b/src/integrationTest/java/org/opensearch/security/FlsAndFieldMaskingTests.java
@@ -62,6 +62,7 @@
 import org.opensearch.test.framework.cluster.TestRestClient;
 import org.opensearch.test.framework.log.LogsRule;
 
+import static org.apache.http.HttpStatus.SC_OK;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
@@ -230,6 +231,12 @@ public class FlsAndFieldMaskingTests {
         "cluster_composite_ops_ro"
     ).indexPermissions("read").fls(String.format("~%s", FIELD_TITLE)).on(FIRST_INDEX_NAME);
 
+    static final TestSecurityConfig.Role ROLE_NO_FIELD_TITLE_WILDCARD_INDEX_FLS = new TestSecurityConfig.Role("example_exclusive_fls")
+        .clusterPermissions("cluster_composite_ops_ro")
+        .indexPermissions("read", "indices:admin/mappings/get")
+        .fls(String.format("~%s", FIELD_TITLE))
+        .on("*");
+
     static final TestSecurityConfig.Role ROLE_ONLY_FIELD_TITLE_MASKED = new TestSecurityConfig.Role("example_mask").clusterPermissions(
         "cluster_composite_ops_ro"
     ).indexPermissions("read").maskedFields(FIELD_TITLE.concat("::/(?<=.{1})./::").concat(MASK_VALUE)).on(FIRST_INDEX_NAME);
@@ -248,6 +255,12 @@ public class FlsAndFieldMaskingTests {
         ROLE_NO_FIELD_TITLE_FLS
     );
 
+    /**
+     * Example user with fls filter in which the user can see every field but the {@link Song#FIELD_TITLE} field.
+     */
+    static final TestSecurityConfig.User USER_NO_FIELD_TITLE_WILDCARD_INDEX_FLS = new TestSecurityConfig.User("exclusive_wildcard_fls_user")
+        .roles(ROLE_NO_FIELD_TITLE_WILDCARD_INDEX_FLS);
+
     /**
      * Example user in which {@link Song#FIELD_TITLE} field is masked.
      */
@@ -305,6 +318,7 @@ public class FlsAndFieldMaskingTests {
             ALL_INDICES_STARS_LESS_THAN_ZERO_READER,
             TWINS_FIRST_ARTIST_READER,
             USER_ONLY_FIELD_TITLE_FLS,
+            USER_NO_FIELD_TITLE_WILDCARD_INDEX_FLS,
             USER_NO_FIELD_TITLE_FLS,
             USER_ONLY_FIELD_TITLE_MASKED,
             USER_BOTH_ONLY_AND_NO_FIELD_TITLE_FLS,
@@ -1778,4 +1792,26 @@ public void testFlsOnAClosedAndReopenedIndex() throws IOException {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    @Test
+    public void testGetMappingsOnAClosedIndexWithFlsRestrictions() throws IOException {
+        String indexName = "fls_excludes_mappings";
+        List<String> docIds = createIndexWithDocs(indexName, SONGS[0], SONGS[1]);
+
+        try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) {
+            TestRestClient.HttpResponse mappingsResponse = client.get(indexName + "/_mapping");
+            mappingsResponse.assertStatusCode(SC_OK);
+            assertThat(mappingsResponse.getBody(), containsString("title"));
+
+            TestRestClient.HttpResponse closeResponse = client.post(indexName + "/_close");
+            closeResponse.assertStatusCode(SC_OK);
+        }
+
+        try (TestRestClient client = cluster.getRestClient(USER_NO_FIELD_TITLE_WILDCARD_INDEX_FLS)) {
+            TestRestClient.HttpResponse mappingsResponse = client.get(indexName + "/_mapping");
+            mappingsResponse.assertStatusCode(SC_OK);
+            assertThat(mappingsResponse.getBody(), not(containsString("title")));
+        }
+    }
+
 }
diff --git a/src/integrationTest/java/org/opensearch/security/SystemIndexTests.java b/src/integrationTest/java/org/opensearch/security/SystemIndexTests.java
@@ -80,4 +80,29 @@ public void adminShouldNotBeAbleToDeleteSecurityIndex() {
             assertThat(response4.getStatusCode(), equalTo(RestStatus.FORBIDDEN.getStatus()));
         }
     }
+
+    @Test
+    public void regularUserShouldGetNoResultsWhenSearchingSystemIndex() {
+        // Create system index and index a dummy document as the super admin user, data returned to super admin
+        try (TestRestClient client = cluster.getRestClient(cluster.getAdminCertificate())) {
+            HttpResponse response1 = client.put(".system-index1");
+
+            assertThat(response1.getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+            String doc = "{\"field\":\"value\"}";
+            HttpResponse adminPostResponse = client.postJson(".system-index1/_doc/1?refresh=true", doc);
+            assertThat(adminPostResponse.getStatusCode(), equalTo(RestStatus.CREATED.getStatus()));
+            HttpResponse response2 = client.get(".system-index1/_search");
+
+            assertThat(response2.getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+            assertThat(response2.getBody(), response2.getBody().contains("\"hits\":{\"total\":{\"value\":1,\"relation\":\"eq\"}"));
+        }
+
+        // Regular users should not be able to read it
+        try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) {
+            // regular user cannot read system index
+            HttpResponse response1 = client.get(".system-index1/_search");
+
+            assertThat(response1.getBody(), response1.getBody().contains("\"hits\":{\"total\":{\"value\":0,\"relation\":\"eq\"}"));
+        }
+    }
 }
diff --git a/...tegrationTest/java/org/opensearch/security/api/DefaultApiAvailabilityIntegrationTest.java b/...tegrationTest/java/org/opensearch/security/api/DefaultApiAvailabilityIntegrationTest.java
@@ -95,16 +95,11 @@ private void verifyAuthInfoApi(final TestRestClient client) throws Exception {
 
     @Test
     public void flushCache() throws Exception {
-        withUser(NEW_USER, client -> {
-            forbidden(() -> client.get(apiPath("cache")));
-            forbidden(() -> client.postJson(apiPath("cache"), EMPTY_BODY));
-            forbidden(() -> client.putJson(apiPath("cache"), EMPTY_BODY));
-            forbidden(() -> client.delete(apiPath("cache")));
-        });
+        withUser(NEW_USER, client -> { forbidden(() -> client.delete(apiPath("cache"))); });
         withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), client -> {
-            notImplemented(() -> client.get(apiPath("cache")));
-            notImplemented(() -> client.postJson(apiPath("cache"), EMPTY_BODY));
-            notImplemented(() -> client.putJson(apiPath("cache"), EMPTY_BODY));
+            methodNotAllowed(() -> client.get(apiPath("cache")));
+            methodNotAllowed(() -> client.postJson(apiPath("cache"), EMPTY_BODY));
+            methodNotAllowed(() -> client.putJson(apiPath("cache"), EMPTY_BODY));
             final var response = ok(() -> client.delete(apiPath("cache")));
             assertThat(response.getBody(), response.getTextFromJsonBody("/message"), is("Cache flushed successfully."));
         });