Enable queryable built-in roles feature by default (#120323)

Making the `es.queryable_built_in_roles_enabled` feature flag enabled by default.
This feature makes the built-in roles automatically indexed in `.security` index and available
for querying via Query Role API. The consequence of this is that `.security` index is now 
created eagerly (if it's not existing) on cluster formation.

In order to keep the scope of this PR small, the feature is disabled for some of the tests, 
because they are either non-trivial to adjust or the gain is not worthy the effort to do it now. 
The tests will be adjusted in a follow-up PR and later the flag will be removed completely.

Relates to #117581

Author: slobodanadamovic

docs/build.gradle

@@ -120,6 +120,8 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
+
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml

@@ -240,7 +240,8 @@
             test: {}
 
   - do:
-      indices.get_alias: { }
+      indices.get_alias:
+        index: test*
   - match: { test1.aliases.test: { } }
   - match: { test2.aliases.test: { } }
   - match: { test3.aliases.test: { } }
@@ -255,7 +256,8 @@
   - is_true: acknowledged
 
   - do:
-      indices.get_alias: {}
+      indices.get_alias:
+        index: test*
   - match: {test1.aliases: {}}
   - match: {test2.aliases: {}}
   - match: {test3.aliases: {}}

modules/dot-prefix-validation/build.gradle

@@ -26,4 +26,5 @@ tasks.named('yamlRestTest') {
 
 tasks.named('yamlRestCompatTest') {
   usesDefaultDistribution()
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }

modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java

@@ -21,6 +21,8 @@
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.junit.ClassRule;
 
+import java.util.Objects;
+
 import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED;
 
 public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -55,6 +57,10 @@ private static ElasticsearchCluster createCluster() {
         if (setNodes) {
             clusterBuilder.nodes(2);
         }
+        clusterBuilder.systemProperty("es.queryable_built_in_roles_enabled", () -> {
+            final String enabled = System.getProperty("es.queryable_built_in_roles_enabled");
+            return Objects.requireNonNullElse(enabled, "");
+        });
         return clusterBuilder.build();
     }
 

modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml

@@ -2,7 +2,7 @@
 teardown:
   - do:
       indices.delete:
-        index: .*
+        index: .*,-.security-*
 
 ---
 "Index creation with a dot-prefix is deprecated unless x-elastic-product-origin set":

qa/packaging/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java

@@ -20,6 +20,7 @@
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
@@ -47,7 +48,9 @@ public void test010Install() throws Exception {
     public void test20GeneratePasswords() throws Exception {
         assertWhileRunning(() -> {
             ServerUtils.waitForElasticsearch(installation);
-            Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
+            Shell.Result result = retryOnAuthenticationErrors(
+                () -> installation.executables().setupPasswordsTool.run("auto --batch", null)
+            );
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout());
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
                 String response = ServerUtils.makeRequest(
@@ -102,20 +105,26 @@ public void test30AddBootstrapPassword() throws Exception {
         installation.executables().keystoreTool.run("add --stdin bootstrap.password", BOOTSTRAP_PASSWORD);
 
         assertWhileRunning(() -> {
-            String response = ServerUtils.makeRequest(
-                Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
-                "elastic",
-                BOOTSTRAP_PASSWORD,
-                null
+            ServerUtils.waitForElasticsearch("green", null, installation, "elastic", BOOTSTRAP_PASSWORD, null);
+            final String response = retryOnAuthenticationErrors(
+                () -> ServerUtils.makeRequest(
+                    Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
+                    "elastic",
+                    BOOTSTRAP_PASSWORD,
+                    null
+                )
             );
             assertThat(response, containsString("\"status\":\"green\""));
         });
+
     }
 
     public void test40GeneratePasswordsBootstrapAlreadySet() throws Exception {
         assertWhileRunning(() -> {
-
-            Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
+            ServerUtils.waitForElasticsearch("green", null, installation, "elastic", BOOTSTRAP_PASSWORD, null);
+            Shell.Result result = retryOnAuthenticationErrors(
+                () -> installation.executables().setupPasswordsTool.run("auto --batch", null)
+            );
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout());
             assertThat(userpasses, hasKey("elastic"));
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
@@ -130,6 +139,48 @@ public void test40GeneratePasswordsBootstrapAlreadySet() throws Exception {
         });
     }
 
+    /**
+     * The security index is created on startup.
+     * It can happen that even when the security index exists, we get an authentication failure as `elastic`
+     * user because the reserved realm checks the security index first.
+     * This is because we check the security index too early (just after the creation) when all shards did not get allocated yet.
+     * Hence, the call can result in an `UnavailableShardsException` and cause the authentication to fail.
+     * We retry here on authentication errors for a couple of seconds just to verify that this is not the case.
+     */
+    private <R> R retryOnAuthenticationErrors(final Callable<R> callable) throws Exception {
+        Exception failure = null;
+        int retries = 5;
+        while (retries-- > 0) {
+            try {
+                return callable.call();
+            } catch (Exception e) {
+                if (e.getMessage() != null
+                    && (e.getMessage().contains("401 Unauthorized") || e.getMessage().contains("Failed to authenticate user"))) {
+                    logger.info(
+                        "Authentication failed (possibly due to UnavailableShardsException for the security index), retrying [{}].",
+                        retries,
+                        e
+                    );
+                    if (failure == null) {
+                        failure = e;
+                    } else {
+                        failure.addSuppressed(e);
+                    }
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException interrupted) {
+                        Thread.currentThread().interrupt();
+                        failure.addSuppressed(interrupted);
+                        throw failure;
+                    }
+                } else {
+                    throw e;
+                }
+            }
+        }
+        throw failure;
+    }
+
     private Map<String, String> parseUsersAndPasswords(String output) {
         Matcher matcher = USERPASS_REGEX.matcher(output);
         assertNotNull(matcher);

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -66,7 +66,7 @@ public class ServerUtils {
     private static final long waitTime = TimeUnit.MINUTES.toMillis(3);
     private static final long timeoutLength = TimeUnit.SECONDS.toMillis(30);
     private static final long requestInterval = TimeUnit.SECONDS.toMillis(5);
-    private static final long dockerWaitForSecurityIndex = TimeUnit.SECONDS.toMillis(25);
+    private static final long dockerWaitForSecurityIndex = TimeUnit.SECONDS.toMillis(60);
 
     public static void waitForElasticsearch(Installation installation) throws Exception {
         final boolean securityEnabled;
@@ -260,9 +260,7 @@ public static void waitForElasticsearch(
                         // `elastic` , the reserved realm checks the security index first. It can happen that we check the security index
                         // too early after the security index creation in DockerTests causing an UnavailableShardsException. We retry
                         // authentication errors for a couple of seconds just to verify this is not the case.
-                        if (installation.distribution.isDocker()
-                            && timeElapsed < dockerWaitForSecurityIndex
-                            && response.getStatusLine().getStatusCode() == 401) {
+                        if (timeElapsed < dockerWaitForSecurityIndex && response.getStatusLine().getStatusCode() == 401) {
                             logger.info(
                                 "Authentication against docker failed (possibly due to UnavailableShardsException for the security index)"
                                     + ", retrying..."

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -18,6 +18,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
@@ -146,6 +147,8 @@
 import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING;
 import static org.elasticsearch.test.ESTestCase.TEST_REQUEST_TIMEOUT;
 import static org.elasticsearch.test.ESTestCase.assertBusy;
+import static org.elasticsearch.test.ESTestCase.assertFalse;
+import static org.elasticsearch.test.ESTestCase.assertTrue;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.test.ESTestCase.runInParallel;
 import static org.elasticsearch.test.ESTestCase.safeAwait;
@@ -160,9 +163,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -1240,16 +1241,29 @@ public synchronized void validateClusterFormed() {
         }
         logger.trace("validating cluster formed, expecting {}", expectedNodes);
 
-        assertFalse(
-            client().admin()
-                .cluster()
-                .prepareHealth(TEST_REQUEST_TIMEOUT)
-                .setWaitForEvents(Priority.LANGUID)
-                .setWaitForNodes(Integer.toString(expectedNodes.size()))
-                .get(TimeValue.timeValueSeconds(40))
-                .isTimedOut()
-        );
         try {
+            assertBusy(() -> {
+                try {
+                    final boolean timeout = client().admin()
+                        .cluster()
+                        .prepareHealth(TEST_REQUEST_TIMEOUT)
+                        .setWaitForEvents(Priority.LANGUID)
+                        .setWaitForNodes(Integer.toString(expectedNodes.size()))
+                        .get(TimeValue.timeValueSeconds(40))
+                        .isTimedOut();
+                    if (timeout) {
+                        throw new IllegalStateException("timed out waiting for cluster to form");
+                    }
+                } catch (UnavailableShardsException e) {
+                    if (e.getMessage() != null && e.getMessage().contains(".security")) {
+                        // security index may not be ready yet, throwing assertion error to retry
+                        throw new AssertionError(e);
+                    } else {
+                        throw e;
+                    }
+                }
+            }, 30, TimeUnit.SECONDS);
+
             final Object[] previousStates = new Object[1];
             assertBusy(() -> {
                 final List<ClusterState> states = nodes.values()

x-pack/plugin/build.gradle

@@ -104,3 +104,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
 
 })
 
+tasks.named('yamlRestCompatTest').configure {
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
+}

x-pack/plugin/core/build.gradle

@@ -158,6 +158,7 @@ testClusters.configureEach {
   keystore 'bootstrap.password', 'x-pack-test-password'
   user username: "x_pack_rest_user", password: "x-pack-test-password"
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.15.0")
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }
 
 if (buildParams.isSnapshotBuild() == false) {

x-pack/plugin/fleet/build.gradle

@@ -29,4 +29,5 @@ testClusters.configureEach {
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.autoconfiguration.enabled', 'false'
   user username: 'x_pack_rest_user', password: 'x-pack-test-password'
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }

x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java

@@ -48,6 +48,11 @@ protected Settings restAdminSettings() {
             .build();
     }
 
+    @Override
+    protected boolean preserveSecurityIndicesUponCompletion() {
+        return true;
+    }
+
     public void testAliasWithSystemDataStream() throws Exception {
         // Create a system data stream
         Request initialDocResponse = new Request("POST", ".fleet-actions-results/_doc");

x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySpecialUserIT.java

@@ -218,7 +218,7 @@ public void testAnonymousUserFromQueryClusterWorks() throws Exception {
                 { "password": "%s" }""", PASS));
             assertOK(client().performRequest(changePasswordRequest));
 
-            final Request elasticUserSearchRequest = new Request("GET", "/*:.security*/_search");
+            final Request elasticUserSearchRequest = new Request("GET", "/*:.security*/_search?size=1");
             elasticUserSearchRequest.setOptions(
                 RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue("elastic", PASS))
             );

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java

@@ -132,7 +132,8 @@ public void testQueryDLSFLSRolesShowAsDisabled() throws Exception {
                     .build() };
             createRoleWithIndicesPrivileges(adminClient(), "role_with_FLS_and_DLS", indicesPrivileges);
         }
-        assertQuery(client(), "", 4, roles -> {
+        assertQuery(client(), """
+            {"query":{"bool":{"must_not":{"term":{"metadata._reserved":true}}}}}""", 4, roles -> {
             roles.sort(Comparator.comparing(o -> ((String) o.get("name"))));
             assertThat(roles, iterableWithSize(4));
             assertThat(roles.get(0).get("name"), equalTo("role_with_DLS"));
@@ -152,7 +153,8 @@ public void testQueryDLSFLSRolesShowAsDisabled() throws Exception {
         assertTrue(((Boolean) responseMap.get("basic_was_started")));
         assertTrue(((Boolean) responseMap.get("acknowledged")));
         // now the same roles show up as disabled ("enabled" is "false")
-        assertQuery(client(), "", 4, roles -> {
+        assertQuery(client(), """
+            {"query":{"bool":{"must_not":{"term":{"metadata._reserved":true}}}}}""", 4, roles -> {
             roles.sort(Comparator.comparing(o -> ((String) o.get("name"))));
             assertThat(roles, iterableWithSize(4));
             assertThat(roles.get(0).get("name"), equalTo("role_with_DLS"));

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java

@@ -16,8 +16,10 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityMigrations;
 import org.hamcrest.Matchers;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -49,15 +51,23 @@ public final class QueryRoleIT extends SecurityInBasicRestTestCase {
 
     private static final String READ_SECURITY_USER_AUTH_HEADER = "Basic cmVhZF9zZWN1cml0eV91c2VyOnJlYWQtc2VjdXJpdHktcGFzc3dvcmQ=";
 
-    public void testSimpleQueryAllRoles() throws IOException {
-        assertQuery("", 0, roles -> assertThat(roles, emptyIterable()));
-        RoleDescriptor createdRole = createRandomRole();
-        assertQuery("", 1, roles -> {
-            assertThat(roles, iterableWithSize(1));
-            assertRoleMap(roles.get(0), createdRole);
+    @Before
+    public void initialize() {
+        new ReservedRolesStore();
+    }
+
+    public void testSimpleQueryAllRoles() throws Exception {
+        createRandomRole();
+        assertQuery("", 1 + ReservedRolesStore.names().size(), roles -> {
+            // default size is 10
+            assertThat(roles, iterableWithSize(10));
         });
-        assertQuery("""
-            {"query":{"match_all":{}},"from":1}""", 1, roles -> assertThat(roles, emptyIterable()));
+        assertQuery(
+            Strings.format("""
+                {"query":{"match_all":{}},"from":%d}""", 1 + ReservedRolesStore.names().size()),
+            1 + ReservedRolesStore.names().size(),
+            roles -> assertThat(roles, emptyIterable())
+        );
     }
 
     public void testDisallowedFields() throws Exception {