wip, no tests

libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java

@@ -213,6 +213,13 @@ private static PrivateKey parsePKCS8(BufferedReader bReader) throws IOException,
         return keyFactory.generatePrivate(new PKCS8EncodedKeySpec(keyBytes));
     }
 
+    public static PrivateKey parsePKCS8PemString(String pem) throws IOException, GeneralSecurityException{
+        byte[] keyBytes = Base64.getDecoder().decode(pem);
+        String keyAlgo = getKeyAlgorithmIdentifier(keyBytes);
+        KeyFactory keyFactory = KeyFactory.getInstance(keyAlgo);
+        return keyFactory.generatePrivate(new PKCS8EncodedKeySpec(keyBytes));
+    }
+
     /**
      * Creates a {@link PrivateKey} from the contents of {@code bReader} that contains an EC private key encoded in
      * OpenSSL traditional format.

x-pack/plugin/security/cli/build.gradle

@@ -8,6 +8,7 @@ archivesBaseName = 'elasticsearch-security-cli'
 dependencies {
   compileOnly project(":server")
   compileOnly project(path: xpackModule('core'))
+  compileOnly project(path: xpackModule('security'))
   api "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
   api "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
   testImplementation("com.google.jimfs:jimfs:${versions.jimfs}") {

x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/ConfigInitialNode.java

@@ -266,6 +266,7 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th
                 // the PKCS12 keystore and the contained private key use the same password
                 transportKeystore.setKeyEntry(TRANSPORT_AUTOGENERATED_KEYSTORE_NAME, transportKeyPair.getPrivate(),
                         transportKeystorePassword.getChars(), new Certificate[]{transportCert});
+                transportKeystore.setCertificateEntry("catransport", transportCert);
                 fullyWriteFile(instantAutoConfigDir, TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12", false,
                         stream -> transportKeystore.store(stream, transportKeystorePassword.getChars()));
                 nodeKeystore.setString("xpack.security.transport.ssl.keystore.secure_password", transportKeystorePassword.getChars());

x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/ConfigAdditionalNodesTests.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.cli;
+
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+
+import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.PathUtilsForTesting;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.security.tool.CommandLineHttpClient;
+import org.elasticsearch.xpack.security.tool.HttpResponse;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ConfigAdditionalNodesTests extends ESTestCase {
+
+    static FileSystem jimfs;
+    private Path confDir;
+    private CommandLineHttpClient client;
+
+    @BeforeClass
+    public static void setupJimfs() {
+        Configuration conf = Configuration.unix().toBuilder().setAttributeViews("posix").build();
+        jimfs = Jimfs.newFileSystem(conf);
+        PathUtilsForTesting.installMock(jimfs);
+    }
+
+    @Before
+    public void setup() throws Exception {
+        Path homeDir = jimfs.getPath("eshome");
+        IOUtils.rm(homeDir);
+        confDir = homeDir.resolve("config");
+        Files.createDirectories(confDir);
+
+        HttpResponse nodeEnrollResponse = new HttpResponse(
+            HttpURLConnection.HTTP_OK,
+            Map.of("status", randomFrom("yellow", "green"))
+        );
+        this.client = mock(CommandLineHttpClient.class);
+        when(
+            client.execute(
+                anyString(),
+                any(URL.class),
+                anyString(),
+                any(SecureString.class),
+                any(CheckedSupplier.class),
+                any(CheckedFunction.class)
+            )
+        ).thenReturn(nodeEnrollResponse);
+
+    }
+
+    @AfterClass
+    public static void closeJimfs() throws IOException {
+        if (jimfs != null) {
+            jimfs.close();
+            jimfs = null;
+        }
+    }
+
+
+}

x-pack/plugin/security/src/main/bin/elasticsearch-enroll-node

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
+
+ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.ConfigAdditionalNodes \
+  ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
+  ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
+  "$(dirname "$0")/elasticsearch-cli" \
+  "$@"

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/EnrollmentToken.java

@@ -8,14 +8,22 @@
 package org.elasticsearch.xpack.security.enrollment;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ParseField;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 public class EnrollmentToken {
     private final String apiKey;
     private final String fingerprint;
@@ -27,6 +35,21 @@ public class EnrollmentToken {
     public String getVersion() { return version; }
     public List<String> getBoundAddress() { return boundAddress; }
 
+    private static final ParseField API_KEY = new ParseField("key");
+    private static final ParseField FINGERPRINT = new ParseField("fgr");
+    private static final ParseField VERSION = new ParseField("ver");
+    private static final ParseField ADDRESS = new ParseField("adr");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<EnrollmentToken, Void> PARSER = new ConstructingObjectParser<>("enrollment_token", false,
+        a -> new EnrollmentToken((String) a[0], (String) a[1], (String) a[2], (List<String>) a[3]));
+
+    static {
+        PARSER.declareString(constructorArg(), API_KEY);
+        PARSER.declareString(constructorArg(), FINGERPRINT);
+        PARSER.declareString(constructorArg(), VERSION);
+        PARSER.declareStringArray(constructorArg(), ADDRESS);
+    }
     /**
      * Create an EnrollmentToken
      *
@@ -61,4 +84,36 @@ public String getEncoded() throws Exception {
         final String jsonString = getRaw();
         return Base64.getUrlEncoder().encodeToString(jsonString.getBytes(StandardCharsets.UTF_8));
     }
+
+    /**
+     * Decodes and parses an enrollment token from it's serialized form (created with {@link EnrollmentToken#getEncoded()}
+     * @param encoded The Base64 encoded JSON representation of the enrollment token
+     * @return the parsed EnrollmentToken
+     * @throws IOException when failing to decode the serialized token
+     */
+    public static EnrollmentToken decodeFromString(String encoded) throws IOException {
+        if (Strings.isNullOrEmpty(encoded)) {
+            throw new IOException("Cannot decode enrollment token from an empty string");
+        }
+        final XContentParser jsonParser = JsonXContent.jsonXContent.createParser(
+            NamedXContentRegistry.EMPTY,
+            DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+            Base64.getDecoder().decode(encoded)
+        );
+        return EnrollmentToken.PARSER.parse(jsonParser, null);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        EnrollmentToken that = (EnrollmentToken) o;
+        return apiKey.equals(that.apiKey) && fingerprint.equals(that.fingerprint) && version.equals(that.version) && boundAddress.equals(
+            that.boundAddress);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(apiKey, fingerprint, version, boundAddress);
+    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/CommandLineHttpClient.java

@@ -6,7 +6,9 @@
  */
 package org.elasticsearch.xpack.security.tool;
 
+import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.core.CharArrays;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.Strings;
@@ -26,6 +28,9 @@
 import org.elasticsearch.xpack.security.tool.HttpResponse.HttpResponseBuilder;
 
 import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -34,9 +39,16 @@
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.CharBuffer;
 import java.nio.charset.StandardCharsets;
 import java.security.AccessController;
-import java.security.PrivilegedAction;
+import java.security.MessageDigest;
+import java.security.PrivilegedExceptionAction;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -60,9 +72,16 @@ public class CommandLineHttpClient {
     private static final int READ_TIMEOUT = 35 * 1000;
 
     private final Environment env;
+    private final String pinnedCaCertFingerprint;
 
     public CommandLineHttpClient(Environment env) {
         this.env = env;
+        this.pinnedCaCertFingerprint = null;
+    }
+
+    public CommandLineHttpClient(Environment env, String pinnedCaCertFingerprint) {
+        this.env = env;
+        this.pinnedCaCertFingerprint = pinnedCaCertFingerprint;
     }
 
     /**
@@ -79,22 +98,55 @@ public CommandLineHttpClient(Environment env) {
      *            handler of the response Input Stream.
      * @return HTTP protocol response code.
      */
-    @SuppressForbidden(reason = "We call connect in doPrivileged and provide SocketPermission")
     public HttpResponse execute(String method, URL url, String user, SecureString password,
                                 CheckedSupplier<String, Exception> requestBodySupplier,
                                 CheckedFunction<InputStream, HttpResponseBuilder, Exception> responseHandler) throws Exception {
+
+        final String authorizationHeader = UsernamePasswordToken.basicAuthHeaderValue(user, password);
+        return execute(method, url, authorizationHeader, requestBodySupplier, responseHandler);
+    }
+
+    /**
+     * General purpose HTTP(S) call with JSON Content-Type and Authorization Header.
+     * SSL settings are read from the settings file, if any.
+     *
+     * @param apiKey
+     *            API key value to be used in the Authorization header
+     * @param requestBodySupplier
+     *            supplier for the JSON string body of the request.
+     * @param responseHandler
+     *            handler of the response Input Stream.
+     * @return HTTP protocol response code.
+     */
+    public HttpResponse execute(String method, URL url, SecureString apiKey,
+        CheckedSupplier<String, Exception> requestBodySupplier,
+        CheckedFunction<InputStream, HttpResponseBuilder, Exception> responseHandler) throws Exception {
+        final String authorizationHeaderValue = apiKeyHeaderValue(apiKey);
+        return execute(method, url, authorizationHeaderValue, requestBodySupplier, responseHandler);
+    }
+
+    @SuppressForbidden(reason = "We call connect in doPrivileged and provide SocketPermission")
+    private HttpResponse execute(String method, URL url, String authorizationHeader,
+        CheckedSupplier<String, Exception> requestBodySupplier,
+        CheckedFunction<InputStream, HttpResponseBuilder, Exception> responseHandler) throws Exception {
         final HttpURLConnection conn;
         // If using SSL, need a custom service because it's likely a self-signed certificate
         if ("https".equalsIgnoreCase(url.getProtocol())) {
             final SSLService sslService = new SSLService(env);
             final HttpsURLConnection httpsConn = (HttpsURLConnection) url.openConnection();
-            AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
-                final SslConfiguration sslConfiguration = sslService.getHttpTransportSSLConfiguration();
-                // Requires permission java.lang.RuntimePermission "setFactory";
-                httpsConn.setSSLSocketFactory(sslService.sslSocketFactory(sslConfiguration));
-                final boolean isHostnameVerificationEnabled = sslConfiguration.getVerificationMode().isHostnameVerificationEnabled();
-                if (isHostnameVerificationEnabled == false) {
-                    httpsConn.setHostnameVerifier((hostname, session) -> true);
+            AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                if (pinnedCaCertFingerprint != null) {
+                    final SSLContext sslContext = SSLContext.getInstance("TLS");
+                    sslContext.init(null, new TrustManager[] { fingerprintTrustingTrustManager(pinnedCaCertFingerprint) }, null);
+                    httpsConn.setSSLSocketFactory(sslContext.getSocketFactory());
+                } else {
+                    final SslConfiguration sslConfiguration = sslService.getHttpTransportSSLConfiguration();
+                    // Requires permission java.lang.RuntimePermission "setFactory";
+                    httpsConn.setSSLSocketFactory(sslService.sslSocketFactory(sslConfiguration));
+                    final boolean isHostnameVerificationEnabled = sslConfiguration.getVerificationMode().isHostnameVerificationEnabled();
+                    if (isHostnameVerificationEnabled == false) {
+                        httpsConn.setHostnameVerifier((hostname, session) -> true);
+                    }
                 }
                 return null;
             });
@@ -105,8 +157,7 @@ public HttpResponse execute(String method, URL url, String user, SecureString pa
         conn.setRequestMethod(method);
         conn.setReadTimeout(READ_TIMEOUT);
         // Add basic-auth header
-        String token = UsernamePasswordToken.basicAuthHeaderValue(user, password);
-        conn.setRequestProperty("Authorization", token);
+        conn.setRequestProperty("Authorization", authorizationHeader);
         conn.setRequestProperty("Content-Type", XContentType.JSON.mediaType());
         String bodyString = requestBodySupplier.get();
         conn.setDoOutput(bodyString != null); // set true if we are sending a body
@@ -253,4 +304,48 @@ public static HttpResponse.HttpResponseBuilder responseBuilder(InputStream is) t
     public static URL createURL(URL url, String path, String query) throws MalformedURLException, URISyntaxException {
         return new URL(url, (url.toURI().getPath() + path).replaceAll("/+", "/") + query);
     }
+
+    public static String apiKeyHeaderValue(SecureString apiKey) {
+        CharBuffer chars = CharBuffer.allocate(apiKey.length());
+        byte[] charBytes = null;
+        try {
+            chars.put(apiKey.getChars());
+            charBytes = CharArrays.toUtf8Bytes(chars.array());
+
+            //TODO we still have passwords in Strings in headers. Maybe we can look into using a CharSequence?
+            String apiKeyToken = Base64.getEncoder().encodeToString(charBytes);
+            return "ApiKey " + apiKeyToken;
+        } finally {
+            Arrays.fill(chars.array(), (char) 0);
+            if (charBytes != null) {
+                Arrays.fill(charBytes, (byte) 0);
+            }
+        }
+    }
+
+    /**
+     * Returns a TrustManager to be used in a client SSLContext, which trusts all certificates that are signed
+     * by a specific CA certificate ( identified by its SHA256 fingerprint, {@code pinnedCaCertFingerPrint} )
+     */
+    private TrustManager fingerprintTrustingTrustManager(String pinnedCaCertFingerprint) {
+        final TrustManager trustManager = new X509TrustManager() {
+            public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+            }
+
+            public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                final Certificate caCertFromChain = chain[chain.length-1];
+                MessageDigest sha256 = MessageDigests.sha256();
+                sha256.update(caCertFromChain.getEncoded());
+                if (MessageDigests.toHexString(sha256.digest()).equals(pinnedCaCertFingerprint) == false ) {
+                    throw new CertificateException();
+                }
+            }
+
+            @Override public X509Certificate[] getAcceptedIssuers() {
+                return new X509Certificate[0];
+            }
+        };
+
+        return trustManager;
+    }
 }