diff --git a/tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java b/tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java index c4e5104bdd..cb8380e519 100644 --- a/tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java +++ b/tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java @@ -18,8 +18,10 @@ */ package org.apache.tez.client.registry.zookeeper; +import java.util.Optional; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -27,6 +29,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.tez.dag.api.TezConfiguration; +import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.common.ClientX509Util; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -49,6 +53,11 @@ public class ZkConfig { private final int curatorMaxRetries; private final int sessionTimeoutMs; private final int connectionTimeoutMs; + private final String sslEnabled; + private final String sslKeystoreLocation; + private final String sslKeystorePassword; + private final String sslTruststoreLocation; + private final String sslTruststorePassword; public ZkConfig(Configuration conf) { zkQuorum = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM); @@ -84,6 +93,16 @@ public ZkConfig(Configuration conf) { TezConfiguration.TEZ_AM_CURATOR_SESSION_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS)); connectionTimeoutMs = Math.toIntExact(conf.getTimeDuration(TezConfiguration.TEZ_AM_CURATOR_CONNECTION_TIMEOUT, TezConfiguration.TEZ_AM_CURATOR_CONNECTION_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS)); + sslEnabled = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE); + Preconditions.checkArgument( + isValidSslEnabledValue(sslEnabled), + "If the optional %s setting is set, then the value should be a boolean value instead of '%s'", + TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE, + sslEnabled); + sslKeystoreLocation = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_KEYSTORE_LOCATION); + sslKeystorePassword = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD); + sslTruststoreLocation = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION); + sslTruststorePassword = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD); } public String getZkQuorum() { @@ -110,17 +129,91 @@ public int getConnectionTimeoutMs() { return connectionTimeoutMs; } + public String getZookeeperTrustStorePassword() { + return sslTruststorePassword; + } + + public String getZookeeperTrustStoreLocation() { + return sslTruststoreLocation; + } + + public String getZookeeperKeyStorePassword() { + return sslKeystorePassword; + } + + public String getZookeeperKeyStoreLocation() { + return sslKeystoreLocation; + } + + /** + * Returns whether the zookeeper connection will be secure or insecure. + * @return An Optional containing the boolean value that indicates whether zookeeper client + * uses a secure zookeeper connection. An empty Optional indicates that it is not specified, + * and in this case the default settings of zookeeper are used, which can be controlled by + * specific JVM properties. + * @see TezConfiguration#TEZ_AM_ZOOKEEPER_SSL_ENABLE + */ + public Optional isSslEnabled() { + if (this.sslEnabled == null || this.sslEnabled.isEmpty()) { + return Optional.empty(); + } + return Optional.of(Boolean.parseBoolean(sslEnabled)); + } + public RetryPolicy getRetryPolicy() { return new ExponentialBackoffRetry(getCuratorBackoffSleepMs(), getCuratorMaxRetries()); } public CuratorFramework createCuratorFramework() { - return CuratorFrameworkFactory.newClient( - getZkQuorum(), - getSessionTimeoutMs(), - getConnectionTimeoutMs(), - getRetryPolicy() - ); + if (isSslEnabled().isEmpty()) { + return CuratorFrameworkFactory.newClient( + getZkQuorum(), + getSessionTimeoutMs(), + getConnectionTimeoutMs(), + getRetryPolicy() + ); + } + + ZKClientConfig zkClientConfig = new ZKClientConfig(); + zkClientConfig.setProperty(ZKClientConfig.SECURE_CLIENT, Boolean.toString(isSslEnabled().get())); + zkClientConfig.setProperty(ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET, + "org.apache.zookeeper.ClientCnxnSocketNetty"); + if (isSslEnabled().get()) { + try (ClientX509Util x509Util = new ClientX509Util()) { + setStoreConfig(zkClientConfig, x509Util.getSslKeystoreLocationProperty(), getZookeeperKeyStoreLocation(), + x509Util.getSslKeystorePasswdProperty(), getZookeeperKeyStorePassword(), "keystore"); + + setStoreConfig(zkClientConfig, x509Util.getSslTruststoreLocationProperty(), getZookeeperTrustStoreLocation(), + x509Util.getSslTruststorePasswdProperty(), getZookeeperTrustStorePassword(), "truststore"); + } + } + + return CuratorFrameworkFactory.builder() + .connectString(getZkQuorum()) + .sessionTimeoutMs(getSessionTimeoutMs()) + .connectionTimeoutMs(getConnectionTimeoutMs()) + .retryPolicy(getRetryPolicy()) + .zkClientConfig(zkClientConfig) + .build(); + } + + private void setStoreConfig(ZKClientConfig config, String locationProp, String locationVal, String passwordProp, + String passwordVal, String storeName) { + if (StringUtils.isEmpty(locationVal)) { + LOG.info("No {} location configured, using ZooKeeper client defaults", storeName); + return; + } + + config.setProperty(locationProp, locationVal); + if (StringUtils.isNotEmpty(passwordVal)) { + config.setProperty(passwordProp, passwordVal); + } + } + + private boolean isValidSslEnabledValue(String value) { + return value == null || value.isEmpty() + || value.trim().equalsIgnoreCase("true") + || value.trim().equalsIgnoreCase("false"); } /** diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 0f7d6d3754..14bff85cab 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -2268,6 +2268,63 @@ static Set getPropertySet() { public static final String TEZ_SHARED_EXECUTOR_MAX_THREADS = "tez.shared-executor.max-threads"; public static final int TEZ_SHARED_EXECUTOR_MAX_THREADS_DEFAULT = -1; + /** + * Optional boolean value represented by string type. A value of "true" enables secure + * Zookeeper connection in ZkAMRegistry and ZkAMRegistryClient classes, while a value + * of "false" disables secure Zookeeper connection. + * If not specified or empty string, then zookeeper enables/disables the secure Zookeeper + * connection based on JVM properties. + * Default: Empty + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_AM_ZOOKEEPER_SSL_ENABLE = TEZ_AM_PREFIX + + "zookeeper.ssl.client.enable"; + + /** + * String value + * An optional setting that specifies the path to the keystore used for the secure + * zookeeper connection. + * Default: Empty + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_AM_ZOOKEEPER_SSL_KEYSTORE_LOCATION = TEZ_AM_PREFIX + + "zookeeper.ssl.keystore.location"; + + /** + * String value + * An optional setting that specifies the password of the keystore used for the secure + * zookeeper connection. + * Default: Empty + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_AM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD = TEZ_AM_PREFIX + + "zookeeper.ssl.keystore.password"; + + /** + * String value + * An optional setting that specifies the path to the truststore used for the secure + * zookeeper connection. + * Default: Empty + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_AM_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION = TEZ_AM_PREFIX + + "zookeeper.ssl.truststore.location"; + + /** + * String value + * An optional setting that specifies the password of the truststore used for the secure + * zookeeper connection. + * Default: Empty + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_AM_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD = TEZ_AM_PREFIX + + "zookeeper.ssl.truststore.password"; + /** * Acquire all FileSystems info. e.g., all namenodes info of HDFS federation cluster. */ diff --git a/tez-api/src/test/java/org/apache/tez/client/registry/zookeeper/TestZkConfig.java b/tez-api/src/test/java/org/apache/tez/client/registry/zookeeper/TestZkConfig.java index 3927e67710..9dd988d3b7 100644 --- a/tez-api/src/test/java/org/apache/tez/client/registry/zookeeper/TestZkConfig.java +++ b/tez-api/src/test/java/org/apache/tez/client/registry/zookeeper/TestZkConfig.java @@ -19,9 +19,13 @@ package org.apache.tez.client.registry.zookeeper; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Optional; import java.util.concurrent.TimeUnit; import org.apache.curator.RetryPolicy; @@ -231,4 +235,89 @@ public void testDefaultNamespace() { assertEquals("/tez-external-sessions" + TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE_DEFAULT, zkConfig.getZkNamespace()); } + + @Test + public void testZkConfigTezAmZookeeperSslEnableNotSpecified() { + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "dummyZkQuorum"); + ZkConfig zkConf = new ZkConfig(conf); + + assertEquals(Optional.empty(), zkConf.isSslEnabled()); + assertNull(zkConf.getZookeeperKeyStoreLocation()); + assertNull(zkConf.getZookeeperKeyStorePassword()); + assertNull(zkConf.getZookeeperTrustStoreLocation()); + assertNull(zkConf.getZookeeperTrustStorePassword()); + } + + @Test + public void testZkConfigTezAmZookeeperSslEnableEmpty() { + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "dummyZkQuorum"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE, ""); // empty means not set + ZkConfig zkConf = new ZkConfig(conf); + + assertEquals(Optional.empty(), zkConf.isSslEnabled()); + assertNull(zkConf.getZookeeperKeyStoreLocation()); + assertNull(zkConf.getZookeeperKeyStorePassword()); + assertNull(zkConf.getZookeeperTrustStoreLocation()); + assertNull(zkConf.getZookeeperTrustStorePassword()); + } + + @Test + public void testZkConfigSslEnabled() { + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "dummyZkQuorum"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE, "true"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_KEYSTORE_LOCATION, "/keystore.jks"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD, "secret"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION, "/truststore.jks"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD, "changeit"); + ZkConfig zkConf = new ZkConfig(conf); + + assertTrue(zkConf.isSslEnabled().isPresent()); + assertTrue(zkConf.isSslEnabled().get()); + assertEquals(zkConf.getZookeeperKeyStoreLocation(), "/keystore.jks"); + assertEquals(zkConf.getZookeeperKeyStorePassword(), "secret"); + assertEquals(zkConf.getZookeeperTrustStoreLocation(), "/truststore.jks"); + assertEquals(zkConf.getZookeeperTrustStorePassword(), "changeit"); + } + + @Test + public void testZkConfigSslDisabled() { + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "dummyZkQuorum"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE, "false"); + ZkConfig zkConf = new ZkConfig(conf); + + assertTrue(zkConf.isSslEnabled().isPresent()); + assertFalse(zkConf.isSslEnabled().get()); + assertNull(zkConf.getZookeeperKeyStoreLocation()); + assertNull(zkConf.getZookeeperKeyStorePassword()); + assertNull(zkConf.getZookeeperTrustStoreLocation()); + assertNull(zkConf.getZookeeperTrustStorePassword()); + } + + @Test + public void testZkConfigAmZookeeperSslEnableCaseInsensitive() { + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "dummyZkQuorum"); + + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE, "False"); + ZkConfig zkConfFalse = new ZkConfig(conf); + assertTrue(zkConfFalse.isSslEnabled().isPresent()); + assertFalse(zkConfFalse.isSslEnabled().get()); + + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE, "True"); + ZkConfig zkConfTrue = new ZkConfig(conf); + assertTrue(zkConfTrue.isSslEnabled().isPresent()); + assertTrue(zkConfTrue.isSslEnabled().get()); + } + + @Test + public void testZkConfigAmZookeeperSslEnableInvalid() { + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "dummyZkQuorum"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE, "invalidValue"); + assertThrows(IllegalArgumentException.class, () -> new ZkConfig(conf)); + } } diff --git a/tez-tests/pom.xml b/tez-tests/pom.xml index ba75496363..8920c17e65 100644 --- a/tez-tests/pom.xml +++ b/tez-tests/pom.xml @@ -133,6 +133,11 @@ org.junit.jupiter junit-jupiter + + org.apache.curator + curator-test + test + diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestZkAMRegistryClient.java b/tez-tests/src/test/java/org/apache/tez/test/TestZkAMRegistryClient.java new file mode 100644 index 0000000000..9cda1fbfbb --- /dev/null +++ b/tez-tests/src/test/java/org/apache/tez/test/TestZkAMRegistryClient.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tez.test; + +import static org.apache.tez.test.TestSecureShuffle.generateCertificate; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.util.HashMap; +import java.util.Map; + +import org.apache.curator.test.InstanceSpec; +import org.apache.curator.test.TestingServer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.tez.client.registry.AMRecord; +import org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient; +import org.apache.tez.dag.api.TezConfiguration; +import org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry; + +import com.google.common.collect.ImmutableMap; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +public class TestZkAMRegistryClient { + + private static final String KEYSTORE_PASSWORD = "secret"; + private static final String TRUSTSTORE_PASSWORD = "changeit"; + private static String testRootDir = "target" + Path.SEPARATOR + + TestZkAMRegistryClient.class.getName() + "-tmpDir"; + private static File keysStoresDir = new File(testRootDir, "keystores"); + private static String serverKS; + private static String trustKS; + + private static TestingServer zkServer; + private static Integer clientPort; + private static Integer secureClientPort; + + @BeforeAll + public static void setupZookeeperTestServer() throws Exception { + clientPort = InstanceSpec.getRandomPort(); + secureClientPort = InstanceSpec.getRandomPort(); + + setupKeyStores(); + + Map customProperties = ImmutableMap.of( + // NettyServerCnxnFactory required for SSL/TLS support + "serverCnxnFactory", "org.apache.zookeeper.server.NettyServerCnxnFactory", + // secureClientPort opens a new port for secure connections + "secureClientPort", Integer.toString(secureClientPort), + "ssl.clientAuth", "none", + "ssl.keyStore.location", serverKS, + "ssl.keyStore.password", KEYSTORE_PASSWORD, + "ssl.trustStore.location", trustKS, + "ssl.trustStore.password", TRUSTSTORE_PASSWORD, + "ssl.keyStore.type", "JKS", + "ssl.trustStore.type", "JKS" + ); + + // the clientPort parameter causes an insecure port to be opened + InstanceSpec spec = new InstanceSpec(null, clientPort, -1, -1, true, 1, -1, -1, customProperties); + zkServer = new TestingServer(spec, true); + } + + @AfterAll + public static void shutdownZookeeperTestServer() throws IOException { + zkServer.stop(); + } + + private void enableZookeeperSecureClientWithJVMProperties() { + System.setProperty("zookeeper.client.secure", "true"); + System.setProperty("zookeeper.clientCnxnSocket", "org.apache.zookeeper.ClientCnxnSocketNetty"); + } + + @AfterEach + public void clearZookeeperSecureClientJVMProperties() { + System.clearProperty("zookeeper.client.secure"); + System.clearProperty("zookeeper.clientCnxnSocket"); + } + + @Test + @Timeout(30) + public void testZkAMRegistryClient() throws Exception { + // configure zookeeper connection to use the insecure client port + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE, "/test-am-registry"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "localhost:" + clientPort); + + runAmRecordTestWithConfiguration(conf); + } + + @Test + @Timeout(30) + public void testZkAMRegistryClientWithSecureClientJVMProperties() throws Exception { + // this affects all zookeeper clients in JVM + enableZookeeperSecureClientWithJVMProperties(); + + // configure zookeeper connection to use the secure client port + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE, "/test-am-registry-with-secure-client-jvm-properties"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "localhost:" + secureClientPort); + + runAmRecordTestWithConfiguration(conf); + } + + @Test + @Timeout(30) + public void testZkAMRegistryClientWithSecureZookeeperPort() throws Exception { + // configure zookeeper connection to use the secure client port without JVM properties + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE, "/test-am-registry-with-secure-connection"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "localhost:" + secureClientPort); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE, "true"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION, trustKS); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD, TRUSTSTORE_PASSWORD); + + runAmRecordTestWithConfiguration(conf); + } + + @Test + @Timeout(30) + public void testZkAMRegistryClientWithInsecureZookeeperPort() throws Exception { + // this affects all zookeeper clients in JVM + enableZookeeperSecureClientWithJVMProperties(); + + // override the JVM properties above and configure zookeeper connection + // to use the insecure client port + Configuration conf = new Configuration(); + conf.set(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE, "/test-am-registry-with-insecure-connection"); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM, "localhost:" + clientPort); + conf.set(TezConfiguration.TEZ_AM_ZOOKEEPER_SSL_ENABLE, "false"); + + runAmRecordTestWithConfiguration(conf); + } + + private void runAmRecordTestWithConfiguration(Configuration conf) throws Exception { + String zkAMRegistryId = "testRegistry" + System.currentTimeMillis(); + try (ZkAMRegistry registry = new ZkAMRegistry(zkAMRegistryId)) { + registry.init(conf); + registry.start(); + + ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); + AMRecord amRecordRegistered = + new AMRecord(appId, "hostName", "testHostIp", 1234, "testExternalId", "testComputeName"); + registry.add(amRecordRegistered); + + ZkAMRegistryClient registryClient = ZkAMRegistryClient.getClient(conf); + registryClient.start(); + + // information registered in registry eventually reaches the registry client + AMRecord amRecordFetched = registryClient.getRecord(appId); + while (amRecordFetched == null) { + Thread.sleep(500); + amRecordFetched = registryClient.getRecord(appId); + } + assertEquals(amRecordFetched, amRecordRegistered); + + registryClient.close(); + } + } + + /** + * Create keystore and truststore for the tests. + */ + private static void setupKeyStores() throws Exception { + keysStoresDir.mkdirs(); + Map certs = new HashMap(); + + String localhostName = InetAddress.getLocalHost().getHostName(); + KeyPair sKP = KeyStoreTestUtil.generateKeyPair("RSA"); + X509Certificate sCert = + generateCertificate("CN="+localhostName+", O=server", sKP, 30, "SHA256WITHRSA"); + serverKS = keysStoresDir.getAbsolutePath() + "/serverKS.jks"; + KeyStoreTestUtil.createKeyStore(serverKS, KEYSTORE_PASSWORD, "server", sKP.getPrivate(), sCert); + certs.put("server", sCert); + trustKS = keysStoresDir.getAbsolutePath() + "/trustKS.jks"; + KeyStoreTestUtil.createTrustStore(trustKS, TRUSTSTORE_PASSWORD, certs); + } + +}