From ee5d5e908b829a94164dbb4bd87ebda215afcf6b Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Tue, 28 Jul 2026 18:13:12 -0700 Subject: [PATCH 01/25] [common][test] Fail fast when pub-sub adapter factory class is not configured PubSubClientsFactory previously defaulted the producer, consumer, and admin adapter factories to the Apache Kafka implementation whenever the corresponding `*.adapter.factory.class` config was absent. On non-Kafka (e.g. xinfra) deployments this silently masked misconfiguration and produced a Kafka client that could not talk to the configured backend. Make the fallback config-driven and fail-fast by default: - Add `pubsub.adapter.factory.kafka.fallback.enabled` (default `false`). - When a factory-class config is missing and the fallback is disabled, `PubSubClientsFactory` now throws a `VeniceException` naming the missing config key instead of silently constructing the Apache Kafka factory. - Set the flag to `true` to restore the legacy implicit-Kafka behavior. To keep integration/e2e tests working after disabling the implicit fallback, `KafkaBrokerFactory.getAdditionalConfig()` now advertises the Apache Kafka producer/consumer/admin factory classes so they propagate through `PubSubBrokerWrapper.getBrokerDetailsForClients()` to every component. Unit tests updated: missing-config now asserts fail-fast for producer, consumer, admin, source-of-truth admin, and the eager instance constructor; a new test covers the explicit fallback-enabled path. --- .../java/com/linkedin/venice/ConfigKeys.java | 13 +++++ .../venice/pubsub/PubSubClientsFactory.java | 23 ++++++++ .../pubsub/PubSubClientsFactoryTest.java | 58 ++++++++++++++++--- .../integration/utils/KafkaBrokerFactory.java | 16 ++++- 4 files changed, 101 insertions(+), 9 deletions(-) diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java index 22ccda8c292..36b178093ed 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java @@ -116,6 +116,19 @@ private ConfigKeys() { public static final String PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS = PUBSUB_CLIENT_CONFIG_PREFIX + "source.of.truth.admin.adapter.factory.class"; + /** + * Configuration key that controls whether the PubSub producer/consumer/admin adapter factories + * silently fall back to the Apache Kafka implementation when their factory-class config keys are + * not explicitly provided. + *

+ * When {@code false} (the default), the {@code PubSubClientsFactory} fails fast by throwing an + * exception if the corresponding factory-class config is missing. This surfaces misconfiguration + * early instead of masking it behind an implicit Kafka default. Set this to {@code true} to + * restore the legacy behavior of defaulting to the Apache Kafka adapter factories. + */ + public static final String PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED = + PUBSUB_CLIENT_CONFIG_PREFIX + "adapter.factory.kafka.fallback.enabled"; + /** * Configuration key for specifying the address of the PubSub broker (e.g., Kafka, Pulsar). *

diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java index cf376c2d178..2b9d465c838 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java @@ -1,5 +1,6 @@ package com.linkedin.venice.pubsub; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; @@ -29,6 +30,14 @@ public class PubSubClientsFactory { private static final Logger LOGGER = LogManager.getLogger(PubSubClientsFactory.class); + /** + * By default the adapter factories do NOT fall back to Apache Kafka when their factory-class config + * is missing; callers must configure the factory classes explicitly so that misconfiguration fails + * fast. Set {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED} to + * {@code true} to restore the legacy implicit-Kafka behavior. + */ + public static final boolean DEFAULT_KAFKA_FALLBACK_ENABLED = false; + private enum FactoryType { PRODUCER, CONSUMER, ADMIN } @@ -112,6 +121,20 @@ private static T createFactory( className = properties.getStringWithAlternative(preferredConfigKey, alternateConfigKey); LOGGER.debug("Creating pub-sub {} adapter factory instance for class: {}", factoryType, className); } else { + boolean kafkaFallbackEnabled = + properties.getBoolean(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, DEFAULT_KAFKA_FALLBACK_ENABLED); + if (!kafkaFallbackEnabled) { + throw new VeniceException( + String.format( + "PubSub %s adapter factory class is not configured. Set '%s' (or the legacy '%s') to the " + + "fully-qualified factory class name. Implicit fallback to the Apache Kafka adapter factory " + + "('%s') is disabled; set '%s=true' to re-enable it.", + factoryType, + preferredConfigKey, + alternateConfigKey, + defaultClassName, + PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED)); + } className = defaultClassName; LOGGER.debug("Creating pub-sub {} adapter factory instance with default class: {}", factoryType, className); } diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java index 02aff808438..a0b7b55be20 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java @@ -1,13 +1,16 @@ package com.linkedin.venice.pubsub; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_PRODUCER_ADAPTER_FACTORY_CLASS; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; import com.linkedin.venice.exceptions.VeniceException; @@ -26,13 +29,6 @@ public class PubSubClientsFactoryTest { @Test public void testCreateInstanceSuccess() { - // default: no config provided - verifyFactoryClasses( - new Properties(), - ApacheKafkaProducerAdapterFactory.class, - ApacheKafkaConsumerAdapterFactory.class, - ApacheKafkaAdminAdapterFactory.class); - // with legacy config names Properties legacyProps = new Properties(); legacyProps.put(PUB_SUB_PRODUCER_ADAPTER_FACTORY_CLASS, TestPubSubProducerAdapterFactory.class.getName()); @@ -56,6 +52,54 @@ public void testCreateInstanceSuccess() { TestPubSubAdminAdapterFactory.class); } + /** + * By default (no factory-class config and no explicit fallback flag) the factory should fail fast + * instead of silently defaulting to the Apache Kafka adapter factories. + */ + @Test + public void testFailFastWhenFactoryClassMissingAndFallbackDisabled() { + VeniceProperties emptyProps = new VeniceProperties(new Properties()); + + assertFailFast(() -> PubSubClientsFactory.createProducerFactory(emptyProps), PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS); + assertFailFast(() -> PubSubClientsFactory.createConsumerFactory(emptyProps), PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS); + assertFailFast(() -> PubSubClientsFactory.createAdminFactory(emptyProps), PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS); + assertFailFast( + () -> PubSubClientsFactory.createSourceOfTruthAdminFactory(emptyProps), + PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS); + // The instance constructor eagerly builds all three factories, so it should fail fast as well. + expectThrows(VeniceException.class, () -> new PubSubClientsFactory(emptyProps)); + + // Explicitly disabling the fallback behaves the same as the default. + Properties fallbackDisabled = new Properties(); + fallbackDisabled.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); + expectThrows(VeniceException.class, () -> new PubSubClientsFactory(new VeniceProperties(fallbackDisabled))); + } + + /** + * When the Kafka fallback is explicitly enabled, missing factory-class configs should resolve to the + * Apache Kafka adapter factories (the legacy behavior). + */ + @Test + public void testKafkaFallbackWhenExplicitlyEnabled() { + Properties fallbackEnabled = new Properties(); + fallbackEnabled.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "true"); + verifyFactoryClasses( + fallbackEnabled, + ApacheKafkaProducerAdapterFactory.class, + ApacheKafkaConsumerAdapterFactory.class, + ApacheKafkaAdminAdapterFactory.class); + } + + private static void assertFailFast(org.testng.Assert.ThrowingRunnable runnable, String expectedConfigKeyInMessage) { + VeniceException e = expectThrows(VeniceException.class, runnable); + assertTrue( + e.getMessage().contains(expectedConfigKeyInMessage), + "Expected fail-fast message to reference '" + expectedConfigKeyInMessage + "' but was: " + e.getMessage()); + assertTrue( + e.getMessage().contains(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED), + "Expected fail-fast message to reference the fallback config key but was: " + e.getMessage()); + } + private void verifyFactoryClasses( Properties props, Class expectedProducer, diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java index 3a00eb22228..18357dfc00e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java @@ -17,7 +17,6 @@ import com.linkedin.venice.utils.VeniceProperties; import java.io.File; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -286,9 +285,22 @@ public String getPubSubClusterName() { @Override public Map getAdditionalConfig() { - return Collections.singletonMap( + Map configs = new HashMap<>(); + configs.put( ConfigKeys.PUBSUB_TYPE_ID_TO_POSITION_CLASS_NAME_MAP, VeniceProperties.mapToString(PubSubPositionTypeRegistry.RESERVED_POSITION_TYPE_ID_TO_CLASS_NAME_MAP)); + // Explicitly advertise the Apache Kafka adapter factories so that clients relying on + // getBrokerDetailsForClients() do not depend on the (now disabled by default) implicit Kafka fallback. + configs.put( + ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + KAFKA_CLIENTS_FACTORY.getProducerAdapterFactory().getClass().getName()); + configs.put( + ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, + KAFKA_CLIENTS_FACTORY.getConsumerAdapterFactory().getClass().getName()); + configs.put( + ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, + KAFKA_CLIENTS_FACTORY.getAdminAdapterFactory().getClass().getName()); + return configs; } @Override From 16a0ffd6d18358dab98db9bb71182bcbbf9efe68 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Tue, 28 Jul 2026 18:33:53 -0700 Subject: [PATCH 02/25] [test] Supply Apache Kafka pub-sub factory configs to server/controller test configs VeniceServerConfig and VeniceControllerClusterConfig eagerly construct a PubSubClientsFactory, which now fails fast when the adapter factory-class config is missing. Add a shared TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs() helper and wire it into the common test config choke points so unit tests that build these configs keep working: - TestUtils.getPropertiesForControllerConfig() - AbstractStorageEngineTest.getServerProperties() - VeniceServerConfigTest.populatedBasicProperties() --- .../config/VeniceServerConfigTest.java | 2 ++ .../store/AbstractStorageEngineTest.java | 2 ++ .../com/linkedin/venice/utils/TestUtils.java | 25 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java index 9135195b45c..567415004e7 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java @@ -28,6 +28,7 @@ import static org.testng.Assert.assertTrue; import com.linkedin.davinci.blobtransfer.client.NettyFileTransferClient; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import java.util.Arrays; import java.util.HashMap; @@ -42,6 +43,7 @@ public class VeniceServerConfigTest { private Properties populatedBasicProperties() { Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); props.setProperty(CLUSTER_NAME, "test_cluster"); props.setProperty(ZOOKEEPER_ADDRESS, "fake_zk_addr"); props.setProperty(KAFKA_BOOTSTRAP_SERVERS, "fake_kafka_addr"); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java index 7a22e97ec2b..01dcf7f7932 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java @@ -17,6 +17,7 @@ import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.RandomGenUtils; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import java.io.File; @@ -45,6 +46,7 @@ public static VeniceProperties getServerProperties(PersistenceType persistenceTy .put(LISTENER_PORT, 7072) .put(ADMIN_PORT, 7073) .put(DATA_BASE_PATH, dataDirectory.getAbsolutePath()) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .put(properties) .build(); } diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java index 432921d2bf7..e75645b1cc7 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java @@ -79,6 +79,9 @@ import com.linkedin.venice.pubsub.PubSubPositionTypeRegistry; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; import com.linkedin.venice.pubsub.PubSubTopicRepository; +import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; +import com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory; +import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubPosition; import com.linkedin.venice.pubsub.api.PubSubTopicType; import com.linkedin.venice.pubsub.manager.TopicManagerRepository; @@ -738,8 +741,30 @@ public static VeniceControllerMultiClusterConfig getMultiClusterConfigFromOneClu return new VeniceControllerMultiClusterConfig(configMap); } + /** + * Returns the Apache Kafka pub-sub adapter factory-class configs (producer, consumer, admin). + *

+ * Tests that build a {@link VeniceServerConfig} or {@link VeniceControllerClusterConfig} (which + * eagerly construct a {@code PubSubClientsFactory}) must supply these now that the implicit Apache + * Kafka fallback is disabled by default. See + * {@code ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED}. + */ + public static Properties getPubSubApacheKafkaAdapterFactoryConfigs() { + Properties properties = new Properties(); + properties.setProperty( + ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + ApacheKafkaProducerAdapterFactory.class.getName()); + properties.setProperty( + ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, + ApacheKafkaConsumerAdapterFactory.class.getName()); + properties + .setProperty(ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, ApacheKafkaAdminAdapterFactory.class.getName()); + return properties; + } + public static Properties getPropertiesForControllerConfig() { Properties properties = new Properties(); + properties.putAll(getPubSubApacheKafkaAdapterFactoryConfigs()); properties.put(ConfigKeys.CLUSTER_NAME, "test-cluster"); properties.put(ConfigKeys.CONTROLLER_NAME, "venice-controller"); properties.put(ConfigKeys.DEFAULT_REPLICA_FACTOR, "1"); From f8b86fcbe3bb1f230ac33be576b683cfcf792a6a Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Tue, 28 Jul 2026 18:41:00 -0700 Subject: [PATCH 03/25] [test] Extend Apache Kafka pub-sub factory configs to more test config sites - Add source-of-truth admin factory class to the shared TestUtils helper (controller configs also build a source-of-truth admin adapter that now fails fast). - Seed StoreIngestionTaskTest's inline server config with the factory configs. - Set the producer factory class in VeniceWriterFactoryTest so the null-factory path resolves to Apache Kafka instead of failing fast. --- .../davinci/kafka/consumer/StoreIngestionTaskTest.java | 1 + .../com/linkedin/venice/writer/VeniceWriterFactoryTest.java | 1 + .../src/main/java/com/linkedin/venice/utils/TestUtils.java | 3 +++ 3 files changed, 5 insertions(+) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java index d4d60f3be06..3d72063ad7a 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java @@ -3298,6 +3298,7 @@ public void testPartitionExceptionIsolation(AAConfig aaConfig) throws Exception private VeniceServerConfig buildVeniceServerConfig(Map extraProperties) { PropertyBuilder propertyBuilder = new PropertyBuilder(); + propertyBuilder.put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); propertyBuilder.put(CLUSTER_NAME, ""); propertyBuilder.put(ZOOKEEPER_ADDRESS, ""); propertyBuilder.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 500L); diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java index aacb7c249be..2425ef0d00d 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java @@ -110,6 +110,7 @@ public void testVeniceWriterFactoryWithProducerCompressionDisabled() { public void testVeniceWriterFactoryCreatesProducerAdapterFactory() { Properties properties = new Properties(); properties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, "kafka:9898"); + properties.put(ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, ApacheKafkaProducerAdapterFactory.class.getName()); VeniceWriterFactory veniceWriterFactory = new VeniceWriterFactory(properties, null, null, null); assertNotNull(veniceWriterFactory.getProducerAdapterFactory()); diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java index e75645b1cc7..a4e332bfcca 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java @@ -759,6 +759,9 @@ public static Properties getPubSubApacheKafkaAdapterFactoryConfigs() { ApacheKafkaConsumerAdapterFactory.class.getName()); properties .setProperty(ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, ApacheKafkaAdminAdapterFactory.class.getName()); + properties.setProperty( + ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS, + ApacheKafkaAdminAdapterFactory.class.getName()); return properties; } From 7be7a0e5da11e95291fc8275cc262ac05ca93c09 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Tue, 28 Jul 2026 18:51:09 -0700 Subject: [PATCH 04/25] [test] Supply Apache Kafka pub-sub factory configs to DaVinci backend/client tests --- .../java/com/linkedin/davinci/DaVinciBackendTest.java | 2 ++ .../test/java/com/linkedin/davinci/StoreBackendTest.java | 9 ++++++--- .../davinci/client/AvroGenericDaVinciClientTest.java | 4 ++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java index 2b581b70642..12ecb361c29 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java @@ -47,6 +47,7 @@ import com.linkedin.venice.schema.writecompute.DerivedSchemaEntry; import com.linkedin.venice.serialization.avro.SchemaPresenceChecker; import com.linkedin.venice.service.ICProvider; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import io.tehuti.metrics.MetricsRepository; import java.util.Optional; @@ -83,6 +84,7 @@ public void setUp() throws Exception { serverProps.setProperty(INGESTION_USE_DA_VINCI_CLIENT, "true"); serverProps.setProperty(DATA_BASE_PATH, "/tmp/test"); serverProps.setProperty(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, "0"); + serverProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); VeniceProperties veniceProperties = new VeniceProperties(serverProps); VeniceConfigLoader configLoader = new VeniceConfigLoader(veniceProperties); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java index fc2273ca23b..2bcb871c425 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java @@ -84,7 +84,8 @@ public class StoreBackendTest { @BeforeMethod void setUp() { baseDataPath = Utils.getTempDataDirectory(); - VeniceProperties backendConfig = new PropertyBuilder().put(ConfigKeys.CLUSTER_NAME, "test-cluster") + VeniceProperties backendConfig = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .put(ConfigKeys.CLUSTER_NAME, "test-cluster") .put(ConfigKeys.ZOOKEEPER_ADDRESS, "test-zookeeper") .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) @@ -668,7 +669,8 @@ public void testResumePausedSITOnTargetPromotion() throws Exception { @Test public void testLegacyNonTargetRegionSubscribesOnOnline() throws Exception { // Re-create storeBackend with paused-SIT disabled (legacy mode). - VeniceProperties legacyConfig = new PropertyBuilder().put(ConfigKeys.CLUSTER_NAME, "test-cluster") + VeniceProperties legacyConfig = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .put(ConfigKeys.CLUSTER_NAME, "test-cluster") .put(ConfigKeys.ZOOKEEPER_ADDRESS, "test-zookeeper") .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) @@ -716,7 +718,8 @@ public void testLegacyNonTargetRegionSubscribesOnOnline() throws Exception { * region stays {@code dc-0} and paused-SIT stays enabled. */ private void rebuildStoreBackendWithRollForwardOrder(String rollForwardOrder) { - VeniceProperties config = new PropertyBuilder().put(ConfigKeys.CLUSTER_NAME, "test-cluster") + VeniceProperties config = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .put(ConfigKeys.CLUSTER_NAME, "test-cluster") .put(ConfigKeys.ZOOKEEPER_ADDRESS, "test-zookeeper") .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java index 664d1ce6c1f..7d112ae4bac 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java @@ -46,6 +46,7 @@ import com.linkedin.venice.utils.DaemonThreadFactory; import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.ReferenceCounted; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import java.lang.reflect.Field; import java.security.AccessController; @@ -85,6 +86,7 @@ public AvroGenericDaVinciClient setUpSpecificClient(ClientConfig clientConfig, b DaVinciConfig daVinciConfig = new DaVinciConfig(); VeniceProperties backendConfig = new PropertyBuilder().put(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, false) .put(DAVINCI_VALIDATE_SPECIFIC_SCHEMA_ENABLED, validateSpecificSchema) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); AvroGenericDaVinciClient dvcClient = @@ -130,6 +132,7 @@ public AvroGenericSeekableDaVinciClient setUpSeekableClient(ClientConfig clientC DaVinciConfig daVinciConfig = new DaVinciConfig(); VeniceProperties backendConfig = new PropertyBuilder().put(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, false) .put(DAVINCI_VALIDATE_SPECIFIC_SCHEMA_ENABLED, validateSpecificSchema) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); AvroGenericSeekableDaVinciClient dvcClient = spy( @@ -210,6 +213,7 @@ public AvroGenericDaVinciClient setUpClientWithRecordTransformer( VeniceProperties backendConfig = new PropertyBuilder().put(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, enableDatabaseChecksumVerification) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); AvroGenericDaVinciClient dvcClient = From 8238262a67ab36c80556f88f89cd21c4f09df977 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Tue, 28 Jul 2026 19:02:19 -0700 Subject: [PATCH 05/25] [test][cc] Supply Apache Kafka pub-sub factory configs to changelog consumer tests --- .../consumer/VeniceChangelogConsumerClientFactoryTest.java | 7 +++++++ ...eChangelogConsumerDaVinciRecordTransformerImplTest.java | 3 +-- .../davinci/consumer/VeniceChangelogConsumerImplTest.java | 3 ++- ...eChangelogConsumerDaVinciRecordTransformerImplTest.java | 3 +-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java index 1ab53dbc84c..bda108b0791 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java @@ -31,6 +31,7 @@ import com.linkedin.venice.pubsub.api.PubSubMessageDeserializer; import com.linkedin.venice.schema.SchemaReader; import com.linkedin.venice.utils.ObjectMapperFactory; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.views.MaterializedView; import io.tehuti.metrics.MetricsRepository; import java.nio.charset.StandardCharsets; @@ -57,6 +58,7 @@ public class VeniceChangelogConsumerClientFactoryTest { @Test public void testGetChangelogConsumer() throws ExecutionException, InterruptedException, JsonProcessingException { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, localKafkaUrl); consumerProperties.put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, localKafkaUrl); @@ -131,6 +133,7 @@ public void testGetChangelogConsumer() throws ExecutionException, InterruptedExc public void testGetChangelogConsumerWithConsumerId() throws ExecutionException, InterruptedException, JsonProcessingException { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, localKafkaUrl); SchemaReader mockSchemaReader = Mockito.mock(SchemaReader.class); @@ -199,6 +202,7 @@ private void setUpMockStoreResponse(D2ControllerClient mockControllerClient, Str @Test public void testGetChangelogConsumerThrowsException() { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, localKafkaUrl); @@ -231,6 +235,7 @@ public void testGetChangelogConsumerThrowsException() { public void testGetStatefulChangelogConsumer() throws ExecutionException, InterruptedException, JsonProcessingException { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, localKafkaUrl); consumerProperties.put(CLUSTER_NAME, TEST_CLUSTER_NAME); @@ -312,6 +317,7 @@ public void testGetStatefulChangelogConsumer() @Test public void testGetStatefulChangelogConsumerThrowsException() { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, localKafkaUrl); consumerProperties.put(CLUSTER_NAME, TEST_CLUSTER_NAME); @@ -365,6 +371,7 @@ public void testCreatePubSubMessageDeserializer( boolean expectKmeWithSchemaReaderCall) { // Build properties Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); if (kmeProp != null) { props.put(ConfigKeys.KME_SCHEMA_READER_FOR_SCHEMA_EVOLUTION_ENABLED, kmeProp); } diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java index 41ac7dc16ac..01ceb499be2 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java @@ -55,7 +55,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Properties; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; @@ -116,7 +115,7 @@ public void setUp() throws NoSuchFieldException, IllegalAccessException { .setStoreName(TEST_STORE_NAME) .setControllerD2ServiceName(D2_SERVICE_NAME) .setD2ServiceName(DEFAULT_CLUSTER_DISCOVERY_D2_SERVICE_NAME) - .setConsumerProperties(new Properties()) + .setConsumerProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .setLocalD2ZkHosts(TEST_ZOOKEEPER_ADDRESS) .setDatabaseSyncBytesInterval(TEST_DB_SYNC_BYTES_INTERVAL) .setD2Client(mock(D2Client.class)) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java index 4b1a8043736..6a6d876ce53 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java @@ -208,6 +208,7 @@ public void testConfig() { assertTrue(config.getConsumerProperties().isEmpty()); assertThrows(NullPointerException.class, () -> config.setConsumerProperties(null)); Properties newProps = new Properties(); + newProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); newProps.setProperty("foo", "bar"); config.setConsumerProperties(newProps); assertNotNull(config.getConsumerProperties()); @@ -1467,7 +1468,7 @@ private ChangelogClientConfig getChangelogClientConfig() { new ChangelogClientConfig<>().setD2ControllerClient(mockD2ControllerClient) .setSchemaReader(schemaReader) .setStoreName(storeName) - .setConsumerProperties(new Properties()) + .setConsumerProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .setViewName(""); changelogClientConfig.getInnerClientConfig() .setMetricsRepository(getVeniceMetricsRepository(CHANGE_DATA_CAPTURE_CLIENT, CONSUMER_METRIC_ENTITIES, true)); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java index c7e042c1ab4..edf78994945 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java @@ -47,7 +47,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -103,7 +102,7 @@ public void setUp() throws NoSuchFieldException, IllegalAccessException { .setStoreName(TEST_STORE_NAME) .setControllerD2ServiceName(D2_SERVICE_NAME) .setD2ServiceName(DEFAULT_CLUSTER_DISCOVERY_D2_SERVICE_NAME) - .setConsumerProperties(new Properties()) + .setConsumerProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .setLocalD2ZkHosts(TEST_ZOOKEEPER_ADDRESS) .setDatabaseSyncBytesInterval(TEST_DB_SYNC_BYTES_INTERVAL) .setD2Client(mock(D2Client.class)) From 8873bb74d2aae2b5c98d3e08ee3836da73b5acac Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 00:01:37 -0700 Subject: [PATCH 06/25] [common][test] Make pub-sub adapter factory fail-fast opt-in Supersedes the earlier default-disabled approach in this PR. Defaulting to fail-fast required threading the pub-sub factory config through the entire test suite; instead make it opt-in. - pubsub.adapter.factory.kafka.fallback.enabled now defaults to true (legacy Apache Kafka fallback preserved) so no unit/mock/integration tests change. Set it to false to opt into fail-fast, where a missing factory-class config throws instead of silently defaulting to Apache Kafka. - Revert the test-wide factory-config additions and the KafkaBrokerFactory change from earlier commits in this PR. - Behavior verified by PubSubClientsFactoryTest and a new integration test, PubSubAdapterFactoryFailFastTest, exercising a real VeniceServerConfig. --- .../linkedin/davinci/DaVinciBackendTest.java | 2 - .../linkedin/davinci/StoreBackendTest.java | 9 +-- .../client/AvroGenericDaVinciClientTest.java | 4 - .../config/VeniceServerConfigTest.java | 2 - ...iceChangelogConsumerClientFactoryTest.java | 7 -- ...sumerDaVinciRecordTransformerImplTest.java | 3 +- .../VeniceChangelogConsumerImplTest.java | 3 +- ...sumerDaVinciRecordTransformerImplTest.java | 3 +- .../consumer/StoreIngestionTaskTest.java | 1 - .../store/AbstractStorageEngineTest.java | 2 - .../java/com/linkedin/venice/ConfigKeys.java | 12 +-- .../venice/pubsub/PubSubClientsFactory.java | 13 +-- .../pubsub/PubSubClientsFactoryTest.java | 63 +++++++++++---- .../writer/VeniceWriterFactoryTest.java | 1 - .../integration/utils/KafkaBrokerFactory.java | 16 +--- .../PubSubAdapterFactoryFailFastTest.java | 79 +++++++++++++++++++ .../com/linkedin/venice/utils/TestUtils.java | 28 ------- 17 files changed, 150 insertions(+), 98 deletions(-) create mode 100644 internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java index 12ecb361c29..2b581b70642 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java @@ -47,7 +47,6 @@ import com.linkedin.venice.schema.writecompute.DerivedSchemaEntry; import com.linkedin.venice.serialization.avro.SchemaPresenceChecker; import com.linkedin.venice.service.ICProvider; -import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import io.tehuti.metrics.MetricsRepository; import java.util.Optional; @@ -84,7 +83,6 @@ public void setUp() throws Exception { serverProps.setProperty(INGESTION_USE_DA_VINCI_CLIENT, "true"); serverProps.setProperty(DATA_BASE_PATH, "/tmp/test"); serverProps.setProperty(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, "0"); - serverProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); VeniceProperties veniceProperties = new VeniceProperties(serverProps); VeniceConfigLoader configLoader = new VeniceConfigLoader(veniceProperties); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java index 2bcb871c425..fc2273ca23b 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java @@ -84,8 +84,7 @@ public class StoreBackendTest { @BeforeMethod void setUp() { baseDataPath = Utils.getTempDataDirectory(); - VeniceProperties backendConfig = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) - .put(ConfigKeys.CLUSTER_NAME, "test-cluster") + VeniceProperties backendConfig = new PropertyBuilder().put(ConfigKeys.CLUSTER_NAME, "test-cluster") .put(ConfigKeys.ZOOKEEPER_ADDRESS, "test-zookeeper") .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) @@ -669,8 +668,7 @@ public void testResumePausedSITOnTargetPromotion() throws Exception { @Test public void testLegacyNonTargetRegionSubscribesOnOnline() throws Exception { // Re-create storeBackend with paused-SIT disabled (legacy mode). - VeniceProperties legacyConfig = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) - .put(ConfigKeys.CLUSTER_NAME, "test-cluster") + VeniceProperties legacyConfig = new PropertyBuilder().put(ConfigKeys.CLUSTER_NAME, "test-cluster") .put(ConfigKeys.ZOOKEEPER_ADDRESS, "test-zookeeper") .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) @@ -718,8 +716,7 @@ public void testLegacyNonTargetRegionSubscribesOnOnline() throws Exception { * region stays {@code dc-0} and paused-SIT stays enabled. */ private void rebuildStoreBackendWithRollForwardOrder(String rollForwardOrder) { - VeniceProperties config = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) - .put(ConfigKeys.CLUSTER_NAME, "test-cluster") + VeniceProperties config = new PropertyBuilder().put(ConfigKeys.CLUSTER_NAME, "test-cluster") .put(ConfigKeys.ZOOKEEPER_ADDRESS, "test-zookeeper") .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java index 7d112ae4bac..664d1ce6c1f 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java @@ -46,7 +46,6 @@ import com.linkedin.venice.utils.DaemonThreadFactory; import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.ReferenceCounted; -import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import java.lang.reflect.Field; import java.security.AccessController; @@ -86,7 +85,6 @@ public AvroGenericDaVinciClient setUpSpecificClient(ClientConfig clientConfig, b DaVinciConfig daVinciConfig = new DaVinciConfig(); VeniceProperties backendConfig = new PropertyBuilder().put(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, false) .put(DAVINCI_VALIDATE_SPECIFIC_SCHEMA_ENABLED, validateSpecificSchema) - .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); AvroGenericDaVinciClient dvcClient = @@ -132,7 +130,6 @@ public AvroGenericSeekableDaVinciClient setUpSeekableClient(ClientConfig clientC DaVinciConfig daVinciConfig = new DaVinciConfig(); VeniceProperties backendConfig = new PropertyBuilder().put(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, false) .put(DAVINCI_VALIDATE_SPECIFIC_SCHEMA_ENABLED, validateSpecificSchema) - .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); AvroGenericSeekableDaVinciClient dvcClient = spy( @@ -213,7 +210,6 @@ public AvroGenericDaVinciClient setUpClientWithRecordTransformer( VeniceProperties backendConfig = new PropertyBuilder().put(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, enableDatabaseChecksumVerification) - .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); AvroGenericDaVinciClient dvcClient = diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java index 567415004e7..9135195b45c 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java @@ -28,7 +28,6 @@ import static org.testng.Assert.assertTrue; import com.linkedin.davinci.blobtransfer.client.NettyFileTransferClient; -import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import java.util.Arrays; import java.util.HashMap; @@ -43,7 +42,6 @@ public class VeniceServerConfigTest { private Properties populatedBasicProperties() { Properties props = new Properties(); - props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); props.setProperty(CLUSTER_NAME, "test_cluster"); props.setProperty(ZOOKEEPER_ADDRESS, "fake_zk_addr"); props.setProperty(KAFKA_BOOTSTRAP_SERVERS, "fake_kafka_addr"); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java index bda108b0791..1ab53dbc84c 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java @@ -31,7 +31,6 @@ import com.linkedin.venice.pubsub.api.PubSubMessageDeserializer; import com.linkedin.venice.schema.SchemaReader; import com.linkedin.venice.utils.ObjectMapperFactory; -import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.views.MaterializedView; import io.tehuti.metrics.MetricsRepository; import java.nio.charset.StandardCharsets; @@ -58,7 +57,6 @@ public class VeniceChangelogConsumerClientFactoryTest { @Test public void testGetChangelogConsumer() throws ExecutionException, InterruptedException, JsonProcessingException { Properties consumerProperties = new Properties(); - consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, localKafkaUrl); consumerProperties.put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, localKafkaUrl); @@ -133,7 +131,6 @@ public void testGetChangelogConsumer() throws ExecutionException, InterruptedExc public void testGetChangelogConsumerWithConsumerId() throws ExecutionException, InterruptedException, JsonProcessingException { Properties consumerProperties = new Properties(); - consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, localKafkaUrl); SchemaReader mockSchemaReader = Mockito.mock(SchemaReader.class); @@ -202,7 +199,6 @@ private void setUpMockStoreResponse(D2ControllerClient mockControllerClient, Str @Test public void testGetChangelogConsumerThrowsException() { Properties consumerProperties = new Properties(); - consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, localKafkaUrl); @@ -235,7 +231,6 @@ public void testGetChangelogConsumerThrowsException() { public void testGetStatefulChangelogConsumer() throws ExecutionException, InterruptedException, JsonProcessingException { Properties consumerProperties = new Properties(); - consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, localKafkaUrl); consumerProperties.put(CLUSTER_NAME, TEST_CLUSTER_NAME); @@ -317,7 +312,6 @@ public void testGetStatefulChangelogConsumer() @Test public void testGetStatefulChangelogConsumerThrowsException() { Properties consumerProperties = new Properties(); - consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, localKafkaUrl); consumerProperties.put(CLUSTER_NAME, TEST_CLUSTER_NAME); @@ -371,7 +365,6 @@ public void testCreatePubSubMessageDeserializer( boolean expectKmeWithSchemaReaderCall) { // Build properties Properties props = new Properties(); - props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); if (kmeProp != null) { props.put(ConfigKeys.KME_SCHEMA_READER_FOR_SCHEMA_EVOLUTION_ENABLED, kmeProp); } diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java index 01ceb499be2..41ac7dc16ac 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java @@ -55,6 +55,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Properties; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; @@ -115,7 +116,7 @@ public void setUp() throws NoSuchFieldException, IllegalAccessException { .setStoreName(TEST_STORE_NAME) .setControllerD2ServiceName(D2_SERVICE_NAME) .setD2ServiceName(DEFAULT_CLUSTER_DISCOVERY_D2_SERVICE_NAME) - .setConsumerProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .setConsumerProperties(new Properties()) .setLocalD2ZkHosts(TEST_ZOOKEEPER_ADDRESS) .setDatabaseSyncBytesInterval(TEST_DB_SYNC_BYTES_INTERVAL) .setD2Client(mock(D2Client.class)) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java index 6a6d876ce53..4b1a8043736 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java @@ -208,7 +208,6 @@ public void testConfig() { assertTrue(config.getConsumerProperties().isEmpty()); assertThrows(NullPointerException.class, () -> config.setConsumerProperties(null)); Properties newProps = new Properties(); - newProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); newProps.setProperty("foo", "bar"); config.setConsumerProperties(newProps); assertNotNull(config.getConsumerProperties()); @@ -1468,7 +1467,7 @@ private ChangelogClientConfig getChangelogClientConfig() { new ChangelogClientConfig<>().setD2ControllerClient(mockD2ControllerClient) .setSchemaReader(schemaReader) .setStoreName(storeName) - .setConsumerProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .setConsumerProperties(new Properties()) .setViewName(""); changelogClientConfig.getInnerClientConfig() .setMetricsRepository(getVeniceMetricsRepository(CHANGE_DATA_CAPTURE_CLIENT, CONSUMER_METRIC_ENTITIES, true)); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java index edf78994945..c7e042c1ab4 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java @@ -47,6 +47,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -102,7 +103,7 @@ public void setUp() throws NoSuchFieldException, IllegalAccessException { .setStoreName(TEST_STORE_NAME) .setControllerD2ServiceName(D2_SERVICE_NAME) .setD2ServiceName(DEFAULT_CLUSTER_DISCOVERY_D2_SERVICE_NAME) - .setConsumerProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .setConsumerProperties(new Properties()) .setLocalD2ZkHosts(TEST_ZOOKEEPER_ADDRESS) .setDatabaseSyncBytesInterval(TEST_DB_SYNC_BYTES_INTERVAL) .setD2Client(mock(D2Client.class)) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java index 3d72063ad7a..d4d60f3be06 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java @@ -3298,7 +3298,6 @@ public void testPartitionExceptionIsolation(AAConfig aaConfig) throws Exception private VeniceServerConfig buildVeniceServerConfig(Map extraProperties) { PropertyBuilder propertyBuilder = new PropertyBuilder(); - propertyBuilder.put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); propertyBuilder.put(CLUSTER_NAME, ""); propertyBuilder.put(ZOOKEEPER_ADDRESS, ""); propertyBuilder.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 500L); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java index 01dcf7f7932..7a22e97ec2b 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java @@ -17,7 +17,6 @@ import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.RandomGenUtils; -import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import java.io.File; @@ -46,7 +45,6 @@ public static VeniceProperties getServerProperties(PersistenceType persistenceTy .put(LISTENER_PORT, 7072) .put(ADMIN_PORT, 7073) .put(DATA_BASE_PATH, dataDirectory.getAbsolutePath()) - .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .put(properties) .build(); } diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java index 36b178093ed..9450807bd9c 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java @@ -118,13 +118,13 @@ private ConfigKeys() { /** * Configuration key that controls whether the PubSub producer/consumer/admin adapter factories - * silently fall back to the Apache Kafka implementation when their factory-class config keys are - * not explicitly provided. + * fall back to the Apache Kafka implementation when their factory-class config keys are not + * explicitly provided. *

- * When {@code false} (the default), the {@code PubSubClientsFactory} fails fast by throwing an - * exception if the corresponding factory-class config is missing. This surfaces misconfiguration - * early instead of masking it behind an implicit Kafka default. Set this to {@code true} to - * restore the legacy behavior of defaulting to the Apache Kafka adapter factories. + * Defaults to {@code true} (fall back to Apache Kafka) for backward compatibility. Set this to + * {@code false} to opt into fail-fast: the {@code PubSubClientsFactory} then throws when a + * factory-class config is missing, surfacing misconfiguration early instead of masking it behind + * an implicit Kafka default. Recommended for non-Kafka (e.g. xinfra) deployments. */ public static final String PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED = PUBSUB_CLIENT_CONFIG_PREFIX + "adapter.factory.kafka.fallback.enabled"; diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java index 2b9d465c838..34c20ae7c30 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java @@ -31,12 +31,15 @@ public class PubSubClientsFactory { private static final Logger LOGGER = LogManager.getLogger(PubSubClientsFactory.class); /** - * By default the adapter factories do NOT fall back to Apache Kafka when their factory-class config - * is missing; callers must configure the factory classes explicitly so that misconfiguration fails - * fast. Set {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED} to - * {@code true} to restore the legacy implicit-Kafka behavior. + * Controls the default behavior when a pub-sub adapter factory-class config is not provided. + *

+ * Defaults to {@code true} (fall back to the Apache Kafka adapter factories) to preserve backward + * compatibility: existing callers that never set the factory-class configs keep working. Set + * {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED} to + * {@code false} to opt into fail-fast, where a missing factory-class config raises an exception + * instead of silently defaulting to Apache Kafka (recommended for non-Kafka deployments). */ - public static final boolean DEFAULT_KAFKA_FALLBACK_ENABLED = false; + public static final boolean DEFAULT_KAFKA_FALLBACK_ENABLED = true; private enum FactoryType { PRODUCER, CONSUMER, ADMIN diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java index a0b7b55be20..365eeebf940 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java @@ -53,31 +53,45 @@ public void testCreateInstanceSuccess() { } /** - * By default (no factory-class config and no explicit fallback flag) the factory should fail fast - * instead of silently defaulting to the Apache Kafka adapter factories. + * By default (no explicit fallback flag) a missing factory-class config resolves to the Apache Kafka + * adapter factories, preserving backward compatibility. */ @Test - public void testFailFastWhenFactoryClassMissingAndFallbackDisabled() { - VeniceProperties emptyProps = new VeniceProperties(new Properties()); + public void testDefaultFallsBackToApacheKafka() { + verifyFactoryClasses( + new Properties(), + ApacheKafkaProducerAdapterFactory.class, + ApacheKafkaConsumerAdapterFactory.class, + ApacheKafkaAdminAdapterFactory.class); + } - assertFailFast(() -> PubSubClientsFactory.createProducerFactory(emptyProps), PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS); - assertFailFast(() -> PubSubClientsFactory.createConsumerFactory(emptyProps), PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS); - assertFailFast(() -> PubSubClientsFactory.createAdminFactory(emptyProps), PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS); + /** + * When the Kafka fallback is explicitly disabled (opt-in fail-fast), a missing factory-class config + * throws instead of silently defaulting to the Apache Kafka adapter factories. + */ + @Test + public void testFailFastWhenFallbackDisabledAndFactoryClassMissing() { + Properties props = new Properties(); + props.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); + VeniceProperties failFastProps = new VeniceProperties(props); + + assertFailFast( + () -> PubSubClientsFactory.createProducerFactory(failFastProps), + PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS); + assertFailFast( + () -> PubSubClientsFactory.createConsumerFactory(failFastProps), + PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS); + assertFailFast(() -> PubSubClientsFactory.createAdminFactory(failFastProps), PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS); assertFailFast( - () -> PubSubClientsFactory.createSourceOfTruthAdminFactory(emptyProps), + () -> PubSubClientsFactory.createSourceOfTruthAdminFactory(failFastProps), PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS); // The instance constructor eagerly builds all three factories, so it should fail fast as well. - expectThrows(VeniceException.class, () -> new PubSubClientsFactory(emptyProps)); - - // Explicitly disabling the fallback behaves the same as the default. - Properties fallbackDisabled = new Properties(); - fallbackDisabled.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); - expectThrows(VeniceException.class, () -> new PubSubClientsFactory(new VeniceProperties(fallbackDisabled))); + expectThrows(VeniceException.class, () -> new PubSubClientsFactory(failFastProps)); } /** - * When the Kafka fallback is explicitly enabled, missing factory-class configs should resolve to the - * Apache Kafka adapter factories (the legacy behavior). + * When the Kafka fallback is explicitly enabled, missing factory-class configs resolve to the + * Apache Kafka adapter factories (same as the default). */ @Test public void testKafkaFallbackWhenExplicitlyEnabled() { @@ -90,6 +104,23 @@ public void testKafkaFallbackWhenExplicitlyEnabled() { ApacheKafkaAdminAdapterFactory.class); } + /** + * Explicit factory-class configs are honored regardless of the fallback flag. + */ + @Test + public void testExplicitConfigWinsOverFailFast() { + Properties props = new Properties(); + props.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); + props.put(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, TestPubSubProducerAdapterFactory.class.getName()); + props.put(PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, TestPubSubConsumerAdapterFactory.class.getName()); + props.put(PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, TestPubSubAdminAdapterFactory.class.getName()); + verifyFactoryClasses( + props, + TestPubSubProducerAdapterFactory.class, + TestPubSubConsumerAdapterFactory.class, + TestPubSubAdminAdapterFactory.class); + } + private static void assertFailFast(org.testng.Assert.ThrowingRunnable runnable, String expectedConfigKeyInMessage) { VeniceException e = expectThrows(VeniceException.class, runnable); assertTrue( diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java index 2425ef0d00d..aacb7c249be 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java @@ -110,7 +110,6 @@ public void testVeniceWriterFactoryWithProducerCompressionDisabled() { public void testVeniceWriterFactoryCreatesProducerAdapterFactory() { Properties properties = new Properties(); properties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, "kafka:9898"); - properties.put(ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, ApacheKafkaProducerAdapterFactory.class.getName()); VeniceWriterFactory veniceWriterFactory = new VeniceWriterFactory(properties, null, null, null); assertNotNull(veniceWriterFactory.getProducerAdapterFactory()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java index 18357dfc00e..3a00eb22228 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java @@ -17,6 +17,7 @@ import com.linkedin.venice.utils.VeniceProperties; import java.io.File; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -285,22 +286,9 @@ public String getPubSubClusterName() { @Override public Map getAdditionalConfig() { - Map configs = new HashMap<>(); - configs.put( + return Collections.singletonMap( ConfigKeys.PUBSUB_TYPE_ID_TO_POSITION_CLASS_NAME_MAP, VeniceProperties.mapToString(PubSubPositionTypeRegistry.RESERVED_POSITION_TYPE_ID_TO_CLASS_NAME_MAP)); - // Explicitly advertise the Apache Kafka adapter factories so that clients relying on - // getBrokerDetailsForClients() do not depend on the (now disabled by default) implicit Kafka fallback. - configs.put( - ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, - KAFKA_CLIENTS_FACTORY.getProducerAdapterFactory().getClass().getName()); - configs.put( - ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, - KAFKA_CLIENTS_FACTORY.getConsumerAdapterFactory().getClass().getName()); - configs.put( - ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, - KAFKA_CLIENTS_FACTORY.getAdminAdapterFactory().getClass().getName()); - return configs; } @Override diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java new file mode 100644 index 00000000000..14e25073a6a --- /dev/null +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java @@ -0,0 +1,79 @@ +package com.linkedin.venice.pubsub; + +import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; +import static com.linkedin.venice.ConfigKeys.INGESTION_USE_DA_VINCI_CLIENT; +import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.ZOOKEEPER_ADDRESS; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import com.linkedin.davinci.config.VeniceServerConfig; +import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; +import com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory; +import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; +import com.linkedin.venice.utils.VeniceProperties; +import java.util.Properties; +import org.testng.annotations.Test; + + +/** + * End-to-end coverage for the opt-in fail-fast behavior of the pub-sub adapter factories, exercised + * through a real production config object ({@link VeniceServerConfig}) rather than the factory in + * isolation. {@link VeniceServerConfig} eagerly constructs a {@link PubSubClientsFactory} from its + * properties, so it is representative of how a Venice component resolves its pub-sub clients at + * startup. + *

+ * The behavior is opt-in via {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED}: + * by default the config keeps working (Apache Kafka fallback); with the fallback disabled it fails + * fast when the adapter factory class is not configured. + */ +public class PubSubAdapterFactoryFailFastTest { + private static Properties baseServerProperties() { + Properties props = new Properties(); + props.setProperty(CLUSTER_NAME, "test_cluster"); + props.setProperty(ZOOKEEPER_ADDRESS, "localhost:2181"); + props.setProperty(KAFKA_BOOTSTRAP_SERVERS, "localhost:9092"); + props.setProperty(INGESTION_USE_DA_VINCI_CLIENT, "true"); + return props; + } + + @Test + public void serverConfigDefaultsToApacheKafkaWhenFallbackNotConfigured() { + VeniceServerConfig config = new VeniceServerConfig(new VeniceProperties(baseServerProperties())); + assertNotNull(config.getPubSubClientsFactory()); + assertNotNull(config.getPubSubClientsFactory().getProducerAdapterFactory()); + } + + @Test + public void serverConfigFailsFastWhenFallbackDisabledAndFactoryClassMissing() { + Properties props = baseServerProperties(); + props.setProperty(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); + + VeniceException e = expectThrows(VeniceException.class, () -> new VeniceServerConfig(new VeniceProperties(props))); + assertTrue( + e.getMessage().contains(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS), + "Expected fail-fast message to name the missing factory-class config but was: " + e.getMessage()); + assertTrue( + e.getMessage().contains(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED), + "Expected fail-fast message to name the fallback config key but was: " + e.getMessage()); + } + + @Test + public void serverConfigSucceedsWhenFallbackDisabledButFactoryClassesProvided() { + Properties props = baseServerProperties(); + props.setProperty(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); + props.setProperty(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, ApacheKafkaProducerAdapterFactory.class.getName()); + props.setProperty(PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, ApacheKafkaConsumerAdapterFactory.class.getName()); + props.setProperty(PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, ApacheKafkaAdminAdapterFactory.class.getName()); + + VeniceServerConfig config = new VeniceServerConfig(new VeniceProperties(props)); + assertNotNull(config.getPubSubClientsFactory()); + assertNotNull(config.getPubSubClientsFactory().getProducerAdapterFactory()); + } +} diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java index a4e332bfcca..432921d2bf7 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java @@ -79,9 +79,6 @@ import com.linkedin.venice.pubsub.PubSubPositionTypeRegistry; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; import com.linkedin.venice.pubsub.PubSubTopicRepository; -import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; -import com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory; -import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubPosition; import com.linkedin.venice.pubsub.api.PubSubTopicType; import com.linkedin.venice.pubsub.manager.TopicManagerRepository; @@ -741,33 +738,8 @@ public static VeniceControllerMultiClusterConfig getMultiClusterConfigFromOneClu return new VeniceControllerMultiClusterConfig(configMap); } - /** - * Returns the Apache Kafka pub-sub adapter factory-class configs (producer, consumer, admin). - *

- * Tests that build a {@link VeniceServerConfig} or {@link VeniceControllerClusterConfig} (which - * eagerly construct a {@code PubSubClientsFactory}) must supply these now that the implicit Apache - * Kafka fallback is disabled by default. See - * {@code ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED}. - */ - public static Properties getPubSubApacheKafkaAdapterFactoryConfigs() { - Properties properties = new Properties(); - properties.setProperty( - ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, - ApacheKafkaProducerAdapterFactory.class.getName()); - properties.setProperty( - ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, - ApacheKafkaConsumerAdapterFactory.class.getName()); - properties - .setProperty(ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, ApacheKafkaAdminAdapterFactory.class.getName()); - properties.setProperty( - ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS, - ApacheKafkaAdminAdapterFactory.class.getName()); - return properties; - } - public static Properties getPropertiesForControllerConfig() { Properties properties = new Properties(); - properties.putAll(getPubSubApacheKafkaAdapterFactoryConfigs()); properties.put(ConfigKeys.CLUSTER_NAME, "test-cluster"); properties.put(ConfigKeys.CONTROLLER_NAME, "venice-controller"); properties.put(ConfigKeys.DEFAULT_REPLICA_FACTOR, "1"); From 829a488334fc9a352db967f261d17dbeb5c68439 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 01:01:15 -0700 Subject: [PATCH 07/25] [common][test] Fail fast by default for unconfigured pub-sub adapter factory Default pubsub.adapter.factory.kafka.fallback.enabled to false so a missing producer/consumer/admin/source-of-truth factory-class config fails fast instead of silently defaulting to Apache Kafka (which masks misconfiguration on non-Kafka deployments). The default is overridable via the same-named system property. To avoid changing the large existing test suite, the test JVM sets that system property to true (root build.gradle), so tests keep resolving to the Apache Kafka adapters. The behavior is verified explicitly by PubSubClientsFactoryTest and the PubSubAdapterFactoryFailFastTest integration test (which sets the flag per case). --- build.gradle | 5 +++ .../java/com/linkedin/venice/ConfigKeys.java | 9 +++--- .../venice/pubsub/PubSubClientsFactory.java | 32 ++++++++++++++----- .../pubsub/PubSubClientsFactoryTest.java | 15 ++++----- .../PubSubAdapterFactoryFailFastTest.java | 26 +++++++++------ 5 files changed, 57 insertions(+), 30 deletions(-) diff --git a/build.gradle b/build.gradle index 438b0cb1fe6..ea705b77113 100644 --- a/build.gradle +++ b/build.gradle @@ -434,6 +434,11 @@ subprojects { systemProperty 'pubSubBrokerFactory', System.getProperty('pubSubBrokerFactory', "com.linkedin.venice.integration.utils.KafkaBrokerFactory") + // Default the pub-sub adapter factories to the Apache Kafka fallback for tests, so the existing + // test suite does not need to configure the factory classes explicitly. Production leaves this + // unset and therefore fails fast when the factory class is missing (see PubSubClientsFactory). + systemProperty 'pubsub.adapter.factory.kafka.fallback.enabled', System.getProperty('pubsub.adapter.factory.kafka.fallback.enabled', 'true') + System.getProperty('jvmArgs')?.eachMatch(/(?:[^\s'"]+|'[^']*'|"[^"]*")+/) { jvmArgs it } doFirst { diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java index 9450807bd9c..7a5a704929a 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java @@ -121,10 +121,11 @@ private ConfigKeys() { * fall back to the Apache Kafka implementation when their factory-class config keys are not * explicitly provided. *

- * Defaults to {@code true} (fall back to Apache Kafka) for backward compatibility. Set this to - * {@code false} to opt into fail-fast: the {@code PubSubClientsFactory} then throws when a - * factory-class config is missing, surfacing misconfiguration early instead of masking it behind - * an implicit Kafka default. Recommended for non-Kafka (e.g. xinfra) deployments. + * Defaults to {@code false} (fail fast): the {@code PubSubClientsFactory} throws when a factory-class + * config is missing, surfacing misconfiguration early instead of masking it behind an implicit Kafka + * default. Set this to {@code true} to fall back to Apache Kafka. The default may also be overridden + * via the same-named system property (the test JVM sets it to {@code true} so existing tests keep + * resolving to the Apache Kafka adapters without configuring the factory classes explicitly). */ public static final String PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED = PUBSUB_CLIENT_CONFIG_PREFIX + "adapter.factory.kafka.fallback.enabled"; diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java index 34c20ae7c30..de71cc39d25 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java @@ -31,15 +31,17 @@ public class PubSubClientsFactory { private static final Logger LOGGER = LogManager.getLogger(PubSubClientsFactory.class); /** - * Controls the default behavior when a pub-sub adapter factory-class config is not provided. + * The hard default for {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED} + * when it is not provided in the properties: {@code false}, i.e. fail fast. A missing factory-class + * config then raises an exception instead of silently defaulting to Apache Kafka, which surfaces + * misconfiguration early (important for non-Kafka deployments). *

- * Defaults to {@code true} (fall back to the Apache Kafka adapter factories) to preserve backward - * compatibility: existing callers that never set the factory-class configs keep working. Set - * {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED} to - * {@code false} to opt into fail-fast, where a missing factory-class config raises an exception - * instead of silently defaulting to Apache Kafka (recommended for non-Kafka deployments). + * This default can be overridden by the same-named system property. Production never sets it, so it + * stays fail-fast; the test JVM sets it to {@code true} (see the root {@code build.gradle} test + * configuration) so the large body of existing tests keep resolving to the Apache Kafka adapters + * without each having to configure the factory classes explicitly. */ - public static final boolean DEFAULT_KAFKA_FALLBACK_ENABLED = true; + public static final boolean DEFAULT_KAFKA_FALLBACK_ENABLED = false; private enum FactoryType { PRODUCER, CONSUMER, ADMIN @@ -125,7 +127,7 @@ private static T createFactory( LOGGER.debug("Creating pub-sub {} adapter factory instance for class: {}", factoryType, className); } else { boolean kafkaFallbackEnabled = - properties.getBoolean(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, DEFAULT_KAFKA_FALLBACK_ENABLED); + properties.getBoolean(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, isKafkaFallbackEnabledByDefault()); if (!kafkaFallbackEnabled) { throw new VeniceException( String.format( @@ -145,6 +147,20 @@ private static T createFactory( return createInstance(className); } + /** + * Resolves the default for {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED} + * when the supplied properties do not set it. Returns {@link #DEFAULT_KAFKA_FALLBACK_ENABLED} (fail + * fast) unless overridden by the same-named system property. Production leaves the system property + * unset; the test JVM sets it to {@code true} so existing tests keep resolving to the Apache Kafka + * adapters without configuring the factory classes explicitly. + */ + private static boolean isKafkaFallbackEnabledByDefault() { + return Boolean.parseBoolean( + System.getProperty( + PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, + Boolean.toString(DEFAULT_KAFKA_FALLBACK_ENABLED))); + } + public static T createInstance(String className) { try { return (T) Class.forName(className).getDeclaredConstructor().newInstance(); diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java index 365eeebf940..fe6c8767620 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java @@ -9,6 +9,7 @@ import static com.linkedin.venice.ConfigKeys.PUB_SUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_PRODUCER_ADAPTER_FACTORY_CLASS; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; @@ -53,16 +54,14 @@ public void testCreateInstanceSuccess() { } /** - * By default (no explicit fallback flag) a missing factory-class config resolves to the Apache Kafka - * adapter factories, preserving backward compatibility. + * The hard default is fail-fast (no implicit Kafka fallback). The test JVM overrides this to + * {@code true} via a system property (see the root {@code build.gradle}) so the rest of the suite + * keeps resolving to the Apache Kafka adapters; the mode tests below set the flag explicitly so they + * are independent of that ambient default. */ @Test - public void testDefaultFallsBackToApacheKafka() { - verifyFactoryClasses( - new Properties(), - ApacheKafkaProducerAdapterFactory.class, - ApacheKafkaConsumerAdapterFactory.class, - ApacheKafkaAdminAdapterFactory.class); + public void testDefaultIsFailFast() { + assertFalse(PubSubClientsFactory.DEFAULT_KAFKA_FALLBACK_ENABLED); } /** diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java index 14e25073a6a..122b24fd9af 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java @@ -23,15 +23,16 @@ /** - * End-to-end coverage for the opt-in fail-fast behavior of the pub-sub adapter factories, exercised - * through a real production config object ({@link VeniceServerConfig}) rather than the factory in - * isolation. {@link VeniceServerConfig} eagerly constructs a {@link PubSubClientsFactory} from its - * properties, so it is representative of how a Venice component resolves its pub-sub clients at - * startup. + * End-to-end coverage for the fail-fast behavior of the pub-sub adapter factories, exercised through a + * real production config object ({@link VeniceServerConfig}) rather than the factory in isolation. + * {@link VeniceServerConfig} eagerly constructs a {@link PubSubClientsFactory} from its properties, so + * it is representative of how a Venice component resolves its pub-sub clients at startup. *

- * The behavior is opt-in via {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED}: - * by default the config keeps working (Apache Kafka fallback); with the fallback disabled it fails - * fast when the adapter factory class is not configured. + * Fail-fast is the production default: when the adapter factory class is not configured, + * {@link PubSubClientsFactory} throws instead of silently defaulting to Apache Kafka. The default can be + * flipped with {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED}; the + * test JVM sets that flag to {@code true} by default, so these tests set it explicitly to assert each + * mode independently of the ambient test default. */ public class PubSubAdapterFactoryFailFastTest { private static Properties baseServerProperties() { @@ -44,14 +45,19 @@ private static Properties baseServerProperties() { } @Test - public void serverConfigDefaultsToApacheKafkaWhenFallbackNotConfigured() { - VeniceServerConfig config = new VeniceServerConfig(new VeniceProperties(baseServerProperties())); + public void serverConfigUsesApacheKafkaWhenFallbackEnabled() { + Properties props = baseServerProperties(); + props.setProperty(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "true"); + + VeniceServerConfig config = new VeniceServerConfig(new VeniceProperties(props)); assertNotNull(config.getPubSubClientsFactory()); assertNotNull(config.getPubSubClientsFactory().getProducerAdapterFactory()); } @Test public void serverConfigFailsFastWhenFallbackDisabledAndFactoryClassMissing() { + // Represents the production default (fail fast). Set explicitly because the test JVM defaults the + // flag to true. Properties props = baseServerProperties(); props.setProperty(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); From e7f21c33fa6f3f40caa9110b83b7c0b9ab30449c Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 02:03:36 -0700 Subject: [PATCH 08/25] [common][test] Forward pub-sub fallback setting to forked Venice processes ForkedJavaProcess only propagated -X args, so forked Venice processes (isolated ingestion, test apps like DaVinciUserApp) did not inherit the parent's pubsub.adapter.factory.kafka.fallback.enabled system property. With fail-fast as the default, a forked DaVinci app whose backend config does not set the factory class would fail to build its pub-sub client. Forward the property when set so forks resolve pub-sub the same way as the parent (no-op in production, where it is unset and the factory classes come from config). --- .../com/linkedin/venice/utils/ForkedJavaProcess.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java b/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java index 73eae7e9a3d..08b5deb19af 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java @@ -1,5 +1,7 @@ package com.linkedin.venice.utils; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; + import com.linkedin.venice.exceptions.VeniceException; import io.github.classgraph.ClassGraph; import io.github.classgraph.ScanResult; @@ -315,6 +317,13 @@ private static List prepareCommandArgList( command.add("-Djava.io.tmpdir=" + System.getProperty("java.io.tmpdir")); // Inherit IPv6 preference setting from parent process. command.add("-Djava.net.preferIPv6Addresses=" + System.getProperty("java.net.preferIPv6Addresses", "false")); + // Inherit the pub-sub adapter-factory fallback setting so forked Venice processes (e.g. isolated + // ingestion or test apps) resolve their pub-sub clients the same way as this process. No-op when the + // property is unset (e.g. in production, where the factory classes come from config instead). + String pubSubAdapterFallback = System.getProperty(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED); + if (pubSubAdapterFallback != null) { + command.add("-D" + PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED + "=" + pubSubAdapterFallback); + } /** Add log4j2 configuration file and JVM arguments. From f1b81fa73acfcd4b5c997bfffba59a71fc5ce83d Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 10:07:46 -0700 Subject: [PATCH 09/25] [docker][pulsar] Configure pub-sub adapter factory in container deployment configs Containerized Venice processes cannot inherit the test JVM's fallback system property, so with fail-fast as the default the docker venice-server/controller crash on startup (their configs did not set the adapter factory class) and the Pulsar sink's VeniceSystemProducer fails to build its writer. Set the Apache Kafka adapter factory classes explicitly in the docker server/controller configs (single-dc and multi-dc) and in the Pulsar sink's producer config, so these Kafka-backed deployments resolve their pub-sub clients without relying on the removed implicit default. --- .../multi-dc-configs/dc-0.venice.controller.properties | 7 +++++++ .../dc-parent.venice.controller.properties | 7 +++++++ .../single-dc-configs/controller.properties | 7 +++++++ .../venice-server/multi-dc-configs/dc-0/server.properties | 6 ++++++ .../venice-server/multi-dc-configs/dc-1/server.properties | 6 ++++++ docker/venice-server/single-dc-configs/server.properties | 6 ++++++ .../com/linkedin/venice/pulsar/sink/VenicePulsarSink.java | 6 ++++++ 7 files changed, 45 insertions(+) diff --git a/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties b/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties index 9ff7fe9fa4e..c82c7f8f4ff 100644 --- a/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties +++ b/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties @@ -55,3 +55,10 @@ active.active.real.time.source.fabric.list=dc-0,dc-1 controller.enable.batch.push.from.admin.in.child=false default.partition.size=100 topic.cleanup.delay.factor=2 + +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast +# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory +pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory +pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory +pubsub.source.of.truth.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties b/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties index 1db1dc23f4f..44d3bc4cf6c 100644 --- a/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties +++ b/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties @@ -56,3 +56,10 @@ kafka.replication.factor=1 native.replication.source.fabric.as.default.for.batch.only.stores=dc-0 default.partition.size=100 topic.cleanup.delay.factor=2 + +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast +# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory +pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory +pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory +pubsub.source.of.truth.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-controller/single-dc-configs/controller.properties b/docker/venice-controller/single-dc-configs/controller.properties index 0e2422d8fb9..6ab4df2ba3e 100644 --- a/docker/venice-controller/single-dc-configs/controller.properties +++ b/docker/venice-controller/single-dc-configs/controller.properties @@ -27,3 +27,10 @@ enable.offline.push.ssl.whitelist=false kafka.linger.ms=0 default.partition.count=1 controller.zk.shared.metadata.system.schema.store.auto.creation.enabled=true + +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast +# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory +pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory +pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory +pubsub.source.of.truth.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-server/multi-dc-configs/dc-0/server.properties b/docker/venice-server/multi-dc-configs/dc-0/server.properties index e96f60cc807..101d6fcdcdf 100644 --- a/docker/venice-server/multi-dc-configs/dc-0/server.properties +++ b/docker/venice-server/multi-dc-configs/dc-0/server.properties @@ -16,3 +16,9 @@ persistence.type=ROCKS_DB rocksdb.block.cache.size.in.bytes=2147483648 rocksdb.sst.file.manager.delete.rate.bytes.per.second=524288000 rocksdb.sst.file.manager.max.trash.db.ratio=0.25 + +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast +# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory +pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory +pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-server/multi-dc-configs/dc-1/server.properties b/docker/venice-server/multi-dc-configs/dc-1/server.properties index 40e4190f677..131b0db43ce 100644 --- a/docker/venice-server/multi-dc-configs/dc-1/server.properties +++ b/docker/venice-server/multi-dc-configs/dc-1/server.properties @@ -16,3 +16,9 @@ persistence.type=ROCKS_DB rocksdb.block.cache.size.in.bytes=2147483648 rocksdb.sst.file.manager.delete.rate.bytes.per.second=524288000 rocksdb.sst.file.manager.max.trash.db.ratio=0.25 + +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast +# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory +pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory +pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-server/single-dc-configs/server.properties b/docker/venice-server/single-dc-configs/server.properties index 4d00fa10887..ab55076ef03 100644 --- a/docker/venice-server/single-dc-configs/server.properties +++ b/docker/venice-server/single-dc-configs/server.properties @@ -19,3 +19,9 @@ persistence.type=ROCKS_DB rocksdb.block.cache.size.in.bytes=2147483648 rocksdb.sst.file.manager.delete.rate.bytes.per.second=524288000 rocksdb.sst.file.manager.max.trash.db.ratio=0.25 + +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast +# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory +pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory +pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/integrations/venice-pulsar/src/main/java/com/linkedin/venice/pulsar/sink/VenicePulsarSink.java b/integrations/venice-pulsar/src/main/java/com/linkedin/venice/pulsar/sink/VenicePulsarSink.java index fa5fc1ed81f..51e730314ea 100644 --- a/integrations/venice-pulsar/src/main/java/com/linkedin/venice/pulsar/sink/VenicePulsarSink.java +++ b/integrations/venice-pulsar/src/main/java/com/linkedin/venice/pulsar/sink/VenicePulsarSink.java @@ -9,6 +9,7 @@ import static com.linkedin.venice.samza.VeniceSystemFactory.VENICE_ROUTER_URL; import static com.linkedin.venice.samza.VeniceSystemFactory.VENICE_STORE; +import com.linkedin.venice.ConfigKeys; import com.linkedin.venice.meta.Version; import com.linkedin.venice.samza.VeniceSystemFactory; import com.linkedin.venice.samza.VeniceSystemProducer; @@ -242,6 +243,11 @@ public static Map getConfig(VenicePulsarSinkConfig veniceCfg, St config.put(VENICE_ROUTER_URL, veniceCfg.getVeniceRouterUrl()); config.put(DEPLOYMENT_ID, Utils.getUniqueString("venice-push-id-pulsar-sink")); config.put(SSL_ENABLED, "false"); + // The sink's Venice producer writes to the (Kafka-backed) real-time topic; set the pub-sub producer + // adapter factory explicitly so it does not fail fast when the factory class is unconfigured. + config.put( + ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + "com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory"); if (veniceCfg.getKafkaSaslConfig() != null && !veniceCfg.getKafkaSaslConfig().isEmpty()) { config.put("kafka.sasl.jaas.config", veniceCfg.getKafkaSaslConfig()); } From c4cdd782f782dd561a1217c7003b5514b4f4c945 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 10:11:15 -0700 Subject: [PATCH 10/25] [pulsar][test] Enable Kafka fallback for admin-tool commands in Pulsar sink test --- .../pulsar/sink/PulsarVeniceSinkTest.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java b/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java index 3d96f49e7dc..b332efd3e64 100644 --- a/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java +++ b/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java @@ -158,8 +158,8 @@ public void testPulsarVeniceSink() throws Exception { // Wait for the store to become queryable before proceeding LOGGER.info("Waiting for Venice store to be ready"); - String readinessCmd = "java -jar " + jar + " --describe-store --url " + veniceControllerUrl + " --cluster " - + clusterName + " --store " + storeName; + String readinessCmd = "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar + + " --describe-store --url " + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName; Awaitility.await().atMost(30, TimeUnit.SECONDS).pollInterval(2, TimeUnit.SECONDS).untilAsserted(() -> { ExecResult res = execByService("venice-client", "bash", "-c", readinessCmd); String stdout = res.getStdout(); @@ -269,8 +269,9 @@ private void initVeniceStore(String veniceControllerUrl, String jar, String clus "venice-client", "bash", "-c", - "java -jar " + jar + " --empty-push --url " + veniceControllerUrl + " --cluster " + clusterName + " --store " - + storeName + " --push-id init --store-size 1000"); + "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar + " --empty-push --url " + + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName + + " --push-id init --store-size 1000"); } private void updateVeniceStoreQuotas(String veniceControllerUrl, String jar, String clusterName, String storeName) @@ -279,15 +280,16 @@ private void updateVeniceStoreQuotas(String veniceControllerUrl, String jar, Str "venice-client", "bash", "-c", - "java -jar " + jar + " --update-store --url " + veniceControllerUrl + " --cluster " + clusterName + " --store " - + storeName + " --storage-quota -1 --incremental-push-enabled true"); + "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar + " --update-store --url " + + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName + + " --storage-quota -1 --incremental-push-enabled true"); execByServiceAsssertNoStdErr( "venice-client", "bash", "-c", - "java -jar " + jar + " --update-store --url " + veniceControllerUrl + " --cluster " + clusterName + " --store " - + storeName + " --read-quota 1000000"); + "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar + " --update-store --url " + + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName + " --read-quota 1000000"); } private void createVeniceStore( @@ -301,8 +303,9 @@ private void createVeniceStore( "venice-client", "bash", "-c", - "java -jar " + jar + " --new-store --url " + veniceControllerUrl + " --cluster " + clusterName + " --store " - + storeName + " --key-schema-file " + keyFile + " --value-schema-file " + valueFile); + "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar + " --new-store --url " + + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName + " --key-schema-file " + + keyFile + " --value-schema-file " + valueFile); } private void saveKeyValueSchemaFiles(String keyAsvc, String valueAsvc, String keyFile, String valueFile) From d54abb97dcfbdf64c39224346674c4184e1ed1a0 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 13:54:41 -0700 Subject: [PATCH 11/25] [common][test] Resolve pub-sub adapter factory class at runtime; no implicit Kafka default Replace the opt-in Kafka fallback flag with strict runtime resolution of the pub-sub adapter factory class. PubSubClientsFactory now resolves each factory class from the VeniceProperties, else from a JVM system property, else fails fast. There is no implicit Apache Kafka default, so any context that supplies the class in neither place throws immediately and surfaces the misconfiguration. - PubSubClientsFactory.createFactory: config -> system property -> throw. Drops the fallback flag and the ApacheKafka* default classes. - ConfigKeys: remove PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. - build.gradle: provide the four factory-class values (Apache Kafka) to all test JVMs as system properties, so the suite runs unchanged while a missing value still throws in any context that does not supply it. - ForkedJavaProcess: forward the factory-class system properties to forked Venice processes so they resolve their pub-sub clients the same way. - Docker server/controller configs and the Pulsar sink test admin-tool commands supply the factory classes explicitly, since containers/CLI cannot inherit the test JVM properties. - PubSubClientsFactoryTest: cover config resolution, system-property resolution, explicit config precedence, and fail-fast when neither is provided. Remove the integration PubSubAdapterFactoryFailFastTest whose fail-fast assertion required clearing global system properties (unsafe in the concurrent integration suite); the unit test covers it. --- build.gradle | 12 +- .../dc-0.venice.controller.properties | 2 +- .../dc-parent.venice.controller.properties | 2 +- .../single-dc-configs/controller.properties | 2 +- .../multi-dc-configs/dc-0/server.properties | 2 +- .../multi-dc-configs/dc-1/server.properties | 2 +- .../single-dc-configs/server.properties | 2 +- .../java/com/linkedin/venice/ConfigKeys.java | 14 -- .../venice/pubsub/PubSubClientsFactory.java | 59 ++------- .../venice/utils/ForkedJavaProcess.java | 33 +++-- .../pubsub/PubSubClientsFactoryTest.java | 123 +++++++++--------- .../PubSubAdapterFactoryFailFastTest.java | 85 ------------ .../pulsar/sink/PulsarVeniceSinkTest.java | 32 +++-- 13 files changed, 136 insertions(+), 234 deletions(-) delete mode 100644 internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java diff --git a/build.gradle b/build.gradle index ea705b77113..b69922efb93 100644 --- a/build.gradle +++ b/build.gradle @@ -434,10 +434,14 @@ subprojects { systemProperty 'pubSubBrokerFactory', System.getProperty('pubSubBrokerFactory', "com.linkedin.venice.integration.utils.KafkaBrokerFactory") - // Default the pub-sub adapter factories to the Apache Kafka fallback for tests, so the existing - // test suite does not need to configure the factory classes explicitly. Production leaves this - // unset and therefore fails fast when the factory class is missing (see PubSubClientsFactory). - systemProperty 'pubsub.adapter.factory.kafka.fallback.enabled', System.getProperty('pubsub.adapter.factory.kafka.fallback.enabled', 'true') + // Provide the pub-sub adapter factory classes to the test JVMs at runtime. The factory fails fast + // when these are not supplied (there is no implicit default); passing them here means the existing + // test suite runs against the Apache Kafka adapters without each test having to set them, while a + // missing value still throws in any context that does not provide it (e.g. containers, production). + systemProperty 'pubsub.producer.adapter.factory.class', System.getProperty('pubsub.producer.adapter.factory.class', 'com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory') + systemProperty 'pubsub.consumer.adapter.factory.class', System.getProperty('pubsub.consumer.adapter.factory.class', 'com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory') + systemProperty 'pubsub.admin.adapter.factory.class', System.getProperty('pubsub.admin.adapter.factory.class', 'com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory') + systemProperty 'pubsub.source.of.truth.admin.adapter.factory.class', System.getProperty('pubsub.source.of.truth.admin.adapter.factory.class', 'com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory') System.getProperty('jvmArgs')?.eachMatch(/(?:[^\s'"]+|'[^']*'|"[^"]*")+/) { jvmArgs it } diff --git a/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties b/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties index c82c7f8f4ff..acc87823a09 100644 --- a/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties +++ b/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties @@ -57,7 +57,7 @@ default.partition.size=100 topic.cleanup.delay.factor=2 # Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +# when the class is provided neither here nor as a JVM system property (there is no implicit default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties b/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties index 44d3bc4cf6c..6c74f6c7377 100644 --- a/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties +++ b/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties @@ -58,7 +58,7 @@ default.partition.size=100 topic.cleanup.delay.factor=2 # Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +# when the class is provided neither here nor as a JVM system property (there is no implicit default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-controller/single-dc-configs/controller.properties b/docker/venice-controller/single-dc-configs/controller.properties index 6ab4df2ba3e..71ecff77f43 100644 --- a/docker/venice-controller/single-dc-configs/controller.properties +++ b/docker/venice-controller/single-dc-configs/controller.properties @@ -29,7 +29,7 @@ default.partition.count=1 controller.zk.shared.metadata.system.schema.store.auto.creation.enabled=true # Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +# when the class is provided neither here nor as a JVM system property (there is no implicit default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-server/multi-dc-configs/dc-0/server.properties b/docker/venice-server/multi-dc-configs/dc-0/server.properties index 101d6fcdcdf..d78fa5266f7 100644 --- a/docker/venice-server/multi-dc-configs/dc-0/server.properties +++ b/docker/venice-server/multi-dc-configs/dc-0/server.properties @@ -18,7 +18,7 @@ rocksdb.sst.file.manager.delete.rate.bytes.per.second=524288000 rocksdb.sst.file.manager.max.trash.db.ratio=0.25 # Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +# when the class is provided neither here nor as a JVM system property (there is no implicit default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-server/multi-dc-configs/dc-1/server.properties b/docker/venice-server/multi-dc-configs/dc-1/server.properties index 131b0db43ce..15bc3d9b273 100644 --- a/docker/venice-server/multi-dc-configs/dc-1/server.properties +++ b/docker/venice-server/multi-dc-configs/dc-1/server.properties @@ -18,7 +18,7 @@ rocksdb.sst.file.manager.delete.rate.bytes.per.second=524288000 rocksdb.sst.file.manager.max.trash.db.ratio=0.25 # Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +# when the class is provided neither here nor as a JVM system property (there is no implicit default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-server/single-dc-configs/server.properties b/docker/venice-server/single-dc-configs/server.properties index ab55076ef03..495bf3931dc 100644 --- a/docker/venice-server/single-dc-configs/server.properties +++ b/docker/venice-server/single-dc-configs/server.properties @@ -21,7 +21,7 @@ rocksdb.sst.file.manager.delete.rate.bytes.per.second=524288000 rocksdb.sst.file.manager.max.trash.db.ratio=0.25 # Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when unset; see ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED. +# when the class is provided neither here nor as a JVM system property (there is no implicit default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java index 7a5a704929a..22ccda8c292 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java @@ -116,20 +116,6 @@ private ConfigKeys() { public static final String PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS = PUBSUB_CLIENT_CONFIG_PREFIX + "source.of.truth.admin.adapter.factory.class"; - /** - * Configuration key that controls whether the PubSub producer/consumer/admin adapter factories - * fall back to the Apache Kafka implementation when their factory-class config keys are not - * explicitly provided. - *

- * Defaults to {@code false} (fail fast): the {@code PubSubClientsFactory} throws when a factory-class - * config is missing, surfacing misconfiguration early instead of masking it behind an implicit Kafka - * default. Set this to {@code true} to fall back to Apache Kafka. The default may also be overridden - * via the same-named system property (the test JVM sets it to {@code true} so existing tests keep - * resolving to the Apache Kafka adapters without configuring the factory classes explicitly). - */ - public static final String PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED = - PUBSUB_CLIENT_CONFIG_PREFIX + "adapter.factory.kafka.fallback.enabled"; - /** * Configuration key for specifying the address of the PubSub broker (e.g., Kafka, Pulsar). *

diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java index de71cc39d25..90fc9e079cc 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java @@ -1,6 +1,5 @@ package com.linkedin.venice.pubsub; -import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; @@ -11,9 +10,6 @@ import static com.linkedin.venice.ConfigKeys.PUB_SUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; import com.linkedin.venice.exceptions.VeniceException; -import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; -import com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory; -import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubAdminAdapter; import com.linkedin.venice.pubsub.api.PubSubConsumerAdapter; import com.linkedin.venice.pubsub.api.PubSubProducerAdapter; @@ -30,19 +26,6 @@ public class PubSubClientsFactory { private static final Logger LOGGER = LogManager.getLogger(PubSubClientsFactory.class); - /** - * The hard default for {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED} - * when it is not provided in the properties: {@code false}, i.e. fail fast. A missing factory-class - * config then raises an exception instead of silently defaulting to Apache Kafka, which surfaces - * misconfiguration early (important for non-Kafka deployments). - *

- * This default can be overridden by the same-named system property. Production never sets it, so it - * stays fail-fast; the test JVM sets it to {@code true} (see the root {@code build.gradle} test - * configuration) so the large body of existing tests keep resolving to the Apache Kafka adapters - * without each having to configure the factory classes explicitly. - */ - public static final boolean DEFAULT_KAFKA_FALLBACK_ENABLED = false; - private enum FactoryType { PRODUCER, CONSUMER, ADMIN } @@ -82,7 +65,6 @@ public static PubSubProducerAdapterFactory createProducer veniceProperties, PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, PUB_SUB_PRODUCER_ADAPTER_FACTORY_CLASS, - ApacheKafkaProducerAdapterFactory.class.getName(), FactoryType.PRODUCER); } @@ -92,7 +74,6 @@ public static PubSubConsumerAdapterFactory createConsumer veniceProperties, PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, PUB_SUB_CONSUMER_ADAPTER_FACTORY_CLASS, - ApacheKafkaConsumerAdapterFactory.class.getName(), FactoryType.CONSUMER); } @@ -101,7 +82,6 @@ public static PubSubAdminAdapterFactory createAdminFactory(V veniceProperties, PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, PUB_SUB_ADMIN_ADAPTER_FACTORY_CLASS, - ApacheKafkaAdminAdapterFactory.class.getName(), FactoryType.ADMIN); } @@ -111,7 +91,6 @@ public static PubSubAdminAdapterFactory createSourceOfTruthA veniceProperties, PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS, PUB_SUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS, - ApacheKafkaAdminAdapterFactory.class.getName(), FactoryType.ADMIN); } @@ -119,48 +98,34 @@ private static T createFactory( VeniceProperties properties, String preferredConfigKey, String alternateConfigKey, - String defaultClassName, FactoryType factoryType) { String className; if (properties.containsKey(preferredConfigKey) || properties.containsKey(alternateConfigKey)) { className = properties.getStringWithAlternative(preferredConfigKey, alternateConfigKey); LOGGER.debug("Creating pub-sub {} adapter factory instance for class: {}", factoryType, className); } else { - boolean kafkaFallbackEnabled = - properties.getBoolean(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, isKafkaFallbackEnabledByDefault()); - if (!kafkaFallbackEnabled) { + // No implicit fallback to a default (e.g. Apache Kafka) adapter factory. The factory class must be + // provided at runtime, either in the properties above or as a JVM system property. If neither + // provides it, fail fast so the misconfiguration surfaces immediately. + className = System.getProperty(preferredConfigKey, System.getProperty(alternateConfigKey)); + if (className == null) { throw new VeniceException( String.format( - "PubSub %s adapter factory class is not configured. Set '%s' (or the legacy '%s') to the " - + "fully-qualified factory class name. Implicit fallback to the Apache Kafka adapter factory " - + "('%s') is disabled; set '%s=true' to re-enable it.", + "PubSub %s adapter factory class is not configured. Provide '%s' (or the legacy '%s') in the " + + "properties or as a JVM system property.", factoryType, preferredConfigKey, - alternateConfigKey, - defaultClassName, - PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED)); + alternateConfigKey)); } - className = defaultClassName; - LOGGER.debug("Creating pub-sub {} adapter factory instance with default class: {}", factoryType, className); + LOGGER.debug( + "Creating pub-sub {} adapter factory instance for class supplied via system property: {}", + factoryType, + className); } return createInstance(className); } - /** - * Resolves the default for {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED} - * when the supplied properties do not set it. Returns {@link #DEFAULT_KAFKA_FALLBACK_ENABLED} (fail - * fast) unless overridden by the same-named system property. Production leaves the system property - * unset; the test JVM sets it to {@code true} so existing tests keep resolving to the Apache Kafka - * adapters without configuring the factory classes explicitly. - */ - private static boolean isKafkaFallbackEnabledByDefault() { - return Boolean.parseBoolean( - System.getProperty( - PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, - Boolean.toString(DEFAULT_KAFKA_FALLBACK_ENABLED))); - } - public static T createInstance(String className) { try { return (T) Class.forName(className).getDeclaredConstructor().newInstance(); diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java b/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java index 08b5deb19af..e1b576720d7 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java @@ -1,6 +1,9 @@ package com.linkedin.venice.utils; -import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; import com.linkedin.venice.exceptions.VeniceException; import io.github.classgraph.ClassGraph; @@ -317,13 +320,14 @@ private static List prepareCommandArgList( command.add("-Djava.io.tmpdir=" + System.getProperty("java.io.tmpdir")); // Inherit IPv6 preference setting from parent process. command.add("-Djava.net.preferIPv6Addresses=" + System.getProperty("java.net.preferIPv6Addresses", "false")); - // Inherit the pub-sub adapter-factory fallback setting so forked Venice processes (e.g. isolated - // ingestion or test apps) resolve their pub-sub clients the same way as this process. No-op when the - // property is unset (e.g. in production, where the factory classes come from config instead). - String pubSubAdapterFallback = System.getProperty(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED); - if (pubSubAdapterFallback != null) { - command.add("-D" + PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED + "=" + pubSubAdapterFallback); - } + // Forward the pub-sub adapter factory classes (when provided as system properties) so forked Venice + // processes (e.g. isolated ingestion or test apps) resolve their pub-sub clients the same way as this + // process. There is no implicit default, so a fork that neither inherits these nor sets them in its + // config will fail fast. + forwardSystemPropertyIfSet(command, PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS); + forwardSystemPropertyIfSet(command, PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS); + forwardSystemPropertyIfSet(command, PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS); + forwardSystemPropertyIfSet(command, PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS); /** Add log4j2 configuration file and JVM arguments. @@ -351,6 +355,19 @@ private static List prepareCommandArgList( return command; } + /** + * Forwards a system property to the forked process command as a {@code -D} argument, but only when it + * is set in this (parent) process. This lets forked Venice processes inherit values that are supplied + * at runtime (e.g. the pub-sub adapter factory classes in tests) without injecting anything when the + * property is unset (e.g. in production, where such values come from config). + */ + private static void forwardSystemPropertyIfSet(List command, String propertyKey) { + String value = System.getProperty(propertyKey); + if (value != null) { + command.add("-D" + propertyKey + "=" + value); + } + } + public long pid() { return getPidOfProcess(process); } diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java index fe6c8767620..69ce390e8d6 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java @@ -1,22 +1,17 @@ package com.linkedin.venice.pubsub; -import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_PRODUCER_ADAPTER_FACTORY_CLASS; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; import com.linkedin.venice.exceptions.VeniceException; -import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; -import com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory; import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubAdminAdapter; import com.linkedin.venice.pubsub.api.PubSubConsumerAdapter; @@ -54,80 +49,92 @@ public void testCreateInstanceSuccess() { } /** - * The hard default is fail-fast (no implicit Kafka fallback). The test JVM overrides this to - * {@code true} via a system property (see the root {@code build.gradle}) so the rest of the suite - * keeps resolving to the Apache Kafka adapters; the mode tests below set the flag explicitly so they - * are independent of that ambient default. + * When the factory class is not present in the config, it is resolved from a JVM system property + * (the mechanism by which the value is "provided at runtime" — see the root {@code build.gradle}, + * which sets these for the whole test suite). There is no implicit Apache Kafka default. */ @Test - public void testDefaultIsFailFast() { - assertFalse(PubSubClientsFactory.DEFAULT_KAFKA_FALLBACK_ENABLED); + public void testResolvesFactoryClassFromSystemProperty() { + String key = PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; + String saved = System.getProperty(key); + try { + System.setProperty(key, TestPubSubProducerAdapterFactory.class.getName()); + PubSubProducerAdapterFactory factory = + PubSubClientsFactory.createProducerFactory(new VeniceProperties(new Properties())); + assertNotNull(factory); + assertEquals(factory.getClass().getName(), TestPubSubProducerAdapterFactory.class.getName()); + } finally { + restoreProperty(key, saved); + } } /** - * When the Kafka fallback is explicitly disabled (opt-in fail-fast), a missing factory-class config - * throws instead of silently defaulting to the Apache Kafka adapter factories. + * When the factory class is provided neither in the config nor as a system property, factory + * creation fails fast instead of silently defaulting to the Apache Kafka adapter factories. */ @Test - public void testFailFastWhenFallbackDisabledAndFactoryClassMissing() { - Properties props = new Properties(); - props.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); - VeniceProperties failFastProps = new VeniceProperties(props); - + public void testFailFastWhenFactoryClassNotProvided() { assertFailFast( - () -> PubSubClientsFactory.createProducerFactory(failFastProps), - PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS); + () -> PubSubClientsFactory.createProducerFactory(new VeniceProperties(new Properties())), + PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + PUB_SUB_PRODUCER_ADAPTER_FACTORY_CLASS); assertFailFast( - () -> PubSubClientsFactory.createConsumerFactory(failFastProps), - PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS); - assertFailFast(() -> PubSubClientsFactory.createAdminFactory(failFastProps), PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS); + () -> PubSubClientsFactory.createConsumerFactory(new VeniceProperties(new Properties())), + PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, + PUB_SUB_CONSUMER_ADAPTER_FACTORY_CLASS); assertFailFast( - () -> PubSubClientsFactory.createSourceOfTruthAdminFactory(failFastProps), - PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS); - // The instance constructor eagerly builds all three factories, so it should fail fast as well. - expectThrows(VeniceException.class, () -> new PubSubClientsFactory(failFastProps)); + () -> PubSubClientsFactory.createAdminFactory(new VeniceProperties(new Properties())), + PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, + PUB_SUB_ADMIN_ADAPTER_FACTORY_CLASS); } /** - * When the Kafka fallback is explicitly enabled, missing factory-class configs resolve to the - * Apache Kafka adapter factories (same as the default). + * An explicit factory-class config takes precedence over a system property. */ @Test - public void testKafkaFallbackWhenExplicitlyEnabled() { - Properties fallbackEnabled = new Properties(); - fallbackEnabled.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "true"); - verifyFactoryClasses( - fallbackEnabled, - ApacheKafkaProducerAdapterFactory.class, - ApacheKafkaConsumerAdapterFactory.class, - ApacheKafkaAdminAdapterFactory.class); + public void testExplicitConfigWinsOverSystemProperty() { + String key = PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; + String saved = System.getProperty(key); + try { + System.setProperty(key, ApacheKafkaProducerAdapterFactory.class.getName()); + Properties props = new Properties(); + props.put(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, TestPubSubProducerAdapterFactory.class.getName()); + PubSubProducerAdapterFactory factory = PubSubClientsFactory.createProducerFactory(new VeniceProperties(props)); + assertEquals(factory.getClass().getName(), TestPubSubProducerAdapterFactory.class.getName()); + } finally { + restoreProperty(key, saved); + } } /** - * Explicit factory-class configs are honored regardless of the fallback flag. + * Invokes {@code runnable} with the given factory-class system properties cleared, and asserts it + * fails fast with a message naming the missing config key. */ - @Test - public void testExplicitConfigWinsOverFailFast() { - Properties props = new Properties(); - props.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); - props.put(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, TestPubSubProducerAdapterFactory.class.getName()); - props.put(PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, TestPubSubConsumerAdapterFactory.class.getName()); - props.put(PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, TestPubSubAdminAdapterFactory.class.getName()); - verifyFactoryClasses( - props, - TestPubSubProducerAdapterFactory.class, - TestPubSubConsumerAdapterFactory.class, - TestPubSubAdminAdapterFactory.class); + private static void assertFailFast( + org.testng.Assert.ThrowingRunnable runnable, + String configKey, + String legacyConfigKey) { + String saved = System.getProperty(configKey); + String savedLegacy = System.getProperty(legacyConfigKey); + try { + System.clearProperty(configKey); + System.clearProperty(legacyConfigKey); + VeniceException e = expectThrows(VeniceException.class, runnable); + assertTrue( + e.getMessage().contains(configKey), + "Expected fail-fast message to reference '" + configKey + "' but was: " + e.getMessage()); + } finally { + restoreProperty(configKey, saved); + restoreProperty(legacyConfigKey, savedLegacy); + } } - private static void assertFailFast(org.testng.Assert.ThrowingRunnable runnable, String expectedConfigKeyInMessage) { - VeniceException e = expectThrows(VeniceException.class, runnable); - assertTrue( - e.getMessage().contains(expectedConfigKeyInMessage), - "Expected fail-fast message to reference '" + expectedConfigKeyInMessage + "' but was: " + e.getMessage()); - assertTrue( - e.getMessage().contains(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED), - "Expected fail-fast message to reference the fallback config key but was: " + e.getMessage()); + private static void restoreProperty(String key, String value) { + if (value == null) { + System.clearProperty(key); + } else { + System.setProperty(key, value); + } } private void verifyFactoryClasses( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java deleted file mode 100644 index 122b24fd9af..00000000000 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/pubsub/PubSubAdapterFactoryFailFastTest.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.linkedin.venice.pubsub; - -import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; -import static com.linkedin.venice.ConfigKeys.INGESTION_USE_DA_VINCI_CLIENT; -import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; -import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.ZOOKEEPER_ADDRESS; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.expectThrows; - -import com.linkedin.davinci.config.VeniceServerConfig; -import com.linkedin.venice.exceptions.VeniceException; -import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; -import com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory; -import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; -import com.linkedin.venice.utils.VeniceProperties; -import java.util.Properties; -import org.testng.annotations.Test; - - -/** - * End-to-end coverage for the fail-fast behavior of the pub-sub adapter factories, exercised through a - * real production config object ({@link VeniceServerConfig}) rather than the factory in isolation. - * {@link VeniceServerConfig} eagerly constructs a {@link PubSubClientsFactory} from its properties, so - * it is representative of how a Venice component resolves its pub-sub clients at startup. - *

- * Fail-fast is the production default: when the adapter factory class is not configured, - * {@link PubSubClientsFactory} throws instead of silently defaulting to Apache Kafka. The default can be - * flipped with {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED}; the - * test JVM sets that flag to {@code true} by default, so these tests set it explicitly to assert each - * mode independently of the ambient test default. - */ -public class PubSubAdapterFactoryFailFastTest { - private static Properties baseServerProperties() { - Properties props = new Properties(); - props.setProperty(CLUSTER_NAME, "test_cluster"); - props.setProperty(ZOOKEEPER_ADDRESS, "localhost:2181"); - props.setProperty(KAFKA_BOOTSTRAP_SERVERS, "localhost:9092"); - props.setProperty(INGESTION_USE_DA_VINCI_CLIENT, "true"); - return props; - } - - @Test - public void serverConfigUsesApacheKafkaWhenFallbackEnabled() { - Properties props = baseServerProperties(); - props.setProperty(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "true"); - - VeniceServerConfig config = new VeniceServerConfig(new VeniceProperties(props)); - assertNotNull(config.getPubSubClientsFactory()); - assertNotNull(config.getPubSubClientsFactory().getProducerAdapterFactory()); - } - - @Test - public void serverConfigFailsFastWhenFallbackDisabledAndFactoryClassMissing() { - // Represents the production default (fail fast). Set explicitly because the test JVM defaults the - // flag to true. - Properties props = baseServerProperties(); - props.setProperty(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); - - VeniceException e = expectThrows(VeniceException.class, () -> new VeniceServerConfig(new VeniceProperties(props))); - assertTrue( - e.getMessage().contains(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS), - "Expected fail-fast message to name the missing factory-class config but was: " + e.getMessage()); - assertTrue( - e.getMessage().contains(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED), - "Expected fail-fast message to name the fallback config key but was: " + e.getMessage()); - } - - @Test - public void serverConfigSucceedsWhenFallbackDisabledButFactoryClassesProvided() { - Properties props = baseServerProperties(); - props.setProperty(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); - props.setProperty(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, ApacheKafkaProducerAdapterFactory.class.getName()); - props.setProperty(PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, ApacheKafkaConsumerAdapterFactory.class.getName()); - props.setProperty(PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, ApacheKafkaAdminAdapterFactory.class.getName()); - - VeniceServerConfig config = new VeniceServerConfig(new VeniceProperties(props)); - assertNotNull(config.getPubSubClientsFactory()); - assertNotNull(config.getPubSubClientsFactory().getProducerAdapterFactory()); - } -} diff --git a/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java b/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java index b332efd3e64..be4e64a08e1 100644 --- a/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java +++ b/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java @@ -31,6 +31,15 @@ public class PulsarVeniceSinkTest { private static final Logger LOGGER = LogManager.getLogger(PulsarVeniceSinkTest.class); + // Apache Kafka pub-sub adapter factory classes passed to admin-tool JVM invocations run inside the + // venice-client container. The admin tool fails fast when these are not provided (no implicit default). + private static final String PUBSUB_ADAPTER_FACTORY_JVM_ARGS = "-Dpubsub.producer.adapter.factory.class=" + + "com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory " + + "-Dpubsub.consumer.adapter.factory.class=" + + "com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory " + + "-Dpubsub.admin.adapter.factory.class=" + + "com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory"; + private DockerComposeContainer environment; // Generated by delombok @@ -158,8 +167,8 @@ public void testPulsarVeniceSink() throws Exception { // Wait for the store to become queryable before proceeding LOGGER.info("Waiting for Venice store to be ready"); - String readinessCmd = "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar - + " --describe-store --url " + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName; + String readinessCmd = "java " + PUBSUB_ADAPTER_FACTORY_JVM_ARGS + " -jar " + jar + " --describe-store --url " + + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName; Awaitility.await().atMost(30, TimeUnit.SECONDS).pollInterval(2, TimeUnit.SECONDS).untilAsserted(() -> { ExecResult res = execByService("venice-client", "bash", "-c", readinessCmd); String stdout = res.getStdout(); @@ -269,9 +278,8 @@ private void initVeniceStore(String veniceControllerUrl, String jar, String clus "venice-client", "bash", "-c", - "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar + " --empty-push --url " - + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName - + " --push-id init --store-size 1000"); + "java " + PUBSUB_ADAPTER_FACTORY_JVM_ARGS + " -jar " + jar + " --empty-push --url " + veniceControllerUrl + + " --cluster " + clusterName + " --store " + storeName + " --push-id init --store-size 1000"); } private void updateVeniceStoreQuotas(String veniceControllerUrl, String jar, String clusterName, String storeName) @@ -280,16 +288,16 @@ private void updateVeniceStoreQuotas(String veniceControllerUrl, String jar, Str "venice-client", "bash", "-c", - "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar + " --update-store --url " - + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName + "java " + PUBSUB_ADAPTER_FACTORY_JVM_ARGS + " -jar " + jar + " --update-store --url " + veniceControllerUrl + + " --cluster " + clusterName + " --store " + storeName + " --storage-quota -1 --incremental-push-enabled true"); execByServiceAsssertNoStdErr( "venice-client", "bash", "-c", - "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar + " --update-store --url " - + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName + " --read-quota 1000000"); + "java " + PUBSUB_ADAPTER_FACTORY_JVM_ARGS + " -jar " + jar + " --update-store --url " + veniceControllerUrl + + " --cluster " + clusterName + " --store " + storeName + " --read-quota 1000000"); } private void createVeniceStore( @@ -303,9 +311,9 @@ private void createVeniceStore( "venice-client", "bash", "-c", - "java -Dpubsub.adapter.factory.kafka.fallback.enabled=true -jar " + jar + " --new-store --url " - + veniceControllerUrl + " --cluster " + clusterName + " --store " + storeName + " --key-schema-file " - + keyFile + " --value-schema-file " + valueFile); + "java " + PUBSUB_ADAPTER_FACTORY_JVM_ARGS + " -jar " + jar + " --new-store --url " + veniceControllerUrl + + " --cluster " + clusterName + " --store " + storeName + " --key-schema-file " + keyFile + + " --value-schema-file " + valueFile); } private void saveKeyValueSchemaFiles(String keyAsvc, String valueAsvc, String keyFile, String valueFile) From 4815af703333cbd2c03fe8e12dd0aaa10553d166 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 16:03:59 -0700 Subject: [PATCH 12/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured PubSubClientsFactory silently defaulted the producer/consumer/admin/source-of-truth adapter factories to Apache Kafka when their factory-class config was missing, masking misconfiguration on non-Kafka deployments. Make the fallback config-driven and fail-fast by default: - Add pubsub.adapter.factory.kafka.fallback.enabled (default false). When a factory-class config is missing and the fallback is disabled, PubSubClientsFactory throws a VeniceException naming the missing key instead of constructing the Apache Kafka factory. Set it to true to restore the legacy implicit-Kafka behavior. Fix every place that relied on the implicit default to configure the factory classes explicitly: - KafkaBrokerFactory.getAdditionalConfig() advertises the Apache Kafka producer/consumer/admin/ source-of-truth factory classes so they propagate to every integration-test component. - TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs() seeds unit-test server/controller configs; the affected DaVinci / changelog-consumer / writer / storage-engine tests use it. - Docker venice-server / venice-controller configs and VenicePulsarSink set the factory classes; the Pulsar sink test passes them to its admin-tool JVM invocations. PubSubClientsFactoryTest covers config resolution, fail-fast-by-default, and the explicit fallback-enabled path. --- build.gradle | 9 -- .../linkedin/davinci/DaVinciBackendTest.java | 2 + .../linkedin/davinci/StoreBackendTest.java | 9 +- .../client/AvroGenericDaVinciClientTest.java | 4 + .../config/VeniceServerConfigTest.java | 2 + ...iceChangelogConsumerClientFactoryTest.java | 7 ++ ...sumerDaVinciRecordTransformerImplTest.java | 3 +- .../VeniceChangelogConsumerImplTest.java | 3 +- ...sumerDaVinciRecordTransformerImplTest.java | 3 +- .../consumer/StoreIngestionTaskTest.java | 1 + .../store/AbstractStorageEngineTest.java | 2 + .../dc-0.venice.controller.properties | 4 +- .../dc-parent.venice.controller.properties | 4 +- .../single-dc-configs/controller.properties | 4 +- .../multi-dc-configs/dc-0/server.properties | 4 +- .../multi-dc-configs/dc-1/server.properties | 4 +- .../single-dc-configs/server.properties | 4 +- .../java/com/linkedin/venice/ConfigKeys.java | 13 ++ .../venice/pubsub/PubSubClientsFactory.java | 40 +++++-- .../venice/utils/ForkedJavaProcess.java | 26 ---- .../pubsub/PubSubClientsFactoryTest.java | 113 ++++++------------ .../writer/VeniceWriterFactoryTest.java | 1 + .../integration/utils/KafkaBrokerFactory.java | 19 ++- .../com/linkedin/venice/utils/TestUtils.java | 28 +++++ 24 files changed, 165 insertions(+), 144 deletions(-) diff --git a/build.gradle b/build.gradle index b69922efb93..438b0cb1fe6 100644 --- a/build.gradle +++ b/build.gradle @@ -434,15 +434,6 @@ subprojects { systemProperty 'pubSubBrokerFactory', System.getProperty('pubSubBrokerFactory', "com.linkedin.venice.integration.utils.KafkaBrokerFactory") - // Provide the pub-sub adapter factory classes to the test JVMs at runtime. The factory fails fast - // when these are not supplied (there is no implicit default); passing them here means the existing - // test suite runs against the Apache Kafka adapters without each test having to set them, while a - // missing value still throws in any context that does not provide it (e.g. containers, production). - systemProperty 'pubsub.producer.adapter.factory.class', System.getProperty('pubsub.producer.adapter.factory.class', 'com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory') - systemProperty 'pubsub.consumer.adapter.factory.class', System.getProperty('pubsub.consumer.adapter.factory.class', 'com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory') - systemProperty 'pubsub.admin.adapter.factory.class', System.getProperty('pubsub.admin.adapter.factory.class', 'com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory') - systemProperty 'pubsub.source.of.truth.admin.adapter.factory.class', System.getProperty('pubsub.source.of.truth.admin.adapter.factory.class', 'com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory') - System.getProperty('jvmArgs')?.eachMatch(/(?:[^\s'"]+|'[^']*'|"[^"]*")+/) { jvmArgs it } doFirst { diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java index 2b581b70642..12ecb361c29 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/DaVinciBackendTest.java @@ -47,6 +47,7 @@ import com.linkedin.venice.schema.writecompute.DerivedSchemaEntry; import com.linkedin.venice.serialization.avro.SchemaPresenceChecker; import com.linkedin.venice.service.ICProvider; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import io.tehuti.metrics.MetricsRepository; import java.util.Optional; @@ -83,6 +84,7 @@ public void setUp() throws Exception { serverProps.setProperty(INGESTION_USE_DA_VINCI_CLIENT, "true"); serverProps.setProperty(DATA_BASE_PATH, "/tmp/test"); serverProps.setProperty(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, "0"); + serverProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); VeniceProperties veniceProperties = new VeniceProperties(serverProps); VeniceConfigLoader configLoader = new VeniceConfigLoader(veniceProperties); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java index fc2273ca23b..2bcb871c425 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/StoreBackendTest.java @@ -84,7 +84,8 @@ public class StoreBackendTest { @BeforeMethod void setUp() { baseDataPath = Utils.getTempDataDirectory(); - VeniceProperties backendConfig = new PropertyBuilder().put(ConfigKeys.CLUSTER_NAME, "test-cluster") + VeniceProperties backendConfig = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .put(ConfigKeys.CLUSTER_NAME, "test-cluster") .put(ConfigKeys.ZOOKEEPER_ADDRESS, "test-zookeeper") .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) @@ -668,7 +669,8 @@ public void testResumePausedSITOnTargetPromotion() throws Exception { @Test public void testLegacyNonTargetRegionSubscribesOnOnline() throws Exception { // Re-create storeBackend with paused-SIT disabled (legacy mode). - VeniceProperties legacyConfig = new PropertyBuilder().put(ConfigKeys.CLUSTER_NAME, "test-cluster") + VeniceProperties legacyConfig = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .put(ConfigKeys.CLUSTER_NAME, "test-cluster") .put(ConfigKeys.ZOOKEEPER_ADDRESS, "test-zookeeper") .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) @@ -716,7 +718,8 @@ public void testLegacyNonTargetRegionSubscribesOnOnline() throws Exception { * region stays {@code dc-0} and paused-SIT stays enabled. */ private void rebuildStoreBackendWithRollForwardOrder(String rollForwardOrder) { - VeniceProperties config = new PropertyBuilder().put(ConfigKeys.CLUSTER_NAME, "test-cluster") + VeniceProperties config = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .put(ConfigKeys.CLUSTER_NAME, "test-cluster") .put(ConfigKeys.ZOOKEEPER_ADDRESS, "test-zookeeper") .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java index 664d1ce6c1f..7d112ae4bac 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java @@ -46,6 +46,7 @@ import com.linkedin.venice.utils.DaemonThreadFactory; import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.ReferenceCounted; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import java.lang.reflect.Field; import java.security.AccessController; @@ -85,6 +86,7 @@ public AvroGenericDaVinciClient setUpSpecificClient(ClientConfig clientConfig, b DaVinciConfig daVinciConfig = new DaVinciConfig(); VeniceProperties backendConfig = new PropertyBuilder().put(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, false) .put(DAVINCI_VALIDATE_SPECIFIC_SCHEMA_ENABLED, validateSpecificSchema) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); AvroGenericDaVinciClient dvcClient = @@ -130,6 +132,7 @@ public AvroGenericSeekableDaVinciClient setUpSeekableClient(ClientConfig clientC DaVinciConfig daVinciConfig = new DaVinciConfig(); VeniceProperties backendConfig = new PropertyBuilder().put(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, false) .put(DAVINCI_VALIDATE_SPECIFIC_SCHEMA_ENABLED, validateSpecificSchema) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); AvroGenericSeekableDaVinciClient dvcClient = spy( @@ -210,6 +213,7 @@ public AvroGenericDaVinciClient setUpClientWithRecordTransformer( VeniceProperties backendConfig = new PropertyBuilder().put(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, enableDatabaseChecksumVerification) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); AvroGenericDaVinciClient dvcClient = diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java index 9135195b45c..567415004e7 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/VeniceServerConfigTest.java @@ -28,6 +28,7 @@ import static org.testng.Assert.assertTrue; import com.linkedin.davinci.blobtransfer.client.NettyFileTransferClient; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import java.util.Arrays; import java.util.HashMap; @@ -42,6 +43,7 @@ public class VeniceServerConfigTest { private Properties populatedBasicProperties() { Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); props.setProperty(CLUSTER_NAME, "test_cluster"); props.setProperty(ZOOKEEPER_ADDRESS, "fake_zk_addr"); props.setProperty(KAFKA_BOOTSTRAP_SERVERS, "fake_kafka_addr"); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java index 1ab53dbc84c..bda108b0791 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerClientFactoryTest.java @@ -31,6 +31,7 @@ import com.linkedin.venice.pubsub.api.PubSubMessageDeserializer; import com.linkedin.venice.schema.SchemaReader; import com.linkedin.venice.utils.ObjectMapperFactory; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.views.MaterializedView; import io.tehuti.metrics.MetricsRepository; import java.nio.charset.StandardCharsets; @@ -57,6 +58,7 @@ public class VeniceChangelogConsumerClientFactoryTest { @Test public void testGetChangelogConsumer() throws ExecutionException, InterruptedException, JsonProcessingException { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, localKafkaUrl); consumerProperties.put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, localKafkaUrl); @@ -131,6 +133,7 @@ public void testGetChangelogConsumer() throws ExecutionException, InterruptedExc public void testGetChangelogConsumerWithConsumerId() throws ExecutionException, InterruptedException, JsonProcessingException { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, localKafkaUrl); SchemaReader mockSchemaReader = Mockito.mock(SchemaReader.class); @@ -199,6 +202,7 @@ private void setUpMockStoreResponse(D2ControllerClient mockControllerClient, Str @Test public void testGetChangelogConsumerThrowsException() { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, localKafkaUrl); @@ -231,6 +235,7 @@ public void testGetChangelogConsumerThrowsException() { public void testGetStatefulChangelogConsumer() throws ExecutionException, InterruptedException, JsonProcessingException { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, localKafkaUrl); consumerProperties.put(CLUSTER_NAME, TEST_CLUSTER_NAME); @@ -312,6 +317,7 @@ public void testGetStatefulChangelogConsumer() @Test public void testGetStatefulChangelogConsumerThrowsException() { Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); String localKafkaUrl = "http://www.fooAddress.linkedin.com:16337"; consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, localKafkaUrl); consumerProperties.put(CLUSTER_NAME, TEST_CLUSTER_NAME); @@ -365,6 +371,7 @@ public void testCreatePubSubMessageDeserializer( boolean expectKmeWithSchemaReaderCall) { // Build properties Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); if (kmeProp != null) { props.put(ConfigKeys.KME_SCHEMA_READER_FOR_SCHEMA_EVOLUTION_ENABLED, kmeProp); } diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java index 41ac7dc16ac..01ceb499be2 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImplTest.java @@ -55,7 +55,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Properties; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; @@ -116,7 +115,7 @@ public void setUp() throws NoSuchFieldException, IllegalAccessException { .setStoreName(TEST_STORE_NAME) .setControllerD2ServiceName(D2_SERVICE_NAME) .setD2ServiceName(DEFAULT_CLUSTER_DISCOVERY_D2_SERVICE_NAME) - .setConsumerProperties(new Properties()) + .setConsumerProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .setLocalD2ZkHosts(TEST_ZOOKEEPER_ADDRESS) .setDatabaseSyncBytesInterval(TEST_DB_SYNC_BYTES_INTERVAL) .setD2Client(mock(D2Client.class)) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java index 4b1a8043736..6a6d876ce53 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImplTest.java @@ -208,6 +208,7 @@ public void testConfig() { assertTrue(config.getConsumerProperties().isEmpty()); assertThrows(NullPointerException.class, () -> config.setConsumerProperties(null)); Properties newProps = new Properties(); + newProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); newProps.setProperty("foo", "bar"); config.setConsumerProperties(newProps); assertNotNull(config.getConsumerProperties()); @@ -1467,7 +1468,7 @@ private ChangelogClientConfig getChangelogClientConfig() { new ChangelogClientConfig<>().setD2ControllerClient(mockD2ControllerClient) .setSchemaReader(schemaReader) .setStoreName(storeName) - .setConsumerProperties(new Properties()) + .setConsumerProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .setViewName(""); changelogClientConfig.getInnerClientConfig() .setMetricsRepository(getVeniceMetricsRepository(CHANGE_DATA_CAPTURE_CLIENT, CONSUMER_METRIC_ENTITIES, true)); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java index c7e042c1ab4..edf78994945 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/VersionSpecificVeniceChangelogConsumerDaVinciRecordTransformerImplTest.java @@ -47,7 +47,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -103,7 +102,7 @@ public void setUp() throws NoSuchFieldException, IllegalAccessException { .setStoreName(TEST_STORE_NAME) .setControllerD2ServiceName(D2_SERVICE_NAME) .setD2ServiceName(DEFAULT_CLUSTER_DISCOVERY_D2_SERVICE_NAME) - .setConsumerProperties(new Properties()) + .setConsumerProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .setLocalD2ZkHosts(TEST_ZOOKEEPER_ADDRESS) .setDatabaseSyncBytesInterval(TEST_DB_SYNC_BYTES_INTERVAL) .setD2Client(mock(D2Client.class)) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java index d4d60f3be06..3d72063ad7a 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java @@ -3298,6 +3298,7 @@ public void testPartitionExceptionIsolation(AAConfig aaConfig) throws Exception private VeniceServerConfig buildVeniceServerConfig(Map extraProperties) { PropertyBuilder propertyBuilder = new PropertyBuilder(); + propertyBuilder.put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); propertyBuilder.put(CLUSTER_NAME, ""); propertyBuilder.put(ZOOKEEPER_ADDRESS, ""); propertyBuilder.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 500L); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java index 7a22e97ec2b..01dcf7f7932 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java @@ -17,6 +17,7 @@ import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.RandomGenUtils; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import java.io.File; @@ -45,6 +46,7 @@ public static VeniceProperties getServerProperties(PersistenceType persistenceTy .put(LISTENER_PORT, 7072) .put(ADMIN_PORT, 7073) .put(DATA_BASE_PATH, dataDirectory.getAbsolutePath()) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .put(properties) .build(); } diff --git a/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties b/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties index acc87823a09..e2ce3b607dc 100644 --- a/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties +++ b/docker/venice-controller/multi-dc-configs/dc-0.venice.controller.properties @@ -56,8 +56,8 @@ controller.enable.batch.push.from.admin.in.child=false default.partition.size=100 topic.cleanup.delay.factor=2 -# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when the class is provided neither here nor as a JVM system property (there is no implicit default). +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast when the +# class is unset and pubsub.adapter.factory.kafka.fallback.enabled is false (the default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties b/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties index 6c74f6c7377..828965b709f 100644 --- a/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties +++ b/docker/venice-controller/multi-dc-configs/dc-parent.venice.controller.properties @@ -57,8 +57,8 @@ native.replication.source.fabric.as.default.for.batch.only.stores=dc-0 default.partition.size=100 topic.cleanup.delay.factor=2 -# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when the class is provided neither here nor as a JVM system property (there is no implicit default). +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast when the +# class is unset and pubsub.adapter.factory.kafka.fallback.enabled is false (the default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-controller/single-dc-configs/controller.properties b/docker/venice-controller/single-dc-configs/controller.properties index 71ecff77f43..9aff1f261fd 100644 --- a/docker/venice-controller/single-dc-configs/controller.properties +++ b/docker/venice-controller/single-dc-configs/controller.properties @@ -28,8 +28,8 @@ kafka.linger.ms=0 default.partition.count=1 controller.zk.shared.metadata.system.schema.store.auto.creation.enabled=true -# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when the class is provided neither here nor as a JVM system property (there is no implicit default). +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast when the +# class is unset and pubsub.adapter.factory.kafka.fallback.enabled is false (the default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-server/multi-dc-configs/dc-0/server.properties b/docker/venice-server/multi-dc-configs/dc-0/server.properties index d78fa5266f7..a9fec9041b2 100644 --- a/docker/venice-server/multi-dc-configs/dc-0/server.properties +++ b/docker/venice-server/multi-dc-configs/dc-0/server.properties @@ -17,8 +17,8 @@ rocksdb.block.cache.size.in.bytes=2147483648 rocksdb.sst.file.manager.delete.rate.bytes.per.second=524288000 rocksdb.sst.file.manager.max.trash.db.ratio=0.25 -# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when the class is provided neither here nor as a JVM system property (there is no implicit default). +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast when the +# class is unset and pubsub.adapter.factory.kafka.fallback.enabled is false (the default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-server/multi-dc-configs/dc-1/server.properties b/docker/venice-server/multi-dc-configs/dc-1/server.properties index 15bc3d9b273..472d678eb45 100644 --- a/docker/venice-server/multi-dc-configs/dc-1/server.properties +++ b/docker/venice-server/multi-dc-configs/dc-1/server.properties @@ -17,8 +17,8 @@ rocksdb.block.cache.size.in.bytes=2147483648 rocksdb.sst.file.manager.delete.rate.bytes.per.second=524288000 rocksdb.sst.file.manager.max.trash.db.ratio=0.25 -# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when the class is provided neither here nor as a JVM system property (there is no implicit default). +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast when the +# class is unset and pubsub.adapter.factory.kafka.fallback.enabled is false (the default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/docker/venice-server/single-dc-configs/server.properties b/docker/venice-server/single-dc-configs/server.properties index 495bf3931dc..9122dd07a05 100644 --- a/docker/venice-server/single-dc-configs/server.properties +++ b/docker/venice-server/single-dc-configs/server.properties @@ -20,8 +20,8 @@ rocksdb.block.cache.size.in.bytes=2147483648 rocksdb.sst.file.manager.delete.rate.bytes.per.second=524288000 rocksdb.sst.file.manager.max.trash.db.ratio=0.25 -# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast -# when the class is provided neither here nor as a JVM system property (there is no implicit default). +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast when the +# class is unset and pubsub.adapter.factory.kafka.fallback.enabled is false (the default). pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java index 22ccda8c292..36b178093ed 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java @@ -116,6 +116,19 @@ private ConfigKeys() { public static final String PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS = PUBSUB_CLIENT_CONFIG_PREFIX + "source.of.truth.admin.adapter.factory.class"; + /** + * Configuration key that controls whether the PubSub producer/consumer/admin adapter factories + * silently fall back to the Apache Kafka implementation when their factory-class config keys are + * not explicitly provided. + *

+ * When {@code false} (the default), the {@code PubSubClientsFactory} fails fast by throwing an + * exception if the corresponding factory-class config is missing. This surfaces misconfiguration + * early instead of masking it behind an implicit Kafka default. Set this to {@code true} to + * restore the legacy behavior of defaulting to the Apache Kafka adapter factories. + */ + public static final String PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED = + PUBSUB_CLIENT_CONFIG_PREFIX + "adapter.factory.kafka.fallback.enabled"; + /** * Configuration key for specifying the address of the PubSub broker (e.g., Kafka, Pulsar). *

diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java index 90fc9e079cc..2b9d465c838 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/PubSubClientsFactory.java @@ -1,5 +1,6 @@ package com.linkedin.venice.pubsub; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; @@ -10,6 +11,9 @@ import static com.linkedin.venice.ConfigKeys.PUB_SUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; +import com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory; +import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubAdminAdapter; import com.linkedin.venice.pubsub.api.PubSubConsumerAdapter; import com.linkedin.venice.pubsub.api.PubSubProducerAdapter; @@ -26,6 +30,14 @@ public class PubSubClientsFactory { private static final Logger LOGGER = LogManager.getLogger(PubSubClientsFactory.class); + /** + * By default the adapter factories do NOT fall back to Apache Kafka when their factory-class config + * is missing; callers must configure the factory classes explicitly so that misconfiguration fails + * fast. Set {@link com.linkedin.venice.ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED} to + * {@code true} to restore the legacy implicit-Kafka behavior. + */ + public static final boolean DEFAULT_KAFKA_FALLBACK_ENABLED = false; + private enum FactoryType { PRODUCER, CONSUMER, ADMIN } @@ -65,6 +77,7 @@ public static PubSubProducerAdapterFactory createProducer veniceProperties, PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, PUB_SUB_PRODUCER_ADAPTER_FACTORY_CLASS, + ApacheKafkaProducerAdapterFactory.class.getName(), FactoryType.PRODUCER); } @@ -74,6 +87,7 @@ public static PubSubConsumerAdapterFactory createConsumer veniceProperties, PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, PUB_SUB_CONSUMER_ADAPTER_FACTORY_CLASS, + ApacheKafkaConsumerAdapterFactory.class.getName(), FactoryType.CONSUMER); } @@ -82,6 +96,7 @@ public static PubSubAdminAdapterFactory createAdminFactory(V veniceProperties, PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, PUB_SUB_ADMIN_ADAPTER_FACTORY_CLASS, + ApacheKafkaAdminAdapterFactory.class.getName(), FactoryType.ADMIN); } @@ -91,6 +106,7 @@ public static PubSubAdminAdapterFactory createSourceOfTruthA veniceProperties, PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS, PUB_SUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS, + ApacheKafkaAdminAdapterFactory.class.getName(), FactoryType.ADMIN); } @@ -98,29 +114,29 @@ private static T createFactory( VeniceProperties properties, String preferredConfigKey, String alternateConfigKey, + String defaultClassName, FactoryType factoryType) { String className; if (properties.containsKey(preferredConfigKey) || properties.containsKey(alternateConfigKey)) { className = properties.getStringWithAlternative(preferredConfigKey, alternateConfigKey); LOGGER.debug("Creating pub-sub {} adapter factory instance for class: {}", factoryType, className); } else { - // No implicit fallback to a default (e.g. Apache Kafka) adapter factory. The factory class must be - // provided at runtime, either in the properties above or as a JVM system property. If neither - // provides it, fail fast so the misconfiguration surfaces immediately. - className = System.getProperty(preferredConfigKey, System.getProperty(alternateConfigKey)); - if (className == null) { + boolean kafkaFallbackEnabled = + properties.getBoolean(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, DEFAULT_KAFKA_FALLBACK_ENABLED); + if (!kafkaFallbackEnabled) { throw new VeniceException( String.format( - "PubSub %s adapter factory class is not configured. Provide '%s' (or the legacy '%s') in the " - + "properties or as a JVM system property.", + "PubSub %s adapter factory class is not configured. Set '%s' (or the legacy '%s') to the " + + "fully-qualified factory class name. Implicit fallback to the Apache Kafka adapter factory " + + "('%s') is disabled; set '%s=true' to re-enable it.", factoryType, preferredConfigKey, - alternateConfigKey)); + alternateConfigKey, + defaultClassName, + PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED)); } - LOGGER.debug( - "Creating pub-sub {} adapter factory instance for class supplied via system property: {}", - factoryType, - className); + className = defaultClassName; + LOGGER.debug("Creating pub-sub {} adapter factory instance with default class: {}", factoryType, className); } return createInstance(className); diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java b/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java index e1b576720d7..73eae7e9a3d 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/utils/ForkedJavaProcess.java @@ -1,10 +1,5 @@ package com.linkedin.venice.utils; -import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; - import com.linkedin.venice.exceptions.VeniceException; import io.github.classgraph.ClassGraph; import io.github.classgraph.ScanResult; @@ -320,14 +315,6 @@ private static List prepareCommandArgList( command.add("-Djava.io.tmpdir=" + System.getProperty("java.io.tmpdir")); // Inherit IPv6 preference setting from parent process. command.add("-Djava.net.preferIPv6Addresses=" + System.getProperty("java.net.preferIPv6Addresses", "false")); - // Forward the pub-sub adapter factory classes (when provided as system properties) so forked Venice - // processes (e.g. isolated ingestion or test apps) resolve their pub-sub clients the same way as this - // process. There is no implicit default, so a fork that neither inherits these nor sets them in its - // config will fail fast. - forwardSystemPropertyIfSet(command, PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS); - forwardSystemPropertyIfSet(command, PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS); - forwardSystemPropertyIfSet(command, PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS); - forwardSystemPropertyIfSet(command, PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS); /** Add log4j2 configuration file and JVM arguments. @@ -355,19 +342,6 @@ private static List prepareCommandArgList( return command; } - /** - * Forwards a system property to the forked process command as a {@code -D} argument, but only when it - * is set in this (parent) process. This lets forked Venice processes inherit values that are supplied - * at runtime (e.g. the pub-sub adapter factory classes in tests) without injecting anything when the - * property is unset (e.g. in production, where such values come from config). - */ - private static void forwardSystemPropertyIfSet(List command, String propertyKey) { - String value = System.getProperty(propertyKey); - if (value != null) { - command.add("-D" + propertyKey + "=" + value); - } - } - public long pid() { return getPidOfProcess(process); } diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java index 69ce390e8d6..a0b7b55be20 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/PubSubClientsFactoryTest.java @@ -1,8 +1,10 @@ package com.linkedin.venice.pubsub; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED; import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_CONSUMER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.PUB_SUB_PRODUCER_ADAPTER_FACTORY_CLASS; @@ -12,6 +14,8 @@ import static org.testng.Assert.expectThrows; import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; +import com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory; import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubAdminAdapter; import com.linkedin.venice.pubsub.api.PubSubConsumerAdapter; @@ -49,92 +53,51 @@ public void testCreateInstanceSuccess() { } /** - * When the factory class is not present in the config, it is resolved from a JVM system property - * (the mechanism by which the value is "provided at runtime" — see the root {@code build.gradle}, - * which sets these for the whole test suite). There is no implicit Apache Kafka default. + * By default (no factory-class config and no explicit fallback flag) the factory should fail fast + * instead of silently defaulting to the Apache Kafka adapter factories. */ @Test - public void testResolvesFactoryClassFromSystemProperty() { - String key = PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; - String saved = System.getProperty(key); - try { - System.setProperty(key, TestPubSubProducerAdapterFactory.class.getName()); - PubSubProducerAdapterFactory factory = - PubSubClientsFactory.createProducerFactory(new VeniceProperties(new Properties())); - assertNotNull(factory); - assertEquals(factory.getClass().getName(), TestPubSubProducerAdapterFactory.class.getName()); - } finally { - restoreProperty(key, saved); - } - } + public void testFailFastWhenFactoryClassMissingAndFallbackDisabled() { + VeniceProperties emptyProps = new VeniceProperties(new Properties()); - /** - * When the factory class is provided neither in the config nor as a system property, factory - * creation fails fast instead of silently defaulting to the Apache Kafka adapter factories. - */ - @Test - public void testFailFastWhenFactoryClassNotProvided() { - assertFailFast( - () -> PubSubClientsFactory.createProducerFactory(new VeniceProperties(new Properties())), - PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, - PUB_SUB_PRODUCER_ADAPTER_FACTORY_CLASS); + assertFailFast(() -> PubSubClientsFactory.createProducerFactory(emptyProps), PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS); + assertFailFast(() -> PubSubClientsFactory.createConsumerFactory(emptyProps), PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS); + assertFailFast(() -> PubSubClientsFactory.createAdminFactory(emptyProps), PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS); assertFailFast( - () -> PubSubClientsFactory.createConsumerFactory(new VeniceProperties(new Properties())), - PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, - PUB_SUB_CONSUMER_ADAPTER_FACTORY_CLASS); - assertFailFast( - () -> PubSubClientsFactory.createAdminFactory(new VeniceProperties(new Properties())), - PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, - PUB_SUB_ADMIN_ADAPTER_FACTORY_CLASS); + () -> PubSubClientsFactory.createSourceOfTruthAdminFactory(emptyProps), + PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS); + // The instance constructor eagerly builds all three factories, so it should fail fast as well. + expectThrows(VeniceException.class, () -> new PubSubClientsFactory(emptyProps)); + + // Explicitly disabling the fallback behaves the same as the default. + Properties fallbackDisabled = new Properties(); + fallbackDisabled.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "false"); + expectThrows(VeniceException.class, () -> new PubSubClientsFactory(new VeniceProperties(fallbackDisabled))); } /** - * An explicit factory-class config takes precedence over a system property. + * When the Kafka fallback is explicitly enabled, missing factory-class configs should resolve to the + * Apache Kafka adapter factories (the legacy behavior). */ @Test - public void testExplicitConfigWinsOverSystemProperty() { - String key = PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; - String saved = System.getProperty(key); - try { - System.setProperty(key, ApacheKafkaProducerAdapterFactory.class.getName()); - Properties props = new Properties(); - props.put(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, TestPubSubProducerAdapterFactory.class.getName()); - PubSubProducerAdapterFactory factory = PubSubClientsFactory.createProducerFactory(new VeniceProperties(props)); - assertEquals(factory.getClass().getName(), TestPubSubProducerAdapterFactory.class.getName()); - } finally { - restoreProperty(key, saved); - } - } - - /** - * Invokes {@code runnable} with the given factory-class system properties cleared, and asserts it - * fails fast with a message naming the missing config key. - */ - private static void assertFailFast( - org.testng.Assert.ThrowingRunnable runnable, - String configKey, - String legacyConfigKey) { - String saved = System.getProperty(configKey); - String savedLegacy = System.getProperty(legacyConfigKey); - try { - System.clearProperty(configKey); - System.clearProperty(legacyConfigKey); - VeniceException e = expectThrows(VeniceException.class, runnable); - assertTrue( - e.getMessage().contains(configKey), - "Expected fail-fast message to reference '" + configKey + "' but was: " + e.getMessage()); - } finally { - restoreProperty(configKey, saved); - restoreProperty(legacyConfigKey, savedLegacy); - } + public void testKafkaFallbackWhenExplicitlyEnabled() { + Properties fallbackEnabled = new Properties(); + fallbackEnabled.put(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED, "true"); + verifyFactoryClasses( + fallbackEnabled, + ApacheKafkaProducerAdapterFactory.class, + ApacheKafkaConsumerAdapterFactory.class, + ApacheKafkaAdminAdapterFactory.class); } - private static void restoreProperty(String key, String value) { - if (value == null) { - System.clearProperty(key); - } else { - System.setProperty(key, value); - } + private static void assertFailFast(org.testng.Assert.ThrowingRunnable runnable, String expectedConfigKeyInMessage) { + VeniceException e = expectThrows(VeniceException.class, runnable); + assertTrue( + e.getMessage().contains(expectedConfigKeyInMessage), + "Expected fail-fast message to reference '" + expectedConfigKeyInMessage + "' but was: " + e.getMessage()); + assertTrue( + e.getMessage().contains(PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED), + "Expected fail-fast message to reference the fallback config key but was: " + e.getMessage()); } private void verifyFactoryClasses( diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java index aacb7c249be..2425ef0d00d 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/writer/VeniceWriterFactoryTest.java @@ -110,6 +110,7 @@ public void testVeniceWriterFactoryWithProducerCompressionDisabled() { public void testVeniceWriterFactoryCreatesProducerAdapterFactory() { Properties properties = new Properties(); properties.put(ConfigKeys.PUBSUB_BROKER_ADDRESS, "kafka:9898"); + properties.put(ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, ApacheKafkaProducerAdapterFactory.class.getName()); VeniceWriterFactory veniceWriterFactory = new VeniceWriterFactory(properties, null, null, null); assertNotNull(veniceWriterFactory.getProducerAdapterFactory()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java index 3a00eb22228..afbf479d63c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/KafkaBrokerFactory.java @@ -17,7 +17,6 @@ import com.linkedin.venice.utils.VeniceProperties; import java.io.File; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -286,9 +285,25 @@ public String getPubSubClusterName() { @Override public Map getAdditionalConfig() { - return Collections.singletonMap( + Map configs = new HashMap<>(); + configs.put( ConfigKeys.PUBSUB_TYPE_ID_TO_POSITION_CLASS_NAME_MAP, VeniceProperties.mapToString(PubSubPositionTypeRegistry.RESERVED_POSITION_TYPE_ID_TO_CLASS_NAME_MAP)); + // Explicitly advertise the Apache Kafka adapter factories so that clients relying on + // getBrokerDetailsForClients() do not depend on the (now disabled by default) implicit Kafka fallback. + configs.put( + ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + KAFKA_CLIENTS_FACTORY.getProducerAdapterFactory().getClass().getName()); + configs.put( + ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, + KAFKA_CLIENTS_FACTORY.getConsumerAdapterFactory().getClass().getName()); + configs.put( + ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, + KAFKA_CLIENTS_FACTORY.getAdminAdapterFactory().getClass().getName()); + configs.put( + ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS, + KAFKA_CLIENTS_FACTORY.getAdminAdapterFactory().getClass().getName()); + return configs; } @Override diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java index 432921d2bf7..a4e332bfcca 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java @@ -79,6 +79,9 @@ import com.linkedin.venice.pubsub.PubSubPositionTypeRegistry; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; import com.linkedin.venice.pubsub.PubSubTopicRepository; +import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; +import com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory; +import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubPosition; import com.linkedin.venice.pubsub.api.PubSubTopicType; import com.linkedin.venice.pubsub.manager.TopicManagerRepository; @@ -738,8 +741,33 @@ public static VeniceControllerMultiClusterConfig getMultiClusterConfigFromOneClu return new VeniceControllerMultiClusterConfig(configMap); } + /** + * Returns the Apache Kafka pub-sub adapter factory-class configs (producer, consumer, admin). + *

+ * Tests that build a {@link VeniceServerConfig} or {@link VeniceControllerClusterConfig} (which + * eagerly construct a {@code PubSubClientsFactory}) must supply these now that the implicit Apache + * Kafka fallback is disabled by default. See + * {@code ConfigKeys#PUBSUB_ADAPTER_FACTORY_KAFKA_FALLBACK_ENABLED}. + */ + public static Properties getPubSubApacheKafkaAdapterFactoryConfigs() { + Properties properties = new Properties(); + properties.setProperty( + ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + ApacheKafkaProducerAdapterFactory.class.getName()); + properties.setProperty( + ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, + ApacheKafkaConsumerAdapterFactory.class.getName()); + properties + .setProperty(ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, ApacheKafkaAdminAdapterFactory.class.getName()); + properties.setProperty( + ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS, + ApacheKafkaAdminAdapterFactory.class.getName()); + return properties; + } + public static Properties getPropertiesForControllerConfig() { Properties properties = new Properties(); + properties.putAll(getPubSubApacheKafkaAdapterFactoryConfigs()); properties.put(ConfigKeys.CLUSTER_NAME, "test-cluster"); properties.put(ConfigKeys.CONTROLLER_NAME, "venice-controller"); properties.put(ConfigKeys.DEFAULT_REPLICA_FACTOR, "1"); From e0e737872aa96bcaec781701883afd4cfe9f6b01 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 16:39:02 -0700 Subject: [PATCH 13/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured PubSubClientsFactory silently defaulted the producer/consumer/admin/source-of-truth adapter factories to Apache Kafka when their factory-class config was missing, masking misconfiguration on non-Kafka deployments. Make the fallback config-driven and fail-fast by default: - Add pubsub.adapter.factory.kafka.fallback.enabled (default false). When a factory-class config is missing and the fallback is disabled, PubSubClientsFactory throws a VeniceException naming the missing key instead of constructing the Apache Kafka factory. Set it to true to restore the legacy implicit-Kafka behavior. Fix every place that relied on the implicit default to configure the factory classes explicitly: - KafkaBrokerFactory.getAdditionalConfig() advertises the Apache Kafka producer/consumer/admin/ source-of-truth factory classes so they propagate to every integration-test component. - TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs() seeds unit-test server/controller configs; TestWriteUtils seeds VPJ defaults (with pub-sub pass-through) and IntegrationTestPushUtils seeds the Samza producer / writer-factory configs, covering the shared push paths. - Per-path fixes for admin-tool, push-job, fast-client, and online-producer / DaVinci-record-transformer integration tests that assemble their own producer config. - Docker venice-server / venice-controller configs and VenicePulsarSink set the factory classes; the Pulsar sink test passes them to its admin-tool JVM invocations. PubSubClientsFactoryTest covers config resolution, fail-fast-by-default, and the explicit fallback-enabled path. --- .../com/linkedin/venice/TestAdminTool.java | 31 +++++++++++++++++++ .../datawriter/reduce/TestVeniceReducer.java | 5 +++ .../heartbeat/TestPushJobHeartbeatSender.java | 5 +-- ...VinciRecordTransformerIntegrationTest.java | 2 +- .../samza/VeniceSystemProducerTest.java | 17 ++++++++++ .../DaVinciClientRecordTransformerTest.java | 4 +-- ...VeniceProducerOrderingIntegrationTest.java | 3 +- .../linkedin/venice/endToEnd/TestHybrid.java | 2 +- .../VersionSpecificDaVinciClientTest.java | 2 +- .../utils/AbstractClientEndToEndSetup.java | 1 + .../utils/IntegrationTestPushUtils.java | 9 ++++++ .../linkedin/venice/utils/TestWriteUtils.java | 4 +++ 12 files changed, 77 insertions(+), 8 deletions(-) diff --git a/clients/venice-admin-tool/src/test/java/com/linkedin/venice/TestAdminTool.java b/clients/venice-admin-tool/src/test/java/com/linkedin/venice/TestAdminTool.java index 5a062459b0a..d4e7e0521f5 100644 --- a/clients/venice-admin-tool/src/test/java/com/linkedin/venice/TestAdminTool.java +++ b/clients/venice-admin-tool/src/test/java/com/linkedin/venice/TestAdminTool.java @@ -57,6 +57,7 @@ import com.linkedin.venice.serialization.avro.AvroProtocolDefinition; import com.linkedin.venice.serializer.FastSerializerDeserializerFactory; import com.linkedin.venice.serializer.RecordSerializer; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import com.linkedin.venice.views.MaterializedView; import java.io.IOException; @@ -76,10 +77,40 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; public class TestAdminTool { + private static final String[] PUBSUB_ADAPTER_FACTORY_CONFIG_KEYS = + TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs().stringPropertyNames().toArray(new String[0]); + private static final Properties ORIGINAL_PUBSUB_ADAPTER_FACTORY_SYSTEM_PROPERTIES = new Properties(); + + @BeforeClass(alwaysRun = true) + public void setUpPubSubAdapterFactorySystemProperties() { + Properties factoryConfigs = TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs(); + for (String key: PUBSUB_ADAPTER_FACTORY_CONFIG_KEYS) { + String originalValue = System.getProperty(key); + if (originalValue != null) { + ORIGINAL_PUBSUB_ADAPTER_FACTORY_SYSTEM_PROPERTIES.setProperty(key, originalValue); + } + System.setProperty(key, factoryConfigs.getProperty(key)); + } + } + + @AfterClass(alwaysRun = true) + public void restorePubSubAdapterFactorySystemProperties() { + for (String key: PUBSUB_ADAPTER_FACTORY_CONFIG_KEYS) { + if (ORIGINAL_PUBSUB_ADAPTER_FACTORY_SYSTEM_PROPERTIES.containsKey(key)) { + System.setProperty(key, ORIGINAL_PUBSUB_ADAPTER_FACTORY_SYSTEM_PROPERTIES.getProperty(key)); + } else { + System.clearProperty(key); + } + } + ORIGINAL_PUBSUB_ADAPTER_FACTORY_SYSTEM_PROPERTIES.clear(); + } + @Test public void testPrintObject() { List output = new ArrayList<>(); diff --git a/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/mapreduce/datawriter/reduce/TestVeniceReducer.java b/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/mapreduce/datawriter/reduce/TestVeniceReducer.java index 77145c00b45..16857d6a745 100644 --- a/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/mapreduce/datawriter/reduce/TestVeniceReducer.java +++ b/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/mapreduce/datawriter/reduce/TestVeniceReducer.java @@ -1,5 +1,7 @@ package com.linkedin.venice.hadoop.mapreduce.datawriter.reduce; +import static com.linkedin.venice.ConfigKeys.PASS_THROUGH_CONFIG_PREFIXES_LIST_KEY; +import static com.linkedin.venice.ConfigKeys.PUBSUB_CLIENT_CONFIG_PREFIX; import static com.linkedin.venice.ConfigKeys.PUSH_JOB_VIEW_CONFIGS; import static com.linkedin.venice.hadoop.mapreduce.counter.MRJobCounterHelper.TOTAL_KEY_SIZE_GROUP_COUNTER_NAME; import static com.linkedin.venice.hadoop.mapreduce.counter.MRJobCounterHelper.TOTAL_VALUE_SIZE_GROUP_COUNTER_NAME; @@ -52,6 +54,7 @@ import com.linkedin.venice.pubsub.api.PubSubProduceResult; import com.linkedin.venice.pubsub.api.PubSubProducerCallback; import com.linkedin.venice.serialization.avro.VeniceAvroKafkaSerializer; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.VeniceProperties; import com.linkedin.venice.views.MaterializedView; import com.linkedin.venice.views.VeniceView; @@ -123,6 +126,8 @@ public void testReducerUpdateWithTooLargeValueAndChunkingDisabled() { private VeniceProperties getTestProps() { Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + props.put(PASS_THROUGH_CONFIG_PREFIXES_LIST_KEY, PUBSUB_CLIENT_CONFIG_PREFIX); props.put(MAP_REDUCE_JOB_ID_PROP, "job_200707121733_0003"); props.put(VALUE_SCHEMA_ID_PROP, 1); props.put(VENICE_PUSH_DESTINATION_PUBSUB_BROKER, "localhost:8090"); /* Destination Kafka cluster */ diff --git a/clients/venice-push-job/src/test/java/com/linkedin/venice/heartbeat/TestPushJobHeartbeatSender.java b/clients/venice-push-job/src/test/java/com/linkedin/venice/heartbeat/TestPushJobHeartbeatSender.java index 6a25bb9a758..02f057a3abd 100644 --- a/clients/venice-push-job/src/test/java/com/linkedin/venice/heartbeat/TestPushJobHeartbeatSender.java +++ b/clients/venice-push-job/src/test/java/com/linkedin/venice/heartbeat/TestPushJobHeartbeatSender.java @@ -15,6 +15,7 @@ import com.linkedin.venice.serialization.avro.AvroProtocolDefinition; import com.linkedin.venice.status.protocol.BatchJobHeartbeatKey; import com.linkedin.venice.status.protocol.BatchJobHeartbeatValue; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import java.util.Optional; @@ -28,8 +29,8 @@ public class TestPushJobHeartbeatSender { public void testHeartbeatSenderCreation() { String kafkaUrl = "localhost:1234"; String heartbeatStoreName = AvroProtocolDefinition.BATCH_JOB_HEARTBEAT.getSystemStoreName(); - VeniceProperties properties = VeniceProperties.empty(); - Optional sslProperties = Optional.empty(); + VeniceProperties properties = new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + Optional sslProperties = Optional.of(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); DefaultPushJobHeartbeatSenderFactory pushJobHeartbeatSenderFactory = new DefaultPushJobHeartbeatSenderFactory(); // Prepare controller client. diff --git a/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java b/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java index ff24637a106..722a04dde3e 100644 --- a/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java +++ b/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java @@ -172,7 +172,7 @@ public void testRecordTransformer() throws Exception { ClientConfig.defaultGenericClientConfig(storeName) .setD2Client(d2Client) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME), - VeniceProperties.empty(), + new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), null)) { producer.asyncDelete(getKey(1)).get(); } diff --git a/integrations/venice-samza/src/test/java/com/linkedin/venice/samza/VeniceSystemProducerTest.java b/integrations/venice-samza/src/test/java/com/linkedin/venice/samza/VeniceSystemProducerTest.java index 20c0971d8c9..058c534f7d5 100644 --- a/integrations/venice-samza/src/test/java/com/linkedin/venice/samza/VeniceSystemProducerTest.java +++ b/integrations/venice-samza/src/test/java/com/linkedin/venice/samza/VeniceSystemProducerTest.java @@ -2,6 +2,7 @@ import static com.linkedin.venice.CommonConfigKeys.SSL_ENABLED; import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.VALIDATE_VENICE_INTERNAL_SCHEMA_VERSION; import static com.linkedin.venice.ConfigKeys.VENICE_PARTITIONERS; import static com.linkedin.venice.VeniceConstants.SYSTEM_PROPERTY_FOR_APP_RUNNING_REGION; @@ -35,6 +36,7 @@ import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.meta.Version; import com.linkedin.venice.meta.VersionImpl; +import com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubProducerAdapter; import com.linkedin.venice.pushmonitor.ExecutionStatus; import com.linkedin.venice.pushmonitor.RouterBasedPushMonitor; @@ -48,12 +50,14 @@ import com.linkedin.venice.writer.update.UpdateBuilderImpl; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Properties; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; +import org.apache.samza.config.MapConfig; import org.apache.samza.system.OutgoingMessageEnvelope; import org.apache.samza.system.SystemProducer; import org.apache.samza.system.SystemStream; @@ -72,6 +76,11 @@ public void testPartialUpdateConversion() { .setSamzaJobId("push-job-id-1") .setRunningFabric("dc-0") .setFactory(mock(VeniceSystemFactory.class)) + .setSamzaConfig( + new MapConfig( + Collections.singletonMap( + PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + ApacheKafkaProducerAdapterFactory.class.getName()))) .setVeniceChildD2ZkHost("zookeeper.com:2181") .setPrimaryControllerColoD2ZKHost("zookeeper.com:2181") .setPrimaryControllerD2ServiceName("ChildController") @@ -142,6 +151,11 @@ public void testGetVeniceWriter(Version.PushType pushType) { .setSamzaJobId("push-job-id-1") .setRunningFabric("dc-0") .setFactory(mock(VeniceSystemFactory.class)) + .setSamzaConfig( + new MapConfig( + Collections.singletonMap( + PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + ApacheKafkaProducerAdapterFactory.class.getName()))) .setVeniceChildD2ZkHost("zookeeper.com:2181") .setPrimaryControllerColoD2ZKHost("zookeeper.com:2181") .setPrimaryControllerD2ServiceName("ChildController") @@ -171,6 +185,9 @@ public void testGetVeniceWriter(Version.PushType pushType) { assertNotNull(resultantVeniceWriter); assertEquals(resultantVeniceWriter, veniceWriterMock); assertEquals(capturedProperties.getProperty(KAFKA_BOOTSTRAP_SERVERS), "venice-kafka.db:2023"); + assertEquals( + capturedProperties.getProperty(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS), + ApacheKafkaProducerAdapterFactory.class.getName()); assertEquals(capturedVwo.getTopicName(), "test_store_v1"); if (pushType != Version.PushType.BATCH && pushType != Version.PushType.STREAM_REPROCESSING) { // invoke create venice write without partition count diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java index 6f6d10a44fc..b5086412081 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java @@ -146,7 +146,7 @@ public void testRecordTransformer() throws Exception { ClientConfig.defaultGenericClientConfig(recordTransformerStoreName) .setD2Client(d2Client) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME), - VeniceProperties.empty(), + new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), null)) { producer.asyncDelete(1).get(); @@ -378,7 +378,7 @@ public void testRecordTransformerOnRecovery() throws Exception { ClientConfig.defaultGenericClientConfig(recordTransformerStoreName) .setD2Client(d2Client) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME), - VeniceProperties.empty(), + new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), null)) { int key = numKeys + 1; String value = "a" + key; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OnlineVeniceProducerOrderingIntegrationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OnlineVeniceProducerOrderingIntegrationTest.java index e04eec2717f..79c283cf62c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OnlineVeniceProducerOrderingIntegrationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OnlineVeniceProducerOrderingIntegrationTest.java @@ -186,7 +186,8 @@ private VeniceProducer createConfiguredProducer( int callbackThreadCount, int callbackQueueCapacity, MetricsRepository metricsRepository) { - VeniceProperties producerConfig = new PropertyBuilder().put(CLIENT_PRODUCER_WORKER_COUNT, workerCount) + VeniceProperties producerConfig = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .put(CLIENT_PRODUCER_WORKER_COUNT, workerCount) .put(CLIENT_PRODUCER_WORKER_QUEUE_CAPACITY, workerQueueCapacity) .put(CLIENT_PRODUCER_CALLBACK_THREAD_COUNT, callbackThreadCount) .put(CLIENT_PRODUCER_CALLBACK_QUEUE_CAPACITY, callbackQueueCapacity) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java index ce402b309c1..6c3aba29c07 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java @@ -1020,7 +1020,7 @@ public void testHybridMultipleVersions() throws Exception { try (VeniceProducer veniceOnlineProducer = OnlineProducerFactory.createProducer( ClientConfig.defaultGenericClientConfig(storeName).setVeniceURL(cluster.getRandomRouterURL()), - VeniceProperties.empty(), + new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), null)) { for (int i = keyCount; i < keyCount * 2; i++) { veniceOnlineProducer.asyncPut(i, i * 2).get(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/VersionSpecificDaVinciClientTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/VersionSpecificDaVinciClientTest.java index 97ad6639598..50ba59c70e5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/VersionSpecificDaVinciClientTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/VersionSpecificDaVinciClientTest.java @@ -127,7 +127,7 @@ public void testVersionSpecificDaVinciClient() throws Exception { ClientConfig.defaultGenericClientConfig(storeName) .setD2Client(d2Client) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME), - VeniceProperties.empty(), + new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), null)) { producer.asyncPut(streamingKey1, customValue).get(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java index f0f4beafe92..c51b224a7a8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java @@ -170,6 +170,7 @@ protected Properties getExtraServerProperties() { public void setUp() throws Exception { Utils.thisIsLocalhost(); Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); props.put(SERVER_HTTP2_INBOUND_ENABLED, "true"); props.put(SERVER_QUOTA_ENFORCEMENT_ENABLED, "true"); props.putAll(getExtraServerProperties()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java index 4ab816ca3fe..9a974979777 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java @@ -286,6 +286,7 @@ public static Map getSamzaProducerConfig( samzaConfig.put(VENICE_PARENT_CONTROLLER_D2_SERVICE, PARENT_D2_SERVICE_NAME); samzaConfig.put(DEPLOYMENT_ID, Utils.getUniqueString("venice-push-id")); samzaConfig.put(SSL_ENABLED, "false"); + addPubSubApacheKafkaAdapterFactoryConfigs(samzaConfig); samzaConfig.putAll( PubSubBrokerWrapper.getBrokerDetailsForClients(Collections.singletonList(venice.getPubSubBrokerWrapper()))); return samzaConfig; @@ -306,6 +307,7 @@ private static Map getSamzaProducerConfig( samzaConfig.put(VENICE_PARENT_CONTROLLER_D2_SERVICE, PARENT_D2_SERVICE_NAME); samzaConfig.put(DEPLOYMENT_ID, "DC_" + index + "_" + storeName); samzaConfig.put(SSL_ENABLED, "false"); + samzaConfig.putAll(clusterWrapper.getChildRegions().get(index).getPubSubClientProperties()); return samzaConfig; } @@ -324,9 +326,15 @@ private static Map getSamzaProducerConfigForBatch( samzaConfig.put(DEPLOYMENT_ID, Utils.getUniqueString("venice-push-id")); samzaConfig.put(SSL_ENABLED, "false"); samzaConfig.put(configPrefix + VENICE_AGGREGATE, "true"); + samzaConfig.putAll(clusterWrapper.getChildRegions().get(0).getPubSubClientProperties()); return samzaConfig; } + private static void addPubSubApacheKafkaAdapterFactoryConfigs(Map config) { + TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs() + .forEach((key, value) -> config.put(key.toString(), value.toString())); + } + /** * Create Samza Producer in Single-Region setup with optional configs. */ @@ -602,6 +610,7 @@ public static VeniceWriterFactory getVeniceWriterFactory( PubSubBrokerWrapper pubSubBrokerWrapper, PubSubProducerAdapterFactory pubSubProducerAdapterFactory) { Properties veniceWriterProperties = new Properties(); + veniceWriterProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); veniceWriterProperties.put(KAFKA_BOOTSTRAP_SERVERS, pubSubBrokerWrapper.getAddress()); veniceWriterProperties .putAll(PubSubBrokerWrapper.getBrokerDetailsForClients(Collections.singletonList(pubSubBrokerWrapper))); diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestWriteUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestWriteUtils.java index 61571b7ba9f..aea4e862715 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestWriteUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestWriteUtils.java @@ -1,6 +1,8 @@ package com.linkedin.venice.utils; import static com.linkedin.venice.ConfigKeys.MULTI_REGION; +import static com.linkedin.venice.ConfigKeys.PASS_THROUGH_CONFIG_PREFIXES_LIST_KEY; +import static com.linkedin.venice.ConfigKeys.PUBSUB_CLIENT_CONFIG_PREFIX; import static com.linkedin.venice.vpj.VenicePushJobConstants.CONTROLLER_REQUEST_RETRY_ATTEMPTS; import static com.linkedin.venice.vpj.VenicePushJobConstants.D2_ZK_HOSTS_PREFIX; import static com.linkedin.venice.vpj.VenicePushJobConstants.DEFAULT_KEY_FIELD_PROP; @@ -987,6 +989,8 @@ public static Properties defaultVPJProps( } private static Properties defaultVPJPropsInternal(Properties props, String inputDirPath, String storeName) { + TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs().forEach(props::putIfAbsent); + props.putIfAbsent(PASS_THROUGH_CONFIG_PREFIXES_LIST_KEY, PUBSUB_CLIENT_CONFIG_PREFIX); props.put(VENICE_STORE_NAME_PROP, storeName); props.put(INPUT_PATH_PROP, inputDirPath); // No need for a big close timeout in tests. This is just to speed up discovery of certain regressions. From 6aa83bda2108afc5a46fd5cdfb40f72f4345f29a Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 17:13:51 -0700 Subject: [PATCH 14/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../venice/controller/AdminToolE2ETest.java | 9 +++++++ .../TestAdminToolClusterConfig.java | 11 ++++++++- .../TestAdminToolDataOperations.java | 11 ++++++++- .../controller/TestAdminToolEndToEnd.java | 11 ++++++++- .../venice/controller/TestFabricBuildout.java | 9 +++++++ .../ActiveActiveReplicationForHybridTest.java | 3 +++ .../DaVinciClientP2PBlobTransferTest.java | 2 ++ ...inciClientRecordTransformerFilterTest.java | 1 + .../endToEnd/DaVinciClusterAgnosticTest.java | 11 ++++++++- .../DaVinciP2PBlobTransferRecoveryTest.java | 2 ++ ...inciP2PBlobTransferReportDisabledTest.java | 1 + .../venice/endToEnd/PushStatusStoreTest.java | 1 + .../endToEnd/StoreMetadataRecoveryTest.java | 9 +++++++ ...ActiveActiveReplicationWithDownRegion.java | 4 ++++ ...TestAdminOperationWithPreviousVersion.java | 9 +++++++ .../venice/endToEnd/TestBatchForRocksDB.java | 2 ++ .../endToEnd/TestDeferredVersionSwapDvc.java | 9 +++++++ .../endToEnd/TestDumpIngestionContext.java | 16 +++++++++++++ .../TestPushJobWithNativeReplication.java | 2 ++ .../venice/endToEnd/TestStoreMigration.java | 9 +++++++ .../TestStoreMigrationMultiRegion.java | 9 +++++++ .../endToEnd/TestVTConsistencyCheckerJob.java | 4 ++++ .../input/kafka/TestKafkaInputFormat.java | 3 +++ .../com/linkedin/venice/utils/TestUtils.java | 24 +++++++++++++++++++ 24 files changed, 168 insertions(+), 4 deletions(-) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java index a79413fd1ba..005e5cd42ec 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java @@ -53,9 +53,11 @@ public class AdminToolE2ETest { private List childDatacenters; private VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass public void setUp() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); // Disable auto materialization here as we need to test the back-fill command. Properties parentControllerProperties = new Properties(); parentControllerProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, "false"); @@ -80,6 +82,13 @@ public void setUp() { @AfterClass(alwaysRun = true) public void cleanUp() { multiRegionMultiClusterWrapper.close(); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = TEST_TIMEOUT) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolClusterConfig.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolClusterConfig.java index 74a7fec3463..6a4803b112b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolClusterConfig.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolClusterConfig.java @@ -37,9 +37,11 @@ public class TestAdminToolClusterConfig { String clusterName; VeniceClusterWrapper venice; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass public void setUp() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Properties properties = new Properties(); String regionName = "dc-0"; properties.setProperty(LOCAL_REGION_NAME, regionName); @@ -67,9 +69,16 @@ public void setUp() { clusterName = venice.getClusterName(); } - @AfterClass + @AfterClass(alwaysRun = true) public void cleanUp() { venice.close(); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = TEST_TIMEOUT) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolDataOperations.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolDataOperations.java index 842eb34db9b..fb89ab5f22d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolDataOperations.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolDataOperations.java @@ -48,9 +48,11 @@ public class TestAdminToolDataOperations { String clusterName; VeniceClusterWrapper venice; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass public void setUp() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Properties properties = new Properties(); String regionName = "dc-0"; properties.setProperty(LOCAL_REGION_NAME, regionName); @@ -78,9 +80,16 @@ public void setUp() { clusterName = venice.getClusterName(); } - @AfterClass + @AfterClass(alwaysRun = true) public void cleanUp() { venice.close(); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = TEST_TIMEOUT * 4) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java index a7eb3c6d04b..ae8b1ef2c92 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java @@ -41,9 +41,11 @@ public class TestAdminToolEndToEnd { String clusterName; VeniceClusterWrapper venice; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass public void setUp() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Properties properties = new Properties(); String regionName = "dc-0"; properties.setProperty(LOCAL_REGION_NAME, regionName); @@ -71,9 +73,16 @@ public void setUp() { clusterName = venice.getClusterName(); } - @AfterClass + @AfterClass(alwaysRun = true) public void cleanUp() { venice.close(); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = TEST_TIMEOUT) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java index e36468fcc7a..085caa448d9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java @@ -49,9 +49,11 @@ public class TestFabricBuildout { private List childDatacenters; private List parentControllers; private VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass public void setUp() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Properties childControllerProperties = new Properties(); childControllerProperties.setProperty(ALLOW_CLUSTER_WIPE, "true"); Properties serverProperties = new Properties(); @@ -79,6 +81,13 @@ public void setUp() { @AfterClass(alwaysRun = true) public void cleanUp() { multiRegionMultiClusterWrapper.close(); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = TEST_TIMEOUT) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java index e9263aa2f4c..5aed7fdab05 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java @@ -70,6 +70,7 @@ import org.apache.helix.manager.zk.ZKHelixAdmin; import org.apache.helix.model.IdealState; import org.apache.http.HttpStatus; +import org.apache.samza.config.MapConfig; import org.apache.samza.system.OutgoingMessageEnvelope; import org.apache.samza.system.SystemStream; import org.testng.annotations.AfterClass; @@ -469,6 +470,7 @@ public void testAAReplicationCanResolveConflicts(boolean useLogicalTimestamp, bo VeniceMultiClusterWrapper childDataCenter = childDatacenters.get(0); try (VeniceSystemProducer producerInDC0 = new VeniceSystemProducer( new VeniceSystemProducerConfig.Builder().setFactory(new VeniceSystemFactory()) + .setSamzaConfig(new MapConfig(childDataCenter.getPubSubClientProperties())) .setStoreName(storeName) .setPushType(Version.PushType.STREAM) .setSamzaJobId(Utils.getUniqueString("venice-push-id")) @@ -563,6 +565,7 @@ public void testAAReplicationCanResolveConflicts(boolean useLogicalTimestamp, bo VeniceMultiClusterWrapper childDataCenter1 = childDatacenters.get(1); try (VeniceSystemProducer producerInDC1 = new VeniceSystemProducer( new VeniceSystemProducerConfig.Builder().setFactory(new VeniceSystemFactory()) + .setSamzaConfig(new MapConfig(childDataCenter1.getPubSubClientProperties())) .setStoreName(storeName) .setPushType(Version.PushType.STREAM) .setSamzaJobId(Utils.getUniqueString("venice-push-id")) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientP2PBlobTransferTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientP2PBlobTransferTest.java index 704eae29fd4..5ff02547b6f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientP2PBlobTransferTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientP2PBlobTransferTest.java @@ -143,6 +143,7 @@ public void testBlobP2PTransferAmongDVC(boolean batchPushReportEnable, Boolean i File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); @@ -291,6 +292,7 @@ public void testBlobP2PTransferForNonLaggingDaVinciClient() throws Exception { File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerFilterTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerFilterTest.java index e6d37c25922..b8752015de5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerFilterTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerFilterTest.java @@ -463,6 +463,7 @@ private List getDataMessages(String storeName, int keyCoun // Consume all the RT messages and validated how many data records were produced. PubSubBrokerWrapper pubSubBrokerWrapper = cluster.getPubSubBrokerWrapper(); Properties properties = new Properties(); + properties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); properties.setProperty(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, pubSubBrokerWrapper.getAddress()); List messages = new ArrayList<>(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java index 278f5491f69..ed2dc5bf7fb 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java @@ -75,12 +75,14 @@ public class DaVinciClusterAgnosticTest { private VeniceMultiClusterWrapper multiClusterVenice; private String[] clusterNames; private String parentControllerURLs; + private Properties originalPubSubAdapterFactorySystemProperties; /** * Set up a multi-cluster Venice environment with meta system store enabled Venice stores. */ @BeforeClass public void setUp() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Utils.thisIsLocalhost(); Properties parentControllerProps = new Properties(); parentControllerProps.put(OFFLINE_JOB_START_TIMEOUT_MS, "180000"); @@ -109,9 +111,16 @@ public void setUp() { IntegrationTestUtils.waitForParticipantStorePush(clusterNames, multiClusterVenice.getControllerConnectString()); } - @AfterClass + @AfterClass(alwaysRun = true) public void cleanUp() { Utils.closeQuietlyWithErrorLogged(multiRegionMultiClusterWrapper); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = 180 * Time.MS_PER_SECOND) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferRecoveryTest.java index c21ffeea9eb..e5af4afd9f8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferRecoveryTest.java @@ -120,6 +120,7 @@ public void testBlobP2PTransferAmongDVCWithServerShutdown(boolean isGracefulShut File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); @@ -265,6 +266,7 @@ public void testBlobP2PinDVCWithRestoreTempFolderSuccessfully() throws Exception File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferReportDisabledTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferReportDisabledTest.java index e94dd565dc7..8013d2c886c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferReportDisabledTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferReportDisabledTest.java @@ -109,6 +109,7 @@ public void testBlobP2PTransferAmongDVC() throws Exception { File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java index d61625eb257..7067bc13b81 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java @@ -94,6 +94,7 @@ public class PushStatusStoreTest { @BeforeClass public void setUp() { Properties extraProperties = new Properties(); + extraProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); // all tests in this class will be reading incremental push status from push status store extraProperties.setProperty(USE_PUSH_STATUS_STORE_FOR_INCREMENTAL_PUSH, String.valueOf(true)); extraProperties.setProperty( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java index 6f641c6b06f..d367f0456c5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java @@ -45,9 +45,11 @@ public class StoreMetadataRecoveryTest { private String parentZKUrl; private String parentKafkaUrl; private String childControllerUrl; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass public void setUp() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Utils.thisIsLocalhost(); Properties parentControllerProperties = new Properties(); // Disable topic cleanup since parent and child are sharing the same kafka cluster. @@ -88,6 +90,13 @@ public void setUp() { @AfterClass(alwaysRun = true) public void cleanUp() { Utils.closeQuietlyWithErrorLogged(twoLayerClusterWrapper); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = TEST_TIMEOUT) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java index 86d18947132..804ddb75907 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java @@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.samza.config.MapConfig; import org.apache.samza.system.OutgoingMessageEnvelope; import org.apache.samza.system.SystemProducer; import org.apache.samza.system.SystemStream; @@ -111,6 +112,7 @@ public void testDownedKafka() throws Exception { // Build a system producer that writes nearline to dc-0 SystemProducer producerInDC0 = new VeniceSystemProducer( new VeniceSystemProducerConfig.Builder().setFactory(new VeniceSystemFactory()) + .setSamzaConfig(new MapConfig(childDatacenters.get(0).getPubSubClientProperties())) .setStoreName(storeName) .setPushType(Version.PushType.STREAM) .setSamzaJobId(Utils.getUniqueString("venice-push-id")) @@ -124,6 +126,7 @@ public void testDownedKafka() throws Exception { SystemProducer producerInDC1 = new VeniceSystemProducer( new VeniceSystemProducerConfig.Builder().setFactory(new VeniceSystemFactory()) + .setSamzaConfig(new MapConfig(childDatacenters.get(1).getPubSubClientProperties())) .setStoreName(storeName) .setPushType(Version.PushType.STREAM) .setSamzaJobId(Utils.getUniqueString("venice-push-id")) @@ -138,6 +141,7 @@ public void testDownedKafka() throws Exception { // Build another one which will write some batch data SystemProducer batchProducer = new VeniceSystemProducer( new VeniceSystemProducerConfig.Builder().setFactory(new VeniceSystemFactory()) + .setSamzaConfig(new MapConfig(childDatacenters.get(0).getPubSubClientProperties())) .setStoreName(storeName) .setPushType(Version.PushType.BATCH) .setSamzaJobId(Utils.getUniqueString("venice-push-id")) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestAdminOperationWithPreviousVersion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestAdminOperationWithPreviousVersion.java index ee96be2ccc0..5c8bef070b4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestAdminOperationWithPreviousVersion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestAdminOperationWithPreviousVersion.java @@ -216,9 +216,11 @@ public class TestAdminOperationWithPreviousVersion { private Admin veniceAdmin; private List childControllerClients; private VeniceMultiClusterWrapper multiClusterWrapperRegion0; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass(alwaysRun = true) public void setUp() throws Exception { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Utils.thisIsLocalhost(); // Validate that all operations have test coverage BEFORE running any tests @@ -294,6 +296,13 @@ void beforeEachTest() { @AfterClass(alwaysRun = true) public void cleanUp() { multiRegionMultiClusterWrapper.close(); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } /** diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java index 7457acc434f..8fe773f2222 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java @@ -15,6 +15,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; +import com.linkedin.venice.utils.TestUtils; import java.util.Properties; import org.testng.annotations.Test; @@ -31,6 +32,7 @@ public VeniceClusterWrapper initializeVeniceCluster() { VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options); Properties serverProperties = new Properties(); + serverProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); serverProperties.put(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB); serverProperties.setProperty(ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED, "false"); serverProperties.setProperty(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, "true"); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwapDvc.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwapDvc.java index a2f3853051e..e96f6d6b3d6 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwapDvc.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwapDvc.java @@ -75,9 +75,11 @@ public class TestDeferredVersionSwapDvc { IntStream.range(0, NUMBER_OF_CLUSTERS).mapToObj(i -> "venice-cluster" + i).toArray(String[]::new); private static final int TEST_TIMEOUT = 120_000; private List childDatacenters; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass public void setUp() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Properties controllerProps = new Properties(); controllerProps.put(CONTROLLER_DEFERRED_VERSION_SWAP_SLEEP_MS, 100); controllerProps.put(CONTROLLER_DEFERRED_VERSION_SWAP_SERVICE_ENABLED, true); @@ -103,6 +105,13 @@ public void setUp() { @AfterClass(alwaysRun = true) public void cleanUp() { Utils.closeQuietlyWithErrorLogged(multiRegionMultiClusterWrapper); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = TEST_TIMEOUT * 2) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java index b9a9c63b48b..fce2d10ddc5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java @@ -21,17 +21,33 @@ import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.Map; +import java.util.Properties; import java.util.concurrent.TimeUnit; import org.apache.avro.Schema; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; public class TestDumpIngestionContext extends AbstractMultiRegionTest { private static final Logger LOGGER = LogManager.getLogger(TestDumpIngestionContext.class); private static final int TEST_TIMEOUT_MS = 180_000; + private Properties originalPubSubAdapterFactorySystemProperties; + + @BeforeClass(alwaysRun = true) + public void setUpAdminToolSystemProperties() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + } + + @AfterClass(alwaysRun = true) + public void restoreAdminToolSystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } + } @Test(timeOut = TEST_TIMEOUT_MS) public void testDumpHostHeartbeatLag() { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java index 8e1b7c8258e..ffc282b3fba 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java @@ -104,6 +104,7 @@ protected boolean shouldCreateD2Client() { @Override protected Properties getExtraServerProperties() { Properties serverProperties = new Properties(); + serverProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); serverProperties.setProperty(SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE, "300"); return serverProperties; } @@ -111,6 +112,7 @@ protected Properties getExtraServerProperties() { @Override protected Properties getExtraControllerProperties() { Properties controllerProps = new Properties(); + controllerProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); // This property is required for test stores that have 10 partitions controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 10); controllerProps.put(BatchJobHeartbeatConfigs.HEARTBEAT_STORE_CLUSTER_CONFIG.getConfigName(), SYSTEM_STORE_CLUSTER); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java index 0993e2c499f..5679a6f8fc2 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java @@ -129,9 +129,11 @@ public class TestStoreMigration { private String parentControllerUrl; private String childControllerUrl0; protected Client r2Client; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass public void setUp() throws Exception { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Utils.thisIsLocalhost(); Properties parentControllerProperties = new Properties(); // Disable topic cleanup since parent and child are sharing the same kafka cluster. @@ -179,6 +181,13 @@ public void setUp() throws Exception { @AfterClass(alwaysRun = true) public void cleanUp() { Utils.closeQuietlyWithErrorLogged(twoLayerMultiRegionMultiClusterWrapper); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = TEST_TIMEOUT) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigrationMultiRegion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigrationMultiRegion.java index 6e2e65a397d..86f9ca67874 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigrationMultiRegion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigrationMultiRegion.java @@ -66,9 +66,11 @@ public class TestStoreMigrationMultiRegion { private String parentControllerUrl; private String childControllerUrl0; private String childControllerUrl1; + private Properties originalPubSubAdapterFactorySystemProperties; @BeforeClass(timeOut = 180_000) public void setUp() { + originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); Utils.thisIsLocalhost(); Properties controllerProperties = new Properties(); controllerProperties.setProperty(TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS, String.valueOf(4000)); @@ -111,6 +113,13 @@ public void setUp() { @AfterClass(alwaysRun = true) public void cleanUp() { Utils.closeQuietlyWithErrorLogged(twoLayerMultiRegionMultiClusterWrapper); + restorePubSubAdapterFactorySystemProperties(); + } + + private void restorePubSubAdapterFactorySystemProperties() { + if (originalPubSubAdapterFactorySystemProperties != null) { + TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + } } @Test(timeOut = TEST_TIMEOUT) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java index fd60fad2463..c3703318107 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java @@ -48,6 +48,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.avro.Schema; +import org.apache.samza.config.MapConfig; import org.apache.samza.system.OutgoingMessageEnvelope; import org.apache.samza.system.SystemStream; import org.apache.spark.sql.Dataset; @@ -143,6 +144,7 @@ public void testFullPipelineWithBatchPushRTWritesAndInjectedInconsistency() thro // 2. RT writes from both DCs via Samza VeniceSystemProducer producerInDC0 = new VeniceSystemProducer( new VeniceSystemProducerConfig.Builder().setFactory(new VeniceSystemFactory()) + .setSamzaConfig(new MapConfig(childDatacenters.get(0).getPubSubClientProperties())) .setStoreName(storeName) .setPushType(Version.PushType.STREAM) .setSamzaJobId(Utils.getUniqueString("venice-push-id")) @@ -162,6 +164,7 @@ public void testFullPipelineWithBatchPushRTWritesAndInjectedInconsistency() thro VeniceSystemProducer producerInDC1 = new VeniceSystemProducer( new VeniceSystemProducerConfig.Builder().setFactory(new VeniceSystemFactory()) + .setSamzaConfig(new MapConfig(childDatacenters.get(1).getPubSubClientProperties())) .setStoreName(storeName) .setPushType(Version.PushType.STREAM) .setSamzaJobId(Utils.getUniqueString("venice-push-id")) @@ -205,6 +208,7 @@ public void testFullPipelineWithBatchPushRTWritesAndInjectedInconsistency() thro // 5. Send more RT writes after injection to advance HW and make the scenario more realistic VeniceSystemProducer postInjectionProducer = new VeniceSystemProducer( new VeniceSystemProducerConfig.Builder().setFactory(new VeniceSystemFactory()) + .setSamzaConfig(new MapConfig(childDatacenters.get(0).getPubSubClientProperties())) .setStoreName(storeName) .setPushType(Version.PushType.STREAM) .setSamzaJobId(Utils.getUniqueString("venice-push-id")) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/input/kafka/TestKafkaInputFormat.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/input/kafka/TestKafkaInputFormat.java index 75740f63074..a9f0737d7af 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/input/kafka/TestKafkaInputFormat.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/input/kafka/TestKafkaInputFormat.java @@ -17,6 +17,7 @@ import com.linkedin.venice.pubsub.api.PubSubTopicPartition; import com.linkedin.venice.pubsub.manager.TopicManager; import com.linkedin.venice.utils.IntegrationTestPushUtils; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.writer.VeniceWriter; @@ -136,6 +137,8 @@ public void testGetSplits() { KafkaInputFormat kafkaInputFormat = new KafkaInputFormat(); PubSubTopic topic = getTopic(1000, 3); JobConf conf = new JobConf(); + TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs() + .forEach((key, value) -> conf.set(key.toString(), value.toString())); conf.set(VENICE_REPUSH_SOURCE_PUBSUB_BROKER, pubSubBrokerWrapper.getAddress()); conf.set(KAFKA_INPUT_TOPIC, topic.getName()); diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java index a4e332bfcca..a27c885e597 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java @@ -765,6 +765,30 @@ public static Properties getPubSubApacheKafkaAdapterFactoryConfigs() { return properties; } + public static Properties setPubSubApacheKafkaAdapterFactorySystemProperties() { + Properties originalProperties = new Properties(); + Properties factoryConfigs = getPubSubApacheKafkaAdapterFactoryConfigs(); + for (String key: factoryConfigs.stringPropertyNames()) { + String originalValue = System.getProperty(key); + if (originalValue != null) { + originalProperties.setProperty(key, originalValue); + } + System.setProperty(key, factoryConfigs.getProperty(key)); + } + return originalProperties; + } + + public static void restorePubSubApacheKafkaAdapterFactorySystemProperties(Properties originalProperties) { + for (String key: getPubSubApacheKafkaAdapterFactoryConfigs().stringPropertyNames()) { + if (originalProperties.containsKey(key)) { + System.setProperty(key, originalProperties.getProperty(key)); + } else { + System.clearProperty(key); + } + } + originalProperties.clear(); + } + public static Properties getPropertiesForControllerConfig() { Properties properties = new Properties(); properties.putAll(getPubSubApacheKafkaAdapterFactoryConfigs()); From d92bc3352f7ec6f476cb34f4fdf30bbaa46a6dca Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 17:42:10 -0700 Subject: [PATCH 15/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../linkedin/venice/consumer/ChangelogConsumerTestUtils.java | 2 +- .../java/com/linkedin/venice/endToEnd/TestBatch.java | 1 + .../linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java index 328f0d3a77d..8eeb5c2c1bc 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java @@ -13,7 +13,6 @@ import com.linkedin.venice.client.store.AvroSpecificStoreClient; import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; -import com.linkedin.venice.common.VeniceSystemStoreType; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; @@ -90,6 +89,7 @@ private static Properties buildConsumerProperties( String zkAddress) { Properties consumerProperties = new Properties(); consumerProperties.putAll(pubSubClientProperties); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, kafkaBootstrapServers); consumerProperties.put(CLUSTER_NAME, clusterName); consumerProperties.put(ZOOKEEPER_ADDRESS, zkAddress); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java index 9cda85d71b7..20c1b34831d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java @@ -455,6 +455,7 @@ public void testNewPushWithNewerDictionaryIsServedCorrectly() throws Exception { // Verify that v1 and v2 have different dictionaries (different data produces different dictionaries) Properties props = new Properties(); props.setProperty(KAFKA_BOOTSTRAP_SERVERS, veniceCluster.getPubSubBrokerWrapper().getAddress()); + props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); VeniceProperties veniceProperties = new VeniceProperties(props); ByteBuffer v1Dict = DictionaryUtils.readDictionaryFromKafka(Version.composeKafkaTopic(storeName, 1), veniceProperties); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java index c3703318107..47eca87a9a8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java @@ -239,6 +239,7 @@ public void testFullPipelineWithBatchPushRTWritesAndInjectedInconsistency() thro File outputDir = new File(tempRoot, "output"); try { Properties jobProps = new Properties(); + jobProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); jobProps.setProperty( VTConsistencyCheckerJob.DC0_BROKER_URL, childDatacenters.get(0).getPubSubBrokerWrapper().getAddress()); From 51a9a68831614a14767a7d1e44c7701c24b83674 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 17:56:54 -0700 Subject: [PATCH 16/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java index 8eeb5c2c1bc..9f636727a9f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java @@ -13,6 +13,7 @@ import com.linkedin.venice.client.store.AvroSpecificStoreClient; import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; +import com.linkedin.venice.common.VeniceSystemStoreType; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; From 7c955714978e02678c56942c4d2150ad35a496d7 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 18:23:27 -0700 Subject: [PATCH 17/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../consumer/VeniceChangelogConsumerImpl.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImpl.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImpl.java index 1d64062081c..89325412452 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImpl.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImpl.java @@ -5,6 +5,10 @@ import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.ZOOKEEPER_ADDRESS; import static com.linkedin.venice.kafka.protocol.enums.ControlMessageType.START_OF_SEGMENT; import static com.linkedin.venice.stats.dimensions.VeniceResponseStatusCategory.FAIL; @@ -235,6 +239,17 @@ public VeniceChangelogConsumerImpl( .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, changelogClientConfig.getRocksDBBlockCacheSizeInBytes()); rocksDBBufferProperties .put(KAFKA_BOOTSTRAP_SERVERS, changelogClientConfig.getConsumerProperties().get(KAFKA_BOOTSTRAP_SERVERS)); + // VeniceServerConfig below eagerly builds a PubSubClientsFactory, which fails fast when the adapter + // factory classes are not configured. Forward them from the consumer properties even though the local + // RocksDB buffer does not itself use a pub-sub client. + for (String factoryClassConfigKey: new String[] { PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, + PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS }) { + Object factoryClassName = changelogClientConfig.getConsumerProperties().get(factoryClassConfigKey); + if (factoryClassName != null) { + rocksDBBufferProperties.put(factoryClassConfigKey, factoryClassName); + } + } VeniceProperties rocksDBBufferVeniceProperties = new VeniceProperties(rocksDBBufferProperties); VeniceServerConfig serverConfig = new VeniceServerConfig(rocksDBBufferVeniceProperties); rocksDBStorageEngineFactory = new RocksDBStorageEngineFactory(serverConfig); From 6166c85f6bf2d7c3d653d3c9249b9e9cf64bca01 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 18:49:00 -0700 Subject: [PATCH 18/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../java/com/linkedin/venice/endToEnd/TestRepushCore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRepushCore.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRepushCore.java index 6d5bd413644..06fec60ddef 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRepushCore.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRepushCore.java @@ -307,6 +307,7 @@ public void testRepushWithDeleteRecord(boolean useSpark) { .put(ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED, "false") .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DAVINCI_PUSH_STATUS_CHECK_INTERVAL_IN_MS, 1000) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); MetricsRepository metricsRepository = new VeniceMetricsRepository(); From 08b99a270e43f1165315462aef24f6142c281296 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 19:07:44 -0700 Subject: [PATCH 19/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../consumer/VeniceChangelogConsumerImpl.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImpl.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImpl.java index 89325412452..56e07a446a4 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImpl.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerImpl.java @@ -5,10 +5,6 @@ import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; -import static com.linkedin.venice.ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.ConfigKeys.ZOOKEEPER_ADDRESS; import static com.linkedin.venice.kafka.protocol.enums.ControlMessageType.START_OF_SEGMENT; import static com.linkedin.venice.stats.dimensions.VeniceResponseStatusCategory.FAIL; @@ -224,6 +220,10 @@ public VeniceChangelogConsumerImpl( this.storeName = VeniceView.getViewStoreName(changelogClientConfig.getStoreName(), changelogClientConfig.getViewName()); Properties rocksDBBufferProperties = new Properties(); + // VeniceServerConfig below eagerly builds a PubSubClientsFactory, which fails fast when the pub-sub + // adapter factory classes are not configured. Seed the buffer config with the consumer properties (which + // carry those classes) so it resolves them; the RocksDB buffer itself does not use a pub-sub client. + rocksDBBufferProperties.putAll(changelogClientConfig.getConsumerProperties()); String rocksDBBufferPath = changelogClientConfig.getBootstrapFileSystemPath(); if (rocksDBBufferPath == null || rocksDBBufferPath.isEmpty()) { throw new VeniceException("bootstrapFileSystemPath must be configured for consuming view store: " + storeName); @@ -239,17 +239,6 @@ public VeniceChangelogConsumerImpl( .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, changelogClientConfig.getRocksDBBlockCacheSizeInBytes()); rocksDBBufferProperties .put(KAFKA_BOOTSTRAP_SERVERS, changelogClientConfig.getConsumerProperties().get(KAFKA_BOOTSTRAP_SERVERS)); - // VeniceServerConfig below eagerly builds a PubSubClientsFactory, which fails fast when the adapter - // factory classes are not configured. Forward them from the consumer properties even though the local - // RocksDB buffer does not itself use a pub-sub client. - for (String factoryClassConfigKey: new String[] { PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, - PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, - PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS }) { - Object factoryClassName = changelogClientConfig.getConsumerProperties().get(factoryClassConfigKey); - if (factoryClassName != null) { - rocksDBBufferProperties.put(factoryClassConfigKey, factoryClassName); - } - } VeniceProperties rocksDBBufferVeniceProperties = new VeniceProperties(rocksDBBufferProperties); VeniceServerConfig serverConfig = new VeniceServerConfig(rocksDBBufferVeniceProperties); rocksDBStorageEngineFactory = new RocksDBStorageEngineFactory(serverConfig); From 87ac7c59ef2f8404052077f8abf7c49ec3567408 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 19:33:48 -0700 Subject: [PATCH 20/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../consumer/ChangelogClientConfigTest.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/ChangelogClientConfigTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/ChangelogClientConfigTest.java index fb7dd603a35..6bc1e7cea8a 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/ChangelogClientConfigTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/consumer/ChangelogClientConfigTest.java @@ -1,8 +1,10 @@ package com.linkedin.davinci.consumer; +import com.linkedin.venice.utils.TestUtils; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Properties; import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; @@ -33,6 +35,13 @@ public class ChangelogClientConfigTest { private static final String GLOBAL_STORE = "global_store"; + @SuppressWarnings("rawtypes") + private static ChangelogClientConfig newGlobalConfig() { + Properties consumerProperties = new Properties(); + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + return new ChangelogClientConfig(GLOBAL_STORE).setConsumerProperties(consumerProperties); + } + /** * Deterministic (single-threaded) regression test. * @@ -47,7 +56,7 @@ public class ChangelogClientConfigTest { @Test @SuppressWarnings("rawtypes") public void testCloneConfigDoesNotShareInnerClientConfig() { - ChangelogClientConfig global = new ChangelogClientConfig(GLOBAL_STORE); + ChangelogClientConfig global = newGlobalConfig(); ChangelogClientConfig clone1 = ChangelogClientConfig.cloneConfig(global).setStoreName("store_A"); ChangelogClientConfig clone2 = ChangelogClientConfig.cloneConfig(global).setStoreName("store_B"); @@ -68,7 +77,7 @@ public void testCloneConfigDoesNotShareInnerClientConfig() { @Test public void testBackgroundReporterThreadSleepIntervalCloned() { - ChangelogClientConfig global = new ChangelogClientConfig(GLOBAL_STORE); + ChangelogClientConfig global = newGlobalConfig(); // Default should be 60 Assert.assertEquals(global.getBackgroundReporterThreadSleepIntervalInSeconds(), 60L); @@ -109,7 +118,7 @@ public void testBackgroundReporterThreadSleepIntervalCloned() { public void testCloneConfigIsThreadSafe() throws InterruptedException { final int numThreads = 20; - ChangelogClientConfig global = new ChangelogClientConfig(GLOBAL_STORE); + ChangelogClientConfig global = newGlobalConfig(); ChangelogClientConfig[] clones = new ChangelogClientConfig[numThreads]; String[] expectedNames = new String[numThreads]; From 467cf16e0c6c09337933fa1ed010c8ee32eeaf2a Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 20:04:36 -0700 Subject: [PATCH 21/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../java/com/linkedin/davinci/VersionBackendTest.java | 2 ++ .../src/test/resources/config/server.properties | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/VersionBackendTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/VersionBackendTest.java index 55252da2c01..b4e6a1eba5f 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/VersionBackendTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/VersionBackendTest.java @@ -167,6 +167,7 @@ public void testRecordTransformerSubscribe() { .put(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, "test-kafka") .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) .put(ConfigKeys.LOCAL_REGION_NAME, "dc-0") + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); VeniceConfigLoader veniceConfigLoader = new VeniceConfigLoader(backendConfig); when(mockDaVinciBackend.getConfigLoader()).thenReturn(veniceConfigLoader); @@ -259,6 +260,7 @@ public void testPushStatusDisabledForVersionSpecificClient() { .put(ConfigKeys.DATA_BASE_PATH, baseDataPath.getAbsolutePath()) .put(ConfigKeys.LOCAL_REGION_NAME, "dc-0") .put(ConfigKeys.PUSH_STATUS_STORE_ENABLED, true) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) .build(); VeniceConfigLoader veniceConfigLoader = new VeniceConfigLoader(backendConfig); when(mockDaVinciBackend.getConfigLoader()).thenReturn(veniceConfigLoader); diff --git a/clients/da-vinci-client/src/test/resources/config/server.properties b/clients/da-vinci-client/src/test/resources/config/server.properties index a2ed1f4a402..8ce5388f357 100644 --- a/clients/da-vinci-client/src/test/resources/config/server.properties +++ b/clients/da-vinci-client/src/test/resources/config/server.properties @@ -1,4 +1,10 @@ node.id=0 listener.port=7072 admin.port=7073 -kafka.threads.per.partition=1 \ No newline at end of file +kafka.threads.per.partition=1 +# Pub-sub adapter factory classes (Apache Kafka). Required because the factory fails fast when the +# class is unset and pubsub.adapter.factory.kafka.fallback.enabled is false (the default). +pubsub.producer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.producer.ApacheKafkaProducerAdapterFactory +pubsub.consumer.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.consumer.ApacheKafkaConsumerAdapterFactory +pubsub.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory +pubsub.source.of.truth.admin.adapter.factory.class=com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory \ No newline at end of file From a721d52e2b5dd6a63bfc468d48ab5b840f5b54c2 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 20:37:26 -0700 Subject: [PATCH 22/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../java/com/linkedin/davinci/DaVinciUserApp.java | 5 +++++ .../linkedin/venice/endToEnd/AbstractMultiRegionTest.java | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java index 08fc76bc2fc..0a6462dc442 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java @@ -30,6 +30,7 @@ import com.linkedin.venice.D2.D2ClientUtils; import com.linkedin.venice.endToEnd.TestStringRecordTransformer; import com.linkedin.venice.integration.utils.DaVinciTestContext; +import com.linkedin.venice.utils.TestUtils; import io.tehuti.metrics.MetricsRepository; import java.io.FileInputStream; import java.io.IOException; @@ -100,6 +101,10 @@ public static void main(String[] args) throws Exception { D2ClientUtils.startClient(d2Client); Map extraBackendConfig = new HashMap<>(); + // This forked DaVinci process must configure the pub-sub adapter factory classes explicitly; the + // factory fails fast otherwise (there is no implicit default). + TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs() + .forEach((key, value) -> extraBackendConfig.put(key.toString(), value)); extraBackendConfig.put(DATA_BASE_PATH, baseDataPath); extraBackendConfig.put(PUSH_STATUS_STORE_ENABLED, true); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/AbstractMultiRegionTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/AbstractMultiRegionTest.java index 32ce6c9ba2b..a697623f06d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/AbstractMultiRegionTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/AbstractMultiRegionTest.java @@ -10,6 +10,7 @@ import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; import java.util.Properties; @@ -85,12 +86,15 @@ protected boolean shouldCreateD2Client() { public void setUp() { Properties serverProperties = new Properties(); serverProperties.putAll(getExtraServerProperties()); + serverProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); Properties parentControllerProps = new Properties(); parentControllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, true); parentControllerProps.putAll(getExtraParentControllerProperties()); + parentControllerProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); Properties childControllerProps = new Properties(); childControllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, true); childControllerProps.putAll(getExtraChildControllerProperties()); + childControllerProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(getNumberOfRegions()) .numberOfClusters(getNumberOfClusters()) From 6cbc50c99850a82430eaae2bfa254aa2a85fc95c Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 21:00:51 -0700 Subject: [PATCH 23/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../ChangelogConsumerDaVinciRecordTransformerUserApp.java | 3 +++ .../linkedin/venice/integration/utils/DaVinciTestContext.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerDaVinciRecordTransformerUserApp.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerDaVinciRecordTransformerUserApp.java index 60f942eb963..8660204c393 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerDaVinciRecordTransformerUserApp.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerDaVinciRecordTransformerUserApp.java @@ -91,6 +91,9 @@ public static void main(String[] args) throws InterruptedException, ExecutionExc getVeniceMetricsRepository(CHANGE_DATA_CAPTURE_CLIENT, CONSUMER_METRIC_ENTITIES, true); Properties consumerProperties = new Properties(); + // This forked process must configure the pub-sub adapter factory classes explicitly; the factory + // fails fast otherwise (there is no implicit default). + consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, kafkaUrl); consumerProperties.put(CLUSTER_NAME, clusterName); consumerProperties.put(ZOOKEEPER_ADDRESS, zkUrl); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java index 508a7e2ec3d..0508848600c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java @@ -18,6 +18,7 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.utils.PropertyBuilder; +import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import io.tehuti.metrics.MetricsRepository; @@ -197,6 +198,7 @@ public static PropertyBuilder getDaVinciPropertyBuilder(String zkAddress) { .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(D2_ZK_HOSTS_ADDRESS, zkAddress) .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 4 * 1024 * 1024 * 1024L) - .put(CLUSTER_DISCOVERY_D2_SERVICE, VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME); + .put(CLUSTER_DISCOVERY_D2_SERVICE, VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME) + .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); } } From 8030f2a0c54ba33cb7ddeea6b3c2733dcc5cb241 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Thu, 30 Jul 2026 21:22:52 -0700 Subject: [PATCH 24/25] [common][server][controller][pulsar][test] Fail fast (config-gated) when a pub-sub adapter factory class is not configured --- .../DefaultPushJobHeartbeatSenderFactory.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/clients/venice-push-job/src/main/java/com/linkedin/venice/heartbeat/DefaultPushJobHeartbeatSenderFactory.java b/clients/venice-push-job/src/main/java/com/linkedin/venice/heartbeat/DefaultPushJobHeartbeatSenderFactory.java index ab2ec4d4f29..c79bd1aa432 100644 --- a/clients/venice-push-job/src/main/java/com/linkedin/venice/heartbeat/DefaultPushJobHeartbeatSenderFactory.java +++ b/clients/venice-push-job/src/main/java/com/linkedin/venice/heartbeat/DefaultPushJobHeartbeatSenderFactory.java @@ -1,6 +1,7 @@ package com.linkedin.venice.heartbeat; import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.status.BatchJobHeartbeatConfigs.HEARTBEAT_INITIAL_DELAY_CONFIG; import static com.linkedin.venice.status.BatchJobHeartbeatConfigs.HEARTBEAT_INTERVAL_CONFIG; import static com.linkedin.venice.status.BatchJobHeartbeatConfigs.HEARTBEAT_LAST_HEARTBEAT_IS_DELETE_CONFIG; @@ -61,7 +62,7 @@ public PushJobHeartbeatSender createHeartbeatSender( VeniceWriter veniceWriter = getVeniceWriter( heartbeatKafkaTopicName, partitionerConfig, - getVeniceWriterProperties(sslProperties, kafkaUrl), + getVeniceWriterProperties(sslProperties, kafkaUrl, properties), partitionNum); Schema heartbeatKeySchema = getHeartbeatKeySchema(controllerClient, retryAttempts, heartbeatStoreName); Map valueSchemasById = @@ -91,13 +92,22 @@ public PushJobHeartbeatSender createHeartbeatSender( return defaultPushJobHeartbeatSender; } - private Properties getVeniceWriterProperties(Optional sslProperties, String kafkaBootstrapUrl) { + private Properties getVeniceWriterProperties( + Optional sslProperties, + String kafkaBootstrapUrl, + VeniceProperties jobProperties) { Properties veniceWriterProperties = new Properties(); veniceWriterProperties.put(KAFKA_BOOTSTRAP_SERVERS, kafkaBootstrapUrl); if (sslProperties.isPresent()) { veniceWriterProperties.putAll(sslProperties.get()); } + // Forward the pub-sub producer adapter factory class so the heartbeat VeniceWriterFactory can resolve + // it; the factory fails fast otherwise (there is no implicit default). + if (jobProperties.containsKey(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS)) { + veniceWriterProperties + .put(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, jobProperties.getString(PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS)); + } return veniceWriterProperties; } From 46667683efc2f5f9df9cb9b799a0a8b01d6769f7 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Fri, 31 Jul 2026 01:57:19 -0700 Subject: [PATCH 25/25] Resolve pub-sub adapter factory configs from the configured backend in integration tests Integration tests no longer hard-code the Apache Kafka adapter factory classes. ServiceFactory.getPubSubClientConfigs() (plus set/restore system-property helpers) resolves the producer/consumer/admin/source-of-truth adapter factory class names from the configured PubSubBrokerFactory (pubSubBrokerFactory), so the suite exercises whatever client configs the pub-sub backend under test exposes. venice-test-common main helpers and pure unit tests keep the Apache Kafka default because the pluggable backend lives in the integrationTest source set and no broker is started there. --- ...VinciRecordTransformerIntegrationTest.java | 2 +- .../com/linkedin/davinci/DaVinciUserApp.java | 5 +- ...nsumerDaVinciRecordTransformerUserApp.java | 3 +- .../consumer/ChangelogConsumerTestUtils.java | 3 +- .../AbstractTestVeniceHelixAdmin.java | 1 + .../venice/controller/AdminToolE2ETest.java | 4 +- .../TestAdminToolClusterConfig.java | 4 +- .../TestAdminToolDataOperations.java | 4 +- .../controller/TestAdminToolEndToEnd.java | 4 +- .../venice/controller/TestFabricBuildout.java | 4 +- .../endToEnd/AbstractMultiRegionTest.java | 7 +-- .../DaVinciClientP2PBlobTransferTest.java | 4 +- ...inciClientRecordTransformerFilterTest.java | 3 +- .../DaVinciClientRecordTransformerTest.java | 5 +- .../endToEnd/DaVinciClusterAgnosticTest.java | 4 +- .../DaVinciP2PBlobTransferRecoveryTest.java | 5 +- ...inciP2PBlobTransferReportDisabledTest.java | 3 +- ...VeniceProducerOrderingIntegrationTest.java | 2 +- .../venice/endToEnd/PushStatusStoreTest.java | 2 +- .../endToEnd/StoreMetadataRecoveryTest.java | 4 +- ...TestAdminOperationWithPreviousVersion.java | 4 +- .../linkedin/venice/endToEnd/TestBatch.java | 3 +- .../venice/endToEnd/TestBatchForRocksDB.java | 3 +- .../endToEnd/TestDeferredVersionSwapDvc.java | 4 +- .../endToEnd/TestDumpIngestionContext.java | 5 +- .../linkedin/venice/endToEnd/TestHybrid.java | 2 +- .../TestPushJobWithNativeReplication.java | 4 +- .../venice/endToEnd/TestRepushCore.java | 3 +- .../venice/endToEnd/TestStoreMigration.java | 4 +- .../TestStoreMigrationMultiRegion.java | 4 +- .../endToEnd/TestVTConsistencyCheckerJob.java | 3 +- .../VersionSpecificDaVinciClientTest.java | 2 +- .../utils/AbstractClientEndToEndSetup.java | 2 +- .../input/kafka/TestKafkaInputFormat.java | 4 +- .../integration/utils/DaVinciTestContext.java | 3 +- .../integration/utils/ServiceFactory.java | 63 ++++++++++++++++++- .../utils/IntegrationTestPushUtils.java | 6 +- .../com/linkedin/venice/utils/TestUtils.java | 24 ------- 38 files changed, 129 insertions(+), 87 deletions(-) diff --git a/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java b/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java index 722a04dde3e..141a5151a16 100644 --- a/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java +++ b/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java @@ -172,7 +172,7 @@ public void testRecordTransformer() throws Exception { ClientConfig.defaultGenericClientConfig(storeName) .setD2Client(d2Client) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME), - new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), + new VeniceProperties(ServiceFactory.getPubSubClientConfigs()), null)) { producer.asyncDelete(getKey(1)).get(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java index 0a6462dc442..f76a552e6e8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java @@ -30,7 +30,7 @@ import com.linkedin.venice.D2.D2ClientUtils; import com.linkedin.venice.endToEnd.TestStringRecordTransformer; import com.linkedin.venice.integration.utils.DaVinciTestContext; -import com.linkedin.venice.utils.TestUtils; +import com.linkedin.venice.integration.utils.ServiceFactory; import io.tehuti.metrics.MetricsRepository; import java.io.FileInputStream; import java.io.IOException; @@ -103,8 +103,7 @@ public static void main(String[] args) throws Exception { Map extraBackendConfig = new HashMap<>(); // This forked DaVinci process must configure the pub-sub adapter factory classes explicitly; the // factory fails fast otherwise (there is no implicit default). - TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs() - .forEach((key, value) -> extraBackendConfig.put(key.toString(), value)); + ServiceFactory.getPubSubClientConfigs().forEach((key, value) -> extraBackendConfig.put(key.toString(), value)); extraBackendConfig.put(DATA_BASE_PATH, baseDataPath); extraBackendConfig.put(PUSH_STATUS_STORE_ENABLED, true); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerDaVinciRecordTransformerUserApp.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerDaVinciRecordTransformerUserApp.java index 8660204c393..8c05a600408 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerDaVinciRecordTransformerUserApp.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerDaVinciRecordTransformerUserApp.java @@ -43,6 +43,7 @@ import com.linkedin.venice.D2.D2ClientUtils; import com.linkedin.venice.endToEnd.TestChangelogKey; import com.linkedin.venice.endToEnd.TestChangelogValue; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.pubsub.api.PubSubMessage; import com.linkedin.venice.utils.SslUtils; @@ -93,7 +94,7 @@ public static void main(String[] args) throws InterruptedException, ExecutionExc Properties consumerProperties = new Properties(); // This forked process must configure the pub-sub adapter factory classes explicitly; the factory // fails fast otherwise (there is no implicit default). - consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + consumerProperties.putAll(ServiceFactory.getPubSubClientConfigs()); consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, kafkaUrl); consumerProperties.put(CLUSTER_NAME, clusterName); consumerProperties.put(ZOOKEEPER_ADDRESS, zkUrl); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java index 9f636727a9f..6323c8b9879 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ChangelogConsumerTestUtils.java @@ -17,6 +17,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; @@ -90,7 +91,7 @@ private static Properties buildConsumerProperties( String zkAddress) { Properties consumerProperties = new Properties(); consumerProperties.putAll(pubSubClientProperties); - consumerProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + consumerProperties.putAll(ServiceFactory.getPubSubClientConfigs()); consumerProperties.put(KAFKA_BOOTSTRAP_SERVERS, kafkaBootstrapServers); consumerProperties.put(CLUSTER_NAME, clusterName); consumerProperties.put(ZOOKEEPER_ADDRESS, zkAddress); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AbstractTestVeniceHelixAdmin.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AbstractTestVeniceHelixAdmin.java index d5f0ddd5442..836ac237922 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AbstractTestVeniceHelixAdmin.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AbstractTestVeniceHelixAdmin.java @@ -330,6 +330,7 @@ void stopParticipant(String nodeId) { Properties getControllerProperties(String clusterName) throws IOException { Properties properties = TestUtils.getPropertiesForControllerConfig(); + properties.putAll(ServiceFactory.getPubSubClientConfigs()); properties.put(DEFAULT_OFFLINE_PUSH_STRATEGY, OfflinePushStrategy.WAIT_ALL_REPLICAS.name()); properties.put(DELAY_TO_REBALANCE_MS, 0); properties.put(KAFKA_REPLICATION_FACTOR, 1); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java index 005e5cd42ec..ee343714c29 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java @@ -57,7 +57,7 @@ public class AdminToolE2ETest { @BeforeClass public void setUp() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); // Disable auto materialization here as we need to test the back-fill command. Properties parentControllerProperties = new Properties(); parentControllerProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, "false"); @@ -87,7 +87,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolClusterConfig.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolClusterConfig.java index 6a4803b112b..45f69de5ef6 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolClusterConfig.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolClusterConfig.java @@ -41,7 +41,7 @@ public class TestAdminToolClusterConfig { @BeforeClass public void setUp() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Properties properties = new Properties(); String regionName = "dc-0"; properties.setProperty(LOCAL_REGION_NAME, regionName); @@ -77,7 +77,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolDataOperations.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolDataOperations.java index fb89ab5f22d..c3c472240ef 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolDataOperations.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolDataOperations.java @@ -52,7 +52,7 @@ public class TestAdminToolDataOperations { @BeforeClass public void setUp() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Properties properties = new Properties(); String regionName = "dc-0"; properties.setProperty(LOCAL_REGION_NAME, regionName); @@ -88,7 +88,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java index 1744ff94408..cf01aad93e3 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java @@ -46,7 +46,7 @@ public class TestAdminToolEndToEnd { @BeforeClass public void setUp() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Properties properties = new Properties(); String regionName = "dc-0"; properties.setProperty(LOCAL_REGION_NAME, regionName); @@ -82,7 +82,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java index 085caa448d9..302fe772cba 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java @@ -53,7 +53,7 @@ public class TestFabricBuildout { @BeforeClass public void setUp() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Properties childControllerProperties = new Properties(); childControllerProperties.setProperty(ALLOW_CLUSTER_WIPE, "true"); Properties serverProperties = new Properties(); @@ -86,7 +86,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/AbstractMultiRegionTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/AbstractMultiRegionTest.java index a697623f06d..391d79ef0d2 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/AbstractMultiRegionTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/AbstractMultiRegionTest.java @@ -10,7 +10,6 @@ import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; -import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; import java.util.Properties; @@ -86,15 +85,15 @@ protected boolean shouldCreateD2Client() { public void setUp() { Properties serverProperties = new Properties(); serverProperties.putAll(getExtraServerProperties()); - serverProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + serverProperties.putAll(ServiceFactory.getPubSubClientConfigs()); Properties parentControllerProps = new Properties(); parentControllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, true); parentControllerProps.putAll(getExtraParentControllerProperties()); - parentControllerProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + parentControllerProps.putAll(ServiceFactory.getPubSubClientConfigs()); Properties childControllerProps = new Properties(); childControllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, true); childControllerProps.putAll(getExtraChildControllerProperties()); - childControllerProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + childControllerProps.putAll(ServiceFactory.getPubSubClientConfigs()); VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(getNumberOfRegions()) .numberOfClusters(getNumberOfClusters()) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientP2PBlobTransferTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientP2PBlobTransferTest.java index 5ff02547b6f..53c8b0b9829 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientP2PBlobTransferTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientP2PBlobTransferTest.java @@ -143,7 +143,7 @@ public void testBlobP2PTransferAmongDVC(boolean batchPushReportEnable, Boolean i File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); - props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + props.putAll(ServiceFactory.getPubSubClientConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); @@ -292,7 +292,7 @@ public void testBlobP2PTransferForNonLaggingDaVinciClient() throws Exception { File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); - props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + props.putAll(ServiceFactory.getPubSubClientConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerFilterTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerFilterTest.java index b8752015de5..72a844c7d37 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerFilterTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerFilterTest.java @@ -56,6 +56,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.DaVinciTestContext; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.meta.Version; @@ -463,7 +464,7 @@ private List getDataMessages(String storeName, int keyCoun // Consume all the RT messages and validated how many data records were produced. PubSubBrokerWrapper pubSubBrokerWrapper = cluster.getPubSubBrokerWrapper(); Properties properties = new Properties(); - properties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + properties.putAll(ServiceFactory.getPubSubClientConfigs()); properties.setProperty(ConfigKeys.KAFKA_BOOTSTRAP_SERVERS, pubSubBrokerWrapper.getAddress()); List messages = new ArrayList<>(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java index b5086412081..5947bd39c19 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java @@ -28,6 +28,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.DaVinciTestContext; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.producer.online.OnlineProducerFactory; @@ -146,7 +147,7 @@ public void testRecordTransformer() throws Exception { ClientConfig.defaultGenericClientConfig(recordTransformerStoreName) .setD2Client(d2Client) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME), - new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), + new VeniceProperties(ServiceFactory.getPubSubClientConfigs()), null)) { producer.asyncDelete(1).get(); @@ -378,7 +379,7 @@ public void testRecordTransformerOnRecovery() throws Exception { ClientConfig.defaultGenericClientConfig(recordTransformerStoreName) .setD2Client(d2Client) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME), - new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), + new VeniceProperties(ServiceFactory.getPubSubClientConfigs()), null)) { int key = numKeys + 1; String value = "a" + key; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java index ed2dc5bf7fb..ef9db9831a9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java @@ -82,7 +82,7 @@ public class DaVinciClusterAgnosticTest { */ @BeforeClass public void setUp() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Utils.thisIsLocalhost(); Properties parentControllerProps = new Properties(); parentControllerProps.put(OFFLINE_JOB_START_TIMEOUT_MS, "180000"); @@ -119,7 +119,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferRecoveryTest.java index e5af4afd9f8..a8b53197555 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferRecoveryTest.java @@ -50,6 +50,7 @@ import com.linkedin.venice.compression.CompressionStrategy; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.store.rocksdb.RocksDBUtils; @@ -120,7 +121,7 @@ public void testBlobP2PTransferAmongDVCWithServerShutdown(boolean isGracefulShut File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); - props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + props.putAll(ServiceFactory.getPubSubClientConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); @@ -266,7 +267,7 @@ public void testBlobP2PinDVCWithRestoreTempFolderSuccessfully() throws Exception File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); - props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + props.putAll(ServiceFactory.getPubSubClientConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferReportDisabledTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferReportDisabledTest.java index 8013d2c886c..22ddcede91e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferReportDisabledTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciP2PBlobTransferReportDisabledTest.java @@ -45,6 +45,7 @@ import com.linkedin.venice.compression.CompressionStrategy; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.store.rocksdb.RocksDBUtils; @@ -109,7 +110,7 @@ public void testBlobP2PTransferAmongDVC() throws Exception { File configDir = Utils.getTempDataDirectory(); File configFile = new File(configDir, "dvc-config.properties"); Properties props = new Properties(); - props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + props.putAll(ServiceFactory.getPubSubClientConfigs()); props.setProperty("zk.hosts", zkHosts); props.setProperty("base.data.path", dvcPath1); props.setProperty("store.name", storeName); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OnlineVeniceProducerOrderingIntegrationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OnlineVeniceProducerOrderingIntegrationTest.java index 79c283cf62c..9103b390787 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OnlineVeniceProducerOrderingIntegrationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OnlineVeniceProducerOrderingIntegrationTest.java @@ -186,7 +186,7 @@ private VeniceProducer createConfiguredProducer( int callbackThreadCount, int callbackQueueCapacity, MetricsRepository metricsRepository) { - VeniceProperties producerConfig = new PropertyBuilder().put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + VeniceProperties producerConfig = new PropertyBuilder().put(ServiceFactory.getPubSubClientConfigs()) .put(CLIENT_PRODUCER_WORKER_COUNT, workerCount) .put(CLIENT_PRODUCER_WORKER_QUEUE_CAPACITY, workerQueueCapacity) .put(CLIENT_PRODUCER_CALLBACK_THREAD_COUNT, callbackThreadCount) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java index 7067bc13b81..4b1c23ec9fe 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java @@ -94,7 +94,7 @@ public class PushStatusStoreTest { @BeforeClass public void setUp() { Properties extraProperties = new Properties(); - extraProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + extraProperties.putAll(ServiceFactory.getPubSubClientConfigs()); // all tests in this class will be reading incremental push status from push status store extraProperties.setProperty(USE_PUSH_STATUS_STORE_FOR_INCREMENTAL_PUSH, String.valueOf(true)); extraProperties.setProperty( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java index d367f0456c5..4892cb47277 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java @@ -49,7 +49,7 @@ public class StoreMetadataRecoveryTest { @BeforeClass public void setUp() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Utils.thisIsLocalhost(); Properties parentControllerProperties = new Properties(); // Disable topic cleanup since parent and child are sharing the same kafka cluster. @@ -95,7 +95,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestAdminOperationWithPreviousVersion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestAdminOperationWithPreviousVersion.java index 5c8bef070b4..7669f4702bd 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestAdminOperationWithPreviousVersion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestAdminOperationWithPreviousVersion.java @@ -220,7 +220,7 @@ public class TestAdminOperationWithPreviousVersion { @BeforeClass(alwaysRun = true) public void setUp() throws Exception { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Utils.thisIsLocalhost(); // Validate that all operations have test coverage BEFORE running any tests @@ -301,7 +301,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java index 20c1b34831d..fa397b8939d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java @@ -59,6 +59,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.jobs.StageMetricsSnapshot; @@ -455,7 +456,7 @@ public void testNewPushWithNewerDictionaryIsServedCorrectly() throws Exception { // Verify that v1 and v2 have different dictionaries (different data produces different dictionaries) Properties props = new Properties(); props.setProperty(KAFKA_BOOTSTRAP_SERVERS, veniceCluster.getPubSubBrokerWrapper().getAddress()); - props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + props.putAll(ServiceFactory.getPubSubClientConfigs()); VeniceProperties veniceProperties = new VeniceProperties(props); ByteBuffer v1Dict = DictionaryUtils.readDictionaryFromKafka(Version.composeKafkaTopic(storeName, 1), veniceProperties); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java index 8fe773f2222..4e82f9bd85c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java @@ -15,7 +15,6 @@ import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; -import com.linkedin.venice.utils.TestUtils; import java.util.Properties; import org.testng.annotations.Test; @@ -32,7 +31,7 @@ public VeniceClusterWrapper initializeVeniceCluster() { VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options); Properties serverProperties = new Properties(); - serverProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + serverProperties.putAll(ServiceFactory.getPubSubClientConfigs()); serverProperties.put(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB); serverProperties.setProperty(ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED, "false"); serverProperties.setProperty(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, "true"); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwapDvc.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwapDvc.java index e96f6d6b3d6..a9a30a3a28b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwapDvc.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwapDvc.java @@ -79,7 +79,7 @@ public class TestDeferredVersionSwapDvc { @BeforeClass public void setUp() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Properties controllerProps = new Properties(); controllerProps.put(CONTROLLER_DEFERRED_VERSION_SWAP_SLEEP_MS, 100); controllerProps.put(CONTROLLER_DEFERRED_VERSION_SWAP_SERVICE_ENABLED, true); @@ -110,7 +110,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java index fce2d10ddc5..2157d86c5a1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java @@ -14,6 +14,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.Store; @@ -39,13 +40,13 @@ public class TestDumpIngestionContext extends AbstractMultiRegionTest { @BeforeClass(alwaysRun = true) public void setUpAdminToolSystemProperties() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); } @AfterClass(alwaysRun = true) public void restoreAdminToolSystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java index 6c3aba29c07..21af82b4110 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java @@ -1020,7 +1020,7 @@ public void testHybridMultipleVersions() throws Exception { try (VeniceProducer veniceOnlineProducer = OnlineProducerFactory.createProducer( ClientConfig.defaultGenericClientConfig(storeName).setVeniceURL(cluster.getRandomRouterURL()), - new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), + new VeniceProperties(ServiceFactory.getPubSubClientConfigs()), null)) { for (int i = keyCount; i < keyCount * 2; i++) { veniceOnlineProducer.asyncPut(i, i * 2).get(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java index ffc282b3fba..5cf102fe544 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java @@ -104,7 +104,7 @@ protected boolean shouldCreateD2Client() { @Override protected Properties getExtraServerProperties() { Properties serverProperties = new Properties(); - serverProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + serverProperties.putAll(ServiceFactory.getPubSubClientConfigs()); serverProperties.setProperty(SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE, "300"); return serverProperties; } @@ -112,7 +112,7 @@ protected Properties getExtraServerProperties() { @Override protected Properties getExtraControllerProperties() { Properties controllerProps = new Properties(); - controllerProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + controllerProps.putAll(ServiceFactory.getPubSubClientConfigs()); // This property is required for test stores that have 10 partitions controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 10); controllerProps.put(BatchJobHeartbeatConfigs.HEARTBEAT_STORE_CLUSTER_CONFIG.getConfigName(), SYSTEM_STORE_CLUSTER); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRepushCore.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRepushCore.java index 06fec60ddef..0ff8dba7032 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRepushCore.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRepushCore.java @@ -46,6 +46,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.hadoop.VenicePushJob; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; @@ -307,7 +308,7 @@ public void testRepushWithDeleteRecord(boolean useSpark) { .put(ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED, "false") .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DAVINCI_PUSH_STATUS_CHECK_INTERVAL_IN_MS, 1000) - .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()) + .put(ServiceFactory.getPubSubClientConfigs()) .build(); MetricsRepository metricsRepository = new VeniceMetricsRepository(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java index 5679a6f8fc2..423d6250564 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java @@ -133,7 +133,7 @@ public class TestStoreMigration { @BeforeClass public void setUp() throws Exception { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Utils.thisIsLocalhost(); Properties parentControllerProperties = new Properties(); // Disable topic cleanup since parent and child are sharing the same kafka cluster. @@ -186,7 +186,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigrationMultiRegion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigrationMultiRegion.java index 86f9ca67874..140dd33f38c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigrationMultiRegion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigrationMultiRegion.java @@ -70,7 +70,7 @@ public class TestStoreMigrationMultiRegion { @BeforeClass(timeOut = 180_000) public void setUp() { - originalPubSubAdapterFactorySystemProperties = TestUtils.setPubSubApacheKafkaAdapterFactorySystemProperties(); + originalPubSubAdapterFactorySystemProperties = ServiceFactory.setPubSubClientConfigsAsSystemProperties(); Utils.thisIsLocalhost(); Properties controllerProperties = new Properties(); controllerProperties.setProperty(TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS, String.valueOf(4000)); @@ -118,7 +118,7 @@ public void cleanUp() { private void restorePubSubAdapterFactorySystemProperties() { if (originalPubSubAdapterFactorySystemProperties != null) { - TestUtils.restorePubSubApacheKafkaAdapterFactorySystemProperties(originalPubSubAdapterFactorySystemProperties); + ServiceFactory.restorePubSubClientConfigsSystemProperties(originalPubSubAdapterFactorySystemProperties); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java index 47eca87a9a8..59484a30cb9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVTConsistencyCheckerJob.java @@ -24,6 +24,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.adapter.kafka.common.ApacheKafkaOffsetPosition; @@ -239,7 +240,7 @@ public void testFullPipelineWithBatchPushRTWritesAndInjectedInconsistency() thro File outputDir = new File(tempRoot, "output"); try { Properties jobProps = new Properties(); - jobProps.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + jobProps.putAll(ServiceFactory.getPubSubClientConfigs()); jobProps.setProperty( VTConsistencyCheckerJob.DC0_BROKER_URL, childDatacenters.get(0).getPubSubBrokerWrapper().getAddress()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/VersionSpecificDaVinciClientTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/VersionSpecificDaVinciClientTest.java index 50ba59c70e5..01e6029940c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/VersionSpecificDaVinciClientTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/VersionSpecificDaVinciClientTest.java @@ -127,7 +127,7 @@ public void testVersionSpecificDaVinciClient() throws Exception { ClientConfig.defaultGenericClientConfig(storeName) .setD2Client(d2Client) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME), - new VeniceProperties(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()), + new VeniceProperties(ServiceFactory.getPubSubClientConfigs()), null)) { producer.asyncPut(streamingKey1, customValue).get(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java index c51b224a7a8..1ba8560f2d6 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java @@ -170,7 +170,7 @@ protected Properties getExtraServerProperties() { public void setUp() throws Exception { Utils.thisIsLocalhost(); Properties props = new Properties(); - props.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + props.putAll(ServiceFactory.getPubSubClientConfigs()); props.put(SERVER_HTTP2_INBOUND_ENABLED, "true"); props.put(SERVER_QUOTA_ENFORCEMENT_ENABLED, "true"); props.putAll(getExtraServerProperties()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/input/kafka/TestKafkaInputFormat.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/input/kafka/TestKafkaInputFormat.java index a9f0737d7af..0f2d9b0f6da 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/input/kafka/TestKafkaInputFormat.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/input/kafka/TestKafkaInputFormat.java @@ -17,7 +17,6 @@ import com.linkedin.venice.pubsub.api.PubSubTopicPartition; import com.linkedin.venice.pubsub.manager.TopicManager; import com.linkedin.venice.utils.IntegrationTestPushUtils; -import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.writer.VeniceWriter; @@ -137,8 +136,7 @@ public void testGetSplits() { KafkaInputFormat kafkaInputFormat = new KafkaInputFormat(); PubSubTopic topic = getTopic(1000, 3); JobConf conf = new JobConf(); - TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs() - .forEach((key, value) -> conf.set(key.toString(), value.toString())); + ServiceFactory.getPubSubClientConfigs().forEach((key, value) -> conf.set(key.toString(), value.toString())); conf.set(VENICE_REPUSH_SOURCE_PUBSUB_BROKER, pubSubBrokerWrapper.getAddress()); conf.set(KAFKA_INPUT_TOPIC, topic.getName()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java index 0508848600c..19985bc7ad7 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java @@ -18,7 +18,6 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.utils.PropertyBuilder; -import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import io.tehuti.metrics.MetricsRepository; @@ -199,6 +198,6 @@ public static PropertyBuilder getDaVinciPropertyBuilder(String zkAddress) { .put(D2_ZK_HOSTS_ADDRESS, zkAddress) .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 4 * 1024 * 1024 * 1024L) .put(CLUSTER_DISCOVERY_D2_SERVICE, VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME) - .put(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + .put(ServiceFactory.getPubSubClientConfigs()); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java index 47a13bf3287..3698f318e40 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java @@ -3,6 +3,10 @@ import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.D2_ZK_HOSTS_ADDRESS; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; +import static com.linkedin.venice.ConfigKeys.PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS; +import static com.linkedin.venice.ConfigKeys.PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_MAX_ATTEMPT; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_WAIT_TIME_FOR_CLUSTER_START_S; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.STANDALONE_REGION_NAME; @@ -120,10 +124,67 @@ public static void withMaxAttempt(int maxAttempt, Runnable action) { /** * @return an instance of {@link PubSubClientsFactory} */ - static PubSubClientsFactory getPubSubClientsFactory() { + public static PubSubClientsFactory getPubSubClientsFactory() { return PUBSUB_BROKER_FACTORY.getClientsFactory(); } + /** + * Resolves the pub-sub adapter factory class configs from the configured pub-sub backend (selected via the + * {@code pubSubBrokerFactory} system property) instead of hard-coding a specific implementation. Tests use this so + * the suite exercises whatever client configs the pub-sub backend under test exposes. + * + * @return a {@link Properties} carrying the producer, consumer, admin, and source-of-truth-admin adapter factory + * class names exposed by the configured backend. + */ + public static Properties getPubSubClientConfigs() { + PubSubClientsFactory clientsFactory = getPubSubClientsFactory(); + String adminAdapterFactoryClass = clientsFactory.getAdminAdapterFactory().getClass().getName(); + Properties properties = new Properties(); + properties.setProperty( + PUBSUB_PRODUCER_ADAPTER_FACTORY_CLASS, + clientsFactory.getProducerAdapterFactory().getClass().getName()); + properties.setProperty( + PUBSUB_CONSUMER_ADAPTER_FACTORY_CLASS, + clientsFactory.getConsumerAdapterFactory().getClass().getName()); + properties.setProperty(PUBSUB_ADMIN_ADAPTER_FACTORY_CLASS, adminAdapterFactoryClass); + properties.setProperty(PUBSUB_SOURCE_OF_TRUTH_ADMIN_ADAPTER_FACTORY_CLASS, adminAdapterFactoryClass); + return properties; + } + + /** + * Publishes the configured backend's pub-sub adapter factory class configs as JVM system properties so components + * that read from {@link System#getProperties()} (e.g. the admin tool) pick them up. + * + * @return the prior values of the affected system properties, to be passed to + * {@link #restorePubSubClientConfigsSystemProperties(Properties)} for cleanup. + */ + public static Properties setPubSubClientConfigsAsSystemProperties() { + Properties originalProperties = new Properties(); + Properties factoryConfigs = getPubSubClientConfigs(); + for (String key: factoryConfigs.stringPropertyNames()) { + String originalValue = System.getProperty(key); + if (originalValue != null) { + originalProperties.setProperty(key, originalValue); + } + System.setProperty(key, factoryConfigs.getProperty(key)); + } + return originalProperties; + } + + /** + * Restores the system properties previously mutated by {@link #setPubSubClientConfigsAsSystemProperties()}. + */ + public static void restorePubSubClientConfigsSystemProperties(Properties originalProperties) { + for (String key: getPubSubClientConfigs().stringPropertyNames()) { + if (originalProperties.containsKey(key)) { + System.setProperty(key, originalProperties.getProperty(key)); + } else { + System.clearProperty(key); + } + } + originalProperties.clear(); + } + /** * @return an instance of {@link ZkServerWrapper} */ diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java index 9a974979777..3ad9c1f9def 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java @@ -51,6 +51,7 @@ import com.linkedin.venice.helix.VeniceJsonSerializer; import com.linkedin.venice.integration.utils.KafkaTestUtils; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; +import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; @@ -331,8 +332,7 @@ private static Map getSamzaProducerConfigForBatch( } private static void addPubSubApacheKafkaAdapterFactoryConfigs(Map config) { - TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs() - .forEach((key, value) -> config.put(key.toString(), value.toString())); + ServiceFactory.getPubSubClientConfigs().forEach((key, value) -> config.put(key.toString(), value.toString())); } /** @@ -610,7 +610,7 @@ public static VeniceWriterFactory getVeniceWriterFactory( PubSubBrokerWrapper pubSubBrokerWrapper, PubSubProducerAdapterFactory pubSubProducerAdapterFactory) { Properties veniceWriterProperties = new Properties(); - veniceWriterProperties.putAll(TestUtils.getPubSubApacheKafkaAdapterFactoryConfigs()); + veniceWriterProperties.putAll(ServiceFactory.getPubSubClientConfigs()); veniceWriterProperties.put(KAFKA_BOOTSTRAP_SERVERS, pubSubBrokerWrapper.getAddress()); veniceWriterProperties .putAll(PubSubBrokerWrapper.getBrokerDetailsForClients(Collections.singletonList(pubSubBrokerWrapper))); diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java index a27c885e597..a4e332bfcca 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java @@ -765,30 +765,6 @@ public static Properties getPubSubApacheKafkaAdapterFactoryConfigs() { return properties; } - public static Properties setPubSubApacheKafkaAdapterFactorySystemProperties() { - Properties originalProperties = new Properties(); - Properties factoryConfigs = getPubSubApacheKafkaAdapterFactoryConfigs(); - for (String key: factoryConfigs.stringPropertyNames()) { - String originalValue = System.getProperty(key); - if (originalValue != null) { - originalProperties.setProperty(key, originalValue); - } - System.setProperty(key, factoryConfigs.getProperty(key)); - } - return originalProperties; - } - - public static void restorePubSubApacheKafkaAdapterFactorySystemProperties(Properties originalProperties) { - for (String key: getPubSubApacheKafkaAdapterFactoryConfigs().stringPropertyNames()) { - if (originalProperties.containsKey(key)) { - System.setProperty(key, originalProperties.getProperty(key)); - } else { - System.clearProperty(key); - } - } - originalProperties.clear(); - } - public static Properties getPropertiesForControllerConfig() { Properties properties = new Properties(); properties.putAll(getPubSubApacheKafkaAdapterFactoryConfigs());