Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
import com.linkedin.venice.serialization.avro.InternalAvroSpecificSerializer;
import com.linkedin.venice.serializer.RecordDeserializer;
import com.linkedin.venice.stats.StatsErrorCode;
import com.linkedin.venice.stats.dimensions.VeniceGlobalRtDivErrorType;
import com.linkedin.venice.stats.dimensions.VeniceIngestionFailureReason;
import com.linkedin.venice.stats.dimensions.VenicePartialUpdateOperation;
import com.linkedin.venice.stats.dimensions.VeniceRegionLocality;
Expand Down Expand Up @@ -4608,6 +4609,8 @@ CompletableFuture<Void> sendGlobalRtDivMessage(
true);

pcs.resetConsumedBytesSinceLastGlobalRtDivSync(brokerUrl);
versionedIngestionStats
.recordGlobalRtDivSent(storeName, versionNumber, valueBytes.length, rtDivPartitionStates.size());
return vtDivSyncedFuture;
}

Expand All @@ -4622,6 +4625,7 @@ private byte[] createGlobalRtDivValueBytes(
try {
valueBytes = compressor.get().compress(valueBytes);
} catch (IOException e) {
versionedIngestionStats.recordGlobalRtDivError(storeName, versionNumber, VeniceGlobalRtDivErrorType.SEND);
LOGGER.error(
"Failed to compress GlobalRtDivState for replica: {}. Will proceed without {} compression.",
topicPartition,
Expand Down Expand Up @@ -4715,6 +4719,7 @@ GlobalRtDivState readGlobalRtDivState(
// GlobalRtDivState loading is best-effort. Any failure (e.g. storage not initialized,
// compressor dictionary not yet available before SOP is consumed on secondary-fabric leaders)
// should not propagate and kill ingestion.
versionedIngestionStats.recordGlobalRtDivError(storeName, versionNumber, VeniceGlobalRtDivErrorType.LOAD);
LOGGER.warn(
"Unable to read Global RT DIV state for topic-partition: {}, brokerUrl: {}",
topicPartition,
Expand All @@ -4736,6 +4741,7 @@ private GlobalRtDivState deserializeGlobalRtDivState(
return globalRtDivStateSerializer
.deserialize(serializedValueBytes, AvroProtocolDefinition.GLOBAL_RT_DIV_STATE.getCurrentProtocolVersion());
} catch (Exception e) {
versionedIngestionStats.recordGlobalRtDivError(storeName, versionNumber, VeniceGlobalRtDivErrorType.LOAD);
// TODO: evaluate whether these logs can be set to debug
LOGGER.error(
"Unable to deserialize stored value bytes for key: {}, topic-partition: {}",
Expand Down Expand Up @@ -4854,6 +4860,7 @@ void loadGlobalRtDiv(int partition, String brokerUrl) {
new ChunkedValueManifestContainer());

if (globalRtDivState == null) {
versionedIngestionStats.recordGlobalRtDivLoadNotFound(storeName, versionNumber);
// If the GlobalRtDivState is not present, it could be acceptable if this could be the first leader to be elected
// Object not existing could be problematic if this isn't the first leader (detected via nonzero leaderPosition)
PubSubPosition leaderPosition = pcs.getLeaderPosition(brokerUrl, false);
Expand All @@ -4868,6 +4875,7 @@ void loadGlobalRtDiv(int partition, String brokerUrl) {
}

final Map<CharSequence, ProducerPartitionState> producerStates = globalRtDivState.getProducerStates();
versionedIngestionStats.recordGlobalRtDivLoaded(storeName, versionNumber, producerStates.size());
PartitionTracker.TopicType realTimeTopicType = PartitionTracker.TopicType.of(REALTIME_TOPIC_TYPE, brokerUrl);
getConsumerDiv().setPartitionState(realTimeTopicType, pcs.getPartition(), producerStates);
ByteBuffer checkpointBytes = globalRtDivState.getLatestPubSubPosition(); // LCRP
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@
import com.linkedin.venice.serializer.RecordDeserializer;
import com.linkedin.venice.server.VersionRole;
import com.linkedin.venice.stats.dimensions.ReplicaType;
import com.linkedin.venice.stats.dimensions.VeniceGlobalRtDivErrorType;
import com.linkedin.venice.stats.dimensions.VeniceIngestionFailureReason;
import com.linkedin.venice.stats.dimensions.VeniceRecordType;
import com.linkedin.venice.storage.protocol.ChunkedValueManifest;
Expand Down Expand Up @@ -2327,7 +2328,16 @@ protected void updateAndSyncOffsetFromSnapshot(PartitionTracker vtDivSnapshot, P
}
vtDivSnapshot.updateOffsetRecord(PartitionTracker.VERSION_TOPIC, pcs.getOffsetRecord());
updateOffsetMetadataInOffsetRecord(pcs);
syncOffset(pcs);
try {
syncOffset(pcs);
} catch (Exception e) {
versionedIngestionStats.recordGlobalRtDivError(storeName, versionNumber, VeniceGlobalRtDivErrorType.VT_SYNC);
throw e;
}
versionedIngestionStats.recordGlobalRtDivVtSynced(
storeName,
versionNumber,
vtDivSnapshot.getPartitionStates(PartitionTracker.VERSION_TOPIC).size());
}

/**
Expand Down Expand Up @@ -4818,9 +4828,15 @@ protected void putInStorageEngine(int partition, byte[] keyBytes, Put put) {
}

protected void putGlobalRtDivStateInMetadata(int partition, byte[] keyBytes, Put put) {
storageEngine.putGlobalRtDivMetadata(
keyBytes,
ByteUtils.prependIntHeaderToByteBuffer(put.putValue, put.schemaId, false).array());
try {
storageEngine.putGlobalRtDivMetadata(
keyBytes,
ByteUtils.prependIntHeaderToByteBuffer(put.putValue, put.schemaId, false).array());
} catch (Exception e) {
versionedIngestionStats.recordGlobalRtDivError(storeName, versionNumber, VeniceGlobalRtDivErrorType.PERSIST);
throw e;
}
versionedIngestionStats.recordGlobalRtDivPersisted(storeName, versionNumber);
}

protected void removeFromStorageEngine(int partition, byte[] keyBytes, Delete delete) {
Expand Down Expand Up @@ -5421,7 +5437,12 @@ private int processKafkaDataMessage(

keyLen = keyBytes.length;
if (kafkaKey.isGlobalRtDiv()) {
storageEngine.deleteGlobalRtDivMetadata(keyBytes);
try {
storageEngine.deleteGlobalRtDivMetadata(keyBytes);
} catch (Exception e) {
versionedIngestionStats.recordGlobalRtDivError(storeName, versionNumber, VeniceGlobalRtDivErrorType.DELETE);
throw e;
}
} else {
deleteFromStorageEngine(producedPartition, keyBytes, delete);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.linkedin.venice.stats.dimensions.ReplicaType;
import com.linkedin.venice.stats.dimensions.VeniceDCREvent;
import com.linkedin.venice.stats.dimensions.VeniceDCROperation;
import com.linkedin.venice.stats.dimensions.VeniceGlobalRtDivErrorType;
import com.linkedin.venice.stats.dimensions.VeniceIngestionDestinationComponent;
import com.linkedin.venice.stats.dimensions.VeniceIngestionFailureReason;
import com.linkedin.venice.stats.dimensions.VeniceIngestionSourceComponent;
Expand Down Expand Up @@ -589,4 +590,28 @@ public void recordPartialUpdateAmplificationAlertCount(String storeName, int ver
public void recordActiveKeyCountInvalidation(String storeName, int version) {
getIngestionOtelStats(storeName).recordActiveKeyCountInvalidation(version);
}

public void recordGlobalRtDivSent(String storeName, int version, long payloadSizeBytes, int rtProducerCount) {
getIngestionOtelStats(storeName).recordGlobalRtDivSent(version, payloadSizeBytes, rtProducerCount);
}

public void recordGlobalRtDivPersisted(String storeName, int version) {
getIngestionOtelStats(storeName).recordGlobalRtDivPersisted(version);
}

public void recordGlobalRtDivVtSynced(String storeName, int version, int vtProducerCount) {
getIngestionOtelStats(storeName).recordGlobalRtDivVtSynced(version, vtProducerCount);
}

public void recordGlobalRtDivError(String storeName, int version, VeniceGlobalRtDivErrorType errorType) {
getIngestionOtelStats(storeName).recordGlobalRtDivError(version, errorType);
}

public void recordGlobalRtDivLoaded(String storeName, int version, int rtProducerCount) {
getIngestionOtelStats(storeName).recordGlobalRtDivLoaded(version, rtProducerCount);
}

public void recordGlobalRtDivLoadNotFound(String storeName, int version) {
getIngestionOtelStats(storeName).recordGlobalRtDivLoadNotFound(version);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import static com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions.VENICE_DCR_EVENT;
import static com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions.VENICE_DCR_OPERATION;
import static com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions.VENICE_DESTINATION_REGION;
import static com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions.VENICE_GLOBAL_RT_DIV_ERROR_TYPE;
import static com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions.VENICE_GLOBAL_RT_DIV_LOAD_OUTCOME;
import static com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions.VENICE_INGESTION_DESTINATION_COMPONENT;
import static com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions.VENICE_INGESTION_FAILURE_REASON;
import static com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions.VENICE_INGESTION_SOURCE_COMPONENT;
Expand Down Expand Up @@ -410,6 +412,50 @@ public enum IngestionOtelMetricEntity implements ModuleMetricEntityInterface {
"ingestion.key.active_count_invalidation", MetricType.COUNTER, MetricUnit.NUMBER,
"Count of active key count invalidations due to underflow drift, keyExists failures, leader-propagated invalidation, or corrupt kcs signals",
setOf(VENICE_STORE_NAME, VENICE_CLUSTER_NAME, VENICE_VERSION_ROLE)
),

GLOBAL_RT_DIV_SEND_SIZE(
"ingestion.global_rt_div.send_size", MetricType.MIN_MAX_COUNT_SUM_AGGREGATIONS, MetricUnit.BYTES,
"Byte size of each Global RT DIV payload produced to the version topic by the leader. The count aggregation "
+ "doubles as the number of Global RT DIV messages sent",
setOf(VENICE_STORE_NAME, VENICE_CLUSTER_NAME, VENICE_VERSION_ROLE)
),

GLOBAL_RT_DIV_SEND_RT_PRODUCER_COUNT(
"ingestion.global_rt_div.send_rt_producer_count", MetricType.MIN_MAX_COUNT_SUM_AGGREGATIONS, MetricUnit.NUMBER,
"Number of RT producers tracked per broker in each Global RT DIV payload sent by the leader",
setOf(VENICE_STORE_NAME, VENICE_CLUSTER_NAME, VENICE_VERSION_ROLE)
),

GLOBAL_RT_DIV_PERSIST_COUNT(
"ingestion.global_rt_div.persist_count", MetricType.COUNTER, MetricUnit.NUMBER,
"Count of Global RT DIV states successfully persisted to metadata storage by the drainer",
setOf(VENICE_STORE_NAME, VENICE_CLUSTER_NAME, VENICE_VERSION_ROLE)
),

GLOBAL_RT_DIV_VT_SYNC_PRODUCER_COUNT(
"ingestion.global_rt_div.vt_sync_producer_count", MetricType.MIN_MAX_COUNT_SUM_AGGREGATIONS, MetricUnit.NUMBER,
"Number of VT producers in the snapshot checkpointed into the OffsetRecord. The count aggregation doubles as "
+ "the number of VT position syncs",
setOf(VENICE_STORE_NAME, VENICE_CLUSTER_NAME, VENICE_VERSION_ROLE)
),

GLOBAL_RT_DIV_ERROR_COUNT(
"ingestion.global_rt_div.error_count", MetricType.COUNTER, MetricUnit.NUMBER,
"Count of errors in any Global RT DIV operation phase (send, persist, vt_sync, delete, load)",
setOf(VENICE_STORE_NAME, VENICE_CLUSTER_NAME, VENICE_VERSION_ROLE, VENICE_GLOBAL_RT_DIV_ERROR_TYPE)
),

GLOBAL_RT_DIV_LOAD_COUNT(
"ingestion.global_rt_div.load_count", MetricType.COUNTER, MetricUnit.NUMBER,
"Count of RT DIV load attempts on F→L leader promotion, dimensioned by whether state was found on disk",
setOf(VENICE_STORE_NAME, VENICE_CLUSTER_NAME, VENICE_VERSION_ROLE, VENICE_GLOBAL_RT_DIV_LOAD_OUTCOME)
),

GLOBAL_RT_DIV_LOAD_RT_PRODUCER_COUNT(
"ingestion.global_rt_div.load_rt_producer_count", MetricType.MIN_MAX_COUNT_SUM_AGGREGATIONS, MetricUnit.NUMBER,
"Number of RT producers restored from disk when loading Global RT DIV state on leader promotion",
setOf(VENICE_STORE_NAME, VENICE_CLUSTER_NAME, VENICE_VERSION_ROLE)
);

private final MetricEntity metricEntity;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.DCR_TOTAL_COUNT;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.DISK_QUOTA_USED;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.DUPLICATE_KEY_UPDATE_COUNT;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.GLOBAL_RT_DIV_ERROR_COUNT;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.GLOBAL_RT_DIV_LOAD_COUNT;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.GLOBAL_RT_DIV_LOAD_RT_PRODUCER_COUNT;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.GLOBAL_RT_DIV_PERSIST_COUNT;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.GLOBAL_RT_DIV_SEND_RT_PRODUCER_COUNT;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.GLOBAL_RT_DIV_SEND_SIZE;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.GLOBAL_RT_DIV_VT_SYNC_PRODUCER_COUNT;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.INGESTION_BYTES_CONSUMED;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.INGESTION_BYTES_PRODUCED;
import static com.linkedin.davinci.stats.ingestion.IngestionOtelMetricEntity.INGESTION_FAILURE_COUNT;
Expand Down Expand Up @@ -71,6 +78,8 @@
import com.linkedin.venice.stats.dimensions.VeniceDCREvent;
import com.linkedin.venice.stats.dimensions.VeniceDCROperation;
import com.linkedin.venice.stats.dimensions.VeniceDimensionInterface;
import com.linkedin.venice.stats.dimensions.VeniceGlobalRtDivErrorType;
import com.linkedin.venice.stats.dimensions.VeniceGlobalRtDivLoadOutcome;
import com.linkedin.venice.stats.dimensions.VeniceIngestionDestinationComponent;
import com.linkedin.venice.stats.dimensions.VeniceIngestionFailureReason;
import com.linkedin.venice.stats.dimensions.VeniceIngestionSourceComponent;
Expand Down Expand Up @@ -201,6 +210,15 @@ public class IngestionOtelStats {
private final MetricEntityStateTwoEnums<VersionRole, VeniceRecordType> recordAssembledSizeMetric;
private final MetricEntityStateOneEnum<VersionRole> recordAssembledSizeRatioMetric;

// Global RT DIV metrics
private final MetricEntityStateOneEnum<VersionRole> globalRtDivSendSizeMetric;
private final MetricEntityStateOneEnum<VersionRole> globalRtDivSendRtProducerCountMetric;
private final MetricEntityStateOneEnum<VersionRole> globalRtDivPersistCountMetric;
private final MetricEntityStateOneEnum<VersionRole> globalRtDivVtSyncProducerCountMetric;
private final MetricEntityStateTwoEnums<VersionRole, VeniceGlobalRtDivErrorType> globalRtDivErrorCountMetric;
private final MetricEntityStateTwoEnums<VersionRole, VeniceGlobalRtDivLoadOutcome> globalRtDivLoadCountMetric;
private final MetricEntityStateOneEnum<VersionRole> globalRtDivLoadRtProducerCountMetric;

// Async gauge metrics
private final AsyncMetricEntityStateOneEnum<VersionRole> ingestionTaskCountByRole;
private final AsyncMetricEntityStateTwoEnums<VersionRole, ReplicaType> activeKeyCountByRoleAndReplicaType;
Expand Down Expand Up @@ -273,6 +291,13 @@ public class IngestionOtelStats {
this.recordValueSizeMetric = null;
this.recordAssembledSizeMetric = null;
this.recordAssembledSizeRatioMetric = null;
this.globalRtDivSendSizeMetric = null;
this.globalRtDivSendRtProducerCountMetric = null;
this.globalRtDivPersistCountMetric = null;
this.globalRtDivVtSyncProducerCountMetric = null;
this.globalRtDivErrorCountMetric = null;
this.globalRtDivLoadCountMetric = null;
this.globalRtDivLoadRtProducerCountMetric = null;
this.ingestionTaskCountByRole = null;
this.activeKeyCountByRoleAndReplicaType = null;
this.uniqueIngestedKeyCountByRoleAndReplicaType = null;
Expand Down Expand Up @@ -412,6 +437,17 @@ public IngestionOtelStats(
recordAssembledSizeMetric = createTwoEnumMetric(RECORD_ASSEMBLED_SIZE.getMetricEntity(), VeniceRecordType.class);
recordAssembledSizeRatioMetric = createOneEnumMetric(RECORD_ASSEMBLED_SIZE_RATIO.getMetricEntity());

// Global RT DIV metrics
globalRtDivSendSizeMetric = createOneEnumMetric(GLOBAL_RT_DIV_SEND_SIZE.getMetricEntity());
globalRtDivSendRtProducerCountMetric = createOneEnumMetric(GLOBAL_RT_DIV_SEND_RT_PRODUCER_COUNT.getMetricEntity());
globalRtDivPersistCountMetric = createOneEnumMetric(GLOBAL_RT_DIV_PERSIST_COUNT.getMetricEntity());
globalRtDivVtSyncProducerCountMetric = createOneEnumMetric(GLOBAL_RT_DIV_VT_SYNC_PRODUCER_COUNT.getMetricEntity());
globalRtDivErrorCountMetric =
createTwoEnumMetric(GLOBAL_RT_DIV_ERROR_COUNT.getMetricEntity(), VeniceGlobalRtDivErrorType.class);
globalRtDivLoadCountMetric =
createTwoEnumMetric(GLOBAL_RT_DIV_LOAD_COUNT.getMetricEntity(), VeniceGlobalRtDivLoadOutcome.class);
globalRtDivLoadRtProducerCountMetric = createOneEnumMetric(GLOBAL_RT_DIV_LOAD_RT_PRODUCER_COUNT.getMetricEntity());

ingestionTaskCountByRole =
createAsyncByRole(INGESTION_TASK_COUNT.getMetricEntity(), this::getTaskForRole, (task, role) -> 1L);

Expand Down Expand Up @@ -830,6 +866,36 @@ public void recordPartialUpdateAmplificationAlertCount(int version, long value)
partialUpdateAmplificationAlertCountMetric.record(value, classifyVersion(version, versionInfo));
}

// Global RT DIV recording methods

public void recordGlobalRtDivSent(int version, long payloadSizeBytes, int rtProducerCount) {
VersionRole versionRole = classifyVersion(version, versionInfo);
globalRtDivSendSizeMetric.record(payloadSizeBytes, versionRole);
globalRtDivSendRtProducerCountMetric.record(rtProducerCount, versionRole);
}

public void recordGlobalRtDivPersisted(int version) {
globalRtDivPersistCountMetric.record(1, classifyVersion(version, versionInfo));
}

public void recordGlobalRtDivVtSynced(int version, int vtProducerCount) {
globalRtDivVtSyncProducerCountMetric.record(vtProducerCount, classifyVersion(version, versionInfo));
}

public void recordGlobalRtDivError(int version, VeniceGlobalRtDivErrorType errorType) {
globalRtDivErrorCountMetric.record(1, classifyVersion(version, versionInfo), errorType);
}

public void recordGlobalRtDivLoaded(int version, int rtProducerCount) {
VersionRole versionRole = classifyVersion(version, versionInfo);
globalRtDivLoadCountMetric.record(1, versionRole, VeniceGlobalRtDivLoadOutcome.FOUND);
globalRtDivLoadRtProducerCountMetric.record(rtProducerCount, versionRole);
}

public void recordGlobalRtDivLoadNotFound(int version) {
globalRtDivLoadCountMetric.record(1, classifyVersion(version, versionInfo), VeniceGlobalRtDivLoadOutcome.NOT_FOUND);
}

public void recordActiveKeyCountInvalidation(int version) {
if (activeKeyCountInvalidationMetric != null) {
activeKeyCountInvalidationMetric.record(1, classifyVersion(version, versionInfo));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.linkedin.venice.stats.dimensions.ReplicaType;
import com.linkedin.venice.stats.dimensions.VeniceDCREvent;
import com.linkedin.venice.stats.dimensions.VeniceDCROperation;
import com.linkedin.venice.stats.dimensions.VeniceGlobalRtDivErrorType;
import com.linkedin.venice.stats.dimensions.VeniceIngestionDestinationComponent;
import com.linkedin.venice.stats.dimensions.VeniceIngestionFailureReason;
import com.linkedin.venice.stats.dimensions.VeniceIngestionSourceComponent;
Expand Down Expand Up @@ -264,4 +265,28 @@ public void recordPartialUpdateAmplificationAlertCount(int version, long value)
@Override
public void recordActiveKeyCountInvalidation(int version) {
}

@Override
public void recordGlobalRtDivSent(int version, long payloadSizeBytes, int rtProducerCount) {
}

@Override
public void recordGlobalRtDivPersisted(int version) {
}

@Override
public void recordGlobalRtDivVtSynced(int version, int vtProducerCount) {
}

@Override
public void recordGlobalRtDivError(int version, VeniceGlobalRtDivErrorType errorType) {
}

@Override
public void recordGlobalRtDivLoaded(int version, int rtProducerCount) {
}

@Override
public void recordGlobalRtDivLoadNotFound(int version) {
}
}
Loading