From 7c3eade1aa11d474aa50fd5a1c3dc827e5609c97 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:34:48 +0800 Subject: [PATCH] Expose consensus subscription queue readiness --- .../confignode/manager/ConfigManager.java | 28 +++++- .../procedure/env/ConfigNodeProcedureEnv.java | 9 +- .../impl/DataNodeInternalRPCServiceImpl.java | 20 +++- .../subscription/ShowSubscriptionsTask.java | 6 ++ .../ConsensusSubscriptionSetupHandler.java | 99 +++++++++++++++++++ .../ShowSubscriptionsTaskTest.java | 43 ++++---- ...ConsensusSubscriptionSetupHandlerTest.java | 23 +++++ .../schema/column/ColumnHeaderConstant.java | 4 +- .../src/main/thrift/confignode.thrift | 1 + .../src/main/thrift/datanode.thrift | 9 ++ 10 files changed, 217 insertions(+), 25 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index cb9dab67bc6c..daaa62cab628 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -274,6 +274,7 @@ import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; import org.apache.iotdb.db.schemaengine.template.alter.TemplateAlterOperationUtil; +import org.apache.iotdb.mpp.rpc.thrift.TConsensusSubscriptionQueueReadiness; import org.apache.iotdb.mpp.rpc.thrift.TPullCommitProgressResp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -310,6 +311,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -2828,11 +2830,28 @@ public TShowSubscriptionResp showSubscription(TShowSubscriptionReq req) { info.getTopicName() + "\u0000" + info.getConsumerGroupId())); } final List progressInfoList = new ArrayList<>(); + final Map> queuesBoundDataNodeIds = new HashMap<>(); final Map dataNodeResponses = - getProcedureManager().getEnv().pullCommitProgressFromDataNodesBestEffort(); + getProcedureManager().getEnv().pullCommitProgressFromDataNodesBestEffort(true); for (final Map.Entry dataNodeEntry : dataNodeResponses.entrySet()) { final TPullCommitProgressResp dataNodeResponse = dataNodeEntry.getValue(); + if (dataNodeResponse.isSetStatus() + && dataNodeResponse.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + && dataNodeResponse.isSetConsensusQueueReadiness()) { + for (final TConsensusSubscriptionQueueReadiness readiness : + dataNodeResponse.getConsensusQueueReadiness()) { + if (readiness.isTableModel() != (req.isSetIsTableModel() && req.isIsTableModel())) { + continue; + } + final String key = readiness.getTopicName() + "\u0000" + readiness.getConsumerGroupId(); + if (visibleSubscriptions.contains(key)) { + queuesBoundDataNodeIds + .computeIfAbsent(key, ignored -> new HashSet<>()) + .add(dataNodeEntry.getKey()); + } + } + } if (!dataNodeResponse.isSetSubscriptionProgress()) { continue; } @@ -2917,6 +2936,13 @@ public TShowSubscriptionResp showSubscription(TShowSubscriptionReq req) { progressInfo.getConsumerGroupId(), tableModel, registeredDataNodeIds)); + final Set queuesBoundIds = + new TreeSet<>( + queuesBoundDataNodeIds.getOrDefault( + progressInfo.getTopicName() + "\u0000" + progressInfo.getConsumerGroupId(), + Collections.emptySet())); + queuesBoundIds.retainAll(registeredDataNodeIds); + progressInfo.setQueuesBoundOnDns(new ArrayList<>(queuesBoundIds)); } progressInfoList.sort( Comparator.comparing(TSubscriptionProgressInfo::getTopicName) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java index 8fd27ab0fc09..eb01e326a394 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java @@ -921,9 +921,16 @@ public Map pullCommitProgressFromDataNodes() { } public Map pullCommitProgressFromDataNodesBestEffort() { + return pullCommitProgressFromDataNodesBestEffort(false); + } + + public Map pullCommitProgressFromDataNodesBestEffort( + final boolean includeConsensusQueueReadiness) { final Map dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); - final TPullCommitProgressReq request = new TPullCommitProgressReq(); + final TPullCommitProgressReq request = + new TPullCommitProgressReq() + .setIncludeConsensusQueueReadiness(includeConsensusQueueReadiness); final DataNodeAsyncRequestContext clientHandler = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index ec97a2ff963f..1069eb2b7b7f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -254,6 +254,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceResp; import org.apache.iotdb.mpp.rpc.thrift.TCleanDataNodeCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TConsensusSubscriptionQueueReadiness; import org.apache.iotdb.mpp.rpc.thrift.TConstructSchemaBlackListReq; import org.apache.iotdb.mpp.rpc.thrift.TConstructSchemaBlackListWithTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TConstructViewSchemaBlackListReq; @@ -1736,9 +1737,22 @@ public TPullCommitProgressResp pullCommitProgress(TPullCommitProgressReq req) { SubscriptionAgent.broker().collectAllRegionCommitProgress(dataNodeId); final Map subscriptionProgress = SubscriptionAgent.broker().collectAllProgressSnapshots(); - return new TPullCommitProgressResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())) - .setCommitRegionProgress(regionProgress) - .setSubscriptionProgress(subscriptionProgress); + final TPullCommitProgressResp response = + new TPullCommitProgressResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())) + .setCommitRegionProgress(regionProgress) + .setSubscriptionProgress(subscriptionProgress); + if (req.isSetIncludeConsensusQueueReadiness() && req.isIncludeConsensusQueueReadiness()) { + final List consensusQueueReadiness = + new ArrayList<>(); + for (final ConsensusSubscriptionSetupHandler.QueueReadiness readiness : + ConsensusSubscriptionSetupHandler.getQueueReadiness()) { + consensusQueueReadiness.add( + new TConsensusSubscriptionQueueReadiness( + readiness.topicName(), readiness.consumerGroupId(), readiness.tableModel())); + } + response.setConsensusQueueReadiness(consensusQueueReadiness); + } + return response; } catch (Exception e) { LOGGER.warn( DataNodeMiscMessages.MISC_LOG_ERROR_OCCURRED_WHEN_PULLING_COMMIT_PROGRESS_48C12E4B, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTask.java index 15d5f6673f81..ed83a7d4cc60 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTask.java @@ -141,6 +141,12 @@ public static void buildDetailsTSBlock( progressInfo.isSetReadyOnDns() ? progressInfo.getReadyOnDns().toString() : Collections.emptyList().toString()); + writeText( + builder, + 21, + progressInfo.isSetQueuesBoundOnDns() + ? progressInfo.getQueuesBoundOnDns().toString() + : Collections.emptyList().toString()); builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java index 654030eaa2be..4b79ceaabdc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java @@ -79,6 +79,9 @@ public class ConsensusSubscriptionSetupHandler { private static final long FIRST_CONSENSUS_SEARCH_INDEX = 1L; + /** A point-in-time result that all matching local Region queues are bound for a subscription. */ + public record QueueReadiness(String topicName, String consumerGroupId, boolean tableModel) {} + /** Last-known preferred writer node ID per region, used to detect routing changes. */ private static final ConcurrentHashMap lastKnownPreferredWriter = new ConcurrentHashMap<>(); @@ -389,6 +392,102 @@ public static boolean isConsensusBasedTopic(final String topicName, final boolea } } + public static Set getQueueReadiness() { + final Set readySubscriptions = new LinkedHashSet<>(); + for (final String consumerGroupId : + SubscriptionAgent.consumer().getAllSubscriptions().keySet()) { + SubscriptionAgent.consumer() + .executeWithConsumerGroupLock( + consumerGroupId, + () -> { + final boolean isTableModel = + SubscriptionAgent.consumer().isTableModel(consumerGroupId); + for (final String topicName : + SubscriptionAgent.consumer() + .getTopicNamesSubscribedByConsumerGroup(consumerGroupId)) { + if (!isConsensusBasedTopic(topicName, isTableModel)) { + continue; + } + try { + if (!areAllLocalQueuesBound(consumerGroupId, topicName, isTableModel)) { + continue; + } + readySubscriptions.add( + new QueueReadiness(topicName, consumerGroupId, isTableModel)); + } catch (final RuntimeException ignored) { + // An uncertain readiness probe must not suppress the normal progress snapshot. + } + } + }); + } + return readySubscriptions; + } + + static boolean isQueueBindingComplete( + final Set matchingRegionIds, + final Set boundRegionIds, + final boolean hasPendingPeerCreationCallback, + final boolean regionSetStable) { + return !hasPendingPeerCreationCallback + && regionSetStable + && boundRegionIds.containsAll(matchingRegionIds); + } + + private static boolean areAllLocalQueuesBound( + final String consumerGroupId, final String topicName, final boolean isTableModel) { + final IConsensus dataRegionConsensus = DataRegionConsensusImpl.getInstance(); + if (!(dataRegionConsensus instanceof IoTConsensus)) { + return false; + } + + final Map topicConfigs = + SubscriptionAgent.topic().getTopicConfigs(Collections.singleton(topicName), isTableModel); + final TopicConfig topicConfig = topicConfigs.get(topicName); + if (topicConfig == null) { + return false; + } + + final IoTConsensus ioTConsensus = (IoTConsensus) dataRegionConsensus; + final Set regionIdsBeforeCheck = getLocalDataRegionIds(ioTConsensus); + final Set matchingRegionIds = new LinkedHashSet<>(); + boolean hasPendingPeerCreationCallback = false; + for (final ConsensusGroupId groupId : ioTConsensus.getAllConsensusGroupIds()) { + if (!(groupId instanceof DataRegionId)) { + continue; + } + final IoTConsensusServerImpl serverImpl = ioTConsensus.getImpl(groupId); + final DataRegion dataRegion = + StorageEngine.getInstance().getDataRegion((DataRegionId) groupId); + if (serverImpl == null || dataRegion == null) { + return false; + } + if (matchesTopicDataRegion(dataRegion.getDatabaseName(), topicConfig, isTableModel)) { + // A pending callback may still be responsible for binding this matching queue. + hasPendingPeerCreationCallback |= ioTConsensus.isPeerCreationCallbackPending(groupId); + matchingRegionIds.add(groupId.toString()); + } + } + + final Set boundRegionIds = + SubscriptionAgent.broker() + .getConsensusQueueRegionIds(consumerGroupId, Collections.singleton(topicName)) + .getOrDefault(topicName, Collections.emptySet()); + final boolean regionSetStable = + regionIdsBeforeCheck.equals(getLocalDataRegionIds(ioTConsensus)); + return isQueueBindingComplete( + matchingRegionIds, boundRegionIds, hasPendingPeerCreationCallback, regionSetStable); + } + + private static Set getLocalDataRegionIds(final IoTConsensus ioTConsensus) { + final Set regionIds = new LinkedHashSet<>(); + for (final ConsensusGroupId groupId : ioTConsensus.getAllConsensusGroupIds()) { + if (groupId instanceof DataRegionId) { + regionIds.add(groupId.toString()); + } + } + return regionIds; + } + private static boolean isConsensusBasedTopicRequired( final String topicName, final boolean isTableModel) { final String topicMode = SubscriptionAgent.topic().getTopicMode(topicName, isTableModel); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTaskTest.java index e3235c77e142..4b5385778c8f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTaskTest.java @@ -39,25 +39,26 @@ public class ShowSubscriptionsTaskTest { public void testBuildDetailsTSBlock() throws Exception { final TSubscriptionProgressInfo progressInfo = new TSubscriptionProgressInfo( - "topic", - "group", - "DataRegion[1]", - 3, - true, - true, - "CATCHING_UP", - 5L, - 20L, - 6L, - 2L, - 1L, - 2L, - 100L, - 81L, - 123L, - 120L, - "consumer", - 4L); + "topic", + "group", + "DataRegion[1]", + 3, + true, + true, + "CATCHING_UP", + 5L, + 20L, + 6L, + 2L, + 1L, + 2L, + 100L, + 81L, + 123L, + 120L, + "consumer", + 4L) + .setQueuesBoundOnDns(Collections.singletonList(3)); final SettableFuture future = SettableFuture.create(); ShowSubscriptionsTask.buildDetailsTSBlock(Collections.singletonList(progressInfo), future); @@ -71,6 +72,9 @@ public void testBuildDetailsTSBlock() throws Exception { ColumnHeaderConstant.SEEK_GENERATION, result.getResultSetHeader().getRespColumns().get(19)); assertEquals( ColumnHeaderConstant.READY_ON_DNS, result.getResultSetHeader().getRespColumns().get(20)); + assertEquals( + ColumnHeaderConstant.QUEUES_BOUND_ON_DNS, + result.getResultSetHeader().getRespColumns().get(21)); assertEquals(1, resultSet.getPositionCount()); assertEquals("topic_group", resultSet.getColumn(0).getBinary(0).toString()); assertEquals("CATCHING_UP", resultSet.getColumn(5).getBinary(0).toString()); @@ -79,5 +83,6 @@ public void testBuildDetailsTSBlock() throws Exception { assertEquals(20L, resultSet.getColumn(9).getLong(0)); assertEquals(4L, resultSet.getColumn(19).getLong(0)); assertEquals("[]", resultSet.getColumn(20).getBinary(0).toString()); + assertEquals("[3]", resultSet.getColumn(21).getBinary(0).toString()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java index 19f944b52d08..17e8fbd19e24 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java @@ -93,6 +93,29 @@ public void testRuntimeUpdatesAreIgnoredWhenSubscriptionIsDisabled() { ConsensusSubscriptionSetupHandler.onRegionRouteChanged(null, 0); } + @Test + public void testQueueBindingReadinessRequiresEveryStableRegionAndCompletedCallbacks() { + final Set matchingRegionIds = new LinkedHashSet<>(Arrays.asList("1", "2")); + final Set allBoundRegionIds = new LinkedHashSet<>(Arrays.asList("1", "2")); + final Set partiallyBoundRegionIds = Collections.singleton("1"); + + assertTrue( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + matchingRegionIds, allBoundRegionIds, false, true)); + assertFalse( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + matchingRegionIds, partiallyBoundRegionIds, false, true)); + assertFalse( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + matchingRegionIds, allBoundRegionIds, true, true)); + assertFalse( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + matchingRegionIds, allBoundRegionIds, false, false)); + assertTrue( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + Collections.emptySet(), Collections.emptySet(), false, true)); + } + @Test public void testSingleTopicSetupFailurePropagates() { SubscriptionException failure = null; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index d439b3560946..92f81eac3fd2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -157,6 +157,7 @@ private ColumnHeaderConstant() { public static final String LAST_CONSUMER_ID = "LastConsumerId"; public static final String SEEK_GENERATION = "SeekGeneration"; public static final String READY_ON_DNS = "ready_on_dns"; + public static final String QUEUES_BOUND_ON_DNS = "queues_bound_on_dns"; // show cluster status public static final String NODE_TYPE_CONFIG_NODE = "ConfigNode"; @@ -740,7 +741,8 @@ private ColumnHeaderConstant() { new ColumnHeader(LAST_POLL_TIME, TSDataType.INT64), new ColumnHeader(LAST_CONSUMER_ID, TSDataType.TEXT), new ColumnHeader(SEEK_GENERATION, TSDataType.INT64), - new ColumnHeader(READY_ON_DNS, TSDataType.TEXT)); + new ColumnHeader(READY_ON_DNS, TSDataType.TEXT), + new ColumnHeader(QUEUES_BOUND_ON_DNS, TSDataType.TEXT)); public static final List selectIntoColumnHeaders = ImmutableList.of( diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 41da9c53b000..643bff4b22ae 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1116,6 +1116,7 @@ struct TSubscriptionProgressInfo { 18: required string lastConsumerId 19: required i64 seekGeneration 20: optional list readyOnDns + 21: optional list queuesBoundOnDns // DataNodes reporting all matching local queues bound. } struct TDropSubscriptionReq { diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index aaf76ff73418..795fb2deb9d0 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -673,12 +673,21 @@ struct TPushConsumerGroupMetaRespExceptionMessage { } struct TPullCommitProgressReq { + 1: optional bool includeConsensusQueueReadiness // Request a point-in-time queue binding snapshot. +} + +// One entry means all matching local DataRegion queues for this subscription are bound. +struct TConsensusSubscriptionQueueReadiness { + 1: required string topicName + 2: required string consumerGroupId + 3: required bool tableModel } struct TPullCommitProgressResp { 1: required common.TSStatus status 2: optional map commitRegionProgress 3: optional map subscriptionProgress + 4: optional list consensusQueueReadiness } struct TSyncSubscriptionProgressReq {