Skip to content

Commit 6d81f75

Browse files
mergify[bot]mergify[bot]
authored
[BugFix] Fix physical partition for rebalance (backport #46402) (backport #52182) (#52214)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
1 parent 87e46cb commit 6d81f75

File tree

9 files changed

+162
-137
lines changed

9 files changed

+162
-137
lines changed

‎fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1851,7 +1851,9 @@ public TTableDescriptor toThrift(List<ReferencedPartitionInfo> partitions) {
18511851
public long getRowCount() {
18521852
long rowCount = 0;
18531853
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
1854-
rowCount += entry.getValue().getBaseIndex().getRowCount();
1854+
for (PhysicalPartition partition : entry.getValue().getSubPartitions()) {
1855+
rowCount += partition.getBaseIndex().getRowCount();
1856+
}
18551857
}
18561858
return rowCount;
18571859
}
@@ -3151,9 +3153,11 @@ public void removeTabletsFromInvertedIndex() {
31513153
TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex();
31523154
Collection<Partition> allPartitions = getAllPartitions();
31533155
for (Partition partition : allPartitions) {
3154-
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
3155-
for (Tablet tablet : index.getTablets()) {
3156-
invertedIndex.deleteTablet(tablet.getId());
3156+
for (PhysicalPartition subPartition : partition.getSubPartitions()) {
3157+
for (MaterializedIndex index : subPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
3158+
for (Tablet tablet : index.getTablets()) {
3159+
invertedIndex.deleteTablet(tablet.getId());
3160+
}
31573161
}
31583162
}
31593163
}

‎fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java‎

Lines changed: 111 additions & 109 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/com/starrocks/load/ExportJob.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
import com.starrocks.catalog.Database;
5555
import com.starrocks.catalog.MaterializedIndex;
5656
import com.starrocks.catalog.MysqlTable;
57-
import com.starrocks.catalog.Partition;
57+
import com.starrocks.catalog.PhysicalPartition;
5858
import com.starrocks.catalog.PrimitiveType;
5959
import com.starrocks.catalog.Replica;
6060
import com.starrocks.catalog.Table;
@@ -361,7 +361,7 @@ private void genTaskFragments(List<PlanFragment> fragments, List<ScanNode> scanN
361361
TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId);
362362
long dataSize = 0L;
363363
if (tabletMeta.isLakeTablet()) {
364-
Partition partition = exportTable.getPartition(tabletMeta.getPartitionId());
364+
PhysicalPartition partition = exportTable.getPhysicalPartition(tabletMeta.getPhysicalPartitionId());
365365
if (partition != null) {
366366
MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId());
367367
if (index != null) {

‎fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import com.starrocks.catalog.Partition;
2828
import com.starrocks.catalog.PartitionInfo;
2929
import com.starrocks.catalog.PartitionType;
30+
import com.starrocks.catalog.PhysicalPartition;
3031
import com.starrocks.catalog.SinglePartitionInfo;
3132
import com.starrocks.catalog.Table;
3233
import com.starrocks.catalog.Tablet;
@@ -408,9 +409,11 @@ private void gc(boolean isReplay) {
408409

409410
Partition partition = targetTable.getPartition(pid);
410411
if (partition != null) {
411-
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
412-
// hash set is able to deduplicate the elements
413-
sourceTablets.addAll(index.getTablets());
412+
for (PhysicalPartition subPartition : partition.getSubPartitions()) {
413+
for (MaterializedIndex index : subPartition.getMaterializedIndices(
414+
MaterializedIndex.IndexExtState.ALL)) {
415+
sourceTablets.addAll(index.getTablets());
416+
}
414417
}
415418
targetTable.dropTempPartition(partition.getName(), true);
416419
} else {
@@ -470,8 +473,10 @@ private void doCommit(boolean isReplay) {
470473
Set<Tablet> sourceTablets = Sets.newHashSet();
471474
sourcePartitionNames.forEach(name -> {
472475
Partition partition = targetTable.getPartition(name);
473-
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
474-
sourceTablets.addAll(index.getTablets());
476+
for (PhysicalPartition subPartition : partition.getSubPartitions()) {
477+
for (MaterializedIndex index : subPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
478+
sourceTablets.addAll(index.getTablets());
479+
}
475480
}
476481
});
477482
long sumSourceRows = job.getSourcePartitionIds().stream()

‎fe/fe-core/src/main/java/com/starrocks/load/PartitionUtils.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.starrocks.catalog.Partition;
2525
import com.starrocks.catalog.PartitionInfo;
2626
import com.starrocks.catalog.PartitionKey;
27+
import com.starrocks.catalog.PhysicalPartition;
2728
import com.starrocks.catalog.RangePartitionInfo;
2829
import com.starrocks.catalog.Table;
2930
import com.starrocks.catalog.Tablet;
@@ -149,9 +150,12 @@ public static void createAndAddTempPartitionsForTable(Database db, OlapTable tar
149150
public static void clearTabletsFromInvertedIndex(List<Partition> partitions) {
150151
TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex();
151152
for (Partition partition : partitions) {
152-
for (MaterializedIndex materializedIndex : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
153-
for (Tablet tablet : materializedIndex.getTablets()) {
154-
invertedIndex.deleteTablet(tablet.getId());
153+
for (PhysicalPartition subPartition : partition.getSubPartitions()) {
154+
for (MaterializedIndex materializedIndex : subPartition.getMaterializedIndices(
155+
MaterializedIndex.IndexExtState.ALL)) {
156+
for (Tablet tablet : materializedIndex.getTablets()) {
157+
invertedIndex.deleteTablet(tablet.getId());
158+
}
155159
}
156160
}
157161
}

‎fe/fe-core/src/main/java/com/starrocks/planner/MetaScanNode.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
import com.starrocks.catalog.LocalTablet;
2121
import com.starrocks.catalog.MaterializedIndex;
2222
import com.starrocks.catalog.OlapTable;
23-
import com.starrocks.catalog.Partition;
23+
import com.starrocks.catalog.PhysicalPartition;
2424
import com.starrocks.catalog.Replica;
2525
import com.starrocks.catalog.Tablet;
2626
import com.starrocks.lake.LakeTablet;
@@ -65,8 +65,8 @@ public MetaScanNode(PlanNodeId id, TupleDescriptor desc, OlapTable olapTable,
6565
}
6666

6767
public void computeRangeLocations() {
68-
Collection<Partition> partitions = olapTable.getPartitions();
69-
for (Partition partition : partitions) {
68+
Collection<PhysicalPartition> partitions = olapTable.getPhysicalPartitions();
69+
for (PhysicalPartition partition : partitions) {
7070
MaterializedIndex index = partition.getBaseIndex();
7171
int schemaHash = olapTable.getSchemaHashByIndexId(index.getId());
7272
List<Tablet> tablets = index.getTablets();

‎fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4971,10 +4971,12 @@ public void onEraseDatabase(long dbId) {
49714971
public void onErasePartition(Partition partition) {
49724972
// remove tablet in inverted index
49734973
TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex();
4974-
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
4975-
for (Tablet tablet : index.getTablets()) {
4976-
long tabletId = tablet.getId();
4977-
invertedIndex.deleteTablet(tabletId);
4974+
for (PhysicalPartition subPartition : partition.getSubPartitions()) {
4975+
for (MaterializedIndex index : subPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
4976+
for (Tablet tablet : index.getTablets()) {
4977+
long tabletId = tablet.getId();
4978+
invertedIndex.deleteTablet(tabletId);
4979+
}
49784980
}
49794981
}
49804982
}

‎fe/fe-core/src/main/java/com/starrocks/sql/Explain.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.starrocks.catalog.MaterializedIndex;
2525
import com.starrocks.catalog.OlapTable;
2626
import com.starrocks.catalog.Partition;
27+
import com.starrocks.catalog.PhysicalPartition;
2728
import com.starrocks.sql.common.ErrorType;
2829
import com.starrocks.sql.common.StarRocksPlannerException;
2930
import com.starrocks.sql.optimizer.ExpressionContext;
@@ -204,8 +205,10 @@ public OperatorStr visitPhysicalOlapScan(OptExpression optExpression, OperatorPr
204205
int totalTabletsNum = 0;
205206
for (Long partitionId : scan.getSelectedPartitionId()) {
206207
final Partition partition = ((OlapTable) scan.getTable()).getPartition(partitionId);
207-
final MaterializedIndex selectedTable = partition.getIndex(scan.getSelectedIndexId());
208-
totalTabletsNum += selectedTable.getTablets().size();
208+
for (PhysicalPartition subPartition : partition.getSubPartitions()) {
209+
final MaterializedIndex selectedTable = subPartition.getIndex(scan.getSelectedIndexId());
210+
totalTabletsNum += selectedTable.getTablets().size();
211+
}
209212
}
210213
String partitionAndBucketInfo = "partitionRatio: " +
211214
scan.getSelectedPartitionId().size() +

‎fe/fe-core/src/test/java/com/starrocks/clone/DiskAndTabletLoadReBalancerTest.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,8 @@ public void testBalance(@Mocked GlobalStateMgr globalStateMgr) {
150150
result = Lists.newArrayList(table);
151151
minTimes = 0;
152152

153-
GlobalStateMgr.getCurrentState().getLocalMetastore().getPartitionIncludeRecycleBin((OlapTable) any, anyLong);
153+
GlobalStateMgr.getCurrentState().getLocalMetastore()
154+
.getPhysicalPartitionIncludeRecycleBin((OlapTable) any, anyLong);
154155
result = partition;
155156
minTimes = 0;
156157

@@ -312,7 +313,8 @@ public void testBalanceWithSameHost(@Mocked GlobalStateMgr globalStateMgr) {
312313
result = Lists.newArrayList(table);
313314
minTimes = 0;
314315

315-
GlobalStateMgr.getCurrentState().getLocalMetastore().getPartitionIncludeRecycleBin((OlapTable) any, anyLong);
316+
GlobalStateMgr.getCurrentState().getLocalMetastore()
317+
.getPhysicalPartitionIncludeRecycleBin((OlapTable) any, anyLong);
316318
result = partition;
317319
minTimes = 0;
318320

@@ -494,11 +496,13 @@ public void testBalanceBackendTablet(@Mocked GlobalStateMgr globalStateMgr) {
494496
result = Lists.newArrayList(table);
495497
minTimes = 0;
496498

497-
GlobalStateMgr.getCurrentState().getLocalMetastore().getPartitionIncludeRecycleBin((OlapTable) any, partitionId1);
499+
GlobalStateMgr.getCurrentState().getLocalMetastore()
500+
.getPhysicalPartitionIncludeRecycleBin((OlapTable) any, partitionId1);
498501
result = partition1;
499502
minTimes = 0;
500503

501-
GlobalStateMgr.getCurrentState().getLocalMetastore().getPartitionIncludeRecycleBin((OlapTable) any, partitionId2);
504+
GlobalStateMgr.getCurrentState().getLocalMetastore()
505+
.getPhysicalPartitionIncludeRecycleBin((OlapTable) any, partitionId2);
502506
result = partition2;
503507
minTimes = 0;
504508

@@ -685,7 +689,8 @@ public void testBalanceParallel(@Mocked GlobalStateMgr globalStateMgr) {
685689
result = Lists.newArrayList(table);
686690
minTimes = 0;
687691

688-
GlobalStateMgr.getCurrentState().getLocalMetastore().getPartitionIncludeRecycleBin((OlapTable) any, anyLong);
692+
GlobalStateMgr.getCurrentState().getLocalMetastore()
693+
.getPhysicalPartitionIncludeRecycleBin((OlapTable) any, anyLong);
689694
result = partition;
690695
minTimes = 0;
691696

0 commit comments

Comments
 (0)