Add ut about empty memtable with 1000 columns

Signed-off-by: JaySon-Huang <[email protected]>

Author: JaySon-Huang

dbms/src/Columns/ColumnString.cpp

@@ -302,8 +302,10 @@ void ColumnString::reserveWithStrategy(size_t n, IColumn::ReserveStrategy strate
         chars.reserve(n * APPROX_STRING_SIZE);
         break;
     case IColumn::ReserveStrategy::ScaleFactor1_5:
-        offsets.reserve_exact(n / 2 * 3);
-        chars.reserve_exact(n * APPROX_STRING_SIZE / 2 * 3);
+        // offsets.reserve_exact(n / 2 * 3);
+        // chars.reserve_exact(n * APPROX_STRING_SIZE / 2 * 3);
+        offsets.resize(n / 2 * 3);
+        chars.resize(n / 2 * APPROX_STRING_SIZE * 3);
         break;
     }
 }

dbms/src/Core/Block.cpp

@@ -370,7 +370,10 @@ MutableColumns Block::cloneEmptyColumns() const
     size_t num_columns = data.size();
     MutableColumns columns(num_columns);
     for (size_t i = 0; i < num_columns; ++i)
+    {
         columns[i] = data[i].column ? data[i].column->cloneEmpty() : data[i].type->createColumn();
+        columns[i]->reserveWithStrategy(0, IColumn::ReserveStrategy::ScaleFactor1_5);
+    }
     return columns;
 }
 

dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.cpp

@@ -123,6 +123,19 @@ ColumnFile::AppendResult ColumnFileInMemory::append(
         max_capacity,
         rows,
         bytes);
+    for (size_t i = 0; i < cache->block.columns(); ++i)
+    {
+        const auto & cache_col = *cache->block.getByPosition(i).column;
+        LOG_INFO(
+            Logger::get(""),
+            "col i={} name={} type={} rows={} bytes={} alloc_bytes={}",
+            i,
+            cache->block.getByPosition(i).name,
+            cache_col.getName(),
+            cache_col.size(),
+            cache_col.byteSize(),
+            cache_col.allocatedBytes());
+    }
     return AppendResult{true, new_alloc_block_bytes};
 }
 

dbms/src/Storages/DeltaMerge/ColumnFile/tests/gtest_dm_column_file.cpp

@@ -38,6 +38,8 @@
 #include <ext/scope_guard.h>
 #include <vector>
 
+#include "Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.h"
+
 namespace DB::DM::tests
 {
 
@@ -228,6 +230,35 @@ try
 }
 CATCH
 
+TEST_P(ColumnFileTest, CreateColumnFileInMemory)
+try
+{
+    size_t num_rows_write = 10;
+    Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false);
+    for (Int64 i = 0; i < 1000; ++i)
+    {
+        auto null_col = ColumnNullable::create(ColumnString::create(), ColumnUInt8::create());
+        for (size_t j = 0; j < num_rows_write; ++j)
+        {
+            null_col->insertDefault();
+        }
+        block.insert(ColumnWithTypeAndName{
+            std::move(null_col),
+            std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>()),
+            fmt::format("field_{}", i),
+        });
+    }
+    // A 1000 columns empty block takes about 238KB
+    auto schema = ColumnFileSchema(block);
+    ColumnFileInMemoryPtr cf_in_mem = std::make_shared<ColumnFileInMemory>(std::make_shared<ColumnFileSchema>(schema));
+    LOG_INFO(
+        Logger::get(),
+        "ColumnFileInMemory, bytes={} alloc_bytes={}",
+        cf_in_mem->getBytes(),
+        cf_in_mem->getAllocateBytes());
+}
+CATCH
+
 TEST_P(ColumnFileTest, ReadColumns)
 try
 {

dbms/src/Storages/DeltaMerge/tests/gtest_segment.cpp

@@ -665,9 +665,10 @@ TEST_F(SegmentOperationTest, TestMassiveSegment)
 try
 {
     const size_t level = 1;
+    const size_t segment_range_size = 500;
     for (size_t lvl = 0; lvl < level; ++lvl)
     {
-        size_t num_expected_segs = 500;
+        size_t num_expected_segs = 200;
         // size_t num_expected_segs = 10;
         size_t progress_interval = 100;
         const auto lvl_beg_seg_id = segments.rbegin()->first;
@@ -678,7 +679,7 @@ try
         auto next_split_seg_id = lvl_beg_seg_id;
         for (size_t i = 0; i < num_expected_segs; ++i)
         {
-            auto split_point = (lvl * num_expected_segs + 1 + i) * 500;
+            auto split_point = (lvl * num_expected_segs + 1 + i) * segment_range_size;
             auto n_seg_id = splitSegmentAt(next_split_seg_id, split_point, Segment::SplitMode::Logical);
             ASSERT_TRUE(n_seg_id.has_value()) << fmt::format("i={} sp={}", i, split_point);
             next_split_seg_id = *n_seg_id;
@@ -710,7 +711,8 @@ try
             if (seg_id < lvl_beg_seg_id)
                 continue; // skip segments created in previous levels
 
-            auto write_rows = 500;
+            size_t write_rows = 4;
+            size_t write_rows_sub = 2;
             if (round % progress_interval == 0)
             {
                 auto mu = get_process_mem_usage();
@@ -722,29 +724,34 @@ try
                     write_rows * round,
                     mu.resident_mb);
             }
-            writeToCache(
-                seg_id,
-                write_rows,
-                /* start_at */ lvl * num_expected_segs * write_rows + round * write_rows,
-                false,
-                std::nullopt);
-            LOG_INFO(
-                log,
-                "lvl={} round={} seg_id={} written_rows={} mem_tbl_bytes={}",
-                lvl,
-                round,
-                seg_id,
-                write_rows,
-                segments[seg_id]->getDelta()->getTotalCacheBytes());
+            for (size_t k = 0; k < 2; ++k)
+            {
+                writeToCache(
+                    seg_id,
+                    write_rows_sub,
+                    /* start_at */ lvl * num_expected_segs * segment_range_size + round * segment_range_size,
+                    false,
+                    std::nullopt);
+                LOG_INFO(
+                    log,
+                    "lvl={} round={} k={} seg_id={} written_rows={} mem_tbl_bytes={} mem_tbl_alloc_bytes={}",
+                    lvl,
+                    round,
+                    k,
+                    seg_id,
+                    write_rows,
+                    segments[seg_id]->getDelta()->getTotalCacheBytes(),
+                    segments[seg_id]->getDelta()->getTotalAllocatedBytes());
+            }
             round++;
         }
         {
             auto mu = get_process_mem_usage();
             LOG_INFO(
                 log,
                 "TestMassiveSegment done, segments.size()={} lvl={} mem_resident_set={:.3f}MB",
-                lvl,
                 segments.size(),
+                lvl,
                 mu.resident_mb);
         }
     }