search

StarRocks / starrocks

The world's fastest open query engine for sub-second analytics both on and off the data lakehouse. With the flexibility to support nearly any scenario, StarRocks provides best-in-class performance for multi-dimensional analytics, real-time analytics, and ad-hoc queries. A Linux Foundation project.
https://starrocks.io
Apache License 2.0
8.98k stars 1.8k forks source link

[Bug] Routine load mem leak #6957

Closed yongbingwang closed 2 years ago

yongbingwang commented 2 years ago

Steps to reproduce the behavior (Required)

  1. CREATE TABLE '...' 
    CREATE TABLE `primary_200cols_0par_300tablets` (
`k0` date NULL COMMENT "",
`k1` varchar(146) NULL COMMENT "",
`k2` date NULL COMMENT "",
`k3` varchar(65533) NULL COMMENT "",
`k4` datetime NULL COMMENT "",
`k5` datetime NULL COMMENT "",
`k6` varchar(65533) NULL COMMENT "",
`k7` decimal64(7, 1) NULL COMMENT "",
`k8` bigint(20) NULL COMMENT "",
`k9` datetime NULL COMMENT "",
`k10` decimal64(8, 4) NULL COMMENT "",
`k11` varchar(65533) NULL COMMENT "",
`k12` char(126) NULL COMMENT "",
`k13` int(11) NULL COMMENT "",
`k14` decimal64(4, 2) NULL COMMENT "",
`k15` datetime NULL COMMENT "",
`k16` datetime NULL COMMENT "",
`k17` varchar(65533) NULL COMMENT "",
`k18` char(40) NULL COMMENT "",
`k19` varchar(165) NULL COMMENT "",
`k20` varchar(65533) NULL COMMENT "",
`k21` char(228) NULL COMMENT "",
`k22` date NULL COMMENT "",
`k23` int(11) NULL COMMENT "",
`k24` int(11) NULL COMMENT "",
`k25` varchar(810) NULL COMMENT "",
`k26` decimal64(11, 10) NULL COMMENT "",
`k27` varchar(65533) NULL COMMENT "",
`k28` varchar(56) NULL COMMENT "",
`k29` datetime NULL COMMENT "",
`k30` varchar(820) NULL COMMENT "",
`k31` varchar(17) NULL COMMENT "",
`k32` varchar(92) NULL COMMENT "",
`k33` datetime NULL COMMENT "",
`k34` char(248) NULL COMMENT "",
`k35` varchar(950) NULL COMMENT "",
`k36` varchar(65533) NULL COMMENT "",
`k37` varchar(65533) NULL COMMENT "",
`k38` date NULL COMMENT "",
`k39` datetime NULL COMMENT "",
`v0` date NULL COMMENT "",
`v1` largeint(40) NULL COMMENT "",
`v2` varchar(65533) NULL COMMENT "",
`v3` datetime NULL COMMENT "",
`v4` datetime NULL COMMENT "",
`v5` varchar(65533) NULL COMMENT "",
`v6` varchar(65533) NULL COMMENT "",
`v7` decimal64(2, 0) NULL COMMENT "",
`v8` datetime NULL COMMENT "",
`v9` char(109) NULL COMMENT "",
`v10` varchar(49) NULL COMMENT "",
`v11` char(135) NULL COMMENT "",
`v12` varchar(986) NULL COMMENT "",
`v13` varchar(65490) NULL COMMENT "",
`v14` varchar(65533) NULL COMMENT "",
`v15` decimal64(12, 1) NULL COMMENT "",
`v16` decimal128(27, 1) NULL COMMENT "",
`v17` varchar(65533) NULL COMMENT "",
`v18` varchar(65519) NULL COMMENT "",
`v19` varchar(65533) NULL COMMENT "",
`v20` decimal128(32, 17) NULL COMMENT "",
`v21` int(11) NULL COMMENT "",
`v22` varchar(65533) NULL COMMENT "",
`v23` tinyint(4) NULL COMMENT "",
`v24` varchar(65533) NULL COMMENT "",
`v25` varchar(65533) NULL COMMENT "",
`v26` bigint(20) NULL COMMENT "",
`v27` varchar(65533) NULL COMMENT "",
`v28` varchar(135) NULL COMMENT "",
`v29` char(165) NULL COMMENT "",
`v30` char(202) NULL COMMENT "",
`v31` largeint(40) NULL COMMENT "",
`v32` int(11) NULL COMMENT "",
`v33` varchar(65533) NULL COMMENT "",
`v34` largeint(40) NULL COMMENT "",
`v35` char(234) NULL COMMENT "",
`v36` varchar(12) NULL COMMENT "",
`v37` varchar(886) NULL COMMENT "",
`v38` datetime NULL COMMENT "",
`v39` largeint(40) NULL COMMENT "",
`v40` char(149) NULL COMMENT "",
`v41` varchar(65533) NULL COMMENT "",
`v42` smallint(6) NULL COMMENT "",
`v43` varchar(65533) NULL COMMENT "",
`v44` date NULL COMMENT "",
`v45` varchar(74) NULL COMMENT "",
`v46` largeint(40) NULL COMMENT "",
`v47` datetime NULL COMMENT "",
`v48` int(11) NULL COMMENT "",
`v49` tinyint(4) NULL COMMENT "",
`v50` varchar(65415) NULL COMMENT "",
`v51` varchar(160) NULL COMMENT "",
`v52` char(10) NULL COMMENT "",
`v53` varchar(65458) NULL COMMENT "",
`v54` int(11) NULL COMMENT "",
`v55` int(11) NULL COMMENT "",
`v56` decimal128(31, 14) NULL COMMENT "",
`v57` varchar(95) NULL COMMENT "",
`v58` varchar(65533) NULL COMMENT "",
`v59` int(11) NULL COMMENT "",
`v60` decimal64(18, 10) NULL COMMENT "",
`v61` tinyint(4) NULL COMMENT "",
`v62` decimal128(27, 6) NULL COMMENT "",
`v63` decimal128(27, 5) NULL COMMENT "",
`v64` varchar(69) NULL COMMENT "",
`v65` largeint(40) NULL COMMENT "",
`v66` date NULL COMMENT "",
`v67` largeint(40) NULL COMMENT "",
`v68` bigint(20) NULL COMMENT "",
`v69` varchar(65533) NULL COMMENT "",
`v70` smallint(6) NULL COMMENT "",
`v71` int(11) NULL COMMENT "",
`v72` tinyint(4) NULL COMMENT "",
`v73` decimal64(6, 2) NULL COMMENT "",
`v74` datetime NULL COMMENT "",
`v75` date NULL COMMENT "",
`v76` date NULL COMMENT "",
`v77` decimal64(15, 12) NULL COMMENT "",
`v78` largeint(40) NULL COMMENT "",
`v79` bigint(20) NULL COMMENT "",
`v80` varchar(818) NULL COMMENT "",
`v81` decimal128(29, 28) NULL COMMENT "",
`v82` varchar(65533) NULL COMMENT "",
`v83` varchar(65533) NULL COMMENT "",
`v84` char(20) NULL COMMENT "",
`v85` varchar(65418) NULL COMMENT "",
`v86` largeint(40) NULL COMMENT "",
`v87` smallint(6) NULL COMMENT "",
`v88` int(11) NULL COMMENT "",
`v89` varchar(65533) NULL COMMENT "",
`v90` varchar(65533) NULL COMMENT "",
`v91` bigint(20) NULL COMMENT "",
`v92` largeint(40) NULL COMMENT "",
`v93` varchar(838) NULL COMMENT "",
`v94` datetime NULL COMMENT "",
`v95` varchar(65447) NULL COMMENT "",
`v96` datetime NULL COMMENT "",
`v97` largeint(40) NULL COMMENT "",
`v98` decimal128(34, 30) NULL COMMENT "",
`v99` varchar(139) NULL COMMENT "",
`v100` char(209) NULL COMMENT "",
`v101` bigint(20) NULL COMMENT "",
`v102` decimal128(25, 24) NULL COMMENT "",
`v103` bigint(20) NULL COMMENT "",
`v104` char(194) NULL COMMENT "",
`v105` varchar(65533) NULL COMMENT "",
`v106` bigint(20) NULL COMMENT "",
`v107` varchar(65533) NULL COMMENT "",
`v108` decimal128(34, 1) NULL COMMENT "",
`v109` bigint(20) NULL COMMENT "",
`v110` varchar(65506) NULL COMMENT "",
`v111` varchar(65533) NULL COMMENT "",
`v112` decimal128(36, 8) NULL COMMENT "",
`v113` varchar(65533) NULL COMMENT "",
`v114` int(11) NULL COMMENT "",
`v115` largeint(40) NULL COMMENT "",
`v116` varchar(65500) NULL COMMENT "",
`v117` varchar(29) NULL COMMENT "",
`v118` varchar(157) NULL COMMENT "",
`v119` int(11) NULL COMMENT "",
`v120` datetime NULL COMMENT "",
`v121` char(221) NULL COMMENT "",
`v122` decimal128(23, 6) NULL COMMENT "",
`v123` varchar(65533) NULL COMMENT "",
`v124` varchar(65533) NULL COMMENT "",
`v125` varchar(65533) NULL COMMENT "",
`v126` decimal64(8, 5) NULL COMMENT "",
`v127` varchar(65477) NULL COMMENT "",
`v128` varchar(65533) NULL COMMENT "",
`v129` date NULL COMMENT "",
`v130` varchar(14) NULL COMMENT "",
`v131` largeint(40) NULL COMMENT "",
`v132` char(50) NULL COMMENT "",
`v133` varchar(65495) NULL COMMENT "",
`v134` largeint(40) NULL COMMENT "",
`v135` datetime NULL COMMENT "",
`v136` varchar(197) NULL COMMENT "",
`v137` bigint(20) NULL COMMENT "",
`v138` char(238) NULL COMMENT "",
`v139` varchar(42) NULL COMMENT "",
`v140` int(11) NULL COMMENT "",
`v141` largeint(40) NULL COMMENT "",
`v142` decimal64(8, 2) NULL COMMENT "",
`v143` char(255) NULL COMMENT "",
`v144` varchar(807) NULL COMMENT "",
`v145` varchar(65484) NULL COMMENT "",
`v146` varchar(65533) NULL COMMENT "",
`v147` varchar(65533) NULL COMMENT "",
`v148` varchar(810) NULL COMMENT "",
`v149` varchar(65533) NULL COMMENT "",
`v150` int(11) NULL COMMENT "",
`v151` varchar(87) NULL COMMENT "",
`v152` bigint(20) NULL COMMENT "",
`v153` varchar(65533) NULL COMMENT "",
`v154` decimal128(25, 9) NULL COMMENT "",
`v155` char(38) NULL COMMENT "",
`v156` char(79) NULL COMMENT "",
`v157` varchar(109) NULL COMMENT "",
`v158` bigint(20) NULL COMMENT "",
`v159` varchar(65533) NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`k0`, `k1`, `k2`, `k3`, `k4`, `k5`, `k6`)
COMMENT "OLAP"
DISTRIBUTED BY HASH(`k0`, `k1`, `k2`, `k3`) BUCKETS 100
PROPERTIES (
"replication_num" = "3",
"in_memory" = "false",
"storage_format" = "DEFAULT",
"enable_persistent_index" = "false"
);
  2. routine load '....' 
    CREATE ROUTINE LOAD routine_load_primary_200cols_con5 on primary_200cols_0par_300tablets COLUMNS TERMINATED BY '\t' PROPERTIES ("desired_concurrent_number"="5","max_error_number"="1000","max_batch_interval"="5") FROM KAFKA ("kafka_broker_list"="[xxx.xxx.xxx.xxx](http://172.26.194.239:9092/)","kafka_topic"="routine_load_200cols_10par_test","property.kafka_default_offsets"="OFFSET_BEGINNING");

    Expected behavior (Required)

Real behavior (Required)

starrocks_be_chunk_allocator_mem_bytes 36279056
starrocks_be_clone_mem_bytes 0
starrocks_be_column_pool_mem_bytes 241893520
starrocks_be_compaction_mem_bytes 0
starrocks_be_consistency_mem_bytes 0
starrocks_be_load_mem_bytes 0
starrocks_be_process_mem_bytes 48795380640
starrocks_be_query_mem_bytes 0
starrocks_be_schema_change_mem_bytes 0
starrocks_be_storage_page_cache_mem_bytes 0
starrocks_be_tablet_meta_mem_bytes 432563912
starrocks_be_tcmalloc_bytes_in_use 48770672056
starrocks_be_tcmalloc_pageheap_unmapped_bytes 2727936
starrocks_be_tcmalloc_total_bytes_reserved 51936796672
starrocks_be_update_mem_bytes 0

StarRocks version (Required)

kangkaisen commented 2 years ago

8222b08dc71e23 introduced this bug