[opt](memory) Replace boxed Long-keyed maps with fastutil primitive collections in FE

fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java

@@ -69,6 +69,9 @@
 import org.apache.doris.transaction.GlobalTransactionMgr;
 import org.apache.doris.transaction.TransactionState;
 
+import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
@@ -100,9 +103,9 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
 
     // partition id -> (rollup tablet id -> base tablet id)
     @SerializedName(value = "partitionIdToBaseRollupTabletIdMap")
-    protected Map<Long, Map<Long, Long>> partitionIdToBaseRollupTabletIdMap = Maps.newHashMap();
+    protected Long2ObjectOpenHashMap<Map<Long, Long>> partitionIdToBaseRollupTabletIdMap = new Long2ObjectOpenHashMap<>();
     @SerializedName(value = "partitionIdToRollupIndex")
-    protected Map<Long, MaterializedIndex> partitionIdToRollupIndex = Maps.newHashMap();
+    protected Long2ObjectOpenHashMap<MaterializedIndex> partitionIdToRollupIndex = new Long2ObjectOpenHashMap<>();
 
     // rollup and base schema info
     @SerializedName(value = "baseIndexId")
@@ -172,7 +175,7 @@ public RollupJobV2(String rawSql, long jobId, long dbId, long tableId, String ta
 
     public void addTabletIdMap(long partitionId, long rollupTabletId, long baseTabletId) {
         Map<Long, Long> tabletIdMap = partitionIdToBaseRollupTabletIdMap
-                .computeIfAbsent(partitionId, k -> Maps.newHashMap());
+                .computeIfAbsent(partitionId, k -> new Long2LongOpenHashMap());
         tabletIdMap.put(rollupTabletId, baseTabletId);
     }
 

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java

@@ -70,6 +70,9 @@
 import org.apache.doris.transaction.GlobalTransactionMgr;
 import org.apache.doris.transaction.TransactionState;
 
+import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.HashBasedTable;
@@ -109,22 +112,22 @@ public class SchemaChangeJobV2 extends AlterJobV2 implements GsonPostProcessable
     protected Table<Long, Long, MaterializedIndex> partitionIndexMap = HashBasedTable.create();
     // shadow index id -> origin index id
     @SerializedName(value = "indexIdMap")
-    protected Map<Long, Long> indexIdMap = Maps.newHashMap();
+    protected Long2LongOpenHashMap indexIdMap = new Long2LongOpenHashMap();
     // partition id -> origin index id
     @SerializedName(value = "partitionOriginIndexIdMap")
-    private Map<Long, Long> partitionOriginIndexIdMap = Maps.newHashMap();
+    private Long2LongOpenHashMap partitionOriginIndexIdMap = new Long2LongOpenHashMap();
     // shadow index id -> shadow index name(__doris_shadow_xxx)
     @SerializedName(value = "indexIdToName")
-    private Map<Long, String> indexIdToName = Maps.newHashMap();
+    private Long2ObjectOpenHashMap<String> indexIdToName = new Long2ObjectOpenHashMap<>();
     // shadow index id -> index schema
     @SerializedName(value = "indexSchemaMap")
-    protected Map<Long, List<Column>> indexSchemaMap = Maps.newHashMap();
+    protected Long2ObjectOpenHashMap<List<Column>> indexSchemaMap = new Long2ObjectOpenHashMap<>();
     // shadow index id -> (shadow index schema version : schema hash)
     @SerializedName(value = "indexSchemaVersionAndHashMap")
-    protected Map<Long, SchemaVersionAndHash> indexSchemaVersionAndHashMap = Maps.newHashMap();
+    protected Long2ObjectOpenHashMap<SchemaVersionAndHash> indexSchemaVersionAndHashMap = new Long2ObjectOpenHashMap<>();
     // shadow index id -> shadow index short key count
     @SerializedName(value = "indexShortKeyMap")
-    protected Map<Long, Short> indexShortKeyMap = Maps.newHashMap();
+    protected Long2ObjectOpenHashMap<Short> indexShortKeyMap = new Long2ObjectOpenHashMap<>();
 
     // bloom filter info
     @SerializedName(value = "hasBfChange")
@@ -168,7 +171,7 @@ public SchemaChangeJobV2(String rawSql, long jobId, long dbId, long tableId, Str
     public void addTabletIdMap(long partitionId, long shadowIdxId, long shadowTabletId, long originTabletId) {
         Map<Long, Long> tabletMap = partitionIndexTabletMap.get(partitionId, shadowIdxId);
         if (tabletMap == null) {
-            tabletMap = Maps.newHashMap();
+            tabletMap = new Long2LongOpenHashMap();
             partitionIndexTabletMap.put(partitionId, shadowIdxId, tabletMap);
         }
         tabletMap.put(shadowTabletId, originTabletId);

fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java

@@ -19,11 +19,12 @@
 
 import org.apache.doris.persist.gson.GsonPostProcessable;
 
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.collect.Lists;
 import com.google.gson.annotations.SerializedName;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -72,7 +73,7 @@ public enum IndexExtState {
 
     public MaterializedIndex() {
         this.state = IndexState.NORMAL;
-        this.idToTablets = new HashMap<>();
+        this.idToTablets = new Long2ObjectOpenHashMap<>();
         this.tablets = new ArrayList<>();
     }
 
@@ -84,7 +85,7 @@ public MaterializedIndex(long id, IndexState state) {
             this.state = IndexState.NORMAL;
         }
 
-        this.idToTablets = new HashMap<>();
+        this.idToTablets = new Long2ObjectOpenHashMap<>();
         this.tablets = new ArrayList<>();
 
         this.rowCount = -1;

fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

@@ -95,6 +95,8 @@
 import org.apache.doris.thrift.TTableDescriptor;
 import org.apache.doris.thrift.TTableType;
 
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
@@ -177,7 +179,7 @@ public enum OlapTableState {
 
     // index id -> index meta
     @SerializedName(value = "itm", alternate = {"indexIdToMeta"})
-    private Map<Long, MaterializedIndexMeta> indexIdToMeta = Maps.newHashMap();
+    private Long2ObjectOpenHashMap<MaterializedIndexMeta> indexIdToMeta = new Long2ObjectOpenHashMap<>();
     // index name -> index id
     @SerializedName(value = "inti", alternate = {"indexNameToId"})
     private Map<String, Long> indexNameToId = Maps.newHashMap();
@@ -841,7 +843,7 @@ public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restore
 
         // reset all 'indexIdToXXX' map
         Map<Long, MaterializedIndexMeta> origIdxIdToMeta = indexIdToMeta;
-        indexIdToMeta = Maps.newHashMap();
+        indexIdToMeta = new Long2ObjectOpenHashMap<>();
         for (Map.Entry<Long, String> entry : origIdxIdToName.entrySet()) {
             long newIdxId = env.getNextId();
             if (entry.getValue().equals(name)) {
@@ -3859,7 +3861,7 @@ public OlapTable copyTableMeta() {
         table.tempPartitions = new TempPartitions();
 
         table.state = state;
-        table.indexIdToMeta = ImmutableMap.copyOf(indexIdToMeta);
+        table.indexIdToMeta = new Long2ObjectOpenHashMap<>(indexIdToMeta);
         table.indexNameToId = ImmutableMap.copyOf(indexNameToId);
         table.keysType = keysType;
         table.partitionInfo = partitionInfo;

fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java

@@ -25,6 +25,8 @@
 import org.apache.doris.common.FeConstants;
 import org.apache.doris.rpc.RpcException;
 
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -68,14 +70,14 @@ public enum PartitionState {
      * User can do query on them, show them in related 'show' stmt.
      */
     @SerializedName(value = "ivr", alternate = {"idToVisibleRollupIndex"})
-    private Map<Long, MaterializedIndex> idToVisibleRollupIndex = Maps.newHashMap();
+    private Long2ObjectOpenHashMap<MaterializedIndex> idToVisibleRollupIndex = new Long2ObjectOpenHashMap<>();
     /**
      * Shadow indexes are indexes which are not visible to user.
      * Query will not run on these shadow indexes, and user can not see them neither.
      * But load process will load data into these shadow indexes.
      */
     @SerializedName(value = "isi", alternate = {"idToShadowIndex"})
-    private Map<Long, MaterializedIndex> idToShadowIndex = Maps.newHashMap();
+    private Long2ObjectOpenHashMap<MaterializedIndex> idToShadowIndex = new Long2ObjectOpenHashMap<>();
 
     /**
      * committed version(hash): after txn is committed, set committed version(hash)

fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java

@@ -29,16 +29,16 @@
 import org.apache.doris.thrift.TStorageMedium;
 import org.apache.doris.thrift.TTabletType;
 
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
 import com.google.gson.annotations.SerializedName;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -58,30 +58,30 @@ public class PartitionInfo {
     protected List<Column> partitionColumns = Lists.newArrayList();
     // formal partition id -> partition item
     @SerializedName("IdToItem")
-    protected Map<Long, PartitionItem> idToItem = Maps.newHashMap();
+    protected Long2ObjectOpenHashMap<PartitionItem> idToItem = new Long2ObjectOpenHashMap<>();
     @SerializedName("IdToTempItem")
     // temp partition id -> partition item
-    protected Map<Long, PartitionItem> idToTempItem = Maps.newHashMap();
+    protected Long2ObjectOpenHashMap<PartitionItem> idToTempItem = new Long2ObjectOpenHashMap<>();
     // partition id -> data property
     @SerializedName("IdToDataProperty")
-    protected Map<Long, DataProperty> idToDataProperty;
+    protected Long2ObjectOpenHashMap<DataProperty> idToDataProperty;
     // partition id -> storage policy
     @SerializedName("IdToStoragePolicy")
-    protected Map<Long, String> idToStoragePolicy;
+    protected Long2ObjectOpenHashMap<String> idToStoragePolicy;
     // partition id -> replication allocation
     @SerializedName("IdToReplicaAllocation")
-    protected Map<Long, ReplicaAllocation> idToReplicaAllocation;
+    protected Long2ObjectOpenHashMap<ReplicaAllocation> idToReplicaAllocation;
     // true if the partition has multi partition columns
     @SerializedName("isM")
     protected boolean isMultiColumnPartition = false;
 
     @SerializedName("IdToInMemory")
-    protected Map<Long, Boolean> idToInMemory;
+    protected Long2ObjectOpenHashMap<Boolean> idToInMemory;
 
     // partition id -> tablet type
     // Note: currently it's only used for testing, it may change/add more meta field later,
     // so we defer adding meta serialization until memory engine feature is more complete.
-    protected Map<Long, TTabletType> idToTabletType;
+    protected Long2ObjectOpenHashMap<TTabletType> idToTabletType;
 
     // the enable automatic partition will hold this, could create partition by expr result
     @SerializedName("PartitionExprs")
@@ -92,21 +92,21 @@ public class PartitionInfo {
 
     public PartitionInfo() {
         this.type = PartitionType.UNPARTITIONED;
-        this.idToDataProperty = new HashMap<>();
-        this.idToReplicaAllocation = new HashMap<>();
-        this.idToInMemory = new HashMap<>();
-        this.idToTabletType = new HashMap<>();
-        this.idToStoragePolicy = new HashMap<>();
+        this.idToDataProperty = new Long2ObjectOpenHashMap<>();
+        this.idToReplicaAllocation = new Long2ObjectOpenHashMap<>();
+        this.idToInMemory = new Long2ObjectOpenHashMap<>();
+        this.idToTabletType = new Long2ObjectOpenHashMap<>();
+        this.idToStoragePolicy = new Long2ObjectOpenHashMap<>();
         this.partitionExprs = new ArrayList<>();
     }
 
     public PartitionInfo(PartitionType type) {
         this.type = type;
-        this.idToDataProperty = new HashMap<>();
-        this.idToReplicaAllocation = new HashMap<>();
-        this.idToInMemory = new HashMap<>();
-        this.idToTabletType = new HashMap<>();
-        this.idToStoragePolicy = new HashMap<>();
+        this.idToDataProperty = new Long2ObjectOpenHashMap<>();
+        this.idToReplicaAllocation = new Long2ObjectOpenHashMap<>();
+        this.idToInMemory = new Long2ObjectOpenHashMap<>();
+        this.idToTabletType = new Long2ObjectOpenHashMap<>();
+        this.idToStoragePolicy = new Long2ObjectOpenHashMap<>();
         this.partitionExprs = new ArrayList<>();
     }
 
@@ -150,7 +150,7 @@ public Map<Long, PartitionItem> getIdToItem(boolean isTemp) {
      * @return both normal partition and temp partition
      */
     public Map<Long, PartitionItem> getAllPartitions() {
-        HashMap all = new HashMap<>();
+        Long2ObjectOpenHashMap<PartitionItem> all = new Long2ObjectOpenHashMap<>();
         all.putAll(idToTempItem);
         all.putAll(idToItem);
         return all;
@@ -420,11 +420,11 @@ public void resetPartitionIdForRestore(
         Map<Long, PartitionItem> origIdToItem = idToItem;
         Map<Long, Boolean> origIdToInMemory = idToInMemory;
         Map<Long, String> origIdToStoragePolicy = idToStoragePolicy;
-        idToDataProperty = Maps.newHashMap();
-        idToReplicaAllocation = Maps.newHashMap();
-        idToItem = Maps.newHashMap();
-        idToInMemory = Maps.newHashMap();
-        idToStoragePolicy = Maps.newHashMap();
+        idToDataProperty = new Long2ObjectOpenHashMap<>();
+        idToReplicaAllocation = new Long2ObjectOpenHashMap<>();
+        idToItem = new Long2ObjectOpenHashMap<>();
+        idToInMemory = new Long2ObjectOpenHashMap<>();
+        idToStoragePolicy = new Long2ObjectOpenHashMap<>();
 
         for (Map.Entry<Long, Long> entry : partitionIdMap.entrySet()) {
             idToDataProperty.put(entry.getKey(), origIdToDataProperty.get(entry.getValue()));

fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java

@@ -26,6 +26,8 @@
 import org.apache.doris.thrift.TTablet;
 import org.apache.doris.thrift.TTabletMetaInfo;
 
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Maps;
@@ -60,7 +62,7 @@ public abstract class TabletInvertedIndex {
     private StampedLock lock = new StampedLock();
 
     // tablet id -> tablet meta
-    protected Map<Long, TabletMeta> tabletMetaMap = Maps.newHashMap();
+    protected Long2ObjectOpenHashMap<TabletMeta> tabletMetaMap = new Long2ObjectOpenHashMap<>();
 
     public TabletInvertedIndex() {
     }

fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java

@@ -19,6 +19,8 @@
 
 import org.apache.doris.persist.gson.GsonPostProcessable;
 
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
@@ -38,7 +40,7 @@
 // to make a overwrite load.
 public class TempPartitions implements GsonPostProcessable {
     @SerializedName(value = "idToPartition")
-    private Map<Long, Partition> idToPartition = Maps.newHashMap();
+    private Long2ObjectOpenHashMap<Partition> idToPartition = new Long2ObjectOpenHashMap<>();
     private Map<String, Partition> nameToPartition = Maps.newHashMap();
     @Deprecated
     // the range info of temp partitions has been moved to "partitionInfo" in OlapTable.

fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java

@@ -63,6 +63,8 @@
 import org.apache.doris.thrift.TStorageMedium;
 import org.apache.doris.transaction.TransactionState;
 
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.EvictingQueue;
@@ -118,9 +120,9 @@ public class TabletScheduler extends MasterDaemon {
      * pendingTablets, allTabletTypes, runningTablets and schedHistory are protected by 'synchronized'
      */
     private MinMaxPriorityQueue<TabletSchedCtx> pendingTablets = MinMaxPriorityQueue.create();
-    private Map<Long, TabletSchedCtx.Type> allTabletTypes = Maps.newHashMap();
+    private Long2ObjectOpenHashMap<TabletSchedCtx.Type> allTabletTypes = new Long2ObjectOpenHashMap<>();
     // contains all tabletCtxs which state are RUNNING
-    private Map<Long, TabletSchedCtx> runningTablets = Maps.newHashMap();
+    private Long2ObjectOpenHashMap<TabletSchedCtx> runningTablets = new Long2ObjectOpenHashMap<>();
     // save the latest 1000 scheduled tablet info
     private Queue<TabletSchedCtx> schedHistory = EvictingQueue.create(1000);
 

fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java

@@ -20,6 +20,8 @@
 import org.apache.doris.catalog.Replica;
 import org.apache.doris.catalog.TabletInvertedIndex;
 
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 import org.apache.logging.log4j.LogManager;
@@ -34,7 +36,7 @@ public class CloudTabletInvertedIndex extends TabletInvertedIndex {
 
     // tablet id -> replica
     // for cloud mode, no need to know the replica's backend
-    private Map<Long, Replica> replicaMetaMap = Maps.newHashMap();
+    private Long2ObjectOpenHashMap<Replica> replicaMetaMap = new Long2ObjectOpenHashMap<>();
 
     public CloudTabletInvertedIndex() {
         super();