[opt](profile) Add new config profile_waiting_time_for_spill_s (apach…

…e#40302)

In previous, we use auto_profile_shreshold to control how many time
duration will ProfileManager waiting for profile collection after query
has finished. The default value of auto_profile_shreshold is -1, makes
profile need to be reported immediately after query finished, or it will
be not complete. So introduce a new config in Config to solve the
problem.

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -2805,6 +2805,12 @@ public class Config extends ConfigBase {
     @ConfField
     public static long spilled_profile_storage_limit_bytes = 1 * 1024 * 1024 * 1024; // 1GB
 
+    // Profile will be spilled to storage after query has finished for this time.
+    @ConfField(mutable = true, description = {
+            "Profile 在 query 完成后等待多久后才会被写入磁盘",
+            "Profile will be spilled to storage after query has finished for this time"})
+    public static int profile_waiting_time_for_spill_seconds = 10;
+
     @ConfField(mutable = true, description = {
             "是否通过检测协调者BE心跳来 abort 事务",
             "SHould abort txn by checking coorinator be heartbeat"})

fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java

@@ -17,6 +17,7 @@
 
 package org.apache.doris.common.profile;
 
+import org.apache.doris.common.Config;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.util.DebugUtil;
 import org.apache.doris.common.util.ProfileManager;
@@ -493,16 +494,18 @@ public boolean shouldStoreToStorage() {
             return false;
         }
 
+        long currentTimeMillis = System.currentTimeMillis();
         if (this.queryFinishTimestamp != Long.MAX_VALUE
-                    && (System.currentTimeMillis() - this.queryFinishTimestamp) > autoProfileDurationMs) {
+                    && (currentTimeMillis - this.queryFinishTimestamp)
+                        > Config.profile_waiting_time_for_spill_seconds * 1000) {
             LOG.warn("Profile {} should be stored to storage without waiting for incoming profile,"
-                    + " since it has been waiting for {} ms, query finished time: {}", id,
-                    System.currentTimeMillis() - this.queryFinishTimestamp, this.queryFinishTimestamp);
+                    + " since it has been waiting for {} ms, current time {} query finished time: {}",
+                    id, currentTimeMillis - this.queryFinishTimestamp, currentTimeMillis, this.queryFinishTimestamp);
 
             this.summaryProfile.setSystemMessage(
                             "This profile is not complete, since its collection does not finish in time."
-                            + " Maybe increase auto_profile_threshold_ms current val: "
-                            + String.valueOf(autoProfileDurationMs));
+                            + " Maybe increase profile_waiting_time_for_spill_secs in fe.conf current val: "
+                            + String.valueOf(Config.profile_waiting_time_for_spill_seconds));
             return true;
         }