From a8e32147b372111ca7d24a3eeead605d5dfb1af2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 24 Jun 2025 15:02:17 +0100 Subject: [PATCH] [BUGFIX] Top-level: Update GOGC before loading TSDB (#16521) We should use the configured value, or Prometheus' default of 75%, while initializing and loading the WAL. Since the Go default is 100%, most Prometheus users would experience higher memory usage before the value is configured. Signed-off-by: Bryan Boreham --- cmd/prometheus/main.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 8db2f2c5eb..31f061329e 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -554,6 +554,7 @@ func main() { if cfgFile.StorageConfig.TSDBConfig != nil { cfg.tsdb.OutOfOrderTimeWindow = cfgFile.StorageConfig.TSDBConfig.OutOfOrderTimeWindow } + updateGoGC(cfgFile, logger) // Now that the validity of the config is established, set the config // success metrics accordingly, although the config isn't really loaded @@ -1386,6 +1387,15 @@ func reloadConfig(filename string, expandExternalLabels, enableExemplarStorage b return fmt.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename) } + updateGoGC(conf, logger) + + noStepSuqueryInterval.Set(conf.GlobalConfig.EvaluationInterval) + l := []interface{}{"msg", "Completed loading of configuration file", "filename", filename, "totalDuration", time.Since(start)} + level.Info(logger).Log(append(l, timings...)...) + return nil +} + +func updateGoGC(conf *config.Config, logger log.Logger) { oldGoGC := debug.SetGCPercent(conf.Runtime.GoGC) if oldGoGC != conf.Runtime.GoGC { level.Info(logger).Log("msg", "updated GOGC", "old", oldGoGC, "new", conf.Runtime.GoGC) @@ -1396,11 +1406,6 @@ func reloadConfig(filename string, expandExternalLabels, enableExemplarStorage b } else { os.Setenv("GOGC", "off") } - - noStepSuqueryInterval.Set(conf.GlobalConfig.EvaluationInterval) - l := []interface{}{"msg", "Completed loading of configuration file", "filename", filename, "totalDuration", time.Since(start)} - level.Info(logger).Log(append(l, timings...)...) - return nil } func startsOrEndsWithQuote(s string) bool {