Support newly added --storage.tsdb.delay-compact-file.path Prometheus flag

Prometheus has a new flag --storage.tsdb.delay-compact-file.path - prometheus/prometheus#17435.
When this flag is passed Prometheus will check which blocks are marked as uploaded in external file and only compact these.
Thanos should look for this flag and if it's set then it can stop forcing people to disable compactions.

Signed-off-by: Lukasz Mierzwa <[email protected]>

Author: prymitive

cmd/thanos/sidecar.go

@@ -189,7 +189,7 @@ func runSidecar(
 					iterCtx, iterCancel := context.WithTimeout(context.Background(), conf.prometheus.getConfigTimeout)
 					defer iterCancel()
 
-					if err := validatePrometheus(iterCtx, m.client, logger, conf.shipper.ignoreBlockSize, m); err != nil {
+					if err := validatePrometheus(iterCtx, m.client, logger, conf.shipper.ignoreBlockSize, conf.shipper.metaFileName, m); err != nil {
 						level.Warn(logger).Log(
 							"msg", "failed to validate prometheus flags. Is Prometheus running? Retrying",
 							"err", err,
@@ -443,7 +443,7 @@ func runSidecar(
 	return nil
 }
 
-func validatePrometheus(ctx context.Context, client *promclient.Client, logger log.Logger, ignoreBlockSize bool, m *promMetadata) error {
+func validatePrometheus(ctx context.Context, client *promclient.Client, logger log.Logger, ignoreBlockSize bool, metaFileName string, m *promMetadata) error {
 	var (
 		flagErr error
 		flags   promclient.Flags
@@ -464,8 +464,16 @@ func validatePrometheus(ctx context.Context, client *promclient.Client, logger l
 		return nil
 	}
 
+	if flags.TSDBDelayCompact != "" && flags.TSDBDelayCompact != metaFileName {
+		return errors.Errorf(
+			"found that Prometheus and Thanos use different paths for tracking block uploads. "+
+				"Prometheus uses --storage.tsdb.delay-compact-file.path=%s while Thanos uses --shipper.meta-file-name=%s, they must both use the same path.",
+			flags.TSDBDelayCompact, metaFileName,
+		)
+	}
+
 	// Check if compaction is disabled.
-	if flags.TSDBMinTime != flags.TSDBMaxTime {
+	if flags.TSDBMinTime != flags.TSDBMaxTime && flags.TSDBDelayCompact == "" {
 		if !ignoreBlockSize {
 			return errors.Errorf("found that TSDB Max time is %s and Min time is %s. "+
 				"Compaction needs to be disabled (storage.tsdb.min-block-duration = storage.tsdb.max-block-duration)", flags.TSDBMaxTime, flags.TSDBMinTime)

pkg/promclient/promclient.go

@@ -217,6 +217,7 @@ type Flags struct {
 	TSDBRetention      model.Duration `json:"storage.tsdb.retention"`
 	TSDBMinTime        model.Duration `json:"storage.tsdb.min-block-duration"`
 	TSDBMaxTime        model.Duration `json:"storage.tsdb.max-block-duration"`
+	TSDBDelayCompact   string         `json:"storage.tsdb.delay-compact-file.path"`
 	WebEnableAdminAPI  bool           `json:"web.enable-admin-api"`
 	WebEnableLifecycle bool           `json:"web.enable-lifecycle"`
 }
@@ -231,6 +232,7 @@ func (f *Flags) UnmarshalJSON(b []byte) error {
 		TSDBRetention      modelDuration `json:"storage.tsdb.retention"`
 		TSDBMinTime        modelDuration `json:"storage.tsdb.min-block-duration"`
 		TSDBMaxTime        modelDuration `json:"storage.tsdb.max-block-duration"`
+		TSDBDelayCompact   string        `json:"storage.tsdb.delay-compact-file.path"`
 		WebEnableAdminAPI  modelBool     `json:"web.enable-admin-api"`
 		WebEnableLifecycle modelBool     `json:"web.enable-lifecycle"`
 	}{}
@@ -244,6 +246,7 @@ func (f *Flags) UnmarshalJSON(b []byte) error {
 		TSDBRetention:      model.Duration(parsableFlags.TSDBRetention),
 		TSDBMinTime:        model.Duration(parsableFlags.TSDBMinTime),
 		TSDBMaxTime:        model.Duration(parsableFlags.TSDBMaxTime),
+		TSDBDelayCompact:   parsableFlags.TSDBDelayCompact,
 		WebEnableAdminAPI:  bool(parsableFlags.WebEnableAdminAPI),
 		WebEnableLifecycle: bool(parsableFlags.WebEnableLifecycle),
 	}