eta logging

Author: MohdElgaar

open_instruct/grpo_fast.py

@@ -3013,6 +3013,8 @@ def health_check_fn():
     else:
         num_total_tokens = 0
 
+    num_prompts_to_refill = 0
+    avg_step_time = 0.0
     training_start_time = time.perf_counter()  # Track overall training start time
     for training_step in range(resume_training_step, args.num_training_steps + 1):
         start_time = time.perf_counter()
@@ -3095,6 +3097,22 @@ def health_check_fn():
             iter_dataloader,
         )
 
+        logger.debug(f"[Main Thread] Triggered weight sync for step {training_step}")
+        weight_sync_trigger_event.set()
+
+        # Print ETA (estimated time remaining)
+        current_step_time = time.perf_counter() - start_time
+        if avg_step_time == 0:
+            avg_step_time = current_step_time
+        else:
+            avg_step_time = 0.1 * current_step_time + 0.9 * avg_step_time
+
+        remaining_steps = args.num_training_steps - training_step
+        eta_seconds = remaining_steps * avg_step_time
+        logger.info(
+            f"[Main Thread] ⏳ ETA to finish: {utils.format_eta(eta_seconds)} ({avg_step_time:.2f}s/step, remaining {remaining_steps} steps)"
+        )
+
         # Checkpoint after one_training_step (or even if it was skipped)
         # This ensures we checkpoint progress even if the exact checkpoint step has no data
         if (