|
| 1 | +//go:build integration |
| 2 | +// +build integration |
| 3 | + |
| 4 | +package integration |
| 5 | + |
| 6 | +import ( |
| 7 | + "context" |
| 8 | + "strings" |
| 9 | + "testing" |
| 10 | + "time" |
| 11 | +) |
| 12 | + |
| 13 | +// TestCLIHibernationOperations tests hibernation and resume operations via CLI |
| 14 | +// Priority: HIGHEST - Phase 3 hibernation is the cornerstone of cost optimization |
| 15 | +func TestCLIHibernationOperations(t *testing.T) { |
| 16 | + ctx := NewCLITestContext(t) |
| 17 | + defer ctx.Cleanup() |
| 18 | + |
| 19 | + // Use a hibernation-capable template (requires specific configuration) |
| 20 | + // Python ML workstation should support hibernation on compatible instance types |
| 21 | + instanceName := GenerateTestName("test-hibernate") |
| 22 | + |
| 23 | + // Launch hibernation-capable instance |
| 24 | + t.Run("LaunchHibernationCapable", func(t *testing.T) { |
| 25 | + t.Logf("Launching hibernation-capable instance: %s", instanceName) |
| 26 | + |
| 27 | + // Launch with size M (should support hibernation) |
| 28 | + result, err := ctx.LaunchInstanceCLI("python-ml-workstation", instanceName, "M") |
| 29 | + AssertNoError(t, err, "launch hibernation-capable instance") |
| 30 | + result.AssertSuccess(t, "launch command should succeed") |
| 31 | + |
| 32 | + // Verify instance is running |
| 33 | + instance := ctx.AssertInstanceExists(instanceName) |
| 34 | + AssertEqual(t, "running", instance.State, "instance should be running") |
| 35 | + |
| 36 | + t.Logf("✓ Instance %s launched successfully", instanceName) |
| 37 | + }) |
| 38 | + |
| 39 | + // Check hibernation support status |
| 40 | + t.Run("CheckHibernationStatus", func(t *testing.T) { |
| 41 | + t.Logf("Checking hibernation support for instance: %s", instanceName) |
| 42 | + |
| 43 | + // Get instance details via API |
| 44 | + instance, err := ctx.Client.GetInstance(context.Background(), instanceName) |
| 45 | + AssertNoError(t, err, "get instance details") |
| 46 | + |
| 47 | + // Check if hibernation fields are present (implementation may vary) |
| 48 | + t.Logf("Instance ID: %s, State: %s", instance.ID, instance.State) |
| 49 | + |
| 50 | + // Note: HibernationSupported field may not exist yet in types.Instance |
| 51 | + // This test documents the expected behavior for Phase 3 completion |
| 52 | + }) |
| 53 | + |
| 54 | + // Test hibernate operation |
| 55 | + t.Run("HibernateRunningInstance", func(t *testing.T) { |
| 56 | + t.Logf("Hibernating instance: %s", instanceName) |
| 57 | + |
| 58 | + // Execute hibernate command |
| 59 | + result := ctx.Prism("workspace", "hibernate", instanceName) |
| 60 | + |
| 61 | + // Check if command exists and works |
| 62 | + if result.ExitCode != 0 { |
| 63 | + // If hibernate command doesn't exist yet, skip gracefully |
| 64 | + if strings.Contains(result.Stderr, "unknown command") || |
| 65 | + strings.Contains(result.Stderr, "not found") { |
| 66 | + t.Skip("Hibernate command not yet implemented - test documents expected behavior") |
| 67 | + return |
| 68 | + } |
| 69 | + |
| 70 | + // If instance doesn't support hibernation, verify graceful fallback |
| 71 | + if strings.Contains(result.Stderr, "not supported") || |
| 72 | + strings.Contains(result.Stderr, "fallback to stop") { |
| 73 | + t.Logf("Instance doesn't support hibernation, should fallback to regular stop") |
| 74 | + // Verify instance was stopped (fallback behavior) |
| 75 | + time.Sleep(5 * time.Second) // Give AWS time to process |
| 76 | + instance, err := ctx.Client.GetInstance(context.Background(), instanceName) |
| 77 | + AssertNoError(t, err, "get instance after fallback") |
| 78 | + if instance.State != "stopped" && instance.State != "stopping" { |
| 79 | + t.Errorf("Expected instance to be stopped/stopping after hibernation fallback, got: %s", instance.State) |
| 80 | + } |
| 81 | + return |
| 82 | + } |
| 83 | + |
| 84 | + // Unexpected error |
| 85 | + t.Fatalf("Hibernate command failed unexpectedly: %s", result.Stderr) |
| 86 | + } |
| 87 | + |
| 88 | + result.AssertSuccess(t, "hibernate command should succeed") |
| 89 | + |
| 90 | + // Wait for hibernation to complete (or stopping state) |
| 91 | + // Hibernation can take 1-3 minutes depending on RAM size |
| 92 | + t.Logf("Waiting for hibernation to complete (may take 1-3 minutes)...") |
| 93 | + time.Sleep(5 * time.Second) // Initial delay |
| 94 | + |
| 95 | + // Check instance state |
| 96 | + instance, err := ctx.Client.GetInstance(context.Background(), instanceName) |
| 97 | + AssertNoError(t, err, "get instance after hibernate") |
| 98 | + |
| 99 | + // Valid states: hibernated, stopped, or stopping |
| 100 | + validStates := []string{"hibernated", "stopped", "stopping"} |
| 101 | + stateValid := false |
| 102 | + for _, state := range validStates { |
| 103 | + if instance.State == state { |
| 104 | + stateValid = true |
| 105 | + break |
| 106 | + } |
| 107 | + } |
| 108 | + |
| 109 | + if !stateValid { |
| 110 | + t.Errorf("Expected instance in hibernated/stopped/stopping state, got: %s", instance.State) |
| 111 | + } |
| 112 | + |
| 113 | + t.Logf("✓ Instance hibernated successfully (state: %s)", instance.State) |
| 114 | + }) |
| 115 | + |
| 116 | + // Test resume operation |
| 117 | + t.Run("ResumeHibernatedInstance", func(t *testing.T) { |
| 118 | + t.Logf("Resuming hibernated instance: %s", instanceName) |
| 119 | + |
| 120 | + // Wait for hibernation to fully complete (not just "stopping" state) |
| 121 | + t.Logf("Waiting for instance to reach stopped/hibernated state...") |
| 122 | + maxWaitTime := 5 * time.Minute |
| 123 | + pollInterval := 10 * time.Second |
| 124 | + startTime := time.Now() |
| 125 | + |
| 126 | + for { |
| 127 | + instance, err := ctx.Client.GetInstance(context.Background(), instanceName) |
| 128 | + if err == nil && (instance.State == "stopped" || instance.State == "hibernated") { |
| 129 | + t.Logf("✓ Instance fully hibernated/stopped (state: %s)", instance.State) |
| 130 | + break |
| 131 | + } |
| 132 | + |
| 133 | + elapsed := time.Since(startTime) |
| 134 | + if elapsed > maxWaitTime { |
| 135 | + t.Fatalf("Timeout waiting for instance to hibernate (waited %v, state: %s)", elapsed, instance.State) |
| 136 | + } |
| 137 | + |
| 138 | + time.Sleep(pollInterval) |
| 139 | + } |
| 140 | + |
| 141 | + // Execute resume command |
| 142 | + result := ctx.Prism("workspace", "resume", instanceName) |
| 143 | + |
| 144 | + // Check if command exists |
| 145 | + if result.ExitCode != 0 { |
| 146 | + if strings.Contains(result.Stderr, "unknown command") || |
| 147 | + strings.Contains(result.Stderr, "not found") { |
| 148 | + t.Skip("Resume command not yet implemented - using start instead") |
| 149 | + // Fallback to start command |
| 150 | + result = ctx.Prism("workspace", "start", instanceName) |
| 151 | + } |
| 152 | + } |
| 153 | + |
| 154 | + result.AssertSuccess(t, "resume/start command should succeed") |
| 155 | + |
| 156 | + // Wait for instance to return to running state |
| 157 | + t.Logf("Waiting for instance to resume...") |
| 158 | + err := ctx.WaitForInstanceState(instanceName, "running", InstanceReadyTimeout) |
| 159 | + AssertNoError(t, err, "instance should return to running state") |
| 160 | + |
| 161 | + // Verify instance is running |
| 162 | + instance := ctx.AssertInstanceExists(instanceName) |
| 163 | + AssertEqual(t, "running", instance.State, "instance should be running after resume") |
| 164 | + |
| 165 | + t.Logf("✓ Instance resumed successfully") |
| 166 | + }) |
| 167 | + |
| 168 | + // Test hibernate command on stopped instance (edge case) |
| 169 | + t.Run("HibernateStoppedInstance", func(t *testing.T) { |
| 170 | + t.Logf("Testing hibernate on stopped instance (edge case)") |
| 171 | + |
| 172 | + // Stop the instance first |
| 173 | + err := ctx.StopInstanceCLI(instanceName) |
| 174 | + AssertNoError(t, err, "stop instance") |
| 175 | + |
| 176 | + // Try to hibernate stopped instance |
| 177 | + result := ctx.Prism("workspace", "hibernate", instanceName) |
| 178 | + |
| 179 | + // Should either succeed (no-op) or fail gracefully with clear message |
| 180 | + if result.ExitCode != 0 { |
| 181 | + if strings.Contains(result.Stderr, "unknown command") { |
| 182 | + t.Skip("Hibernate command not yet implemented") |
| 183 | + return |
| 184 | + } |
| 185 | + |
| 186 | + // Check for helpful error message |
| 187 | + if !strings.Contains(result.Stderr, "already stopped") && |
| 188 | + !strings.Contains(result.Stderr, "not running") { |
| 189 | + t.Logf("Warning: Hibernate error message could be more helpful: %s", result.Stderr) |
| 190 | + } |
| 191 | + } |
| 192 | + |
| 193 | + t.Logf("✓ Edge case handled appropriately") |
| 194 | + }) |
| 195 | + |
| 196 | + // Cleanup |
| 197 | + t.Run("Cleanup", func(t *testing.T) { |
| 198 | + // Start instance if stopped/hibernated before deletion |
| 199 | + instance, err := ctx.Client.GetInstance(context.Background(), instanceName) |
| 200 | + if err == nil && (instance.State == "stopped" || instance.State == "hibernated") { |
| 201 | + ctx.Prism("workspace", "start", instanceName) |
| 202 | + time.Sleep(10 * time.Second) // Give it time to start |
| 203 | + } |
| 204 | + |
| 205 | + err = ctx.DeleteInstanceCLI(instanceName) |
| 206 | + AssertNoError(t, err, "cleanup should succeed") |
| 207 | + |
| 208 | + t.Logf("✓ Cleanup complete") |
| 209 | + }) |
| 210 | +} |
| 211 | + |
| 212 | +// TestCLIHibernationNonCapable tests hibernation fallback on non-capable instances |
| 213 | +func TestCLIHibernationNonCapable(t *testing.T) { |
| 214 | + ctx := NewCLITestContext(t) |
| 215 | + defer ctx.Cleanup() |
| 216 | + |
| 217 | + instanceName := GenerateTestName("test-no-hibernate") |
| 218 | + |
| 219 | + t.Run("LaunchNonCapable", func(t *testing.T) { |
| 220 | + t.Logf("Launching instance without hibernation support") |
| 221 | + |
| 222 | + // Launch with size S (may not support hibernation) |
| 223 | + result, err := ctx.LaunchInstanceCLI("python-ml-workstation", instanceName, "S") |
| 224 | + if err != nil { |
| 225 | + t.Skipf("Launch failed (may be environmental): %v", err) |
| 226 | + return |
| 227 | + } |
| 228 | + |
| 229 | + if result.ExitCode != 0 { |
| 230 | + t.Skipf("Launch command failed (may be environmental): %s", result.Stderr) |
| 231 | + return |
| 232 | + } |
| 233 | + |
| 234 | + t.Logf("✓ Instance launched") |
| 235 | + }) |
| 236 | + |
| 237 | + t.Run("HibernateWithFallback", func(t *testing.T) { |
| 238 | + t.Logf("Testing hibernation fallback to stop") |
| 239 | + |
| 240 | + result := ctx.Prism("workspace", "hibernate", instanceName) |
| 241 | + |
| 242 | + if result.ExitCode != 0 { |
| 243 | + if strings.Contains(result.Stderr, "unknown command") { |
| 244 | + t.Skip("Hibernate command not yet implemented") |
| 245 | + return |
| 246 | + } |
| 247 | + } |
| 248 | + |
| 249 | + // Should succeed (fallback to stop) or provide clear message |
| 250 | + if result.ExitCode == 0 { |
| 251 | + t.Logf("✓ Hibernation attempted (may fallback to stop)") |
| 252 | + |
| 253 | + // Verify instance stopped |
| 254 | + time.Sleep(5 * time.Second) |
| 255 | + instance, err := ctx.Client.GetInstance(context.Background(), instanceName) |
| 256 | + AssertNoError(t, err, "get instance") |
| 257 | + |
| 258 | + if instance.State != "stopped" && instance.State != "stopping" && instance.State != "hibernated" { |
| 259 | + t.Errorf("Expected instance stopped/stopping/hibernated, got: %s", instance.State) |
| 260 | + } |
| 261 | + } else { |
| 262 | + // Should have clear message about lack of hibernation support |
| 263 | + t.Logf("Hibernation not supported (expected): %s", result.Stderr) |
| 264 | + } |
| 265 | + }) |
| 266 | + |
| 267 | + t.Run("Cleanup", func(t *testing.T) { |
| 268 | + err := ctx.DeleteInstanceCLI(instanceName) |
| 269 | + if err != nil { |
| 270 | + // Try to start first if needed |
| 271 | + ctx.Prism("workspace", "start", instanceName) |
| 272 | + time.Sleep(5 * time.Second) |
| 273 | + err = ctx.DeleteInstanceCLI(instanceName) |
| 274 | + } |
| 275 | + AssertNoError(t, err, "cleanup should succeed") |
| 276 | + }) |
| 277 | +} |
| 278 | + |
| 279 | +// TestCLIHibernationHelp tests help output for hibernation commands |
| 280 | +func TestCLIHibernationHelp(t *testing.T) { |
| 281 | + ctx := NewCLITestContext(t) |
| 282 | + defer ctx.Cleanup() |
| 283 | + |
| 284 | + t.Run("HibernateHelp", func(t *testing.T) { |
| 285 | + result := ctx.Prism("workspace", "hibernate", "--help") |
| 286 | + |
| 287 | + if result.ExitCode != 0 { |
| 288 | + if strings.Contains(result.Stderr, "unknown command") { |
| 289 | + t.Skip("Hibernate command not yet implemented") |
| 290 | + return |
| 291 | + } |
| 292 | + } |
| 293 | + |
| 294 | + result.AssertSuccess(t, "hibernate help should succeed") |
| 295 | + |
| 296 | + // Check for key information in help |
| 297 | + helpText := result.Stdout + result.Stderr |
| 298 | + expectedTerms := []string{"hibernate", "save state", "cost"} |
| 299 | + |
| 300 | + for _, term := range expectedTerms { |
| 301 | + if !strings.Contains(strings.ToLower(helpText), strings.ToLower(term)) { |
| 302 | + t.Logf("Warning: Help text might be missing '%s' explanation", term) |
| 303 | + } |
| 304 | + } |
| 305 | + |
| 306 | + t.Logf("✓ Hibernate help available") |
| 307 | + }) |
| 308 | + |
| 309 | + t.Run("ResumeHelp", func(t *testing.T) { |
| 310 | + result := ctx.Prism("workspace", "resume", "--help") |
| 311 | + |
| 312 | + if result.ExitCode != 0 { |
| 313 | + if strings.Contains(result.Stderr, "unknown command") { |
| 314 | + t.Skip("Resume command not yet implemented") |
| 315 | + return |
| 316 | + } |
| 317 | + } |
| 318 | + |
| 319 | + result.AssertSuccess(t, "resume help should succeed") |
| 320 | + |
| 321 | + t.Logf("✓ Resume help available") |
| 322 | + }) |
| 323 | +} |
0 commit comments