fix: fetch cluster view after leader recover (#92)

* fix: fetch cluster view after leader recover

* fix: fetch cluster again view only when leader recover

Co-authored-by: shaimoria <shai.moria@zooz.com>

Author: shyimo

controllers/rediscluster.go

@@ -627,6 +627,7 @@ func (r *RedisClusterReconciler) handleFailover(redisCluster *dbv1.RedisCluster,
 }
 
 func (r *RedisClusterReconciler) recoverCluster(redisCluster *dbv1.RedisCluster) error {
+	var runLeaderRecover bool = false
 	clusterView, err := r.NewRedisClusterView(redisCluster)
 	if err != nil {
 		return err
@@ -635,6 +636,8 @@ func (r *RedisClusterReconciler) recoverCluster(redisCluster *dbv1.RedisCluster)
 	r.Log.Info(clusterView.String())
 	for i, leader := range *clusterView {
 		if leader.Failed {
+			runLeaderRecover = true
+
 			if leader.Terminating {
 				if err = r.waitForPodDelete(*leader.Pod); err != nil {
 					return errors.Errorf("Failed to wait for leader pod to be deleted %s: %v", leader.NodeNumber, err)
@@ -666,6 +669,17 @@ func (r *RedisClusterReconciler) recoverCluster(redisCluster *dbv1.RedisCluster)
 		}
 	}
 
+	if runLeaderRecover {
+		// we fetch again the cluster view in case the state has changed
+		// since the last check (before handling the failed leaders)
+		clusterView, err = r.NewRedisClusterView(redisCluster)
+		if err != nil {
+			return err
+		}
+
+		r.Log.Info(clusterView.String())
+	}
+
 	for _, leader := range *clusterView {
 		var missingFollowers []NodeNumbers
 		var failedFollowerIPs []string