lots of little fixes

api/models.py

@@ -78,10 +78,22 @@ def success_bool(self):
                 return ret["return"]["result"]
         return self.jid
 
+    def valid_for_highstate(self):
+        valid_for_highstate = True
+        if not self.loaded_ret()["fun_args"]:
+            valid_for_highstate = True
+        if isinstance(self.loaded_ret()["fun_args"], list) and self.loaded_ret()["fun_args"]:
+            if self.loaded_ret()["fun_args"][0] == {"test": True}:
+                valid_for_highstate = False
+            if self.loaded_ret()["fun_args"][0] == "test=True":
+                valid_for_highstate = False
+        return valid_for_highstate
+
     class Meta:
         managed = False
         db_table = "salt_returns"
         app_label = "api"
+        ordering = ['-id']
 
 
 class SaltEvents(models.Model):
@@ -95,6 +107,7 @@ class Meta:
         managed = False
         db_table = "salt_events"
         app_label = "api"
+        ordering = ['-id']
 
 
 # Alcali custom.
@@ -145,16 +158,12 @@ def last_highstate(self):
         # Get all potential jobs.
         states = SaltReturns.objects.filter(
             Q(fun="state.apply") | Q(fun="state.highstate"), id=self.minion_id
-        )
-        states = sorted(states, key=lambda x: x.jid, reverse=True)
+        ).order_by('-jid')[0:2]
+        states = sorted(states, key=lambda x: x.jid)
 
         # Remove jobs with arguments.
         for state in states:
-            if (
-                not state.loaded_ret()["fun_args"]
-                or state.loaded_ret()["fun_args"][0] == {"test": True}
-                or state.loaded_ret()["fun_args"][0] == "test=True"
-            ):
+            if state.valid_for_highstate():
                 return state
         return None
 
@@ -171,8 +180,9 @@ def conformity(self):
 
         for state in return_item:
             # One of the state is not ok
-            if not return_item.get(state, {}).get("result"):
-                return False
+            if type(return_item) == dict:
+                if not return_item.get(state, {}).get("result"):
+                    return False
         return True
 
     def custom_conformity(self, fun, *args):

api/views/alcali.py

@@ -124,6 +124,7 @@ def refresh_minions(self, request):
             return Response({"result": "refreshed {}".format(minion_id)})
 
         # Run test.ping to list currently connected minions
+        print("sending test.ping to salt for minions list")
         connected = run_raw(
             [
                 {
@@ -132,15 +133,35 @@ def refresh_minions(self, request):
                     "tgt_type": "glob",
                     "tgt": "*",
                     "fun": "test.ping",
+                    "timeout": 20
                 }
             ]
         )
         accepted_minions = [i for i in connected if connected.get(i) is True]
+        print(f"{len(accepted_minions)} responded to test.ping, processing...")
+        refresh_failures = {}
+        known_minions = []
+        refreshed_minions = [] 
         for minion in accepted_minions:
+            if not Minions.objects.filter(minion_id=minion).exists():
+                print(f"attempting to add minion: {minion}")
+                ret = refresh_minion(minion)
+                if "error" in ret:
+                    print(f"minion refresh failed: {ret['error']}")
+                    refresh_failures[minion] = ret['error']
+                else:
+                    refreshed_minions.append(minion)
+            else:
+                known_minions.append(minion)
+        print("starting refresh of known minions")
+        for minion in known_minions:
+            print(minion)
             ret = refresh_minion(minion)
             if "error" in ret:
-                return Response(ret["error"], status=401)
-        return Response({"refreshed": accepted_minions})
+                refresh_failures[minion] = ret['error']
+            else:
+                refreshed_minions.append(minion)
+        return Response({"refreshed": refreshed_minions, "errors": refresh_failures})
 
     @action(detail=False)
     def conformity(self, request):
@@ -208,6 +229,13 @@ def conformity_detail(self, request, minion_id):
             }
         )
 
+    @action(detail=False, methods=["get"])
+    def add_minion(self, request):
+        if request.GET.get("minion_id"):
+            minion_id = request.GET.get("minion_id")
+            ret = refresh_minion(minion_id)
+        return Response(None, status=201)
+
 
 class MinionsCustomFieldsViewSet(viewsets.ModelViewSet):
     queryset = MinionsCustomFields.objects.all()
@@ -259,11 +287,15 @@ def render(self, request):
                     succeeded, unchanged, failed = None, None, 1
                 else:
                     for state in last_highstate:
-                        if last_highstate[state]["result"] is True:
-                            succeeded += 1
-                        elif last_highstate[state]["result"] is None:
-                            unchanged += 1
+                        if isinstance(last_highstate, dict):
+                            if last_highstate[state]["result"] is True:
+                                succeeded += 1
+                            elif last_highstate[state]["result"] is None:
+                                unchanged += 1
+                            else:
+                                failed += 1
                         else:
+                            # most likely a string response containing "Unhandled exception running state.highstate"
                             failed += 1
             else:
                 last_highstate_date, succeeded, unchanged, failed = (

api/views/salt.py

@@ -57,14 +57,17 @@ class SaltReturnsListJid(generics.ListAPIView):
 
     def get_queryset(self):
         jid = self.kwargs["jid"]
-        queryset = SaltReturns.objects.all()
-        return queryset.filter(jid=jid).order_by("-alter_time")
+        queryset = SaltReturns.objects.filter(jid=jid).order_by("-alter_time")
+        return queryset
 
 
 @api_view(["GET"])
 def jobs_filters(request):
     # Filter options.
-    user_list = list({i.user() for i in Jids.objects.all()})
+
+    # THIS is eating through ram and cpu...
+    # user_list = list({i.user() for i in Jids.objects.all()}) # THIS is eating through ram and cpu...
+    user_list = list()
     minion_list = SaltReturns.objects.values_list("id", flat=True).distinct()
     return Response({"users": user_list, "minions": minion_list})
 
@@ -97,5 +100,5 @@ class EventsViewSet(viewsets.ReadOnlyModelViewSet):
     A simple ViewSet for viewing accounts.
     """
 
-    queryset = SaltEvents.objects.all().order_by("-alter_time")[:200]
+    queryset = SaltEvents.objects.all().order_by("-id")[:200]
     serializer_class = EventsSerializer

config/settings.py

@@ -42,6 +42,7 @@
     DJANGO_DEBUG = False
 DEBUG = DJANGO_DEBUG
 
+
 ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "127.0.0.1").split(" ")
 
 # Application definition
@@ -126,6 +127,9 @@
 
 USE_L10N = True
 
+USE_TZ = True
+TIME_ZONE = "UTC"
+
 # Static files (CSS, JavaScript, Images)
 # Place static in the same location as webpack build files
 STATIC_URL = "/static/"

docker/Dockerfile-dev

@@ -32,6 +32,6 @@ WORKDIR /opt/alcali/code
 RUN pip install --user -U setuptools
 
 # Install project
-RUN pip install --user -e .[dev,ldap,social] mysqlclient
+RUN pip install --user -e .[dev,ldap,social] mysqlclient psycopg2
 
 ENTRYPOINT ["/opt/alcali/code/docker/utils/entrypoint-dev.sh"]

src/views/MinionDetail.vue

@@ -63,6 +63,7 @@
       loadData() {
         this.$http.get(`api/minions/${this.minion_id}/`).then(response => this.minion = addedGrains(response.data)).catch((error) => {
           this.$toast.error(this.$i18n.t("components.MinionDetail.MissingMinion", [this.minion_id]))
+          this.$http.get("/api/minions/add_minion?minion_id=${this.minion_id}")
           this.$router.push("/minions")
         })
       },