handle_task catch all exception (#3441)

### What problem does this PR solve?

handle_task catch all exception
Report heartbeats

### Type of change

- [x] Refactoring

api/apps/system_app.py

@@ -13,7 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License
 #
-import json
+import logging
 from datetime import datetime
 
 from flask_login import login_required, current_user
@@ -154,26 +154,16 @@ def status():
             "error": str(e),
         }
 
+    task_executor_heartbeats = {}
     try:
-        v = REDIS_CONN.get("TASKEXE")
-        if not v:
-            raise Exception("No task executor running!")
-        obj = json.loads(v)
-        color = "green"
-        for id in obj.keys():
-            arr = obj[id]
-            if len(arr) == 1:
-                obj[id] = [0]
-            else:
-                obj[id] = [arr[i + 1] - arr[i] for i in range(len(arr) - 1)]
-            elapsed = max(obj[id])
-            if elapsed > 50:
-                color = "yellow"
-            if elapsed > 120:
-                color = "red"
-        res["task_executor"] = {"status": color, "elapsed": obj}
-    except Exception as e:
-        res["task_executor"] = {"status": "red", "error": str(e)}
+        task_executors = REDIS_CONN.smembers("TASKEXE")
+        now = datetime.now().timestamp()
+        for task_executor_id in task_executors:
+            heartbeats = REDIS_CONN.zrangebyscore(task_executor_id, now - 60*30, now)
+            task_executor_heartbeats[task_executor_id] = heartbeats
+    except Exception:
+        logging.exception("get task executor heartbeats failed!")
+    res["task_executor_heartbeats"] = task_executor_heartbeats
 
     return get_json_result(data=res)