[k8s] add even more debug logs around k8s list_namespaced_pod (#7852)

* add more debug logs around k8s list_namespaced_pod

* more info

* reduce overhead

* label selector

* clean

* only debug

Author: rohansonecha

sky/provision/kubernetes/instance.py

@@ -1691,29 +1691,49 @@ def query_instances(
     context = kubernetes_utils.get_context_from_config(provider_config)
     is_ssh = context.startswith('ssh-') if context else False
     identity = 'SSH Node Pool' if is_ssh else 'Kubernetes cluster'
+    label_selector = (f'{constants.TAG_SKYPILOT_CLUSTER_NAME}='
+                      f'{cluster_name_on_cloud}')
 
     # Get all the pods with the label skypilot-cluster-name: <cluster_name>
     try:
-        logger.debug(
-            f'Querying k8s api for pods in context: {context} and '
-            f'namespace: {namespace} with '
-            f'`skypilot-cluster-name={cluster_name_on_cloud}` label selector.')
+        # log the query parameters we pass to the k8s api
+        logger.debug(f'Querying k8s api for pods:\n'
+                     f'context: {context}\n'
+                     f'namespace: {namespace}\n'
+                     f'label selector:`{label_selector}`.')
 
-        label_selector = (f'{constants.TAG_SKYPILOT_CLUSTER_NAME}='
-                          f'{cluster_name_on_cloud}')
         response = kubernetes.core_api(context).list_namespaced_pod(
             namespace,
             label_selector=label_selector,
             _request_timeout=kubernetes.API_TIMEOUT)
+
+        # log PodList response info
+        if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+            logger.debug(f'k8s api response for `{label_selector}`:\n'
+                         f'apiVersion={response.api_version}, '
+                         f'kind={response.kind},\n'
+                         f'metadata={response.metadata}')
+
         pods = response.items
 
-        # Log response metadata
-        # pylint: disable=protected-access
-        logger.debug(
-            f'Query response for skypilot cluster {cluster_name_on_cloud}: '
-            f'resource_version={response.metadata.resource_version}, '
-            f'pod_count={len(pods)}, '
-            f'continue_token={response.metadata._continue}')
+        # log detailed Pod info
+        if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+            logger.debug(f'k8s api response for `{label_selector}`: '
+                         f'len(pods)={len(pods)}')
+            for pod in pods:
+                logger.debug(f'k8s pod info for `{label_selector}`: '
+                             f'pod.apiVersion={pod.api_version}, '
+                             f'pod.kind={pod.kind}, \n'
+                             f'pod.name={pod.metadata.name}, '
+                             f'pod.namespace={pod.metadata.namespace}, \n'
+                             f'pod.labels={pod.metadata.labels}, \n'
+                             f'pod.annotations={pod.metadata.annotations}, \n'
+                             'pod.creationTimestamp='
+                             f'{pod.metadata.creation_timestamp}, '
+                             'pod.deletionTimestamp='
+                             f'{pod.metadata.deletion_timestamp}, \n'
+                             f'pod.status={pod.status}')
+
     except kubernetes.max_retry_error():
         with ux_utils.print_exception_no_traceback():
             if is_ssh: