Serve --server-log in API server helm deployment (#7226)

* Server server log in container and rotate it

Signed-off-by: Aylei <[email protected]>

* Update

Signed-off-by: Aylei <[email protected]>

* Update doc

Signed-off-by: Aylei <[email protected]>

* Update

Signed-off-by: Aylei <[email protected]>

* Tune defaults

Signed-off-by: Aylei <[email protected]>

* Update

Signed-off-by: Aylei <[email protected]>

---------

Signed-off-by: Aylei <[email protected]>

Author: aylei

Dockerfile

@@ -68,7 +68,7 @@ ARG NEXT_BASE_PATH=/dashboard
 RUN apt-get update -y && \
     apt-get install --no-install-recommends -y \
         git gcc rsync sudo patch openssh-server \
-        pciutils nano fuse socat netcat-openbsd curl tini autossh jq && \
+        pciutils nano fuse socat netcat-openbsd curl tini autossh jq logrotate && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Install the session manager plugin for AWS CLI.

charts/skypilot/templates/api-deployment.yaml

@@ -213,7 +213,8 @@ spec:
           {{- end }}
 
           if sky api start -h | grep -q -- "--foreground"; then
-            exec sky api start {{ include "skypilot.apiArgs" . }} --foreground
+            mkdir -p /root/.sky/api_server
+            exec sky api start {{ include "skypilot.apiArgs" . }} --foreground  2>&1 | tee -a /root/.sky/api_server/server.log
           else
             # For backward compatibility, run in background if --foreground is not supported.
             # TODO(aylei): this will be dropped in 0.11.0.
@@ -249,10 +250,10 @@ spec:
           periodSeconds: 5
           initialDelaySeconds: 5
         volumeMounts:
-        {{- if .Values.storage.enabled }}
         - name: state-volume
           mountPath: /root/.sky
           subPath: .sky
+        {{- if .Values.storage.enabled }}
         - name: state-volume
           mountPath: /root/.ssh # To preserve the SSH keys for the user when using the API server
           subPath: .ssh
@@ -320,6 +321,38 @@ spec:
         {{- if .Values.apiService.extraVolumeMounts }}
         {{- toYaml .Values.apiService.extraVolumeMounts | nindent 8 }}
         {{- end }}
+      {{- if .Values.apiService.logs.retention.enabled }}
+      - name: logrotate
+        image: {{ .Values.apiService.image }}
+        command: ["/bin/sh", "-c"]
+        # Warn if logrotate is missing in the image and keep the sidecar running
+        args:
+        - |
+          if ! command -v logrotate >/dev/null 2>&1; then
+            echo "warning: logrotate not found; skipping log retention" >&2
+            while true; do
+              sleep 3600
+            done
+          fi
+          set -x
+          cat > /etc/logrotate.conf <<EOF
+          /root/.sky/api_server/server.log {
+              size {{ .Values.apiService.logs.retention.size }}
+              rotate 1
+              missingok
+              notifempty
+              copytruncate
+          }
+          EOF
+          while true; do
+            logrotate -s /var/lib/logrotate/status /etc/logrotate.conf
+            sleep 60;
+          done
+        volumeMounts:
+        - name: state-volume
+          mountPath: /root/.sky
+          subPath: .sky
+      {{- end }}
       initContainers:
       {{- if .Values.awsCredentials.enabled }}
       - name: create-aws-credentials
@@ -369,19 +402,19 @@ spec:
           set -e
           echo "Setting up R2 credentials..."
           mkdir -p /root/.cloudflare
-          
+
           # Direct echo of environment variables (no quotes around EOF)
           cat > /root/.cloudflare/r2.credentials <<EOF
           $R2_CREDENTIALS
           EOF
-          
+
           cat > /root/.cloudflare/accountid <<EOF
           $R2_ACCOUNT_ID
           EOF
-          
+
           chmod 600 /root/.cloudflare/r2.credentials
           chmod 600 /root/.cloudflare/accountid
-          
+
           echo "R2 credentials setup complete:"
           ls -la /root/.cloudflare/
           echo "First line of r2.credentials:"
@@ -571,7 +604,7 @@ spec:
       - name: skypilot-ssh-identity
         secret:
           secretName: {{ .Values.apiService.sshKeySecret }}
-          defaultMode: 0600 
+          defaultMode: 0600
       {{- end }}
       {{- if .Values.apiService.extraVolumes }}
       {{- toYaml .Values.apiService.extraVolumes | nindent 6 }}

charts/skypilot/values.schema.json

@@ -80,6 +80,22 @@
                         "null"
                     ]
                 },
+                "logs": {
+                    "type": "object",
+                    "properties": {
+                        "retention": {
+                            "type": "object",
+                            "properties": {
+                                "enabled": {
+                                    "type": "boolean"
+                                },
+                                "size": {
+                                    "type": "string"
+                                }
+                            }
+                        }
+                    }
+                },
                 "metrics": {
                     "type": "object",
                     "properties": {

charts/skypilot/values.yaml

@@ -153,6 +153,14 @@ apiService:
   # [Internal] Enable developer mode for SkyPilot
   skypilotDev: false
 
+  # Configure how to manage API server logs
+  logs:
+    retention:
+      # Whether to enable log retention
+      enabled: true
+      # The size of the log file to retain
+      size: 10M
+
 # Authentication configuration for the API server
 # Refer to https://docs.skypilot.co/en/latest/reference/auth.html for more details.
 # @schema type: [object, null]

docs/source/reference/api-server/helm-values-spec.rst

@@ -73,6 +73,10 @@ Below is the available helm value keys and the default value of each key:
     :ref:`extraEnvs <helm-values-apiService-extraEnvs>`: null
     :ref:`extraVolumes <helm-values-apiService-extraVolumes>`: null
     :ref:`extraVolumeMounts <helm-values-apiService-extraVolumeMounts>`: null
+    :ref:`logs <helm-values-apiService-logs>`:
+      :ref:`retention <helm-values-apiService-logs-retention>`:
+        :ref:`enabled <helm-values-apiService-logs-retention-enabled>`: false
+        :ref:`size <helm-values-apiService-logs-retention-size>`: 10M
 
   :ref:`auth <helm-values-auth>`:
     :ref:`oauth <helm-values-auth-oauth>`:
@@ -691,6 +695,62 @@ Default: ``null``
         mountPath: /my-path
         subPath: my-file
 
+.. _helm-values-apiService-logs:
+
+``apiService.logs``
+^^^^^^^^^^^^^^^^^^^
+
+Configuration for managing API server logs.
+
+Default: see the yaml below.
+
+.. code-block:: yaml
+
+  apiService:
+    logs:
+      retention:
+        enabled: false
+        size: 10M
+
+.. _helm-values-apiService-logs-retention:
+
+``apiService.logs.retention``
+'''''''''''''''''''''''''''''
+
+Configuration for log retention settings.
+
+.. _helm-values-apiService-logs-retention-enabled:
+
+``apiService.logs.retention.enabled``
+''''''''''''''''''''''''''''''''''''''
+
+Whether to enable log retention for the API server. When enabled, logs will be automatically rotated and managed according to the specified size limit.
+
+Default: ``false``
+
+.. code-block:: yaml
+
+  apiService:
+    logs:
+      retention:
+        enabled: true
+
+.. _helm-values-apiService-logs-retention-size:
+
+``apiService.logs.retention.size``
+'''''''''''''''''''''''''''''''''''
+
+The maximum size of the log file before rotation. When the log file reaches this size, it will be rotated to preserve disk space. Only used when ``enabled`` is ``true``.
+
+Default: ``10M``
+
+.. code-block:: yaml
+
+  apiService:
+    logs:
+      retention:
+        size: 50M
+
 
 .. _helm-values-auth: