Merge branch 'master' into test_801715791

Signed-off-by: Alex <[email protected]>

Author: droctothorpe

.github/OWNERS

@@ -2,9 +2,7 @@ approvers:
   - hbelmiro
   - DharmitD
   - mprahl
-  - nsingla
 reviewers:
   - rimolive
   - droctothorpe
   - gmfrasca
-  - nsingla

.github/actions/deploy/action.yml

@@ -27,6 +27,10 @@ inputs:
     description: "If KFP should be deployed in multi-user mode"
     required: false
     default: 'false'
+  artifact_proxy:
+    description: "Enables artifact proxy"
+    required: false
+    default: 'false'
   storage_backend:
     description: "Storage backend to use (minio or seaweedfs)"
     required: false
@@ -38,6 +42,10 @@ inputs:
     required: false
     default: 'true'
     description: "If you want to forward API server port to localhost:8888"
+  pod_to_pod_tls_enabled:
+    description: "If KFP should be deployed with TLS pod-to-pod communication."
+    required: false
+    default: 'false'
 
 runs:
   using: "composite"
@@ -56,7 +64,7 @@ runs:
     - name: Load Docker Images
       shell: bash
       run: |
-        APPS=("apiserver" "driver" "launcher" "scheduledworkflow" "persistenceagent" "frontend")
+        APPS=("apiserver" "driver" "launcher" "scheduledworkflow" "persistenceagent" "frontend" "metadata-writer")
         for app in "${APPS[@]}"; do
           docker image load -i ${{ inputs.image_path }}/$app/$app.tar
           docker push ${{ inputs.image_registry }}/$app:${{ inputs.image_tag }}
@@ -90,6 +98,11 @@ runs:
           ARGS="${ARGS} --multi-user"
         fi
 
+        if [ "${{ inputs.artifact_proxy }}" = "true" ]; then
+          echo "Enabling artifact proxy"
+          ARGS="${ARGS} --artifact-proxy"
+        fi
+
         if [ "${{ inputs.storage_backend }}" != "seaweedfs" ] && [ -n "${{ inputs.storage_backend }}" ]; then
           echo "Deploying with artifact storage ${{ inputs.storage_backend }}"
           ARGS="${ARGS} --storage ${{ inputs.storage_backend }}"
@@ -99,6 +112,9 @@ runs:
           echo "Deploying with argo version ${{ inputs.argo_version }}"
           ARGS="${ARGS} --argo-version ${{ inputs.argo_version }}"
         fi
+        if [ "${{inputs.pod_to_pod_tls_enabled }}" = "true" ]; then
+          ARGS="${ARGS} --tls-enabled"
+        fi
         echo "ARGS=$ARGS" >> $GITHUB_OUTPUT
 
     - name: Deploy KFP

.github/actions/test-and-report/action.yml

@@ -48,6 +48,14 @@ inputs:
     required: false
     default: ""
     description: "Override it if you want a custom name for your test report file"
+  tls_enabled:
+    description: "If KFP should be deployed with TLS pod-to-pod communication."
+    required: false
+    default: 'false'
+  ca_cert_path:
+    description: "Path to the CA certificate file."
+    required: false
+    default: ""
 
 
 runs:
@@ -73,9 +81,17 @@ runs:
         if [ -z $MULTI_USER ]; then
           MULTI_USER='false'
         fi
+        TLS_ENABLED=${{ inputs.tls_enabled }}
+        if [ -z $TLS_ENABLED ]; then
+          TLS_ENABLED='false'
+        fi
+        CA_CERT_PATH=${{ inputs.ca_cert_path }}
+        if [ -z $CA_CERT_PATH ]; then
+          CA_CERT_PATH=''
+        fi
         PULL_NUMBER="${{ github.event.inputs.pull_number || github.event.pull_request.number }}"
         REPO_NAME="${{ github.repository }}"
-        go run github.com/onsi/ginkgo/v2/ginkgo -r -v --cover -p --keep-going --github-output=true --nodes=${{ inputs.num_parallel_nodes }} -v --label-filter=${{ inputs.test_label }} -- -namespace=${{ inputs.default_namespace }} -multiUserMode=$MULTI_USER -useProxy=$USE_PROXY -userNamespace=${{ inputs.user_namespace }} -uploadPipelinesWithKubernetes=${{ inputs.upload_pipelines_with_kubernetes_client}} -pullNumber=$PULL_NUMBER -repoName=$REPO_NAME
+        go run github.com/onsi/ginkgo/v2/ginkgo -r -v --cover -p --keep-going --github-output=true --nodes=${{ inputs.num_parallel_nodes }} -v --label-filter=${{ inputs.test_label }} -- -namespace=${{ inputs.default_namespace }} -multiUserMode=$MULTI_USER -useProxy=$USE_PROXY -userNamespace=${{ inputs.user_namespace }} -uploadPipelinesWithKubernetes=${{ inputs.upload_pipelines_with_kubernetes_client}} -tlsEnabled=$TLS_ENABLED -caCertPath=$CA_CERT_PATH -pullNumber=$PULL_NUMBER -repoName=$REPO_NAME
       continue-on-error: true
 
     - name: Collect Pod logs in case of Test Failures

.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../default
+
+patches:
+- target:
+    kind: ConfigMap
+    name: pipeline-install-config
+  patch: |
+    - op: add
+      path: /data/ARTIFACTS_PROXY_ENABLED
+      value: "true"
+- target:
+    kind: Deployment
+    name: kubeflow-pipelines-profile-controller
+  patch: |
+    - op: add
+      path: /spec/template/spec/containers/0/env/-
+      value:
+        name: DISABLE_ISTIO_SIDECAR 
+        value: "true" # Dangerous and only for CI/CD, not production usage
+

.github/resources/manifests/standalone/tls-enabled/apiserver-env.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: V2_DRIVER_IMAGE
+              value: kind-registry:5000/driver:latest
+            - name: V2_LAUNCHER_IMAGE
+              value: kind-registry:5000/launcher:latest
+            - name: LOG_LEVEL
+              value: "debug"
+            - name: TLS_ENABLED
+              value: "true"

.github/resources/manifests/standalone/tls-enabled/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../../../manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls
+
+images:
+  - name: ghcr.io/kubeflow/kfp-api-server
+    newName: kind-registry:5000/apiserver
+    newTag: latest
+  - name: ghcr.io/kubeflow/kfp-persistence-agent
+    newName: kind-registry:5000/persistenceagent
+    newTag: latest
+  - name: ghcr.io/kubeflow/kfp-scheduled-workflow-controller
+    newName: kind-registry:5000/scheduledworkflow
+    newTag: latest
+  - name: ghcr.io/kubeflow/kfp-frontend
+    newName: kind-registry:5000/frontend
+    newTag: latest
+  - name: ghcr.io/kubeflow/kfp-metadata-writer
+    newName: kind-registry:5000/metadata-writer
+    newTag: latest
+patches:
+  - path: apiserver-env.yaml

.github/resources/scripts/deploy-kfp.sh

@@ -28,9 +28,12 @@ TEST_MANIFESTS=".github/resources/manifests"
 PIPELINES_STORE="database"
 USE_PROXY=false
 CACHE_DISABLED=false
+ARTIFACT_PROXY_ENABLED=false
 MULTI_USER=false
 STORAGE_BACKEND="seaweedfs"
 AWF_VERSION=""
+POD_TO_POD_TLS_ENABLED=false
+SEAWEEDFS_INIT_TIMEOUT=300s
 
 # Loop over script arguments passed. This uses a single switch-case
 # block with default value in case we want to make alternative deployments
@@ -53,6 +56,10 @@ while [ "$#" -gt 0 ]; do
       MULTI_USER=true
       shift
       ;;
+    --artifact-proxy)
+      ARTIFACT_PROXY_ENABLED=true
+      shift
+      ;;
     --storage)
       STORAGE_BACKEND="$2"
       shift 2
@@ -67,6 +74,10 @@ while [ "$#" -gt 0 ]; do
         exit 1
       fi
       ;;
+    --tls-enabled)
+      POD_TO_POD_TLS_ENABLED=true
+      shift
+      ;;
   esac
 done
 
@@ -80,11 +91,6 @@ if [ "${MULTI_USER}" == "true" ] && [ "${USE_PROXY}" == "true" ]; then
   exit 1
 fi
 
-if [ "${STORAGE_BACKEND}" != "minio" ] && [ "${STORAGE_BACKEND}" != "seaweedfs" ]; then
-  echo "ERROR: Storage backend must be either 'minio' or 'seaweedfs'."
-  exit 1
-fi
-
 if [ -n "${AWF_VERSION}"  ]; then
   echo "NOTE: Argo version ${AWF_VERSION} specified, updating Argo Workflow manifests..."
   echo "${AWF_VERSION}" > third_party/argo/VERSION
@@ -100,8 +106,8 @@ if [[ $EXIT_CODE -ne 0 ]]; then
   exit $EXIT_CODE
 fi
 
-# If pipelines store is set to 'kubernetes', cert-manager must be deployed
-if [ "${PIPELINES_STORE}" == "kubernetes" ]; then
+# If pipelines store is set to 'kubernetes' or pod-to-pod TLS is set to 'true', cert-manager must be deployed
+if [ "${PIPELINES_STORE}" == "kubernetes" ] || [ "${POD_TO_POD_TLS_ENABLED}" == "true" ]; then
   #Install cert-manager
   make -C ./backend install-cert-manager || EXIT_CODE=$?
   if [[ $EXIT_CODE -ne 0 ]]
@@ -128,15 +134,6 @@ if [ "${MULTI_USER}" == "true" ]; then
   echo "Installing Profile Controller Resources..."
   kubectl apply -k https://github.com/kubeflow/manifests/applications/profiles/upstream/overlays/kubeflow?ref=master
   kubectl -n kubeflow wait --for=condition=Ready pods -l kustomize.component=profiles --timeout 180s
-
-  echo "Creating KF Profile..."
-  kubectl apply -f test_data/kubernetes/seaweedfs/test-profiles.yaml
-
-  echo "Applying kubeflow-edit ClusterRole with proper aggregation..."
-  kubectl apply -f test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml
-
-  echo "Applying network policy to allow user namespace access to kubeflow services..."
-  kubectl apply -f test_data/kubernetes/seaweedfs/allow-user-namespace-access.yaml
 fi
 
 # Manifests will be deployed according to the flag provided
@@ -156,6 +153,8 @@ if [ "${MULTI_USER}" == "false" ] && [ "${PIPELINES_STORE}" != "kubernetes" ]; t
     TEST_MANIFESTS="${TEST_MANIFESTS}/proxy-minio"
   elif $CACHE_DISABLED && $USE_PROXY && [ "${STORAGE_BACKEND}" == "minio" ]; then
     TEST_MANIFESTS="${TEST_MANIFESTS}/cache-disabled-proxy-minio"
+  elif $POD_TO_POD_TLS_ENABLED; then
+    TEST_MANIFESTS="${TEST_MANIFESTS}/tls-enabled"
   else
     TEST_MANIFESTS="${TEST_MANIFESTS}/default"
   fi
@@ -168,17 +167,20 @@ elif [ "${MULTI_USER}" == "false" ] && [ "${PIPELINES_STORE}" == "kubernetes" ];
   fi
 elif [ "${MULTI_USER}" == "true" ]; then
   TEST_MANIFESTS="${TEST_MANIFESTS}/multiuser"
-  if [ "${STORAGE_BACKEND}" == "minio" ]; then
+  if $ARTIFACT_PROXY_ENABLED && [ "${STORAGE_BACKEND}" == "seaweedfs" ]; then
+    TEST_MANIFESTS="${TEST_MANIFESTS}/artifact-proxy"
+  elif [ "${STORAGE_BACKEND}" == "minio" ]; then
     TEST_MANIFESTS="${TEST_MANIFESTS}/minio"
-  elif $CACHE_DISABLED; then
-    TEST_MANIFESTS="${TEST_MANIFESTS}/cache-disabled"
   elif $CACHE_DISABLED && [ "${STORAGE_BACKEND}" == "minio" ]; then
     TEST_MANIFESTS="${TEST_MANIFESTS}/cache-disabled-minio"
+  elif $CACHE_DISABLED; then
+    TEST_MANIFESTS="${TEST_MANIFESTS}/cache-disabled"
   else
     TEST_MANIFESTS="${TEST_MANIFESTS}/default"
   fi
 fi
 
+
 echo "Deploying ${TEST_MANIFESTS}..."
 
 kubectl apply -k "${TEST_MANIFESTS}" || EXIT_CODE=$?
@@ -196,6 +198,29 @@ then
   exit 1
 fi
 
+# Ensure SeaweedFS S3 auth is configured before proceeding
+if [ "${STORAGE_BACKEND}" == "seaweedfs" ]; then
+  wait_for_seaweedfs_init kubeflow "${SEAWEEDFS_INIT_TIMEOUT}" || EXIT_CODE=$?
+  if [[ $EXIT_CODE -ne 0 ]]
+  then
+    echo "SeaweedFS init job did not complete successfully."
+    exit 1
+  fi
+  echo "SeaweedFS init job completed successfully."
+fi
+
+if [ "${MULTI_USER}" == "true" ]; then
+  echo "Creating KF Profile..."
+  kubectl apply -f test_data/kubernetes/seaweedfs/test-profiles.yaml
+  sleep 30 # Let the profile controler reconcile the namespace
+
+  echo "Applying kubeflow-edit ClusterRole with proper aggregation..."
+  kubectl apply -f test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml
+
+  echo "Applying network policy to allow user namespace access to kubeflow services..."
+  kubectl apply -f test_data/kubernetes/seaweedfs/allow-user-namespace-access.yaml
+fi
+
 # Verify pipeline integration for multi-user mode
 if [ "${MULTI_USER}" == "true" ]; then
   echo "Verifying Pipeline Integration..."

.github/resources/scripts/forward-port.sh

@@ -23,7 +23,7 @@ APP_NAME=$2
 LOCAL_PORT=$3
 REMOTE_PORT=$4
 
-POD_NAME=$(kubectl get pods -n "$KUBEFLOW_NS" -l "app=$APP_NAME" -o jsonpath='{.items[0].metadata.name}')
+POD_NAME=$(kubectl get pods -n "$KUBEFLOW_NS" -l "app=$APP_NAME" --field-selector=status.phase=Running --sort-by='.metadata.creationTimestamp' -o jsonpath='{.items[-1].metadata.name}')
 echo "POD_NAME=$POD_NAME"
 
 if [ $QUIET -eq 1 ]; then

.github/resources/scripts/helper-functions.sh

@@ -61,6 +61,17 @@ wait_for_pods () {
     python "${C_DIR}"/kfp-readiness/wait_for_pods.py
 }
 
+wait_for_seaweedfs_init () {
+    # Wait for SeaweedFS init job to complete to ensure S3 auth is configured
+    local namespace="$1"
+    local timeout="$2"
+    if kubectl -n "$namespace" get job init-seaweedfs > /dev/null 2>&1; then
+        if ! kubectl -n "$namespace" wait --for=condition=complete --timeout="$timeout" job/init-seaweedfs; then
+            return 1
+        fi
+    fi
+}
+
 deploy_with_retries () {
     if [[ $# -ne 4 ]]
     then