[flaky] pkg/activator/net: TestRevisionBackendManagerAddEndpoint #16204
Description
There's a data race in the test
From: https://prow.knative.dev/view/gs/knative-prow/logs/nightly_serving_main_periodic/1983836148849446912
=== Failed
=== FAIL: pkg/activator/net TestRevisionBackendManagerAddEndpoint (0.21s)
logger.go:146: 2025-10-30T10:08:49.561Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.562Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
testing.go:1617: race detected during execution of test
logger.go:146: 2025-10-30T10:08:49.353Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.353Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.354Z DEBUG net/revision_backends.go:437 Updating Endpoints: ready backends: 1, not-ready backends: 0 {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.354Z WARN net/revision_backends.go:342 Failed probing pods {"knative.dev/key": "test-namespace/test-revision", "curDests": {"ready": "128.0.0.1:1234", "notReady": ""}, "error": "unexpected status code: want [200], got 503"}
logger.go:146: 2025-10-30T10:08:49.354Z DEBUG net/revision_backends.go:348 Done probing, got 0 healthy pods {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.354Z ERROR net/revision_backends.go:398 Failed to probe clusterIP 129.0.0.1:1234 {"knative.dev/key": "test-namespace/test-revision", "error": "error roundtripping http://129.0.0.1:1234/healthz: clusterIP transport error"}
logger.go:146: 2025-10-30T10:08:49.354Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "128.0.0.1:1234", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.354Z DEBUG net/revision_backends.go:426 Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.439Z DEBUG net/revision_backends.go:348 Done probing, got 1 healthy pods {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.439Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "128.0.0.1:1234", "notReady": ""}, "healthy": {"keys": "128.0.0.1:1234"}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.439Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
--- PASS: TestRevisionBackendManagerAddEndpoint/add_slow_healthy (0.09s)
logger.go:146: 2025-10-30T10:08:49.441Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.442Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.442Z DEBUG net/revision_backends.go:437 Updating Endpoints: ready backends: 1, not-ready backends: 0 {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.442Z WARN net/revision_backends.go:342 Failed probing pods {"knative.dev/key": "test-namespace/test-revision", "curDests": {"ready": "128.0.0.1:1234", "notReady": ""}, "error": "unexpected status code: want [200], got 503"}
logger.go:146: 2025-10-30T10:08:49.442Z DEBUG net/revision_backends.go:348 Done probing, got 0 healthy pods {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.442Z ERROR net/revision_backends.go:398 Failed to probe clusterIP 129.0.0.1:1234 {"knative.dev/key": "test-namespace/test-revision", "error": "error roundtripping http://129.0.0.1:1234/healthz: clusterIP transport error"}
logger.go:146: 2025-10-30T10:08:49.442Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "128.0.0.1:1234", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.443Z DEBUG net/revision_backends.go:426 Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.492Z DEBUG net/revision_backends.go:348 Done probing, got 1 healthy pods {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.492Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "128.0.0.1:1234", "notReady": ""}, "healthy": {"keys": "128.0.0.1:1234"}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.492Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
--- PASS: TestRevisionBackendManagerAddEndpoint/add_slow_ready_http2 (0.05s)
logger.go:146: 2025-10-30T10:08:49.540Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision1", "dests": {"ready": "", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.540Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision1"}
logger.go:146: 2025-10-30T10:08:49.541Z DEBUG net/revision_backends.go:437 Updating Endpoints: ready backends: 1, not-ready backends: 0 {"knative.dev/key": "test-namespace/test-revision1"}
logger.go:146: 2025-10-30T10:08:49.541Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision2", "dests": {"ready": "", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.541Z DEBUG net/revision_backends.go:348 Done probing, got 1 healthy pods {"knative.dev/key": "test-namespace/test-revision1"}
logger.go:146: 2025-10-30T10:08:49.541Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision2"}
logger.go:146: 2025-10-30T10:08:49.541Z DEBUG net/revision_backends.go:437 Updating Endpoints: ready backends: 1, not-ready backends: 0 {"knative.dev/key": "test-namespace/test-revision2"}
logger.go:146: 2025-10-30T10:08:49.541Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision1", "dests": {"ready": "128.0.0.1:1234", "notReady": ""}, "healthy": {"keys": "128.0.0.1:1234"}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.541Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision1"}
logger.go:146: 2025-10-30T10:08:49.542Z DEBUG net/revision_backends.go:348 Done probing, got 1 healthy pods {"knative.dev/key": "test-namespace/test-revision2"}
logger.go:146: 2025-10-30T10:08:49.542Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision2", "dests": {"ready": "128.1.0.2:1235", "notReady": ""}, "healthy": {"keys": "128.1.0.2:1235"}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.542Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision2"}
--- PASS: TestRevisionBackendManagerAddEndpoint/multiple_revisions (0.05s)
--- PASS: TestRevisionBackendManagerAddEndpoint/no_pods_available,_but_non-mesh-related_error (0.00s)
logger.go:146: 2025-10-30T10:08:49.548Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.548Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.548Z DEBUG net/revision_backends.go:437 Updating Endpoints: ready backends: 1, not-ready backends: 0 {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.549Z WARN net/revision_backends.go:342 Failed probing pods {"knative.dev/key": "test-namespace/test-revision", "curDests": {"ready": "128.0.0.1:1234", "notReady": ""}, "error": "unexpected status code: want [200], got 503"}
logger.go:146: 2025-10-30T10:08:49.549Z DEBUG net/revision_backends.go:348 Done probing, got 0 healthy pods {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.549Z DEBUG net/revision_backends.go:403 ClusterIP is successfully probed: 129.0.0.1:1234 (ready backends: 1) {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.549Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "128.0.0.1:1234", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": true}
logger.go:146: 2025-10-30T10:08:49.549Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
--- PASS: TestRevisionBackendManagerAddEndpoint/no_pod_addressability (0.00s)
--- PASS: TestRevisionBackendManagerAddEndpoint/unhealthy (0.00s)
logger.go:146: 2025-10-30T10:08:49.553Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.553Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.553Z DEBUG net/revision_backends.go:437 Updating Endpoints: ready backends: 0, not-ready backends: 1 {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.554Z DEBUG net/revision_backends.go:348 Done probing, got 1 healthy pods {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.554Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "", "notReady": "128.0.0.1:1234"}, "healthy": {"keys": "128.0.0.1:1234"}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.554Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
--- PASS: TestRevisionBackendManagerAddEndpoint/unready_pod_successfully_probed (0.00s)
--- PASS: TestRevisionBackendManagerAddEndpoint/pod_with_exec_probe_only_goes_ready_when_kubernetes_agrees (0.00s)
logger.go:146: 2025-10-30T10:08:49.558Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.558Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.558Z DEBUG net/revision_backends.go:437 Updating Endpoints: ready backends: 1, not-ready backends: 0 {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.559Z DEBUG net/revision_backends.go:348 Done probing, got 1 healthy pods {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.559Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "128.0.0.1:1234", "notReady": ""}, "healthy": {"keys": "128.0.0.1:1234"}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.559Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
--- PASS: TestRevisionBackendManagerAddEndpoint/pod_with_exec_probe_goes_ready_when_kubernetes_agrees (0.00s)
--- PASS: TestRevisionBackendManagerAddEndpoint/pod_with_sidecar_container_with_exec_probe_only_goes_ready_when_kubernetes_agrees (0.00s)
==================
WARNING: DATA RACE
Read at 0x00c0006c458b by goroutine 492:
testing.(*common).destination()
/root/.gvm/gos/go1.25.3/src/testing/testing.go:1049 +0x96
testing.(*common).log()
/root/.gvm/gos/go1.25.3/src/testing/testing.go:1027 +0xbe
testing.(*common).Logf()
/root/.gvm/gos/go1.25.3/src/testing/testing.go:1191 +0x8e
go.uber.org/zap/zaptest.TestingWriter.Write()
/home/prow/go/src/knative.dev/serving/vendor/go.uber.org/zap/zaptest/logger.go:146 +0x119
go.uber.org/zap/zaptest.(*TestingWriter).Write()
<autogenerated>:1 +0x74
go.uber.org/zap/zapcore.(*ioCore).Write()
/home/prow/go/src/knative.dev/serving/vendor/go.uber.org/zap/zapcore/core.go:99 +0x18d
go.uber.org/zap/zapcore.(*CheckedEntry).Write()
/home/prow/go/src/knative.dev/serving/vendor/go.uber.org/zap/zapcore/entry.go:253 +0x1ec
go.uber.org/zap.(*SugaredLogger).log()
/home/prow/go/src/knative.dev/serving/vendor/go.uber.org/zap/sugar.go:355 +0x12c
go.uber.org/zap.(*SugaredLogger).Debugw()
/home/prow/go/src/knative.dev/serving/vendor/go.uber.org/zap/sugar.go:251 +0x55e
knative.dev/serving/pkg/activator/net.(*revisionWatcher).run()
/home/prow/go/src/knative.dev/serving/pkg/activator/net/revision_backends.go:422 +0x18c
knative.dev/serving/pkg/activator/net.(*revisionBackendsManager).getOrCreateRevisionWatcher.gowrap2()
/home/prow/go/src/knative.dev/serving/pkg/activator/net/revision_backends.go:558 +0x44
Previous write at 0x00c0006c458b by goroutine 480:
testing.tRunner.func1()
/root/.gvm/gos/go1.25.3/src/testing/testing.go:1921 +0x904
runtime.deferreturn()
/root/.gvm/gos/go1.25.3/src/runtime/panic.go:589 +0x5d
testing.(*T).Run.gowrap1()
/root/.gvm/gos/go1.25.3/src/testing/testing.go:1997 +0x44
Goroutine 492 (running) created at:
knative.dev/serving/pkg/activator/net.(*revisionBackendsManager).getOrCreateRevisionWatcher()
/home/prow/go/src/knative.dev/serving/pkg/activator/net/revision_backends.go:558 +0xf67
knative.dev/serving/pkg/activator/net.(*revisionBackendsManager).endpointsUpdated()
/home/prow/go/src/knative.dev/serving/pkg/activator/net/revision_backends.go:577 +0x147
knative.dev/serving/pkg/activator/net.(*revisionBackendsManager).endpointsUpdated-fm()
<autogenerated>:1 +0x47
k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd()
/home/prow/go/src/knative.dev/serving/vendor/k8s.io/client-go/tools/cache/controller.go:257 +0x63
k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd()
<autogenerated>:1 +0x1b
k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd()
/home/prow/go/src/knative.dev/serving/vendor/k8s.io/client-go/tools/cache/controller.go:320 +0x82
k8s.io/client-go/tools/cache.(*FilteringResourceEventHandler).OnAdd()
<autogenerated>:1 +0x75
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
/home/prow/go/src/knative.dev/serving/vendor/k8s.io/client-go/tools/cache/shared_informer.go:1077 +0x1e4
k8s.io/client-go/tools/cache.(*processorListener).run()
/home/prow/go/src/knative.dev/serving/vendor/k8s.io/client-go/tools/cache/shared_informer.go:1087 +0x5e
k8s.io/client-go/tools/cache.(*processorListener).run-fm()
<autogenerated>:1 +0x33
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
/home/prow/go/src/knative.dev/serving/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:72 +0x82
Goroutine 480 (running) created at:
testing.(*T).Run()
/root/.gvm/gos/go1.25.3/src/testing/testing.go:1997 +0x9d2
knative.dev/serving/pkg/activator/net.TestRevisionBackendManagerAddEndpoint()
/home/prow/go/src/knative.dev/serving/pkg/activator/net/revision_backends_test.go:967 +0x4df7
testing.tRunner()
/root/.gvm/gos/go1.25.3/src/testing/testing.go:1934 +0x21c
testing.(*T).Run.gowrap1()
/root/.gvm/gos/go1.25.3/src/testing/testing.go:1997 +0x44
==================
logger.go:146: 2025-10-30T10:08:49.564Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "", "notReady": ""}, "healthy": {"keys": ""}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.565Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.565Z DEBUG net/revision_backends.go:437 Updating Endpoints: ready backends: 1, not-ready backends: 0 {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.565Z DEBUG net/revision_backends.go:348 Done probing, got 1 healthy pods {"knative.dev/key": "test-namespace/test-revision"}
logger.go:146: 2025-10-30T10:08:49.565Z DEBUG net/revision_backends.go:422 Revision state {"knative.dev/key": "test-namespace/test-revision", "dests": {"ready": "128.0.0.1:1234", "notReady": ""}, "healthy": {"keys": "128.0.0.1:1234"}, "clusterIPHealthy": false}
logger.go:146: 2025-10-30T10:08:49.565Z DEBUG net/revision_backends.go:429 Not Probing on timer {"knative.dev/key": "test-namespace/test-revision"}
--- PASS: TestRevisionBackendManagerAddEndpoint/pod_with_sidecar_container_with_exec_probe_goes_ready_when_kubernetes_agrees (0.00s)