Skip to content

Commit 4ff5fa9

Browse files
wenyingdwenyingd
committed
Add support for VPC mixed mode
1. Add controller for CR Network to maintain the default Network resource in system Namespaces. 2. Add watch in Pod/Namepsace/Service/NetworkPolicy controller to requeue resources when Network type is switch to VPC. 3. Add precheck on if the default Network type is VPC or not in resource create events. 4. Add webhook in NCP owned resource creations to check if the default Network type is VPC or not in a Namespace.
1 parent 566bc19 commit 4ff5fa9

File tree

18 files changed

+1410
-75
lines changed

18 files changed

+1410
-75
lines changed

‎build/yaml/webhook/manifests.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,36 @@ webhooks:
3030
resources:
3131
- subnetsets
3232
sideEffects: None
33+
- admissionReviewVersions:
34+
- v1
35+
clientConfig:
36+
service:
37+
name: subnetset
38+
namespace: vmware-system-nsx
39+
path: /validate-vpc-enablement
40+
failurePolicy: Fail
41+
name: vpcnetwork.validating.nsx.vmware.com
42+
rules:
43+
- apiGroups:
44+
- nsx.vmware.com
45+
apiVersions:
46+
- v1alpha1
47+
operations:
48+
- CREATE
49+
resources:
50+
- networkinfos
51+
- nsxserviceaccount
52+
- securitypolicies
53+
- staticroutes
54+
- subnetports
55+
- subnets
56+
- subnetsets
57+
- apiGroups:
58+
- nsx.vmware.com
59+
apiVersions:
60+
- v1alpha2
61+
operations:
62+
- CREATE
63+
resources:
64+
- ippools
65+
sideEffects: None

‎cmd/main.go‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"sigs.k8s.io/controller-runtime/pkg/healthz"
1818
logf "sigs.k8s.io/controller-runtime/pkg/log"
1919
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
20+
"sigs.k8s.io/controller-runtime/pkg/webhook"
2021

2122
"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
2223
"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2"
@@ -35,6 +36,7 @@ import (
3536
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnet"
3637
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetport"
3738
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetset"
39+
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
3840
"github.com/vmware-tanzu/nsx-operator/pkg/logger"
3941
"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
4042
"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
@@ -133,12 +135,13 @@ func StartNetworkInfoController(mgr ctrl.Manager, vpcService *vpc.VPCService) {
133135
}
134136
}
135137

136-
func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider) {
138+
func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider, networkProvider vpcnetwork.VPCNetworkProvider) {
137139
nsReconciler := &namespacecontroller.NamespaceReconciler{
138-
Client: mgr.GetClient(),
139-
Scheme: mgr.GetScheme(),
140-
NSXConfig: cf,
141-
VPCService: vpcService,
140+
Client: mgr.GetClient(),
141+
Scheme: mgr.GetScheme(),
142+
NSXConfig: cf,
143+
VPCService: vpcService,
144+
NetworkProvider: networkProvider,
142145
}
143146

144147
if err := nsReconciler.Start(mgr); err != nil {
@@ -149,14 +152,26 @@ func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vp
149152

150153
func main() {
151154
log.Info("starting NSX Operator")
152-
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
155+
mgrOptions := ctrl.Options{
153156
Scheme: scheme,
154157
HealthProbeBindAddress: config.ProbeAddr,
155158
Metrics: metricsserver.Options{BindAddress: config.MetricsAddr},
156159
LeaderElection: cf.HAEnabled(),
157160
LeaderElectionNamespace: nsxOperatorNamespace,
158161
LeaderElectionID: "nsx-operator",
159-
})
162+
}
163+
164+
enableWebhook := true
165+
if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
166+
log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
167+
enableWebhook = false
168+
} else {
169+
mgrOptions.WebhookServer = webhook.NewServer(webhook.Options{
170+
Port: config.WebhookServerPort,
171+
CertDir: config.WebhookCertDir,
172+
})
173+
}
174+
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOptions)
160175
if err != nil {
161176
log.Error(err, "failed to init manager")
162177
os.Exit(1)
@@ -181,6 +196,10 @@ func main() {
181196
var vpcService *vpc.VPCService
182197

183198
if cf.CoeConfig.EnableVPCNetwork {
199+
if !enableWebhook {
200+
log.Error(nil, "Webhook cert is not provided, can't filter out the CRs in a non-VPC namespace")
201+
os.Exit(1)
202+
}
184203
// Check NSX version for VPC networking mode
185204
if !commonService.NSXClient.NSXCheckVersion(nsx.VPC) {
186205
log.Error(nil, "VPC mode cannot be enabled if NSX version is lower than 4.1.1")
@@ -218,28 +237,24 @@ func main() {
218237
os.Exit(1)
219238
}
220239
// Start controllers which only supports VPC
240+
vpcNetworkProvider := vpcnetwork.StartNetworkController(mgr)
221241
StartNetworkInfoController(mgr, vpcService)
222-
StartNamespaceController(mgr, cf, vpcService)
242+
StartNamespaceController(mgr, cf, vpcService, vpcNetworkProvider)
223243
// Start subnet/subnetset controller.
224244
if err := subnet.StartSubnetController(mgr, subnetService, subnetPortService, vpcService); err != nil {
225245
os.Exit(1)
226246
}
227-
enableWebhook := true
228-
if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
229-
log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
230-
enableWebhook = false
231-
}
232247
if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService, enableWebhook); err != nil {
233248
os.Exit(1)
234249
}
235250

236251
node.StartNodeController(mgr, nodeService)
237252
staticroutecontroller.StartStaticRouteController(mgr, staticRouteService)
238253
subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService)
239-
pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService)
254+
pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService, vpcNetworkProvider)
240255
StartIPPoolController(mgr, ipPoolService, vpcService)
241-
networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService)
242-
service.StartServiceLbController(mgr, commonService)
256+
networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService, vpcNetworkProvider)
257+
service.StartServiceLbController(mgr, commonService, vpcNetworkProvider)
243258
}
244259
// Start controllers which can run in non-VPC mode
245260
securitypolicycontroller.StartSecurityPolicyController(mgr, commonService, vpcService)

‎go.mod‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@ replace (
66
github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
77
github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1 => ./pkg/apis/v1alpha1
88
github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2 => ./pkg/apis/v1alpha2
9-
github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
9+
github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
1010
)
1111

1212
require (
1313
github.com/agiledragon/gomonkey/v2 v2.9.0
1414
github.com/apparentlymart/go-cidr v1.1.0
1515
github.com/deckarep/golang-set v1.8.0
1616
github.com/go-logr/logr v1.3.0
17+
github.com/go-logr/zapr v1.2.4
1718
github.com/golang-jwt/jwt v3.2.2+incompatible
1819
github.com/golang/mock v1.6.0
1920
github.com/google/uuid v1.3.0
@@ -23,6 +24,7 @@ require (
2324
github.com/prometheus/client_golang v1.16.0
2425
github.com/sirupsen/logrus v1.9.0
2526
github.com/stretchr/testify v1.8.4
27+
github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1
2628
github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240305035435-c992c623aad3
2729
github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f
2830
github.com/vmware-tanzu/vm-operator/api v1.8.2
@@ -53,7 +55,6 @@ require (
5355
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
5456
github.com/fsnotify/fsnotify v1.7.0 // indirect
5557
github.com/gibson042/canonicaljson-go v1.0.3 // indirect
56-
github.com/go-logr/zapr v1.2.4 // indirect
5758
github.com/go-openapi/jsonpointer v0.19.6 // indirect
5859
github.com/go-openapi/jsonreference v0.20.2 // indirect
5960
github.com/go-openapi/swag v0.22.3 // indirect

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,8 +177,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
177177
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
178178
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
179179
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
180-
github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f h1:EV4eiUQr3QpUGfTtqdVph0+bmE+3cj0aNJpd9n2qTdo=
181-
github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f/go.mod h1:dzob8tUzpAREQPtbbjQs4b1UyQDR37B2TiIdg8WJSRM=
180+
github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1 h1:r8RuSJnLEStOdGTfaZeOjH4rvRB4Gm/N1+qtU16VrI0=
181+
github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1/go.mod h1:w6QJGm3crIA16ZIz1FVQXD2NVeJhOgGXxW05RbVTSTo=
182182
github.com/vmware-tanzu/vm-operator/api v1.8.2 h1:7cZHVusqAmAMFWvsiU7X5xontxdjasknI/sVfe0p0Z4=
183183
github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wfrj+0nH2zSEc9Q7AzWJ54=
184184
github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I=

‎pkg/controllers/namespace/namespace_controller.go‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,21 @@ import (
88
"errors"
99
"fmt"
1010

11+
netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
1112
v1 "k8s.io/api/core/v1"
1213
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1314
apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
15+
apitypes "k8s.io/apimachinery/pkg/types"
1416
"k8s.io/client-go/util/retry"
1517
ctrl "sigs.k8s.io/controller-runtime"
18+
"sigs.k8s.io/controller-runtime/pkg/builder"
1619
"sigs.k8s.io/controller-runtime/pkg/client"
1720
"sigs.k8s.io/controller-runtime/pkg/controller"
1821

1922
"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
2023
"github.com/vmware-tanzu/nsx-operator/pkg/config"
2124
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
25+
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
2226
"github.com/vmware-tanzu/nsx-operator/pkg/logger"
2327
"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
2428
_ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
@@ -35,10 +39,11 @@ var (
3539
// Using vpcservice provider instead of vpc service to prevent
3640
// invoking method that should be exposed to other module.
3741
type NamespaceReconciler struct {
38-
Client client.Client
39-
Scheme *apimachineryruntime.Scheme
40-
NSXConfig *config.NSXOperatorConfig
41-
VPCService types.VPCServiceProvider
42+
Client client.Client
43+
Scheme *apimachineryruntime.Scheme
44+
NSXConfig *config.NSXOperatorConfig
45+
VPCService types.VPCServiceProvider
46+
NetworkProvider vpcnetwork.VPCNetworkProvider
4247
}
4348

4449
func (r *NamespaceReconciler) getDefaultNetworkConfigName() (string, error) {
@@ -179,6 +184,10 @@ func (r *NamespaceReconciler) insertNamespaceNetworkconfigBinding(ns string, ann
179184
return nil
180185
}
181186

187+
func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
188+
return r.NetworkProvider.ReconcileWithVPCFilters("namespace", ctx, req, r.reconcile)
189+
}
190+
182191
/*
183192
VPC creation strategy:
184193
@@ -193,7 +202,7 @@ We suppose namespace should have following annotations:
193202
- If the ns do not have either of the annotation above, then we believe it is using default VPC, try to search
194203
default VPC in network config CR store. The default VPC network config CR's name is "default".
195204
*/
196-
func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
205+
func (r *NamespaceReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
197206
obj := &v1.Namespace{}
198207
log.Info("reconciling K8s namespace", "namespace", req.NamespacedName)
199208
metrics.CounterInc(r.NSXConfig, metrics.ControllerSyncTotal, common.MetricResTypeNamespace)
@@ -251,6 +260,14 @@ func (r *NamespaceReconciler) setupWithManager(mgr ctrl.Manager) error {
251260
controller.Options{
252261
MaxConcurrentReconciles: common.NumReconcile(),
253262
}).
263+
Watches(
264+
&netopv1alpha1.Network{},
265+
&vpcnetwork.EnqueueRequestForNetwork{Client: r.Client, Lister: func(namespace string) ([]apitypes.NamespacedName, error) {
266+
obj := apitypes.NamespacedName{Name: namespace, Namespace: namespace}
267+
return []apitypes.NamespacedName{obj}, nil
268+
}},
269+
builder.WithPredicates(vpcnetwork.PredicateFuncsByNetwork),
270+
).
254271
Complete(r)
255272
}
256273

‎pkg/controllers/namespace/namespace_controller_test.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"sigs.k8s.io/controller-runtime/pkg/client/fake"
1515

1616
"github.com/vmware-tanzu/nsx-operator/pkg/config"
17+
vpcnetworktesting "github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork/testing"
1718
"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
1819
"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
1920
"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
@@ -40,9 +41,10 @@ func createNameSpaceReconciler() *NamespaceReconciler {
4041
}
4142

4243
return &NamespaceReconciler{
43-
Client: fake.NewClientBuilder().Build(),
44-
Scheme: fake.NewClientBuilder().Build().Scheme(),
45-
VPCService: service,
44+
Client: fake.NewClientBuilder().Build(),
45+
Scheme: fake.NewClientBuilder().Build().Scheme(),
46+
VPCService: service,
47+
NetworkProvider: &vpcnetworktesting.FakeVPCNetworkProvider{},
4648
}
4749
}
4850

‎pkg/controllers/networkpolicy/networkpolicy_controller.go‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,21 @@ import (
1111
"sync"
1212
"time"
1313

14+
"github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
1415
v1 "k8s.io/api/core/v1"
1516
networkingv1 "k8s.io/api/networking/v1"
1617
apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
1718
"k8s.io/apimachinery/pkg/types"
1819
"k8s.io/apimachinery/pkg/util/sets"
1920
"k8s.io/client-go/tools/record"
2021
ctrl "sigs.k8s.io/controller-runtime"
22+
"sigs.k8s.io/controller-runtime/pkg/builder"
2123
"sigs.k8s.io/controller-runtime/pkg/client"
2224
"sigs.k8s.io/controller-runtime/pkg/controller"
2325
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2426

2527
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
28+
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
2629
"github.com/vmware-tanzu/nsx-operator/pkg/logger"
2730
"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
2831
_ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
@@ -42,10 +45,11 @@ var (
4245

4346
// NetworkPolicyReconciler reconciles a NetworkPolicy object
4447
type NetworkPolicyReconciler struct {
45-
Client client.Client
46-
Scheme *apimachineryruntime.Scheme
47-
Service *securitypolicy.SecurityPolicyService
48-
Recorder record.EventRecorder
48+
Client client.Client
49+
Scheme *apimachineryruntime.Scheme
50+
Service *securitypolicy.SecurityPolicyService
51+
Recorder record.EventRecorder
52+
NetworkProvider vpcnetwork.VPCNetworkProvider
4953
}
5054

5155
func updateFail(r *NetworkPolicyReconciler, c *context.Context, o *networkingv1.NetworkPolicy, e *error) {
@@ -71,6 +75,10 @@ func deleteSuccess(r *NetworkPolicyReconciler, _ *context.Context, o *networking
7175
func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
7276
// Use once.Do to ensure gc is called only once
7377
once.Do(func() { go r.GarbageCollector(make(chan bool), servicecommon.GCInterval) })
78+
return r.NetworkProvider.ReconcileWithVPCFilters("networkpolicy", ctx, req, r.reconcile)
79+
}
80+
81+
func (r *NetworkPolicyReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
7482
networkPolicy := &networkingv1.NetworkPolicy{}
7583
log.Info("reconciling networkpolicy", "networkpolicy", req.NamespacedName)
7684
metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType)
@@ -135,6 +143,11 @@ func (r *NetworkPolicyReconciler) setupWithManager(mgr ctrl.Manager) error {
135143
controller.Options{
136144
MaxConcurrentReconciles: common.NumReconcile(),
137145
}).
146+
Watches(
147+
&v1alpha1.Network{},
148+
&vpcnetwork.EnqueueRequestForNetwork{Client: r.Client, Lister: r.listNetworkPolicies},
149+
builder.WithPredicates(vpcnetwork.PredicateFuncsByNetwork),
150+
).
138151
Complete(r)
139152
}
140153

@@ -191,11 +204,28 @@ func (r *NetworkPolicyReconciler) GarbageCollector(cancel chan bool, timeout tim
191204
}
192205
}
193206

194-
func StartNetworkPolicyController(mgr ctrl.Manager, commonService servicecommon.Service, vpcService servicecommon.VPCServiceProvider) {
207+
func (r *NetworkPolicyReconciler) listNetworkPolicies(ns string) ([]types.NamespacedName, error) {
208+
npList := &networkingv1.NetworkPolicyList{}
209+
err := r.Client.List(context.Background(), npList, client.InNamespace(ns))
210+
if err != nil {
211+
return nil, err
212+
}
213+
nsNames := make([]types.NamespacedName, 0)
214+
for _, np := range npList.Items {
215+
nsNames = append(nsNames, types.NamespacedName{
216+
Namespace: np.Namespace,
217+
Name: np.Name,
218+
})
219+
}
220+
return nsNames, nil
221+
}
222+
223+
func StartNetworkPolicyController(mgr ctrl.Manager, commonService servicecommon.Service, vpcService servicecommon.VPCServiceProvider, networkProvider vpcnetwork.VPCNetworkProvider) {
195224
networkPolicyReconcile := NetworkPolicyReconciler{
196-
Client: mgr.GetClient(),
197-
Scheme: mgr.GetScheme(),
198-
Recorder: mgr.GetEventRecorderFor("networkpolicy-controller"),
225+
Client: mgr.GetClient(),
226+
Scheme: mgr.GetScheme(),
227+
Recorder: mgr.GetEventRecorderFor("networkpolicy-controller"),
228+
NetworkProvider: networkProvider,
199229
}
200230
networkPolicyReconcile.Service = securitypolicy.GetSecurityService(commonService, vpcService)
201231
if err := networkPolicyReconcile.Start(mgr); err != nil {

0 commit comments

Comments
 (0)