Skip to content

Commit 13812b3

Browse files
FryguyFryguy
authored
Merge pull request #1003 from bdunne/node_affinity
Add Node affinity
2 parents afd62fe + 70cad9d commit 13812b3

File tree

10 files changed

+132
-36
lines changed

10 files changed

+132
-36
lines changed

‎manageiq-operator/api/v1alpha1/helpers/miq-components/httpd.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
1010
tlstools "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/helpers/tlstools"
11+
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
1112
routev1 "github.com/openshift/api/route/v1"
1213
appsv1 "k8s.io/api/apps/v1"
1314
corev1 "k8s.io/api/core/v1"
@@ -470,6 +471,8 @@ func HttpdDeployment(client client.Client, cr *miqv1alpha1.ManageIQ, scheme *run
470471
deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: volumeName, VolumeSource: corev1.VolumeSource{Secret: &secretVolumeSource}})
471472
}
472473

474+
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)
475+
473476
return nil
474477
}
475478

‎manageiq-operator/api/v1alpha1/helpers/miq-components/kafka.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
7+
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
78
appsv1 "k8s.io/api/apps/v1"
89
corev1 "k8s.io/api/core/v1"
910
resource "k8s.io/apimachinery/pkg/api/resource"
@@ -197,7 +198,7 @@ func ZookeeperService(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*corev1
197198
return service, f
198199
}
199200

200-
func KafkaDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
201+
func KafkaDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
201202
deploymentLabels := map[string]string{
202203
"name": "kafka",
203204
"app": cr.Spec.AppName,
@@ -315,13 +316,15 @@ func KafkaDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.
315316
},
316317
},
317318
}
319+
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)
320+
318321
return nil
319322
}
320323

321324
return deployment, f, nil
322325
}
323326

324-
func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
327+
func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
325328
deploymentLabels := map[string]string{
326329
"name": "zookeeper",
327330
"app": cr.Spec.AppName,
@@ -398,6 +401,8 @@ func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*app
398401
},
399402
},
400403
}
404+
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)
405+
401406
return nil
402407
}
403408

‎manageiq-operator/api/v1alpha1/helpers/miq-components/memcached.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package miqtools
22

33
import (
44
miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
5+
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
56
appsv1 "k8s.io/api/apps/v1"
67
corev1 "k8s.io/api/core/v1"
78
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -103,6 +104,8 @@ func NewMemcachedDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl
103104
deployment.Spec.Template.Spec.Containers[0].Env = addOrUpdateEnvVar(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "MEMCACHED_EXTRA_PARAMETERS", Value: "-Z -o ssl_chain_cert=/root/server.crt -o ssl_key=/root/server.key -p 11211"})
104105
}
105106

107+
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)
108+
106109
return nil
107110
}
108111

‎manageiq-operator/api/v1alpha1/helpers/miq-components/operator.go‎

Lines changed: 8 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
7+
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
78
appsv1 "k8s.io/api/apps/v1"
89
corev1 "k8s.io/api/core/v1"
910
rbacv1 "k8s.io/api/rbac/v1"
@@ -14,7 +15,10 @@ import (
1415
)
1516

1617
func ManageOperator(cr *miqv1alpha1.ManageIQ, client client.Client) (*appsv1.Deployment, controllerutil.MutateFn) {
17-
deployment := operatorDeployment(cr, client)
18+
podName := os.Getenv("POD_NAME")
19+
pod := miqutilsv1alpha1.FindPodByName(client, cr.Namespace, podName)
20+
replicaSet := miqutilsv1alpha1.FindReplicaSetByName(client, cr.Namespace, pod.ObjectMeta.OwnerReferences[0].Name)
21+
deployment := miqutilsv1alpha1.FindDeploymentByName(client, cr.Namespace, replicaSet.ObjectMeta.OwnerReferences[0].Name)
1822

1923
f := func() error {
2024
addAppLabel(cr.Spec.AppName, &deployment.ObjectMeta)
@@ -106,38 +110,10 @@ func ManageOperatorRoleBinding(cr *miqv1alpha1.ManageIQ, client client.Client) (
106110
return operatorRoleBinding, f
107111
}
108112

109-
func operatorPod(cr *miqv1alpha1.ManageIQ, client client.Client) *corev1.Pod {
110-
operatorPodName := os.Getenv("POD_NAME")
111-
podKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorPodName}
112-
pod := &corev1.Pod{}
113-
client.Get(context.TODO(), podKey, pod)
114-
115-
return pod
116-
}
117-
118-
func operatorReplicaSet(cr *miqv1alpha1.ManageIQ, client client.Client) *appsv1.ReplicaSet {
119-
pod := operatorPod(cr, client)
120-
operatorReplicaSetName := pod.ObjectMeta.OwnerReferences[0].Name
121-
replicaSetKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorReplicaSetName}
122-
replicaSet := &appsv1.ReplicaSet{}
123-
client.Get(context.TODO(), replicaSetKey, replicaSet)
124-
125-
return replicaSet
126-
}
127-
128-
func operatorDeployment(cr *miqv1alpha1.ManageIQ, client client.Client) *appsv1.Deployment {
129-
replicaSet := operatorReplicaSet(cr, client)
130-
operatorDeploymentName := replicaSet.ObjectMeta.OwnerReferences[0].Name
131-
deploymentKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorDeploymentName}
132-
deployment := &appsv1.Deployment{}
133-
client.Get(context.TODO(), deploymentKey, deployment)
134-
135-
return deployment
136-
}
137-
138113
func operatorServiceAccount(cr *miqv1alpha1.ManageIQ, client client.Client) *corev1.ServiceAccount {
139-
deployment := operatorDeployment(cr, client)
140-
operatorServiceAccountName := deployment.Spec.Template.Spec.ServiceAccountName
114+
podName := os.Getenv("POD_NAME")
115+
pod := miqutilsv1alpha1.FindPodByName(client, cr.Namespace, podName)
116+
operatorServiceAccountName := pod.Spec.ServiceAccountName
141117
serviceAccountKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorServiceAccountName}
142118
serviceAccount := &corev1.ServiceAccount{}
143119
client.Get(context.TODO(), serviceAccountKey, serviceAccount)

‎manageiq-operator/api/v1alpha1/helpers/miq-components/orchestrator.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
7+
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
78
appsv1 "k8s.io/api/apps/v1"
89
corev1 "k8s.io/api/core/v1"
910
rbacv1 "k8s.io/api/rbac/v1"
@@ -322,6 +323,8 @@ func OrchestratorDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl
322323
}}
323324
deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: "database-secret", VolumeSource: corev1.VolumeSource{Secret: &databaseSecretVolumeSource}})
324325

326+
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)
327+
325328
return nil
326329
}
327330

‎manageiq-operator/api/v1alpha1/helpers/miq-components/postgresql.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
7+
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
78
appsv1 "k8s.io/api/apps/v1"
89
corev1 "k8s.io/api/core/v1"
910
resource "k8s.io/apimachinery/pkg/api/resource"
@@ -286,6 +287,8 @@ func PostgresqlDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme
286287

287288
addInternalCertificate(cr, deployment, client, "postgresql", "/opt/app-root/src/certificates")
288289

290+
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)
291+
289292
return nil
290293
}
291294

‎manageiq-operator/api/v1alpha1/miqutils/find.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package miqutils
2+
3+
import (
4+
"context"
5+
appsv1 "k8s.io/api/apps/v1"
6+
corev1 "k8s.io/api/core/v1"
7+
"k8s.io/apimachinery/pkg/types"
8+
"sigs.k8s.io/controller-runtime/pkg/client"
9+
)
10+
11+
func FindPodByName(client client.Client, namespace string, name string) *corev1.Pod {
12+
podKey := types.NamespacedName{Namespace: namespace, Name: name}
13+
pod := &corev1.Pod{}
14+
client.Get(context.TODO(), podKey, pod)
15+
16+
return pod
17+
}
18+
19+
func FindReplicaSetByName(client client.Client, namespace string, name string) *appsv1.ReplicaSet {
20+
replicaSetKey := types.NamespacedName{Namespace: namespace, Name: name}
21+
replicaSet := &appsv1.ReplicaSet{}
22+
client.Get(context.TODO(), replicaSetKey, replicaSet)
23+
24+
return replicaSet
25+
}
26+
27+
func FindDeploymentByName(client client.Client, namespace string, name string) *appsv1.Deployment {
28+
deploymentKey := types.NamespacedName{Namespace: namespace, Name: name}
29+
deployment := &appsv1.Deployment{}
30+
client.Get(context.TODO(), deploymentKey, deployment)
31+
32+
return deployment
33+
}

‎manageiq-operator/api/v1alpha1/miqutils/node_affinity.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package miqutils
2+
3+
import (
4+
appsv1 "k8s.io/api/apps/v1"
5+
corev1 "k8s.io/api/core/v1"
6+
"os"
7+
"sigs.k8s.io/controller-runtime/pkg/client"
8+
)
9+
10+
func OperatorNodeAffinityArchValues(deployment *appsv1.Deployment, client client.Client) []string {
11+
podName := os.Getenv("POD_NAME")
12+
pod := FindPodByName(client, deployment.ObjectMeta.Namespace, podName)
13+
values := []string{"amd64"}
14+
15+
if pod.Spec.Affinity == nil {
16+
// In case we don't find the operator pod (local testing) or it doesn't have affinities
17+
return values
18+
}
19+
20+
nodeSelectorTerms := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms
21+
22+
for _, selector := range nodeSelectorTerms {
23+
for _, matchExpression := range selector.MatchExpressions {
24+
if matchExpression.Key == "kubernetes.io/arch" {
25+
values = matchExpression.Values
26+
}
27+
}
28+
}
29+
30+
return values
31+
}
32+
33+
func SetDeploymentNodeAffinity(deployment *appsv1.Deployment, client client.Client) {
34+
operatorNodeAffinityArchValues := OperatorNodeAffinityArchValues(deployment, client)
35+
if len(operatorNodeAffinityArchValues) == 0 {
36+
// We're running local, can't find the operator pod, or it doesn't have any affinities to use as a template. Skip it.
37+
return
38+
}
39+
40+
matchExpression := corev1.NodeSelectorRequirement{
41+
Key: "kubernetes.io/arch",
42+
Operator: corev1.NodeSelectorOpIn,
43+
Values: operatorNodeAffinityArchValues,
44+
}
45+
46+
matchExpressions := []corev1.NodeSelectorRequirement{matchExpression}
47+
48+
nodeSelectorTerm := corev1.NodeSelectorTerm{
49+
MatchExpressions: matchExpressions,
50+
}
51+
52+
nodeSelectionTerms := []corev1.NodeSelectorTerm{nodeSelectorTerm}
53+
54+
deployment.Spec.Template.Spec.Affinity = &corev1.Affinity{
55+
NodeAffinity: &corev1.NodeAffinity{
56+
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
57+
NodeSelectorTerms: nodeSelectionTerms,
58+
},
59+
},
60+
}
61+
}

‎manageiq-operator/config/manager/manager.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,15 @@ spec:
1212
labels:
1313
name: manageiq-operator
1414
spec:
15+
affinity:
16+
nodeAffinity:
17+
requiredDuringSchedulingIgnoredDuringExecution:
18+
nodeSelectorTerms:
19+
- matchExpressions:
20+
- key: kubernetes.io/arch
21+
operator: In
22+
values:
23+
- amd64
1524
serviceAccountName: manageiq-operator
1625
containers:
1726
- name: manageiq-operator

‎manageiq-operator/internal/controller/manageiq_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -563,7 +563,7 @@ func (r *ManageIQReconciler) generateKafkaResources(cr *miqv1alpha1.ManageIQ) er
563563
logger.Info("Service has been reconciled", "component", "zookeeper", "result", result)
564564
}
565565

566-
kafkaDeployment, mutateFunc, err := miqtool.KafkaDeployment(cr, r.Scheme)
566+
kafkaDeployment, mutateFunc, err := miqtool.KafkaDeployment(cr, r.Client, r.Scheme)
567567
if err != nil {
568568
return err
569569
}
@@ -574,7 +574,7 @@ func (r *ManageIQReconciler) generateKafkaResources(cr *miqv1alpha1.ManageIQ) er
574574
logger.Info("Deployment has been reconciled", "component", "kafka", "result", result)
575575
}
576576

577-
zookeeperDeployment, mutateFunc, err := miqtool.ZookeeperDeployment(cr, r.Scheme)
577+
zookeeperDeployment, mutateFunc, err := miqtool.ZookeeperDeployment(cr, r.Client, r.Scheme)
578578
if err != nil {
579579
return err
580580
}

0 commit comments

Comments
 (0)