Skip to content

Commit 3db2e89

Browse files
Add support for pause scale out annotation (#7023)
* Add support for pause scale out annotation Signed-off-by: Ali Aqel <aliaqel@stripe.com> * add missing test file Signed-off-by: Ali Aqel <aliaqel@stripe.com> * Update pkg/scaling/executor/scale_scaledobjects.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: aliaqel-stripe <120822631+aliaqel-stripe@users.noreply.github.com> * Update controllers/keda/hpa.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: aliaqel-stripe <120822631+aliaqel-stripe@users.noreply.github.com> * Trigger Build Signed-off-by: Ali Aqel <aliaqel@stripe.com> --------- Signed-off-by: Ali Aqel <aliaqel@stripe.com> Signed-off-by: aliaqel-stripe <120822631+aliaqel-stripe@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 72f94ad commit 3db2e89

9 files changed

Lines changed: 534 additions & 1 deletion

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio
6666
- **General**: Add error and event for mismatching input property ([#6721](https://github.com/kedacore/keda/issues/6721))
6767
- **General**: Add fallback support for triggers of `Value` metric type ([#6655](https://github.com/kedacore/keda/pull/6655))
6868
- **General**: Add support for pause scale in annotation ([#6902](https://github.com/kedacore/keda/issues/6902))
69+
- **General**: Add support for pause scale out annotation ([#7022](https://github.com/kedacore/keda/issues/7022))
6970
- **General**: Enable support on s390x for KEDA ([#6543](https://github.com/kedacore/keda/issues/6543))
7071
- **General**: Implement Force Activation annotation ([#6903](https://github.com/kedacore/keda/issues/6903))
7172
- **General**: Introduce new Forgejo Scaler ([#6488](https://github.com/kedacore/keda/issues/6488))

‎apis/keda/v1alpha1/scaledobject_types.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ const ValidationsHpaOwnershipAnnotation = "validations.keda.sh/hpa-ownership"
5858
const PausedReplicasAnnotation = "autoscaling.keda.sh/paused-replicas"
5959
const PausedAnnotation = "autoscaling.keda.sh/paused"
6060
const PausedScaleInAnnotation = "autoscaling.keda.sh/paused-scale-in"
61+
const PausedScaleOutAnnotation = "autoscaling.keda.sh/paused-scale-out"
6162
const FallbackBehaviorStatic = "static"
6263
const FallbackBehaviorCurrentReplicas = "currentReplicas"
6364
const FallbackBehaviorCurrentReplicasIfHigher = "currentReplicasIfHigher"
@@ -240,6 +241,11 @@ func (so *ScaledObject) NeedToForceActivation() bool {
240241
return getBoolAnnotation(so, ForceActivationAnnotation)
241242
}
242243

244+
// NeedToPauseScaleOut checks whether Scale Out actions for a ScaledObject need to be blocked based on the PausedScaleOut annotation
245+
func (so *ScaledObject) NeedToPauseScaleOut() bool {
246+
return getBoolAnnotation(so, PausedScaleOutAnnotation)
247+
}
248+
243249
func getBoolAnnotation(so *ScaledObject, annotation string) bool {
244250
value, found := so.GetAnnotations()[annotation]
245251
if !found {

‎controllers/keda/hpa.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,28 @@ func (r *ScaledObjectReconciler) newHPAForScaledObject(ctx context.Context, logg
106106
)
107107
}
108108

109+
if scaledObject.NeedToPauseScaleOut() {
110+
// If the paused-scale-out annotation is set, set the HPA ScaleUp Select policy to Disabled
111+
// to prevent the HPA from scaling up the scale target
112+
if behavior == nil {
113+
behavior = &autoscalingv2.HorizontalPodAutoscalerBehavior{}
114+
}
115+
if behavior.ScaleUp == nil {
116+
behavior.ScaleUp = &autoscalingv2.HPAScalingRules{}
117+
}
118+
119+
disabledPolicy := autoscalingv2.DisabledPolicySelect
120+
behavior.ScaleUp.SelectPolicy = &disabledPolicy
121+
122+
logger.Info(
123+
"Scale out paused by annotation, setting HPA Scale Up Select Behavior to Disabled",
124+
"HPA.Namespace",
125+
scaledObject.Namespace,
126+
"HPA.Name",
127+
getHPAName(scaledObject),
128+
)
129+
}
130+
109131
// label can have max 63 chars
110132
labelName := getHPAName(scaledObject)
111133
if len(labelName) > 63 {

‎controllers/keda/scaledobject_controller.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ func (r *ScaledObjectReconciler) SetupWithManager(mgr ctrl.Manager, options cont
132132
kedacontrollerutil.PausedPredicate{},
133133
kedacontrollerutil.PausedReplicasPredicate{},
134134
kedacontrollerutil.PausedScaleInPredicate{},
135+
kedacontrollerutil.PausedScaleOutPredicate{},
135136
kedacontrollerutil.ScaleObjectReadyConditionPredicate{},
136137
kedacontrollerutil.ForceActivationPredicate{},
137138
predicate.GenerationChangedPredicate{},
@@ -234,6 +235,7 @@ func (r *ScaledObjectReconciler) reconcileScaledObject(ctx context.Context, logg
234235
// and if so, stop the scale loop and delete the HPA on the scaledObject.
235236
// Additionally, if the following annotations are present:
236237
// - "autoscaling.keda.sh/paused-scale-in"
238+
// - "autoscaling.keda.sh/paused-scale-out"
237239
// we also set the status to paused but we allow the scale loop to continue and do not delete the HPA because these are unidirectional pauses.
238240
needsToPause := scaledObject.NeedToBePausedByAnnotation()
239241
switch {
@@ -259,7 +261,7 @@ func (r *ScaledObjectReconciler) reconcileScaledObject(ctx context.Context, logg
259261
conditions.SetPausedCondition(metav1.ConditionTrue, kedav1alpha1.ScaledObjectConditionPausedReason, msg)
260262
return msg, nil
261263
}
262-
case scaledObject.NeedToPauseScaleIn():
264+
case scaledObject.NeedToPauseScaleIn() || scaledObject.NeedToPauseScaleOut():
263265
conditions.SetPausedCondition(metav1.ConditionTrue, kedav1alpha1.ScaledObjectConditionPausedReason, kedav1alpha1.ScaledObjectConditionPausedMessage)
264266
case conditions.GetPausedCondition().Status == metav1.ConditionTrue:
265267
conditions.SetPausedCondition(metav1.ConditionFalse, "ScaledObjectUnpaused", "pause annotation removed for ScaledObject")

‎controllers/keda/scaledobject_controller_test.go‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -810,6 +810,233 @@ var _ = Describe("ScaledObjectController", func() {
810810
}).WithTimeout(1 * time.Minute).WithPolling(10 * time.Second).Should(Equal(autoscalingv2.MinChangePolicySelect))
811811
})
812812

813+
It("deploys ScaledObject and creates HPA with scale up select policy disabled when pause scale-out annotation set", func() {
814+
deploymentName := "disable-scale-out"
815+
soName := "so-" + deploymentName
816+
817+
// Create the scaling target.
818+
err := k8sClient.Create(context.Background(), generateDeployment(deploymentName))
819+
Expect(err).ToNot(HaveOccurred())
820+
821+
// Create the ScaledObject
822+
so := &kedav1alpha1.ScaledObject{
823+
ObjectMeta: metav1.ObjectMeta{
824+
Name: soName,
825+
Namespace: "default",
826+
Annotations: map[string]string{
827+
kedav1alpha1.PausedScaleOutAnnotation: "true",
828+
}},
829+
Spec: kedav1alpha1.ScaledObjectSpec{
830+
ScaleTargetRef: &kedav1alpha1.ScaleTarget{
831+
Name: deploymentName,
832+
},
833+
Triggers: []kedav1alpha1.ScaleTriggers{
834+
{
835+
Type: "cron",
836+
Metadata: map[string]string{
837+
"timezone": "UTC",
838+
"start": "0 * * * *",
839+
"end": "1 * * * *",
840+
"desiredReplicas": "1",
841+
},
842+
},
843+
},
844+
},
845+
}
846+
err = k8sClient.Create(context.Background(), so)
847+
Ω(err).ToNot(HaveOccurred())
848+
849+
Eventually(func() metav1.ConditionStatus {
850+
err = k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so)
851+
Ω(err).ToNot(HaveOccurred())
852+
return so.Status.Conditions.GetPausedCondition().Status
853+
}, 5*time.Second).Should(Equal(metav1.ConditionTrue))
854+
855+
// Get and confirm the HPA
856+
hpa := &autoscalingv2.HorizontalPodAutoscaler{}
857+
Eventually(func() error {
858+
return k8sClient.Get(context.Background(), types.NamespacedName{Name: "keda-hpa-" + soName, Namespace: "default"}, hpa)
859+
}).ShouldNot(HaveOccurred())
860+
861+
Ω(*hpa.Spec.Behavior.ScaleUp.SelectPolicy).To(Equal(autoscalingv2.DisabledPolicySelect))
862+
})
863+
864+
It("sets scale up select policy to disable when annotation added to existing object", func() {
865+
deploymentName := "disable-scale-out-old-so"
866+
soName := "so-" + deploymentName
867+
868+
// Create the scaling target.
869+
err := k8sClient.Create(context.Background(), generateDeployment(deploymentName))
870+
Expect(err).ToNot(HaveOccurred())
871+
872+
minReplicaCount := int32(1)
873+
maxReplicaCount := int32(10)
874+
875+
// Create the ScaledObject with 1 trigger.
876+
so := &kedav1alpha1.ScaledObject{
877+
ObjectMeta: metav1.ObjectMeta{
878+
Name: soName,
879+
Namespace: "default",
880+
Annotations: make(map[string]string),
881+
},
882+
Spec: kedav1alpha1.ScaledObjectSpec{
883+
ScaleTargetRef: &kedav1alpha1.ScaleTarget{
884+
Name: deploymentName,
885+
},
886+
Triggers: []kedav1alpha1.ScaleTriggers{
887+
{
888+
Type: "cron",
889+
Metadata: map[string]string{
890+
"timezone": "UTC",
891+
"start": "0 * * * *",
892+
"end": "1 * * * *",
893+
"desiredReplicas": "2",
894+
},
895+
},
896+
},
897+
MinReplicaCount: &minReplicaCount,
898+
MaxReplicaCount: &maxReplicaCount,
899+
},
900+
}
901+
err = k8sClient.Create(context.Background(), so)
902+
Expect(err).ToNot(HaveOccurred())
903+
904+
testLogger.Info("Created scaled object")
905+
906+
// Get and confirm the HPA.
907+
hpa := &autoscalingv2.HorizontalPodAutoscaler{}
908+
Eventually(func() error {
909+
return k8sClient.Get(context.Background(), types.NamespacedName{Name: "keda-hpa-" + soName, Namespace: "default"}, hpa)
910+
}).ShouldNot(HaveOccurred())
911+
Expect(hpa.Spec.Behavior).To(Equal((*autoscalingv2.HorizontalPodAutoscalerBehavior)(nil)))
912+
913+
Eventually(func() metav1.ConditionStatus {
914+
err = k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so)
915+
Ω(err).ToNot(HaveOccurred())
916+
return so.Status.Conditions.GetPausedCondition().Status
917+
}, 2*time.Minute).WithPolling(5 * time.Second).Should(Or(Equal(metav1.ConditionFalse), Equal(metav1.ConditionUnknown)))
918+
919+
// Set the annotation
920+
Eventually(func() error {
921+
err = k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so)
922+
Expect(err).ToNot(HaveOccurred())
923+
924+
if so.Annotations == nil {
925+
so.Annotations = make(map[string]string)
926+
}
927+
so.Annotations[kedav1alpha1.PausedScaleOutAnnotation] = "true"
928+
929+
return k8sClient.Update(context.Background(), so)
930+
}).ShouldNot(HaveOccurred())
931+
932+
Eventually(func() metav1.ConditionStatus {
933+
err = k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so)
934+
Ω(err).ToNot(HaveOccurred())
935+
return so.Status.Conditions.GetPausedCondition().Status
936+
}, 2*time.Minute).WithPolling(5 * time.Second).Should(Equal(metav1.ConditionTrue))
937+
938+
Eventually(func() autoscalingv2.ScalingPolicySelect {
939+
err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "keda-hpa-" + soName, Namespace: "default"}, hpa)
940+
Expect(err).ToNot(HaveOccurred())
941+
if hpa.Spec.Behavior != nil && hpa.Spec.Behavior.ScaleUp != nil && hpa.Spec.Behavior.ScaleUp.SelectPolicy != nil {
942+
return *hpa.Spec.Behavior.ScaleUp.SelectPolicy
943+
} else {
944+
return ""
945+
}
946+
}).WithTimeout(1 * time.Minute).WithPolling(10 * time.Second).Should(Equal(autoscalingv2.DisabledPolicySelect))
947+
})
948+
949+
It("sets scale up select policy to original policy when annotation removed", func() {
950+
deploymentName := "remove-scale-out-disabled-annotation"
951+
soName := "so-" + deploymentName
952+
953+
// Create the scaling target.
954+
err := k8sClient.Create(context.Background(), generateDeployment(deploymentName))
955+
Expect(err).ToNot(HaveOccurred())
956+
957+
// Create the ScaledObject with 1 trigger.
958+
minPolicy := autoscalingv2.MinChangePolicySelect
959+
minReplicaCount := int32(1)
960+
maxReplicaCount := int32(10)
961+
so := &kedav1alpha1.ScaledObject{
962+
ObjectMeta: metav1.ObjectMeta{
963+
Name: soName,
964+
Namespace: "default",
965+
Annotations: map[string]string{
966+
kedav1alpha1.PausedScaleOutAnnotation: "true",
967+
},
968+
},
969+
970+
Spec: kedav1alpha1.ScaledObjectSpec{
971+
ScaleTargetRef: &kedav1alpha1.ScaleTarget{
972+
Name: deploymentName,
973+
},
974+
MinReplicaCount: &minReplicaCount,
975+
MaxReplicaCount: &maxReplicaCount,
976+
Triggers: []kedav1alpha1.ScaleTriggers{
977+
{
978+
Type: "cron",
979+
Metadata: map[string]string{
980+
"timezone": "UTC",
981+
"start": "0 * * * *",
982+
"end": "1 * * * *",
983+
"desiredReplicas": "2",
984+
},
985+
},
986+
},
987+
Advanced: &kedav1alpha1.AdvancedConfig{
988+
HorizontalPodAutoscalerConfig: &kedav1alpha1.HorizontalPodAutoscalerConfig{
989+
Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{
990+
ScaleUp: &autoscalingv2.HPAScalingRules{
991+
SelectPolicy: &minPolicy,
992+
},
993+
},
994+
},
995+
},
996+
},
997+
}
998+
err = k8sClient.Create(context.Background(), so)
999+
Expect(err).ToNot(HaveOccurred())
1000+
testLogger.Info("Created scaled object")
1001+
1002+
// Get and confirm the HPA.
1003+
hpa := &autoscalingv2.HorizontalPodAutoscaler{}
1004+
Eventually(func() error {
1005+
return k8sClient.Get(context.Background(), types.NamespacedName{Name: "keda-hpa-" + soName, Namespace: "default"}, hpa)
1006+
}).ShouldNot(HaveOccurred())
1007+
Expect(*hpa.Spec.Behavior.ScaleUp.SelectPolicy).To(Equal(autoscalingv2.DisabledPolicySelect))
1008+
1009+
Eventually(func() metav1.ConditionStatus {
1010+
err = k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so)
1011+
Ω(err).ToNot(HaveOccurred())
1012+
return so.Status.Conditions.GetPausedCondition().Status
1013+
}, 2*time.Minute).WithPolling(5 * time.Second).Should(Equal(metav1.ConditionTrue))
1014+
1015+
// Remove the annotation.
1016+
Eventually(func() error {
1017+
err = k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so)
1018+
Expect(err).ToNot(HaveOccurred())
1019+
delete(so.ObjectMeta.Annotations, kedav1alpha1.PausedScaleOutAnnotation)
1020+
return k8sClient.Update(context.Background(), so)
1021+
}).ShouldNot(HaveOccurred())
1022+
1023+
Eventually(func() metav1.ConditionStatus {
1024+
err = k8sClient.Get(context.Background(), types.NamespacedName{Name: soName, Namespace: "default"}, so)
1025+
Ω(err).ToNot(HaveOccurred())
1026+
return so.Status.Conditions.GetPausedCondition().Status
1027+
}, 2*time.Minute).WithPolling(5 * time.Second).Should(Or(Equal(metav1.ConditionFalse), Equal(metav1.ConditionUnknown)))
1028+
1029+
Eventually(func() autoscalingv2.ScalingPolicySelect {
1030+
err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "keda-hpa-" + soName, Namespace: "default"}, hpa)
1031+
Expect(err).ToNot(HaveOccurred())
1032+
if hpa.Spec.Behavior != nil && hpa.Spec.Behavior.ScaleUp != nil && hpa.Spec.Behavior.ScaleUp.SelectPolicy != nil {
1033+
return *hpa.Spec.Behavior.ScaleUp.SelectPolicy
1034+
} else {
1035+
return ""
1036+
}
1037+
}).WithTimeout(1 * time.Minute).WithPolling(10 * time.Second).Should(Equal(autoscalingv2.MinChangePolicySelect))
1038+
})
1039+
8131040
It("doesn't allow MinReplicaCount > MaxReplicaCount", func() {
8141041
deploymentName := "minmax"
8151042
soName := "so-" + deploymentName

‎controllers/keda/util/predicate.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,14 @@ func (ForceActivationPredicate) Update(e event.UpdateEvent) bool {
7272
return checkAnnotation(e, kedav1alpha1.ForceActivationAnnotation)
7373
}
7474

75+
type PausedScaleOutPredicate struct {
76+
predicate.Funcs
77+
}
78+
79+
func (PausedScaleOutPredicate) Update(e event.UpdateEvent) bool {
80+
return checkAnnotation(e, kedav1alpha1.PausedScaleOutAnnotation)
81+
}
82+
7583
func checkAnnotation(e event.UpdateEvent, annotation string) bool {
7684
if e.ObjectOld == nil || e.ObjectNew == nil {
7785
return false

‎pkg/scaling/executor/scale_scaledobjects.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,12 @@ func (e *scaleExecutor) scaleToZeroOrIdle(ctx context.Context, logger logr.Logge
286286
}
287287

288288
func (e *scaleExecutor) scaleFromZeroOrIdle(ctx context.Context, logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject, scale *autoscalingv1.Scale, activeTriggers []string) {
289+
if scaledObject.NeedToPauseScaleOut() {
290+
// The Pause Scale Out annotation is set so we should not scale up (out) this target
291+
logger.Info("Pause Scale Out annotation set on ScaledObject, no scaling out on active trigger")
292+
return
293+
}
294+
289295
var replicas int32
290296
if scaledObject.Spec.MinReplicaCount != nil && *scaledObject.Spec.MinReplicaCount > 0 {
291297
replicas = *scaledObject.Spec.MinReplicaCount

0 commit comments

Comments
 (0)