Merge pull request #1343 from relu/autoscaler-replicas-overrides

Support for overriding primary scaler replicas
This commit is contained in:
Stefan Prodan
2023-01-17 16:01:14 +02:00
committed by GitHub
12 changed files with 150 additions and 18 deletions
+7
View File
@@ -121,6 +121,13 @@ spec:
type: object
additionalProperties:
type: string
primaryScalerReplicas:
type: object
properties:
minReplicas:
type: number
maxReplicas:
type: number
ingressRef:
description: Ingress selector
type: object
+7
View File
@@ -121,6 +121,13 @@ spec:
type: object
additionalProperties:
type: string
primaryScalerReplicas:
type: object
properties:
minReplicas:
type: number
maxReplicas:
type: number
ingressRef:
description: Ingress selector
type: object
@@ -99,6 +99,11 @@ spec:
# ScaledObject targeting the primary deployment. (Optional)
primaryScalerQueries:
prom-trigger: sum(rate(http_requests_total{ app="podinfo-primary" }[30s]))
# Overriding replica scaling configuration for the generated ScaledObject
# targeting the primary deployment. (Optional)
primaryScalerReplicas:
minReplicas: 2
maxReplicas: 5
# the maximum time in seconds for the canary deployment
# to make progress before rollback (default 600s)
progressDeadlineSeconds: 60
@@ -167,6 +172,9 @@ If, the generated query does not meet your requirements, you can specify the que
`.spec.autoscalerRef.primaryScalerQueries`, which lets you define a query for each trigger. Please note that, your ScaledObject's `.spec.triggers[@].name` must
not be blank, as Flagger needs that to identify each trigger uniquely.
In the situation when it is desired to have different scaling replica configuration between the canary and primary deployment ScaledObject you can use
the `.spec.autoscalerRef.primaryScalerReplicas` to override these values for the generated primary ScaledObject.
After the boostrap, the podinfo deployment will be scaled to zero and the traffic to `podinfo.test` will be routed to the primary pods. To keep the podinfo deployment
at 0 replicas and pause auto scaling, Flagger will add an annotation to your ScaledObject: `autoscaling.keda.sh/paused-replicas: 0`.
During the canary analysis, the annotation is removed, to enable auto scaling for the podinfo deployment.
+8
View File
@@ -68,6 +68,9 @@ spec:
apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
name: podinfo
primaryScalerReplicas:
minReplicas: 2
maxReplicas: 5
```
Based on the above configuration, Flagger generates the following Kubernetes objects:
@@ -80,6 +83,11 @@ by default all traffic is routed to this version and the target deployment is sc
Flagger will detect changes to the target deployment (including secrets and configmaps)
and will perform a canary analysis before promoting the new version as primary.
Use `.spec.autoscalerRef.primaryScalerReplicas` to override the replica scaling
configuration for the generated primary HorizontalPodAutoscaler. This is useful
for situations when you want to have a different scaling configuration for the
primary workload as opposed to using the same values from the original workload HorizontalPodAutoscaler.
**Note** that the target deployment must have a single label selector in the format `app: <DEPLOYMENT-NAME>`:
```yaml
-18
View File
@@ -20,12 +20,8 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0=
cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
cloud.google.com/go/compute v1.13.0 h1:AYrLkB8NPdDRslNp4Jxmzrhdr03fUAIDbiGFjLWowoU=
cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE=
cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
cloud.google.com/go/compute/metadata v0.2.2 h1:aWKAjYaBaOSrpKl57+jnS/3fJRQnxL7TvR/u1VVbt6k=
cloud.google.com/go/compute/metadata v0.2.2/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
@@ -45,8 +41,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
@@ -58,8 +52,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/aws/aws-sdk-go v1.44.144 h1:mMWdnYL8HZsobrQe1mwvQ18Xt8UbOVhWgipjuma5Mkg=
github.com/aws/aws-sdk-go v1.44.144/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.155 h1:PMHMuUS0atPD4LhiXuYrLasrlIm4u3lpNQBl9h+Lr2s=
github.com/aws/aws-sdk-go v1.44.155/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@@ -205,8 +197,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/influxdata/influxdb-client-go/v2 v2.12.0 h1:LGct9uIp36IT+8RAJdmJGQbNonGi26YfYYSpDIyq8fI=
github.com/influxdata/influxdb-client-go/v2 v2.12.0/go.mod h1:YteV91FiQxRdccyJ2cHvj2f/5sq4y4Njqu1fQzsQCOU=
github.com/influxdata/influxdb-client-go/v2 v2.12.1 h1:RrjoDNyBGFYvjKfjmtIyYAn6GY/SrtocSo4RPlt+Lng=
github.com/influxdata/influxdb-client-go/v2 v2.12.1/go.mod h1:YteV91FiQxRdccyJ2cHvj2f/5sq4y4Njqu1fQzsQCOU=
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
@@ -346,8 +336,6 @@ go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -577,8 +565,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ=
google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0=
google.golang.org/api v0.104.0 h1:KBfmLRqdZEbwQleFlSLnzpQJwhjpmNOk4cKQIBDZ9mg=
google.golang.org/api v0.104.0/go.mod h1:JCspTXJbBxa5ySXw4UgUqVer7DfVxbvc/CTUFqAED5U=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
@@ -619,10 +605,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c=
google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 h1:AGXp12e/9rItf6/4QymU7WsAUwCf+ICW75cuR91nJIc=
google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6/go.mod h1:1dOng4TWOomJrDGhpXjfCD35wQC6jnC7HpRmOFRqEV0=
google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 h1:jmIfw8+gSvXcZSgaFAGyInDXeWzUhvYH57G/5GKMn70=
google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+7
View File
@@ -121,6 +121,13 @@ spec:
type: object
additionalProperties:
type: string
primaryScalerReplicas:
type: object
properties:
minReplicas:
type: number
maxReplicas:
type: number
ingressRef:
description: Ingress selector
type: object
+13
View File
@@ -447,6 +447,19 @@ type AutoscalerRefernce struct {
// scaler, if a scaler supports scaling using queries.
// +optional
PrimaryScalerQueries map[string]string `json:"primaryScalerQueries"`
// PrimaryScalerReplicas defines overrides for the primary
// autoscaler replicas.
// +optional
PrimaryScalerReplicas *ScalerReplicas `json:"primaryScalerReplicas,omitempty"`
}
// ScalerReplicas holds overrides for autoscaler replicas
type ScalerReplicas struct {
// +optional
MinReplicas *int32 `json:"minReplicas,omitempty"`
// +optional
MaxReplicas *int32 `json:"maxReplicas,omitempty"`
}
// CustomMetadata holds labels and annotations to set on generated objects.
@@ -161,6 +161,11 @@ func (in *AutoscalerRefernce) DeepCopyInto(out *AutoscalerRefernce) {
(*out)[key] = val
}
}
if in.PrimaryScalerReplicas != nil {
in, out := &in.PrimaryScalerReplicas, &out.PrimaryScalerReplicas
*out = new(ScalerReplicas)
(*in).DeepCopyInto(*out)
}
return
}
@@ -826,6 +831,32 @@ func (in *MetricTemplateStatus) DeepCopy() *MetricTemplateStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ScalerReplicas) DeepCopyInto(out *ScalerReplicas) {
*out = *in
if in.MinReplicas != nil {
in, out := &in.MinReplicas, &out.MinReplicas
*out = new(int32)
**out = **in
}
if in.MaxReplicas != nil {
in, out := &in.MaxReplicas, &out.MaxReplicas
*out = new(int32)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScalerReplicas.
func (in *ScalerReplicas) DeepCopy() *ScalerReplicas {
if in == nil {
return nil
}
out := new(ScalerReplicas)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SessionAffinity) DeepCopyInto(out *SessionAffinity) {
*out = *in
+18
View File
@@ -77,6 +77,15 @@ func (hr *HPAReconciler) reconcilePrimaryHpaV2(cd *flaggerv1.Canary, hpa *hpav2.
Behavior: hpa.Spec.Behavior,
}
if replicas := cd.Spec.AutoscalerRef.PrimaryScalerReplicas; replicas != nil {
if minReplicas := replicas.MinReplicas; minReplicas != nil {
hpaSpec.MinReplicas = minReplicas
}
if maxReplicas := replicas.MaxReplicas; maxReplicas != nil {
hpaSpec.MaxReplicas = *maxReplicas
}
}
primaryHpaName := fmt.Sprintf("%s-primary", cd.Spec.AutoscalerRef.Name)
primaryHpa, err := hr.kubeClient.AutoscalingV2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{})
@@ -161,6 +170,15 @@ func (hr *HPAReconciler) reconcilePrimaryHpaV2Beta2(cd *flaggerv1.Canary, hpa *h
Behavior: hpa.Spec.Behavior,
}
if replicas := cd.Spec.AutoscalerRef.PrimaryScalerReplicas; replicas != nil {
if minReplicas := replicas.MinReplicas; minReplicas != nil {
hpaSpec.MinReplicas = minReplicas
}
if maxReplicas := replicas.MaxReplicas; maxReplicas != nil {
hpaSpec.MaxReplicas = *maxReplicas
}
}
primaryHpaName := fmt.Sprintf("%s-primary", cd.Spec.AutoscalerRef.Name)
primaryHpa, err := hr.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{})
+27
View File
@@ -4,6 +4,7 @@ import (
"context"
"testing"
flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
hpav2 "k8s.io/api/autoscaling/v2"
@@ -73,6 +74,19 @@ func Test_reconcilePrimaryHpaV2(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, int(*primaryHPA.Spec.Metrics[0].Resource.Target.AverageUtilization), 50)
assert.Equal(t, int(primaryHPA.Spec.MaxReplicas), 10)
// Test reconcile with PrimaryHorizontalPodAutoscalerOverride
mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas = &flaggerv1.ScalerReplicas{
MinReplicas: int32p(2),
MaxReplicas: int32p(15),
}
err = hpaReconciler.reconcilePrimaryHpaV2(mocks.canary, hpa, false)
require.NoError(t, err)
primaryHPA, err = mocks.kubeClient.AutoscalingV2().HorizontalPodAutoscalers("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
require.NoError(t, err)
assert.Equal(t, primaryHPA.Spec.MinReplicas, mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MinReplicas)
assert.Equal(t, primaryHPA.Spec.MaxReplicas, *mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MaxReplicas)
}
func Test_reconcilePrimaryHpaV2Beta2(t *testing.T) {
@@ -105,4 +119,17 @@ func Test_reconcilePrimaryHpaV2Beta2(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, int(*primaryHPA.Spec.Metrics[0].Resource.Target.AverageUtilization), 50)
assert.Equal(t, int(primaryHPA.Spec.MaxReplicas), 10)
// Test reconcile with PrimaryHorizontalPodAutoscalerOverride
mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas = &flaggerv1.ScalerReplicas{
MinReplicas: int32p(2),
MaxReplicas: int32p(15),
}
err = hpaReconciler.reconcilePrimaryHpaV2Beta2(mocks.canary, hpa, false)
require.NoError(t, err)
primaryHPA, err = mocks.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
require.NoError(t, err)
assert.Equal(t, primaryHPA.Spec.MinReplicas, mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MinReplicas)
assert.Equal(t, primaryHPA.Spec.MaxReplicas, *mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MaxReplicas)
}
+10
View File
@@ -63,6 +63,16 @@ func (sor *ScaledObjectReconciler) reconcilePrimaryScaler(cd *flaggerv1.Canary,
Fallback: targetSoClone.Spec.Fallback,
IdleReplicaCount: targetSoClone.Spec.IdleReplicaCount,
}
if replicas := cd.Spec.AutoscalerRef.PrimaryScalerReplicas; replicas != nil {
if minReplicas := replicas.MinReplicas; minReplicas != nil {
soSpec.MinReplicaCount = minReplicas
}
if maxReplicas := replicas.MaxReplicas; maxReplicas != nil {
soSpec.MaxReplicaCount = maxReplicas
}
}
primarySoName := fmt.Sprintf("%s-primary", cd.Spec.AutoscalerRef.Name)
primarySo, err := sor.flaggerClient.KedaV1alpha1().ScaledObjects(cd.Namespace).Get(context.TODO(), primarySoName, metav1.GetOptions{})
if errors.IsNotFound(err) {
@@ -46,6 +46,20 @@ func Test_reconcilePrimaryScaledObject(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, int(*primarySO.Spec.PollingInterval), 20)
assert.Equal(t, primarySO.Spec.Triggers[0].Metadata["query"], `sum(rate(http_requests_total{app="podinfo-primary"}[10m]))`)
// Test reconcile with PrimaryScaledObjectOverride
mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas = &flaggerv1.ScalerReplicas{
MinReplicas: int32p(2),
MaxReplicas: int32p(15),
}
err = soReconciler.reconcilePrimaryScaler(mocks.canary, false)
require.NoError(t, err)
primarySO, err = mocks.flaggerClient.KedaV1alpha1().ScaledObjects("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
require.NoError(t, err)
assert.Equal(t, int(*primarySO.Spec.PollingInterval), 20)
assert.Equal(t, primarySO.Spec.MinReplicaCount, mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MinReplicas)
assert.Equal(t, primarySO.Spec.MaxReplicaCount, mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MaxReplicas)
}
func Test_pauseScaledObject(t *testing.T) {