Enrich deploy. template for acs-engine-autoscaler (#5662)

Add a number of configuration options for the pod in
`acs-engine-autoscaler`. Specifically, allow setting
resource requests/limits, pod/deployment annotations, pod tolerations,
pod node selectors, and pod affinity.

I tested that running `helm install acs-engine-autoscaler` works without
setting any of these values (i.e. relying on the defaults). I also ran
`helm install acs-engine-autoscaler` with the following new config,
and verified it successfully applied this configuration to the
pod/deployment.

```
resources:
  limits:
    cpu: 100m
    memory: 128Mi
  requests:
    cpu: 100m
    memory: 128Mi

nodeSelector:
  beta.kubernetes.io/os: linux

podAnnotations:
  mattjmcnaughton.com/test: test-this-pod-annotation

deploymentAnnotations:
  mattjmcnaughton.com/test: test-this-deployment-annotation

tolerations:
- key: node-type
  Operator: Equal
  value: production
  effect: NoSchedule

affinity:
  nodeAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
    - weight: 80
      preference:
        matchExpressions:
        - key: availability-zone
          operator: In
          values:
          - zone1
```

Finally, I updated the README to include these new default options.

stable/acs-engine-autoscaler/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 description: Scales worker nodes within agent pools
 icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png
 name: acs-engine-autoscaler
-version: 2.1.5
+version: 2.2.0
 appVersion: 2.1.1
 home: https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler
 sources:

stable/acs-engine-autoscaler/README.md

@@ -103,6 +103,15 @@ The following tables list the configurable parameters of the acs-engine-autoscal
 
 Parameter | Description | Default
 --- | --- | ---
+`image.repository` | Image repository | `wbuchwalter/kubernetes-acs-engine-autoscaler` |
+`image.tag` | Image tag. Possible values listed [here](https://hub.docker.com/r/wbuchwalter/kubernetes-acs-engine-autoscaler/tags/).| `2.1.1`|
+`image.pullPolicy` | Image pull policy | `IfNotPresent` |
+`podAnnotations` | Pod annotations | `{}` |
+`deploymentAnnotations` | Deployment annotations | `{}` |
+`resources` | CPU/Memory resource requests/limits | `{}` |
+`nodeSelector` | Node labels for pod assignment | `{}` |
+`tolerations` | Toleration labels for pod assignment | `[]` |
+`affinity` | Affinity settings for pod assignment | `{}` |
 `resourcegroup`| Name of the resource group containing the cluster | None. You *must* supply one.
 `azurespappid`| An Azure service principal id | None. You *must* supply one.
 `azurespsecret`| An Azure service principal secret | None. You *must* supply one.

stable/acs-engine-autoscaler/templates/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
 {{ include "acs-engine-autoscaler.labels.standard" . | indent 4 }}
   annotations:
     description: {{ .Chart.Description }}
+{{- with .Values.deploymentAnnotations }}
+{{ toYaml . | indent 4 }}
+{{- end }}
 spec:
   replicas: {{ .Values.replicaCount }}
   template:
@@ -14,10 +17,15 @@ spec:
       labels:
 {{ include "acs-engine-autoscaler.labels.standard" . | indent 8 }}
         openai/do-not-drain: "true"
+{{- with .Values.podAnnotations }}
+      annotations:
+{{ toYaml . | indent 8 }}
+{{- end }}
     spec:
       containers:
       - name: {{ template "acs-engine-autoscaler.fullname" . }}
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
         env:
         - name: AZURE_SP_APP_ID
           valueFrom:
@@ -79,7 +87,20 @@ spec:
             - {{ .Values.acsenginecluster.overprovision | quote }}
             {{- end }}
             - -vvv
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        resources:
+{{ toYaml .Values.resources | indent 12 }}
+{{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+{{- end }}
+{{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+{{- end }}
+{{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+{{- end }}
       restartPolicy: Always
       dnsPolicy: Default
 {{ end }}

stable/acs-engine-autoscaler/values.yaml

@@ -8,6 +8,37 @@ image:
   repository: wbuchwalter/kubernetes-acs-engine-autoscaler
   tag: 2.1.1
   pullPolicy: IfNotPresent
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #  cpu: 100m
+  #  memory: 128Mi
+  # requests:
+  #  cpu: 100m
+  #  memory: 128Mi
+
+## Node labels for pod assignment
+## ref: https://kubernetes.io/docs/user-guide/node-selection/
+nodeSelector: {}
+
+## Pod Annotations
+podAnnotations: {}
+
+## Deployment Annotations
+deploymentAnnotations: {}
+
+## Tolerations for pod assignment
+## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+tolerations: []
+
+## Affinity for pod assignment
+## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+affinity: {}
+
 acsenginecluster:
   resourcegroup:
   azurespappid: