diff --git a/stable/mysqldump/Chart.yaml b/stable/mysqldump/Chart.yaml index c05e42810e..e8ea0a1eb4 100644 --- a/stable/mysqldump/Chart.yaml +++ b/stable/mysqldump/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: 2.4.1 description: A Helm chart to help backup MySQL databases using mysqldump name: mysqldump -version: 2.5.1 +version: 2.6.0 keywords: - mysql - mysqldump diff --git a/stable/mysqldump/README.md b/stable/mysqldump/README.md index 0c1a63a38d..4912066ca1 100644 --- a/stable/mysqldump/README.md +++ b/stable/mysqldump/README.md @@ -85,10 +85,24 @@ The following tables lists the configurable parameters of the mysqldump chart an | upload.ssh.host | ssh server url | yourdomain.com | | upload.ssh.dir | directory on server | /backup | | upload.ssh.privatekey | ssh user private key | "" | +| upload.openstack.enabled | upload backups via swift to openstack | false | +| upload.openstack.user | user name | backup@mydomain | +| upload.openstack.userDomain | user domain | default | +| upload.openstack.password | user password, overriden by `existingSecret`/`existingSecretKey` if set | | +| upload.openstack.authUrl | openstack auth url (v3) | https://mydomain:5000/v3 | +| upload.openstack.project | project name | my_project | +| upload.openstack.projectDomain | project domain | default | +| upload.openstack.destination | destination path, starting witch container | backup/mysql | +| upload.openstack.existingSecret | optional, specify a secret name to use for password | | +| upload.openstack.existingSecretKey | optional, specify a secret key to use for password | openstack-backup-password | +| upload.openstack.ttlDays | days to set time-to-live on uploaded objects (0 to disable) | 30 | | resources | resource definitions | {} | | nodeSelector | node selector | {} | | tolerations | tolerations | \[] | | affinity | affinity | {} | +| securityContext.enabled | set true to change default security context of job/cronjob | false | +| securityContext.fsGroup | group id to use | 999 | +| securityContext.runAsUser | user id to use | 999 | ### Auto generating the gcp service account diff --git a/stable/mysqldump/files/job.tpl b/stable/mysqldump/files/job.tpl index 4411745ac8..d84707901c 100755 --- a/stable/mysqldump/files/job.tpl +++ b/stable/mysqldump/files/job.tpl @@ -1,11 +1,18 @@ spec: + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} + runAsUser: {{ .Values.securityContext.runAsUser }} + {{- end }} containers: - name: mysql-backup image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["/bin/bash", "/scripts/backup.sh"] -{{- if .Values.mysql.existingSecret }} +{{- if or .Values.mysql.existingSecret .Values.upload.openstack.existingSecret }} env: +{{- end }} +{{- if .Values.mysql.existingSecret }} - name: MYSQL_PWD valueFrom: secretKeyRef: @@ -15,6 +22,17 @@ spec: {{- else }} key: "mysql-root-password" {{- end }} +{{- end }} +{{- if .Values.upload.openstack.existingSecret }} + - name: OS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.upload.openstack.existingSecret | quote }} + {{- if .Values.upload.openstack.existingSecretKey }} + key: {{ .Values.upload.openstack.existingSecretKey | quote }} + {{- else }} + key: "openstack-backup-password" + {{- end }} {{- end }} envFrom: - configMapRef: diff --git a/stable/mysqldump/files/openstack-upload.py b/stable/mysqldump/files/openstack-upload.py new file mode 100644 index 0000000000..7ae387cf7d --- /dev/null +++ b/stable/mysqldump/files/openstack-upload.py @@ -0,0 +1,212 @@ +#!/usr/bin/python + +from optparse import OptionParser +import os, sys, subprocess, json, re, threading + +global VERBOSE +VERBOSE = False + +class Expando(object): + pass + +def curl(args, shell=True, check=True, input=None, timeout_sec = 30, **kwargs): + '''python3 subprocess.run() workalike with more appropriate defaults ''' + + args = 'curl ' + args + if VERBOSE: print("Run:\n {}".format(args)) + p = subprocess.Popen(args, shell=shell, stdin=subprocess.PIPE if input else None, stdout=subprocess.PIPE, **kwargs) + + timer = threading.Timer(timeout_sec, p.kill) + try: + timer.start() + (stdout, stderr) = p.communicate(input.encode('utf-8') if input else '') + finally: + timer.cancel() + + r = Expando() + r.stdout = stdout + r.stderr = stderr or '' + r.returncode = p.returncode + + if VERBOSE: print("Result:\n{}".format(r.stdout).replace('\n', '\n ')) + + if check and r.returncode != 0: + raise Exception("Command {} exited with code {}:\n{}".format(args, r.returncode, r.stdout)) + + return r + +def upload(source, destination, ttl_days): + try: + auth_url = os.environ['OS_AUTH_URL'] + '/auth/tokens' + print('Getting authentication token from {} ...'.format(auth_url)) + + auth_json = '''{{ + "auth": {{ + "identity": {{ + "methods": ["password"], + "password": {{ + "user": {{ + "domain": {{"name": "{OS_USER_DOMAIN_NAME}"}}, + "name": "{OS_USERNAME}", + "password": "{OS_PASSWORD}" + }} + }} + }}, + "scope": {{ + "project": {{ + "domain": {{"name": "{OS_PROJECT_DOMAIN_NAME}"}}, + "name": "{OS_PROJECT_NAME}" + }} + }} + }} + }}'''.format(**dict({k:v.strip() for (k,v) in os.environ.iteritems() if k.startswith("OS_")})) + except KeyError as e: + raise Exception('These environment variables must be set with login info:\n'+ + ' OS_USER_DOMAIN_NAME, OS_USERNAME, OS_PASSWORD, OS_PROJECT_DOMAIN_NAME, OS_PROJECT_NAME, OS_AUTH_URL', e) + + if VERBOSE: + print("Login data:\n " + auth_json) + + # check that it is valid + json.loads(auth_json) + + p = curl('--silent --show-error --include --header "Content-Type: application/json" --data @- {}'.format(auth_url), + input=auth_json.encode('utf-8')) + + lines = [s.decode('utf-8') for s in p.stdout.splitlines()] + header_lines = lines[0:-2] + body = lines[-1] + + if p.returncode: + raise Exception("Failed to log in to Openstack at {}: exit code {}".format(auth_url, p)) + + if header_lines[0].split()[1] != '201': + raise Exception("Failed to log in to Openstack at {}: {}\n{}".format(auth_url, header_lines[0], body)) + + for line in header_lines: + if line.startswith('X-Subject-Token:'): + auth_token = line.split()[1] + break + else: + raise Exception("Failed to find X-Subject-Token in returned headers:\n {}".format("\n ".join(header_lines))) + + auth_token_header = 'X-Auth-Token: {0}\nX-Storage-Token: {0}'.format(auth_token) + if VERBOSE: print("\nHeaders:\n {}\n".format(auth_token_header)) + + print('Locating public object store URL in catalog ...') + object_store_url = None + for i in json.loads(body)["token"]["catalog"]: + if i["type"] != "object-store": + continue + for j in i["endpoints"]: + if j["interface"] == "public": + object_store_url = j["url"] + break + else: + continue + break + + if object_store_url is None: + raise Exception("Failed to find object-store public endpoint URL in returned JSON:\n{}".format(body)) + + print(" {}".format(object_store_url)) + + full_destination_url = '{}/{}'.format(object_store_url, destination) + + # Check container + container_name = destination.split('/')[0] + container_url = object_store_url + "/" + container_name + print('Checking container: {} ...'.format(container_name)) + p =curl('--fail --silent --show-error --head --header @- {}'.format(container_url), + input=auth_token_header) + if not VERBOSE: + print(" {}".format(p.stdout.splitlines()[0])) + + # Upload + print('\nUploading\n from: {}\n to: {}'.format(os.path.join(os.getcwd(), source), full_destination_url)) + if ttl_days: + ttl_seconds = ttl_days * 24 * 60 * 60 + print(' ttl: {} day(s)\n'.format(ttl_days)) + else: + print(' ttl: None\n') + + found = 0 + count = 0 + errs = 0 + for path, _, files in os.walk(source): + abspath = os.path.join(os.getcwd(), path) + if VERBOSE: print("\nEntering '{}'".format(abspath)) + + for f in files: + found += 1 + + local_file = os.path.join(abspath, f) + dest_url = (full_destination_url + '/' + path + '/' + f).replace('//', '/') # avoid double slash, screws up web UI + + if VERBOSE: + print ("\n{} ==> {}\n".format(local_file, dest_url)) + else: + print (' {}'.format(local_file)) + + try: + p = curl('--silent --show-error --head --header @- -o /dev/null --write-out "%{{http_code}}" {}'.format(dest_url), + input=auth_token_header) + + if p.stdout.strip() != '404': + if VERBOSE: print(" File found on server, not uploading.") + continue # Don't upload existing file, or on error + + print(" Uploading ...") + + upload_args = '--silent --show-error --fail --request PUT --header @- --upload-file {} {}'.format(local_file, dest_url) + if ttl_days: + upload_args += ' --header "X-Delete-After: {}"'.format(ttl_seconds) + + p = curl(upload_args, shell=True, check=True, timeout_sec=30*60, input=auth_token_header) + + count += 1 + except Exception as e: + print ("ERROR: {}".format(e)) + errs += 1 + + print ('\nUploaded {} of {} file(s), {} failure(s)'.format(count, found, errs)) + + if errs: + print("FAIL: Error(s) occured") + return 2 + + if not count and found: + print("WARN: No files uploaded?") + return 1 + + if not found: + print("WARN: No files found?") + return 1 + + return 0 + +if __name__ == '__main__': + print('This script uploads files to an OpenStack object store container\n') + # Inspired by http://doc.swift.surfsara.nl/en/latest/Pages/Clients/curl_token.html#curl-token + + parser = OptionParser() + parser.add_option("-s", "--source", action="store", type="string", dest="source", + help="Source directory to copy from") + parser.add_option("-d", "--destination", action="store", type="string", dest="destination", + help="Destination URL to copy to, starting with container name") + parser.add_option("-t", "--ttl-days", action="store", type="int", dest="ttl_days", default=0, + help="Days to set TTL/Delete-After, 0 to disable") + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, + help="Verbose logging output") + + (options, args) = parser.parse_args() + + for r in ['source', 'destination']: + if not options.__dict__.get(r): + parser.error("parameter {} required".format(r)) + + + VERBOSE = options.verbose + + r = upload(source=options.source, destination=options.destination, ttl_days=options.ttl_days) + sys.exit(r) diff --git a/stable/mysqldump/templates/NOTES.txt b/stable/mysqldump/templates/NOTES.txt index 22ad7e8937..b6cd0c45d0 100644 --- a/stable/mysqldump/templates/NOTES.txt +++ b/stable/mysqldump/templates/NOTES.txt @@ -40,6 +40,10 @@ $ kubectl logs $(kubectl get pods --selector \ --output=jsonpath='{.items[-1:].metadata.name}') \ --output=jsonpath={.items..metadata.name}) +To run cronjob now run: + +$ kubectl create job {{ template "mysqldump.fullname" . }} --from=cronjob/{{ template "mysqldump.fullname" . }} + mysqldump contents can be found in: {{- if .Values.persistentVolumeClaim }} $ kubectl get persistentvolumeclaim {{ .Values.persistentVolumeClaim }} diff --git a/stable/mysqldump/templates/configmap.yaml b/stable/mysqldump/templates/configmap.yaml index 3c922e85e2..eb1731342c 100755 --- a/stable/mysqldump/templates/configmap.yaml +++ b/stable/mysqldump/templates/configmap.yaml @@ -10,6 +10,13 @@ data: MYSQL_PORT: {{ .Values.mysql.port | quote }} MYSQL_OPTS: {{ .Values.options | quote }} KEEP_DAYS: {{ .Values.housekeeping.keepDays | quote }} +{{- if .Values.upload.openstack.enabled }} + OS_AUTH_URL: {{ .Values.upload.openstack.authUrl }} + OS_PROJECT_NAME: {{ .Values.upload.openstack.project }} + OS_PROJECT_DOMAIN_NAME: {{ .Values.upload.openstack.projectDomain }} + OS_USERNAME: {{ .Values.upload.openstack.user }} + OS_USER_DOMAIN_NAME: {{ .Values.upload.openstack.userDomain }} +{{- end }} --- apiVersion: v1 kind: ConfigMap @@ -18,6 +25,8 @@ metadata: labels: {{ include "mysqldump.labels" . | indent 4 }} data: + openstack-upload.py: |- +{{ .Files.Get "files/openstack-upload.py" | indent 4 }} backup.sh: |- #!/bin/sh # @@ -37,12 +46,12 @@ data: echo "started" > ${BACKUP_DIR}/${TIMESTAMP}.state - {{- if (.Values.persistence.enabled) or (.Values.persistentVolumeClaim) }} - + {{ if (.Values.persistence.enabled) or (.Values.persistentVolumeClaim) }} {{ if .Values.housekeeping.enabled }} echo "delete old backups" find ${BACKUP_DIR} -maxdepth 2 -mtime +${KEEP_DAYS} -regex "^${BACKUP_DIR}/.*[0-9]*_.*\.sql\.gz$" -type f -exec rm {} \; {{ end -}} + {{ end -}} {{ if and (.Values.mysql.db) (eq .Values.allDatabases.enabled false) }} MYSQL_DB="{{ .Values.mysql.db }}" @@ -66,20 +75,30 @@ data: rc=$? {{- end -}} - {{- if or (.Values.upload.googlestoragebucket.enabled) (.Values.upload.ssh.enabled) -}} + {{- if or (.Values.upload.googlestoragebucket.enabled) (.Values.upload.ssh.enabled) (.Values.upload.openstack.enabled) -}} {{ if .Values.upload.ssh.enabled -}} echo "upload files via ssh to {{ .Values.upload.ssh.user }}@{{ .Values.upload.ssh.host }}:{{ .Values.upload.ssh.dir }}" rsync -av --delete --exclude=*.state -e 'ssh -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null' ${BACKUP_DIR}/ {{ .Values.upload.ssh.user }}@{{ .Values.upload.ssh.host }}:{{ .Values.upload.ssh.dir }} rcu=$? {{ end -}} - {{ if .Values.upload.googlestoragebucket.enabled }} + {{ if .Values.upload.googlestoragebucket.enabled -}} echo "upload files to google storage bucket {{ .Values.upload.googlestoragebucket.bucketname }}" gcloud auth activate-service-account --key-file /root/gcloud/{{ .Values.upload.googlestoragebucket.secretFileName }} gsutil -m rsync -r -x '.*\.state' -d ${BACKUP_DIR}/ {{ .Values.upload.googlestoragebucket.bucketname }} rcu=$? {{ end }} + {{ if .Values.upload.openstack.enabled -}} + echo "upload files to openstack at {{ .Values.upload.openstack.destination }}" + python /scripts/openstack-upload.py \ + --source=${BACKUP_DIR} \ + --destination={{ .Values.upload.openstack.destination }} \ + {{ if .Values.upload.openstack.ttlDays -}} --ttl-days={{ .Values.upload.openstack.ttlDays }} {{- end }} \ + {{ if .Values.debug -}} --verbose {{- end }} + rcu=$? + {{ end }} + if [ "$rcu" != "0" ]; then echo "upload failed" exit 1 @@ -91,7 +110,6 @@ data: rc=$? {{ end }} {{ end }} - {{ end }} {{ if .Values.debug }} echo Contents of ${BACKUP_DIR} @@ -114,4 +132,4 @@ data: echo "Disk usage in ${BACKUP_DIR}" du -h -d 2 ${BACKUP_DIR} - echo "Backup successful! :-)" \ No newline at end of file + echo "Backup successful! :-)" diff --git a/stable/mysqldump/templates/secret.yaml b/stable/mysqldump/templates/secret.yaml index e0731fc912..b4de860160 100644 --- a/stable/mysqldump/templates/secret.yaml +++ b/stable/mysqldump/templates/secret.yaml @@ -8,6 +8,9 @@ metadata: type: Opaque data: MYSQL_PWD: {{ .Values.mysql.password | b64enc | quote }} +{{- if and (.Values.upload.openstack.enabled) (not .Values.upload.openstack.existingSecret) }} + OS_PASSWORD: {{ .Values.upload.openstack.password | b64enc | quote }} +{{- end }} --- {{- end }} {{- if .Values.upload.ssh.enabled }} diff --git a/stable/mysqldump/values.yaml b/stable/mysqldump/values.yaml index c3041202b7..dae931520f 100644 --- a/stable/mysqldump/values.yaml +++ b/stable/mysqldump/values.yaml @@ -107,6 +107,24 @@ upload: # id_rsa private key as string privatekey: "" + openstack: + enabled: false + user: backup@mydomain + userDomain: default + # overriden by used if secretName/secretKey set + password: + authUrl: https://mydomain:5000/v3 + project: my_project + projectDomain: default + # container/folder(/subfolder ...) + destination: backup/mysql + # existingSecret can be enabled to use an existing secret + # existingSecret: + # existingSecretKey defines the key to use, or 'openstack-backup-password' if not set + # existingSecretKey: + # set to 0 to disable TTL on uploaded files + ttlDays: 30 + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -124,3 +142,9 @@ nodeSelector: {} tolerations: [] affinity: {} + +## Pod Security Context +securityContext: + enabled: false + fsGroup: 999 + runAsUser: 999