diff --git a/stable/luigi/.helmignore b/stable/luigi/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/stable/luigi/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/stable/luigi/Chart.yaml b/stable/luigi/Chart.yaml new file mode 100644 index 0000000000..e370685cb9 --- /dev/null +++ b/stable/luigi/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +description: Luigi is a Python module that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization etc. It also comes with Hadoop support built in. +home: https://luigi.readthedocs.io/en/stable/ +maintainers: +- name: sammerry + email: sam.merry@gmail.com +sources: +- https://github.com/spotify/luigi/ +- https://github.com/sammerry/luigi-docker-image/ +icon: https://raw.githubusercontent.com/spotify/luigi/master/doc/luigi.png +keywords: +- luigi +- scheduler +- dependency graph +- web +- task management +- hadoop +- spark +- kubernetes job manager +name: luigi +version: 2.7.2 diff --git a/stable/luigi/README.md b/stable/luigi/README.md new file mode 100644 index 0000000000..a48731971f --- /dev/null +++ b/stable/luigi/README.md @@ -0,0 +1,79 @@ +# Luigi Scheduler + +[Luigi](https://github.com/spotify/luigi) is a Python module that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization etc. It also comes with Hadoop support built in. + +## TL;DR; + +```console +$ helm install incubator/luigi +``` + +## Introduction + +This chart bootstraps a [Luigi](https://github.com/spotify/luigi) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +$ helm install --name my-release stable/luigi +``` +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete my-release --purge +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Luigi Configuration + +[Luigi](https://github.com/spotify/luigi) configs are set as a block of text through a configmap and mouted as a file in /etc/luigi Any value in this text block should match the defined luigi configuration. There are several values here that will have to match our kubernetes configuration. + +## Configuration + +The following tables lists the configurable parameters of the Sentry chart and their default values. + +| Parameter | Description | Default | +| ------------------------------- | ------------------------------- | ---------------------------------------------------------- | +| `image.repository` | Luigi image | `getpolymorph/luigi` | +| `image.tag` | Luigi image tag | `2.7.2` | +| `image.pullPolicy` | Luigi image pull policy | `IfNotPresent` | +| `service.name` | Luigi service name | `luigi` | +| `service.type` | The kube service type | `LoadBalancer` | +| `service.externalPort` | The service external port | `3000` | +| `service.internalPort` | The service internal port | `8082` | +| `service.config` | The luigi service configs | View this default in the values.yaml file | +| `persistence.enabled` | Persistence flag | `false` | +| `ingressUI.enabled` | UI Ingress Flag | `false` | +| `ingressAPI.enabled` | API Ingress Flag | `false` | + +Dependent charts can also have values overwritten. Preface values with postgresql.* or redis.* + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name my-release \ + --set persistence.enabled=false,email.host=email \ + stable/luigi +``` + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install --name my-release -f values.yaml stable/luigi +``` + +Read through the [values.yaml](values.yaml) file. It has several commented out suggested values. + +## Persistence + +Luigi requires a pickled state file. To maintain state after a restart you'll need to enable persistence. +`--set persistence.enabled=true` + +## Ingress + +This chart provides support for two Ingress resources. This is to allow authentication in the ui via reverse proxy with something like oauth-proxy and a separate form of authentication for luigi worker access. diff --git a/stable/luigi/requirements.lock b/stable/luigi/requirements.lock new file mode 100644 index 0000000000..d47711d6f3 --- /dev/null +++ b/stable/luigi/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: mysql + repository: https://kubernetes-charts.storage.googleapis.com/ + version: 0.3.4 +digest: sha256:4bfa5b90139dd5d0a9ef9e9fecaac9df407d3a24ee2f54bc05d4eb5cad5ab9c6 +generated: 2018-02-04T22:05:57.911294008-08:00 diff --git a/stable/luigi/requirements.yaml b/stable/luigi/requirements.yaml new file mode 100644 index 0000000000..14ea33e6ce --- /dev/null +++ b/stable/luigi/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: + - name: mysql + version: 0.3.4 + repository: https://kubernetes-charts.storage.googleapis.com/ diff --git a/stable/luigi/templates/NOTES.txt b/stable/luigi/templates/NOTES.txt new file mode 100644 index 0000000000..8c8865ed55 --- /dev/null +++ b/stable/luigi/templates/NOTES.txt @@ -0,0 +1,25 @@ + _ _ _ + | | (_) (_) + | | _ _ _ __ _ _ + | | | | | | |/ _` | | + | |___| |_| | | (_| | | + |______\__,_|_|\__, |_| + __/ | + |___/ + +Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "luigi.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "luigi.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "luigi.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.externalPort }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "luigi.name" . }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:{{ .Values.service.externalPort }} +{{- end }} + diff --git a/stable/luigi/templates/_helpers.tpl b/stable/luigi/templates/_helpers.tpl new file mode 100644 index 0000000000..b8444fda6c --- /dev/null +++ b/stable/luigi/templates/_helpers.tpl @@ -0,0 +1,25 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "luigi.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "luigi.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/stable/luigi/templates/configmap.yaml b/stable/luigi/templates/configmap.yaml new file mode 100644 index 0000000000..bf7e8a2b82 --- /dev/null +++ b/stable/luigi/templates/configmap.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "luigi.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "luigi.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + client.cfg: | +{{ printf .Values.service.config | indent 4 }} + + logging.cfg: | + [loggers] + keys=root,tornado,client,scheduler,server + + [logger_root] + level=DEBUG + handlers=console + + [logger_client] + level=DEBUG + handlers=console + qualname=luigi-interface + propagate=0 + + [logger_server] + level=DEBUG + handlers=console + qualname=luigi.server + propagate=0 + + [logger_scheduler] + level=DEBUG + handlers=console + qualname=luigi.scheduler + propagate=0 + + [logger_tornado] + level=DEBUG + handlers=warnconsole + qualname=tornado + propagate=0 + + [formatters] + keys=detail + + [formatter_detail] + class=logging.Formatter + format=%(asctime)s %(name)-15s %(levelname)-8s %(message)s + + [handlers] + keys=console,warnconsole + + [handler_console] + level=INFO + class=StreamHandler + args=(sys.stdout,) + formatter=detail + + [handler_warnconsole] + level=WARNING + class=StreamHandler + args=(sys.stdout,) + formatter=detail + diff --git a/stable/luigi/templates/deployment.yaml b/stable/luigi/templates/deployment.yaml new file mode 100644 index 0000000000..c5d5510afb --- /dev/null +++ b/stable/luigi/templates/deployment.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ template "luigi.fullname" . }} + labels: + app: {{ template "luigi.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ template "luigi.name" . }} + release: {{ .Release.Name }} + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ template "luigi.name" . }} + release: {{ .Release.Name }} + spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["/bin/sh", "-c", "luigid"] + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 8082 + livenessProbe: + httpGet: + path: /static/visualiser/index.html + port: 8082 + readinessProbe: + httpGet: + path: /static/visualiser/index.html + port: 8082 + resources: +{{ toYaml .Values.resources | indent 12 }} + volumeMounts: + - name: {{ template "luigi.name" . }}-luigi-conf + mountPath: /etc/luigi/client.cfg + subPath: client.cfg + - name: {{ template "luigi.name" . }}-logging-conf + mountPath: /etc/luigi/logging.cfg + subPath: logging.cfg + - name: luigi-state + mountPath: /luigi/state/ + + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + + volumes: + - name: {{ template "luigi.name" . }}-luigi-conf + configMap: + name: {{ template "luigi.fullname" . }} + items: + - key: client.cfg + path: client.cfg + + - name: {{ template "luigi.name" . }}-logging-conf + configMap: + name: {{ template "luigi.fullname" . }} + items: + - key: logging.cfg + path: logging.cfg + + - name: luigi-state + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ template "luigi.fullname" . }} + {{- else }} + emptyDir: {} + {{ end }} diff --git a/stable/luigi/templates/ingress_api.yaml b/stable/luigi/templates/ingress_api.yaml new file mode 100644 index 0000000000..0ab7519b92 --- /dev/null +++ b/stable/luigi/templates/ingress_api.yaml @@ -0,0 +1,33 @@ +{{- if .Values.ingressAPI.enabled -}} +{{- $serviceName := include "luigi.fullname" . -}} +{{- $servicePort := .Values.service.externalPort -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "luigi.fullname" . }}-api + labels: + app: {{ template "luigi.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + {{- range $key, $value := .Values.ingressAPI.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: + {{- range $host := .Values.ingressAPI.hosts }} + - host: {{ $host }} + http: + paths: + - path: / + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end -}} + {{- if .Values.ingressAPI.tls }} + tls: +{{ toYaml .Values.ingressAPI.tls | indent 4 }} + {{- end -}} +{{- end -}} + diff --git a/stable/luigi/templates/ingress_ui.yaml b/stable/luigi/templates/ingress_ui.yaml new file mode 100644 index 0000000000..176b8e455b --- /dev/null +++ b/stable/luigi/templates/ingress_ui.yaml @@ -0,0 +1,32 @@ +{{- if .Values.ingressUI.enabled -}} +{{- $serviceName := include "luigi.fullname" . -}} +{{- $servicePort := .Values.service.externalPort -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "luigi.fullname" . }}-ui + labels: + app: {{ template "luigi.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + {{- range $key, $value := .Values.ingressUI.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: + {{- range $host := .Values.ingressUI.hosts }} + - host: {{ $host }} + http: + paths: + - path: / + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end -}} + {{- if .Values.ingressUI.tls }} + tls: +{{ toYaml .Values.ingressUI.tls | indent 4 }} + {{- end -}} +{{- end -}} diff --git a/stable/luigi/templates/pvc.yaml b/stable/luigi/templates/pvc.yaml new file mode 100644 index 0000000000..61f24bf328 --- /dev/null +++ b/stable/luigi/templates/pvc.yaml @@ -0,0 +1,25 @@ +{{- if .Values.persistence.enabled -}} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "luigi.fullname" . }} + labels: + app: {{ template "luigi.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} +{{- if .Values.persistence.storageClass }} +{{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" +{{- end }} +{{- end }} +{{- end -}} + diff --git a/stable/luigi/templates/service.yaml b/stable/luigi/templates/service.yaml new file mode 100644 index 0000000000..25f172b460 --- /dev/null +++ b/stable/luigi/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "luigi.fullname" . }} + labels: + app: {{ template "luigi.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.externalPort }} + targetPort: 8082 + protocol: TCP + name: http + selector: + app: {{ template "luigi.name" . }} + release: {{ .Release.Name }} diff --git a/stable/luigi/values.yaml b/stable/luigi/values.yaml new file mode 100644 index 0000000000..48ae982d1a --- /dev/null +++ b/stable/luigi/values.yaml @@ -0,0 +1,85 @@ +# Default values for luigi. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +replicaCount: 1 +image: + repository: axiom/docker-luigi + tag: 2.7.2-alpine + pullPolicy: IfNotPresent + +service: + name: luigi + type: LoadBalancer + externalPort: 80 + + # Luigi config: these values should mattch the luigi documentation + # https://luigi.readthedocs.io/en/stable/configuration.html + config: | + [core] + logging_conf_file=/etc/luigi/logging.cfg + + [scheduler] + record_task_history=true + state-path=/luigi/state/luigi-state.pickle + + [task_history] + db_connection=mysql://luigi-mysql/luigidb + +# creates a persistent volume claim for +# luigi state pickel +persistence: + enabled: false + size: 1G + accessMode: ReadWriteOnce + +# Ingress for ui access for use with authentication like oauth-proxy +# depending on the authentication you use. You may only need one ingress. +ingressUI: + enabled: false + path: / + # Used to create an Ingress record. + # hosts: + # - chart-example.local + # annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # tls: + # Secrets must be manually created in the namespace. + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + +# Ingress for api access viahttps and whatever authentication you use +ingressAPI: + enabled: false + path: / + # Used to create an Ingress record. + # hosts: + # - chart-example.local + # annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # tls: + # Secrets must be manually created in the namespace. + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +mysql: + mysqlDatabase: luigidb + mysqlAllowEmptyPassword: true + persistence: + enabled: false