[stable/luigi] Initial Commit (#3567)

* fist pass at luigi scheduler * updating to match comments * updating to match comments * fixing ports and adding selector
2026-08-23 22:37:45 +00:00 · 2018-02-24 02:36:45 -08:00
parent 95c24c136c
commit 76acc43b9d
14 changed files with 518 additions and 0 deletions
@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
@@ -0,0 +1,21 @@
+apiVersion: v1
+description: Luigi is a Python module that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization etc. It also comes with Hadoop support built in.
+home: https://luigi.readthedocs.io/en/stable/
+maintainers:
+- name: sammerry
+  email: sam.merry@gmail.com
+sources:
+- https://github.com/spotify/luigi/
+- https://github.com/sammerry/luigi-docker-image/
+icon: https://raw.githubusercontent.com/spotify/luigi/master/doc/luigi.png
+keywords:
+- luigi
+- scheduler
+- dependency graph
+- web
+- task management
+- hadoop
+- spark
+- kubernetes job manager
+name: luigi
+version: 2.7.2
@@ -0,0 +1,79 @@
+# Luigi Scheduler
+
+[Luigi](https://github.com/spotify/luigi) is a Python module that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization etc. It also comes with Hadoop support built in.
+
+## TL;DR;
+
+```console
+$ helm install incubator/luigi
+```
+
+## Introduction
+
+This chart bootstraps a [Luigi](https://github.com/spotify/luigi) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
+
+## Installing the Chart
+
+To install the chart with the release name `my-release`:
+
+```console
+$ helm install --name my-release stable/luigi
+```
+## Uninstalling the Chart
+
+To uninstall/delete the `my-release` deployment:
+
+```console
+$ helm delete my-release --purge
+```
+
+The command removes all the Kubernetes components associated with the chart and deletes the release.
+
+## Luigi Configuration
+
+[Luigi](https://github.com/spotify/luigi) configs are set as a block of text through a configmap and mouted as a file in /etc/luigi Any value in this text block should match the defined luigi configuration. There are several values here that will have to match our kubernetes configuration.
+
+## Configuration
+
+The following tables lists the configurable parameters of the Sentry chart and their default values.
+
+| Parameter                            | Description                                | Default                                                    |
+| -------------------------------      | -------------------------------            | ---------------------------------------------------------- |
+| `image.repository`                   | Luigi image                                | `getpolymorph/luigi`                                       |
+| `image.tag`                          | Luigi image tag                            | `2.7.2`                                                    |
+| `image.pullPolicy`                   | Luigi image pull policy                    | `IfNotPresent`                                                   |
+| `service.name`                       | Luigi service name                         | `luigi`                                                    |
+| `service.type`                       | The kube service type                      | `LoadBalancer`                                             |
+| `service.externalPort`               | The service external port                  | `3000`                                                     |
+| `service.internalPort`               | The service internal port                  | `8082`                                                     |
+| `service.config`                     | The luigi service configs                  | View this default in the values.yaml file                  |
+| `persistence.enabled`                | Persistence flag                           | `false`                                                    |
+| `ingressUI.enabled`                  | UI Ingress Flag                            | `false`                                                    |
+| `ingressAPI.enabled`                 | API Ingress Flag                           | `false`                                                    |
+
+Dependent charts can also have values overwritten. Preface values with postgresql.* or redis.*
+
+Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
+
+```console
+$ helm install --name my-release \
+  --set persistence.enabled=false,email.host=email \
+    stable/luigi
+```
+
+Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example,
+
+```console
+$ helm install --name my-release -f values.yaml stable/luigi
+```
+
+Read through the [values.yaml](values.yaml) file. It has several commented out suggested values.
+
+## Persistence
+
+Luigi requires a pickled state file. To maintain state after a restart you'll need to enable persistence.
+`--set persistence.enabled=true`
+
+## Ingress
+
+This chart provides support for two Ingress resources. This is to allow authentication in the ui via reverse proxy with something like oauth-proxy and a separate form of authentication for luigi worker access.
@@ -0,0 +1,6 @@
+dependencies:
+- name: mysql
+  repository: https://kubernetes-charts.storage.googleapis.com/
+  version: 0.3.4
+digest: sha256:4bfa5b90139dd5d0a9ef9e9fecaac9df407d3a24ee2f54bc05d4eb5cad5ab9c6
+generated: 2018-02-04T22:05:57.911294008-08:00
@@ -0,0 +1,4 @@
+dependencies:
+  - name: mysql
+    version: 0.3.4
+    repository: https://kubernetes-charts.storage.googleapis.com/
@@ -0,0 +1,25 @@
+  _           _       _ 
+ | |         (_)     (_)
+ | |    _   _ _  __ _ _ 
+ | |   | | | | |/ _` | |
+ | |___| |_| | | (_| | |
+ |______\__,_|_|\__, |_|
+                 __/ |  
+                |___/   
+
+Get the application URL by running these commands:
+{{- if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "luigi.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get svc -w {{ template "luigi.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "luigi.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.service.externalPort }}
+{{- else if contains "ClusterIP"  .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "luigi.name" . }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl port-forward $POD_NAME 8080:{{ .Values.service.externalPort }}
+{{- end }}
+
@@ -0,0 +1,25 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "luigi.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "luigi.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "luigi.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "luigi.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  client.cfg: |
+{{ printf .Values.service.config | indent 4 }}
+
+  logging.cfg: |
+    [loggers]
+    keys=root,tornado,client,scheduler,server
+
+    [logger_root]
+    level=DEBUG
+    handlers=console
+
+    [logger_client]
+    level=DEBUG
+    handlers=console
+    qualname=luigi-interface
+    propagate=0
+
+    [logger_server]
+    level=DEBUG
+    handlers=console
+    qualname=luigi.server
+    propagate=0
+
+    [logger_scheduler]
+    level=DEBUG
+    handlers=console
+    qualname=luigi.scheduler
+    propagate=0
+
+    [logger_tornado]
+    level=DEBUG
+    handlers=warnconsole
+    qualname=tornado
+    propagate=0
+
+    [formatters]
+    keys=detail
+
+    [formatter_detail]
+    class=logging.Formatter
+    format=%(asctime)s %(name)-15s %(levelname)-8s %(message)s
+
+    [handlers]
+    keys=console,warnconsole
+
+    [handler_console]
+    level=INFO
+    class=StreamHandler
+    args=(sys.stdout,)
+    formatter=detail
+
+    [handler_warnconsole]
+    level=WARNING
+    class=StreamHandler
+    args=(sys.stdout,)
+    formatter=detail
+
@@ -0,0 +1,75 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: {{ template "luigi.fullname" . }}
+  labels:
+    app: {{ template "luigi.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ template "luigi.name" . }}
+      release: {{ .Release.Name }}
+  replicas: {{ .Values.replicaCount }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "luigi.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          command: ["/bin/sh", "-c", "luigid"]
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - containerPort: 8082
+          livenessProbe:
+            httpGet:
+              path: /static/visualiser/index.html
+              port: 8082
+          readinessProbe:
+            httpGet:
+              path: /static/visualiser/index.html
+              port: 8082
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+          volumeMounts:
+          - name: {{ template "luigi.name" . }}-luigi-conf
+            mountPath: /etc/luigi/client.cfg
+            subPath: client.cfg
+          - name: {{ template "luigi.name" . }}-logging-conf
+            mountPath: /etc/luigi/logging.cfg
+            subPath: logging.cfg
+          - name: luigi-state
+            mountPath: /luigi/state/
+
+    {{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+    {{- end }}
+
+      volumes:
+      - name: {{ template "luigi.name" . }}-luigi-conf
+        configMap:
+          name: {{ template "luigi.fullname" . }}
+          items:
+          - key: client.cfg
+            path: client.cfg
+
+      - name: {{ template "luigi.name" . }}-logging-conf
+        configMap:
+          name: {{ template "luigi.fullname" . }}
+          items:
+          - key: logging.cfg
+            path: logging.cfg
+
+      - name: luigi-state
+      {{- if .Values.persistence.enabled }}
+        persistentVolumeClaim:
+          claimName: {{ template "luigi.fullname" . }}
+      {{- else }}
+        emptyDir: {}
+      {{ end }}
@@ -0,0 +1,33 @@
+{{- if .Values.ingressAPI.enabled -}}
+{{- $serviceName := include "luigi.fullname" . -}}
+{{- $servicePort := .Values.service.externalPort -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: {{ template "luigi.fullname" . }}-api
+  labels:
+    app: {{ template "luigi.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    {{- range $key, $value := .Values.ingressAPI.annotations }}
+      {{ $key }}: {{ $value | quote }}
+    {{- end }}
+spec:
+  rules:
+    {{- range $host := .Values.ingressAPI.hosts }}
+    - host: {{ $host }}
+      http:
+        paths:
+          - path: /
+            backend:
+              serviceName: {{ $serviceName }}
+              servicePort: {{ $servicePort }}
+    {{- end -}}
+  {{- if .Values.ingressAPI.tls }}
+  tls:
+{{ toYaml .Values.ingressAPI.tls | indent 4 }}
+  {{- end -}}
+{{- end -}}
+
@@ -0,0 +1,32 @@
+{{- if .Values.ingressUI.enabled -}}
+{{- $serviceName := include "luigi.fullname" . -}}
+{{- $servicePort := .Values.service.externalPort -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: {{ template "luigi.fullname" . }}-ui
+  labels:
+    app: {{ template "luigi.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    {{- range $key, $value := .Values.ingressUI.annotations }}
+      {{ $key }}: {{ $value | quote }}
+    {{- end }}
+spec:
+  rules:
+    {{- range $host := .Values.ingressUI.hosts }}
+    - host: {{ $host }}
+      http:
+        paths:
+          - path: /
+            backend:
+              serviceName: {{ $serviceName }}
+              servicePort: {{ $servicePort }}
+    {{- end -}}
+  {{- if .Values.ingressUI.tls }}
+  tls:
+{{ toYaml .Values.ingressUI.tls | indent 4 }}
+  {{- end -}}
+{{- end -}}
@@ -0,0 +1,25 @@
+{{- if .Values.persistence.enabled -}}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "luigi.fullname" . }}
+  labels:
+    app: {{ template "luigi.fullname" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+spec:
+  accessModes:
+    - {{ .Values.persistence.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size | quote }}
+{{- if .Values.persistence.storageClass }}
+{{- if (eq "-" .Values.persistence.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.persistence.storageClass }}"
+{{- end }}
+{{- end }}
+{{- end -}}
+
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "luigi.fullname" . }}
+  labels:
+    app: {{ template "luigi.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.externalPort }}
+      targetPort: 8082
+      protocol: TCP
+      name: http
+  selector:
+    app: {{ template "luigi.name" . }}
+    release: {{ .Release.Name }}
@@ -0,0 +1,85 @@
+# Default values for luigi.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+replicaCount: 1
+image:
+  repository: axiom/docker-luigi
+  tag: 2.7.2-alpine
+  pullPolicy: IfNotPresent
+
+service:
+  name: luigi
+  type: LoadBalancer
+  externalPort: 80
+
+  # Luigi config: these values should mattch the luigi documentation
+  # https://luigi.readthedocs.io/en/stable/configuration.html
+  config: |
+    [core]
+    logging_conf_file=/etc/luigi/logging.cfg
+
+    [scheduler]
+    record_task_history=true
+    state-path=/luigi/state/luigi-state.pickle
+
+    [task_history]
+    db_connection=mysql://luigi-mysql/luigidb
+
+# creates a persistent volume claim for
+# luigi state pickel
+persistence:
+  enabled: false
+  size: 1G
+  accessMode: ReadWriteOnce
+
+# Ingress for ui access for use with authentication like oauth-proxy
+# depending on the authentication you use. You may only need one ingress.
+ingressUI:
+  enabled: false
+  path: /
+  # Used to create an Ingress record.
+  # hosts:
+  # - chart-example.local
+  # annotations:
+  #   kubernetes.io/ingress.class: nginx
+  #   kubernetes.io/tls-acme: "true"
+  # tls:
+  # Secrets must be manually created in the namespace.
+  # - secretName: chart-example-tls
+  #   hosts:
+  #     - chart-example.local
+
+
+# Ingress for api access viahttps and whatever authentication you use
+ingressAPI:
+  enabled: false
+  path: /
+  # Used to create an Ingress record.
+  # hosts:
+  # - chart-example.local
+  # annotations:
+  #   kubernetes.io/ingress.class: nginx
+  #   kubernetes.io/tls-acme: "true"
+  # tls:
+  # Secrets must be manually created in the namespace.
+  # - secretName: chart-example-tls
+  #   hosts:
+  #     - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #  cpu: 100m
+  #  memory: 128Mi
+  # requests:
+  #  cpu: 100m
+  #  memory: 128Mi
+
+mysql:
+  mysqlDatabase: luigidb
+  mysqlAllowEmptyPassword: true
+  persistence:
+    enabled: false