diff --git a/stable/hadoop/Chart.yaml b/stable/hadoop/Chart.yaml
index 9f7b364e6c..b270800c35 100644
--- a/stable/hadoop/Chart.yaml
+++ b/stable/hadoop/Chart.yaml
@@ -1,8 +1,8 @@
apiVersion: v1
description: The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using simple programming models.
name: hadoop
-version: 1.0.8
-appVersion: 2.7.3
+version: 1.1.0
+appVersion: 2.9.0
home: https://hadoop.apache.org/
sources:
- https://github.com/apache/hadoop
diff --git a/stable/hadoop/README.md b/stable/hadoop/README.md
index 275d26ea8e..0d3f8fcc70 100644
--- a/stable/hadoop/README.md
+++ b/stable/hadoop/README.md
@@ -39,9 +39,10 @@ The following table lists the configurable parameters of the Hadoop chart and th
| Parameter | Description | Default |
| ------------------------------------------------- | ------------------------------- | ---------------------------------------------------------------- |
-| `image` | Hadoop image ([source](https://github.com/Comcast/kube-yarn/tree/master/image)) | `danisla/hadoop:{VERSION}` |
-| `imagePullPolicy` | Pull policy for the images | `IfNotPresent` |
-| `hadoopVersion` | Version of hadoop libraries being used | `{VERSION}` |
+| `image.repository` | Hadoop image ([source](https://github.com/Comcast/kube-yarn/tree/master/image)) | `danisla/hadoop` |
+| `image.tag` | Hadoop image tag | `2.9.0` |
+| `imagee.pullPolicy` | Pull policy for the images | `IfNotPresent` |
+| `hadoopVersion` | Version of hadoop libraries being used | `2.9.0` |
| `antiAffinity` | Pod antiaffinity, `hard` or `soft` | `hard` |
| `hdfs.nameNode.pdbMinAvailable` | PDB for HDFS NameNode | `1` |
| `hdfs.nameNode.resources` | resources for the HDFS NameNode | `requests:memory=256Mi,cpu=10m,limits:memory=2048Mi,cpu=1000m` |
diff --git a/stable/hadoop/templates/NOTES.txt b/stable/hadoop/templates/NOTES.txt
index 5bb3b20387..cd0a12debe 100644
--- a/stable/hadoop/templates/NOTES.txt
+++ b/stable/hadoop/templates/NOTES.txt
@@ -1,24 +1,24 @@
1. You can check the status of HDFS by running this command:
- kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-hdfs-nn-0 -- /usr/local/hadoop/bin/hdfs dfsadmin -report
+ kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-hdfs-nn-0 -- /usr/local/hadoop/bin/hdfs dfsadmin -report
2. You can list the yarn nodes by running this command:
- kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/yarn node -list
+ kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/yarn node -list
3. Create a port-forward to the yarn resource manager UI:
- kubectl port-forward -n {{ .Release.Namespace }} {{ template "hadoop.fullname" . }}-yarn-rm-0 8088:8088
+ kubectl port-forward -n {{ .Release.Namespace }} {{ include "hadoop.fullname" . }}-yarn-rm-0 8088:8088
Then open the ui in your browser:
-
+
open http://localhost:8088
4. You can run included hadoop tests like this:
- kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-yarn-nm-0 -- /usr/local/hadoop/bin/hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-{{ .Values.hadoopVersion }}-tests.jar TestDFSIO -write -nrFiles 5 -fileSize 128MB -resFile /tmp/TestDFSIOwrite.txt
+ kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-yarn-nm-0 -- /usr/local/hadoop/bin/hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-{{ .Values.hadoopVersion }}-tests.jar TestDFSIO -write -nrFiles 5 -fileSize 128MB -resFile /tmp/TestDFSIOwrite.txt
5. You can list the mapreduce jobs like this:
- kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/mapred job -list
+ kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/mapred job -list
6. This chart can also be used with the zeppelin chart
- helm install --namespace {{ .Release.Namespace }} --set hadoop.useConfigMap=true,hadoop.configMapName={{ template "hadoop.fullname" . }} stable/zeppelin
+ helm install --namespace {{ .Release.Namespace }} --set hadoop.useConfigMap=true,hadoop.configMapName={{ include "hadoop.fullname" . }} stable/zeppelin
7. You can scale the number of yarn nodes like this:
helm upgrade {{ .Release.Name }} --set yarn.nodeManager.replicas=4 stable/hadoop
diff --git a/stable/hadoop/templates/_helpers.yaml b/stable/hadoop/templates/_helpers.tpl
similarity index 71%
rename from stable/hadoop/templates/_helpers.yaml
rename to stable/hadoop/templates/_helpers.tpl
index f480938efd..89b72a1c66 100644
--- a/stable/hadoop/templates/_helpers.yaml
+++ b/stable/hadoop/templates/_helpers.tpl
@@ -14,3 +14,10 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "hadoop.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
diff --git a/stable/hadoop/templates/hadoop-configmap.yaml b/stable/hadoop/templates/hadoop-configmap.yaml
index 71dac58460..4064ce4fc5 100644
--- a/stable/hadoop/templates/hadoop-configmap.yaml
+++ b/stable/hadoop/templates/hadoop-configmap.yaml
@@ -1,12 +1,11 @@
apiVersion: v1
kind: ConfigMap
metadata:
- name: {{ template "hadoop.fullname" . }}
+ name: {{ include "hadoop.fullname" . }}
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
data:
bootstrap.sh: |
#!/bin/bash
@@ -41,8 +40,8 @@ data:
if [[ "${HOSTNAME}" =~ "hdfs-dn" ]]; then
mkdir -p /root/hdfs/datanode
- # wait up to 30 seconds for namenode
- (while [[ $count -lt 15 && -z `curl -sf http://{{ template "hadoop.fullname" . }}-hdfs-nn:50070` ]]; do ((count=count+1)) ; echo "Waiting for {{ template "hadoop.fullname" . }}-hdfs-nn" ; sleep 2; done && [[ $count -lt 15 ]])
+ # wait up to 30 seconds for namenode
+ (while [[ $count -lt 15 && -z `curl -sf http://{{ include "hadoop.fullname" . }}-hdfs-nn:50070` ]]; do ((count=count+1)) ; echo "Waiting for {{ include "hadoop.fullname" . }}-hdfs-nn" ; sleep 2; done && [[ $count -lt 15 ]])
[[ $? -ne 0 ]] && echo "Timeout waiting for hdfs-nn, exiting." && exit 1
$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode
@@ -74,7 +73,7 @@ data:
chmod +x start-yarn-nm.sh
# wait up to 30 seconds for resourcemanager
- (while [[ $count -lt 15 && -z `curl -sf http://{{ template "hadoop.fullname" . }}-yarn-rm:8088/ws/v1/cluster/info` ]]; do ((count=count+1)) ; echo "Waiting for {{ template "hadoop.fullname" . }}-yarn-rm" ; sleep 2; done && [[ $count -lt 15 ]])
+ (while [[ $count -lt 15 && -z `curl -sf http://{{ include "hadoop.fullname" . }}-yarn-rm:8088/ws/v1/cluster/info` ]]; do ((count=count+1)) ; echo "Waiting for {{ include "hadoop.fullname" . }}-yarn-rm" ; sleep 2; done && [[ $count -lt 15 ]])
[[ $? -ne 0 ]] && echo "Timeout waiting for yarn-rm, exiting." && exit 1
./start-yarn-nm.sh
@@ -96,7 +95,7 @@ data:
fs.defaultFS
- hdfs://{{ template "hadoop.fullname" . }}-hdfs-nn:9000/
+ hdfs://{{ include "hadoop.fullname" . }}-hdfs-nn:9000/
NameNode URI
@@ -147,7 +146,7 @@ data:
0.0.0.0
-
+
mapred-site.xml: |
@@ -161,11 +160,11 @@ data:
mapreduce.jobhistory.address
- {{ template "hadoop.fullname" . }}-yarn-rm-0.{{ template "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:10020
+ {{ include "hadoop.fullname" . }}-yarn-rm-0.{{ include "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:10020
mapreduce.jobhistory.webapp.address
- {{ template "hadoop.fullname" . }}-yarn-rm-0.{{ template "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:19888
+ {{ include "hadoop.fullname" . }}-yarn-rm-0.{{ include "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:19888
@@ -253,7 +252,7 @@ data:
yarn.resourcemanager.hostname
- {{ template "hadoop.fullname" . }}-yarn-rm
+ {{ include "hadoop.fullname" . }}-yarn-rm
diff --git a/stable/hadoop/templates/hdfs-dn-pdb.yaml b/stable/hadoop/templates/hdfs-dn-pdb.yaml
index ccb77fbed5..e125f368c2 100644
--- a/stable/hadoop/templates/hdfs-dn-pdb.yaml
+++ b/stable/hadoop/templates/hdfs-dn-pdb.yaml
@@ -1,18 +1,16 @@
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
- name: {{ template "hadoop.fullname" . }}-hdfs-dn
+ name: {{ include "hadoop.fullname" . }}-hdfs-dn
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: hdfs-dn
spec:
selector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-dn
minAvailable: {{ .Values.hdfs.dataNode.pdbMinAvailable }}
-
\ No newline at end of file
diff --git a/stable/hadoop/templates/hdfs-dn-pvc.yaml b/stable/hadoop/templates/hdfs-dn-pvc.yaml
index 1d6ec16c0d..e4b0249186 100644
--- a/stable/hadoop/templates/hdfs-dn-pvc.yaml
+++ b/stable/hadoop/templates/hdfs-dn-pvc.yaml
@@ -2,12 +2,11 @@
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
- name: {{ template "hadoop.fullname" . }}-hdfs-dn
+ name: {{ include "hadoop.fullname" . }}-hdfs-dn
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: hdfs-dn
spec:
accessModes:
diff --git a/stable/hadoop/templates/hdfs-dn-statefulset.yaml b/stable/hadoop/templates/hdfs-dn-statefulset.yaml
index 2d29b95a22..fcaf72be28 100644
--- a/stable/hadoop/templates/hdfs-dn-statefulset.yaml
+++ b/stable/hadoop/templates/hdfs-dn-statefulset.yaml
@@ -1,22 +1,21 @@
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
- name: {{ template "hadoop.fullname" . }}-hdfs-dn
+ name: {{ include "hadoop.fullname" . }}-hdfs-dn
annotations:
checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: hdfs-dn
spec:
- serviceName: {{ template "hadoop.fullname" . }}-hdfs-dn
+ serviceName: {{ include "hadoop.fullname" . }}-hdfs-dn
replicas: {{ .Values.hdfs.dataNode.replicas }}
template:
metadata:
labels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-dn
spec:
@@ -27,7 +26,7 @@ spec:
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: hdfs-dn
{{- else if eq .Values.antiAffinity "soft" }}
@@ -37,20 +36,20 @@ spec:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: hdfs-dn
{{- end }}
terminationGracePeriodSeconds: 0
containers:
- name: hdfs-dn
- image: {{ .Values.image }}
- imagePullPolicy: {{ .Values.imagePullPolicy }}
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
command:
- "/bin/bash"
- "/tmp/hadoop-config/bootstrap.sh"
- "-d"
- resources:
+ resources:
{{ toYaml .Values.hdfs.dataNode.resources | indent 10 }}
readinessProbe:
httpGet:
@@ -72,11 +71,11 @@ spec:
volumes:
- name: hadoop-config
configMap:
- name: {{ template "hadoop.fullname" . }}
+ name: {{ include "hadoop.fullname" . }}
- name: dfs
{{- if .Values.persistence.dataNode.enabled }}
persistentVolumeClaim:
- claimName: {{ template "hadoop.fullname" . }}-hdfs-dn
- {{- else }}
+ claimName: {{ include "hadoop.fullname" . }}-hdfs-dn
+ {{- else }}
emptyDir: {}
{{- end }}
diff --git a/stable/hadoop/templates/hdfs-dn-svc.yaml b/stable/hadoop/templates/hdfs-dn-svc.yaml
index e809788044..e833b4ca4c 100644
--- a/stable/hadoop/templates/hdfs-dn-svc.yaml
+++ b/stable/hadoop/templates/hdfs-dn-svc.yaml
@@ -2,12 +2,11 @@
apiVersion: v1
kind: Service
metadata:
- name: {{ template "hadoop.fullname" . }}-hdfs-dn
+ name: {{ include "hadoop.fullname" . }}-hdfs-dn
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: hdfs-dn
spec:
ports:
@@ -18,6 +17,6 @@ spec:
port: 50075
clusterIP: None
selector:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
- component: hdfs-dn
\ No newline at end of file
+ component: hdfs-dn
diff --git a/stable/hadoop/templates/hdfs-nn-pdb.yaml b/stable/hadoop/templates/hdfs-nn-pdb.yaml
index 4fd3b3c82c..fcc7aa3b4e 100644
--- a/stable/hadoop/templates/hdfs-nn-pdb.yaml
+++ b/stable/hadoop/templates/hdfs-nn-pdb.yaml
@@ -1,18 +1,16 @@
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
- name: {{ template "hadoop.fullname" . }}-hdfs-nn
+ name: {{ include "hadoop.fullname" . }}-hdfs-nn
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: hdfs-nn
spec:
selector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-nn
minAvailable: {{ .Values.hdfs.nameNode.pdbMinAvailable }}
-
\ No newline at end of file
diff --git a/stable/hadoop/templates/hdfs-nn-pvc.yaml b/stable/hadoop/templates/hdfs-nn-pvc.yaml
index 6d86211b49..ebae1d92e6 100644
--- a/stable/hadoop/templates/hdfs-nn-pvc.yaml
+++ b/stable/hadoop/templates/hdfs-nn-pvc.yaml
@@ -2,12 +2,11 @@
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
- name: {{ template "hadoop.fullname" . }}-hdfs-nn
+ name: {{ include "hadoop.fullname" . }}-hdfs-nn
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: hdfs-nn
spec:
accessModes:
diff --git a/stable/hadoop/templates/hdfs-nn-statefulset.yaml b/stable/hadoop/templates/hdfs-nn-statefulset.yaml
index 707acfb75b..d2cca90ac3 100644
--- a/stable/hadoop/templates/hdfs-nn-statefulset.yaml
+++ b/stable/hadoop/templates/hdfs-nn-statefulset.yaml
@@ -1,22 +1,21 @@
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
- name: {{ template "hadoop.fullname" . }}-hdfs-nn
+ name: {{ include "hadoop.fullname" . }}-hdfs-nn
annotations:
checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: hdfs-nn
spec:
- serviceName: {{ template "hadoop.fullname" . }}-hdfs-nn
+ serviceName: {{ include "hadoop.fullname" . }}-hdfs-nn
replicas: 1
template:
metadata:
labels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-nn
spec:
@@ -27,7 +26,7 @@ spec:
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: hdfs-nn
{{- else if eq .Values.antiAffinity "soft" }}
@@ -37,15 +36,15 @@ spec:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: hdfs-nn
{{- end }}
terminationGracePeriodSeconds: 0
containers:
- name: hdfs-nn
- image: {{ .Values.image }}
- imagePullPolicy: {{ .Values.imagePullPolicy }}
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
command:
- "/bin/bash"
- "/tmp/hadoop-config/bootstrap.sh"
@@ -72,11 +71,11 @@ spec:
volumes:
- name: hadoop-config
configMap:
- name: {{ template "hadoop.fullname" . }}
+ name: {{ include "hadoop.fullname" . }}
- name: dfs
{{- if .Values.persistence.nameNode.enabled }}
persistentVolumeClaim:
- claimName: {{ template "hadoop.fullname" . }}-hdfs-nn
- {{- else }}
+ claimName: {{ include "hadoop.fullname" . }}-hdfs-nn
+ {{- else }}
emptyDir: {}
{{- end }}
diff --git a/stable/hadoop/templates/hdfs-nn-svc.yaml b/stable/hadoop/templates/hdfs-nn-svc.yaml
index c577629811..2f8b08ac3c 100644
--- a/stable/hadoop/templates/hdfs-nn-svc.yaml
+++ b/stable/hadoop/templates/hdfs-nn-svc.yaml
@@ -2,12 +2,11 @@
apiVersion: v1
kind: Service
metadata:
- name: {{ template "hadoop.fullname" . }}-hdfs-nn
+ name: {{ include "hadoop.fullname" . }}-hdfs-nn
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: hdfs-nn
spec:
ports:
@@ -18,6 +17,6 @@ spec:
port: 50070
clusterIP: None
selector:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-nn
\ No newline at end of file
diff --git a/stable/hadoop/templates/yarn-nm-pdb.yaml b/stable/hadoop/templates/yarn-nm-pdb.yaml
index cd4e459d04..151a2bb607 100644
--- a/stable/hadoop/templates/yarn-nm-pdb.yaml
+++ b/stable/hadoop/templates/yarn-nm-pdb.yaml
@@ -1,18 +1,16 @@
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
- name: {{ template "hadoop.fullname" . }}-yarn-nm
+ name: {{ include "hadoop.fullname" . }}-yarn-nm
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: yarn-nm
spec:
selector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-nm
minAvailable: {{ .Values.yarn.nodeManager.pdbMinAvailable }}
-
\ No newline at end of file
diff --git a/stable/hadoop/templates/yarn-nm-statefulset.yaml b/stable/hadoop/templates/yarn-nm-statefulset.yaml
index ba4aef2011..e3cde56016 100644
--- a/stable/hadoop/templates/yarn-nm-statefulset.yaml
+++ b/stable/hadoop/templates/yarn-nm-statefulset.yaml
@@ -1,17 +1,16 @@
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
- name: {{ template "hadoop.fullname" . }}-yarn-nm
+ name: {{ include "hadoop.fullname" . }}-yarn-nm
annotations:
checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: yarn-nm
spec:
- serviceName: {{ template "hadoop.fullname" . }}-yarn-nm
+ serviceName: {{ include "hadoop.fullname" . }}-yarn-nm
replicas: {{ .Values.yarn.nodeManager.replicas }}
{{- if .Values.yarn.nodeManager.parallelCreate }}
podManagementPolicy: Parallel
@@ -19,7 +18,7 @@ spec:
template:
metadata:
labels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-nm
spec:
@@ -30,7 +29,7 @@ spec:
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: yarn-nm
{{- else if eq .Values.antiAffinity "soft" }}
@@ -40,15 +39,15 @@ spec:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: yarn-nm
{{- end }}
terminationGracePeriodSeconds: 0
containers:
- name: yarn-nm
- image: {{ .Values.image }}
- imagePullPolicy: {{ .Values.imagePullPolicy }}
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
ports:
- containerPort: 8088
name: web
@@ -89,4 +88,4 @@ spec:
volumes:
- name: hadoop-config
configMap:
- name: {{ template "hadoop.fullname" . }}
+ name: {{ include "hadoop.fullname" . }}
diff --git a/stable/hadoop/templates/yarn-nm-svc.yaml b/stable/hadoop/templates/yarn-nm-svc.yaml
index da3297e03f..9638ed0276 100644
--- a/stable/hadoop/templates/yarn-nm-svc.yaml
+++ b/stable/hadoop/templates/yarn-nm-svc.yaml
@@ -2,12 +2,11 @@
apiVersion: v1
kind: Service
metadata:
- name: {{ template "hadoop.fullname" . }}-yarn-nm
+ name: {{ include "hadoop.fullname" . }}-yarn-nm
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: yarn-nm
spec:
ports:
@@ -19,6 +18,6 @@ spec:
name: api
clusterIP: None
selector:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
- component: yarn-nm
\ No newline at end of file
+ component: yarn-nm
diff --git a/stable/hadoop/templates/yarn-rm-pdb.yaml b/stable/hadoop/templates/yarn-rm-pdb.yaml
index f066771527..cc28fdd15d 100644
--- a/stable/hadoop/templates/yarn-rm-pdb.yaml
+++ b/stable/hadoop/templates/yarn-rm-pdb.yaml
@@ -1,18 +1,16 @@
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
- name: {{ template "hadoop.fullname" . }}-yarn-rm
+ name: {{ include "hadoop.fullname" . }}-yarn-rm
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: yarn-rm
spec:
selector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-rm
minAvailable: {{ .Values.yarn.resourceManager.pdbMinAvailable }}
-
\ No newline at end of file
diff --git a/stable/hadoop/templates/yarn-rm-statefulset.yaml b/stable/hadoop/templates/yarn-rm-statefulset.yaml
index da52182b3c..a93c24297e 100644
--- a/stable/hadoop/templates/yarn-rm-statefulset.yaml
+++ b/stable/hadoop/templates/yarn-rm-statefulset.yaml
@@ -1,22 +1,21 @@
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
- name: {{ template "hadoop.fullname" . }}-yarn-rm
+ name: {{ include "hadoop.fullname" . }}-yarn-rm
annotations:
checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: yarn-rm
spec:
- serviceName: {{ template "hadoop.fullname" . }}-yarn-rm
+ serviceName: {{ include "hadoop.fullname" . }}-yarn-rm
replicas: 1
template:
metadata:
labels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-rm
spec:
@@ -27,7 +26,7 @@ spec:
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: yarn-rm
{{- else if eq .Values.antiAffinity "soft" }}
@@ -37,15 +36,15 @@ spec:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: yarn-rm
{{- end }}
terminationGracePeriodSeconds: 0
containers:
- name: yarn-rm
- image: {{ .Values.image }}
- imagePullPolicy: {{ .Values.imagePullPolicy }}
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
ports:
- containerPort: 8088
name: web
@@ -73,4 +72,4 @@ spec:
volumes:
- name: hadoop-config
configMap:
- name: {{ template "hadoop.fullname" . }}
+ name: {{ include "hadoop.fullname" . }}
diff --git a/stable/hadoop/templates/yarn-rm-svc.yaml b/stable/hadoop/templates/yarn-rm-svc.yaml
index ea4e0a585e..81a75cbda6 100644
--- a/stable/hadoop/templates/yarn-rm-svc.yaml
+++ b/stable/hadoop/templates/yarn-rm-svc.yaml
@@ -2,12 +2,11 @@
apiVersion: v1
kind: Service
metadata:
- name: {{ template "hadoop.fullname" . }}-yarn-rm
+ name: {{ include "hadoop.fullname" . }}-yarn-rm
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: yarn-rm
spec:
ports:
@@ -15,6 +14,6 @@ spec:
name: web
clusterIP: None
selector:
- app: {{ template "hadoop.name" . }}
+ app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
- component: yarn-rm
\ No newline at end of file
+ component: yarn-rm
diff --git a/stable/hadoop/templates/yarn-ui-svc.yaml b/stable/hadoop/templates/yarn-ui-svc.yaml
index 48405daedd..1297c812b8 100644
--- a/stable/hadoop/templates/yarn-ui-svc.yaml
+++ b/stable/hadoop/templates/yarn-ui-svc.yaml
@@ -2,17 +2,16 @@
apiVersion: v1
kind: Service
metadata:
- name: {{ template "hadoop.fullname" . }}-yarn-ui
+ name: {{ include "hadoop.fullname" . }}-yarn-ui
labels:
- app: {{ template "hadoop.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app: {{ include "hadoop.name" . }}
+ chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
- heritage: {{ .Release.Service }}
component: yarn-ui
spec:
ports:
- port: 8088
name: web
selector:
- app: {{ template "hadoop.name" . }}
- component: yarn-rm
\ No newline at end of file
+ app: {{ include "hadoop.name" . }}
+ component: yarn-rm
diff --git a/stable/hadoop/tools/calc_resources.sh b/stable/hadoop/tools/calc_resources.sh
index 6005d31c62..85bb22d12a 100755
--- a/stable/hadoop/tools/calc_resources.sh
+++ b/stable/hadoop/tools/calc_resources.sh
@@ -39,7 +39,7 @@ for NODE in ${NODES}; do
# Get available memory
AVAIL_MEM[$i]=$(jq '.node.memory.availableBytes' <<< "${NODE_STATS[$i]}")
AVAIL_MEM[$i]=$(bc -l <<< "scale=0; ${AVAIL_MEM[$i]}/1024/1024")
-
+
# Derive available CPU
USED_CPU=$(jq '.node.cpu.usageNanoCores' <<< "${NODE_STATS[$i]}")
AVAIL_CPU[$i]=$(bc -l <<< "scale=2; (${TOTAL_CPU} - ${USED_CPU})/1000000")
diff --git a/stable/hadoop/values.yaml b/stable/hadoop/values.yaml
index f5f85ccda5..f2582ea769 100644
--- a/stable/hadoop/values.yaml
+++ b/stable/hadoop/values.yaml
@@ -1,10 +1,12 @@
# The base hadoop image to use for all components.
# See this repo for image build details: https://github.com/Comcast/kube-yarn/tree/master/image
-image: danisla/hadoop:2.7.3
-imagePullPolicy: IfNotPresent
+image:
+ repository: danisla/hadoop
+ tag: 2.9.0
+ pullPolicy: IfNotPresent
# The version of the hadoop libraries being used in the image.
-hadoopVersion: 2.7.3
+hadoopVersion: 2.9.0
# Select antiAffinity as either hard or soft, default is soft
antiAffinity: "soft"