github/deprecated-helm-charts
1
0
Fork 0
mirror of https://github.com/helm/charts.git synced 2026-08-23 22:37:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

[stable/hadoop] Hadoop bump to 2.9.0 and some cleanup (#10199)

Browse Source 
* bump hadoop to 2.9.0 and some cleanup

Signed-off-by: cpanato <ctadeu@gmail.com>

* remove heritage label

Signed-off-by: cpanato <ctadeu@gmail.com>
This commit is contained in:
Carlos Tadeu Panato Junior
2019-01-02 08:33:54 -08:00
committed by Kubernetes Prow Robot
parent 5070b5ffbb
commit 967250d3fb
22 changed files with 128 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
stable/hadoop/Chart.yaml
+2 -2
View File
@@ -1,8 +1,8 @@
apiVersion: v1
description: The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using simple programming models.
name: hadoop
version: 1.0.8
appVersion: 2.7.3
version: 1.1.0
appVersion: 2.9.0
home: https://hadoop.apache.org/
sources:
- https://github.com/apache/hadoop
stable/hadoop/README.md
+4 -3
View File
@@ -39,9 +39,10 @@ The following table lists the configurable parameters of the Hadoop chart and th
| Parameter | Description | Default |
| ------------------------------------------------- | ------------------------------- | ---------------------------------------------------------------- |
| `image` | Hadoop image ([source](https://github.com/Comcast/kube-yarn/tree/master/image)) | `danisla/hadoop:{VERSION}` |
| `imagePullPolicy` | Pull policy for the images | `IfNotPresent` |
| `hadoopVersion` | Version of hadoop libraries being used | `{VERSION}` |
| `image.repository` | Hadoop image ([source](https://github.com/Comcast/kube-yarn/tree/master/image)) | `danisla/hadoop` |
| `image.tag` | Hadoop image tag | `2.9.0` |
| `imagee.pullPolicy` | Pull policy for the images | `IfNotPresent` |
| `hadoopVersion` | Version of hadoop libraries being used | `2.9.0` |
| `antiAffinity` | Pod antiaffinity, `hard` or `soft` | `hard` |
| `hdfs.nameNode.pdbMinAvailable` | PDB for HDFS NameNode | `1` |
| `hdfs.nameNode.resources` | resources for the HDFS NameNode | `requests:memory=256Mi,cpu=10m,limits:memory=2048Mi,cpu=1000m` |
stable/hadoop/templates/NOTES.txt
+7 -7
View File
@@ -1,24 +1,24 @@
1. You can check the status of HDFS by running this command:
kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-hdfs-nn-0 -- /usr/local/hadoop/bin/hdfs dfsadmin -report
kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-hdfs-nn-0 -- /usr/local/hadoop/bin/hdfs dfsadmin -report
2. You can list the yarn nodes by running this command:
kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/yarn node -list
kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/yarn node -list
3. Create a port-forward to the yarn resource manager UI:
kubectl port-forward -n {{ .Release.Namespace }} {{ template "hadoop.fullname" . }}-yarn-rm-0 8088:8088
kubectl port-forward -n {{ .Release.Namespace }} {{ include "hadoop.fullname" . }}-yarn-rm-0 8088:8088
Then open the ui in your browser:
open http://localhost:8088
4. You can run included hadoop tests like this:
kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-yarn-nm-0 -- /usr/local/hadoop/bin/hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-{{ .Values.hadoopVersion }}-tests.jar TestDFSIO -write -nrFiles 5 -fileSize 128MB -resFile /tmp/TestDFSIOwrite.txt
kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-yarn-nm-0 -- /usr/local/hadoop/bin/hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-{{ .Values.hadoopVersion }}-tests.jar TestDFSIO -write -nrFiles 5 -fileSize 128MB -resFile /tmp/TestDFSIOwrite.txt
5. You can list the mapreduce jobs like this:
kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/mapred job -list
kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/mapred job -list
6. This chart can also be used with the zeppelin chart
helm install --namespace {{ .Release.Namespace }} --set hadoop.useConfigMap=true,hadoop.configMapName={{ template "hadoop.fullname" . }} stable/zeppelin
helm install --namespace {{ .Release.Namespace }} --set hadoop.useConfigMap=true,hadoop.configMapName={{ include "hadoop.fullname" . }} stable/zeppelin
7. You can scale the number of yarn nodes like this:
helm upgrade {{ .Release.Name }} --set yarn.nodeManager.replicas=4 stable/hadoop
stable/hadoop/templates/_helpers.yaml → stable/hadoop/templates/_helpers.tpl
+7
View File
@@ -14,3 +14,10 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "hadoop.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
stable/hadoop/templates/hadoop-configmap.yaml
+11 -12
View File
@@ -1,12 +1,11 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "hadoop.fullname" . }}
name: {{ include "hadoop.fullname" . }}
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
data:
bootstrap.sh: |
#!/bin/bash
@@ -41,8 +40,8 @@ data:
if [[ "${HOSTNAME}" =~ "hdfs-dn" ]]; then
mkdir -p /root/hdfs/datanode
# wait up to 30 seconds for namenode
(while [[ $count -lt 15 && -z `curl -sf http://{{ template "hadoop.fullname" . }}-hdfs-nn:50070` ]]; do ((count=count+1)) ; echo "Waiting for {{ template "hadoop.fullname" . }}-hdfs-nn" ; sleep 2; done && [[ $count -lt 15 ]])
# wait up to 30 seconds for namenode
(while [[ $count -lt 15 && -z `curl -sf http://{{ include "hadoop.fullname" . }}-hdfs-nn:50070` ]]; do ((count=count+1)) ; echo "Waiting for {{ include "hadoop.fullname" . }}-hdfs-nn" ; sleep 2; done && [[ $count -lt 15 ]])
[[ $? -ne 0 ]] && echo "Timeout waiting for hdfs-nn, exiting." && exit 1
$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode
@@ -74,7 +73,7 @@ data:
chmod +x start-yarn-nm.sh
# wait up to 30 seconds for resourcemanager
(while [[ $count -lt 15 && -z `curl -sf http://{{ template "hadoop.fullname" . }}-yarn-rm:8088/ws/v1/cluster/info` ]]; do ((count=count+1)) ; echo "Waiting for {{ template "hadoop.fullname" . }}-yarn-rm" ; sleep 2; done && [[ $count -lt 15 ]])
(while [[ $count -lt 15 && -z `curl -sf http://{{ include "hadoop.fullname" . }}-yarn-rm:8088/ws/v1/cluster/info` ]]; do ((count=count+1)) ; echo "Waiting for {{ include "hadoop.fullname" . }}-yarn-rm" ; sleep 2; done && [[ $count -lt 15 ]])
[[ $? -ne 0 ]] && echo "Timeout waiting for yarn-rm, exiting." && exit 1
./start-yarn-nm.sh
@@ -96,7 +95,7 @@ data:
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://{{ template "hadoop.fullname" . }}-hdfs-nn:9000/</value>
<value>hdfs://{{ include "hadoop.fullname" . }}-hdfs-nn:9000/</value>
<description>NameNode URI</description>
</property>
</configuration>
@@ -147,7 +146,7 @@ data:
<value>0.0.0.0</value>
</property>
<!-- /Bind to all interfaces -->
</configuration>
mapred-site.xml: |
@@ -161,11 +160,11 @@ data:
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>{{ template "hadoop.fullname" . }}-yarn-rm-0.{{ template "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:10020</value>
<value>{{ include "hadoop.fullname" . }}-yarn-rm-0.{{ include "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>{{ template "hadoop.fullname" . }}-yarn-rm-0.{{ template "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:19888</value>
<value>{{ include "hadoop.fullname" . }}-yarn-rm-0.{{ include "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:19888</value>
</property>
</configuration>
@@ -253,7 +252,7 @@ data:
<configuration>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>{{ template "hadoop.fullname" . }}-yarn-rm</value>
<value>{{ include "hadoop.fullname" . }}-yarn-rm</value>
</property>
<!-- Bind to all interfaces -->
stable/hadoop/templates/hdfs-dn-pdb.yaml
+4 -6
View File
@@ -1,18 +1,16 @@
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: {{ template "hadoop.fullname" . }}-hdfs-dn
name: {{ include "hadoop.fullname" . }}-hdfs-dn
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: hdfs-dn
spec:
selector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-dn
minAvailable: {{ .Values.hdfs.dataNode.pdbMinAvailable }}
stable/hadoop/templates/hdfs-dn-pvc.yaml
+3 -4
View File
@@ -2,12 +2,11 @@
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "hadoop.fullname" . }}-hdfs-dn
name: {{ include "hadoop.fullname" . }}-hdfs-dn
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: hdfs-dn
spec:
accessModes:
stable/hadoop/templates/hdfs-dn-statefulset.yaml
+13 -14
View File
@@ -1,22 +1,21 @@
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: {{ template "hadoop.fullname" . }}-hdfs-dn
name: {{ include "hadoop.fullname" . }}-hdfs-dn
annotations:
checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: hdfs-dn
spec:
serviceName: {{ template "hadoop.fullname" . }}-hdfs-dn
serviceName: {{ include "hadoop.fullname" . }}-hdfs-dn
replicas: {{ .Values.hdfs.dataNode.replicas }}
template:
metadata:
labels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-dn
spec:
@@ -27,7 +26,7 @@ spec:
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: hdfs-dn
{{- else if eq .Values.antiAffinity "soft" }}
@@ -37,20 +36,20 @@ spec:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: hdfs-dn
{{- end }}
terminationGracePeriodSeconds: 0
containers:
- name: hdfs-dn
image: {{ .Values.image }}
imagePullPolicy: {{ .Values.imagePullPolicy }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
command:
- "/bin/bash"
- "/tmp/hadoop-config/bootstrap.sh"
- "-d"
resources:
resources:
{{ toYaml .Values.hdfs.dataNode.resources | indent 10 }}
readinessProbe:
httpGet:
@@ -72,11 +71,11 @@ spec:
volumes:
- name: hadoop-config
configMap:
name: {{ template "hadoop.fullname" . }}
name: {{ include "hadoop.fullname" . }}
- name: dfs
{{- if .Values.persistence.dataNode.enabled }}
persistentVolumeClaim:
claimName: {{ template "hadoop.fullname" . }}-hdfs-dn
{{- else }}
claimName: {{ include "hadoop.fullname" . }}-hdfs-dn
{{- else }}
emptyDir: {}
{{- end }}
stable/hadoop/templates/hdfs-dn-svc.yaml
+5 -6
View File
@@ -2,12 +2,11 @@
apiVersion: v1
kind: Service
metadata:
name: {{ template "hadoop.fullname" . }}-hdfs-dn
name: {{ include "hadoop.fullname" . }}-hdfs-dn
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: hdfs-dn
spec:
ports:
@@ -18,6 +17,6 @@ spec:
port: 50075
clusterIP: None
selector:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-dn
component: hdfs-dn
stable/hadoop/templates/hdfs-nn-pdb.yaml
+4 -6
View File
@@ -1,18 +1,16 @@
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: {{ template "hadoop.fullname" . }}-hdfs-nn
name: {{ include "hadoop.fullname" . }}-hdfs-nn
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: hdfs-nn
spec:
selector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-nn
minAvailable: {{ .Values.hdfs.nameNode.pdbMinAvailable }}
stable/hadoop/templates/hdfs-nn-pvc.yaml
+3 -4
View File
@@ -2,12 +2,11 @@
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "hadoop.fullname" . }}-hdfs-nn
name: {{ include "hadoop.fullname" . }}-hdfs-nn
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: hdfs-nn
spec:
accessModes:
stable/hadoop/templates/hdfs-nn-statefulset.yaml
+12 -13
View File
@@ -1,22 +1,21 @@
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: {{ template "hadoop.fullname" . }}-hdfs-nn
name: {{ include "hadoop.fullname" . }}-hdfs-nn
annotations:
checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: hdfs-nn
spec:
serviceName: {{ template "hadoop.fullname" . }}-hdfs-nn
serviceName: {{ include "hadoop.fullname" . }}-hdfs-nn
replicas: 1
template:
metadata:
labels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-nn
spec:
@@ -27,7 +26,7 @@ spec:
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: hdfs-nn
{{- else if eq .Values.antiAffinity "soft" }}
@@ -37,15 +36,15 @@ spec:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: hdfs-nn
{{- end }}
terminationGracePeriodSeconds: 0
containers:
- name: hdfs-nn
image: {{ .Values.image }}
imagePullPolicy: {{ .Values.imagePullPolicy }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
command:
- "/bin/bash"
- "/tmp/hadoop-config/bootstrap.sh"
@@ -72,11 +71,11 @@ spec:
volumes:
- name: hadoop-config
configMap:
name: {{ template "hadoop.fullname" . }}
name: {{ include "hadoop.fullname" . }}
- name: dfs
{{- if .Values.persistence.nameNode.enabled }}
persistentVolumeClaim:
claimName: {{ template "hadoop.fullname" . }}-hdfs-nn
{{- else }}
claimName: {{ include "hadoop.fullname" . }}-hdfs-nn
{{- else }}
emptyDir: {}
{{- end }}
stable/hadoop/templates/hdfs-nn-svc.yaml
+4 -5
View File
@@ -2,12 +2,11 @@
apiVersion: v1
kind: Service
metadata:
name: {{ template "hadoop.fullname" . }}-hdfs-nn
name: {{ include "hadoop.fullname" . }}-hdfs-nn
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: hdfs-nn
spec:
ports:
@@ -18,6 +17,6 @@ spec:
port: 50070
clusterIP: None
selector:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: hdfs-nn
stable/hadoop/templates/yarn-nm-pdb.yaml
+4 -6
View File
@@ -1,18 +1,16 @@
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: {{ template "hadoop.fullname" . }}-yarn-nm
name: {{ include "hadoop.fullname" . }}-yarn-nm
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: yarn-nm
spec:
selector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-nm
minAvailable: {{ .Values.yarn.nodeManager.pdbMinAvailable }}
stable/hadoop/templates/yarn-nm-statefulset.yaml
+10 -11
View File
@@ -1,17 +1,16 @@
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: {{ template "hadoop.fullname" . }}-yarn-nm
name: {{ include "hadoop.fullname" . }}-yarn-nm
annotations:
checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: yarn-nm
spec:
serviceName: {{ template "hadoop.fullname" . }}-yarn-nm
serviceName: {{ include "hadoop.fullname" . }}-yarn-nm
replicas: {{ .Values.yarn.nodeManager.replicas }}
{{- if .Values.yarn.nodeManager.parallelCreate }}
podManagementPolicy: Parallel
@@ -19,7 +18,7 @@ spec:
template:
metadata:
labels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-nm
spec:
@@ -30,7 +29,7 @@ spec:
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: yarn-nm
{{- else if eq .Values.antiAffinity "soft" }}
@@ -40,15 +39,15 @@ spec:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: yarn-nm
{{- end }}
terminationGracePeriodSeconds: 0
containers:
- name: yarn-nm
image: {{ .Values.image }}
imagePullPolicy: {{ .Values.imagePullPolicy }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
ports:
- containerPort: 8088
name: web
@@ -89,4 +88,4 @@ spec:
volumes:
- name: hadoop-config
configMap:
name: {{ template "hadoop.fullname" . }}
name: {{ include "hadoop.fullname" . }}
stable/hadoop/templates/yarn-nm-svc.yaml
+5 -6
View File
@@ -2,12 +2,11 @@
apiVersion: v1
kind: Service
metadata:
name: {{ template "hadoop.fullname" . }}-yarn-nm
name: {{ include "hadoop.fullname" . }}-yarn-nm
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: yarn-nm
spec:
ports:
@@ -19,6 +18,6 @@ spec:
name: api
clusterIP: None
selector:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-nm
component: yarn-nm
stable/hadoop/templates/yarn-rm-pdb.yaml
+4 -6
View File
@@ -1,18 +1,16 @@
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: {{ template "hadoop.fullname" . }}-yarn-rm
name: {{ include "hadoop.fullname" . }}-yarn-rm
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: yarn-rm
spec:
selector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-rm
minAvailable: {{ .Values.yarn.resourceManager.pdbMinAvailable }}
stable/hadoop/templates/yarn-rm-statefulset.yaml
+10 -11
View File
@@ -1,22 +1,21 @@
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: {{ template "hadoop.fullname" . }}-yarn-rm
name: {{ include "hadoop.fullname" . }}-yarn-rm
annotations:
checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: yarn-rm
spec:
serviceName: {{ template "hadoop.fullname" . }}-yarn-rm
serviceName: {{ include "hadoop.fullname" . }}-yarn-rm
replicas: 1
template:
metadata:
labels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-rm
spec:
@@ -27,7 +26,7 @@ spec:
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: yarn-rm
{{- else if eq .Values.antiAffinity "soft" }}
@@ -37,15 +36,15 @@ spec:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name | quote }}
component: yarn-rm
{{- end }}
terminationGracePeriodSeconds: 0
containers:
- name: yarn-rm
image: {{ .Values.image }}
imagePullPolicy: {{ .Values.imagePullPolicy }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
ports:
- containerPort: 8088
name: web
@@ -73,4 +72,4 @@ spec:
volumes:
- name: hadoop-config
configMap:
name: {{ template "hadoop.fullname" . }}
name: {{ include "hadoop.fullname" . }}
stable/hadoop/templates/yarn-rm-svc.yaml
+5 -6
View File
@@ -2,12 +2,11 @@
apiVersion: v1
kind: Service
metadata:
name: {{ template "hadoop.fullname" . }}-yarn-rm
name: {{ include "hadoop.fullname" . }}-yarn-rm
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: yarn-rm
spec:
ports:
@@ -15,6 +14,6 @@ spec:
name: web
clusterIP: None
selector:
app: {{ template "hadoop.name" . }}
app: {{ include "hadoop.name" . }}
release: {{ .Release.Name }}
component: yarn-rm
component: yarn-rm
stable/hadoop/templates/yarn-ui-svc.yaml
+5 -6
View File
@@ -2,17 +2,16 @@
apiVersion: v1
kind: Service
metadata:
name: {{ template "hadoop.fullname" . }}-yarn-ui
name: {{ include "hadoop.fullname" . }}-yarn-ui
labels:
app: {{ template "hadoop.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app: {{ include "hadoop.name" . }}
chart: {{ include "hadoop.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: yarn-ui
spec:
ports:
- port: 8088
name: web
selector:
app: {{ template "hadoop.name" . }}
component: yarn-rm
app: {{ include "hadoop.name" . }}
component: yarn-rm
stable/hadoop/tools/calc_resources.sh
+1 -1
View File
@@ -39,7 +39,7 @@ for NODE in ${NODES}; do
# Get available memory
AVAIL_MEM[$i]=$(jq '.node.memory.availableBytes' <<< "${NODE_STATS[$i]}")
AVAIL_MEM[$i]=$(bc -l <<< "scale=0; ${AVAIL_MEM[$i]}/1024/1024")
# Derive available CPU
USED_CPU=$(jq '.node.cpu.usageNanoCores' <<< "${NODE_STATS[$i]}")
AVAIL_CPU[$i]=$(bc -l <<< "scale=2; (${TOTAL_CPU} - ${USED_CPU})/1000000")
stable/hadoop/values.yaml
+5 -3
View File
@@ -1,10 +1,12 @@
# The base hadoop image to use for all components.
# See this repo for image build details: https://github.com/Comcast/kube-yarn/tree/master/image
image: danisla/hadoop:2.7.3
imagePullPolicy: IfNotPresent
image:
repository: danisla/hadoop
tag: 2.9.0
pullPolicy: IfNotPresent
# The version of the hadoop libraries being used in the image.
hadoopVersion: 2.7.3
hadoopVersion: 2.9.0
# Select antiAffinity as either hard or soft, default is soft
antiAffinity: "soft"