diff --git a/.dockerignore b/.dockerignore index 79f96a3c2..cb656fc49 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,4 @@ ui/coverage ui/node_modules vendor Dockerfile -Dockerfile.* +demo diff --git a/Dockerfile.web b/Dockerfile.web deleted file mode 100644 index 302d0a5cf..000000000 --- a/Dockerfile.web +++ /dev/null @@ -1,12 +0,0 @@ -FROM alpine:latest -COPY --from=lmierzwa/karma:latest /karma /karma -RUN adduser -D karma -USER karma -ENV LOG_CONFIG=false -ENV ALERTMANAGER_INTERVAL=2400h -ENV ALERTMANAGER_URI=file:///mock -ENV LABELS_COLOR_UNIQUE="@receiver instance cluster" -ENV LABELS_COLOR_STATIC="job" -ENV FILTERS_DEFAULT="@receiver=by-cluster-service" -COPY internal/mock/0.15.2 /mock -CMD /karma diff --git a/Makefile b/Makefile index 93e535a90..61f6c1461 100644 --- a/Makefile +++ b/Makefile @@ -156,9 +156,3 @@ mock-assets: .build/deps-build-go.ok .PHONY: ui ui: .build/artifacts-ui.ok - -.PHONY: heroku -heroku: - docker pull lmierzwa/karma:latest - heroku container:push web -R - heroku container:release web diff --git a/demo/Dockerfile.web b/demo/Dockerfile.web new file mode 100644 index 000000000..dfec65c50 --- /dev/null +++ b/demo/Dockerfile.web @@ -0,0 +1,16 @@ +FROM alpine:latest + +RUN apk add --update supervisor python && rm -rf /tmp/* /var/cache/apk/* +COPY supervisord.conf /etc/supervisord.conf + +COPY --from=prom/alertmanager:latest /bin/alertmanager /alertmanager +COPY alertmanager.yaml /etc/alertmanager.yaml + +COPY generator.py /generator.py + +COPY --from=lmierzwa/karma:latest /karma /karma +COPY karma.yaml /etc/karma.yaml + +RUN adduser -D karma +USER karma +CMD supervisord --nodaemon --configuration /etc/supervisord.conf diff --git a/demo/Makefile b/demo/Makefile new file mode 100644 index 000000000..9d4c4e678 --- /dev/null +++ b/demo/Makefile @@ -0,0 +1,5 @@ +.PHONY: heroku +heroku: + docker pull lmierzwa/karma:latest + heroku container:push web -R + heroku container:release web diff --git a/demo/alertmanager.yaml b/demo/alertmanager.yaml new file mode 100644 index 000000000..2b1ceee09 --- /dev/null +++ b/demo/alertmanager.yaml @@ -0,0 +1,32 @@ +global: + resolve_timeout: 30s +route: + group_by: ['alertname'] + group_wait: 5s + group_interval: 10s + repeat_interval: 999h + receiver: 'default' + routes: + - receiver: 'by-cluster-service' + group_by: ['alertname', 'cluster', 'service'] + match_re: + alertname: .* + continue: true + - receiver: 'by-name' + group_by: [alertname] + match_re: + alertname: .* + continue: true + +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + # Apply inhibition if the alertname is the same. + equal: ['alertname', 'cluster', 'service'] + +receivers: + - name: 'default' + - name: 'by-cluster-service' + - name: 'by-name' diff --git a/demo/generator.py b/demo/generator.py new file mode 100755 index 000000000..bcfb06b2e --- /dev/null +++ b/demo/generator.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python +""" +Generates alerts and sends to Alertmanager API. + +1. Start Alertmanager: + +$ docker run \ + --rm \ + --name prom \ + -p 9093:9093 \ + -v $(pwd)/alertmanager.yml:/etc/alertmanager/alertmanager.yml \ + prom/alertmanager + +2. Start this script +3. Start karma: + +$ karma \ + --alertmanager.uri http://localhost:9093 \ + --alertmanager.interval 10s \ + --annotations.hidden help \ + --labels.color.unique "@receiver instance cluster" \ + --labels.color.static job \ + --filters.default "@receiver=by-cluster-service" +""" + + +import random +import json +import datetime +import time +import urllib2 + + +API = "http://localhost:9093" +MAX_INTERVAL = 10 +MIN_INTERVAL = 5 + + +def jsonGetRequest(uri): + req = urllib2.Request(uri) + response = urllib2.urlopen(req) + return json.load(response) + +def jsonPostRequest(uri, data): + req = urllib2.Request(uri) + req.add_header("Content-Type", "application/json") + response = urllib2.urlopen(req, json.dumps(data)) + + +def addSilence(matchers, startsAt, endsAt, createdBy, comment): + uri = "{}/api/v1/silences".format(API) + + silences = jsonGetRequest(uri) + found = False + for silence in silences["data"]: + if silence["status"]["state"] != "active": + continue + if silence["createdBy"] == createdBy and silence["comment"] == comment: + if json.dumps(silence["matchers"], sort_keys=True) == json.dumps( + matchers, sort_keys=True): + found = True + break + + if not found: + jsonPostRequest(uri, { + "matchers": matchers, + "startsAt": startsAt, + "endsAt": endsAt, + "createdBy": createdBy, + "comment": comment + }) + + +def addAlerts(alerts): + jsonPostRequest("{}/api/v1/alerts".format(API), alerts) + + +def newMatcher(name, value, isRegex): + return {"name": name, "value": value, "isRegex": isRegex} + + +def newAlert(labels, annotations=None, generatorURL="http://localhost:9093"): + return { + "labels": labels, + "annotations": annotations or {}, + "generatorURL": generatorURL + } + + +class AlertGenerator(object): + name = "Fake Alert" + comment = "" + + def __init__(self, interval=15): + self._interval = interval + self._lastSend = 1 + + def _annotations(self, **kwargs): + annotations = {"help": self.comment} + annotations.update(kwargs) + return annotations + + def _labels(self, **kwargs): + labels = {"alertname": self.name} + labels.update(kwargs) + return labels + + def _send(self): + alerts = self.alerts() + if alerts: + addAlerts(alerts) + + for silence in self.silences(): + addSilence(*silence) + + def alerts(self): + return [] + + def silences(self): + return [] + + def tick(self): + if time.time() - self._lastSend >= self._interval: + self._send() + self._lastSend = time.time() + + +class AlwaysOnAlert(AlertGenerator): + name = "Always On Alert" + comment = "This alert is always firing" + + def alerts(self): + def _gen(size, cluster): + return [newAlert( + self._labels(instance="server{}".format(i), cluster=cluster), + self._annotations( + summary="Silence this alert, it's always firing") + ) for i in xrange(1, size)] + return _gen(10, "dev") + _gen(5, "staging") + _gen(3, "prod") + + +class RandomInstances(AlertGenerator): + name = "Random Instances" + comment = "This alerts will have a random number of instances" + + def alerts(self): + instances = random.randint(0, 30) + return [ + newAlert( + self._labels(instance="server{}".format(i), cluster="staging"), + self._annotations( + dashboard="https://www.google.com/search?q=" + "server{}".format(i)) + ) for i in xrange(0, instances) + ] + + +class RandomName(AlertGenerator): + name = "Random Alert Name" + comment = "This alerts will have a random name" + + def alerts(self): + alerts = [] + for i in xrange(0, 1): + throw = random.randint(0, 1000) + alerts.append( + newAlert( + self._labels(alertname="Alert Nr {}".format(throw), + instance="server{}".format(i), + cluster="dev"), + self._annotations( + summary="This is a random alert", + dashboard="https://www.google.com/search?q=" + "server{}".format(i)) + ) + ) + return alerts + + +class LowChance(AlertGenerator): + name = "Low Chance" + comment = "This alert has only a 20% chance of firing" + + def alerts(self): + throw = random.randint(0, 100) + if throw > 20: + return [] + return [ + newAlert( + self._labels(instance="server{}".format(i), cluster="dev"), + self._annotations() + ) for i in xrange(0, 3) + ] + + +class TimeAnnotation(AlertGenerator): + name = "Time Annotation" + comment = "This alert includes a 'time' annotation that changes every N \ + seconds" + + def alerts(self): + return [ + newAlert(self._labels(instance="server1", cluster="prod"), + self._annotations(time=str(int(time.time()))) + ) + ] + + +class DiskFreeLowAlert(AlertGenerator): + name = "Disk Free Low" + comment = "This alert simulates a warning about low disk space" + + def alerts(self): + alerts = [] + for i in xrange(0, 10): + spaceFree = throw = random.randint(0, 10) + alerts.append( + newAlert(self._labels(instance="server{}".format(i), + cluster="prod", + device="/dev/sda{}".format(i), + mount_point="/disk"), + self._annotations( + summary="Only {}% free space left on /disk".format( + spaceFree), + dashboard="https://wikipedia.org/wiki/Disk_storage") + ) + ) + return alerts + + +class SilencedAlert(AlertGenerator): + name = "Always Silenced Alert" + comment = "This alert is always silenced" + + def alerts(self): + return [ + newAlert(self._labels(instance="server1", cluster="prod"), + self._annotations( + alertReference="https://www." + "youtube.com/watch?v=dQw4w9WgXcQ") + ) + ] + + def silences(self): + now = datetime.datetime.utcnow().replace(microsecond=0) + return [ + ( + [newMatcher("alertname", SilencedAlert.name, False)], + "{}Z".format(now.isoformat()), + "{}Z".format((now + datetime.timedelta( + minutes=30)).isoformat()), + "me@example.com", + "Silence '{}''".format(self.name) + ) + ] + + +if __name__ == "__main__": + generators = [ + AlwaysOnAlert(MAX_INTERVAL), + RandomInstances(MAX_INTERVAL), + LowChance(MAX_INTERVAL), + TimeAnnotation(MIN_INTERVAL), + DiskFreeLowAlert(MIN_INTERVAL), + SilencedAlert(MAX_INTERVAL), + RandomName(MAX_INTERVAL), + ] + while True: + for g in generators: + g.tick() + time.sleep(1) diff --git a/demo/karma.yaml b/demo/karma.yaml new file mode 100644 index 000000000..246ea8841 --- /dev/null +++ b/demo/karma.yaml @@ -0,0 +1,27 @@ +alertmanager: + interval: 10s + servers: + - name: demo + uri: "http://localhost:9093" + timeout: 10s + proxy: true +annotations: + hidden: + - help +filters: + default: + - "@receiver=by-cluster-service" +labels: + color: + static: + - job + unique: + - cluster + - instance + - "@receiver" +log: + config: false + level: warning +sentry: + private: https://84a9ef37a6ed4fdb80e9ea2310d1ed26:8c6ee6f0ab02406482ff4b4e824e2c27@sentry.io/1279017 + public: https://84a9ef37a6ed4fdb80e9ea2310d1ed26@sentry.io/1279017 diff --git a/demo/supervisord.conf b/demo/supervisord.conf new file mode 100644 index 000000000..439637d5d --- /dev/null +++ b/demo/supervisord.conf @@ -0,0 +1,31 @@ +[supervisord] +nodaemon=true +pidfile=/tmp/supervisord.pid +logfile = /tmp/supervisord.log +logfile_maxbytes = 1MB +logfile_backups=0 +loglevel = info + +[program:alertmanager] +command=/alertmanager --config.file=/etc/alertmanager.yaml --storage.path=/tmp/alertmanager +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:generator] +command=/generator.py +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:karma] +command=/karma --config.dir /etc/ +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/internal/mock/generator.py b/internal/mock/generator.py deleted file mode 100755 index 08d238c38..000000000 --- a/internal/mock/generator.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python -""" -Generates alerts and sends to Alertmanager API. - -1. Start Alertmanager: - -$ docker run \ - --rm \ - --name prom \ - -p 9093:9093 \ - -v $(pwd)/alertmanager.yml:/etc/alertmanager/alertmanager.yml \ - prom/alertmanager - -2. Start this script -3. Start karma: - -$ karma \ - --alertmanager.uri http://localhost:9093 \ - --alertmanager.interval 10s \ - --annotations.hidden help \ - --labels.color.unique "@receiver instance cluster" \ - --labels.color.static job \ - --filters.default "@receiver=by-cluster-service" -""" - - -import random -import json -import time -import urllib2 - - -API = "http://localhost:9093" - - -def jsonPostRequest(uri, data): - req = urllib2.Request(uri) - req.add_header("Content-Type", "application/json") - response = urllib2.urlopen(req, json.dumps(data)) - - -def addSilence(matchers, startsAt, endsAt, createdBy, comment): - jsonPostRequest("{}/api/v1/silences".format(API), { - "matchers": matchers, - "startsAt": startsAt, - "endsAt": endsAt, - "createdBy": createdBy, - "comment": comment - }) - - -def addAlerts(alerts): - jsonPostRequest("{}/api/v1/alerts".format(API), alerts) - - -def newMatcher(name, value, isRegex): - return {"name": name, "value": value, "isRegex": isRegex} - - -def newAlert(labels, annotations=None, generatorURL="http://localhost:9093"): - return { - "labels": labels, - "annotations": annotations or {}, - "generatorURL": generatorURL - } - - -class AlertGenerator(object): - name = "Fake Alert" - - def __init__(self, interval=15): - self._interval = interval - self._lastSend = 1 - - def _labels(self, **kwargs): - labels = {"alertname": self.name} - labels.update(kwargs) - return labels - - def _send(self): - alerts = self.generate() - if alerts: - print("{} sending {} alert(s)".format(self.name, len(alerts))) - addAlerts(alerts) - - def tick(self): - if time.time() - self._lastSend >= self._interval: - self._send() - self._lastSend = time.time() - - -class AlwaysOnAlert(AlertGenerator): - name = "Always On Alert" - - def generate(self): - return [ - newAlert( - self._labels(instance="server{}".format(i)) - ) for i in xrange(0, 10) - ] - - -class RandomInstances(AlertGenerator): - name = "Random Instances" - - def generate(self): - instances = random.randint(0, 30) - return [ - newAlert( - self._labels(instance="server{}".format(i)) - ) for i in xrange(0, instances) - ] - - -class LowChance(AlertGenerator): - name = "Low Chance" - - def generate(self): - throw = random.randint(0, 100) - if throw > 10: - return [] - return [ - newAlert( - self._labels(instance="server{}".format(i)) - ) for i in xrange(0, 3) - ] - - -class TimeAnnotation(AlertGenerator): - name = "Time Annotation" - - def generate(self): - annotations = {"time": str(int(time.time()))} - return [ - newAlert(self._labels(instance="server1"), annotations) - ] - - -if __name__ == "__main__": - generators = [ - AlwaysOnAlert(15), - RandomInstances(30), - LowChance(60), - TimeAnnotation(5), - ] - while True: - for g in generators: - g.tick() - time.sleep(1)