First commit

This commit is contained in:
Marc Campbell
2019-05-15 23:58:50 +00:00
commit d84f0ad89d
29 changed files with 6008 additions and 0 deletions
+67
View File
@@ -0,0 +1,67 @@
workflow "Deploy to Heroku" {
on = "push"
resolves = ["release registry", "release hooks", "release reaper"]
}
action "heroku login" {
uses = "actions/heroku@master"
args = "container:login"
secrets = ["HEROKU_API_KEY"]
}
action "build registry" {
uses = "actions/docker/cli@master"
needs = "heroku login"
args = "build -t registry.heroku.com/replreg/web registry"
}
action "push registry" {
uses = "actions/docker/cli@master"
needs = "build registry"
args = "push registry.heroku.com/replreg/web"
}
action "release registry" {
uses = "actions/heroku@master"
needs = "push registry"
args = "container:release -a replreg web"
secrets = ["HEROKU_API_KEY"]
}
action "build hooks" {
uses = "actions/docker/cli@master"
needs = "heroku login"
args = "build -f hooks/Dockerfile.hooks -t registry.heroku.com/replreg-hooks/web hooks"
}
action "push hooks" {
uses = "actions/docker/cli@master"
needs = "build hooks"
args = "push registry.heroku.com/replreg-hooks/web"
}
action "release hooks" {
uses = "actions/heroku@master"
needs = "push hooks"
args = "container:release -a replreg-hooks web"
secrets = ["HEROKU_API_KEY"]
}
action "build reaper" {
uses = "actions/docker/cli@master"
needs = "heroku login"
args = "build -f hooks/Dockerfile.reap -t registry.heroku.com/replreg-hooks/reap hooks"
}
action "push reaper" {
uses = "actions/docker/cli@master"
needs = "build reaper"
args = "push registry.heroku.com/replreg-hooks/reap"
}
action "release reaper" {
uses = "actions/heroku@master"
needs = "push reaper"
args = "container:release -a replreg-hooks webreap"
secrets = ["HEROKU_API_KEY"]
}
+11
View File
@@ -0,0 +1,11 @@
# replreg
An ephemeral container registry for CI workflows.
## What is replreg?
replreg is an anonymous, expiring Docker container registry using the official Docker Registry image. This is a set of tools and configuration that can be used to delpoy the registry without authentication, but with self-expiring images.
+2
View File
@@ -0,0 +1,2 @@
node_modules
build
+13
View File
@@ -0,0 +1,13 @@
FROM node:10 as deps
ADD ./package.json /src/package.json
ADD ./Makefile /src/Makefile
WORKDIR /src
RUN make deps
FROM node:10
ADD . /src
WORKDIR /src
COPY --from=0 /src .
RUN make test
ENTRYPOINT ["node"]
CMD ["--no-deprecation", "build/server.js", "hooks"]
+13
View File
@@ -0,0 +1,13 @@
FROM node:10 as deps
ADD ./package.json /src/package.json
ADD ./Makefile /src/Makefile
WORKDIR /src
RUN make deps
FROM node:10
ADD . /src
WORKDIR /src
COPY --from=0 /src .
RUN make test
ENTRYPOINT ["node"]
CMD ["--no-deprecation", "build/server.js", "reap"]
+46
View File
@@ -0,0 +1,46 @@
.PHONY: prebuild
prebuild:
rm -rf build
mkdir -p build
.PHONY: deps
deps:
npm install -g node-gyp
npm i
.PHONY: lint
lint:
`npm bin`/tslint --project ./tsconfig.json --fix
.PHONY: test
test: build
npm test
.PHONY: build
build: prebuild
`npm bin`/tsc
.PHONY: run
run:
node --no-deprecation ./build/server.js hooks
.PHONY: reap
reap:
node --no-deprecation ./build/server.js reap
.PHONY: publish
publish: publish-hooks publish-reap
.PHONY: publish-hooks
publish-hooks: test
docker build -f Dockerfile.hooks -t registry.heroku.com/replreg-hooks/web .
docker push registry.heroku.com/replreg-hooks/web
heroku container:release web -a replreg-hooks
.PHONY: publish-reap
publish-reap: test
docker build -f Dockerfile.reap -t registry.heroku.com/replreg-hooks/reap .
docker push registry.heroku.com/replreg-hooks/reap
heroku container:release reap -a replreg-hooks
+2
View File
@@ -0,0 +1,2 @@
# replreg hooks api serer
+4
View File
@@ -0,0 +1,4 @@
build:
docker:
hooks: Dockerfile.hooks
reap: Dockerfile.reap
+5124
View File
File diff suppressed because it is too large Load Diff
+55
View File
@@ -0,0 +1,55 @@
{
"name": "hooks",
"version": "1.0.0",
"description": "replreg hooks",
"license": "Apache-2.0",
"main": "./build/server.js",
"dependencies": {
"@octokit/rest": "^16.16.3",
"@sentry/node": "^4.6.4",
"@types/node": "^11.9.5",
"apac": "^3.0.2",
"bluebird": "^3.5.3",
"body-parser": "^1.19.0",
"cors": "^2.8.5",
"cron": "^1.7.1",
"express": "^4.16.4",
"fast-crc32c": "^1.0.4",
"jsonwebtoken": "^8.5.0",
"left-pad": "^1.1.3",
"lodash": "^4.17.11",
"moment": "^2.24.0",
"parse-duration": "^0.1.1",
"pg": "^7.8.1",
"pino": "^5.12.3",
"pino-pretty": "^2.6.1",
"randomstring": "^1.1.5",
"redis": "^2.8.0",
"request": "^2.88.0",
"request-promise": "^4.2.4",
"simple-oauth2": "^2.2.1",
"slug": "^1.0.0",
"source-map-support": "^0.5.10",
"ts-express-decorators": "^5.14.1",
"ts-log-debug": "^5.1.0",
"yargs": "^13.2.2"
},
"devDependencies": {
"@types/express": "^4.16.1",
"@types/lodash": "^4.14.121",
"@types/mocha": "^5.2.6",
"chai": "^4.2.0",
"istanbul": "^0.4.5",
"mocha": "^6.0.2",
"mocha-junit-reporter": "^1.13.0",
"mocha-typescript": "^1.0.23",
"ts-node": "^8.0.2",
"tslint": "^5.13.0",
"typemoq": "^2.1.0",
"typescript": "^3.3.3333"
},
"scripts": {
"test": "echo 0"
},
"snyk": true
}
+32
View File
@@ -0,0 +1,32 @@
import * as util from "util";
import { Server } from "../server/server";
import { logger } from "../logger";
import * as Sentry from "@sentry/node";
import { param } from "../util";
exports.name = "hooks";
exports.describe = "Start and run the hook api server";
exports.builder = {
};
exports.handler = async (argv) => {
main(argv).catch((err) => {
console.log(`Failed with error ${util.inspect(err)}`);
process.exit(1);
});
};
async function main(argv): Promise<any> {
process.on('SIGTERM', function onSigterm () {
logger.info(`Got SIGTERM, cleaning up`);
process.exit();
});
Sentry.init({
dsn: param.get("SENTRY_DSN_API"),
environment: param.get("ENVIRONMENT"),
});
await new Server().start();
}
+84
View File
@@ -0,0 +1,84 @@
import * as util from "util";
import { CronJob } from "cron";
import { logger } from "../logger";
import * as redis from "redis";
import { promisify } from "util";
import * as rp from "request-promise";
const client = redis.createClient({url: process.env["REDISCLOUD_URL"]});
const smembersAsync = promisify(client.smembers).bind(client);
const sremAsync = promisify(client.srem).bind(client);
const hgetAsync = promisify(client.hget).bind(client);
const delAsync = promisify(client.del).bind(client);
exports.name = "reap";
exports.describe = "find and purge expirable images";
exports.builder = {
};
exports.handler = async (argv) => {
main(argv).catch((err) => {
console.log(`Failed with error ${util.inspect(err)}`);
process.exit(1);
});
};
async function main(argv): Promise<any> {
process.on('SIGTERM', function onSigterm () {
logger.info(`Got SIGTERM, cleaning up`);
process.exit();
});
const job = new CronJob({
cronTime: "* * * * *",
onTick: async () => {
console.log("-----> beginning to reap expired images");
const now = new Date().getTime();
const images = await smembersAsync("current.images");
console.log(` there are ${images.length} total images to evaluate`);
for (const image of images) {
const expireAt = await hgetAsync(image, "expires");
if (+expireAt > now) {
const minutesLeft = (+expireAt - now) / 1000 / 60;
console.log(`not expiring ${image} for another ~${Math.round(minutesLeft)} minute(s)`);
continue;
}
const imageAndTag = image.split(":");
const headers = {
"Accept": "application/vnd.docker.distribution.manifest.v2+json",
};
// Get the manifest from the tag
const getOptions = {
method: "HEAD",
uri: `https://replreg.is/v2/${imageAndTag[0]}/manifests/${imageAndTag[1]}`,
headers,
resolveWithFullResponse: true,
}
const getResponse = await rp(getOptions);
const deleteURI = `https://replreg.is/v2/${imageAndTag[0]}/manifests/${getResponse.headers.etag.replace(/"/g,"")}`;
// Remove from the registry
const options = {
method: "DELETE",
uri: deleteURI,
headers,
}
await rp(options);
console.log(`expiring ${image}`);
await delAsync(image);
await sremAsync("current.images", image);
}
},
start: true,
});
job.start();
}
+27
View File
@@ -0,0 +1,27 @@
import * as Express from "express";
import {
Controller,
Get,
Res } from "ts-express-decorators";
interface ErrorResponse {
error: any;
}
@Controller("/healthz")
export class HealthzAPI {
/**
* /healthz handler
*
* @param request
* @param response
* @returns {{id: any, name: string}}
*/
@Get("")
public async check(
@Res() response: Express.Response,
): Promise<{}> {
response.status(200);
return {};
}
}
+78
View File
@@ -0,0 +1,78 @@
import * as Express from "express";
import {
Controller,
Post,
Res,
HeaderParams,
BodyParams,
Req} from "ts-express-decorators";
import * as parseDuration from "parse-duration";
import * as redis from "redis";
import { promisify } from "util";
interface ErrorResponse {
error: any;
}
const client = redis.createClient({url: process.env["REDISCLOUD_URL"]});
const saddAsync = promisify(client.sadd).bind(client);
const hsetAsync = promisify(client.hset).bind(client);
@Controller("/v1/hook")
export class HookAPI {
/**
* /v1/exec handler
*
* @param request
* @param response
* @returns {{id: any, name: string}}
*/
@Post("/registry-event")
public async hook(
@Res() response: Express.Response,
@Req() request: Express.Request,
@HeaderParams("Authorization") authorization: string,
@BodyParams("") body: any,
): Promise<ErrorResponse | {}> {
if (authorization !== `Token ${process.env["HOOK_TOKEN"]}`) {
response.status(401);
return {};
}
for (const event of body.events) {
if (event.action === "push") {
const image = event.target.repository;
const tag = event.target.tag;
if (!image || !tag) {
continue;
}
const imageWithTag = `${image}:${tag}`;
// default to 1h
let expireInSeconds = 60 * 60 * 1000;
console.log(`parsing tag ${tag}`);
const parsed = parseDuration(tag);
if (parsed > 0) {
expireInSeconds = parsed;
// enforce a max of 24 hours
if (expireInSeconds > 24 * 60 * 60 * 1000) {
expireInSeconds = 24 * 60 * 60 * 1000;
}
}
await saddAsync("current.images", imageWithTag);
const now = new Date().getTime();
const then = now + expireInSeconds;
await hsetAsync(imageWithTag, "created", now, "expires", then);
}
}
return {};
}
}
+22
View File
@@ -0,0 +1,22 @@
import * as pino from "pino";
import * as process from "process";
function initLogger(): any {
const logOptions = {
name: process.env["LOG_NAME"] || "replreg",
safe: true,
prettyPrint: process.env["LOG_PRETTY"] || false,
};
if (process.env["LOG_PRETTY"]) {
const logger = pino(logOptions);
logger.level = process.env["LOG_LEVEL"] || "warn";
return logger;
} else {
const logger = pino(logOptions)
logger.level = process.env["LOG_LEVEL"] || "warn";
return logger;
}
}
export const logger = initLogger();
+160
View File
@@ -0,0 +1,160 @@
import * as _ from "lodash";
import * as express from "express";
import * as util from "util";
import * as uuid from "uuid";
import { logger } from "./logger";
export interface Response<T> {
status: number;
body: T;
contentType?: string;
headers?: object;
filename?: string;
}
export interface RawResponse extends Response<string> {
}
export const Responses = {
created(entity: any): RawResponse {
return {
status: 201,
body: JSON.stringify(entity),
};
},
};
/*
* This file contains express middleware functions
* for Pre/Post request logging and response generation.
*/
export const onSuccess = (res: express.Response, reqId: string, statusCodeGetter?: () => number | undefined) => (result: any) => {
if (result) {
const statusToSend = result.status || (statusCodeGetter && statusCodeGetter()) || 200;
const body = result.body || JSON.stringify(result);
const contentType = result.contentType || "application/json";
let bodyToLog = body;
if (!bodyToLog) {
bodyToLog = "";
} else if (bodyToLog.length > 512) {
bodyToLog = `${bodyToLog.substring(0, 512)} (... truncated, total ${bodyToLog.length} bytes)`;
}
if (res.statusCode !== 200) {
logger.warn(`[${reqId}] WARN response already has statusCode ${res.statusCode}, a response might have already been sent!`);
logger.warn(util.inspect(res));
}
logger.info(`[${reqId}] => ${statusToSend} ${bodyToLog}`);
const respObj = res.status(statusToSend).type(contentType).set("X-Replreg-RequestId", reqId);
if (result.filename) {
respObj.attachment(result.filename);
}
if (result.headers) {
_.forOwn(result.headers, (value, key) => {
respObj.set(key!, value);
});
}
respObj.send(body);
} else {
const statusToSend = (statusCodeGetter && statusCodeGetter()) || 200;
logger.info(`[${reqId}] => ${statusToSend}`);
res.status(statusToSend).set("X-Replreg-RequestId", reqId).json(result);
}
};
export const onError = (res: express.Response, reqId: string) => (err: any) => {
if (err.status) {
handleFrameworkError(err, reqId, res);
} else {
handleUnexpectedError(err, reqId, res);
}
};
function handleFrameworkError(err: any, reqId: string, res: express.Response) {
// Structured error, specific status code.
const errMsg = err.err ? err.err.message : err.message || "An unexpected error occurred";
logger.info(`[${reqId}] !! ${err.status} ${errMsg} ${err.stack || util.inspect(err)}`);
const errClass = err.constructor.name;
const hasMeaningfulType = ["Object", "Error"].indexOf(errClass) === -1;
const bodyToSend = {
status: err.status,
type: hasMeaningfulType ? errClass : "Error",
error: errMsg,
invalid: err.invalid,
};
res.status(err.status).set("X-Replreg-RequestId", reqId).json(bodyToSend);
}
function handleUnexpectedError(err: any, reqId: string, res: express.Response) {
// Generic error, default code (500).
const bodyToSend = {
status: 500,
error: err.message || "An unexpected error occurred",
type: "Error",
};
if (
(err.message ? err.message : "").indexOf("Can't set headers after they are sent") !== -1 ||
(err.stack ? err.stack : "").indexOf("Can't set headers after they are sent") !== -1) {
logger.error("Middleware error, current response object is", util.inspect(res));
}
logger.error(`[${reqId}] !! 500 ${err.stack || err.message || util.inspect(err)}`);
res.status(500).set("X-Replreg-RequestId", reqId).json(bodyToSend);
}
export const preRequest = (req: express.Request, reqId: string) => {
logger.info(`[${reqId}] <- ${req.method} ${req.originalUrl}`);
if (!_.isEmpty(req.body)) {
let bodyString = JSON.stringify(req.body);
if (bodyString.length > 512) {
bodyString = `${bodyString.substring(0, 512)} (... truncated, total ${bodyString.length} bytes)`;
}
logger.info(`[${reqId}] <- ${bodyString}`);
}
};
export const requestId = (req: express.Request, handlerName: string) => {
const id = `${handlerName}:${uuid.v4().replace("-", "").substring(0, 8)}`;
const clientID: string = <string> req.headers["x-request-uuid"];
if (clientID) {
return `${id}.${clientID.substring(0, 8)}`;
}
return id;
};
export const wrapRoute = (route, handlerName) =>
(req: express.Request, res: express.Response) => {
const reqId = requestId(req, handlerName);
preRequest(req, reqId);
route.handler(req)
.then(onSuccess(res, reqId))
.catch(onError(res, reqId));
};
export function register(route, handler, app) {
// Register this route and callback with express.
if (route.method === "get") {
logger.debug(`GET '${route.path}'`);
app.get(route.path, handler);
} else if (route.method === "post") {
logger.debug(`POST '${route.path}'`);
app.post(route.path, handler);
} else if (route.method === "put") {
logger.debug(`PUT '${route.path}'`);
app.put(route.path, handler);
} else if (route.method === "delete") {
logger.debug(`DELETE '${route.path}'`);
app.delete(route.path, handler);
} else {
logger.debug(`Unhandled HTTP method: '${route.method}'`);
}
}
+8
View File
@@ -0,0 +1,8 @@
import * as yargs from "yargs";
yargs
.commandDir("../build/commands")
.env()
.help()
.demandCommand()
.argv;
+83
View File
@@ -0,0 +1,83 @@
import * as Express from "express";
import * as util from "util";
import {
Err,
IMiddlewareError,
MiddlewareError,
Next,
Request,
Response,
} from "ts-express-decorators";
import { logger } from "../logger";
export class HTTPError extends Error {
public static requireMatch(field: string, pattern: RegExp, name: string) {
const isValid = pattern.test(field);
if (!isValid) {
throw new HTTPError(400, {
message: `Missing or invalid parameters: ${name}`,
code: "bad_request",
extra: {name},
});
}
}
constructor(
public readonly status: number,
public readonly body: any,
) {
super((body && body.message) || "Internal Error");
}
}
export class ServerError extends HTTPError {
constructor() {
super(500, {
error: {
message: "A server error has occurred",
},
});
}
}
export class Unauthorized extends HTTPError {
constructor() {
super(401, {
error: {
message: "Unauthorized",
},
});
}
}
export class Errors {
public static Unauthorized = Unauthorized;
public static ServerError = ServerError;
}
@MiddlewareError()
export class ErrorMiddleware implements IMiddlewareError {
public use(
@Err() error: any,
@Request() request: Express.Request,
@Response() response: Express.Response,
@Next() next: Express.NextFunction,
): any {
logger.debug("Handling error", error);
if (response.headersSent) {
logger.debug("Headers sent, skipping error handling");
return next(error);
}
if (!(error instanceof HTTPError)) {
// its an unhandled error so log it and then return a regular 500
logger.error("Handling internal server error " + util.inspect(error));
error = new ServerError();
}
response.status(error.status).send(JSON.stringify(error.body));
return next();
}
}
+61
View File
@@ -0,0 +1,61 @@
import "source-map-support/register";
import * as cors from "cors";
import {ServerLoader, ServerSettings} from "ts-express-decorators";
import {$log} from "ts-log-debug";
import Path = require("path");
import getPgPool from "../util/pg";
import * as bodyParser from "body-parser";
let port = process.env.PORT;
if (port == null || port == "") {
port = "8000";
}
@ServerSettings({
rootDir: Path.resolve(__dirname),
mount: {
"/": "${rootDir}/../controllers/**/*.js",
},
acceptMimes: ["application/json"],
port: Number(port),
httpsPort: 0,
debug: false,
logger: {
level: "warn",
}
})
export class Server extends ServerLoader {
/**
* This method let you configure the middleware required by your application to works.
* @returns {Server}
*/
public async $onMountingMiddlewares(): Promise<null> {
this.expressApp.enable("trust proxy"); // so we get the real ip from the ELB in amaazon
this.use(bodyParser.json({
type: [
"application/json",
"application/vnd.docker.distribution.events.v1+json",
],
}));
this.use(cors());
const pool = getPgPool();
if (process.env["NODE_ENV"] === "production") {
$log.level = "OFF";
}
return null;
}
public $onReady() {
console.log("Server started...");
}
public $onServerInitError(err) {
console.error(err);
}
}
+2
View File
@@ -0,0 +1,2 @@
export * from "./params";
export * from "./pg";
+5
View File
@@ -0,0 +1,5 @@
export class param {
public static get(envName: string): string {
return process.env[envName] || "";
}
}
+5
View File
@@ -0,0 +1,5 @@
--require ts-node/register
--require source-map-support/register
--colors
src/**/*_test.ts
+29
View File
@@ -0,0 +1,29 @@
{
"compilerOptions": {
"target": "es2017",
"module": "commonjs",
"lib": ["es6", "dom"],
"types": ["reflect-metadata", "node", "mocha"],
"noImplicitAny": false,
"noEmitOnError": true,
"sourceMap": true,
"preserveConstEnums": true,
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"strictNullChecks": true,
"skipLibCheck": true,
"allowJs": true,
"outDir": "build",
"pretty": true
},
"include": [
"src/**/*.js",
"src/**/*.ts"
],
"exclude": [
"node_modules",
"src/**/*_test.ts",
"src/test/**/*.ts",
"src/data/fixtures.js"
]
}
+18
View File
@@ -0,0 +1,18 @@
{
"extends": "tslint:recommended",
"jsRules": {
"no-console": [false],
"object-literal-sort-keys": false,
"max-line-length": [false]
},
"rules": {
"interface-name": [false],
"ordered-imports": [false],
"object-literal-sort-keys": false,
"no-string-literal": false,
"no-console": [false],
"max-line-length": [false],
"no-unused-variable": true,
"max-classes-per-file": [false]
}
}
+1
View File
@@ -0,0 +1 @@
gcs.json
+8
View File
@@ -0,0 +1,8 @@
FROM registry:2.7.1
ADD ./entrypoint.sh /heroku-entrypoint.sh
ADD ./config.yml /etc/docker/registry/config.yml
ADD ./gcs.json /etc/gcs.json
ENTRYPOINT ["/heroku-entrypoint.sh"]
CMD ["/etc/docker/registry/config.yml"]
+29
View File
@@ -0,0 +1,29 @@
version: 0.1
log:
level: info
storage:
delete:
enabled: true
gcs:
bucket: replreg
keyfile: /etc/gcs.json
rootdirectory: /
http:
addr: 0.0.0.0:__PORT__
secret: __REPLREG_SECRET__
host: __REPLREG_HOST__
notifications:
endpoints:
- name: rgstry-hooks
url: __HOOK_URI__
headers:
Authorization: ["Token __HOOK_TOKEN__"]
timeout: 200ms
threshold: 3
backoff: 5s
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -e
sed -i "s/__PORT__/$PORT/g" /etc/docker/registry/config.yml
sed -i "s/__HOOK_TOKEN__/$HOOK_TOKEN/g" /etc/docker/registry/config.yml
sed -i "s/__HOOK_URI__/$HOOK_URI/g" /etc/docker/registry/config.yml
sed -i "s/__REPLREG_HOST__/$REPLREG_HOST/g" /etc/docker/registry/config.yml
sed -i "s/__REPLREG_SECRET__/$REPLREG_SECRET/g" /etc/docker/registry/config.yml
case "$1" in
*.yaml|*.yml) set -- registry serve "$@" ;;
serve|garbage-collect|help|-*) set -- registry "$@" ;;
esac
exec "$@"
+3
View File
@@ -0,0 +1,3 @@
build:
docker:
web: Dockerfile