Merge pull request #1908 from weaveworks/mike/static-content-from-s3

Add option to scope to have static content served from S3 instead
2026-07-19 13:29:35 +00:00 · 2016-10-07 16:24:40 -07:00
parent a2f6090176 ce5fc17972
commit 33ec1acf67
10 changed files with 185 additions and 19 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -50,6 +50,8 @@ extras/fixprobe/*.json
 *sublime-workspace
 *npm-debug.log
 app/static.go
-prog/static.go
 vendor/github.com/ugorji/go/codec/codecgen/bin/*
 *.codecgen.go
+client/build-external/*
+prog/staticui/*
+prog/externalui/*
--- a/42
+++ b/42
@@ -1,4 +1,4 @@
-.PHONY: all deps static clean client-lint client-test client-sync backend frontend shell lint
+.PHONY: all deps static clean client-lint client-test client-sync backend frontend shell lint ui-upload

 # If you can use Docker without being root, you can `make SUDO= <target>`
 SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E")
@@ -50,17 +50,18 @@ $(SCOPE_EXPORT): $(SCOPE_EXE) $(DOCKER_DISTRIB) docker/weave $(RUNSVINIT) docker

 $(RUNSVINIT): vendor/runsvinit/*.go

-$(SCOPE_EXE): $(shell find ./ -path ./vendor -prune -o -type f -name *.go) prog/static.go $(CODECGEN_TARGETS)
+$(SCOPE_EXE): $(shell find ./ -path ./vendor -prune -o -type f -name *.go) prog/staticui/staticui.go prog/externalui/externalui.go $(CODECGEN_TARGETS)

 report/report.codecgen.go: $(call GET_CODECGEN_DEPS,report/)
 render/render.codecgen.go: $(call GET_CODECGEN_DEPS,render/)
 render/detailed/detailed.codecgen.go: $(call GET_CODECGEN_DEPS,render/detailed/)
-static: prog/static.go
-prog/static.go: client/build/app.js
+static: prog/staticui/staticui.go prog/externalui/externalui.go
+prog/staticui/staticui.go: client/build/index.html
+prog/externalui/externalui.go: client/build-external/index.html

 ifeq ($(BUILD_IN_CONTAINER),true)

-$(SCOPE_EXE) $(RUNSVINIT) lint tests shell prog/static.go: $(SCOPE_BACKEND_BUILD_UPTODATE)
+$(SCOPE_EXE) $(RUNSVINIT) lint tests shell prog/staticui/staticui.go prog/externalui/externalui.go: $(SCOPE_BACKEND_BUILD_UPTODATE)
 	@mkdir -p $(shell pwd)/.pkg
 	$(SUDO) docker run $(RM) $(RUN_FLAGS) \
 		-v $(shell pwd):/go/src/github.com/weaveworks/scope \
@@ -104,19 +105,30 @@ lint: $(SCOPE_BACKEND_BUILD_UPTODATE)
 	./tools/lint -ignorespelling "agre " -ignorespelling "AGRE " .
 	./tools/shell-lint tools

-prog/static.go: $(SCOPE_BACKEND_BUILD_UPTODATE)
-	esc -o $@ -prefix client/build client/build
+prog/staticui/staticui.go: $(SCOPE_BACKEND_BUILD_UPTODATE)
+	mkdir -p prog/staticui
+	esc -o $@ -pkg staticui -prefix client/build client/build
+
+prog/externalui/externalui.go: $(SCOPE_BACKEND_BUILD_UPTODATE)
+	mkdir -p prog/externalui
+	esc -o $@ -pkg externalui -prefix client/build-external -include '\.html$$' client/build-external

 endif

 ifeq ($(BUILD_IN_CONTAINER),true)

-client/build/app.js: $(shell find client/app -type f) $(SCOPE_UI_BUILD_UPTODATE)
+client/build/index.html: $(shell find client/app -type f) $(SCOPE_UI_BUILD_UPTODATE)
 	mkdir -p client/build
 	$(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/build:/home/weave/build \
 		$(SCOPE_UI_BUILD_IMAGE) npm run build

+client/build-external/index.html: $(shell find client/app -type f) $(SCOPE_UI_BUILD_UPTODATE)
+	mkdir -p client/build
+	$(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \
+		-v $(shell pwd)/client/build-external:/home/weave/build-external \
+		$(SCOPE_UI_BUILD_IMAGE) npm run build-external
+
 client-test: $(shell find client/app/scripts -type f) $(SCOPE_UI_BUILD_UPTODATE)
 	$(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/test:/home/weave/test \
@@ -134,12 +146,15 @@ client-start: $(SCOPE_UI_BUILD_UPTODATE)

 else

-client/build/app.js:
+client/build/index.html:
 	cd client && npm run build

+client/build-external/index.html:
+	cd client && npm run build-external
+
 endif

-$(SCOPE_UI_BUILD_UPTODATE): client/Dockerfile client/package.json client/webpack.local.config.js client/webpack.production.config.js client/server.js client/.eslintrc
+$(SCOPE_UI_BUILD_UPTODATE): client/Dockerfile client/package.json client/webpack.local.config.js client/webpack.production.config.js client/webpack.external.config.js client/server.js client/.eslintrc
 	$(SUDO) docker build -t $(SCOPE_UI_BUILD_IMAGE) client
 	touch $@

@@ -147,13 +162,18 @@ $(SCOPE_BACKEND_BUILD_UPTODATE): backend/*
 	$(SUDO) docker build -t $(SCOPE_BACKEND_BUILD_IMAGE) backend
 	touch $@

+ui-upload: client/build-external/index.html
+	AWS_ACCESS_KEY_ID=$$UI_BUCKET_KEY_ID \
+	AWS_SECRET_ACCESS_KEY=$$UI_BUCKET_KEY_SECRET \
+	aws s3 cp client/build-external/ s3://static.weave.works/scope-ui/ --recursive --exclude '*.html'
+
 clean:
 	$(GO) clean ./...
 	# Don't actually rmi the build images - rm'ing the .uptodate files is enough to ensure
 	# we rebuild the images, and rmi'ing the images causes us to have to redownload a lot of stuff.
 	# $(SUDO) docker rmi $(SCOPE_UI_BUILD_IMAGE) $(SCOPE_BACKEND_BUILD_IMAGE) >/dev/null 2>&1 || true
 	rm -rf $(SCOPE_EXPORT) $(SCOPE_UI_BUILD_UPTODATE) $(SCOPE_BACKEND_BUILD_UPTODATE) \
-		$(SCOPE_EXE) $(RUNSVINIT) prog/static.go client/build/*.js docker/weave .pkg \
+		$(SCOPE_EXE) $(RUNSVINIT) prog/staticui/staticui.go prog/externalui/externalui.go client/build/*.js client/build-external/*.js docker/weave .pkg \
 		$(CODECGEN_TARGETS) $(CODECGEN_DIR)/bin

 clean-codecgen:
--- a/circle.yml
+++ b/circle.yml
@@ -30,6 +30,7 @@ dependencies:
    - "cd $SRCDIR/backend; ../tools/rebuild-image weaveworks/scope-backend-build . Dockerfile build.sh && touch $SRCDIR/.scope_backend_build.uptodate"
    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh make_template):
        parallel: false
+    - sudo apt-get update && sudo apt-get install python-pip && sudo pip install awscli

 test:
  override:
@@ -79,6 +80,9 @@ deployment:
          docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:$(./tools/image-tag) &&
          docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope-traffic-control-plugin &&
          docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope-traffic-control-plugin:$(./tools/image-tag) &&
+          (test -n "${UI_BUCKET_KEY_ID}" || (
+            make ui-upload
+          )) &&
          (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" != "weaveworks" || (
            wcloud deploy -u circle weaveworks/scope:$(./tools/image-tag)
          ))
--- a/client/Dockerfile
+++ b/client/Dockerfile
@@ -3,4 +3,4 @@ WORKDIR /home/weave
 COPY package.json /home/weave/
 ENV NPM_CONFIG_LOGLEVEL=warn NPM_CONFIG_PROGRESS=false
 RUN npm install
-COPY webpack.local.config.js webpack.production.config.js server.js .babelrc .eslintrc .eslintignore /home/weave/
+COPY webpack.local.config.js webpack.production.config.js webpack.external.config.js server.js .babelrc .eslintrc .eslintignore /home/weave/
--- a/client/package.json
+++ b/client/package.json
@@ -77,6 +77,7 @@
  },
  "scripts": {
    "build": "webpack --config webpack.production.config.js",
+    "build-external": "webpack --config webpack.external.config.js",
    "start": "node server.js",
    "start-production": "NODE_ENV=production node server.js",
    "test": "jest",
--- a/client/webpack.external.config.js
+++ b/client/webpack.external.config.js
@@ -0,0 +1,121 @@
+var webpack = require('webpack');
+var autoprefixer = require('autoprefixer');
+var path = require('path');
+
+var CleanWebpackPlugin = require('clean-webpack-plugin');
+var ExtractTextPlugin = require("extract-text-webpack-plugin");
+var HtmlWebpackPlugin = require('html-webpack-plugin');
+
+var GLOBALS = {
+  'process.env': {NODE_ENV: '"production"'}
+};
+
+/**
+ * This is the Webpack configuration file for hosting most of the static content
+ * (all but index.htm) on an external host (eg. a CDN).
+ * You should change output.publicPath to point to your external host.
+ */
+module.exports = {
+
+  // fail on first error when building release
+  bail: true,
+
+  cache: {},
+
+  entry: {
+    app: './app/scripts/main',
+    'contrast-app': './app/scripts/contrast-main',
+    'terminal-app': './app/scripts/terminal-main',
+    // keep only some in here, to make vendors and app bundles roughly same size
+    vendors: ['babel-polyfill', 'classnames', 'd3', 'immutable',
+      'lodash', 'react', 'react-dom', 'react-redux',
+      'redux', 'redux-thunk']
+  },
+
+  output: {
+    path: path.join(__dirname, 'build-external/'),
+    filename: '[name]-[chunkhash].js',
+	// Change this line to point to resources on an external host.
+	publicPath: 'https://s3.amazonaws.com/static.weave.works/scope-ui/'
+  },
+
+  module: {
+    include: [
+      path.resolve(__dirname, 'app/scripts')
+    ],
+    preLoaders: [
+      {
+        test: /\.js$/,
+        exclude: /node_modules|vendor/,
+        loader: 'eslint-loader'
+      }
+    ],
+    loaders: [
+      {
+        test: /\.less$/,
+        loader: ExtractTextPlugin.extract('style-loader',
+          'css-loader?minimize!postcss-loader!less-loader')
+      },
+      {
+        test: /\.woff(2)?(\?v=[0-9]\.[0-9]\.[0-9])?$/,
+        loader: 'url-loader?limit=10000&minetype=application/font-woff'
+      },
+      {
+        test: /\.(ttf|eot|svg|ico)(\?v=[0-9]\.[0-9]\.[0-9])?$/,
+        loader: 'file-loader'
+      },
+      {
+        test: /\.ico$/,
+        loader: 'file-loader?name=[name].[ext]'
+      },
+      { test: /\.jsx?$/, exclude: /node_modules|vendor/, loader: 'babel' }
+    ]
+  },
+
+  postcss: [
+    autoprefixer({
+      browsers: ['last 2 versions']
+    })
+  ],
+
+  eslint: {
+    failOnError: true
+  },
+
+  resolve: {
+    extensions: ['', '.js', '.jsx']
+  },
+
+  plugins: [
+    new CleanWebpackPlugin(['build']),
+    new webpack.DefinePlugin(GLOBALS),
+    new webpack.optimize.CommonsChunkPlugin('vendors', 'vendors-[chunkhash].js'),
+    new webpack.optimize.OccurenceOrderPlugin(true),
+    new webpack.IgnorePlugin(/^\.\/locale$/, [/moment$/]),
+    new webpack.optimize.UglifyJsPlugin({
+      sourceMap: false,
+      compress: {
+        warnings: false
+      }
+    }),
+    new ExtractTextPlugin('style-[name]-[chunkhash].css'),
+    new HtmlWebpackPlugin({
+      hash: true,
+      chunks: ['vendors', 'contrast-app'],
+      template: 'app/html/index.html',
+      filename: 'contrast.html'
+    }),
+    new HtmlWebpackPlugin({
+      hash: true,
+      chunks: ['vendors', 'terminal-app'],
+      template: 'app/html/index.html',
+      filename: 'terminal.html'
+    }),
+    new HtmlWebpackPlugin({
+      hash: true,
+      chunks: ['vendors', 'app'],
+      template: 'app/html/index.html',
+      filename: 'index.html'
+    })
+  ]
+};
--- a/client/webpack.production.config.js
+++ b/client/webpack.production.config.js
@@ -32,7 +32,7 @@ module.exports = {

  output: {
    path: path.join(__dirname, 'build/'),
-    filename: '[name].js'
+    filename: '[name]-[chunkhash].js',
  },

  module: {
@@ -85,7 +85,7 @@ module.exports = {
  plugins: [
    new CleanWebpackPlugin(['build']),
    new webpack.DefinePlugin(GLOBALS),
-    new webpack.optimize.CommonsChunkPlugin('vendors', 'vendors.js'),
+    new webpack.optimize.CommonsChunkPlugin('vendors', 'vendors-[chunkhash].js'),
    new webpack.optimize.OccurenceOrderPlugin(true),
    new webpack.IgnorePlugin(/^\.\/locale$/, [/moment$/]),
    new webpack.optimize.UglifyJsPlugin({
@@ -94,7 +94,7 @@ module.exports = {
        warnings: false
      }
    }),
-    new ExtractTextPlugin('style-[name].css'),
+    new ExtractTextPlugin('style-[name]-[chunkhash].css'),
    new HtmlWebpackPlugin({
      hash: true,
      chunks: ['vendors', 'contrast-app'],
--- a/prog/app.go
+++ b/prog/app.go
@@ -47,7 +47,7 @@ func init() {
 }

 // Router creates the mux for all the various app components.
-func router(collector app.Collector, controlRouter app.ControlRouter, pipeRouter app.PipeRouter) http.Handler {
+func router(collector app.Collector, controlRouter app.ControlRouter, pipeRouter app.PipeRouter, externalUI bool) http.Handler {
 	router := mux.NewRouter().SkipClean(true)

 	// We pull in the http.DefaultServeMux to get the pprof routes
@@ -59,7 +59,7 @@ func router(collector app.Collector, controlRouter app.ControlRouter, pipeRouter
 	app.RegisterPipeRoutes(router, pipeRouter)
 	app.RegisterTopologyRoutes(router, collector)

-	uiHandler := http.FileServer(FS(false))
+	uiHandler := http.FileServer(GetFS(externalUI))
 	router.PathPrefix("/ui").Name("static").Handler(
 		middleware.PathRewrite(regexp.MustCompile("^/ui"), "").Wrap(
 			uiHandler))
@@ -267,7 +267,7 @@ func appMain(flags appFlags) {
 		}
 	}

-	handler := router(collector, controlRouter, pipeRouter)
+	handler := router(collector, controlRouter, pipeRouter, flags.externalUI)
 	if flags.logHTTP {
 		handler = middleware.Logging.Wrap(handler)
 	}
--- a/prog/main.go
+++ b/prog/main.go
@@ -130,6 +130,7 @@ type appFlags struct {
 	memcachedExpiration       time.Duration
 	memcachedCompressionLevel int
 	userIDHeader              string
+	externalUI                bool

 	blockProfileRate int

@@ -255,6 +256,7 @@ func main() {
 	flag.StringVar(&flags.app.memcachedService, "app.memcached.service", "memcached", "SRV service used to discover memcache servers.")
 	flag.IntVar(&flags.app.memcachedCompressionLevel, "app.memcached.compression", gzip.DefaultCompression, "How much to compress reports stored in memcached.")
 	flag.StringVar(&flags.app.userIDHeader, "app.userid.header", "", "HTTP header to use as userid")
+	flag.BoolVar(&flags.app.externalUI, "app.externalUI", false, "Point to externally hosted static UI assets")

 	flag.IntVar(&flags.app.blockProfileRate, "app.block.profile.rate", 0, "If more than 0, enable block profiling. The profiler aims to sample an average of one blocking event per rate nanoseconds spent blocked.")

--- a/prog/static.go
+++ b/prog/static.go
@@ -0,0 +1,16 @@
+package main
+
+import (
+	"net/http"
+
+	"github.com/weaveworks/scope/prog/staticui"
+	"github.com/weaveworks/scope/prog/externalui"
+)
+
+func GetFS(use_external bool) http.FileSystem {
+	if use_external {
+		return externalui.FS(false)
+	} else {
+		return staticui.FS(false)
+	}
+}