diff --git a/Makefile b/Makefile
index 5a7c1eac4..e4557e4dd 100644
--- a/Makefile
+++ b/Makefile
@@ -49,7 +49,7 @@ endif
 	touch $@
 
 cmd/karma/bindata_assetfs.go: .build/deps-build-go.ok .build/artifacts-bindata_assetfs.$(GO_BINDATA_MODE) .build/artifacts-ui.ok
-	go-bindata-assetfs -o cmd/karma/bindata_assetfs.go ui/build/... ui/src/...
+	go-bindata-assetfs -o cmd/karma/bindata_assetfs.go ui/build/... ui/src/... cmd/karma/tests/bindata/...
 
 $(NAME): .build/deps-build-go.ok go.mod cmd/karma/bindata_assetfs.go $(SOURCES)
 	GO111MODULE=on go build -ldflags "-X main.version=$(VERSION)" ./cmd/karma
@@ -202,7 +202,7 @@ mock-assets: .build/deps-build-go.ok
 	rm -fr ui/build
 	mkdir ui/build
 	cp ui/public/* ui/build/
-	go-bindata-assetfs -o cmd/karma/bindata_assetfs.go -nometadata ui/build/...
+	go-bindata-assetfs -o cmd/karma/bindata_assetfs.go -nometadata ui/build/... cmd/karma/tests/bindata/...
 	# force assets rebuild on next make run
 	rm -f .build/bindata_assetfs.*
 
diff --git a/cmd/karma/assets_test.go b/cmd/karma/assets_test.go
index 16ae1c011..6d3ef5d10 100644
--- a/cmd/karma/assets_test.go
+++ b/cmd/karma/assets_test.go
@@ -132,7 +132,7 @@ func TestLoadTemplateUnparsable(t *testing.T) {
 	var wasFatal bool
 	log.StandardLogger().ExitFunc = func(int) { wasFatal = true }
 
-	loadTemplate(nil, "ui/build/go-test-invalid.html")
+	loadTemplate(nil, "cmd/karma/tests/bindata/go-test-invalid.html")
 
 	if !wasFatal {
 		t.Error("loadTemplate() with unparsable file didn't cause log.Fatal()")
diff --git a/cmd/karma/script_test.go b/cmd/karma/script_test.go
index 4be50a16d..573339cd2 100644
--- a/cmd/karma/script_test.go
+++ b/cmd/karma/script_test.go
@@ -53,6 +53,6 @@ func TestMain(m *testing.M) {
 
 func TestScripts(t *testing.T) {
 	testscript.Run(t, testscript.Params{
-		Dir: "testdata",
+		Dir: "tests/testscript",
 	})
 }
diff --git a/ui/public/go-test-invalid.html b/cmd/karma/tests/bindata/go-test-invalid.html
similarity index 100%
rename from ui/public/go-test-invalid.html
rename to cmd/karma/tests/bindata/go-test-invalid.html
diff --git a/cmd/karma/testdata/check-config.txt b/cmd/karma/tests/testscript/check-config.txt
similarity index 100%
rename from cmd/karma/testdata/check-config.txt
rename to cmd/karma/tests/testscript/check-config.txt
diff --git a/cmd/karma/testdata/debug.txt b/cmd/karma/tests/testscript/debug.txt
similarity index 100%
rename from cmd/karma/testdata/debug.txt
rename to cmd/karma/tests/testscript/debug.txt
diff --git a/cmd/karma/testdata/default_config_file.txt b/cmd/karma/tests/testscript/default_config_file.txt
similarity index 100%
rename from cmd/karma/testdata/default_config_file.txt
rename to cmd/karma/tests/testscript/default_config_file.txt
diff --git a/cmd/karma/testdata/duplicated_alertmanager_name.txt b/cmd/karma/tests/testscript/duplicated_alertmanager_name.txt
similarity index 100%
rename from cmd/karma/testdata/duplicated_alertmanager_name.txt
rename to cmd/karma/tests/testscript/duplicated_alertmanager_name.txt
diff --git a/cmd/karma/testdata/duplicated_alertmanager_uri.txt b/cmd/karma/tests/testscript/duplicated_alertmanager_uri.txt
similarity index 100%
rename from cmd/karma/testdata/duplicated_alertmanager_uri.txt
rename to cmd/karma/tests/testscript/duplicated_alertmanager_uri.txt
diff --git a/cmd/karma/testdata/invalid_config_alertmanager_timeout.txt b/cmd/karma/tests/testscript/invalid_config_alertmanager_timeout.txt
similarity index 100%
rename from cmd/karma/testdata/invalid_config_alertmanager_timeout.txt
rename to cmd/karma/tests/testscript/invalid_config_alertmanager_timeout.txt
diff --git a/cmd/karma/testdata/invalid_flag_alertmanager_timeout.txt b/cmd/karma/tests/testscript/invalid_flag_alertmanager_timeout.txt
similarity index 100%
rename from cmd/karma/testdata/invalid_flag_alertmanager_timeout.txt
rename to cmd/karma/tests/testscript/invalid_flag_alertmanager_timeout.txt
diff --git a/cmd/karma/testdata/invalid_log_format.txt b/cmd/karma/tests/testscript/invalid_log_format.txt
similarity index 100%
rename from cmd/karma/testdata/invalid_log_format.txt
rename to cmd/karma/tests/testscript/invalid_log_format.txt
diff --git a/cmd/karma/testdata/invalid_log_level.txt b/cmd/karma/tests/testscript/invalid_log_level.txt
similarity index 100%
rename from cmd/karma/testdata/invalid_log_level.txt
rename to cmd/karma/tests/testscript/invalid_log_level.txt
diff --git a/cmd/karma/testdata/invalid_proxy_config.txt b/cmd/karma/tests/testscript/invalid_proxy_config.txt
similarity index 100%
rename from cmd/karma/testdata/invalid_proxy_config.txt
rename to cmd/karma/tests/testscript/invalid_proxy_config.txt
diff --git a/cmd/karma/testdata/invalid_tls_config.txt b/cmd/karma/tests/testscript/invalid_tls_config.txt
similarity index 100%
rename from cmd/karma/testdata/invalid_tls_config.txt
rename to cmd/karma/tests/testscript/invalid_tls_config.txt
diff --git a/cmd/karma/testdata/invalid_ttl.txt b/cmd/karma/tests/testscript/invalid_ttl.txt
similarity index 100%
rename from cmd/karma/testdata/invalid_ttl.txt
rename to cmd/karma/tests/testscript/invalid_ttl.txt
diff --git a/cmd/karma/testdata/invalid_uri.txt b/cmd/karma/tests/testscript/invalid_uri.txt
similarity index 100%
rename from cmd/karma/testdata/invalid_uri.txt
rename to cmd/karma/tests/testscript/invalid_uri.txt
diff --git a/cmd/karma/testdata/labels_color_custom_invalid_regex.txt b/cmd/karma/tests/testscript/labels_color_custom_invalid_regex.txt
similarity index 100%
rename from cmd/karma/testdata/labels_color_custom_invalid_regex.txt
rename to cmd/karma/tests/testscript/labels_color_custom_invalid_regex.txt
diff --git a/cmd/karma/testdata/labels_color_custom_missing_value.txt b/cmd/karma/tests/testscript/labels_color_custom_missing_value.txt
similarity index 100%
rename from cmd/karma/testdata/labels_color_custom_missing_value.txt
rename to cmd/karma/tests/testscript/labels_color_custom_missing_value.txt
diff --git a/cmd/karma/testdata/linkDetect_invalid_regex.txt b/cmd/karma/tests/testscript/linkDetect_invalid_regex.txt
similarity index 100%
rename from cmd/karma/testdata/linkDetect_invalid_regex.txt
rename to cmd/karma/tests/testscript/linkDetect_invalid_regex.txt
diff --git a/cmd/karma/testdata/linkDetect_no_regex.txt b/cmd/karma/tests/testscript/linkDetect_no_regex.txt
similarity index 100%
rename from cmd/karma/testdata/linkDetect_no_regex.txt
rename to cmd/karma/tests/testscript/linkDetect_no_regex.txt
diff --git a/cmd/karma/testdata/linkDetect_no_uri.txt b/cmd/karma/tests/testscript/linkDetect_no_uri.txt
similarity index 100%
rename from cmd/karma/testdata/linkDetect_no_uri.txt
rename to cmd/karma/tests/testscript/linkDetect_no_uri.txt
diff --git a/cmd/karma/testdata/linkDetect_valid_config.txt b/cmd/karma/tests/testscript/linkDetect_valid_config.txt
similarity index 100%
rename from cmd/karma/testdata/linkDetect_valid_config.txt
rename to cmd/karma/tests/testscript/linkDetect_valid_config.txt
diff --git a/cmd/karma/testdata/log_format_json.txt b/cmd/karma/tests/testscript/log_format_json.txt
similarity index 100%
rename from cmd/karma/testdata/log_format_json.txt
rename to cmd/karma/tests/testscript/log_format_json.txt
diff --git a/cmd/karma/testdata/log_full_config_env.txt b/cmd/karma/tests/testscript/log_full_config_env.txt
similarity index 100%
rename from cmd/karma/testdata/log_full_config_env.txt
rename to cmd/karma/tests/testscript/log_full_config_env.txt
diff --git a/cmd/karma/testdata/log_full_config_file.txt b/cmd/karma/tests/testscript/log_full_config_file.txt
similarity index 100%
rename from cmd/karma/testdata/log_full_config_file.txt
rename to cmd/karma/tests/testscript/log_full_config_file.txt
diff --git a/cmd/karma/testdata/log_full_config_file_invalid_values.txt b/cmd/karma/tests/testscript/log_full_config_file_invalid_values.txt
similarity index 100%
rename from cmd/karma/testdata/log_full_config_file_invalid_values.txt
rename to cmd/karma/tests/testscript/log_full_config_file_invalid_values.txt
diff --git a/cmd/karma/testdata/log_timestamp.txt b/cmd/karma/tests/testscript/log_timestamp.txt
similarity index 100%
rename from cmd/karma/testdata/log_timestamp.txt
rename to cmd/karma/tests/testscript/log_timestamp.txt
diff --git a/cmd/karma/testdata/missing_config_file.txt b/cmd/karma/tests/testscript/missing_config_file.txt
similarity index 100%
rename from cmd/karma/testdata/missing_config_file.txt
rename to cmd/karma/tests/testscript/missing_config_file.txt
diff --git a/cmd/karma/testdata/no_args.txt b/cmd/karma/tests/testscript/no_args.txt
similarity index 100%
rename from cmd/karma/testdata/no_args.txt
rename to cmd/karma/tests/testscript/no_args.txt
diff --git a/cmd/karma/testdata/proxy-with-readonly.txt b/cmd/karma/tests/testscript/proxy-with-readonly.txt
similarity index 100%
rename from cmd/karma/testdata/proxy-with-readonly.txt
rename to cmd/karma/tests/testscript/proxy-with-readonly.txt
diff --git a/cmd/karma/testdata/proxy.txt b/cmd/karma/tests/testscript/proxy.txt
similarity index 100%
rename from cmd/karma/testdata/proxy.txt
rename to cmd/karma/tests/testscript/proxy.txt
diff --git a/cmd/karma/testdata/readonly.txt b/cmd/karma/tests/testscript/readonly.txt
similarity index 100%
rename from cmd/karma/testdata/readonly.txt
rename to cmd/karma/tests/testscript/readonly.txt
diff --git a/cmd/karma/testdata/sentry.txt b/cmd/karma/tests/testscript/sentry.txt
similarity index 100%
rename from cmd/karma/testdata/sentry.txt
rename to cmd/karma/tests/testscript/sentry.txt
diff --git a/cmd/karma/testdata/silenceform_populatefromheader_invalid_regex.txt b/cmd/karma/tests/testscript/silenceform_populatefromheader_invalid_regex.txt
similarity index 100%
rename from cmd/karma/testdata/silenceform_populatefromheader_invalid_regex.txt
rename to cmd/karma/tests/testscript/silenceform_populatefromheader_invalid_regex.txt
diff --git a/cmd/karma/testdata/silenceform_populatefromheader_missing_header.txt b/cmd/karma/tests/testscript/silenceform_populatefromheader_missing_header.txt
similarity index 100%
rename from cmd/karma/testdata/silenceform_populatefromheader_missing_header.txt
rename to cmd/karma/tests/testscript/silenceform_populatefromheader_missing_header.txt
diff --git a/cmd/karma/testdata/silenceform_populatefromheader_missing_regex.txt b/cmd/karma/tests/testscript/silenceform_populatefromheader_missing_regex.txt
similarity index 100%
rename from cmd/karma/testdata/silenceform_populatefromheader_missing_regex.txt
rename to cmd/karma/tests/testscript/silenceform_populatefromheader_missing_regex.txt
diff --git a/cmd/karma/testdata/simple_config.txt b/cmd/karma/tests/testscript/simple_config.txt
similarity index 100%
rename from cmd/karma/testdata/simple_config.txt
rename to cmd/karma/tests/testscript/simple_config.txt
diff --git a/cmd/karma/testdata/upper_case_keys.txt b/cmd/karma/tests/testscript/upper_case_keys.txt
similarity index 100%
rename from cmd/karma/testdata/upper_case_keys.txt
rename to cmd/karma/tests/testscript/upper_case_keys.txt
diff --git a/cmd/karma/testdata/version.txt b/cmd/karma/tests/testscript/version.txt
similarity index 100%
rename from cmd/karma/testdata/version.txt
rename to cmd/karma/tests/testscript/version.txt