mirror of
https://github.com/kubeshark/kubeshark.git
synced 2026-04-27 21:06:41 +00:00
Compare commits
24 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
d9ed9098eb | ||
|
e09748baff | ||
|
20fcc8e163 | ||
|
fd64c1bb14 | ||
|
|
3a51ca21eb | ||
|
|
0a2e55f7bc | ||
|
|
f7c200b821 | ||
|
0846a98bc1 | ||
|
|
602225bb36 | ||
|
|
c0f6f2a049 | ||
|
|
7846d812c1 | ||
|
|
0f6c56986f | ||
|
daf6b3db06 | ||
|
fdf552a9ec | ||
|
|
cbff1837c1 | ||
|
71425928cc | ||
|
|
82db4acb7d | ||
|
|
9bd82191aa | ||
|
|
83b657472b | ||
|
2e5cf13b3f | ||
|
4be7164f20 | ||
|
|
0abd7c06ff | ||
|
|
c2739a68c2 | ||
|
|
d0ef6c9f97 |
@@ -1,4 +1,4 @@
|
||||
name: PR validation
|
||||
name: Build
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
@@ -12,7 +12,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
build-cli:
|
||||
name: Build CLI
|
||||
name: CLI executable build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Go 1.16
|
||||
@@ -27,35 +27,11 @@ jobs:
|
||||
run: make cli
|
||||
|
||||
build-agent:
|
||||
name: Build Agent
|
||||
name: Agent docker image build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Go 1.16
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: '1.16'
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- shell: bash
|
||||
run: |
|
||||
sudo apt-get install libpcap-dev
|
||||
|
||||
- name: Build Agent
|
||||
run: make agent
|
||||
|
||||
build-ui:
|
||||
name: Build UI
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Node 16
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '16'
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Build UI
|
||||
run: make ui
|
||||
run: make agent-docker
|
||||
90
.github/workflows/static_code_analysis.yml
vendored
Normal file
90
.github/workflows/static_code_analysis.yml
vendored
Normal file
@@ -0,0 +1,90 @@
|
||||
name: Static code analysis
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'develop'
|
||||
- 'main'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
golangci:
|
||||
name: Go lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: '^1.16'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y libpcap-dev
|
||||
|
||||
- name: Go lint - agent
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: agent
|
||||
args: --timeout=3m
|
||||
|
||||
- name: Go lint - shared
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: shared
|
||||
args: --timeout=3m
|
||||
|
||||
- name: Go lint - tap
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: tap
|
||||
args: --timeout=3m
|
||||
|
||||
- name: Go lint - CLI
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: cli
|
||||
args: --timeout=3m
|
||||
|
||||
- name: Go lint - acceptanceTests
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: acceptanceTests
|
||||
args: --timeout=3m
|
||||
|
||||
- name: Go lint - tap/api
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: tap/api
|
||||
|
||||
- name: Go lint - tap/extensions/amqp
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: tap/extensions/amqp
|
||||
|
||||
- name: Go lint - tap/extensions/http
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: tap/extensions/http
|
||||
|
||||
- name: Go lint - tap/extensions/kafka
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: tap/extensions/kafka
|
||||
|
||||
- name: Go lint - tap/extensions/redis
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
working-directory: tap/extensions/redis
|
||||
@@ -1,14 +1,10 @@
|
||||
name: tests validation
|
||||
name: Test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'develop'
|
||||
- 'main'
|
||||
push:
|
||||
branches:
|
||||
- 'develop'
|
||||
- 'main'
|
||||
|
||||
concurrency:
|
||||
group: mizu-tests-validation-${{ github.ref }}
|
||||
@@ -16,7 +12,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
run-tests-cli:
|
||||
name: Run CLI tests
|
||||
name: CLI Tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Go 1.16
|
||||
@@ -34,7 +30,7 @@ jobs:
|
||||
uses: codecov/codecov-action@v2
|
||||
|
||||
run-tests-agent:
|
||||
name: Run Agent tests
|
||||
name: Agent Tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Go 1.16
|
||||
@@ -72,10 +72,10 @@ ARG SEM_VER=0.0.0
|
||||
WORKDIR /app/agent-build
|
||||
|
||||
RUN go build -ldflags="-extldflags=-static -s -w \
|
||||
-X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
|
||||
-X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
|
||||
-X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
|
||||
-X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
|
||||
-X 'github.com/up9inc/mizu/agent/pkg/version.GitCommitHash=${COMMIT_HASH}' \
|
||||
-X 'github.com/up9inc/mizu/agent/pkg/version.Branch=${GIT_BRANCH}' \
|
||||
-X 'github.com/up9inc/mizu/agent/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
|
||||
-X 'github.com/up9inc/mizu/agent/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
|
||||
|
||||
# Download Basenine executable, verify the sha1sum
|
||||
ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
|
||||
|
||||
4
Makefile
4
Makefile
@@ -47,6 +47,10 @@ agent-debug: ## Build agent for debug.
|
||||
docker: ## Build and publish agent docker image.
|
||||
$(MAKE) push-docker
|
||||
|
||||
agent-docker: ## Build agent docker image.
|
||||
@echo "Building agent docker image"
|
||||
@docker build -t up9inc/mizu:devlatest .
|
||||
|
||||
push: push-docker push-cli ## Build and publish agent docker image & CLI.
|
||||
|
||||
push-docker: ## Build and publish agent docker image.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"viewportHeight": 1080,
|
||||
"video": false,
|
||||
"screenshotOnRunFailure": false,
|
||||
"defaultCommandTimeout": 5000,
|
||||
"defaultCommandTimeout": 6000,
|
||||
"testFiles": [
|
||||
"tests/GuiPort.js",
|
||||
"tests/MultipleNamespaces.js",
|
||||
|
||||
@@ -3,6 +3,7 @@ import {resizeToHugeMizu, resizeToNormalMizu} from "../testHelpers/TrafficHelper
|
||||
const greenFilterColor = 'rgb(210, 250, 210)';
|
||||
const redFilterColor = 'rgb(250, 214, 220)';
|
||||
const refreshWaitTimeout = 10000;
|
||||
const bodyJsonClass = '.hljs';
|
||||
|
||||
it('opening mizu', function () {
|
||||
cy.visit(Cypress.env('testUrl'));
|
||||
@@ -25,6 +26,37 @@ it('filtering guide check', function () {
|
||||
cy.get('#modal-modal-title').should('not.exist');
|
||||
});
|
||||
|
||||
it('right side sanity test', function () {
|
||||
cy.get('#entryDetailedTitleBodySize').then(sizeTopLine => {
|
||||
const sizeOnTopLine = sizeTopLine.text().replace(' B', '');
|
||||
cy.contains('Response').click();
|
||||
cy.contains('Body Size (bytes)').parent().next().then(size => {
|
||||
const bodySizeByDetails = size.text();
|
||||
expect(sizeOnTopLine).to.equal(bodySizeByDetails, 'The body size in the top line should match the details in the response');
|
||||
|
||||
if (parseInt(bodySizeByDetails) < 0) {
|
||||
throw new Error(`The body size cannot be negative. got the size: ${bodySizeByDetails}`)
|
||||
}
|
||||
|
||||
cy.get('#entryDetailedTitleElapsedTime').then(timeInMs => {
|
||||
const time = timeInMs.text();
|
||||
if (time < '0ms') {
|
||||
throw new Error(`The time in the top line cannot be negative ${time}`);
|
||||
}
|
||||
|
||||
cy.get('#rightSideContainer [title="Status Code"]').then(status => {
|
||||
const statusCode = status.text();
|
||||
cy.contains('Status').parent().next().then(statusInDetails => {
|
||||
const statusCodeInDetails = statusInDetails.text();
|
||||
|
||||
expect(statusCode).to.equal(statusCodeInDetails, 'The status code in the top line should match the status code in details');
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
checkIllegalFilter('invalid filter');
|
||||
|
||||
checkFilter({
|
||||
@@ -161,11 +193,12 @@ function checkFilter(filterDetails){
|
||||
if (!applyByEnter)
|
||||
cy.get('[type="submit"]').click();
|
||||
|
||||
// only one entry in DOM after filtering, checking all four checks on it
|
||||
// only one entry in DOM after filtering, checking all checks on it
|
||||
leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
|
||||
leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
|
||||
rightTextCheck(rightSidePath, rightSideExpectedText);
|
||||
rightOnHoverCheck(rightSidePath, name);
|
||||
checkRightSideResponseBody();
|
||||
|
||||
cy.get('[title="Fetch old records"]').click();
|
||||
resizeToHugeMizu();
|
||||
@@ -196,6 +229,7 @@ function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedTe
|
||||
cy.get(`#list #entry-${entryNum}`).click();
|
||||
rightTextCheck(rightSidePath, rightSideExpectedText);
|
||||
rightOnHoverCheck(rightSidePath, filterName);
|
||||
checkRightSideResponseBody();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -216,3 +250,88 @@ function rightOnHoverCheck(path, expectedText) {
|
||||
cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).trigger('mouseover');
|
||||
cy.get(`.TrafficPage-Container > :nth-child(2) .Queryable-Tooltip`).should('have.text', expectedText);
|
||||
}
|
||||
|
||||
|
||||
function checkRightSideResponseBody() {
|
||||
cy.contains('Response').click();
|
||||
clickCheckbox('Decode Base64');
|
||||
|
||||
cy.get(`${bodyJsonClass}`).then(value => {
|
||||
const encodedBody = value.text();
|
||||
cy.log(encodedBody);
|
||||
|
||||
const decodedBody = atob(encodedBody);
|
||||
const responseBody = JSON.parse(decodedBody);
|
||||
|
||||
const expectdJsonBody = {
|
||||
args: RegExp({}),
|
||||
url: RegExp('http://.*/get'),
|
||||
headers: {
|
||||
"User-Agent": RegExp('[REDACTED]'),
|
||||
"Accept-Encoding": RegExp('gzip'),
|
||||
"X-Forwarded-Uri": RegExp('/api/v1/namespaces/.*/services/.*/proxy/get')
|
||||
}
|
||||
};
|
||||
|
||||
expect(responseBody.args).to.match(expectdJsonBody.args);
|
||||
expect(responseBody.url).to.match(expectdJsonBody.url);
|
||||
expect(responseBody.headers['User-Agent']).to.match(expectdJsonBody.headers['User-Agent']);
|
||||
expect(responseBody.headers['Accept-Encoding']).to.match(expectdJsonBody.headers['Accept-Encoding']);
|
||||
expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectdJsonBody.headers['X-Forwarded-Uri']);
|
||||
|
||||
cy.get(`${bodyJsonClass}`).should('have.text', encodedBody);
|
||||
clickCheckbox('Decode Base64');
|
||||
|
||||
cy.get(`${bodyJsonClass} > `).its('length').should('be.gt', 1).then(linesNum => {
|
||||
cy.get(`${bodyJsonClass} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
|
||||
checkPrettyAndLineNums(jsonItemsNum, decodedBody);
|
||||
|
||||
clickCheckbox('Line numbers');
|
||||
checkPrettyOrNothing(jsonItemsNum, decodedBody);
|
||||
|
||||
clickCheckbox('Pretty');
|
||||
checkPrettyOrNothing(jsonItemsNum, decodedBody);
|
||||
|
||||
clickCheckbox('Line numbers');
|
||||
checkOnlyLineNumberes(jsonItemsNum, decodedBody);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function clickCheckbox(type) {
|
||||
cy.contains(`${type}`).prev().children().click();
|
||||
}
|
||||
|
||||
function checkPrettyAndLineNums(jsonItemsLen, decodedBody) {
|
||||
decodedBody = decodedBody.replaceAll(' ', '');
|
||||
cy.get(`${bodyJsonClass} >`).then(elements => {
|
||||
const lines = Object.values(elements);
|
||||
lines.forEach((line, index) => {
|
||||
if (line.getAttribute) {
|
||||
const cleanLine = getCleanLine(line);
|
||||
const currentLineFromDecodedText = decodedBody.substring(0, cleanLine.length);
|
||||
|
||||
expect(cleanLine).to.equal(currentLineFromDecodedText, `expected the text in line number ${index + 1} to match the text that generated by the base64 decoding`)
|
||||
|
||||
decodedBody = decodedBody.substring(cleanLine.length);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getCleanLine(lineElement) {
|
||||
return (lineElement.innerText.substring(0, lineElement.innerText.length - 1)).replaceAll(' ', '');
|
||||
}
|
||||
|
||||
function checkPrettyOrNothing(jsonItems, decodedBody) {
|
||||
cy.get(`${bodyJsonClass} > `).should('have.length', jsonItems).then(text => {
|
||||
const json = text.text();
|
||||
expect(json).to.equal(decodedBody);
|
||||
});
|
||||
}
|
||||
|
||||
function checkOnlyLineNumberes(jsonItems, decodedText) {
|
||||
cy.get(`${bodyJsonClass} >`).should('have.length', 1).and('have.text', decodedText);
|
||||
cy.get(`${bodyJsonClass} > >`).should('have.length', jsonItems)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package acceptanceTests
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
@@ -11,12 +10,10 @@ import (
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
)
|
||||
|
||||
@@ -28,7 +25,6 @@ const (
|
||||
defaultServiceName = "httpbin"
|
||||
defaultEntriesCount = 50
|
||||
waitAfterTapPodsReady = 3 * time.Second
|
||||
cleanCommandTimeout = 1 * time.Minute
|
||||
)
|
||||
|
||||
type PodDescriptor struct {
|
||||
@@ -36,18 +32,6 @@ type PodDescriptor struct {
|
||||
Namespace string
|
||||
}
|
||||
|
||||
func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool {
|
||||
for _, pod := range pods {
|
||||
podNamespace := pod["namespace"].(string)
|
||||
podName := pod["name"].(string)
|
||||
|
||||
if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func getCliPath() (string, error) {
|
||||
dir, filePathErr := os.Getwd()
|
||||
if filePathErr != nil {
|
||||
@@ -84,10 +68,6 @@ func getApiServerUrl(port uint16) string {
|
||||
return fmt.Sprintf("http://localhost:%v", port)
|
||||
}
|
||||
|
||||
func getWebSocketUrl(port uint16) string {
|
||||
return fmt.Sprintf("ws://localhost:%v/ws", port)
|
||||
}
|
||||
|
||||
func getDefaultCommandArgs() []string {
|
||||
setFlag := "--set"
|
||||
telemetry := "telemetry=false"
|
||||
@@ -130,20 +110,6 @@ func getDefaultConfigCommandArgs() []string {
|
||||
return append([]string{configCommand}, defaultCmdArgs...)
|
||||
}
|
||||
|
||||
func getDefaultCleanCommandArgs() []string {
|
||||
cleanCommand := "clean"
|
||||
defaultCmdArgs := getDefaultCommandArgs()
|
||||
|
||||
return append([]string{cleanCommand}, defaultCmdArgs...)
|
||||
}
|
||||
|
||||
func getDefaultViewCommandArgs() []string {
|
||||
viewCommand := "view"
|
||||
defaultCmdArgs := getDefaultCommandArgs()
|
||||
|
||||
return append([]string{viewCommand}, defaultCmdArgs...)
|
||||
}
|
||||
|
||||
func runCypressTests(t *testing.T, cypressRunCmd string) {
|
||||
cypressCmd := exec.Command("bash", "-c", cypressRunCmd)
|
||||
t.Logf("running command: %v", cypressCmd.String())
|
||||
@@ -268,36 +234,6 @@ func executeHttpPostRequestWithHeaders(url string, headers map[string]string, bo
|
||||
return executeHttpRequest(response, requestErr)
|
||||
}
|
||||
|
||||
func runMizuClean() error {
|
||||
cliPath, err := getCliPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cleanCmdArgs := getDefaultCleanCommandArgs()
|
||||
|
||||
cleanCmd := exec.Command(cliPath, cleanCmdArgs...)
|
||||
|
||||
commandDone := make(chan error)
|
||||
go func() {
|
||||
if err := cleanCmd.Run(); err != nil {
|
||||
commandDone <- err
|
||||
}
|
||||
commandDone <- nil
|
||||
}()
|
||||
|
||||
select {
|
||||
case err = <-commandDone:
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case <-time.After(cleanCommandTimeout):
|
||||
return errors.New("clean command timed out")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupCommand(cmd *exec.Cmd) error {
|
||||
if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
|
||||
return err
|
||||
@@ -310,17 +246,6 @@ func cleanupCommand(cmd *exec.Cmd) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
|
||||
tapPodsInterface := tapStatusInterface.([]interface{})
|
||||
|
||||
var pods []map[string]interface{}
|
||||
for _, podInterface := range tapPodsInterface {
|
||||
pods = append(pods, podInterface.(map[string]interface{}))
|
||||
}
|
||||
|
||||
return pods, nil
|
||||
}
|
||||
|
||||
func getLogsPath() (string, error) {
|
||||
dir, filePathErr := os.Getwd()
|
||||
if filePathErr != nil {
|
||||
@@ -331,77 +256,6 @@ func getLogsPath() (string, error) {
|
||||
return logsPath, nil
|
||||
}
|
||||
|
||||
// waitTimeout waits for the waitgroup for the specified max timeout.
|
||||
// Returns true if waiting timed out.
|
||||
func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
|
||||
channel := make(chan struct{})
|
||||
go func() {
|
||||
defer close(channel)
|
||||
wg.Wait()
|
||||
}()
|
||||
select {
|
||||
case <-channel:
|
||||
return false // completed normally
|
||||
case <-time.After(timeout):
|
||||
return true // timed out
|
||||
}
|
||||
}
|
||||
|
||||
// checkEntriesAtLeast checks whether the number of entries greater than or equal to n
|
||||
func checkEntriesAtLeast(entries []map[string]interface{}, n int) error {
|
||||
if len(entries) < n {
|
||||
return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getDBEntries retrieves the entries from the database before the given timestamp.
|
||||
// Also limits the results according to the limit parameter.
|
||||
// Timeout for the WebSocket connection is defined by the timeout parameter.
|
||||
func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) {
|
||||
query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit)
|
||||
webSocketUrl := getWebSocketUrl(defaultApiServerPort)
|
||||
|
||||
var connection *websocket.Conn
|
||||
connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer connection.Close()
|
||||
|
||||
handleWSConnection := func(wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
for {
|
||||
_, message, err := connection.ReadMessage()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var data map[string]interface{}
|
||||
if err = json.Unmarshal([]byte(message), &data); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if data["messageType"] == "entry" {
|
||||
entries = append(entries, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = connection.WriteMessage(websocket.TextMessage, []byte(query))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
go handleWSConnection(&wg)
|
||||
wg.Add(1)
|
||||
|
||||
waitTimeout(&wg, timeout)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func Contains(slice []string, containsValue string) bool {
|
||||
for _, sliceValue := range slice {
|
||||
if sliceValue == containsValue {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
module mizuserver
|
||||
module github.com/up9inc/mizu/agent
|
||||
|
||||
go 1.16
|
||||
|
||||
@@ -15,6 +15,7 @@ require (
|
||||
github.com/go-playground/validator/v10 v10.5.0
|
||||
github.com/google/uuid v1.1.2
|
||||
github.com/gorilla/websocket v1.4.2
|
||||
github.com/nav-inc/datetime v0.1.3
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
|
||||
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
|
||||
github.com/ory/keto-client-go v0.7.0-alpha.1
|
||||
|
||||
@@ -515,6 +515,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
|
||||
github.com/nav-inc/datetime v0.1.3 h1:PaybPUsScX+Cd3TEa1tYpfwU61deCEhMTlCO2hONm1c=
|
||||
github.com/nav-inc/datetime v0.1.3/go.mod h1:gKGf5G+cW7qkTo5TC/sieNyz6lYdrA9cf1PNV+pXIOE=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/ohler55/ojg v1.12.12 h1:hepbQFn7GHAecTPmwS3j5dCiOLsOpzPLvhiqnlAVAoE=
|
||||
@@ -612,6 +614,7 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
|
||||
@@ -6,17 +6,6 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"mizuserver/pkg/api"
|
||||
"mizuserver/pkg/config"
|
||||
"mizuserver/pkg/controllers"
|
||||
"mizuserver/pkg/elastic"
|
||||
"mizuserver/pkg/middlewares"
|
||||
"mizuserver/pkg/models"
|
||||
"mizuserver/pkg/oas"
|
||||
"mizuserver/pkg/routes"
|
||||
"mizuserver/pkg/servicemap"
|
||||
"mizuserver/pkg/up9"
|
||||
"mizuserver/pkg/utils"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -26,6 +15,21 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/middlewares"
|
||||
"github.com/up9inc/mizu/agent/pkg/models"
|
||||
"github.com/up9inc/mizu/agent/pkg/oas"
|
||||
"github.com/up9inc/mizu/agent/pkg/routes"
|
||||
"github.com/up9inc/mizu/agent/pkg/servicemap"
|
||||
"github.com/up9inc/mizu/agent/pkg/up9"
|
||||
"github.com/up9inc/mizu/agent/pkg/utils"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/elastic"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/api"
|
||||
"github.com/up9inc/mizu/agent/pkg/config"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
|
||||
"github.com/antelman107/net-wait-go/wait"
|
||||
@@ -62,7 +66,6 @@ const (
|
||||
socketConnectionRetries = 30
|
||||
socketConnectionRetryDelay = time.Second * 2
|
||||
socketHandshakeTimeout = time.Second * 2
|
||||
uiIndexPath = "./site/index.html"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -169,7 +172,7 @@ func configureBasenineServer(host string, port string) {
|
||||
wait.WithWait(200*time.Millisecond),
|
||||
wait.WithBreak(50*time.Millisecond),
|
||||
wait.WithDeadline(5*time.Second),
|
||||
wait.WithDebug(true),
|
||||
wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
|
||||
).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
|
||||
logger.Log.Panicf("Basenine is not available!")
|
||||
}
|
||||
@@ -248,16 +251,23 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
|
||||
|
||||
app.Use(DisableRootStaticCache())
|
||||
|
||||
if err := setUIFlags(); err != nil {
|
||||
logger.Log.Errorf("Error setting ui mode, err: %v", err)
|
||||
var staticFolder string
|
||||
if config.Config.StandaloneMode {
|
||||
staticFolder = "./site-standalone"
|
||||
} else {
|
||||
staticFolder = "./site"
|
||||
}
|
||||
|
||||
if config.Config.StandaloneMode {
|
||||
app.Use(static.ServeRoot("/", "./site-standalone"))
|
||||
} else {
|
||||
app.Use(static.ServeRoot("/", "./site"))
|
||||
indexStaticFile := staticFolder + "/index.html"
|
||||
if err := setUIFlags(indexStaticFile); err != nil {
|
||||
logger.Log.Errorf("Error setting ui flags, err: %v", err)
|
||||
}
|
||||
|
||||
app.Use(static.ServeRoot("/", staticFolder))
|
||||
app.NoRoute(func(c *gin.Context) {
|
||||
c.File(indexStaticFile)
|
||||
})
|
||||
|
||||
app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
|
||||
|
||||
api.WebSocketRoutes(app, &eventHandlers, startTime)
|
||||
@@ -278,7 +288,6 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
|
||||
routes.EntriesRoutes(app)
|
||||
routes.MetadataRoutes(app)
|
||||
routes.StatusRoutes(app)
|
||||
routes.NotFoundRoute(app)
|
||||
utils.StartServer(app)
|
||||
}
|
||||
|
||||
@@ -293,7 +302,7 @@ func DisableRootStaticCache() gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
func setUIFlags() error {
|
||||
func setUIFlags(uiIndexPath string) error {
|
||||
read, err := ioutil.ReadFile(uiIndexPath)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -5,22 +5,23 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mizuserver/pkg/elastic"
|
||||
"mizuserver/pkg/har"
|
||||
"mizuserver/pkg/holder"
|
||||
"mizuserver/pkg/providers"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"mizuserver/pkg/servicemap"
|
||||
"github.com/up9inc/mizu/agent/pkg/elastic"
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
"github.com/up9inc/mizu/agent/pkg/holder"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers"
|
||||
|
||||
"mizuserver/pkg/models"
|
||||
"mizuserver/pkg/oas"
|
||||
"mizuserver/pkg/resolver"
|
||||
"mizuserver/pkg/utils"
|
||||
"github.com/up9inc/mizu/agent/pkg/servicemap"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/models"
|
||||
"github.com/up9inc/mizu/agent/pkg/oas"
|
||||
"github.com/up9inc/mizu/agent/pkg/resolver"
|
||||
"github.com/up9inc/mizu/agent/pkg/utils"
|
||||
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
@@ -42,10 +43,8 @@ func StartResolving(namespace string) {
|
||||
res.Start(ctx)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case err := <-errOut:
|
||||
logger.Log.Infof("name resolving error %s", err)
|
||||
}
|
||||
err := <-errOut
|
||||
logger.Log.Infof("name resolving error %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -67,7 +66,7 @@ func startReadingFiles(workingDir string) {
|
||||
return
|
||||
}
|
||||
|
||||
for true {
|
||||
for {
|
||||
dir, _ := os.Open(workingDir)
|
||||
dirFiles, _ := dir.Readdir(-1)
|
||||
|
||||
@@ -124,7 +123,9 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
|
||||
if extension.Protocol.Name == "http" {
|
||||
if !disableOASValidation {
|
||||
var httpPair tapApi.HTTPRequestResponsePair
|
||||
json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
|
||||
if err := json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
|
||||
contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
|
||||
mizuEntry.ContractStatus = contract.Status
|
||||
@@ -139,7 +140,8 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
|
||||
mizuEntry.Rules = rules
|
||||
}
|
||||
|
||||
oas.GetOasGeneratorInstance().PushEntry(harEntry)
|
||||
entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name}
|
||||
oas.GetOasGeneratorInstance().PushEntry(&entryWSource)
|
||||
}
|
||||
|
||||
data, err := json.Marshal(mizuEntry)
|
||||
|
||||
@@ -2,18 +2,17 @@ package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mizuserver/pkg/models"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/models"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/websocket"
|
||||
basenine "github.com/up9inc/basenine/client/go"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/debounce"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
tapApi "github.com/up9inc/mizu/tap/api"
|
||||
)
|
||||
@@ -42,7 +41,7 @@ var connectedWebsocketIdCounter = 0
|
||||
|
||||
func init() {
|
||||
websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
|
||||
connectedWebsockets = make(map[int]*SocketConnection, 0)
|
||||
connectedWebsockets = make(map[int]*SocketConnection)
|
||||
}
|
||||
|
||||
func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
|
||||
@@ -94,7 +93,10 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
|
||||
eventHandlers.WebSocketConnect(socketId, isTapper)
|
||||
|
||||
startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
|
||||
SendToSocket(socketId, startTimeBytes)
|
||||
|
||||
if err = SendToSocket(socketId, startTimeBytes); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
|
||||
for {
|
||||
_, msg, err := ws.ReadMessage()
|
||||
@@ -117,7 +119,9 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
|
||||
AutoClose: 5000,
|
||||
Text: fmt.Sprintf("Syntax error: %s", err.Error()),
|
||||
})
|
||||
SendToSocket(socketId, toastBytes)
|
||||
if err := SendToSocket(socketId, toastBytes); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
@@ -137,7 +141,9 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
|
||||
base := tapApi.Summarize(entry)
|
||||
|
||||
baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
|
||||
SendToSocket(socketId, baseEntryBytes)
|
||||
if err := SendToSocket(socketId, baseEntryBytes); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,7 +162,9 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
|
||||
}
|
||||
|
||||
metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
|
||||
SendToSocket(socketId, metadataBytes)
|
||||
if err := SendToSocket(socketId, metadataBytes); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,14 +191,10 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
|
||||
socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
|
||||
}
|
||||
|
||||
var db = debounce.NewDebouncer(time.Second*5, func() {
|
||||
logger.Log.Error("Successfully sent to socket")
|
||||
})
|
||||
|
||||
func SendToSocket(socketId int, message []byte) error {
|
||||
socketObj := connectedWebsockets[socketId]
|
||||
if socketObj == nil {
|
||||
return errors.New("Socket is disconnected")
|
||||
return fmt.Errorf("Socket %v is disconnected", socketId)
|
||||
}
|
||||
|
||||
var sent = false
|
||||
@@ -204,7 +208,10 @@ func SendToSocket(socketId int, message []byte) error {
|
||||
socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
|
||||
err := socketObj.connection.WriteMessage(1, message)
|
||||
socketObj.lock.Unlock()
|
||||
|
||||
sent = true
|
||||
return err
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to write message to socket %v, err: %w", socketId, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,12 +3,13 @@ package api
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mizuserver/pkg/models"
|
||||
"mizuserver/pkg/providers"
|
||||
"mizuserver/pkg/providers/tappers"
|
||||
"mizuserver/pkg/up9"
|
||||
"sync"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/models"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers/tappers"
|
||||
"github.com/up9inc/mizu/agent/pkg/up9"
|
||||
|
||||
tapApi "github.com/up9inc/mizu/tap/api"
|
||||
|
||||
"github.com/up9inc/mizu/shared"
|
||||
@@ -54,9 +55,8 @@ func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
|
||||
func BroadcastToBrowserClients(message []byte) {
|
||||
for _, socketId := range browserClientSocketUUIDs {
|
||||
go func(socketId int) {
|
||||
err := SendToSocket(socketId, message)
|
||||
if err != nil {
|
||||
logger.Log.Errorf("error sending message to socket ID %d: %v", socketId, err)
|
||||
if err := SendToSocket(socketId, message); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
}(socketId)
|
||||
}
|
||||
|
||||
@@ -2,21 +2,22 @@ package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/up9inc/mizu/agent/pkg/config"
|
||||
"github.com/up9inc/mizu/agent/pkg/models"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers/tapConfig"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers/tappers"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/kubernetes"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
tapApi "github.com/up9inc/mizu/tap/api"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"mizuserver/pkg/config"
|
||||
"mizuserver/pkg/models"
|
||||
"mizuserver/pkg/providers"
|
||||
"mizuserver/pkg/providers/tapConfig"
|
||||
"mizuserver/pkg/providers/tappedPods"
|
||||
"mizuserver/pkg/providers/tappers"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
var cancelTapperSyncer context.CancelFunc
|
||||
|
||||
@@ -2,13 +2,14 @@ package controllers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"mizuserver/pkg/har"
|
||||
"mizuserver/pkg/models"
|
||||
"mizuserver/pkg/validation"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
"github.com/up9inc/mizu/agent/pkg/models"
|
||||
"github.com/up9inc/mizu/agent/pkg/validation"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
basenine "github.com/up9inc/basenine/client/go"
|
||||
@@ -26,7 +27,7 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) {
|
||||
func Error(c *gin.Context, err error) bool {
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Error getting entry: %v", err)
|
||||
c.Error(err)
|
||||
_ = c.Error(err)
|
||||
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
|
||||
"error": true,
|
||||
"type": "error",
|
||||
@@ -131,7 +132,9 @@ func GetEntry(c *gin.Context) {
|
||||
_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
|
||||
isRulesEnabled = _isRulesEnabled
|
||||
inrec, _ := json.Marshal(rulesMatched)
|
||||
json.Unmarshal(inrec, &rules)
|
||||
if err := json.Unmarshal(inrec, &rules); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, tapApi.EntryWrapper{
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/providers"
|
||||
"net/http"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/providers"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"mizuserver/pkg/version"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/up9inc/mizu/agent/pkg/version"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
)
|
||||
|
||||
func GetVersion(c *gin.Context) {
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/chanced/openapi"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/up9inc/mizu/agent/pkg/oas"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"mizuserver/pkg/oas"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func GetOASServers(c *gin.Context) {
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"mizuserver/pkg/oas"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/oas"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func TestGetOASServers(t *testing.T) {
|
||||
@@ -15,7 +17,6 @@ func TestGetOASServers(t *testing.T) {
|
||||
|
||||
GetOASServers(c)
|
||||
t.Logf("Written body: %s", recorder.Body.String())
|
||||
return
|
||||
}
|
||||
|
||||
func TestGetOASAllSpecs(t *testing.T) {
|
||||
@@ -26,7 +27,6 @@ func TestGetOASAllSpecs(t *testing.T) {
|
||||
|
||||
GetOASAllSpecs(c)
|
||||
t.Logf("Written body: %s", recorder.Body.String())
|
||||
return
|
||||
}
|
||||
|
||||
func TestGetOASSpec(t *testing.T) {
|
||||
@@ -39,5 +39,4 @@ func TestGetOASSpec(t *testing.T) {
|
||||
|
||||
GetOASSpec(c)
|
||||
t.Logf("Written body: %s", recorder.Body.String())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/servicemap"
|
||||
"net/http"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/servicemap"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"mizuserver/pkg/servicemap"
|
||||
"github.com/up9inc/mizu/agent/pkg/servicemap"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
@@ -2,17 +2,18 @@ package controllers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/up9inc/mizu/agent/pkg/api"
|
||||
"github.com/up9inc/mizu/agent/pkg/holder"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers/tappers"
|
||||
"github.com/up9inc/mizu/agent/pkg/up9"
|
||||
"github.com/up9inc/mizu/agent/pkg/validation"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"mizuserver/pkg/api"
|
||||
"mizuserver/pkg/holder"
|
||||
"mizuserver/pkg/providers"
|
||||
"mizuserver/pkg/providers/tappedPods"
|
||||
"mizuserver/pkg/providers/tappers"
|
||||
"mizuserver/pkg/up9"
|
||||
"mizuserver/pkg/validation"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func HealthCheck(c *gin.Context) {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/providers"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
|
||||
@@ -123,11 +123,11 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
|
||||
cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
|
||||
|
||||
postData, _ := request["postData"].(map[string]interface{})
|
||||
mimeType, _ := postData["mimeType"]
|
||||
mimeType := postData["mimeType"]
|
||||
if mimeType == nil || len(mimeType.(string)) == 0 {
|
||||
mimeType = "text/html"
|
||||
}
|
||||
text, _ := postData["text"]
|
||||
text := postData["text"]
|
||||
postDataText := ""
|
||||
if text != nil {
|
||||
postDataText = text.(string)
|
||||
@@ -176,12 +176,12 @@ func NewResponse(response map[string]interface{}) (harResponse *Response, err er
|
||||
cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
|
||||
|
||||
content, _ := response["content"].(map[string]interface{})
|
||||
mimeType, _ := content["mimeType"]
|
||||
mimeType := content["mimeType"]
|
||||
if mimeType == nil || len(mimeType.(string)) == 0 {
|
||||
mimeType = "text/html"
|
||||
}
|
||||
encoding, _ := content["encoding"]
|
||||
text, _ := content["text"]
|
||||
encoding := content["encoding"]
|
||||
text := content["text"]
|
||||
bodyText := ""
|
||||
if text != nil {
|
||||
bodyText = text.(string)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package holder
|
||||
|
||||
import "mizuserver/pkg/resolver"
|
||||
import "github.com/up9inc/mizu/agent/pkg/resolver"
|
||||
|
||||
var k8sResolver *resolver.Resolver
|
||||
|
||||
@@ -11,4 +11,3 @@ func SetResolver(param *resolver.Resolver) {
|
||||
func GetResolver() *resolver.Resolver {
|
||||
return k8sResolver
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/config"
|
||||
"mizuserver/pkg/providers"
|
||||
"github.com/up9inc/mizu/agent/pkg/config"
|
||||
"github.com/up9inc/mizu/agent/pkg/providers"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
ory "github.com/ory/kratos-client-go"
|
||||
|
||||
@@ -2,9 +2,10 @@ package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
"github.com/up9inc/mizu/agent/pkg/rules"
|
||||
tapApi "github.com/up9inc/mizu/tap/api"
|
||||
"mizuserver/pkg/har"
|
||||
"mizuserver/pkg/rules"
|
||||
|
||||
basenine "github.com/up9inc/basenine/client/go"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
|
||||
57
agent/pkg/oas/counters.go
Normal file
57
agent/pkg/oas/counters.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package oas
|
||||
|
||||
import "math"
|
||||
|
||||
type Counter struct {
|
||||
Entries int `json:"entries"`
|
||||
Failures int `json:"failures"`
|
||||
FirstSeen float64 `json:"firstSeen"`
|
||||
LastSeen float64 `json:"lastSeen"`
|
||||
SumRT float64 `json:"sumRT"`
|
||||
SumDuration float64 `json:"sumDuration"`
|
||||
}
|
||||
|
||||
func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
|
||||
c.Entries += 1
|
||||
c.SumRT += rt
|
||||
c.SumDuration += dur
|
||||
if !succ {
|
||||
c.Failures += 1
|
||||
}
|
||||
|
||||
if c.FirstSeen == 0 {
|
||||
c.FirstSeen = ts
|
||||
} else {
|
||||
c.FirstSeen = math.Min(c.FirstSeen, ts)
|
||||
}
|
||||
|
||||
c.LastSeen = math.Max(c.LastSeen, ts)
|
||||
}
|
||||
|
||||
func (c *Counter) addOther(other *Counter) {
|
||||
c.Entries += other.Entries
|
||||
c.SumRT += other.SumRT
|
||||
c.Failures += other.Failures
|
||||
c.SumDuration += other.SumDuration
|
||||
|
||||
if c.FirstSeen == 0 {
|
||||
c.FirstSeen = other.FirstSeen
|
||||
} else {
|
||||
c.FirstSeen = math.Min(c.FirstSeen, other.FirstSeen)
|
||||
}
|
||||
|
||||
c.LastSeen = math.Max(c.LastSeen, other.LastSeen)
|
||||
}
|
||||
|
||||
type CounterMap map[string]*Counter
|
||||
|
||||
func (m *CounterMap) addOther(other *CounterMap) {
|
||||
for src, cmap := range *other {
|
||||
if existing, ok := (*m)[src]; ok {
|
||||
existing.addOther(cmap)
|
||||
} else {
|
||||
copied := *cmap
|
||||
(*m)[src] = &copied
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,19 +4,21 @@ import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"mizuserver/pkg/har"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
func getFiles(baseDir string) (result []string, err error) {
|
||||
result = make([]string, 0, 0)
|
||||
result = make([]string, 0)
|
||||
logger.Log.Infof("Reading files from tree: %s", baseDir)
|
||||
|
||||
// https://yourbasic.org/golang/list-files-in-directory/
|
||||
@@ -108,27 +110,26 @@ func feedFromHAR(file string, isSync bool) (int, error) {
|
||||
cnt := 0
|
||||
for _, entry := range harDoc.Log.Entries {
|
||||
cnt += 1
|
||||
feedEntry(&entry, isSync)
|
||||
feedEntry(&entry, "", isSync)
|
||||
}
|
||||
|
||||
return cnt, nil
|
||||
}
|
||||
|
||||
func feedEntry(entry *har.Entry, isSync bool) {
|
||||
func feedEntry(entry *har.Entry, source string, isSync bool) {
|
||||
if entry.Response.Status == 302 {
|
||||
logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
|
||||
}
|
||||
|
||||
if strings.Contains(entry.Request.URL, "taboola") {
|
||||
if strings.Contains(entry.Request.URL, "some") { // for debugging
|
||||
logger.Log.Debugf("Interesting: %s", entry.Request.URL)
|
||||
} else {
|
||||
//return
|
||||
}
|
||||
|
||||
ews := EntryWithSource{Entry: *entry, Source: source}
|
||||
if isSync {
|
||||
GetOasGeneratorInstance().entriesChan <- *entry // blocking variant, right?
|
||||
GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right?
|
||||
} else {
|
||||
GetOasGeneratorInstance().PushEntry(entry)
|
||||
GetOasGeneratorInstance().PushEntry(&ews)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,6 +146,7 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
|
||||
var meta map[string]interface{}
|
||||
buf := strings.Builder{}
|
||||
cnt := 0
|
||||
source := ""
|
||||
for {
|
||||
substr, isPrefix, err := reader.ReadLine()
|
||||
if err == io.EOF {
|
||||
@@ -164,6 +166,9 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if s, ok := meta["_source"]; ok && s != nil {
|
||||
source = s.(string)
|
||||
}
|
||||
} else {
|
||||
var entry har.Entry
|
||||
err := json.Unmarshal([]byte(line), &entry)
|
||||
@@ -171,7 +176,7 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
|
||||
logger.Log.Warningf("Failed decoding entry: %s", line)
|
||||
} else {
|
||||
cnt += 1
|
||||
feedEntry(&entry, isSync)
|
||||
feedEntry(&entry, source, isSync)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,11 +33,7 @@ func IsGibberish(str string) bool {
|
||||
}
|
||||
|
||||
noise := noiseLevel(str)
|
||||
if noise >= 0.2 {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
return noise >= 0.2
|
||||
}
|
||||
|
||||
func noiseLevel(str string) (score float64) {
|
||||
|
||||
@@ -3,10 +3,11 @@ package oas
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"mizuserver/pkg/har"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -29,7 +30,7 @@ func (g *oasGenerator) Start() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
g.cancel = cancel
|
||||
g.ctx = ctx
|
||||
g.entriesChan = make(chan har.Entry, 100) // buffer up to 100 entries for OAS processing
|
||||
g.entriesChan = make(chan EntryWithSource, 100) // buffer up to 100 entries for OAS processing
|
||||
g.ServiceSpecs = &sync.Map{}
|
||||
g.started = true
|
||||
go instance.runGeneretor()
|
||||
@@ -42,11 +43,12 @@ func (g *oasGenerator) runGeneretor() {
|
||||
logger.Log.Infof("OAS Generator was canceled")
|
||||
return
|
||||
|
||||
case entry, ok := <-g.entriesChan:
|
||||
case entryWithSource, ok := <-g.entriesChan:
|
||||
if !ok {
|
||||
logger.Log.Infof("OAS Generator - entries channel closed")
|
||||
break
|
||||
}
|
||||
entry := entryWithSource.Entry
|
||||
u, err := url.Parse(entry.Request.URL)
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
|
||||
@@ -61,7 +63,7 @@ func (g *oasGenerator) runGeneretor() {
|
||||
gen = val.(*SpecGen)
|
||||
}
|
||||
|
||||
opId, err := gen.feedEntry(entry)
|
||||
opId, err := gen.feedEntry(entryWithSource)
|
||||
if err != nil {
|
||||
txt, suberr := json.Marshal(entry)
|
||||
if suberr == nil {
|
||||
@@ -77,12 +79,12 @@ func (g *oasGenerator) runGeneretor() {
|
||||
}
|
||||
}
|
||||
|
||||
func (g *oasGenerator) PushEntry(entry *har.Entry) {
|
||||
func (g *oasGenerator) PushEntry(entryWithSource *EntryWithSource) {
|
||||
if !g.started {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case g.entriesChan <- *entry:
|
||||
case g.entriesChan <- *entryWithSource:
|
||||
default:
|
||||
logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
|
||||
}
|
||||
@@ -98,10 +100,15 @@ func newOasGenerator() *oasGenerator {
|
||||
}
|
||||
}
|
||||
|
||||
type EntryWithSource struct {
|
||||
Source string
|
||||
Entry har.Entry
|
||||
}
|
||||
|
||||
type oasGenerator struct {
|
||||
started bool
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
ServiceSpecs *sync.Map
|
||||
entriesChan chan har.Entry
|
||||
entriesChan chan EntryWithSource
|
||||
}
|
||||
|
||||
@@ -3,17 +3,26 @@ package oas
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/chanced/openapi"
|
||||
"github.com/google/uuid"
|
||||
"github.com/nav-inc/datetime"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"mime"
|
||||
"mizuserver/pkg/har"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
|
||||
"time"
|
||||
)
|
||||
|
||||
const LastSeenTS = "x-last-seen-ts"
|
||||
const CountersTotal = "x-counters-total"
|
||||
const CountersPerSource = "x-counters-per-source"
|
||||
|
||||
type reqResp struct { // hello, generics in Go
|
||||
Req *har.Request
|
||||
Resp *har.Response
|
||||
@@ -43,17 +52,23 @@ func NewGen(server string) *SpecGen {
|
||||
|
||||
func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
|
||||
g.oas = oas
|
||||
g.tree = new(Node)
|
||||
for pathStr, pathObj := range oas.Paths.Items {
|
||||
pathSplit := strings.Split(string(pathStr), "/")
|
||||
g.tree.getOrSet(pathSplit, pathObj)
|
||||
|
||||
// clean "last entry timestamp" markers from the past
|
||||
for _, pathAndOp := range g.tree.listOps() {
|
||||
delete(pathAndOp.op.Extensions, LastSeenTS)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (g *SpecGen) feedEntry(entry har.Entry) (string, error) {
|
||||
func (g *SpecGen) feedEntry(entryWithSource EntryWithSource) (string, error) {
|
||||
g.lock.Lock()
|
||||
defer g.lock.Unlock()
|
||||
|
||||
opId, err := g.handlePathObj(&entry)
|
||||
opId, err := g.handlePathObj(&entryWithSource)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -68,10 +83,23 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
|
||||
|
||||
g.tree.compact()
|
||||
|
||||
for _, pathop := range g.tree.listOps() {
|
||||
if pathop.op.Summary == "" {
|
||||
pathop.op.Summary = pathop.path
|
||||
counters := CounterMaps{counterTotal: Counter{}, counterMapTotal: CounterMap{}}
|
||||
|
||||
for _, pathAndOp := range g.tree.listOps() {
|
||||
opObj := pathAndOp.op
|
||||
if opObj.Summary == "" {
|
||||
opObj.Summary = pathAndOp.path
|
||||
}
|
||||
|
||||
err := counters.processOp(opObj)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err := counters.processOas(g.oas)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// put paths back from tree into OAS
|
||||
@@ -79,6 +107,8 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
|
||||
|
||||
suggestTags(g.oas)
|
||||
|
||||
g.oas.Info.Description = setCounterMsgIfOk(g.oas.Info.Description, &counters.counterTotal)
|
||||
|
||||
// to make a deep copy, no better idea than marshal+unmarshal
|
||||
specText, err := json.MarshalIndent(g.oas, "", "\t")
|
||||
if err != nil {
|
||||
@@ -167,7 +197,8 @@ func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
|
||||
return keys
|
||||
}
|
||||
|
||||
func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
|
||||
func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error) {
|
||||
entry := entryWithSource.Entry
|
||||
urlParsed, err := url.Parse(entry.Request.URL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -211,7 +242,7 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
|
||||
split = strings.Split(urlParsed.Path, "/")
|
||||
}
|
||||
node := g.tree.getOrSet(split, new(openapi.PathObj))
|
||||
opObj, err := handleOpObj(entry, node.pathObj)
|
||||
opObj, err := handleOpObj(entryWithSource, node.pathObj)
|
||||
|
||||
if opObj != nil {
|
||||
return opObj.OperationID, err
|
||||
@@ -220,7 +251,8 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation, error) {
|
||||
func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*openapi.Operation, error) {
|
||||
entry := entryWithSource.Entry
|
||||
isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
|
||||
opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
|
||||
if err != nil {
|
||||
@@ -242,9 +274,86 @@ func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = handleCounters(opObj, isSuccess, entryWithSource)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return opObj, nil
|
||||
}
|
||||
|
||||
func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *EntryWithSource) error {
|
||||
// TODO: if performance around DecodeExtension+SetExtension is bad, store counters as separate maps
|
||||
counter := Counter{}
|
||||
counterMap := CounterMap{}
|
||||
prevTs := 0.0
|
||||
if opObj.Extensions == nil {
|
||||
opObj.Extensions = openapi.Extensions{}
|
||||
} else {
|
||||
if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
|
||||
err := opObj.Extensions.DecodeExtension(CountersTotal, &counter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
|
||||
err := opObj.Extensions.DecodeExtension(CountersPerSource, &counterMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if _, ok := opObj.Extensions.Extension(LastSeenTS); ok {
|
||||
err := opObj.Extensions.DecodeExtension(LastSeenTS, &prevTs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var counterPerSource *Counter
|
||||
if existing, ok := counterMap[entryWithSource.Source]; ok {
|
||||
counterPerSource = existing
|
||||
} else {
|
||||
counterPerSource = new(Counter)
|
||||
counterMap[entryWithSource.Source] = counterPerSource
|
||||
}
|
||||
|
||||
started, err := datetime.Parse(entryWithSource.Entry.StartedDateTime, time.UTC)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ts := float64(started.UnixNano()) / float64(time.Millisecond) / 1000
|
||||
rt := float64(entryWithSource.Entry.Time) / 1000
|
||||
|
||||
dur := 0.0
|
||||
if prevTs != 0 {
|
||||
dur = ts - prevTs
|
||||
}
|
||||
|
||||
counter.addEntry(ts, rt, success, dur)
|
||||
counterPerSource.addEntry(ts, rt, success, dur)
|
||||
|
||||
err = opObj.Extensions.SetExtension(LastSeenTS, ts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = opObj.Extensions.SetExtension(CountersTotal, counter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = opObj.Extensions.SetExtension(CountersPerSource, counterMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
|
||||
// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
|
||||
|
||||
@@ -272,7 +381,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
|
||||
|
||||
if reqBody != nil {
|
||||
reqCtype := getReqCtype(req)
|
||||
reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, err)
|
||||
reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -294,7 +403,7 @@ func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool
|
||||
|
||||
respCtype := getRespCtype(resp)
|
||||
respContent := respObj.Content
|
||||
respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, err)
|
||||
respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -342,11 +451,9 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err error) (*openapi.MediaType, error) {
|
||||
func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*openapi.MediaType, error) {
|
||||
content, found := respContent[ctype]
|
||||
if !found {
|
||||
respContent[ctype] = &openapi.MediaType{}
|
||||
@@ -367,14 +474,16 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err
|
||||
any, isJSON := anyJSON(text)
|
||||
if isJSON {
|
||||
// re-marshal with forced indent
|
||||
exampleMsg, err = json.MarshalIndent(any, "", "\t")
|
||||
if err != nil {
|
||||
if msg, err := json.MarshalIndent(any, "", "\t"); err != nil {
|
||||
panic("Failed to re-marshal value, super-strange")
|
||||
} else {
|
||||
exampleMsg = msg
|
||||
}
|
||||
} else {
|
||||
exampleMsg, err = json.Marshal(text)
|
||||
if err != nil {
|
||||
if msg, err := json.Marshal(text); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
exampleMsg = msg
|
||||
}
|
||||
}
|
||||
|
||||
@@ -498,3 +607,56 @@ func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*open
|
||||
|
||||
return *op, isMissing, nil
|
||||
}
|
||||
|
||||
type CounterMaps struct {
|
||||
counterTotal Counter
|
||||
counterMapTotal CounterMap
|
||||
}
|
||||
|
||||
func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
|
||||
if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
|
||||
counter := new(Counter)
|
||||
err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.counterTotal.addOther(counter)
|
||||
|
||||
opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
|
||||
}
|
||||
|
||||
if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
|
||||
counterMap := new(CounterMap)
|
||||
err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.counterMapTotal.addOther(counterMap)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
|
||||
if oas.Extensions == nil {
|
||||
oas.Extensions = openapi.Extensions{}
|
||||
}
|
||||
|
||||
err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
|
||||
tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
|
||||
if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
|
||||
return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
|
||||
}
|
||||
return oldStr
|
||||
}
|
||||
|
||||
@@ -2,15 +2,16 @@ package oas
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/chanced/openapi"
|
||||
"github.com/op/go-logging"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"io/ioutil"
|
||||
"mizuserver/pkg/har"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/chanced/openapi"
|
||||
"github.com/op/go-logging"
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
// if started via env, write file into subdir
|
||||
@@ -171,8 +172,6 @@ func loadStartingOAS() {
|
||||
gen.StartFromSpec(doc)
|
||||
|
||||
GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func TestEntriesNegative(t *testing.T) {
|
||||
@@ -224,5 +223,4 @@ func TestLoadValid3_1(t *testing.T) {
|
||||
t.Log(err)
|
||||
t.FailNow()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{"messageType": "http", "_source": null, "firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
|
||||
{"messageType": "http", "_source": "some-source", ",firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
|
||||
{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
|
||||
{"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
|
||||
{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}
|
||||
|
||||
@@ -121,6 +121,120 @@
|
||||
"connect": 262,
|
||||
"ssl": -1
|
||||
}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2019-09-06T06:16:20.047122+00:00",
|
||||
"time": 630,
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "https://httpbin.org/appears-once",
|
||||
"httpVersion": "HTTP/1.1",
|
||||
"cookies": [],
|
||||
"headers": [],
|
||||
"queryString": [],
|
||||
"headersSize": 1542,
|
||||
"bodySize": 0
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"statusText": "OK",
|
||||
"httpVersion": "HTTP/1.1",
|
||||
"cookies": [],
|
||||
"headers": [],
|
||||
"content": {
|
||||
"size": 39,
|
||||
"compression": -20,
|
||||
"mimeType": "application/json",
|
||||
"text": "null"
|
||||
},
|
||||
"redirectURL": "",
|
||||
"headersSize": 248,
|
||||
"bodySize": 39
|
||||
},
|
||||
"cache": {},
|
||||
"timings": {
|
||||
"send": 14,
|
||||
"receive": 4,
|
||||
"wait": 350,
|
||||
"connect": 262,
|
||||
"ssl": -1
|
||||
}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2019-09-06T06:16:20.747122+00:00",
|
||||
"time": 630,
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "https://httpbin.org/appears-twice",
|
||||
"httpVersion": "HTTP/1.1",
|
||||
"cookies": [],
|
||||
"headers": [],
|
||||
"queryString": [],
|
||||
"headersSize": 1542,
|
||||
"bodySize": 0
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"statusText": "OK",
|
||||
"httpVersion": "HTTP/1.1",
|
||||
"cookies": [],
|
||||
"headers": [],
|
||||
"content": {
|
||||
"size": 39,
|
||||
"compression": -20,
|
||||
"mimeType": "application/json",
|
||||
"text": "null"
|
||||
},
|
||||
"redirectURL": "",
|
||||
"headersSize": 248,
|
||||
"bodySize": 39
|
||||
},
|
||||
"cache": {},
|
||||
"timings": {
|
||||
"send": 14,
|
||||
"receive": 4,
|
||||
"wait": 350,
|
||||
"connect": 262,
|
||||
"ssl": -1
|
||||
}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2019-09-06T06:16:21.747122+00:00",
|
||||
"time": 630,
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "https://httpbin.org/appears-twice",
|
||||
"httpVersion": "HTTP/1.1",
|
||||
"cookies": [],
|
||||
"headers": [],
|
||||
"queryString": [],
|
||||
"headersSize": 1542,
|
||||
"bodySize": 0
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"statusText": "OK",
|
||||
"httpVersion": "HTTP/1.1",
|
||||
"cookies": [],
|
||||
"headers": [],
|
||||
"content": {
|
||||
"size": 39,
|
||||
"compression": -20,
|
||||
"mimeType": "application/json",
|
||||
"text": "null"
|
||||
},
|
||||
"redirectURL": "",
|
||||
"headersSize": 248,
|
||||
"bodySize": 39
|
||||
},
|
||||
"cache": {},
|
||||
"timings": {
|
||||
"send": 14,
|
||||
"receive": 4,
|
||||
"wait": 350,
|
||||
"connect": 262,
|
||||
"ssl": -1
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package oas
|
||||
|
||||
import (
|
||||
"github.com/chanced/openapi"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/chanced/openapi"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
type NodePath = []string
|
||||
@@ -162,9 +163,7 @@ func (n *Node) listPaths() *openapi.Paths {
|
||||
strChunk = *n.constant
|
||||
} else if n.pathParam != nil {
|
||||
strChunk = "{" + n.pathParam.Name + "}"
|
||||
} else {
|
||||
// this is the root node
|
||||
}
|
||||
} // else -> this is the root node
|
||||
|
||||
// add self
|
||||
if n.pathObj != nil {
|
||||
|
||||
@@ -3,11 +3,13 @@ package oas
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/chanced/openapi"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"mizuserver/pkg/har"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
|
||||
"github.com/chanced/openapi"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
func exampleResolver(ref string) (*openapi.ExampleObj, error) {
|
||||
@@ -32,16 +34,14 @@ func headerResolver(ref string) (*openapi.HeaderObj, error) {
|
||||
|
||||
func initParams(obj **openapi.ParameterList) {
|
||||
if *obj == nil {
|
||||
var params openapi.ParameterList
|
||||
params = make([]openapi.Parameter, 0)
|
||||
var params openapi.ParameterList = make([]openapi.Parameter, 0)
|
||||
*obj = ¶ms
|
||||
}
|
||||
}
|
||||
|
||||
func initHeaders(respObj *openapi.ResponseObj) {
|
||||
if respObj.Headers == nil {
|
||||
var created openapi.Headers
|
||||
created = map[string]openapi.Header{}
|
||||
var created openapi.Headers = map[string]openapi.Header{}
|
||||
respObj.Headers = created
|
||||
}
|
||||
}
|
||||
@@ -85,7 +85,7 @@ func findParamByName(params *openapi.ParameterList, in openapi.In, name string)
|
||||
continue
|
||||
}
|
||||
|
||||
if paramObj.Name == name || (caseInsensitive && strings.ToLower(paramObj.Name) == strings.ToLower(name)) {
|
||||
if paramObj.Name == name || (caseInsensitive && strings.EqualFold(paramObj.Name, name)) {
|
||||
pathParam = paramObj
|
||||
break
|
||||
}
|
||||
@@ -102,7 +102,7 @@ func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.ToLower(hname) == strings.ToLower(name) {
|
||||
if strings.EqualFold(hname, name) {
|
||||
return hdrObj
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,10 @@ package providers
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"mizuserver/pkg/config"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/config"
|
||||
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
|
||||
ory "github.com/ory/kratos-client-go"
|
||||
)
|
||||
@@ -38,7 +41,9 @@ func CreateAdminUser(password string, ctx context.Context) (token *string, err e
|
||||
|
||||
if err != nil {
|
||||
//Delete the user to prevent a half-setup situation where admin user is created without admin privileges
|
||||
DeleteUser(identityId, ctx)
|
||||
if err := DeleteUser(identityId, ctx); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
|
||||
return nil, err, nil
|
||||
}
|
||||
|
||||
@@ -2,8 +2,9 @@ package providers_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"mizuserver/pkg/providers"
|
||||
"testing"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/providers"
|
||||
)
|
||||
|
||||
func TestNoEntryAddedCount(t *testing.T) {
|
||||
|
||||
@@ -3,12 +3,13 @@ package providers
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/patrickmn/go-cache"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/tap"
|
||||
"mizuserver/pkg/models"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/patrickmn/go-cache"
|
||||
"github.com/up9inc/mizu/agent/pkg/models"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/tap"
|
||||
)
|
||||
|
||||
const tlsLinkRetainmentTime = time.Minute * 15
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
package tapConfig
|
||||
|
||||
import (
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"mizuserver/pkg/models"
|
||||
"mizuserver/pkg/utils"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/models"
|
||||
"github.com/up9inc/mizu/agent/pkg/utils"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
const FilePath = shared.DataDirPath + "tap-config.json"
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
package tappedPods
|
||||
|
||||
import (
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"mizuserver/pkg/providers/tappers"
|
||||
"mizuserver/pkg/utils"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/providers/tappers"
|
||||
"github.com/up9inc/mizu/agent/pkg/utils"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
const FilePath = shared.DataDirPath + "tapped-pods.json"
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package tappers
|
||||
|
||||
import (
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"mizuserver/pkg/utils"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/utils"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
const FilePath = shared.DataDirPath + "tappers-status.json"
|
||||
|
||||
@@ -85,11 +85,11 @@ func DeleteUser(identityId string, ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
if result == nil {
|
||||
return errors.New("unknown error occured during user deletion")
|
||||
return fmt.Errorf("unknown error occured during user deletion %v", identityId)
|
||||
}
|
||||
|
||||
if result.StatusCode < 200 || result.StatusCode > 299 {
|
||||
return errors.New(fmt.Sprintf("user deletion returned bad status %d", result.StatusCode))
|
||||
return fmt.Errorf("user deletion %v returned bad status %d", identityId, result.StatusCode)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -15,7 +15,8 @@ more on keto here: https://www.ory.sh/keto/docs/
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"mizuserver/pkg/utils"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/utils"
|
||||
|
||||
ketoClient "github.com/ory/keto-client-go/client"
|
||||
ketoRead "github.com/ory/keto-client-go/client/read"
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/controllers"
|
||||
"mizuserver/pkg/middlewares"
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
"github.com/up9inc/mizu/agent/pkg/middlewares"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/controllers"
|
||||
"mizuserver/pkg/middlewares"
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
"github.com/up9inc/mizu/agent/pkg/middlewares"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/controllers"
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/controllers"
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// NotFoundRoute defines the 404 Error route.
|
||||
func NotFoundRoute(app *gin.Engine) {
|
||||
app.Use(
|
||||
func(c *gin.Context) {
|
||||
c.JSON(http.StatusNotFound, map[string]interface{}{
|
||||
"error": true,
|
||||
"msg": "sorry, endpoint is not found",
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/controllers"
|
||||
"mizuserver/pkg/middlewares"
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
"github.com/up9inc/mizu/agent/pkg/middlewares"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/controllers"
|
||||
"mizuserver/pkg/middlewares"
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
"github.com/up9inc/mizu/agent/pkg/middlewares"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/controllers"
|
||||
"mizuserver/pkg/middlewares"
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
"github.com/up9inc/mizu/agent/pkg/middlewares"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/controllers"
|
||||
"mizuserver/pkg/middlewares"
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
"github.com/up9inc/mizu/agent/pkg/middlewares"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"mizuserver/pkg/controllers"
|
||||
"github.com/up9inc/mizu/agent/pkg/controllers"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -4,11 +4,12 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mizuserver/pkg/har"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
|
||||
"github.com/up9inc/mizu/shared"
|
||||
@@ -108,7 +109,7 @@ func PassedValidationRules(rulesMatched []RulesMatched) (bool, int64, int) {
|
||||
}
|
||||
|
||||
for _, rule := range rulesMatched {
|
||||
if rule.Matched == false {
|
||||
if !rule.Matched {
|
||||
return false, responseTime, numberOfRulesMatched
|
||||
} else {
|
||||
if strings.ToLower(rule.Rule.Type) == "slo" {
|
||||
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"mizuserver/pkg/har"
|
||||
"mizuserver/pkg/utils"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
@@ -15,6 +13,9 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/up9inc/mizu/agent/pkg/har"
|
||||
"github.com/up9inc/mizu/agent/pkg/utils"
|
||||
|
||||
basenine "github.com/up9inc/basenine/client/go"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
|
||||
@@ -31,10 +31,14 @@ func StartServer(app *gin.Engine) {
|
||||
}
|
||||
|
||||
go func() {
|
||||
_ = <-signals
|
||||
<-signals
|
||||
logger.Log.Infof("Shutting down...")
|
||||
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = srv.Shutdown(ctx)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
err := srv.Shutdown(ctx)
|
||||
if err != nil {
|
||||
logger.Log.Errorf("%v", err)
|
||||
}
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
@@ -91,7 +95,7 @@ func UniqueStringSlice(s []string) []string {
|
||||
uniqueMap := map[string]bool{}
|
||||
|
||||
for _, val := range s {
|
||||
if uniqueMap[val] == true {
|
||||
if uniqueMap[val] {
|
||||
continue
|
||||
}
|
||||
uniqueMap[val] = true
|
||||
|
||||
@@ -50,7 +50,9 @@ func init() {
|
||||
rootCmd.AddCommand(configCmd)
|
||||
|
||||
defaultConfig := config.ConfigStruct{}
|
||||
defaults.Set(&defaultConfig)
|
||||
if err := defaults.Set(&defaultConfig); err != nil {
|
||||
logger.Log.Debug(err)
|
||||
}
|
||||
|
||||
configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfig.Config.Regenerate, fmt.Sprintf("Regenerate the config file with default values to path %s or to chosen path using --%s", defaultConfig.ConfigFilePath, config.ConfigFilePathCommandName))
|
||||
}
|
||||
|
||||
@@ -31,7 +31,9 @@ func runMizuInstall() {
|
||||
var defaultMaxEntriesDBSizeBytes int64 = 200 * 1000 * 1000
|
||||
|
||||
defaultResources := shared.Resources{}
|
||||
defaults.Set(&defaultResources)
|
||||
if err := defaults.Set(&defaultResources); err != nil {
|
||||
logger.Log.Debug(err)
|
||||
}
|
||||
|
||||
mizuAgentConfig := getInstallMizuAgentConfig(defaultMaxEntriesDBSizeBytes, defaultResources)
|
||||
serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig)
|
||||
|
||||
@@ -43,7 +43,9 @@ func init() {
|
||||
rootCmd.AddCommand(logsCmd)
|
||||
|
||||
defaultLogsConfig := configStructs.LogsConfig{}
|
||||
defaults.Set(&defaultLogsConfig)
|
||||
if err := defaults.Set(&defaultLogsConfig); err != nil {
|
||||
logger.Log.Debug(err)
|
||||
}
|
||||
|
||||
logsCmd.Flags().StringP(configStructs.FileLogsName, "f", defaultLogsConfig.FileStr, "Path for zip file (default current <pwd>\\mizu_logs.zip)")
|
||||
}
|
||||
|
||||
@@ -30,7 +30,9 @@ Further info is available at https://github.com/up9inc/mizu`,
|
||||
|
||||
func init() {
|
||||
defaultConfig := config.ConfigStruct{}
|
||||
defaults.Set(&defaultConfig)
|
||||
if err := defaults.Set(&defaultConfig); err != nil {
|
||||
logger.Log.Debug(err)
|
||||
}
|
||||
|
||||
rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName))
|
||||
rootCmd.PersistentFlags().String(config.ConfigFilePathCommandName, defaultConfig.ConfigFilePath, fmt.Sprintf("Override config file path using --%s", config.ConfigFilePathCommandName))
|
||||
|
||||
@@ -104,7 +104,9 @@ func init() {
|
||||
rootCmd.AddCommand(tapCmd)
|
||||
|
||||
defaultTapConfig := configStructs.TapConfig{}
|
||||
defaults.Set(&defaultTapConfig)
|
||||
if err := defaults.Set(&defaultTapConfig); err != nil {
|
||||
logger.Log.Debug(err)
|
||||
}
|
||||
|
||||
tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
|
||||
tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
|
||||
|
||||
@@ -36,7 +36,9 @@ func init() {
|
||||
rootCmd.AddCommand(versionCmd)
|
||||
|
||||
defaultVersionConfig := configStructs.VersionConfig{}
|
||||
defaults.Set(&defaultVersionConfig)
|
||||
if err := defaults.Set(&defaultVersionConfig); err != nil {
|
||||
logger.Log.Debug(err)
|
||||
}
|
||||
|
||||
versionCmd.Flags().BoolP(configStructs.DebugInfoVersionName, "d", defaultVersionConfig.DebugInfo, "Provide all information about version")
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/up9inc/mizu/cli/config"
|
||||
"github.com/up9inc/mizu/cli/config/configStructs"
|
||||
"github.com/up9inc/mizu/cli/telemetry"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
)
|
||||
|
||||
var viewCmd = &cobra.Command{
|
||||
@@ -22,10 +23,14 @@ func init() {
|
||||
rootCmd.AddCommand(viewCmd)
|
||||
|
||||
defaultViewConfig := configStructs.ViewConfig{}
|
||||
defaults.Set(&defaultViewConfig)
|
||||
if err := defaults.Set(&defaultViewConfig); err != nil {
|
||||
logger.Log.Debug(err)
|
||||
}
|
||||
|
||||
viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
|
||||
viewCmd.Flags().StringP(configStructs.UrlViewName, "u", defaultViewConfig.Url, "Provide a custom host")
|
||||
|
||||
viewCmd.Flags().MarkHidden(configStructs.UrlViewName)
|
||||
if err := viewCmd.Flags().MarkHidden(configStructs.UrlViewName); err != nil {
|
||||
logger.Log.Debug(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ type ConfigStruct struct {
|
||||
Logs configStructs.LogsConfig `yaml:"logs"`
|
||||
Auth configStructs.AuthConfig `yaml:"auth"`
|
||||
Config configStructs.ConfigConfig `yaml:"config,omitempty"`
|
||||
AgentImage string `yaml:"agent-image,omitempty"`
|
||||
AgentImage string `yaml:"agent-image,omitempty" readonly:""`
|
||||
KratosImage string `yaml:"kratos-image,omitempty" readonly:""`
|
||||
KetoImage string `yaml:"keto-image,omitempty" readonly:""`
|
||||
ImagePullPolicyStr string `yaml:"image-pull-policy" default:"Always"`
|
||||
@@ -38,7 +38,7 @@ type ConfigStruct struct {
|
||||
ConfigFilePath string `yaml:"config-path,omitempty" readonly:""`
|
||||
HeadlessMode bool `yaml:"headless" default:"false"`
|
||||
LogLevelStr string `yaml:"log-level,omitempty" default:"INFO" readonly:""`
|
||||
ServiceMap bool `yaml:"service-map,omitempty" default:"false" readonly:""`
|
||||
ServiceMap bool `yaml:"service-map" default:"false"`
|
||||
OAS bool `yaml:"oas,omitempty" default:"false" readonly:""`
|
||||
Elastic shared.ElasticConfig `yaml:"elastic"`
|
||||
}
|
||||
|
||||
@@ -60,12 +60,12 @@ func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
|
||||
func (config *TapConfig) Validate() error {
|
||||
_, compileErr := regexp.Compile(config.PodRegexStr)
|
||||
if compileErr != nil {
|
||||
return errors.New(fmt.Sprintf("%s is not a valid regex %s", config.PodRegexStr, compileErr))
|
||||
return fmt.Errorf("%s is not a valid regex %s", config.PodRegexStr, compileErr)
|
||||
}
|
||||
|
||||
_, parseHumanDataSizeErr := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
|
||||
if parseHumanDataSizeErr != nil {
|
||||
return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
|
||||
return fmt.Errorf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize)
|
||||
}
|
||||
|
||||
if config.Workspace != "" {
|
||||
@@ -76,7 +76,7 @@ func (config *TapConfig) Validate() error {
|
||||
}
|
||||
|
||||
if config.Analysis && config.Workspace != "" {
|
||||
return errors.New(fmt.Sprintf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName))
|
||||
return fmt.Errorf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package fsUtils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
@@ -18,7 +17,7 @@ func EnsureDir(dirName string) error {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return errors.New(fmt.Sprintf("path exists but is not a directory: %s", dirName))
|
||||
return fmt.Errorf("path exists but is not a directory: %s", dirName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -23,9 +23,5 @@ func IsTokenValid(tokenString string, envName string) bool {
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
return response.StatusCode == http.StatusOK
|
||||
}
|
||||
|
||||
@@ -2,6 +2,35 @@
|
||||
This document summarizes main and fixes changes published in stable (aka `main`) branch of this project.
|
||||
Ongoing work and development releases are under `develop` branch.
|
||||
|
||||
## 0.24.0
|
||||
|
||||
### main features
|
||||
* ARM64 support -- Mizu is now available for ARM 64bit architecture
|
||||
* Now you can run Mizu with `minikube` on your Apple M1 laptop or any other ARM-based hosts
|
||||
* New command helps user verify Mizu deployment
|
||||
* Run `mizu check` to verify Mizu was deployed successfully
|
||||
* `mizu check` verifies version compatibility, resources and permissions required by Mizu
|
||||
* EXPERIMENTAL: Service Map - graph of all service interactions
|
||||
* Arrow direction show client to server connection
|
||||
* Graph edge width reflects volume of traffic captured between the services
|
||||
* to enable this experimental feature use `--set service-map=true` flag
|
||||
|
||||
### improvements
|
||||
* Mizu container images are now served from [Docker Hub](https://hub.docker.com/r/up9inc/mizu), as multi-architecture images (arm64, amd64)
|
||||
* in Mizu GUI the filter query can now be applied by pressing CONTROL/COMMAND + ENTER
|
||||
* try port-forwarding if http-proxy connection to Mizu API server is not available
|
||||
|
||||
### notable bug fixes
|
||||
* Fixed HTTP/1.0 presentation which was shown as HTTP/1.1
|
||||
* Fixed handling of long-living TCP connections, improves capturing gRPC and HTTP/2 traffic, and helps in service-mesh setups (istio, linkerd)
|
||||
|
||||
|
||||
## 0.23.0
|
||||
### notable bug fixes
|
||||
* fixed errors in Redis protocol parser (better handling of Array and Bulk String message types)
|
||||
|
||||
|
||||
|
||||
## 0.22.0
|
||||
|
||||
### main features
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
This document describes in details all permissions required for full and correct operation of Mizu.
|
||||
|
||||
## Editting permissions
|
||||
## Editing permissions
|
||||
|
||||
During installation, Mizu creates a `ServiceAccount` and the roles it requires. No further action is required.
|
||||
However, if there is a need, it is possible to make changes to Mizu permissions.
|
||||
|
||||
@@ -44,7 +44,7 @@ func (d *Debouncer) SetOn() error {
|
||||
if d.canceled {
|
||||
return fmt.Errorf("debouncer cancelled")
|
||||
}
|
||||
if d.running == true {
|
||||
if d.running {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -93,6 +93,7 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
|
||||
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
|
||||
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
|
||||
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
|
||||
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
|
||||
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
|
||||
github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
|
||||
@@ -307,6 +308,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
|
||||
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
|
||||
github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
|
||||
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
|
||||
@@ -34,7 +34,7 @@ func (wh *EventWatchHelper) Filter(wEvent *WatchEvent) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if strings.ToLower(event.Regarding.Kind) != strings.ToLower(wh.Kind) {
|
||||
if strings.EqualFold(event.Regarding.Kind, wh.Kind) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,21 +3,22 @@ package kubernetes
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/op/go-logging"
|
||||
"github.com/up9inc/mizu/shared"
|
||||
"github.com/up9inc/mizu/shared/debounce"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"github.com/up9inc/mizu/tap/api"
|
||||
core "k8s.io/api/core/v1"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
const updateTappersDelay = 5 * time.Second
|
||||
|
||||
type TappedPodChangeEvent struct {
|
||||
Added []core.Pod
|
||||
Removed []core.Pod
|
||||
Added []core.Pod
|
||||
Removed []core.Pod
|
||||
}
|
||||
|
||||
// MizuTapperSyncer uses a k8s pod watch to update tapper daemonsets when targeted pods are removed or created
|
||||
@@ -222,10 +223,14 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
|
||||
switch wEvent.Type {
|
||||
case EventAdded:
|
||||
logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
|
||||
restartTappersDebouncer.SetOn()
|
||||
if err := restartTappersDebouncer.SetOn(); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
case EventDeleted:
|
||||
logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
|
||||
restartTappersDebouncer.SetOn()
|
||||
if err := restartTappersDebouncer.SetOn(); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
case EventModified:
|
||||
logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
|
||||
// Act only if the modified pod has already obtained an IP address.
|
||||
@@ -235,7 +240,9 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
|
||||
// - Pod reaches ready state
|
||||
// Ready/unready transitions might also trigger this event.
|
||||
if pod.Status.PodIP != "" {
|
||||
restartTappersDebouncer.SetOn()
|
||||
if err := restartTappersDebouncer.SetOn(); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
}
|
||||
case EventBookmark:
|
||||
break
|
||||
@@ -285,8 +292,8 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch
|
||||
tapperSyncer.CurrentlyTappedPods = podsToTap
|
||||
tapperSyncer.nodeToTappedPodMap = GetNodeHostToTappedPodsMap(tapperSyncer.CurrentlyTappedPods)
|
||||
tapperSyncer.TapPodChangesOut <- TappedPodChangeEvent{
|
||||
Added: addedPods,
|
||||
Removed: removedPods,
|
||||
Added: addedPods,
|
||||
Removed: removedPods,
|
||||
}
|
||||
return nil, true
|
||||
}
|
||||
|
||||
@@ -194,19 +194,19 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
|
||||
|
||||
cpuLimit, err := resource.ParseQuantity(opts.Resources.CpuLimit)
|
||||
if err != nil {
|
||||
return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", opts.PodName))
|
||||
return nil, fmt.Errorf("invalid cpu limit for %s container", opts.PodName)
|
||||
}
|
||||
memLimit, err := resource.ParseQuantity(opts.Resources.MemoryLimit)
|
||||
if err != nil {
|
||||
return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", opts.PodName))
|
||||
return nil, fmt.Errorf("invalid memory limit for %s container", opts.PodName)
|
||||
}
|
||||
cpuRequests, err := resource.ParseQuantity(opts.Resources.CpuRequests)
|
||||
if err != nil {
|
||||
return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", opts.PodName))
|
||||
return nil, fmt.Errorf("invalid cpu request for %s container", opts.PodName)
|
||||
}
|
||||
memRequests, err := resource.ParseQuantity(opts.Resources.MemoryRequests)
|
||||
if err != nil {
|
||||
return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", opts.PodName))
|
||||
return nil, fmt.Errorf("invalid memory request for %s container", opts.PodName)
|
||||
}
|
||||
|
||||
command := []string{"./mizuagent", "--api-server"}
|
||||
@@ -395,7 +395,7 @@ func (provider *Provider) CreatePod(ctx context.Context, namespace string, podSp
|
||||
}
|
||||
|
||||
func (provider *Provider) CreateDeployment(ctx context.Context, namespace string, deploymentName string, podSpec *core.Pod) (*v1.Deployment, error) {
|
||||
if _, keyExists := podSpec.ObjectMeta.Labels["app"]; keyExists == false {
|
||||
if _, keyExists := podSpec.ObjectMeta.Labels["app"]; !keyExists {
|
||||
return nil, errors.New("pod spec must contain 'app' label")
|
||||
}
|
||||
podTemplate := &core.PodTemplateSpec{
|
||||
@@ -764,7 +764,7 @@ func (provider *Provider) handleRemovalError(err error) error {
|
||||
}
|
||||
|
||||
func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
|
||||
configMapData := make(map[string]string, 0)
|
||||
configMapData := make(map[string]string)
|
||||
if serializedValidationRules != "" {
|
||||
configMapData[shared.ValidationRulesFileName] = serializedValidationRules
|
||||
}
|
||||
@@ -854,19 +854,19 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
|
||||
)
|
||||
cpuLimit, err := resource.ParseQuantity(resources.CpuLimit)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("invalid cpu limit for %s container", tapperPodName))
|
||||
return fmt.Errorf("invalid cpu limit for %s container", tapperPodName)
|
||||
}
|
||||
memLimit, err := resource.ParseQuantity(resources.MemoryLimit)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("invalid memory limit for %s container", tapperPodName))
|
||||
return fmt.Errorf("invalid memory limit for %s container", tapperPodName)
|
||||
}
|
||||
cpuRequests, err := resource.ParseQuantity(resources.CpuRequests)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("invalid cpu request for %s container", tapperPodName))
|
||||
return fmt.Errorf("invalid cpu request for %s container", tapperPodName)
|
||||
}
|
||||
memRequests, err := resource.ParseQuantity(resources.MemoryRequests)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("invalid memory request for %s container", tapperPodName))
|
||||
return fmt.Errorf("invalid memory request for %s container", tapperPodName)
|
||||
}
|
||||
agentResourceLimits := core.ResourceList{
|
||||
"cpu": cpuLimit,
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
func GetNodeHostToTappedPodsMap(tappedPods []core.Pod) map[string][]core.Pod {
|
||||
nodeToTappedPodMap := make(map[string][]core.Pod, 0)
|
||||
nodeToTappedPodMap := make(map[string][]core.Pod)
|
||||
for _, pod := range tappedPods {
|
||||
minimizedPod := getMinimizedPod(pod)
|
||||
|
||||
|
||||
@@ -4,11 +4,12 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/up9inc/mizu/shared/debounce"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/up9inc/mizu/shared/debounce"
|
||||
"github.com/up9inc/mizu/shared/logger"
|
||||
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
)
|
||||
|
||||
@@ -31,7 +32,7 @@ func FilteredWatch(ctx context.Context, watcherCreator WatchCreator, targetNames
|
||||
|
||||
go func(targetNamespace string) {
|
||||
defer wg.Done()
|
||||
watchRestartDebouncer := debounce.NewDebouncer(1 * time.Minute, func() {})
|
||||
watchRestartDebouncer := debounce.NewDebouncer(1*time.Minute, func() {})
|
||||
|
||||
for {
|
||||
watcher, err := watcherCreator.NewWatcher(ctx, targetNamespace)
|
||||
@@ -44,7 +45,7 @@ func FilteredWatch(ctx context.Context, watcherCreator WatchCreator, targetNames
|
||||
watcher.Stop()
|
||||
|
||||
select {
|
||||
case <- ctx.Done():
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
break
|
||||
@@ -55,7 +56,9 @@ func FilteredWatch(ctx context.Context, watcherCreator WatchCreator, targetNames
|
||||
break
|
||||
} else {
|
||||
if !watchRestartDebouncer.IsOn() {
|
||||
watchRestartDebouncer.SetOn()
|
||||
if err := watchRestartDebouncer.SetOn(); err != nil {
|
||||
logger.Log.Error(err)
|
||||
}
|
||||
logger.Log.Debug("k8s watch channel closed, restarting watcher")
|
||||
time.Sleep(time.Second * 5)
|
||||
continue
|
||||
|
||||
@@ -6,11 +6,12 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/google/martian/har"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/martian/har"
|
||||
)
|
||||
|
||||
type Protocol struct {
|
||||
@@ -282,7 +283,7 @@ func (h HTTPPayload) MarshalJSON() ([]byte, error) {
|
||||
RawResponse: &HTTPResponseWrapper{Response: h.Data.(*http.Response)},
|
||||
})
|
||||
default:
|
||||
panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s", h.Type))
|
||||
panic(fmt.Sprintf("HTTP payload cannot be marshaled: %v", h.Type))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -312,7 +313,7 @@ type HTTPRequestWrapper struct {
|
||||
func (r *HTTPRequestWrapper) MarshalJSON() ([]byte, error) {
|
||||
body, _ := ioutil.ReadAll(r.Request.Body)
|
||||
r.Request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
|
||||
return json.Marshal(&struct {
|
||||
return json.Marshal(&struct { //nolint
|
||||
Body string `json:"Body,omitempty"`
|
||||
GetBody string `json:"GetBody,omitempty"`
|
||||
Cancel string `json:"Cancel,omitempty"`
|
||||
@@ -330,7 +331,7 @@ type HTTPResponseWrapper struct {
|
||||
func (r *HTTPResponseWrapper) MarshalJSON() ([]byte, error) {
|
||||
body, _ := ioutil.ReadAll(r.Response.Body)
|
||||
r.Response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
|
||||
return json.Marshal(&struct {
|
||||
return json.Marshal(&struct { //nolint
|
||||
Body string `json:"Body,omitempty"`
|
||||
GetBody string `json:"GetBody,omitempty"`
|
||||
Cancel string `json:"Cancel,omitempty"`
|
||||
|
||||
@@ -24,14 +24,14 @@ var connectionMethodMap = map[int]string{
|
||||
61: "connection unblocked",
|
||||
}
|
||||
|
||||
var channelMethodMap = map[int]string{
|
||||
10: "channel open",
|
||||
11: "channel open-ok",
|
||||
20: "channel flow",
|
||||
21: "channel flow-ok",
|
||||
40: "channel close",
|
||||
41: "channel close-ok",
|
||||
}
|
||||
// var channelMethodMap = map[int]string{
|
||||
// 10: "channel open",
|
||||
// 11: "channel open-ok",
|
||||
// 20: "channel flow",
|
||||
// 21: "channel flow-ok",
|
||||
// 40: "channel close",
|
||||
// 41: "channel close-ok",
|
||||
// }
|
||||
|
||||
var exchangeMethodMap = map[int]string{
|
||||
10: "exchange declare",
|
||||
@@ -78,14 +78,14 @@ var basicMethodMap = map[int]string{
|
||||
120: "basic nack",
|
||||
}
|
||||
|
||||
var txMethodMap = map[int]string{
|
||||
10: "tx select",
|
||||
11: "tx select-ok",
|
||||
20: "tx commit",
|
||||
21: "tx commit-ok",
|
||||
30: "tx rollback",
|
||||
31: "tx rollback-ok",
|
||||
}
|
||||
// var txMethodMap = map[int]string{
|
||||
// 10: "tx select",
|
||||
// 11: "tx select-ok",
|
||||
// 20: "tx commit",
|
||||
// 21: "tx commit-ok",
|
||||
// 30: "tx rollback",
|
||||
// 31: "tx rollback-ok",
|
||||
// }
|
||||
|
||||
type AMQPWrapper struct {
|
||||
Method string `json:"method"`
|
||||
@@ -550,14 +550,12 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
|
||||
headers := make([]api.TableData, 0)
|
||||
for name, value := range event["serverProperties"].(map[string]interface{}) {
|
||||
var outcome string
|
||||
switch value.(type) {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
outcome = value.(string)
|
||||
break
|
||||
outcome = v
|
||||
case map[string]interface{}:
|
||||
x, _ := json.Marshal(value)
|
||||
outcome = string(x)
|
||||
break
|
||||
default:
|
||||
panic("Unknown data type for the server property!")
|
||||
}
|
||||
|
||||
@@ -47,7 +47,6 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
|
||||
|
||||
var remaining int
|
||||
var header *HeaderFrame
|
||||
var body []byte
|
||||
|
||||
connectionInfo := &api.ConnectionInfo{
|
||||
ClientIP: tcpID.SrcIP,
|
||||
@@ -102,13 +101,10 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
|
||||
eventBasicPublish.Properties = header.Properties
|
||||
case *BasicDeliver:
|
||||
eventBasicDeliver.Properties = header.Properties
|
||||
default:
|
||||
frame = nil
|
||||
}
|
||||
|
||||
case *BodyFrame:
|
||||
// continue until terminated
|
||||
body = append(body, f.Body...)
|
||||
remaining -= len(f.Body)
|
||||
switch lastMethodFrameMessage.(type) {
|
||||
case *BasicPublish:
|
||||
@@ -119,9 +115,6 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
|
||||
eventBasicDeliver.Body = f.Body
|
||||
superIdentifier.Protocol = &protocol
|
||||
emitAMQP(*eventBasicDeliver, amqpRequest, basicMethodMap[60], connectionInfo, superTimer.CaptureTime, emitter)
|
||||
default:
|
||||
body = nil
|
||||
frame = nil
|
||||
}
|
||||
|
||||
case *MethodFrame:
|
||||
@@ -211,10 +204,6 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
|
||||
}
|
||||
superIdentifier.Protocol = &protocol
|
||||
emitAMQP(*eventConnectionClose, amqpRequest, connectionMethodMap[50], connectionInfo, superTimer.CaptureTime, emitter)
|
||||
|
||||
default:
|
||||
frame = nil
|
||||
|
||||
}
|
||||
|
||||
default:
|
||||
@@ -231,32 +220,24 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
|
||||
switch request["method"] {
|
||||
case basicMethodMap[40]:
|
||||
summary = reqDetails["exchange"].(string)
|
||||
break
|
||||
case basicMethodMap[60]:
|
||||
summary = reqDetails["exchange"].(string)
|
||||
break
|
||||
case exchangeMethodMap[10]:
|
||||
summary = reqDetails["exchange"].(string)
|
||||
break
|
||||
case queueMethodMap[10]:
|
||||
summary = reqDetails["queue"].(string)
|
||||
break
|
||||
case connectionMethodMap[10]:
|
||||
summary = fmt.Sprintf(
|
||||
"%s.%s",
|
||||
strconv.Itoa(int(reqDetails["versionMajor"].(float64))),
|
||||
strconv.Itoa(int(reqDetails["versionMinor"].(float64))),
|
||||
)
|
||||
break
|
||||
case connectionMethodMap[50]:
|
||||
summary = reqDetails["replyText"].(string)
|
||||
break
|
||||
case queueMethodMap[20]:
|
||||
summary = reqDetails["queue"].(string)
|
||||
break
|
||||
case basicMethodMap[20]:
|
||||
summary = reqDetails["queue"].(string)
|
||||
break
|
||||
}
|
||||
|
||||
request["url"] = summary
|
||||
@@ -288,33 +269,25 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
|
||||
|
||||
func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
|
||||
bodySize = 0
|
||||
representation := make(map[string]interface{}, 0)
|
||||
representation := make(map[string]interface{})
|
||||
var repRequest []interface{}
|
||||
switch request["method"].(string) {
|
||||
case basicMethodMap[40]:
|
||||
repRequest = representBasicPublish(request)
|
||||
break
|
||||
case basicMethodMap[60]:
|
||||
repRequest = representBasicDeliver(request)
|
||||
break
|
||||
case queueMethodMap[10]:
|
||||
repRequest = representQueueDeclare(request)
|
||||
break
|
||||
case exchangeMethodMap[10]:
|
||||
repRequest = representExchangeDeclare(request)
|
||||
break
|
||||
case connectionMethodMap[10]:
|
||||
repRequest = representConnectionStart(request)
|
||||
break
|
||||
case connectionMethodMap[50]:
|
||||
repRequest = representConnectionClose(request)
|
||||
break
|
||||
case queueMethodMap[20]:
|
||||
repRequest = representQueueBind(request)
|
||||
break
|
||||
case basicMethodMap[20]:
|
||||
repRequest = representBasicConsume(request)
|
||||
break
|
||||
}
|
||||
representation["request"] = repRequest
|
||||
object, err = json.Marshal(representation)
|
||||
|
||||
@@ -135,20 +135,6 @@ func readDecimal(r io.Reader) (v Decimal, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func readFloat32(r io.Reader) (v float32, err error) {
|
||||
if err = binary.Read(r, binary.BigEndian, &v); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func readFloat64(r io.Reader) (v float64, err error) {
|
||||
if err = binary.Read(r, binary.BigEndian, &v); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func readTimestamp(r io.Reader) (v time.Time, err error) {
|
||||
var sec int64
|
||||
if err = binary.Read(r, binary.BigEndian, &sec); err != nil {
|
||||
|
||||
@@ -78,15 +78,6 @@ type Error struct {
|
||||
Recover bool // true when this error can be recovered by retrying later or with different parameters
|
||||
}
|
||||
|
||||
func newError(code uint16, text string) *Error {
|
||||
return &Error{
|
||||
Code: int(code),
|
||||
Reason: text,
|
||||
Recover: isSoftExceptionCode(int(code)),
|
||||
Server: true,
|
||||
}
|
||||
}
|
||||
|
||||
func (e Error) Error() string {
|
||||
return fmt.Sprintf("Exception (%d) Reason: %q", e.Code, e.Reason)
|
||||
}
|
||||
@@ -262,19 +253,6 @@ func (t Table) Validate() error {
|
||||
return validateField(t)
|
||||
}
|
||||
|
||||
// Heap interface for maintaining delivery tags
|
||||
type tagSet []uint64
|
||||
|
||||
func (set tagSet) Len() int { return len(set) }
|
||||
func (set tagSet) Less(i, j int) bool { return (set)[i] < (set)[j] }
|
||||
func (set tagSet) Swap(i, j int) { (set)[i], (set)[j] = (set)[j], (set)[i] }
|
||||
func (set *tagSet) Push(tag interface{}) { *set = append(*set, tag.(uint64)) }
|
||||
func (set *tagSet) Pop() interface{} {
|
||||
val := (*set)[len(*set)-1]
|
||||
*set = (*set)[:len(*set)-1]
|
||||
return val
|
||||
}
|
||||
|
||||
type Message interface {
|
||||
id() (uint16, uint16)
|
||||
wait() bool
|
||||
@@ -282,12 +260,6 @@ type Message interface {
|
||||
write(io.Writer) error
|
||||
}
|
||||
|
||||
type messageWithContent interface {
|
||||
Message
|
||||
getContent() (Properties, []byte)
|
||||
setContent(Properties, []byte)
|
||||
}
|
||||
|
||||
/*
|
||||
The base interface implemented as:
|
||||
|
||||
@@ -322,22 +294,6 @@ type AmqpReader struct {
|
||||
R io.Reader
|
||||
}
|
||||
|
||||
type writer struct {
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
// Implements the frame interface for Connection RPC
|
||||
type protocolHeader struct{}
|
||||
|
||||
func (protocolHeader) write(w io.Writer) error {
|
||||
_, err := w.Write([]byte{'A', 'M', 'Q', 'P', 0, 0, 9, 1})
|
||||
return err
|
||||
}
|
||||
|
||||
func (protocolHeader) channel() uint16 {
|
||||
panic("only valid as initial handshake")
|
||||
}
|
||||
|
||||
/*
|
||||
Method frames carry the high-level protocol commands (which we call "methods").
|
||||
One method frame carries one command. The method frame payload has this format:
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
package amqp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
@@ -15,18 +14,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func (w *writer) WriteFrame(frame frame) (err error) {
|
||||
if err = frame.write(w.w); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if buf, ok := w.w.(*bufio.Writer); ok {
|
||||
err = buf.Flush()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (f *MethodFrame) write(w io.Writer) (err error) {
|
||||
var payload bytes.Buffer
|
||||
|
||||
@@ -412,5 +399,5 @@ func writeTable(w io.Writer, table Table) (err error) {
|
||||
}
|
||||
}
|
||||
|
||||
return writeLongstr(w, string(buf.Bytes()))
|
||||
return writeLongstr(w, buf.String())
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ func representSliceAsTable(slice []interface{}, selectorPrefix string) (represen
|
||||
selector := fmt.Sprintf("%s[%d]", selectorPrefix, i)
|
||||
table = append(table, api.TableData{
|
||||
Name: strconv.Itoa(i),
|
||||
Value: item.(interface{}),
|
||||
Value: item,
|
||||
Selector: selector,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -92,21 +92,30 @@ func (d dissecting) Ping() {
|
||||
}
|
||||
|
||||
func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
|
||||
isHTTP2, err := checkIsHTTP2Connection(b, isClient)
|
||||
var err error
|
||||
isHTTP2, _ := checkIsHTTP2Connection(b, isClient)
|
||||
|
||||
var http2Assembler *Http2Assembler
|
||||
if isHTTP2 {
|
||||
prepareHTTP2Connection(b, isClient)
|
||||
err = prepareHTTP2Connection(b, isClient)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
http2Assembler = createHTTP2Assembler(b)
|
||||
}
|
||||
|
||||
dissected := false
|
||||
switchingProtocolsHTTP2 := false
|
||||
for {
|
||||
if switchingProtocolsHTTP2 {
|
||||
switchingProtocolsHTTP2 = false
|
||||
isHTTP2, err = checkIsHTTP2Connection(b, isClient)
|
||||
prepareHTTP2Connection(b, isClient)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
err = prepareHTTP2Connection(b, isClient)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
http2Assembler = createHTTP2Assembler(b)
|
||||
}
|
||||
|
||||
@@ -121,7 +130,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
|
||||
} else if err != nil {
|
||||
continue
|
||||
}
|
||||
dissected = true
|
||||
superIdentifier.Protocol = &http11protocol
|
||||
} else if isClient {
|
||||
var req *http.Request
|
||||
switchingProtocolsHTTP2, req, err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options)
|
||||
@@ -130,7 +139,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
|
||||
} else if err != nil {
|
||||
continue
|
||||
}
|
||||
dissected = true
|
||||
superIdentifier.Protocol = &http11protocol
|
||||
|
||||
// In case of an HTTP2 upgrade, duplicate the HTTP1 request into HTTP2 with stream ID 1
|
||||
if switchingProtocolsHTTP2 {
|
||||
@@ -161,14 +170,14 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
|
||||
} else if err != nil {
|
||||
continue
|
||||
}
|
||||
dissected = true
|
||||
superIdentifier.Protocol = &http11protocol
|
||||
}
|
||||
}
|
||||
|
||||
if !dissected {
|
||||
if superIdentifier.Protocol == nil {
|
||||
return err
|
||||
}
|
||||
superIdentifier.Protocol = &http11protocol
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -341,11 +350,11 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
|
||||
})
|
||||
|
||||
postData, _ := request["postData"].(map[string]interface{})
|
||||
mimeType, _ := postData["mimeType"]
|
||||
mimeType := postData["mimeType"]
|
||||
if mimeType == nil || len(mimeType.(string)) == 0 {
|
||||
mimeType = "text/html"
|
||||
}
|
||||
text, _ := postData["text"]
|
||||
text := postData["text"]
|
||||
if text != nil {
|
||||
repRequest = append(repRequest, api.SectionData{
|
||||
Type: api.BODY,
|
||||
@@ -425,12 +434,12 @@ func representResponse(response map[string]interface{}) (repResponse []interface
|
||||
})
|
||||
|
||||
content, _ := response["content"].(map[string]interface{})
|
||||
mimeType, _ := content["mimeType"]
|
||||
mimeType := content["mimeType"]
|
||||
if mimeType == nil || len(mimeType.(string)) == 0 {
|
||||
mimeType = "text/html"
|
||||
}
|
||||
encoding, _ := content["encoding"]
|
||||
text, _ := content["text"]
|
||||
encoding := content["encoding"]
|
||||
text := content["text"]
|
||||
if text != nil {
|
||||
repResponse = append(repResponse, api.SectionData{
|
||||
Type: api.BODY,
|
||||
@@ -446,7 +455,7 @@ func representResponse(response map[string]interface{}) (repResponse []interface
|
||||
}
|
||||
|
||||
func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
|
||||
representation := make(map[string]interface{}, 0)
|
||||
representation := make(map[string]interface{})
|
||||
repRequest := representRequest(request)
|
||||
repResponse, bodySize := representResponse(response)
|
||||
representation["request"] = repRequest
|
||||
|
||||
@@ -89,7 +89,7 @@ func filterResponseBody(response *http.Response, options *api.TrafficFilteringOp
|
||||
}
|
||||
|
||||
func filterHeaders(headers *http.Header) {
|
||||
for key, _ := range *headers {
|
||||
for key := range *headers {
|
||||
if strings.ToLower(key) == userAgent {
|
||||
continue
|
||||
}
|
||||
@@ -103,7 +103,7 @@ func filterHeaders(headers *http.Header) {
|
||||
}
|
||||
|
||||
func getContentTypeHeaderValue(headers http.Header) string {
|
||||
for key, _ := range headers {
|
||||
for key := range headers {
|
||||
if strings.ToLower(key) == "content-type" {
|
||||
return headers.Get(key)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
@@ -201,25 +200,6 @@ func newPageBuffer() *pageBuffer {
|
||||
return b
|
||||
}
|
||||
|
||||
func (pb *pageBuffer) refTo(ref *pageRef, begin, end int64) {
|
||||
length := end - begin
|
||||
|
||||
if length > math.MaxUint32 {
|
||||
panic("reference to contiguous buffer pages exceeds the maximum size of 4 GB")
|
||||
}
|
||||
|
||||
ref.pages = append(ref.buffer[:0], pb.pages.slice(begin, end)...)
|
||||
ref.pages.ref()
|
||||
ref.offset = begin
|
||||
ref.length = uint32(length)
|
||||
}
|
||||
|
||||
func (pb *pageBuffer) ref(begin, end int64) *pageRef {
|
||||
ref := new(pageRef)
|
||||
pb.refTo(ref, begin, end)
|
||||
return ref
|
||||
}
|
||||
|
||||
func (pb *pageBuffer) unref() {
|
||||
pb.refc.unref(func() {
|
||||
pb.pages.unref()
|
||||
@@ -353,12 +333,12 @@ func (pb *pageBuffer) Write(b []byte) (int, error) {
|
||||
free := tail.Cap() - tail.Len()
|
||||
|
||||
if len(b) <= free {
|
||||
tail.Write(b)
|
||||
_, _ = tail.Write(b)
|
||||
pb.length += len(b)
|
||||
break
|
||||
}
|
||||
|
||||
tail.Write(b[:free])
|
||||
_, _ = tail.Write(b[:free])
|
||||
b = b[free:]
|
||||
|
||||
pb.length += free
|
||||
@@ -374,7 +354,7 @@ func (pb *pageBuffer) WriteAt(b []byte, off int64) (int, error) {
|
||||
return n, err
|
||||
}
|
||||
if n < len(b) {
|
||||
pb.Write(b[n:])
|
||||
_, _ = pb.Write(b[n:])
|
||||
}
|
||||
return len(b), nil
|
||||
}
|
||||
@@ -406,12 +386,6 @@ var (
|
||||
|
||||
type contiguousPages []*page
|
||||
|
||||
func (pages contiguousPages) ref() {
|
||||
for _, p := range pages {
|
||||
p.ref()
|
||||
}
|
||||
}
|
||||
|
||||
func (pages contiguousPages) unref() {
|
||||
for _, p := range pages {
|
||||
p.unref()
|
||||
@@ -480,7 +454,6 @@ var (
|
||||
)
|
||||
|
||||
type pageRef struct {
|
||||
buffer [2]*page
|
||||
pages contiguousPages
|
||||
offset int64
|
||||
cursor int64
|
||||
@@ -590,28 +563,6 @@ var (
|
||||
_ io.WriterTo = (*pageRef)(nil)
|
||||
)
|
||||
|
||||
type pageRefAllocator struct {
|
||||
refs []pageRef
|
||||
head int
|
||||
size int
|
||||
}
|
||||
|
||||
func (a *pageRefAllocator) newPageRef() *pageRef {
|
||||
if a.head == len(a.refs) {
|
||||
a.refs = make([]pageRef, a.size)
|
||||
a.head = 0
|
||||
}
|
||||
ref := &a.refs[a.head]
|
||||
a.head++
|
||||
return ref
|
||||
}
|
||||
|
||||
func unref(x interface{}) {
|
||||
if r, _ := x.(interface{ unref() }); r != nil {
|
||||
r.unref()
|
||||
}
|
||||
}
|
||||
|
||||
func seek(cursor, limit, offset int64, whence int) (int64, error) {
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
@@ -631,15 +582,3 @@ func seek(cursor, limit, offset int64, whence int) (int64, error) {
|
||||
}
|
||||
return offset, nil
|
||||
}
|
||||
|
||||
func closeBytes(b Bytes) {
|
||||
if b != nil {
|
||||
b.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func resetBytes(b Bytes) {
|
||||
if r, _ := b.(interface{ Reset() }); r != nil {
|
||||
r.Reset()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/segmentio/kafka-go/compress"
|
||||
)
|
||||
|
||||
@@ -12,19 +10,3 @@ type CompressionCodec = compress.Codec
|
||||
|
||||
// TODO: this file should probably go away once the internals of the package
|
||||
// have moved to use the protocol package.
|
||||
const (
|
||||
compressionCodecMask = 0x07
|
||||
)
|
||||
|
||||
var (
|
||||
errUnknownCodec = errors.New("the compression code is invalid or its codec has not been imported")
|
||||
)
|
||||
|
||||
// resolveCodec looks up a codec by Code()
|
||||
func resolveCodec(code int8) (CompressionCodec, error) {
|
||||
codec := compress.Compression(code).Codec()
|
||||
if codec == nil {
|
||||
return nil, errUnknownCodec
|
||||
}
|
||||
return codec, nil
|
||||
}
|
||||
|
||||
@@ -58,14 +58,6 @@ func (d *decoder) ReadByte() (byte, error) {
|
||||
return c, d.err
|
||||
}
|
||||
|
||||
func (d *decoder) done() bool {
|
||||
return d.remain == 0 || d.err != nil
|
||||
}
|
||||
|
||||
func (d *decoder) setCRC(table *crc32.Table) {
|
||||
d.table, d.crc32 = table, 0
|
||||
}
|
||||
|
||||
func (d *decoder) decodeBool(v value) {
|
||||
v.setBool(d.readBool())
|
||||
}
|
||||
@@ -199,19 +191,6 @@ func (d *decoder) read(n int) []byte {
|
||||
return b
|
||||
}
|
||||
|
||||
func (d *decoder) writeTo(w io.Writer, n int) {
|
||||
limit := d.remain
|
||||
if n < limit {
|
||||
d.remain = n
|
||||
}
|
||||
c, err := io.Copy(w, d)
|
||||
if int(c) < n && err == nil {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
d.remain = limit - int(c)
|
||||
d.setError(err)
|
||||
}
|
||||
|
||||
func (d *decoder) setError(err error) {
|
||||
if d.err == nil && err != nil {
|
||||
d.err = err
|
||||
@@ -272,14 +251,6 @@ func (d *decoder) readString() string {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) readVarString() string {
|
||||
if n := d.readVarInt(); n < 0 {
|
||||
return ""
|
||||
} else {
|
||||
return bytesToString(d.read(int(n)))
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) readCompactString() string {
|
||||
if n := d.readUnsignedVarInt(); n < 1 {
|
||||
return ""
|
||||
@@ -296,32 +267,6 @@ func (d *decoder) readBytes() []byte {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) readBytesTo(w io.Writer) bool {
|
||||
if n := d.readInt32(); n < 0 {
|
||||
return false
|
||||
} else {
|
||||
d.writeTo(w, int(n))
|
||||
return d.err == nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) readVarBytes() []byte {
|
||||
if n := d.readVarInt(); n < 0 {
|
||||
return nil
|
||||
} else {
|
||||
return d.read(int(n))
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) readVarBytesTo(w io.Writer) bool {
|
||||
if n := d.readVarInt(); n < 0 {
|
||||
return false
|
||||
} else {
|
||||
d.writeTo(w, int(n))
|
||||
return d.err == nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) readCompactBytes() []byte {
|
||||
if n := d.readUnsignedVarInt(); n < 1 {
|
||||
return nil
|
||||
@@ -330,15 +275,6 @@ func (d *decoder) readCompactBytes() []byte {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) readCompactBytesTo(w io.Writer) bool {
|
||||
if n := d.readUnsignedVarInt(); n < 1 {
|
||||
return false
|
||||
} else {
|
||||
d.writeTo(w, int(n-1))
|
||||
return d.err == nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) readVarInt() int64 {
|
||||
n := 11 // varints are at most 11 bytes
|
||||
|
||||
|
||||
@@ -14,37 +14,3 @@ func discardN(r *bufio.Reader, sz int, n int) (int, error) {
|
||||
}
|
||||
return sz - n, err
|
||||
}
|
||||
|
||||
func discardInt8(r *bufio.Reader, sz int) (int, error) {
|
||||
return discardN(r, sz, 1)
|
||||
}
|
||||
|
||||
func discardInt16(r *bufio.Reader, sz int) (int, error) {
|
||||
return discardN(r, sz, 2)
|
||||
}
|
||||
|
||||
func discardInt32(r *bufio.Reader, sz int) (int, error) {
|
||||
return discardN(r, sz, 4)
|
||||
}
|
||||
|
||||
func discardInt64(r *bufio.Reader, sz int) (int, error) {
|
||||
return discardN(r, sz, 8)
|
||||
}
|
||||
|
||||
func discardString(r *bufio.Reader, sz int) (int, error) {
|
||||
return readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (int, error) {
|
||||
if n < 0 {
|
||||
return sz, nil
|
||||
}
|
||||
return discardN(r, sz, n)
|
||||
})
|
||||
}
|
||||
|
||||
func discardBytes(r *bufio.Reader, sz int) (int, error) {
|
||||
return readBytesWith(r, sz, func(r *bufio.Reader, sz int, n int) (int, error) {
|
||||
if n < 0 {
|
||||
return sz, nil
|
||||
}
|
||||
return discardN(r, sz, n)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package kafka
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"reflect"
|
||||
@@ -95,10 +94,6 @@ func (e *encoder) WriteString(s string) (int, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (e *encoder) setCRC(table *crc32.Table) {
|
||||
e.table, e.crc32 = table, 0
|
||||
}
|
||||
|
||||
func (e *encoder) update(b []byte) {
|
||||
if e.table != nil {
|
||||
e.crc32 = crc32.Update(e.crc32, e.table, b)
|
||||
@@ -133,10 +128,6 @@ func (e *encoder) encodeString(v value) {
|
||||
e.writeString(v.string())
|
||||
}
|
||||
|
||||
func (e *encoder) encodeVarString(v value) {
|
||||
e.writeVarString(v.string())
|
||||
}
|
||||
|
||||
func (e *encoder) encodeCompactString(v value) {
|
||||
e.writeCompactString(v.string())
|
||||
}
|
||||
@@ -145,10 +136,6 @@ func (e *encoder) encodeNullString(v value) {
|
||||
e.writeNullString(v.string())
|
||||
}
|
||||
|
||||
func (e *encoder) encodeVarNullString(v value) {
|
||||
e.writeVarNullString(v.string())
|
||||
}
|
||||
|
||||
func (e *encoder) encodeCompactNullString(v value) {
|
||||
e.writeCompactNullString(v.string())
|
||||
}
|
||||
@@ -157,10 +144,6 @@ func (e *encoder) encodeBytes(v value) {
|
||||
e.writeBytes(v.bytes())
|
||||
}
|
||||
|
||||
func (e *encoder) encodeVarBytes(v value) {
|
||||
e.writeVarBytes(v.bytes())
|
||||
}
|
||||
|
||||
func (e *encoder) encodeCompactBytes(v value) {
|
||||
e.writeCompactBytes(v.bytes())
|
||||
}
|
||||
@@ -169,10 +152,6 @@ func (e *encoder) encodeNullBytes(v value) {
|
||||
e.writeNullBytes(v.bytes())
|
||||
}
|
||||
|
||||
func (e *encoder) encodeVarNullBytes(v value) {
|
||||
e.writeVarNullBytes(v.bytes())
|
||||
}
|
||||
|
||||
func (e *encoder) encodeCompactNullBytes(v value) {
|
||||
e.writeCompactNullBytes(v.bytes())
|
||||
}
|
||||
@@ -228,37 +207,32 @@ func (e *encoder) encodeCompactNullArray(v value, elemType reflect.Type, encodeE
|
||||
|
||||
func (e *encoder) writeInt8(i int8) {
|
||||
writeInt8(e.buffer[:1], i)
|
||||
e.Write(e.buffer[:1])
|
||||
_, _ = e.Write(e.buffer[:1])
|
||||
}
|
||||
|
||||
func (e *encoder) writeInt16(i int16) {
|
||||
writeInt16(e.buffer[:2], i)
|
||||
e.Write(e.buffer[:2])
|
||||
_, _ = e.Write(e.buffer[:2])
|
||||
}
|
||||
|
||||
func (e *encoder) writeInt32(i int32) {
|
||||
writeInt32(e.buffer[:4], i)
|
||||
e.Write(e.buffer[:4])
|
||||
_, _ = e.Write(e.buffer[:4])
|
||||
}
|
||||
|
||||
func (e *encoder) writeInt64(i int64) {
|
||||
writeInt64(e.buffer[:8], i)
|
||||
e.Write(e.buffer[:8])
|
||||
_, _ = e.Write(e.buffer[:8])
|
||||
}
|
||||
|
||||
func (e *encoder) writeString(s string) {
|
||||
e.writeInt16(int16(len(s)))
|
||||
e.WriteString(s)
|
||||
}
|
||||
|
||||
func (e *encoder) writeVarString(s string) {
|
||||
e.writeVarInt(int64(len(s)))
|
||||
e.WriteString(s)
|
||||
_, _ = e.WriteString(s)
|
||||
}
|
||||
|
||||
func (e *encoder) writeCompactString(s string) {
|
||||
e.writeUnsignedVarInt(uint64(len(s)) + 1)
|
||||
e.WriteString(s)
|
||||
_, _ = e.WriteString(s)
|
||||
}
|
||||
|
||||
func (e *encoder) writeNullString(s string) {
|
||||
@@ -266,16 +240,7 @@ func (e *encoder) writeNullString(s string) {
|
||||
e.writeInt16(-1)
|
||||
} else {
|
||||
e.writeInt16(int16(len(s)))
|
||||
e.WriteString(s)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *encoder) writeVarNullString(s string) {
|
||||
if s == "" {
|
||||
e.writeVarInt(-1)
|
||||
} else {
|
||||
e.writeVarInt(int64(len(s)))
|
||||
e.WriteString(s)
|
||||
_, _ = e.WriteString(s)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,23 +249,18 @@ func (e *encoder) writeCompactNullString(s string) {
|
||||
e.writeUnsignedVarInt(0)
|
||||
} else {
|
||||
e.writeUnsignedVarInt(uint64(len(s)) + 1)
|
||||
e.WriteString(s)
|
||||
_, _ = e.WriteString(s)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *encoder) writeBytes(b []byte) {
|
||||
e.writeInt32(int32(len(b)))
|
||||
e.Write(b)
|
||||
}
|
||||
|
||||
func (e *encoder) writeVarBytes(b []byte) {
|
||||
e.writeVarInt(int64(len(b)))
|
||||
e.Write(b)
|
||||
_, _ = e.Write(b)
|
||||
}
|
||||
|
||||
func (e *encoder) writeCompactBytes(b []byte) {
|
||||
e.writeUnsignedVarInt(uint64(len(b)) + 1)
|
||||
e.Write(b)
|
||||
_, _ = e.Write(b)
|
||||
}
|
||||
|
||||
func (e *encoder) writeNullBytes(b []byte) {
|
||||
@@ -308,16 +268,7 @@ func (e *encoder) writeNullBytes(b []byte) {
|
||||
e.writeInt32(-1)
|
||||
} else {
|
||||
e.writeInt32(int32(len(b)))
|
||||
e.Write(b)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *encoder) writeVarNullBytes(b []byte) {
|
||||
if b == nil {
|
||||
e.writeVarInt(-1)
|
||||
} else {
|
||||
e.writeVarInt(int64(len(b)))
|
||||
e.Write(b)
|
||||
_, _ = e.Write(b)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -326,69 +277,10 @@ func (e *encoder) writeCompactNullBytes(b []byte) {
|
||||
e.writeUnsignedVarInt(0)
|
||||
} else {
|
||||
e.writeUnsignedVarInt(uint64(len(b)) + 1)
|
||||
e.Write(b)
|
||||
_, _ = e.Write(b)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *encoder) writeBytesFrom(b Bytes) error {
|
||||
size := int64(b.Len())
|
||||
e.writeInt32(int32(size))
|
||||
n, err := io.Copy(e, b)
|
||||
if err == nil && n != size {
|
||||
err = fmt.Errorf("size of bytes does not match the number of bytes that were written (size=%d, written=%d): %w", size, n, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *encoder) writeNullBytesFrom(b Bytes) error {
|
||||
if b == nil {
|
||||
e.writeInt32(-1)
|
||||
return nil
|
||||
} else {
|
||||
size := int64(b.Len())
|
||||
e.writeInt32(int32(size))
|
||||
n, err := io.Copy(e, b)
|
||||
if err == nil && n != size {
|
||||
err = fmt.Errorf("size of nullable bytes does not match the number of bytes that were written (size=%d, written=%d): %w", size, n, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (e *encoder) writeVarNullBytesFrom(b Bytes) error {
|
||||
if b == nil {
|
||||
e.writeVarInt(-1)
|
||||
return nil
|
||||
} else {
|
||||
size := int64(b.Len())
|
||||
e.writeVarInt(size)
|
||||
n, err := io.Copy(e, b)
|
||||
if err == nil && n != size {
|
||||
err = fmt.Errorf("size of nullable bytes does not match the number of bytes that were written (size=%d, written=%d): %w", size, n, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (e *encoder) writeCompactNullBytesFrom(b Bytes) error {
|
||||
if b == nil {
|
||||
e.writeUnsignedVarInt(0)
|
||||
return nil
|
||||
} else {
|
||||
size := int64(b.Len())
|
||||
e.writeUnsignedVarInt(uint64(size + 1))
|
||||
n, err := io.Copy(e, b)
|
||||
if err == nil && n != size {
|
||||
err = fmt.Errorf("size of compact nullable bytes does not match the number of bytes that were written (size=%d, written=%d): %w", size, n, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (e *encoder) writeVarInt(i int64) {
|
||||
e.writeUnsignedVarInt(uint64((i << 1) ^ (i >> 63)))
|
||||
}
|
||||
|
||||
func (e *encoder) writeUnsignedVarInt(i uint64) {
|
||||
b := e.buffer[:]
|
||||
n := 0
|
||||
@@ -404,7 +296,7 @@ func (e *encoder) writeUnsignedVarInt(i uint64) {
|
||||
n++
|
||||
}
|
||||
|
||||
e.Write(b[:n])
|
||||
_, _ = e.Write(b[:n])
|
||||
}
|
||||
|
||||
type encodeFunc func(*encoder, value)
|
||||
@@ -530,7 +422,7 @@ func structEncodeFuncOf(typ reflect.Type, version int16, flexible bool) encodeFu
|
||||
se := &encoder{writer: buf}
|
||||
f.encode(se, v.fieldByIndex(f.index))
|
||||
e.writeUnsignedVarInt(uint64(buf.Len()))
|
||||
e.Write(buf.Bytes())
|
||||
_, _ = e.Write(buf.Bytes())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -346,7 +346,13 @@ func representProduceRequest(data map[string]interface{}) []interface{} {
|
||||
})
|
||||
|
||||
obj, err := oj.ParseString(string(partitionsJson))
|
||||
if err != nil {
|
||||
return rep
|
||||
}
|
||||
recordBatchPath, err := jp.ParseString(`partitionData.records.recordBatch`)
|
||||
if err != nil {
|
||||
return rep
|
||||
}
|
||||
recordBatchresults := recordBatchPath.Get(obj)
|
||||
if len(recordBatchresults) > 0 {
|
||||
rep = append(rep, api.SectionData{
|
||||
@@ -357,6 +363,9 @@ func representProduceRequest(data map[string]interface{}) []interface{} {
|
||||
}
|
||||
|
||||
recordsPath, err := jp.ParseString(`partitionData.records.recordBatch.record`)
|
||||
if err != nil {
|
||||
return rep
|
||||
}
|
||||
recordsResults := recordsPath.Get(obj)
|
||||
if len(recordsResults) > 0 {
|
||||
records := recordsResults[0].([]interface{})
|
||||
|
||||
@@ -81,10 +81,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
|
||||
if len(summary) > 0 {
|
||||
summary = summary[:len(summary)-2]
|
||||
}
|
||||
break
|
||||
case ApiVersions:
|
||||
summary = reqDetails["clientID"].(string)
|
||||
break
|
||||
case Produce:
|
||||
_topics := reqDetails["payload"].(map[string]interface{})["topicData"]
|
||||
if _topics == nil {
|
||||
@@ -97,7 +95,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
|
||||
if len(summary) > 0 {
|
||||
summary = summary[:len(summary)-2]
|
||||
}
|
||||
break
|
||||
case Fetch:
|
||||
_topics := reqDetails["payload"].(map[string]interface{})["topics"]
|
||||
if _topics == nil {
|
||||
@@ -110,7 +107,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
|
||||
if len(summary) > 0 {
|
||||
summary = summary[:len(summary)-2]
|
||||
}
|
||||
break
|
||||
case ListOffsets:
|
||||
_topics := reqDetails["payload"].(map[string]interface{})["topics"]
|
||||
if _topics == nil {
|
||||
@@ -123,7 +119,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
|
||||
if len(summary) > 0 {
|
||||
summary = summary[:len(summary)-2]
|
||||
}
|
||||
break
|
||||
case CreateTopics:
|
||||
topics := reqDetails["payload"].(map[string]interface{})["topics"].([]interface{})
|
||||
for _, topic := range topics {
|
||||
@@ -132,13 +127,11 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
|
||||
if len(summary) > 0 {
|
||||
summary = summary[:len(summary)-2]
|
||||
}
|
||||
break
|
||||
case DeleteTopics:
|
||||
topicNames := reqDetails["topicNames"].([]string)
|
||||
for _, name := range topicNames {
|
||||
summary += fmt.Sprintf("%s, ", name)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
request["url"] = summary
|
||||
@@ -173,7 +166,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
|
||||
|
||||
func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
|
||||
bodySize = 0
|
||||
representation := make(map[string]interface{}, 0)
|
||||
representation := make(map[string]interface{})
|
||||
|
||||
apiKey := ApiKey(request["apiKey"].(float64))
|
||||
|
||||
@@ -183,31 +176,24 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
|
||||
case Metadata:
|
||||
repRequest = representMetadataRequest(request)
|
||||
repResponse = representMetadataResponse(response)
|
||||
break
|
||||
case ApiVersions:
|
||||
repRequest = representApiVersionsRequest(request)
|
||||
repResponse = representApiVersionsResponse(response)
|
||||
break
|
||||
case Produce:
|
||||
repRequest = representProduceRequest(request)
|
||||
repResponse = representProduceResponse(response)
|
||||
break
|
||||
case Fetch:
|
||||
repRequest = representFetchRequest(request)
|
||||
repResponse = representFetchResponse(response)
|
||||
break
|
||||
case ListOffsets:
|
||||
repRequest = representListOffsetsRequest(request)
|
||||
repResponse = representListOffsetsResponse(response)
|
||||
break
|
||||
case CreateTopics:
|
||||
repRequest = representCreateTopicsRequest(request)
|
||||
repResponse = representCreateTopicsResponse(response)
|
||||
break
|
||||
case DeleteTopics:
|
||||
repRequest = representDeleteTopicsRequest(request)
|
||||
repResponse = representDeleteTopicsResponse(response)
|
||||
break
|
||||
}
|
||||
|
||||
representation["request"] = repRequest
|
||||
|
||||
@@ -26,9 +26,9 @@ func CreateResponseRequestMatcher() requestResponseMatcher {
|
||||
func (matcher *requestResponseMatcher) registerRequest(key string, request *Request) *RequestResponsePair {
|
||||
if response, found := matcher.openMessagesMap.LoadAndDelete(key); found {
|
||||
// Check for a situation that only occurs when a Kafka broker is initiating
|
||||
switch response.(type) {
|
||||
switch v := response.(type) {
|
||||
case *Response:
|
||||
return matcher.preparePair(request, response.(*Response))
|
||||
return matcher.preparePair(request, v)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -167,10 +167,6 @@ type messageType struct {
|
||||
encode encodeFunc
|
||||
}
|
||||
|
||||
func (t *messageType) new() Message {
|
||||
return reflect.New(t.gotype).Interface().(Message)
|
||||
}
|
||||
|
||||
type apiType struct {
|
||||
requests []messageType
|
||||
responses []messageType
|
||||
@@ -389,16 +385,16 @@ func (b Broker) String() string {
|
||||
func (b Broker) Format(w fmt.State, v rune) {
|
||||
switch v {
|
||||
case 'd':
|
||||
io.WriteString(w, itoa(b.ID))
|
||||
_, _ = io.WriteString(w, itoa(b.ID))
|
||||
case 's':
|
||||
io.WriteString(w, b.String())
|
||||
_, _ = io.WriteString(w, b.String())
|
||||
case 'v':
|
||||
io.WriteString(w, itoa(b.ID))
|
||||
io.WriteString(w, " ")
|
||||
io.WriteString(w, b.String())
|
||||
_, _ = io.WriteString(w, itoa(b.ID))
|
||||
_, _ = io.WriteString(w, " ")
|
||||
_, _ = io.WriteString(w, b.String())
|
||||
if b.Rack != "" {
|
||||
io.WriteString(w, " ")
|
||||
io.WriteString(w, b.Rack)
|
||||
_, _ = io.WriteString(w, " ")
|
||||
_, _ = io.WriteString(w, b.Rack)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,10 +99,8 @@ func (k apiKey) String() string {
|
||||
return strconv.Itoa(int(k))
|
||||
}
|
||||
|
||||
type apiVersion int16
|
||||
|
||||
const (
|
||||
v0 = 0
|
||||
// v0 = 0
|
||||
v1 = 1
|
||||
v2 = 2
|
||||
v3 = 3
|
||||
@@ -113,6 +111,7 @@ const (
|
||||
v8 = 8
|
||||
v9 = 9
|
||||
v10 = 10
|
||||
v11 = 11
|
||||
)
|
||||
|
||||
var apiKeyStrings = [...]string{
|
||||
@@ -166,35 +165,6 @@ var apiKeyStrings = [...]string{
|
||||
offsetDelete: "OffsetDelete",
|
||||
}
|
||||
|
||||
type requestHeader struct {
|
||||
Size int32
|
||||
ApiKey int16
|
||||
ApiVersion int16
|
||||
CorrelationID int32
|
||||
ClientID string
|
||||
}
|
||||
|
||||
func sizeofString(s string) int32 {
|
||||
return 2 + int32(len(s))
|
||||
}
|
||||
|
||||
func (h requestHeader) size() int32 {
|
||||
return 4 + 2 + 2 + 4 + sizeofString(h.ClientID)
|
||||
}
|
||||
|
||||
// func (h requestHeader) writeTo(wb *writeBuffer) {
|
||||
// wb.writeInt32(h.Size)
|
||||
// wb.writeInt16(h.ApiKey)
|
||||
// wb.writeInt16(h.ApiVersion)
|
||||
// wb.writeInt32(h.CorrelationID)
|
||||
// wb.writeString(h.ClientID)
|
||||
// }
|
||||
|
||||
type request interface {
|
||||
size() int32
|
||||
// writable
|
||||
}
|
||||
|
||||
func makeInt8(b []byte) int8 {
|
||||
return int8(b[0])
|
||||
}
|
||||
@@ -210,10 +180,3 @@ func makeInt32(b []byte) int32 {
|
||||
func makeInt64(b []byte) int64 {
|
||||
return int64(binary.BigEndian.Uint64(b))
|
||||
}
|
||||
|
||||
func expectZeroSize(sz int, err error) error {
|
||||
if err == nil && sz != 0 {
|
||||
err = fmt.Errorf("reading a response left %d unread bytes", sz)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
type readable interface {
|
||||
@@ -42,59 +41,6 @@ func readInt64(r *bufio.Reader, sz int, v *int64) (int, error) {
|
||||
return peekRead(r, sz, 8, func(b []byte) { *v = makeInt64(b) })
|
||||
}
|
||||
|
||||
func readVarInt(r *bufio.Reader, sz int, v *int64) (remain int, err error) {
|
||||
// Optimistically assume that most of the time, there will be data buffered
|
||||
// in the reader. If this is not the case, the buffer will be refilled after
|
||||
// consuming zero bytes from the input.
|
||||
input, _ := r.Peek(r.Buffered())
|
||||
x := uint64(0)
|
||||
s := uint(0)
|
||||
|
||||
for {
|
||||
if len(input) > sz {
|
||||
input = input[:sz]
|
||||
}
|
||||
|
||||
for i, b := range input {
|
||||
if b < 0x80 {
|
||||
x |= uint64(b) << s
|
||||
*v = int64(x>>1) ^ -(int64(x) & 1)
|
||||
n, err := r.Discard(i + 1)
|
||||
return sz - n, err
|
||||
}
|
||||
|
||||
x |= uint64(b&0x7f) << s
|
||||
s += 7
|
||||
}
|
||||
|
||||
// Make room in the input buffer to load more data from the underlying
|
||||
// stream. The x and s variables are left untouched, ensuring that the
|
||||
// varint decoding can continue on the next loop iteration.
|
||||
n, _ := r.Discard(len(input))
|
||||
sz -= n
|
||||
if sz == 0 {
|
||||
return 0, errShortRead
|
||||
}
|
||||
|
||||
// Fill the buffer: ask for one more byte, but in practice the reader
|
||||
// will load way more from the underlying stream.
|
||||
if _, err := r.Peek(1); err != nil {
|
||||
if err == io.EOF {
|
||||
err = errShortRead
|
||||
}
|
||||
return sz, err
|
||||
}
|
||||
|
||||
// Grab as many bytes as possible from the buffer, then go on to the
|
||||
// next loop iteration which is going to consume it.
|
||||
input, _ = r.Peek(r.Buffered())
|
||||
}
|
||||
}
|
||||
|
||||
func readBool(r *bufio.Reader, sz int, v *bool) (int, error) {
|
||||
return peekRead(r, sz, 1, func(b []byte) { *v = b[0] != 0 })
|
||||
}
|
||||
|
||||
func readString(r *bufio.Reader, sz int, v *string) (int, error) {
|
||||
return readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) {
|
||||
*v, remain, err = readNewString(r, sz, n)
|
||||
@@ -179,102 +125,6 @@ func readArrayLen(r *bufio.Reader, sz int, n *int) (int, error) {
|
||||
return sz, nil
|
||||
}
|
||||
|
||||
func readArrayWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int) (int, error)) (int, error) {
|
||||
var err error
|
||||
var len int32
|
||||
|
||||
if sz, err = readInt32(r, sz, &len); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
|
||||
for n := int(len); n > 0; n-- {
|
||||
if sz, err = cb(r, sz); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return sz, err
|
||||
}
|
||||
|
||||
func readStringArray(r *bufio.Reader, sz int, v *[]string) (remain int, err error) {
|
||||
var content []string
|
||||
fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
|
||||
var value string
|
||||
if fnRemain, fnErr = readString(r, size, &value); fnErr != nil {
|
||||
return
|
||||
}
|
||||
content = append(content, value)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, sz, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
*v = content
|
||||
return
|
||||
}
|
||||
|
||||
func readMapStringInt32(r *bufio.Reader, sz int, v *map[string][]int32) (remain int, err error) {
|
||||
var len int32
|
||||
if remain, err = readInt32(r, sz, &len); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
content := make(map[string][]int32, len)
|
||||
for i := 0; i < int(len); i++ {
|
||||
var key string
|
||||
var values []int32
|
||||
|
||||
if remain, err = readString(r, remain, &key); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
|
||||
var value int32
|
||||
if fnRemain, fnErr = readInt32(r, size, &value); fnErr != nil {
|
||||
return
|
||||
}
|
||||
values = append(values, value)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, remain, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
content[key] = values
|
||||
}
|
||||
*v = content
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func read(r *bufio.Reader, sz int, a interface{}) (int, error) {
|
||||
switch v := a.(type) {
|
||||
case *int8:
|
||||
return readInt8(r, sz, v)
|
||||
case *int16:
|
||||
return readInt16(r, sz, v)
|
||||
case *int32:
|
||||
return readInt32(r, sz, v)
|
||||
case *int64:
|
||||
return readInt64(r, sz, v)
|
||||
case *bool:
|
||||
return readBool(r, sz, v)
|
||||
case *string:
|
||||
return readString(r, sz, v)
|
||||
case *[]byte:
|
||||
return readBytes(r, sz, v)
|
||||
}
|
||||
switch v := reflect.ValueOf(a).Elem(); v.Kind() {
|
||||
case reflect.Struct:
|
||||
return readStruct(r, sz, v)
|
||||
case reflect.Slice:
|
||||
return readSlice(r, sz, v)
|
||||
default:
|
||||
panic(fmt.Sprintf("unsupported type: %T", a))
|
||||
}
|
||||
}
|
||||
|
||||
func ReadAll(r *bufio.Reader, sz int, ptrs ...interface{}) (int, error) {
|
||||
var err error
|
||||
|
||||
@@ -307,333 +157,3 @@ func readPtr(r *bufio.Reader, sz int, ptr interface{}) (int, error) {
|
||||
panic(fmt.Sprintf("unsupported type: %T", v))
|
||||
}
|
||||
}
|
||||
|
||||
func readStruct(r *bufio.Reader, sz int, v reflect.Value) (int, error) {
|
||||
var err error
|
||||
for i, n := 0, v.NumField(); i != n; i++ {
|
||||
if sz, err = read(r, sz, v.Field(i).Addr().Interface()); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
}
|
||||
return sz, nil
|
||||
}
|
||||
|
||||
func readSlice(r *bufio.Reader, sz int, v reflect.Value) (int, error) {
|
||||
var err error
|
||||
var len int32
|
||||
|
||||
if sz, err = readInt32(r, sz, &len); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
|
||||
if n := int(len); n < 0 {
|
||||
v.Set(reflect.Zero(v.Type()))
|
||||
} else {
|
||||
v.Set(reflect.MakeSlice(v.Type(), n, n))
|
||||
|
||||
for i := 0; i != n; i++ {
|
||||
if sz, err = read(r, sz, v.Index(i).Addr().Interface()); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return sz, nil
|
||||
}
|
||||
|
||||
func readFetchResponseHeaderV2(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {
|
||||
var n int32
|
||||
var p struct {
|
||||
Partition int32
|
||||
ErrorCode int16
|
||||
HighwaterMarkOffset int64
|
||||
MessageSetSize int32
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, size, &throttle); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, remain, &n); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if n != 1 {
|
||||
err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n)
|
||||
return
|
||||
}
|
||||
|
||||
// We ignore the topic name because we've requests messages for a single
|
||||
// topic, unless there's a bug in the kafka server we will have received
|
||||
// the name of the topic that we requested.
|
||||
if remain, err = discardString(r, remain); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, remain, &n); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if n != 1 {
|
||||
err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n)
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = read(r, remain, &p); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if p.ErrorCode != 0 {
|
||||
err = Error(p.ErrorCode)
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if remain != int(p.MessageSetSize) {
|
||||
err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", p.MessageSetSize, remain)
|
||||
return
|
||||
}
|
||||
|
||||
watermark = p.HighwaterMarkOffset
|
||||
return
|
||||
}
|
||||
|
||||
func readFetchResponseHeaderV5(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {
|
||||
var n int32
|
||||
type AbortedTransaction struct {
|
||||
ProducerId int64
|
||||
FirstOffset int64
|
||||
}
|
||||
var p struct {
|
||||
Partition int32
|
||||
ErrorCode int16
|
||||
HighwaterMarkOffset int64
|
||||
LastStableOffset int64
|
||||
LogStartOffset int64
|
||||
}
|
||||
var messageSetSize int32
|
||||
var abortedTransactions []AbortedTransaction
|
||||
|
||||
if remain, err = readInt32(r, size, &throttle); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, remain, &n); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if n != 1 {
|
||||
err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n)
|
||||
return
|
||||
}
|
||||
|
||||
// We ignore the topic name because we've requests messages for a single
|
||||
// topic, unless there's a bug in the kafka server we will have received
|
||||
// the name of the topic that we requested.
|
||||
if remain, err = discardString(r, remain); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, remain, &n); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if n != 1 {
|
||||
err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n)
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = read(r, remain, &p); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var abortedTransactionLen int
|
||||
if remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if abortedTransactionLen == -1 {
|
||||
abortedTransactions = nil
|
||||
} else {
|
||||
abortedTransactions = make([]AbortedTransaction, abortedTransactionLen)
|
||||
for i := 0; i < abortedTransactionLen; i++ {
|
||||
if remain, err = read(r, remain, &abortedTransactions[i]); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if p.ErrorCode != 0 {
|
||||
err = Error(p.ErrorCode)
|
||||
return
|
||||
}
|
||||
|
||||
remain, err = readInt32(r, remain, &messageSetSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if remain != int(messageSetSize) {
|
||||
err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", messageSetSize, remain)
|
||||
return
|
||||
}
|
||||
|
||||
watermark = p.HighwaterMarkOffset
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func readFetchResponseHeaderV10(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {
|
||||
var n int32
|
||||
var errorCode int16
|
||||
type AbortedTransaction struct {
|
||||
ProducerId int64
|
||||
FirstOffset int64
|
||||
}
|
||||
var p struct {
|
||||
Partition int32
|
||||
ErrorCode int16
|
||||
HighwaterMarkOffset int64
|
||||
LastStableOffset int64
|
||||
LogStartOffset int64
|
||||
}
|
||||
var messageSetSize int32
|
||||
var abortedTransactions []AbortedTransaction
|
||||
|
||||
if remain, err = readInt32(r, size, &throttle); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt16(r, remain, &errorCode); err != nil {
|
||||
return
|
||||
}
|
||||
if errorCode != 0 {
|
||||
err = Error(errorCode)
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = discardInt32(r, remain); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, remain, &n); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if n != 1 {
|
||||
err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n)
|
||||
return
|
||||
}
|
||||
|
||||
// We ignore the topic name because we've requests messages for a single
|
||||
// topic, unless there's a bug in the kafka server we will have received
|
||||
// the name of the topic that we requested.
|
||||
if remain, err = discardString(r, remain); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, remain, &n); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if n != 1 {
|
||||
err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n)
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = read(r, remain, &p); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var abortedTransactionLen int
|
||||
if remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if abortedTransactionLen == -1 {
|
||||
abortedTransactions = nil
|
||||
} else {
|
||||
abortedTransactions = make([]AbortedTransaction, abortedTransactionLen)
|
||||
for i := 0; i < abortedTransactionLen; i++ {
|
||||
if remain, err = read(r, remain, &abortedTransactions[i]); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if p.ErrorCode != 0 {
|
||||
err = Error(p.ErrorCode)
|
||||
return
|
||||
}
|
||||
|
||||
remain, err = readInt32(r, remain, &messageSetSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if remain != int(messageSetSize) {
|
||||
err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", messageSetSize, remain)
|
||||
return
|
||||
}
|
||||
|
||||
watermark = p.HighwaterMarkOffset
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func readMessageHeader(r *bufio.Reader, sz int) (offset int64, attributes int8, timestamp int64, remain int, err error) {
|
||||
var version int8
|
||||
|
||||
if remain, err = readInt64(r, sz, &offset); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// On discarding the message size and CRC:
|
||||
// ---------------------------------------
|
||||
//
|
||||
// - Not sure why kafka gives the message size here, we already have the
|
||||
// number of remaining bytes in the response and kafka should only truncate
|
||||
// the trailing message.
|
||||
//
|
||||
// - TCP is already taking care of ensuring data integrity, no need to
|
||||
// waste resources doing it a second time so we just skip the message CRC.
|
||||
//
|
||||
if remain, err = discardN(r, remain, 8); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt8(r, remain, &version); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt8(r, remain, &attributes); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch version {
|
||||
case 0:
|
||||
case 1:
|
||||
remain, err = readInt64(r, remain, ×tamp)
|
||||
default:
|
||||
err = fmt.Errorf("unsupported message version %d found in fetch response", version)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -110,25 +110,6 @@ type RecordSet struct {
|
||||
Records RecordReader
|
||||
}
|
||||
|
||||
// bufferedReader is an interface implemented by types like bufio.Reader, which
|
||||
// we use to optimize prefix reads by accessing the internal buffer directly
|
||||
// through calls to Peek.
|
||||
type bufferedReader interface {
|
||||
Discard(int) (int, error)
|
||||
Peek(int) ([]byte, error)
|
||||
}
|
||||
|
||||
// bytesBuffer is an interface implemented by types like bytes.Buffer, which we
|
||||
// use to optimize prefix reads by accessing the internal buffer directly
|
||||
// through calls to Bytes.
|
||||
type bytesBuffer interface {
|
||||
Bytes() []byte
|
||||
}
|
||||
|
||||
// magicByteOffset is the position of the magic byte in all versions of record
|
||||
// sets in the kafka protocol.
|
||||
const magicByteOffset = 16
|
||||
|
||||
// ReadFrom reads the representation of a record set from r into rs, returning
|
||||
// the number of bytes consumed from r, and an non-nil error if the record set
|
||||
// could not be read.
|
||||
@@ -292,23 +273,7 @@ func (rs *RecordSet) WriteTo(w io.Writer) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func makeTime(t int64) time.Time {
|
||||
return time.Unix(t/1000, (t%1000)*int64(time.Millisecond))
|
||||
}
|
||||
|
||||
func timestamp(t time.Time) int64 {
|
||||
if t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.UnixNano() / int64(time.Millisecond)
|
||||
}
|
||||
|
||||
func packUint32(u uint32) (b [4]byte) {
|
||||
binary.BigEndian.PutUint32(b[:], u)
|
||||
return
|
||||
}
|
||||
|
||||
func packUint64(u uint64) (b [8]byte) {
|
||||
binary.BigEndian.PutUint64(b[:], u)
|
||||
return
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user