Compare commits

...

6 Commits

Author SHA1 Message Date
Igor Gov
a4f7e61a6e Fix proxy retries (#718) 2022-01-30 12:13:06 +02:00
RoyUP9
a5fef90781 Small fix in check resources (#716) 2022-01-30 09:36:22 +02:00
Igor Gov
70cef9dc4b Fix: show agent-image in config after generation (#717) 2022-01-30 09:25:56 +02:00
Igor Gov
0f3dd66d2d Experimental feature: elastic exporter (#713) 2022-01-30 09:22:13 +02:00
Igor Gov
5536e5bb44 Fixing minor bugs and remove unused dependency (#714) 2022-01-30 08:51:17 +02:00
M. Mert Yıldıran
3bab83754f Fix the interface conversion and index out of range errors in the Redis dissector (#710) 2022-01-27 22:58:43 +03:00
14 changed files with 213 additions and 61 deletions

View File

@@ -6,13 +6,13 @@ require (
github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
github.com/chanced/openapi v0.0.6
github.com/djherbis/atime v1.0.0
github.com/elastic/go-elasticsearch/v7 v7.16.0
github.com/getkin/kin-openapi v0.76.0
github.com/gin-contrib/static v0.0.1
github.com/gin-gonic/gin v1.7.7
github.com/go-playground/locales v0.13.0
github.com/go-playground/universal-translator v0.17.0
github.com/go-playground/validator/v10 v10.5.0
github.com/google/martian v2.1.0+incompatible
github.com/google/uuid v1.1.2
github.com/gorilla/websocket v1.4.2
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7

View File

@@ -126,6 +126,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/elastic/go-elasticsearch/v7 v7.16.0 h1:GHsxDFXIAlhSleXun4kwA89P7kQFADRChqvgOPeYP5A=
github.com/elastic/go-elasticsearch/v7 v7.16.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=

View File

@@ -9,6 +9,7 @@ import (
"mizuserver/pkg/api"
"mizuserver/pkg/config"
"mizuserver/pkg/controllers"
"mizuserver/pkg/elastic"
"mizuserver/pkg/middlewares"
"mizuserver/pkg/models"
"mizuserver/pkg/oas"
@@ -159,6 +160,7 @@ func enableExpFeatureIfNeeded() {
if config.Config.ServiceMap {
servicemap.GetInstance().SetConfig(config.Config)
}
elastic.GetInstance().Configure(config.Config.Elastic)
}
func configureBasenineServer(host string, port string) {

View File

@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"mizuserver/pkg/elastic"
"mizuserver/pkg/har"
"mizuserver/pkg/holder"
"mizuserver/pkg/providers"
@@ -16,14 +17,15 @@ import (
"mizuserver/pkg/servicemap"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api"
"mizuserver/pkg/models"
"mizuserver/pkg/oas"
"mizuserver/pkg/resolver"
"mizuserver/pkg/utils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api"
basenine "github.com/up9inc/basenine/client/go"
)
@@ -150,6 +152,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
connection.SendText(string(data))
servicemap.GetInstance().NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
elastic.GetInstance().PushEntry(mizuEntry)
}
}

View File

@@ -0,0 +1,120 @@
package elastic
import (
"bytes"
"crypto/tls"
"encoding/json"
"github.com/elastic/go-elasticsearch/v7"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/tap/api"
"net/http"
"sync"
"time"
)
type client struct {
es *elasticsearch.Client
index string
insertedCount int
}
var instance *client
var once sync.Once
func GetInstance() *client {
once.Do(func() {
instance = newClient()
})
return instance
}
func (client *client) Configure(config shared.ElasticConfig) {
if config.Url == "" || config.User == "" || config.Password == "" {
logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
return
}
transport := http.DefaultTransport
tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
transport.(*http.Transport).TLSClientConfig = tlsClientConfig
cfg := elasticsearch.Config{
Addresses: []string{config.Url},
Username: config.User,
Password: config.Password,
Transport: transport,
}
es, err := elasticsearch.NewClient(cfg)
if err != nil {
logger.Log.Fatalf("Failed to initialize elastic client %v", err)
}
// Have the client instance return a response
res, err := es.Info()
if err != nil {
logger.Log.Fatalf("Elastic client.Info() ERROR: %v", err)
} else {
client.es = es
client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
client.insertedCount = 0
logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
}
defer res.Body.Close()
}
func newClient() *client {
return &client{
es: nil,
index: "",
}
}
type httpEntry struct {
Source *api.TCP `json:"src"`
Destination *api.TCP `json:"dst"`
Outgoing bool `json:"outgoing"`
CreatedAt time.Time `json:"createdAt"`
Request map[string]interface{} `json:"request"`
Response map[string]interface{} `json:"response"`
Summary string `json:"summary"`
Method string `json:"method"`
Status int `json:"status"`
ElapsedTime int64 `json:"elapsedTime"`
Path string `json:"path"`
}
func (client *client) PushEntry(entry *api.Entry) {
if client.es == nil {
return
}
if entry.Protocol.Name != "http" {
return
}
entryToPush := httpEntry{
Source: entry.Source,
Destination: entry.Destination,
Outgoing: entry.Outgoing,
CreatedAt: entry.StartTime,
Request: entry.Request,
Response: entry.Response,
Summary: entry.Summary,
Method: entry.Method,
Status: entry.Status,
ElapsedTime: entry.ElapsedTime,
Path: entry.Path,
}
entryJson, err := json.Marshal(entryToPush)
if err != nil {
logger.Log.Errorf("json.Marshal ERROR: %v", err)
return
}
var buffer bytes.Buffer
buffer.WriteString(string(entryJson))
res, _ := client.es.Index(client.index, &buffer)
if res.StatusCode == 201 {
client.insertedCount += 1
}
}

View File

@@ -23,8 +23,8 @@ type Provider struct {
client *http.Client
}
const DefaultRetries = 20
const DefaultTimeout = 5 * time.Second
const DefaultRetries = 3
const DefaultTimeout = 2 * time.Second
func NewProvider(url string, retries int, timeout time.Duration) *Provider {
return &Provider{
@@ -36,16 +36,6 @@ func NewProvider(url string, retries int, timeout time.Duration) *Provider {
}
}
func NewProviderWithoutRetries(url string, timeout time.Duration) *Provider {
return &Provider{
url: url,
retries: 1,
client: &http.Client{
Timeout: timeout,
},
}
}
func (provider *Provider) TestConnection() error {
retriesLeft := provider.retries
for retriesLeft > 0 {

View File

@@ -3,13 +3,14 @@ package cmd
import (
"context"
"fmt"
"regexp"
"github.com/up9inc/mizu/cli/apiserver"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared/kubernetes"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/shared/semver"
"regexp"
)
func runMizuCheck() {
@@ -96,7 +97,7 @@ func checkServerConnection(kubernetesProvider *kubernetes.Provider) bool {
serverUrl := GetApiServerUrl()
apiServerProvider := apiserver.NewProviderWithoutRetries(serverUrl, apiserver.DefaultTimeout)
apiServerProvider := apiserver.NewProvider(serverUrl, 1, apiserver.DefaultTimeout)
if err := apiServerProvider.TestConnection(); err == nil {
logger.Log.Infof("%v found Mizu server tunnel available and connected successfully to API server", fmt.Sprintf(uiUtils.Green, "√"))
return true
@@ -169,32 +170,32 @@ func checkAllResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.
allResourcesExist := checkResourceExist(config.Config.MizuResourcesNamespace, "namespace", exist, err)
exist, err = kubernetesProvider.DoesConfigMapExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName)
allResourcesExist = checkResourceExist(kubernetes.ConfigMapName, "config map", exist, err)
allResourcesExist = checkResourceExist(kubernetes.ConfigMapName, "config map", exist, err) && allResourcesExist
exist, err = kubernetesProvider.DoesServiceAccountExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName)
allResourcesExist = checkResourceExist(kubernetes.ServiceAccountName, "service account", exist, err)
allResourcesExist = checkResourceExist(kubernetes.ServiceAccountName, "service account", exist, err) && allResourcesExist
if config.Config.IsNsRestrictedMode() {
exist, err = kubernetesProvider.DoesRoleExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleName)
allResourcesExist = checkResourceExist(kubernetes.RoleName, "role", exist, err)
allResourcesExist = checkResourceExist(kubernetes.RoleName, "role", exist, err) && allResourcesExist
exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleBindingName)
allResourcesExist = checkResourceExist(kubernetes.RoleBindingName, "role binding", exist, err)
allResourcesExist = checkResourceExist(kubernetes.RoleBindingName, "role binding", exist, err) && allResourcesExist
} else {
exist, err = kubernetesProvider.DoesClusterRoleExist(ctx, kubernetes.ClusterRoleName)
allResourcesExist = checkResourceExist(kubernetes.ClusterRoleName, "cluster role", exist, err)
allResourcesExist = checkResourceExist(kubernetes.ClusterRoleName, "cluster role", exist, err) && allResourcesExist
exist, err = kubernetesProvider.DoesClusterRoleBindingExist(ctx, kubernetes.ClusterRoleBindingName)
allResourcesExist = checkResourceExist(kubernetes.ClusterRoleBindingName, "cluster role binding", exist, err)
allResourcesExist = checkResourceExist(kubernetes.ClusterRoleBindingName, "cluster role binding", exist, err) && allResourcesExist
}
exist, err = kubernetesProvider.DoesServiceExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
allResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "service", exist, err)
allResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "service", exist, err) && allResourcesExist
if isInstallCommand {
allResourcesExist = checkInstallResourcesExist(ctx, kubernetesProvider)
allResourcesExist = checkInstallResourcesExist(ctx, kubernetesProvider) && allResourcesExist
} else {
allResourcesExist = checkTapResourcesExist(ctx, kubernetesProvider)
allResourcesExist = checkTapResourcesExist(ctx, kubernetesProvider) && allResourcesExist
}
return allResourcesExist
@@ -205,13 +206,13 @@ func checkInstallResourcesExist(ctx context.Context, kubernetesProvider *kuberne
installResourcesExist := checkResourceExist(kubernetes.DaemonRoleName, "role", exist, err)
exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleBindingName)
installResourcesExist = checkResourceExist(kubernetes.DaemonRoleBindingName, "role binding", exist, err)
installResourcesExist = checkResourceExist(kubernetes.DaemonRoleBindingName, "role binding", exist, err) && installResourcesExist
exist, err = kubernetesProvider.DoesPersistentVolumeClaimExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName)
installResourcesExist = checkResourceExist(kubernetes.PersistentVolumeClaimName, "persistent volume claim", exist, err)
installResourcesExist = checkResourceExist(kubernetes.PersistentVolumeClaimName, "persistent volume claim", exist, err) && installResourcesExist
exist, err = kubernetesProvider.DoesDeploymentExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
installResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "deployment", exist, err)
installResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "deployment", exist, err) && installResourcesExist
return installResourcesExist
}

View File

@@ -5,6 +5,10 @@ import (
"encoding/json"
"errors"
"fmt"
"path"
"regexp"
"time"
"github.com/up9inc/mizu/cli/apiserver"
"github.com/up9inc/mizu/cli/config/configStructs"
"github.com/up9inc/mizu/cli/errormessage"
@@ -13,9 +17,6 @@ import (
"github.com/up9inc/mizu/cli/resources"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"path"
"regexp"
"time"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/shared/kubernetes"
@@ -35,10 +36,10 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx con
return
}
apiProvider = apiserver.NewProviderWithoutRetries(GetApiServerUrl(), time.Second) // short check for proxy
apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
if err := apiProvider.TestConnection(); err != nil {
logger.Log.Debugf("Couldn't connect using proxy, stopping proxy and trying to create port-forward")
if err := httpServer.Shutdown(context.Background()); err != nil {
if err := httpServer.Shutdown(ctx); err != nil {
logger.Log.Debugf("Error occurred while stopping proxy %v", errormessage.FormatError(err))
}
@@ -50,7 +51,7 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx con
return
}
apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout) // long check for port-forward
apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
if err := apiProvider.TestConnection(); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
cancel()

View File

@@ -74,6 +74,7 @@ func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Reso
StandaloneMode: true,
ServiceMap: config.Config.ServiceMap,
OAS: config.Config.OAS,
Elastic: config.Config.Elastic,
}
return &mizuAgentConfig

View File

@@ -23,7 +23,7 @@ var logsCmd = &cobra.Command{
if err != nil {
return nil
}
ctx, _ := context.WithCancel(context.Background())
ctx := context.Background()
if validationErr := config.Config.Logs.Validate(); validationErr != nil {
return errormessage.FormatError(validationErr)

View File

@@ -164,6 +164,7 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
AgentDatabasePath: shared.DataDirPath,
ServiceMap: config.Config.ServiceMap,
OAS: config.Config.OAS,
Elastic: config.Config.Elastic,
}
return &mizuAgentConfig
@@ -342,7 +343,7 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
isPodReady = true
postApiServerStarted(ctx, kubernetesProvider, cancel, err)
postApiServerStarted(ctx, kubernetesProvider, cancel)
}
case kubernetes.EventBookmark:
break
@@ -405,7 +406,7 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
case "FailedScheduling", "Failed":
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu API Server status: %s - %s", event.Reason, event.Note))
cancel()
break
}
case err, ok := <-errorChan:
if !ok {
@@ -421,11 +422,11 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
}
}
func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, err error) {
func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel)
options, _ := getMizuApiFilteringOptions()
if err = startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err))
cancel()
}

View File

@@ -27,7 +27,7 @@ type ConfigStruct struct {
Logs configStructs.LogsConfig `yaml:"logs"`
Auth configStructs.AuthConfig `yaml:"auth"`
Config configStructs.ConfigConfig `yaml:"config,omitempty"`
AgentImage string `yaml:"agent-image,omitempty" readonly:""`
AgentImage string `yaml:"agent-image,omitempty"`
KratosImage string `yaml:"kratos-image,omitempty" readonly:""`
KetoImage string `yaml:"keto-image,omitempty" readonly:""`
ImagePullPolicyStr string `yaml:"image-pull-policy" default:"Always"`
@@ -40,6 +40,7 @@ type ConfigStruct struct {
LogLevelStr string `yaml:"log-level,omitempty" default:"INFO" readonly:""`
ServiceMap bool `yaml:"service-map,omitempty" default:"false" readonly:""`
OAS bool `yaml:"oas,omitempty" default:"false" readonly:""`
Elastic shared.ElasticConfig `yaml:"elastic"`
}
func (config *ConfigStruct) validate() error {

View File

@@ -43,6 +43,13 @@ type MizuAgentConfig struct {
StandaloneMode bool `json:"standaloneMode"`
ServiceMap bool `json:"serviceMap"`
OAS bool `json:"oas"`
Elastic ElasticConfig `json:"elastic"`
}
type ElasticConfig struct {
User string `yaml:"user,omitempty" default:"" readonly:""`
Password string `yaml:"password,omitempty" default:"" readonly:""`
Url string `yaml:"url,omitempty" default:"" readonly:""`
}
type WebSocketMessageMetadata struct {

View File

@@ -197,6 +197,12 @@ func (r *RedisInputStream) readLineBytes() ([]byte, error) {
line := make([]byte, N)
j := 0
for i := r.count; i <= N; i++ {
if i >= len(buf) {
return nil, errors.New("Redis buffer index mismatch.")
}
if i >= len(line) {
return nil, errors.New("Redis line index mismatch.")
}
line[j] = buf[i]
j++
}
@@ -295,27 +301,44 @@ func (p *RedisProtocol) Read() (packet *RedisPacket, err error) {
switch x.(type) {
case []interface{}:
array := x.([]interface{})
switch array[0].(type) {
case []uint8:
packet.Command = RedisCommand(strings.ToUpper(string(array[0].([]uint8))))
if len(array) > 1 {
packet.Key = string(array[1].([]uint8))
}
if len(array) > 2 {
packet.Value = string(array[2].([]uint8))
}
if len(array) > 3 {
packet.Value = fmt.Sprintf("[%s", packet.Value)
for _, item := range array[3:] {
packet.Value = fmt.Sprintf("%s, %s", packet.Value, item.([]uint8))
if len(array) > 0 {
switch array[0].(type) {
case []uint8:
packet.Command = RedisCommand(strings.ToUpper(string(array[0].([]uint8))))
if len(array) > 1 {
switch array[1].(type) {
case []uint8:
packet.Key = string(array[1].([]uint8))
case int64:
packet.Key = fmt.Sprintf("%d", array[1].(int64))
}
}
packet.Value = strings.TrimSuffix(packet.Value, ", ")
packet.Value = fmt.Sprintf("%s]", packet.Value)
if len(array) > 2 {
switch array[2].(type) {
case []uint8:
packet.Value = string(array[2].([]uint8))
case int64:
packet.Value = fmt.Sprintf("%d", array[2].(int64))
}
}
if len(array) > 3 {
packet.Value = fmt.Sprintf("[%s", packet.Value)
for _, item := range array[3:] {
switch item.(type) {
case []uint8:
packet.Value = fmt.Sprintf("%s, %s", packet.Value, item.([]uint8))
case int64:
packet.Value = fmt.Sprintf("%s, %d", packet.Value, item.(int64))
}
}
packet.Value = strings.TrimSuffix(packet.Value, ", ")
packet.Value = fmt.Sprintf("%s]", packet.Value)
}
default:
msg := fmt.Sprintf("Unrecognized element in Redis array: %v", reflect.TypeOf(array[0]))
err = errors.New(msg)
return
}
default:
msg := fmt.Sprintf("Unrecognized element in Redis array: %v", reflect.TypeOf(array[0]))
err = errors.New(msg)
return
}
case []uint8:
val := string(x.([]uint8))