Merge pull request #115 from RafayLabs/audit-log-final

Audit log

Dockerfile

@@ -2,8 +2,12 @@ FROM golang:1.17 as build
 LABEL description="Build container"
 
 ENV CGO_ENABLED 0
-COPY . /build
 WORKDIR /build
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+COPY . .
 RUN go build github.com/RafayLabs/rcloud-base
 
 FROM alpine:latest as runtime

internal/dao/permission.go

@@ -234,7 +234,7 @@ func GetAccountBasics(ctx context.Context, db bun.IDB, accountID uuid.UUID) (*mo
 	err := db.NewSelect().Model(&acc).
 		Column("identities.id", "traits", "state").
 		ColumnExpr("max(ks.authenticated_at) as lastlogin").
-		ColumnExpr("identities.traits -> 'email' as username").
+		ColumnExpr("identities.traits ->> 'email' as username").
 		Join("INNER JOIN sessions as ks ON identities.id = ks.identity_id").
 		Where("identities.id = ?", accountID).
 		Where("ks.active = ?", true).

internal/models/bootstrapagenttemplate.go

@@ -30,4 +30,5 @@ type BootstrapAgentTemplate struct {
 	Token                  string          `bun:"token,notnull"`
 	InclusterTemplate      string          `bun:"incluster_template,notnull"`
 	OutofclusterTemplate   string          `bun:"outofcluster_template,notnull"`
+	Trash                  bool            `bun:"type:bool"`
 }

main.go

@@ -189,9 +189,9 @@ func setup() {
 
 	// audit
 	viper.SetDefault(esEndPointEnv, "http://127.0.0.1:9200")
-	viper.SetDefault(esIndexPrefixEnv, "events-core")
-	viper.SetDefault(relayAuditESIndexPrefixEnv, "relay-audits")
-	viper.SetDefault(relayCommandESIndexPrefix, "relay-commands")
+	viper.SetDefault(esIndexPrefixEnv, "ralogs-system")
+	viper.SetDefault(relayAuditESIndexPrefixEnv, "ralogs-relay")
+	viper.SetDefault(relayCommandESIndexPrefix, "ralogs-prompt")
 	viper.SetDefault(auditFileEnv, "audit.log")
 
 	// cd relay
@@ -501,10 +501,7 @@ func runRelayPeerRPC(wg *sync.WaitGroup, ctx context.Context) {
 		_log.Fatalw("unable to get create relay peer service")
 	}
 	clusterAuthzServer := server.NewClusterAuthzServer(bs, aps, gps, krs, kcs, kss)
-
-	/*
-		auditInfoServer := server.NewAuditInfoServer(bs, aps)
-	*/
+	auditInfoServer := server.NewAuditInfoServer(bs, aps)
 
 	s, err := grpc.NewSecureServerWithPEM(cert, key, ca)
 	if err != nil {
@@ -521,7 +518,7 @@ func runRelayPeerRPC(wg *sync.WaitGroup, ctx context.Context) {
 
 	sentryrpc.RegisterRelayPeerServiceServer(s, relayPeerService)
 	sentryrpc.RegisterClusterAuthorizationServer(s, clusterAuthzServer)
-	/*sentryrpc.RegisterAuditInformationServer(s, auditInfoServer)*/
+	sentryrpc.RegisterAuditInformationServer(s, auditInfoServer)
 
 	l, err := net.Listen("tcp", fmt.Sprintf(":%d", rpcRelayPeeringPort))
 	if err != nil {
@@ -549,7 +546,7 @@ func runRPC(wg *sync.WaitGroup, ctx context.Context) {
 
 	bootstrapServer := server.NewBootstrapServer(bs, kekFunc, cs)
 	kubeConfigServer := server.NewKubeConfigServer(bs, aps, gps, kss, krs, kekFunc, ks, os, ps)
-	/*auditInfoServer := rpcv2.NewAuditInfoServer(bs, aps)*/
+	auditInfoServer := server.NewAuditInfoServer(bs, aps)
 	clusterAuthzServer := server.NewClusterAuthzServer(bs, aps, gps, krs, kcs, kss)
 	kubectlClusterSettingsServer := server.NewKubectlClusterSettingsServer(bs, kcs)
 	crpc := server.NewClusterServer(cs, downloadData)
@@ -613,7 +610,7 @@ func runRPC(wg *sync.WaitGroup, ctx context.Context) {
 	sentryrpc.RegisterBootstrapServer(s, bootstrapServer)
 	sentryrpc.RegisterKubeConfigServer(s, kubeConfigServer)
 	sentryrpc.RegisterClusterAuthorizationServer(s, clusterAuthzServer)
-	/*pbrpcv2.RegisterAuditInformationServer(s, auditInfoServer)*/
+	sentryrpc.RegisterAuditInformationServer(s, auditInfoServer)
 	sentryrpc.RegisterKubectlClusterSettingsServer(s, kubectlClusterSettingsServer)
 	schedulerrpc.RegisterClusterServer(s, crpc)
 	systemrpc.RegisterLocationServer(s, mserver)

pkg/audit/events.go

@@ -39,7 +39,6 @@ const (
 
 // EventActorAccount Event's initiator account
 type EventActorAccount struct {
-	ID       string `json:"id"`
 	Username string `json:"username"`
 }
 
@@ -249,7 +248,6 @@ func getEventClientFromContext(ctx context.Context) *EventClient {
 
 func getActor(cOpts createEventOptions) *EventActor {
 	account := EventActorAccount{
-		ID:       cOpts.accountID,
 		Username: cOpts.username,
 	}
 	return &EventActor{
@@ -260,10 +258,8 @@ func getActor(cOpts createEventOptions) *EventActor {
 }
 
 func GetActorFromSessionData(sd *commonv3.SessionData) *EventActor {
-	accountID := sd.GetAccount()
 	username := sd.GetUsername()
 	account := EventActorAccount{
-		ID:       accountID,
 		Username: username,
 	}
 	groups := sd.Groups // TODO: get groups (in interceptor?)

pkg/auth/v3/auth.go

@@ -56,7 +56,7 @@ func SetupAuthContext(auditLogger *zap.Logger) authContext {
 	sqldb := sql.OpenDB(pgdriver.NewConnector(pgdriver.WithDSN(getDSN())))
 	db = bun.NewDB(sqldb, pgdialect.New())
 
-	if v, ok := os.LookupEnv("KRATOS_ADDR"); ok {
+	if v, ok := os.LookupEnv("KRATOS_PUB_ADDR"); ok {
 		kratosAddr = v
 	} else {
 		kratosAddr = "http://localhost:4433"

pkg/service/audit_log_query.go

@@ -36,10 +36,10 @@ func (a *AuditLogService) GetAuditLog(req *v1.AuditLogSearchRequest) (res *v1.Au
 
 func validateQueryString(queryString string) error {
 	if strings.Contains(queryString, "*") {
-		return fmt.Errorf("Sorry. '*' is not supported in search query")
+		return fmt.Errorf("'*' is not supported in search query")
 	}
 	if len(queryString) > 0 && len(queryString) < 3 {
-		return fmt.Errorf("Search string has to be atleast 3 characters")
+		return fmt.Errorf("search string has to be atleast 3 characters")
 	}
 	return nil
 }
@@ -48,13 +48,12 @@ func getPrjectIdFromUrlScope(urlScope string) (string, error) {
 	s := strings.Split(urlScope, "/")
 	if len(s) != 2 {
 		_log.Errorw("Unable to retrieve projectID from urlScope", "urlScope", urlScope)
-		return "", fmt.Errorf("Unable to retrieve projectID from urlScope")
+		return "", fmt.Errorf("unable to retrieve projectID from urlScope")
 	}
 	return s[1], nil
 }
 
 func (a *AuditLogService) GetAuditLogByProjects(req *v1.AuditLogSearchRequest) (res *v1.AuditLogSearchResponse, err error) {
-	// No embedding in golang/protoc (https://github.com/golang/protobuf/issues/192)
 	err = validateQueryString(req.GetFilter().QueryString)
 	if err != nil {
 		return nil, err
@@ -65,34 +64,33 @@ func (a *AuditLogService) GetAuditLogByProjects(req *v1.AuditLogSearchRequest) (
 	var buf bytes.Buffer
 	var r map[string]interface{}
 	query := map[string]interface{}{
-		"_source": true,
+		"_source": []string{"json"},
 		"size":    500,
 		"query": map[string]interface{}{
 			"bool": map[string]interface{}{
 				"must": []map[string]interface{}{
-					// Add org and partner filter once we have to support multi-org
 					{
 						"term": map[string]interface{}{
-							"category": "AUDIT",
+							"json.category": "AUDIT",
 						},
 					},
 				},
 			},
 		},
 		"sort": map[string]interface{}{
-			"timestamp": map[string]interface{}{
+			"json.timestamp": map[string]interface{}{
 				"order": "desc",
 			},
 		},
 		"aggs": map[string]interface{}{
 			"group_by_username": map[string]interface{}{
 				"terms": map[string]interface{}{
-					"field": "actor.account.username",
+					"field": "json.actor.account.username",
 				},
 			},
 			"group_by_type": map[string]interface{}{
 				"terms": map[string]interface{}{
-					"field": "type",
+					"field": "json.type",
 				},
 			},
 		},
@@ -103,19 +101,19 @@ func (a *AuditLogService) GetAuditLogByProjects(req *v1.AuditLogSearchRequest) (
 	if req.GetFilter().DashboardData {
 		agg["group_by_project"] = map[string]interface{}{
 			"terms": map[string]interface{}{
-				"field": "project_id",
+				"field": "json.project_id",
 				"size":  1000,
 			},
 			"aggs": map[string]interface{}{
 				"group_by_username": map[string]interface{}{
 					"terms": map[string]interface{}{
-						"field": "actor.account.username",
+						"field": "json.actor.account.username",
 						"size":  1000,
 					},
 				},
 				"group_by_type": map[string]interface{}{
 					"terms": map[string]interface{}{
-						"field": "type",
+						"field": "json.type",
 						"size":  1000,
 					},
 				},
@@ -136,7 +134,7 @@ func (a *AuditLogService) GetAuditLogByProjects(req *v1.AuditLogSearchRequest) (
 	if req.GetFilter().Timefrom != "" {
 		b["filter"] = map[string]interface{}{
 			"range": map[string]interface{}{
-				"timestamp": map[string]interface{}{
+				"json.timestamp": map[string]interface{}{
 					"gte": req.GetFilter().Timefrom,
 					"lt":  "now",
 				},
@@ -147,7 +145,7 @@ func (a *AuditLogService) GetAuditLogByProjects(req *v1.AuditLogSearchRequest) (
 	if req.GetFilter().Type != "" {
 		t := map[string]interface{}{
 			"term": map[string]interface{}{
-				"type": req.GetFilter().Type,
+				"json.type": req.GetFilter().Type,
 			},
 		}
 		m = append(m, t)
@@ -156,7 +154,7 @@ func (a *AuditLogService) GetAuditLogByProjects(req *v1.AuditLogSearchRequest) (
 	if req.GetFilter().User != "" {
 		t := map[string]interface{}{
 			"term": map[string]interface{}{
-				"actor.account.username": req.GetFilter().User,
+				"json.actor.account.username": req.GetFilter().User,
 			},
 		}
 		m = append(m, t)
@@ -165,7 +163,7 @@ func (a *AuditLogService) GetAuditLogByProjects(req *v1.AuditLogSearchRequest) (
 	if req.GetFilter().Client != "" {
 		t := map[string]interface{}{
 			"term": map[string]interface{}{
-				"client.type": req.GetFilter().Client,
+				"json.client.type": req.GetFilter().Client,
 			},
 		}
 		m = append(m, t)
@@ -174,7 +172,7 @@ func (a *AuditLogService) GetAuditLogByProjects(req *v1.AuditLogSearchRequest) (
 	if len(req.GetFilter().ProjectIds) > 0 {
 		t := map[string]interface{}{
 			"terms": map[string]interface{}{
-				"project_id": req.GetFilter().ProjectIds,
+				"json.project_id": req.GetFilter().ProjectIds,
 			},
 		}
 		m = append(m, t)

pkg/service/relay_audit_query.go

@@ -34,8 +34,6 @@ func (ra *RelayAuditService) GetRelayAudit(req *v1.RelayAuditSearchRequest) (res
 
 func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchRequest) (res *v1.RelayAuditSearchResponse, err error) {
 	err = validateQueryString(req.GetFilter().QueryString)
-	oid := req.GetMetadata().GetOrganization()
-	pid := req.GetMetadata().GetPartner()
 	if err != nil {
 		return &v1.RelayAuditSearchResponse{}, err
 	}
@@ -44,53 +42,42 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 	res = &v1.RelayAuditSearchResponse{}
 	//Handle defaults value
 	query := map[string]interface{}{
-		"_source": true,
+		"_source": []string{"json"},
 		"size":    500,
 		"query": map[string]interface{}{
 			"bool": map[string]interface{}{
-				"must": []map[string]interface{}{
-					{
-						"term": map[string]interface{}{
-							"o": oid,
-						},
-					},
-					{
-						"term": map[string]interface{}{
-							"p": pid,
-						},
-					},
-				},
+				"must": []map[string]interface{}{},
 			},
 		},
 		"sort": map[string]interface{}{
-			"ts": map[string]interface{}{
+			"json.ts": map[string]interface{}{
 				"order": "desc",
 			},
 		},
 		"aggs": map[string]interface{}{
 			"group_by_username": map[string]interface{}{
 				"terms": map[string]interface{}{
-					"field": "un",
+					"field": "json.un",
 				},
 			},
 			"group_by_cluster": map[string]interface{}{
 				"terms": map[string]interface{}{
-					"field": "cn",
+					"field": "json.cn",
 				},
 			},
 			"group_by_namespace": map[string]interface{}{
 				"terms": map[string]interface{}{
-					"field": "ns",
+					"field": "json.ns",
 				},
 			},
 			"group_by_kind": map[string]interface{}{
 				"terms": map[string]interface{}{
-					"field": "k",
+					"field": "json.k",
 				},
 			},
 			"group_by_method": map[string]interface{}{
 				"terms": map[string]interface{}{
-					"field": "m",
+					"field": "json.m",
 				},
 			},
 		},
@@ -101,19 +88,19 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 	if req.GetFilter().DashboardData {
 		agg["group_by_cluster"] = map[string]interface{}{
 			"terms": map[string]interface{}{
-				"field": "cn",
+				"field": "json.cn",
 				"size":  1000,
 			},
 			"aggs": map[string]interface{}{
 				"group_by_username": map[string]interface{}{
 					"terms": map[string]interface{}{
-						"field": "un",
+						"field": "json.un",
 						"size":  1000,
 					},
 				},
 				"group_by_namespace": map[string]interface{}{
 					"terms": map[string]interface{}{
-						"field": "ns",
+						"field": "json.ns",
 						"size":  1000,
 					},
 				},
@@ -142,7 +129,7 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 	if req.Filter.Timefrom != "" {
 		b["filter"] = map[string]interface{}{
 			"range": map[string]interface{}{
-				"ts": map[string]interface{}{
+				"json.ts": map[string]interface{}{
 					"gte": req.Filter.Timefrom,
 					"lt":  "now",
 				},
@@ -153,7 +140,7 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 	if req.Filter.User != "" {
 		t := map[string]interface{}{
 			"term": map[string]interface{}{
-				"un": req.Filter.User,
+				"json.un": req.Filter.User,
 			},
 		}
 		m = append(m, t)
@@ -162,7 +149,7 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 	if req.Filter.Cluster != "" {
 		t := map[string]interface{}{
 			"term": map[string]interface{}{
-				"cn": req.Filter.Cluster,
+				"json.cn": req.Filter.Cluster,
 			},
 		}
 		m = append(m, t)
@@ -171,7 +158,7 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 	if req.Filter.Namespace != "" {
 		t := map[string]interface{}{
 			"term": map[string]interface{}{
-				"ns": req.Filter.Namespace,
+				"json.ns": req.Filter.Namespace,
 			},
 		}
 		m = append(m, t)
@@ -180,7 +167,7 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 	if req.Filter.Kind != "" {
 		t := map[string]interface{}{
 			"term": map[string]interface{}{
-				"k": req.Filter.Kind,
+				"json.k": req.Filter.Kind,
 			},
 		}
 		m = append(m, t)
@@ -189,7 +176,7 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 	if req.Filter.Method != "" {
 		t := map[string]interface{}{
 			"term": map[string]interface{}{
-				"m": req.Filter.Method,
+				"json.m": req.Filter.Method,
 			},
 		}
 		m = append(m, t)
@@ -201,7 +188,7 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 			req.Filter.ClusterNames != nil && len(req.Filter.ClusterNames) > 0 {
 			t := map[string]interface{}{
 				"terms": map[string]interface{}{
-					"cn": req.Filter.ClusterNames,
+					"json.cn": req.Filter.ClusterNames,
 				},
 			}
 			m = append(m, t)
@@ -209,7 +196,7 @@ func (ra *RelayAuditService) GetRelayAuditByProjects(req *v1.RelayAuditSearchReq
 
 			t := map[string]interface{}{
 				"terms": map[string]interface{}{
-					"project_id": req.Filter.ProjectIds,
+					"json.project_id": req.Filter.ProjectIds,
 				},
 			}
 			m = append(m, t)