From 1f2247a8c4aaa879684e62739fb2e1e0e0390e50 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 11 Dec 2017 19:20:27 +0000 Subject: [PATCH 1/2] move node metadata keys into report package Both the probe and the app (for rendering) need to know about them. --- probe/awsecs/reporter.go | 14 +++--- probe/docker/container.go | 26 +++++------ probe/docker/controls.go | 16 +++---- probe/docker/reporter.go | 14 +++--- probe/docker/tagger.go | 5 +-- probe/endpoint/reporter.go | 6 +-- probe/kubernetes/controls.go | 8 ++-- probe/kubernetes/cronjob.go | 8 ++-- probe/kubernetes/daemonset.go | 2 +- probe/kubernetes/deployment.go | 8 ++-- probe/kubernetes/meta.go | 6 +-- probe/kubernetes/pod.go | 6 +-- probe/kubernetes/replica_set.go | 2 +- probe/kubernetes/reporter.go | 10 ++--- probe/kubernetes/service.go | 2 +- probe/process/reporter.go | 10 ++--- report/map_keys.go | 80 +++++++++++++++++++++++++++++++++ 17 files changed, 151 insertions(+), 72 deletions(-) create mode 100644 report/map_keys.go diff --git a/probe/awsecs/reporter.go b/probe/awsecs/reporter.go index 5c3ee9c83..f2cc59d34 100644 --- a/probe/awsecs/reporter.go +++ b/probe/awsecs/reporter.go @@ -13,13 +13,13 @@ import ( // TaskFamily is the key that stores the task family of an ECS Task const ( - Cluster = "ecs_cluster" - CreatedAt = "ecs_created_at" - TaskFamily = "ecs_task_family" - ServiceDesiredCount = "ecs_service_desired_count" - ServiceRunningCount = "ecs_service_running_count" - ScaleUp = "ecs_scale_up" - ScaleDown = "ecs_scale_down" + Cluster = report.ECSCluster + CreatedAt = report.ECSCreatedAt + TaskFamily = report.ECSTaskFamily + ServiceDesiredCount = report.ECSServiceDesiredCount + ServiceRunningCount = report.ECSServiceRunningCount + ScaleUp = report.ECSScaleUp + ScaleDown = report.ECSScaleDown ) var ( diff --git a/probe/docker/container.go b/probe/docker/container.go index b31f2ff37..81a8bbd94 100644 --- a/probe/docker/container.go +++ b/probe/docker/container.go @@ -18,19 +18,19 @@ import ( // These constants are keys used in node metadata const ( - ContainerName = "docker_container_name" - ContainerCommand = "docker_container_command" - ContainerPorts = "docker_container_ports" - ContainerCreated = "docker_container_created" - ContainerNetworks = "docker_container_networks" - ContainerIPs = "docker_container_ips" - ContainerHostname = "docker_container_hostname" - ContainerIPsWithScopes = "docker_container_ips_with_scopes" - ContainerState = "docker_container_state" - ContainerStateHuman = "docker_container_state_human" - ContainerUptime = "docker_container_uptime" - ContainerRestartCount = "docker_container_restart_count" - ContainerNetworkMode = "docker_container_network_mode" + ContainerName = report.DockerContainerName + ContainerCommand = report.DockerContainerCommand + ContainerPorts = report.DockerContainerPorts + ContainerCreated = report.DockerContainerCreated + ContainerNetworks = report.DockerContainerNetworks + ContainerIPs = report.DockerContainerIPs + ContainerHostname = report.DockerContainerHostname + ContainerIPsWithScopes = report.DockerContainerIPsWithScopes + ContainerState = report.DockerContainerState + ContainerStateHuman = report.DockerContainerStateHuman + ContainerUptime = report.DockerContainerUptime + ContainerRestartCount = report.DockerContainerRestartCount + ContainerNetworkMode = report.DockerContainerNetworkMode NetworkRxDropped = "network_rx_dropped" NetworkRxBytes = "network_rx_bytes" diff --git a/probe/docker/controls.go b/probe/docker/controls.go index f51fc4651..5c5d4f211 100644 --- a/probe/docker/controls.go +++ b/probe/docker/controls.go @@ -12,14 +12,14 @@ import ( // Control IDs used by the docker integration. const ( - StopContainer = "docker_stop_container" - StartContainer = "docker_start_container" - RestartContainer = "docker_restart_container" - PauseContainer = "docker_pause_container" - UnpauseContainer = "docker_unpause_container" - RemoveContainer = "docker_remove_container" - AttachContainer = "docker_attach_container" - ExecContainer = "docker_exec_container" + StopContainer = report.DockerStopContainer + StartContainer = report.DockerStartContainer + RestartContainer = report.DockerRestartContainer + PauseContainer = report.DockerPauseContainer + UnpauseContainer = report.DockerUnpauseContainer + RemoveContainer = report.DockerRemoveContainer + AttachContainer = report.DockerAttachContainer + ExecContainer = report.DockerExecContainer ResizeExecTTY = "docker_resize_exec_tty" waitTime = 10 diff --git a/probe/docker/reporter.go b/probe/docker/reporter.go index 079c9a53d..08e9fd934 100644 --- a/probe/docker/reporter.go +++ b/probe/docker/reporter.go @@ -14,15 +14,15 @@ import ( // Keys for use in Node const ( - ImageID = "docker_image_id" - ImageName = "docker_image_name" - ImageSize = "docker_image_size" - ImageVirtualSize = "docker_image_virtual_size" + ImageID = report.DockerImageID + ImageName = report.DockerImageName + ImageSize = report.DockerImageSize + ImageVirtualSize = report.DockerImageVirtualSize + IsInHostNetwork = report.DockerIsInHostNetwork ImageLabelPrefix = "docker_image_label_" - IsInHostNetwork = "docker_is_in_host_network" ImageTableID = "image_table" - ServiceName = "service_name" - StackNamespace = "stack_namespace" + ServiceName = report.DockerServiceName + StackNamespace = report.DockerStackNamespace DefaultNamespace = "No Stack" ) diff --git a/probe/docker/tagger.go b/probe/docker/tagger.go index 066b9ee97..1d9c67051 100644 --- a/probe/docker/tagger.go +++ b/probe/docker/tagger.go @@ -10,9 +10,8 @@ import ( // Node metadata keys. const ( - ContainerID = "docker_container_id" - Domain = "domain" // TODO this is ambiguous, be more specific - Name = "name" // TODO this is ambiguous, be more specific + ContainerID = report.DockerContainerID + Name = report.Name ) // These vars are exported for testing. diff --git a/probe/endpoint/reporter.go b/probe/endpoint/reporter.go index c5fe57e31..86b52dbf5 100644 --- a/probe/endpoint/reporter.go +++ b/probe/endpoint/reporter.go @@ -11,9 +11,9 @@ import ( // Node metadata keys. const ( - ReverseDNSNames = "reverse_dns_names" - SnoopedDNSNames = "snooped_dns_names" - CopyOf = "copy_of" + ReverseDNSNames = report.ReverseDNSNames + SnoopedDNSNames = report.SnoopedDNSNames + CopyOf = report.CopyOf ) // ReporterConfig are the config options for the endpoint reporter. diff --git a/probe/kubernetes/controls.go b/probe/kubernetes/controls.go index b4ac690d7..62ddb806e 100644 --- a/probe/kubernetes/controls.go +++ b/probe/kubernetes/controls.go @@ -11,10 +11,10 @@ import ( // Control IDs used by the kubernetes integration. const ( - GetLogs = "kubernetes_get_logs" - DeletePod = "kubernetes_delete_pod" - ScaleUp = "kubernetes_scale_up" - ScaleDown = "kubernetes_scale_down" + GetLogs = report.KubernetesGetLogs + DeletePod = report.KubernetesDeletePod + ScaleUp = report.KubernetesScaleUp + ScaleDown = report.KubernetesScaleDown ) // GetLogs is the control to get the logs for a kubernetes pod diff --git a/probe/kubernetes/cronjob.go b/probe/kubernetes/cronjob.go index 5e76aeee7..ca4846c8f 100644 --- a/probe/kubernetes/cronjob.go +++ b/probe/kubernetes/cronjob.go @@ -15,10 +15,10 @@ import ( // These constants are keys used in node metadata const ( - Schedule = "kubernetes_schedule" - Suspended = "kubernetes_suspended" - LastScheduled = "kubernetes_last_scheduled" - ActiveJobs = "kubernetes_active_jobs" + Schedule = report.KubernetesSchedule + Suspended = report.KubernetesSuspended + LastScheduled = report.KubernetesLastScheduled + ActiveJobs = report.KubernetesActiveJobs ) // CronJob represents a Kubernetes cron job diff --git a/probe/kubernetes/daemonset.go b/probe/kubernetes/daemonset.go index ad6241c53..45738d950 100644 --- a/probe/kubernetes/daemonset.go +++ b/probe/kubernetes/daemonset.go @@ -12,7 +12,7 @@ import ( // These constants are keys used in node metadata const ( - MisscheduledReplicas = "kubernetes_misscheduled_replicas" + MisscheduledReplicas = report.KubernetesMisscheduledReplicas ) // DaemonSet represents a Kubernetes daemonset diff --git a/probe/kubernetes/deployment.go b/probe/kubernetes/deployment.go index 97edf2a6e..7dbebeffb 100644 --- a/probe/kubernetes/deployment.go +++ b/probe/kubernetes/deployment.go @@ -13,10 +13,10 @@ import ( // These constants are keys used in node metadata const ( - UpdatedReplicas = "kubernetes_updated_replicas" - AvailableReplicas = "kubernetes_available_replicas" - UnavailableReplicas = "kubernetes_unavailable_replicas" - Strategy = "kubernetes_strategy" + UpdatedReplicas = report.KubernetesUpdatedReplicas + AvailableReplicas = report.KubernetesAvailableReplicas + UnavailableReplicas = report.KubernetesUnavailableReplicas + Strategy = report.KubernetesStrategy ) // Deployment represents a Kubernetes deployment diff --git a/probe/kubernetes/meta.go b/probe/kubernetes/meta.go index f3cc3339c..b364d9bdd 100644 --- a/probe/kubernetes/meta.go +++ b/probe/kubernetes/meta.go @@ -10,9 +10,9 @@ import ( // These constants are keys used in node metadata const ( - Name = "kubernetes_name" - Namespace = "kubernetes_namespace" - Created = "kubernetes_created" + Name = report.KubernetesName + Namespace = report.KubernetesNamespace + Created = report.KubernetesCreated LabelPrefix = "kubernetes_labels_" ) diff --git a/probe/kubernetes/pod.go b/probe/kubernetes/pod.go index 2614a14f0..d96336d90 100644 --- a/probe/kubernetes/pod.go +++ b/probe/kubernetes/pod.go @@ -10,9 +10,9 @@ import ( // These constants are keys used in node metadata const ( - State = "kubernetes_state" - IsInHostNetwork = "kubernetes_is_in_host_network" - RestartCount = "kubernetes_restart_count" + State = report.KubernetesState + IsInHostNetwork = report.KubernetesIsInHostNetwork + RestartCount = report.KubernetesRestartCount StateDeleted = "deleted" ) diff --git a/probe/kubernetes/replica_set.go b/probe/kubernetes/replica_set.go index 6424f9bd4..9f537ae51 100644 --- a/probe/kubernetes/replica_set.go +++ b/probe/kubernetes/replica_set.go @@ -13,7 +13,7 @@ import ( // These constants are keys used in node metadata const ( - FullyLabeledReplicas = "kubernetes_fully_labeled_replicas" + FullyLabeledReplicas = report.KubernetesFullyLabeledReplicas ) // ReplicaSet represents a Kubernetes replica set diff --git a/probe/kubernetes/reporter.go b/probe/kubernetes/reporter.go index c222e9b0a..e4000cae8 100644 --- a/probe/kubernetes/reporter.go +++ b/probe/kubernetes/reporter.go @@ -18,11 +18,11 @@ import ( // These constants are keys used in node metadata const ( - IP = "kubernetes_ip" - ObservedGeneration = "kubernetes_observed_generation" - Replicas = "kubernetes_replicas" - DesiredReplicas = "kubernetes_desired_replicas" - NodeType = "kubernetes_node_type" + IP = report.KubernetesIP + ObservedGeneration = report.KubernetesObservedGeneration + Replicas = report.KubernetesReplicas + DesiredReplicas = report.KubernetesDesiredReplicas + NodeType = report.KubernetesNodeType ) // Exposed for testing diff --git a/probe/kubernetes/service.go b/probe/kubernetes/service.go index 599e94b4d..d8b42b131 100644 --- a/probe/kubernetes/service.go +++ b/probe/kubernetes/service.go @@ -9,7 +9,7 @@ import ( // These constants are keys used in node metadata const ( - PublicIP = "kubernetes_public_ip" + PublicIP = report.KubernetesPublicIP ) // Service represents a Kubernetes service diff --git a/probe/process/reporter.go b/probe/process/reporter.go index 0f6f88ef5..6fd3f5511 100644 --- a/probe/process/reporter.go +++ b/probe/process/reporter.go @@ -10,11 +10,11 @@ import ( // We use these keys in node metadata const ( - PID = "pid" - Name = "name" - PPID = "ppid" - Cmdline = "cmdline" - Threads = "threads" + PID = report.PID + Name = report.Name + PPID = report.PPID + Cmdline = report.Cmdline + Threads = report.Threads CPUUsage = "process_cpu_usage_percent" MemoryUsage = "process_memory_usage_bytes" OpenFilesCount = "open_files_count" diff --git a/report/map_keys.go b/report/map_keys.go new file mode 100644 index 000000000..b29ca9bdf --- /dev/null +++ b/report/map_keys.go @@ -0,0 +1,80 @@ +package report + +// node metadata keys +const ( + // probe/endpoint + ReverseDNSNames = "reverse_dns_names" + SnoopedDNSNames = "snooped_dns_names" + CopyOf = "copy_of" + // probe/process + PID = "pid" + Name = "name" // also used by probe/docker + PPID = "ppid" + Cmdline = "cmdline" + Threads = "threads" + // probe/docker + DockerContainerID = "docker_container_id" + DockerImageID = "docker_image_id" + DockerImageName = "docker_image_name" + DockerImageSize = "docker_image_size" + DockerImageVirtualSize = "docker_image_virtual_size" + DockerIsInHostNetwork = "docker_is_in_host_network" + DockerServiceName = "service_name" + DockerStackNamespace = "stack_namespace" + DockerStopContainer = "docker_stop_container" + DockerStartContainer = "docker_start_container" + DockerRestartContainer = "docker_restart_container" + DockerPauseContainer = "docker_pause_container" + DockerUnpauseContainer = "docker_unpause_container" + DockerRemoveContainer = "docker_remove_container" + DockerAttachContainer = "docker_attach_container" + DockerExecContainer = "docker_exec_container" + DockerContainerName = "docker_container_name" + DockerContainerCommand = "docker_container_command" + DockerContainerPorts = "docker_container_ports" + DockerContainerCreated = "docker_container_created" + DockerContainerNetworks = "docker_container_networks" + DockerContainerIPs = "docker_container_ips" + DockerContainerHostname = "docker_container_hostname" + DockerContainerIPsWithScopes = "docker_container_ips_with_scopes" + DockerContainerState = "docker_container_state" + DockerContainerStateHuman = "docker_container_state_human" + DockerContainerUptime = "docker_container_uptime" + DockerContainerRestartCount = "docker_container_restart_count" + DockerContainerNetworkMode = "docker_container_network_mode" + // probe/kubernetes + KubernetesName = "kubernetes_name" + KubernetesNamespace = "kubernetes_namespace" + KubernetesCreated = "kubernetes_created" + KubernetesIP = "kubernetes_ip" + KubernetesObservedGeneration = "kubernetes_observed_generation" + KubernetesReplicas = "kubernetes_replicas" + KubernetesDesiredReplicas = "kubernetes_desired_replicas" + KubernetesNodeType = "kubernetes_node_type" + KubernetesGetLogs = "kubernetes_get_logs" + KubernetesDeletePod = "kubernetes_delete_pod" + KubernetesScaleUp = "kubernetes_scale_up" + KubernetesScaleDown = "kubernetes_scale_down" + KubernetesUpdatedReplicas = "kubernetes_updated_replicas" + KubernetesAvailableReplicas = "kubernetes_available_replicas" + KubernetesUnavailableReplicas = "kubernetes_unavailable_replicas" + KubernetesStrategy = "kubernetes_strategy" + KubernetesFullyLabeledReplicas = "kubernetes_fully_labeled_replicas" + KubernetesState = "kubernetes_state" + KubernetesIsInHostNetwork = "kubernetes_is_in_host_network" + KubernetesRestartCount = "kubernetes_restart_count" + KubernetesMisscheduledReplicas = "kubernetes_misscheduled_replicas" + KubernetesPublicIP = "kubernetes_public_ip" + KubernetesSchedule = "kubernetes_schedule" + KubernetesSuspended = "kubernetes_suspended" + KubernetesLastScheduled = "kubernetes_last_scheduled" + KubernetesActiveJobs = "kubernetes_active_jobs" + // probe/awsecs + ECSCluster = "ecs_cluster" + ECSCreatedAt = "ecs_created_at" + ECSTaskFamily = "ecs_task_family" + ECSServiceDesiredCount = "ecs_service_desired_count" + ECSServiceRunningCount = "ecs_service_running_count" + ECSScaleUp = "ecs_scale_up" + ECSScaleDown = "ecs_scale_down" +) From e24d3e93fe8549cb5e9913d589952b201be27156 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 11 Dec 2017 19:25:36 +0000 Subject: [PATCH 2/2] Add static map of common keys to reduce allocations --- extras/generate_latest_map | 2 +- report/latest_map_generated.go | 4 +- report/map_helpers.go | 2 +- report/map_keys.go | 110 +++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 4 deletions(-) diff --git a/extras/generate_latest_map b/extras/generate_latest_map index cfcfa434c..24d9d66b4 100755 --- a/extras/generate_latest_map +++ b/extras/generate_latest_map @@ -243,7 +243,7 @@ function generate_latest_map() { z.DecSendContainerState(containerMapKey) var key string if !r.TryDecodeAsNil() { - key = r.DecodeString() + key = lookupCommonKey(r.DecodeStringAsBytes()) } i := m.locate(key) (*m)[i].key = key diff --git a/report/latest_map_generated.go b/report/latest_map_generated.go index 33e203a96..86e181086 100644 --- a/report/latest_map_generated.go +++ b/report/latest_map_generated.go @@ -211,7 +211,7 @@ func (m *StringLatestMap) CodecDecodeSelf(decoder *codec.Decoder) { z.DecSendContainerState(containerMapKey) var key string if !r.TryDecodeAsNil() { - key = r.DecodeString() + key = lookupCommonKey(r.DecodeStringAsBytes()) } i := m.locate(key) (*m)[i].key = key @@ -432,7 +432,7 @@ func (m *NodeControlDataLatestMap) CodecDecodeSelf(decoder *codec.Decoder) { z.DecSendContainerState(containerMapKey) var key string if !r.TryDecodeAsNil() { - key = r.DecodeString() + key = lookupCommonKey(r.DecodeStringAsBytes()) } i := m.locate(key) (*m)[i].key = key diff --git a/report/map_helpers.go b/report/map_helpers.go index 080914490..480ddcf26 100644 --- a/report/map_helpers.go +++ b/report/map_helpers.go @@ -110,7 +110,7 @@ func mapRead(decoder *codec.Decoder, decodeValue func(isNil bool) interface{}) p var key string z.DecSendContainerState(containerMapKey) if !r.TryDecodeAsNil() { - key = r.DecodeString() + key = lookupCommonKey(r.DecodeStringAsBytes()) } z.DecSendContainerState(containerMapValue) diff --git a/report/map_keys.go b/report/map_keys.go index b29ca9bdf..865a3143b 100644 --- a/report/map_keys.go +++ b/report/map_keys.go @@ -78,3 +78,113 @@ const ( ECSScaleUp = "ecs_scale_up" ECSScaleDown = "ecs_scale_down" ) + +/* Lookup table to allow msgpack/json decoder to avoid heap allocation + for common ps.Map keys. The map is static so we don't have to lock + access from multiple threads and don't have to worry about it + getting clogged with values that are only used once. +*/ +var commonKeys = map[string]string{ + Endpoint: Endpoint, + Process: Process, + Container: Container, + Pod: Pod, + Service: Service, + Deployment: Deployment, + ReplicaSet: ReplicaSet, + DaemonSet: DaemonSet, + StatefulSet: StatefulSet, + CronJob: CronJob, + ContainerImage: ContainerImage, + Host: Host, + Overlay: Overlay, + ECSService: ECSService, + ECSTask: ECSTask, + SwarmService: SwarmService, + + HostNodeID: HostNodeID, + ControlProbeID: ControlProbeID, + DoesNotMakeConnections: DoesNotMakeConnections, + + ReverseDNSNames: ReverseDNSNames, + SnoopedDNSNames: SnoopedDNSNames, + CopyOf: CopyOf, + + PID: PID, + Name: Name, + PPID: PPID, + Cmdline: Cmdline, + Threads: Threads, + + DockerContainerID: DockerContainerID, + DockerImageID: DockerImageID, + DockerImageName: DockerImageName, + DockerImageSize: DockerImageSize, + DockerImageVirtualSize: DockerImageVirtualSize, + DockerIsInHostNetwork: DockerIsInHostNetwork, + DockerServiceName: DockerServiceName, + DockerStackNamespace: DockerStackNamespace, + DockerStopContainer: DockerStopContainer, + DockerStartContainer: DockerStartContainer, + DockerRestartContainer: DockerRestartContainer, + DockerPauseContainer: DockerPauseContainer, + DockerUnpauseContainer: DockerUnpauseContainer, + DockerRemoveContainer: DockerRemoveContainer, + DockerAttachContainer: DockerAttachContainer, + DockerExecContainer: DockerExecContainer, + DockerContainerName: DockerContainerName, + DockerContainerCommand: DockerContainerCommand, + DockerContainerPorts: DockerContainerPorts, + DockerContainerCreated: DockerContainerCreated, + DockerContainerNetworks: DockerContainerNetworks, + DockerContainerIPs: DockerContainerIPs, + DockerContainerHostname: DockerContainerHostname, + DockerContainerIPsWithScopes: DockerContainerIPsWithScopes, + DockerContainerState: DockerContainerState, + DockerContainerStateHuman: DockerContainerStateHuman, + DockerContainerUptime: DockerContainerUptime, + DockerContainerRestartCount: DockerContainerRestartCount, + DockerContainerNetworkMode: DockerContainerNetworkMode, + + KubernetesName: KubernetesName, + KubernetesNamespace: KubernetesNamespace, + KubernetesCreated: KubernetesCreated, + KubernetesIP: KubernetesIP, + KubernetesObservedGeneration: KubernetesObservedGeneration, + KubernetesReplicas: KubernetesReplicas, + KubernetesDesiredReplicas: KubernetesDesiredReplicas, + KubernetesNodeType: KubernetesNodeType, + KubernetesGetLogs: KubernetesGetLogs, + KubernetesDeletePod: KubernetesDeletePod, + KubernetesScaleUp: KubernetesScaleUp, + KubernetesScaleDown: KubernetesScaleDown, + KubernetesUpdatedReplicas: KubernetesUpdatedReplicas, + KubernetesAvailableReplicas: KubernetesAvailableReplicas, + KubernetesUnavailableReplicas: KubernetesUnavailableReplicas, + KubernetesStrategy: KubernetesStrategy, + KubernetesFullyLabeledReplicas: KubernetesFullyLabeledReplicas, + KubernetesState: KubernetesState, + KubernetesIsInHostNetwork: KubernetesIsInHostNetwork, + KubernetesRestartCount: KubernetesRestartCount, + KubernetesMisscheduledReplicas: KubernetesMisscheduledReplicas, + KubernetesPublicIP: KubernetesPublicIP, + KubernetesSchedule: KubernetesSchedule, + KubernetesSuspended: KubernetesSuspended, + KubernetesLastScheduled: KubernetesLastScheduled, + KubernetesActiveJobs: KubernetesActiveJobs, + + ECSCluster: ECSCluster, + ECSCreatedAt: ECSCreatedAt, + ECSTaskFamily: ECSTaskFamily, + ECSServiceDesiredCount: ECSServiceDesiredCount, + ECSServiceRunningCount: ECSServiceRunningCount, + ECSScaleUp: ECSScaleUp, + ECSScaleDown: ECSScaleDown, +} + +func lookupCommonKey(b []byte) string { + if key, ok := commonKeys[string(b)]; ok { + return key + } + return string(b) +}