From 8a91c913b58d2b8bd7db35da31b0bc50817efb0d Mon Sep 17 00:00:00 2001 From: "M. Mert Yildiran" Date: Mon, 27 Jun 2022 16:17:22 +0300 Subject: [PATCH] Add `ABI` enum and morph the function according to the detected ABI --- tap/tlstapper/bpf/go_uprobes.c | 56 ++++++++++++----- tap/tlstapper/go_hooks.go | 20 ++++-- tap/tlstapper/tlstapper_bpfel_x86.go | 92 ++++++++++++++++------------ 3 files changed, 109 insertions(+), 59 deletions(-) diff --git a/tap/tlstapper/bpf/go_uprobes.c b/tap/tlstapper/bpf/go_uprobes.c index 19eef8603..dd1cfecd3 100644 --- a/tap/tlstapper/bpf/go_uprobes.c +++ b/tap/tlstapper/bpf/go_uprobes.c @@ -35,11 +35,12 @@ using `bpf_probe_read` calls in `go_crypto_tls_get_fd_from_tcp_conn` function. SOURCES: -Tracing Go Functions with eBPF (before 1.17): https://www.grant.pizza/blog/tracing-go-functions-with-ebpf-part-2/ +Tracing Go Functions with eBPF (<=1.16): https://www.grant.pizza/blog/tracing-go-functions-with-ebpf-part-2/ Challenges of BPF Tracing Go: https://blog.0x74696d.com/posts/challenges-of-bpf-tracing-go/ x86 calling conventions: https://en.wikipedia.org/wiki/X86_calling_conventions Plan 9 from Bell Labs: https://en.wikipedia.org/wiki/Plan_9_from_Bell_Labs The issue for calling convention change in Go: https://github.com/golang/go/issues/40724 +Go ABI0 (<=1.16) specification: https://go.dev/doc/asm Proposal of Register-based Go calling convention: https://go.googlesource.com/proposal/+/master/design/40724-register-calling.md Go internal ABI (1.17) specification: https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal-abi.md Go internal ABI (current) specification: https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/abi-internal.md @@ -58,6 +59,11 @@ Capstone Engine: https://www.capstone-engine.org/ #include "include/go_abi_internal.h" #include "include/go_types.h" +enum ABI { + ABI0=0, + ABIInternal=1, +}; + static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs *ctx) { struct go_interface conn; long err; @@ -91,7 +97,7 @@ static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs * return fd; } -static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context) { +static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, enum ABI abi) { __u64 pid_tgid = bpf_get_current_pid_tgid(); __u64 pid = pid_tgid >> 32; if (!should_tap(pid)) { @@ -124,7 +130,7 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf return; } -static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, __u32 flags) { +static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, __u32 flags, enum ABI abi) { __u64 pid_tgid = bpf_get_current_pid_tgid(); __u64 pid = pid_tgid >> 32; if (!should_tap(pid)) { @@ -170,22 +176,42 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct return; } -SEC("uprobe/go_crypto_tls_write") -void BPF_KPROBE(go_crypto_tls_write) { - go_crypto_tls_uprobe(ctx, &go_write_context); +SEC("uprobe/go_crypto_tls_abi0_write") +void BPF_KPROBE(go_crypto_tls_abi0_write) { + go_crypto_tls_uprobe(ctx, &go_write_context, ABI0); } -SEC("uprobe/go_crypto_tls_write_ex") -void BPF_KPROBE(go_crypto_tls_write_ex) { - go_crypto_tls_ex_uprobe(ctx, &go_write_context, 0); +SEC("uprobe/go_crypto_tls_abi0_write_ex") +void BPF_KPROBE(go_crypto_tls_abi0_write_ex) { + go_crypto_tls_ex_uprobe(ctx, &go_write_context, 0, ABI0); } -SEC("uprobe/go_crypto_tls_read") -void BPF_KPROBE(go_crypto_tls_read) { - go_crypto_tls_uprobe(ctx, &go_read_context); +SEC("uprobe/go_crypto_tls_abi0_read") +void BPF_KPROBE(go_crypto_tls_abi0_read) { + go_crypto_tls_uprobe(ctx, &go_read_context, ABI0); } -SEC("uprobe/go_crypto_tls_read_ex") -void BPF_KPROBE(go_crypto_tls_read_ex) { - go_crypto_tls_ex_uprobe(ctx, &go_read_context, FLAGS_IS_READ_BIT); +SEC("uprobe/go_crypto_tls_abi0_read_ex") +void BPF_KPROBE(go_crypto_tls_abi0_read_ex) { + go_crypto_tls_ex_uprobe(ctx, &go_read_context, FLAGS_IS_READ_BIT, ABI0); +} + +SEC("uprobe/go_crypto_tls_abi_internal_write") +void BPF_KPROBE(go_crypto_tls_abi_internal_write) { + go_crypto_tls_uprobe(ctx, &go_write_context, ABIInternal); +} + +SEC("uprobe/go_crypto_tls_abi_internal_write_ex") +void BPF_KPROBE(go_crypto_tls_abi_internal_write_ex) { + go_crypto_tls_ex_uprobe(ctx, &go_write_context, 0, ABIInternal); +} + +SEC("uprobe/go_crypto_tls_abi_internal_read") +void BPF_KPROBE(go_crypto_tls_abi_internal_read) { + go_crypto_tls_uprobe(ctx, &go_read_context, ABIInternal); +} + +SEC("uprobe/go_crypto_tls_abi_internal_read_ex") +void BPF_KPROBE(go_crypto_tls_abi_internal_read_ex) { + go_crypto_tls_ex_uprobe(ctx, &go_read_context, FLAGS_IS_READ_BIT, ABIInternal); } diff --git a/tap/tlstapper/go_hooks.go b/tap/tlstapper/go_hooks.go index 25cdc89fe..9a5af6669 100644 --- a/tap/tlstapper/go_hooks.go +++ b/tap/tlstapper/go_hooks.go @@ -31,9 +31,21 @@ func (s *goHooks) installUprobes(bpfObjects *tlsTapperObjects, filePath string) func (s *goHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable, offsets goOffsets) error { var err error + goCryptoTlsWrite := bpfObjects.GoCryptoTlsAbiInternalWrite + goCryptoTlsWriteEx := bpfObjects.GoCryptoTlsAbiInternalWriteEx + goCryptoTlsRead := bpfObjects.GoCryptoTlsAbiInternalRead + goCryptoTlsReadEx := bpfObjects.GoCryptoTlsAbiInternalReadEx + + if offsets.Abi == ABI0 { + goCryptoTlsWrite = bpfObjects.GoCryptoTlsAbi0Write + goCryptoTlsWriteEx = bpfObjects.GoCryptoTlsAbi0WriteEx + goCryptoTlsRead = bpfObjects.GoCryptoTlsAbi0Read + goCryptoTlsReadEx = bpfObjects.GoCryptoTlsAbi0ReadEx + } + // Symbol points to // [`crypto/tls.(*Conn).Write`](https://github.com/golang/go/blob/go1.17.6/src/crypto/tls/conn.go#L1099) - s.goWriteProbe, err = ex.Uprobe(goWriteSymbol, bpfObjects.GoCryptoTlsWrite, &link.UprobeOptions{ + s.goWriteProbe, err = ex.Uprobe(goWriteSymbol, goCryptoTlsWrite, &link.UprobeOptions{ Offset: offsets.GoWriteOffset.enter, }) @@ -42,7 +54,7 @@ func (s *goHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable } for _, offset := range offsets.GoWriteOffset.exits { - probe, err := ex.Uprobe(goWriteSymbol, bpfObjects.GoCryptoTlsWriteEx, &link.UprobeOptions{ + probe, err := ex.Uprobe(goWriteSymbol, goCryptoTlsWriteEx, &link.UprobeOptions{ Offset: offset, }) @@ -55,7 +67,7 @@ func (s *goHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable // Symbol points to // [`crypto/tls.(*Conn).Read`](https://github.com/golang/go/blob/go1.17.6/src/crypto/tls/conn.go#L1263) - s.goReadProbe, err = ex.Uprobe(goReadSymbol, bpfObjects.GoCryptoTlsRead, &link.UprobeOptions{ + s.goReadProbe, err = ex.Uprobe(goReadSymbol, goCryptoTlsRead, &link.UprobeOptions{ Offset: offsets.GoReadOffset.enter, }) @@ -64,7 +76,7 @@ func (s *goHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable } for _, offset := range offsets.GoReadOffset.exits { - probe, err := ex.Uprobe(goReadSymbol, bpfObjects.GoCryptoTlsReadEx, &link.UprobeOptions{ + probe, err := ex.Uprobe(goReadSymbol, goCryptoTlsReadEx, &link.UprobeOptions{ Offset: offset, }) diff --git a/tap/tlstapper/tlstapper_bpfel_x86.go b/tap/tlstapper/tlstapper_bpfel_x86.go index e0eecfc72..941d0699e 100644 --- a/tap/tlstapper/tlstapper_bpfel_x86.go +++ b/tap/tlstapper/tlstapper_bpfel_x86.go @@ -66,24 +66,28 @@ type tlsTapperSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type tlsTapperProgramSpecs struct { - GoCryptoTlsRead *ebpf.ProgramSpec `ebpf:"go_crypto_tls_read"` - GoCryptoTlsReadEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_read_ex"` - GoCryptoTlsWrite *ebpf.ProgramSpec `ebpf:"go_crypto_tls_write"` - GoCryptoTlsWriteEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_write_ex"` - SslRead *ebpf.ProgramSpec `ebpf:"ssl_read"` - SslReadEx *ebpf.ProgramSpec `ebpf:"ssl_read_ex"` - SslRetRead *ebpf.ProgramSpec `ebpf:"ssl_ret_read"` - SslRetReadEx *ebpf.ProgramSpec `ebpf:"ssl_ret_read_ex"` - SslRetWrite *ebpf.ProgramSpec `ebpf:"ssl_ret_write"` - SslRetWriteEx *ebpf.ProgramSpec `ebpf:"ssl_ret_write_ex"` - SslWrite *ebpf.ProgramSpec `ebpf:"ssl_write"` - SslWriteEx *ebpf.ProgramSpec `ebpf:"ssl_write_ex"` - SysEnterAccept4 *ebpf.ProgramSpec `ebpf:"sys_enter_accept4"` - SysEnterConnect *ebpf.ProgramSpec `ebpf:"sys_enter_connect"` - SysEnterRead *ebpf.ProgramSpec `ebpf:"sys_enter_read"` - SysEnterWrite *ebpf.ProgramSpec `ebpf:"sys_enter_write"` - SysExitAccept4 *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"` - SysExitConnect *ebpf.ProgramSpec `ebpf:"sys_exit_connect"` + GoCryptoTlsAbi0Read *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi0_read"` + GoCryptoTlsAbi0ReadEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi0_read_ex"` + GoCryptoTlsAbi0Write *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi0_write"` + GoCryptoTlsAbi0WriteEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi0_write_ex"` + GoCryptoTlsAbiInternalRead *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi_internal_read"` + GoCryptoTlsAbiInternalReadEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi_internal_read_ex"` + GoCryptoTlsAbiInternalWrite *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi_internal_write"` + GoCryptoTlsAbiInternalWriteEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi_internal_write_ex"` + SslRead *ebpf.ProgramSpec `ebpf:"ssl_read"` + SslReadEx *ebpf.ProgramSpec `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.ProgramSpec `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.ProgramSpec `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.ProgramSpec `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.ProgramSpec `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.ProgramSpec `ebpf:"ssl_write"` + SslWriteEx *ebpf.ProgramSpec `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.ProgramSpec `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.ProgramSpec `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.ProgramSpec `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.ProgramSpec `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.ProgramSpec `ebpf:"sys_exit_connect"` } // tlsTapperMapSpecs contains maps before they are loaded into the kernel. @@ -155,32 +159,40 @@ func (m *tlsTapperMaps) Close() error { // // It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign. type tlsTapperPrograms struct { - GoCryptoTlsRead *ebpf.Program `ebpf:"go_crypto_tls_read"` - GoCryptoTlsReadEx *ebpf.Program `ebpf:"go_crypto_tls_read_ex"` - GoCryptoTlsWrite *ebpf.Program `ebpf:"go_crypto_tls_write"` - GoCryptoTlsWriteEx *ebpf.Program `ebpf:"go_crypto_tls_write_ex"` - SslRead *ebpf.Program `ebpf:"ssl_read"` - SslReadEx *ebpf.Program `ebpf:"ssl_read_ex"` - SslRetRead *ebpf.Program `ebpf:"ssl_ret_read"` - SslRetReadEx *ebpf.Program `ebpf:"ssl_ret_read_ex"` - SslRetWrite *ebpf.Program `ebpf:"ssl_ret_write"` - SslRetWriteEx *ebpf.Program `ebpf:"ssl_ret_write_ex"` - SslWrite *ebpf.Program `ebpf:"ssl_write"` - SslWriteEx *ebpf.Program `ebpf:"ssl_write_ex"` - SysEnterAccept4 *ebpf.Program `ebpf:"sys_enter_accept4"` - SysEnterConnect *ebpf.Program `ebpf:"sys_enter_connect"` - SysEnterRead *ebpf.Program `ebpf:"sys_enter_read"` - SysEnterWrite *ebpf.Program `ebpf:"sys_enter_write"` - SysExitAccept4 *ebpf.Program `ebpf:"sys_exit_accept4"` - SysExitConnect *ebpf.Program `ebpf:"sys_exit_connect"` + GoCryptoTlsAbi0Read *ebpf.Program `ebpf:"go_crypto_tls_abi0_read"` + GoCryptoTlsAbi0ReadEx *ebpf.Program `ebpf:"go_crypto_tls_abi0_read_ex"` + GoCryptoTlsAbi0Write *ebpf.Program `ebpf:"go_crypto_tls_abi0_write"` + GoCryptoTlsAbi0WriteEx *ebpf.Program `ebpf:"go_crypto_tls_abi0_write_ex"` + GoCryptoTlsAbiInternalRead *ebpf.Program `ebpf:"go_crypto_tls_abi_internal_read"` + GoCryptoTlsAbiInternalReadEx *ebpf.Program `ebpf:"go_crypto_tls_abi_internal_read_ex"` + GoCryptoTlsAbiInternalWrite *ebpf.Program `ebpf:"go_crypto_tls_abi_internal_write"` + GoCryptoTlsAbiInternalWriteEx *ebpf.Program `ebpf:"go_crypto_tls_abi_internal_write_ex"` + SslRead *ebpf.Program `ebpf:"ssl_read"` + SslReadEx *ebpf.Program `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.Program `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.Program `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.Program `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.Program `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.Program `ebpf:"ssl_write"` + SslWriteEx *ebpf.Program `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.Program `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.Program `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.Program `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.Program `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.Program `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.Program `ebpf:"sys_exit_connect"` } func (p *tlsTapperPrograms) Close() error { return _TlsTapperClose( - p.GoCryptoTlsRead, - p.GoCryptoTlsReadEx, - p.GoCryptoTlsWrite, - p.GoCryptoTlsWriteEx, + p.GoCryptoTlsAbi0Read, + p.GoCryptoTlsAbi0ReadEx, + p.GoCryptoTlsAbi0Write, + p.GoCryptoTlsAbi0WriteEx, + p.GoCryptoTlsAbiInternalRead, + p.GoCryptoTlsAbiInternalReadEx, + p.GoCryptoTlsAbiInternalWrite, + p.GoCryptoTlsAbiInternalWriteEx, p.SslRead, p.SslReadEx, p.SslRetRead,