From 66ffc2397b9fc174d87cf57599c40c00546cfe3f Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 20 Aug 2015 13:18:50 +0000 Subject: [PATCH 1/2] First working ftrace prototype. --- experimental/ftrace/main.go | 217 ++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 experimental/ftrace/main.go diff --git a/experimental/ftrace/main.go b/experimental/ftrace/main.go new file mode 100644 index 0000000000..0bf4bede7b --- /dev/null +++ b/experimental/ftrace/main.go @@ -0,0 +1,217 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "os" + "path" + "regexp" + "strconv" + "strings" +) + +const ( + perms = 0777 +) + +var ( + lineMatcher = regexp.MustCompile(`^\s*[a-z\-]+\-(\d+)\s+\[(\d{3})] (?:\.|1){4} ([\d\.]+): (.*)$`) + enterMatcher = regexp.MustCompile(`^([\w_]+)\((.*)\)$`) + argMatcher = regexp.MustCompile(`(\w+): (\w+)`) + exitMatcher = regexp.MustCompile(`^([\w_]+) -> (\w+)$`) +) + +// Ftrace is a tracer using ftrace... +type Ftrace struct { + ftraceRoot string + root string + outstanding map[int]*syscall // map from pid (readlly tid) to outstanding syscall +} + +type event struct { + pid int + cpu int + ts float64 +} + +type syscall struct { + event + name string + args map[string]string + returnCode int64 +} + +func findDebugFS() (string, error) { + contents, err := ioutil.ReadFile("/proc/mounts") + if err != nil { + return "", err + } + scanner := bufio.NewScanner(bytes.NewBuffer(contents)) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 3 { + continue + } + if fields[2] == "debugfs" { + return fields[1], nil + } + } + if err := scanner.Err(); err != nil { + return "", err + } + return "", fmt.Errorf("Not found") +} + +// NewFtrace constucts a new Ftrace instance. +func NewFtrace() (*Ftrace, error) { + root, err := findDebugFS() + if err != nil { + return nil, err + } + scopeRoot := path.Join(root, "tracing", "instances", "scope") + if err := os.Mkdir(scopeRoot, perms); err != nil && os.IsExist(err) { + if err := os.Remove(scopeRoot); err != nil { + return nil, err + } + if err := os.Mkdir(scopeRoot, perms); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + return &Ftrace{ + ftraceRoot: root, + root: scopeRoot, + outstanding: map[int]*syscall{}, + }, nil +} + +func (f *Ftrace) destroy() error { + return os.Remove(f.root) +} + +func (f *Ftrace) enableTracing() error { + // need to enable tracing at root to get trace_pipe to block in my instance. Weird + if err := ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms); err != nil { + return err + } + return ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("1"), perms) +} + +func (f *Ftrace) disableTracing() error { + if err := ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("0"), perms); err != nil { + return err + } + + return ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms) +} + +func (f *Ftrace) enableEvent(class, event string) error { + return ioutil.WriteFile(path.Join(f.root, "events", class, event, "enable"), []byte("1"), perms) +} + +func mustAtoi(a string) int { + i, err := strconv.Atoi(a) + if err != nil { + panic(err) + } + return i +} + +func mustAtof(a string) float64 { + i, err := strconv.ParseFloat(a, 64) + if err != nil { + panic(err) + } + return i +} + +func (f *Ftrace) events() error { + file, err := os.Open(path.Join(f.root, "trace_pipe")) + if err != nil { + return err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + matches := lineMatcher.FindStringSubmatch(scanner.Text()) + if matches == nil { + continue + } + e := event{ + pid: mustAtoi(matches[1]), + cpu: mustAtoi(matches[2]), + ts: mustAtof(matches[3]), + } + f.handleEvent(e, matches[4]) + } + + return scanner.Err() +} + +func (f *Ftrace) handleEvent(e event, log string) { + if matches := enterMatcher.FindStringSubmatch(log); matches != nil { + name := matches[1] + args := map[string]string{} + for _, arg := range argMatcher.FindAllStringSubmatch(matches[2], -1) { + args[arg[1]] = arg[2] + } + + s := &syscall{ + event: e, + name: name, + args: args, + } + + f.outstanding[e.pid] = s + return + } + + if matches := exitMatcher.FindStringSubmatch(log); matches != nil { + s, ok := f.outstanding[e.pid] + if !ok { + return + } + delete(f.outstanding, e.pid) + returnCode, err := strconv.ParseUint(matches[2], 0, 64) + if err != nil { + panic(err) + } + s.returnCode = int64(returnCode) + fmt.Printf("%+v\n", *s) + return + } +} + +func main() { + ftrace, err := NewFtrace() + if err != nil { + panic(err) + } + defer ftrace.destroy() + + for _, e := range []struct{ class, event string }{ + {"syscalls", "sys_enter_connect"}, + {"syscalls", "sys_exit_connect"}, + {"syscalls", "sys_enter_accept"}, + {"syscalls", "sys_exit_accept"}, + {"syscalls", "sys_enter_accept4"}, + {"syscalls", "sys_exit_accept4"}, + } { + if err := ftrace.enableEvent(e.class, e.event); err != nil { + panic(err) + } + } + + if err := ftrace.enableTracing(); err != nil { + panic(err) + } + if err := ftrace.events(); err != nil { + panic(err) + } + fmt.Println("done") + defer ftrace.disableTracing() +} From 27c42b0191269d3aad57a2164f05e67a7088a666 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 20 Aug 2015 17:11:19 +0000 Subject: [PATCH 2/2] Lookup local addr of fds we detect via ftrace. --- experimental/ftrace/ftrace.go | 204 ++++++++++++++++++++++++++++ experimental/ftrace/main.go | 244 ++++++++++------------------------ experimental/ftrace/proc.go | 94 +++++++++++++ 3 files changed, 370 insertions(+), 172 deletions(-) create mode 100644 experimental/ftrace/ftrace.go create mode 100644 experimental/ftrace/proc.go diff --git a/experimental/ftrace/ftrace.go b/experimental/ftrace/ftrace.go new file mode 100644 index 0000000000..95952c3bd0 --- /dev/null +++ b/experimental/ftrace/ftrace.go @@ -0,0 +1,204 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "os" + "path" + "regexp" + "strconv" + "strings" +) + +const ( + perms = 0777 +) + +var ( + lineMatcher = regexp.MustCompile(`^\s*[a-z\-]+\-(\d+)\s+\[(\d{3})] (?:\.|1){4} ([\d\.]+): (.*)$`) + enterMatcher = regexp.MustCompile(`^([\w_]+)\((.*)\)$`) + argMatcher = regexp.MustCompile(`(\w+): (\w+)`) + exitMatcher = regexp.MustCompile(`^([\w_]+) -> (\w+)$`) +) + +// Ftrace is a tracer using ftrace... +type Ftrace struct { + ftraceRoot string + root string + outstanding map[int]*syscall // map from pid (readlly tid) to outstanding syscall +} + +type syscall struct { + pid int + cpu int + ts float64 + name string + args map[string]string + returnCode int64 +} + +func findDebugFS() (string, error) { + contents, err := ioutil.ReadFile("/proc/mounts") + if err != nil { + return "", err + } + scanner := bufio.NewScanner(bytes.NewBuffer(contents)) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 3 { + continue + } + if fields[2] == "debugfs" { + return fields[1], nil + } + } + if err := scanner.Err(); err != nil { + return "", err + } + return "", fmt.Errorf("Not found") +} + +// NewFtracer constucts a new Ftrace instance. +func NewFtracer() (*Ftrace, error) { + root, err := findDebugFS() + if err != nil { + return nil, err + } + scopeRoot := path.Join(root, "tracing", "instances", "scope") + if err := os.Mkdir(scopeRoot, perms); err != nil && os.IsExist(err) { + if err := os.Remove(scopeRoot); err != nil { + return nil, err + } + if err := os.Mkdir(scopeRoot, perms); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + return &Ftrace{ + ftraceRoot: root, + root: scopeRoot, + outstanding: map[int]*syscall{}, + }, nil +} + +func (f *Ftrace) destroy() error { + return os.Remove(f.root) +} + +func (f *Ftrace) enableTracing() error { + // need to enable tracing at root to get trace_pipe to block in my instance. Weird + if err := ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms); err != nil { + return err + } + return ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("1"), perms) +} + +func (f *Ftrace) disableTracing() error { + if err := ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("0"), perms); err != nil { + return err + } + + return ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms) +} + +func (f *Ftrace) enableEvent(class, event string) error { + return ioutil.WriteFile(path.Join(f.root, "events", class, event, "enable"), []byte("1"), perms) +} + +func mustAtoi(a string) int { + i, err := strconv.Atoi(a) + if err != nil { + panic(err) + } + return i +} + +func mustAtof(a string) float64 { + i, err := strconv.ParseFloat(a, 64) + if err != nil { + panic(err) + } + return i +} + +func (f *Ftrace) events(out chan<- *syscall) { + file, err := os.Open(path.Join(f.root, "trace_pipe")) + if err != nil { + panic(err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + matches := lineMatcher.FindStringSubmatch(scanner.Text()) + if matches == nil { + continue + } + pid := mustAtoi(matches[1]) + log := matches[4] + + if enterMatches := enterMatcher.FindStringSubmatch(log); enterMatches != nil { + name := enterMatches[1] + args := map[string]string{} + for _, arg := range argMatcher.FindAllStringSubmatch(enterMatches[2], -1) { + args[arg[1]] = arg[2] + } + + s := &syscall{ + pid: pid, + cpu: mustAtoi(matches[2]), + ts: mustAtof(matches[3]), + name: strings.TrimPrefix(name, "sys_"), + args: args, + } + + f.outstanding[pid] = s + + } else if exitMatches := exitMatcher.FindStringSubmatch(log); exitMatches != nil { + s, ok := f.outstanding[pid] + if !ok { + continue + } + delete(f.outstanding, pid) + returnCode, err := strconv.ParseUint(exitMatches[2], 0, 64) + if err != nil { + panic(err) + } + s.returnCode = int64(returnCode) + out <- s + } else { + fmt.Printf("Unmatched: %s", log) + } + } + + if err := scanner.Err(); err != nil { + panic(err) + } +} + +func (f *Ftrace) start() error { + for _, e := range []struct{ class, event string }{ + {"syscalls", "sys_enter_connect"}, + {"syscalls", "sys_exit_connect"}, + {"syscalls", "sys_enter_accept"}, + {"syscalls", "sys_exit_accept"}, + {"syscalls", "sys_enter_accept4"}, + {"syscalls", "sys_exit_accept4"}, + {"syscalls", "sys_enter_close"}, + {"syscalls", "sys_exit_close"}, + } { + if err := f.enableEvent(e.class, e.event); err != nil { + return err + } + } + + return f.enableTracing() +} + +func (f *Ftrace) stop() error { + defer f.destroy() + return f.disableTracing() +} diff --git a/experimental/ftrace/main.go b/experimental/ftrace/main.go index 0bf4bede7b..fe7789678a 100644 --- a/experimental/ftrace/main.go +++ b/experimental/ftrace/main.go @@ -1,217 +1,117 @@ package main import ( - "bufio" - "bytes" "fmt" - "io/ioutil" - "os" - "path" - "regexp" "strconv" - "strings" -) + "time" -const ( - perms = 0777 + "github.com/bluele/gcache" ) -var ( - lineMatcher = regexp.MustCompile(`^\s*[a-z\-]+\-(\d+)\s+\[(\d{3})] (?:\.|1){4} ([\d\.]+): (.*)$`) - enterMatcher = regexp.MustCompile(`^([\w_]+)\((.*)\)$`) - argMatcher = regexp.MustCompile(`(\w+): (\w+)`) - exitMatcher = regexp.MustCompile(`^([\w_]+) -> (\w+)$`) -) +const cacheSize = 500 -// Ftrace is a tracer using ftrace... -type Ftrace struct { - ftraceRoot string - root string - outstanding map[int]*syscall // map from pid (readlly tid) to outstanding syscall -} +// On every connect and accept, we lookup the local addr +// As this is expensive, we cache the result +var fdAddrCache = gcache.New(cacheSize).LRU().Expiration(15 * time.Second).Build() -type event struct { +type fdCacheKey struct { pid int - cpu int - ts float64 + fd int } - -type syscall struct { - event - name string - args map[string]string - returnCode int64 +type fdCacheValue struct { + addr uint32 + port uint16 } -func findDebugFS() (string, error) { - contents, err := ioutil.ReadFile("/proc/mounts") - if err != nil { - return "", err - } - scanner := bufio.NewScanner(bytes.NewBuffer(contents)) - for scanner.Scan() { - fields := strings.Fields(scanner.Text()) - if len(fields) < 3 { - continue - } - if fields[2] == "debugfs" { - return fields[1], nil - } +func getCachedLocalAddr(pid, fd int) (uint32, uint16, error) { + key := fdCacheKey{pid, fd} + val, err := fdAddrCache.Get(key) + if val != nil { + return val.(fdCacheValue).addr, val.(fdCacheValue).port, nil } - if err := scanner.Err(); err != nil { - return "", err - } - return "", fmt.Errorf("Not found") -} -// NewFtrace constucts a new Ftrace instance. -func NewFtrace() (*Ftrace, error) { - root, err := findDebugFS() + addr, port, err := getLocalAddr(pid, fd) if err != nil { - return nil, err + return 0, 0, err } - scopeRoot := path.Join(root, "tracing", "instances", "scope") - if err := os.Mkdir(scopeRoot, perms); err != nil && os.IsExist(err) { - if err := os.Remove(scopeRoot); err != nil { - return nil, err - } - if err := os.Mkdir(scopeRoot, perms); err != nil { - return nil, err - } - } else if err != nil { - return nil, err - } - return &Ftrace{ - ftraceRoot: root, - root: scopeRoot, - outstanding: map[int]*syscall{}, - }, nil + fdAddrCache.Set(key, fdCacheValue{addr, port}) + return addr, port, nil } -func (f *Ftrace) destroy() error { - return os.Remove(f.root) -} +// On every connect or accept, we cache the syscall that caused +// it for matching with a connection from conntrack +var syscallCache = gcache.New(cacheSize).LRU().Expiration(15 * time.Second).Build() -func (f *Ftrace) enableTracing() error { - // need to enable tracing at root to get trace_pipe to block in my instance. Weird - if err := ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms); err != nil { - return err - } - return ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("1"), perms) +type syscallCacheKey struct { + localAddr uint32 + localPort uint16 } +type syscallCacheValue *syscall -func (f *Ftrace) disableTracing() error { - if err := ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("0"), perms); err != nil { - return err - } +// One ever conntrack connection, we cache it by local addr, port to match with +// a future syscall +var conntrackCache = gcache.New(cacheSize).LRU().Expiration(15 * time.Second).Build() - return ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms) -} +type conntrackCacheKey syscallCacheKey -func (f *Ftrace) enableEvent(class, event string) error { - return ioutil.WriteFile(path.Join(f.root, "events", class, event, "enable"), []byte("1"), perms) -} +// And keep a list of successfully matched connection, for us to close out +// when we get the close syscall -func mustAtoi(a string) int { - i, err := strconv.Atoi(a) +func main() { + ftrace, err := NewFtracer() if err != nil { panic(err) } - return i -} -func mustAtof(a string) float64 { - i, err := strconv.ParseFloat(a, 64) - if err != nil { - panic(err) - } - return i -} + ftrace.start() + defer ftrace.stop() -func (f *Ftrace) events() error { - file, err := os.Open(path.Join(f.root, "trace_pipe")) - if err != nil { - return err - } - defer file.Close() + syscalls := make(chan *syscall, 100) + go ftrace.events(syscalls) - scanner := bufio.NewScanner(file) - for scanner.Scan() { - matches := lineMatcher.FindStringSubmatch(scanner.Text()) - if matches == nil { - continue - } - e := event{ - pid: mustAtoi(matches[1]), - cpu: mustAtoi(matches[2]), - ts: mustAtof(matches[3]), + onConnection := func(s *syscall) { + fdStr, ok := s.args["fd"] + if !ok { + panic("no pid") } - f.handleEvent(e, matches[4]) - } - - return scanner.Err() -} - -func (f *Ftrace) handleEvent(e event, log string) { - if matches := enterMatcher.FindStringSubmatch(log); matches != nil { - name := matches[1] - args := map[string]string{} - for _, arg := range argMatcher.FindAllStringSubmatch(matches[2], -1) { - args[arg[1]] = arg[2] + fd64, err := strconv.ParseInt(fdStr, 32, 16) + if err != nil { + panic(err) } + fd := int(fd64) - s := &syscall{ - event: e, - name: name, - args: args, + addr, port, err := getCachedLocalAddr(s.pid, fd) + if err != nil { + fmt.Printf("Failed to get local addr for pid=%d fd=%d: %v\n", s.pid, fd, err) + return } - f.outstanding[e.pid] = s - return + fmt.Printf("%+v %d %d\n", s, addr, port) + syscallCache.Set(syscallCacheKey{addr, port}, s) } - if matches := exitMatcher.FindStringSubmatch(log); matches != nil { - s, ok := f.outstanding[e.pid] - if !ok { - return - } - delete(f.outstanding, e.pid) - returnCode, err := strconv.ParseUint(matches[2], 0, 64) - if err != nil { - panic(err) - } - s.returnCode = int64(returnCode) - fmt.Printf("%+v\n", *s) - return - } -} + onAccept := func(s *syscall) { -func main() { - ftrace, err := NewFtrace() - if err != nil { - panic(err) - } - defer ftrace.destroy() - - for _, e := range []struct{ class, event string }{ - {"syscalls", "sys_enter_connect"}, - {"syscalls", "sys_exit_connect"}, - {"syscalls", "sys_enter_accept"}, - {"syscalls", "sys_exit_accept"}, - {"syscalls", "sys_enter_accept4"}, - {"syscalls", "sys_exit_accept4"}, - } { - if err := ftrace.enableEvent(e.class, e.event); err != nil { - panic(err) - } } - if err := ftrace.enableTracing(); err != nil { - panic(err) + onClose := func(s *syscall) { + } - if err := ftrace.events(); err != nil { - panic(err) + + fmt.Println("Started") + + for { + select { + case s := <-syscalls: + + switch s.name { + case "connect": + onConnection(s) + case "accept", "accept4": + onAccept(s) + case "close": + onClose(s) + } + } } - fmt.Println("done") - defer ftrace.disableTracing() } diff --git a/experimental/ftrace/proc.go b/experimental/ftrace/proc.go new file mode 100644 index 0000000000..15320cdf9b --- /dev/null +++ b/experimental/ftrace/proc.go @@ -0,0 +1,94 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "regexp" + "strconv" +) + +const ( + socketPattern = `^socket:\[(\d+)\]$` + tcpPattern = `^\s*(?P\d+): (?P[A-F0-9]{8}):(?P[A-F0-9]{4}) ` + + `(?P[A-F0-9]{8}):(?P[A-F0-9]{4}) (?:[A-F0-9]{2}) (?:[A-F0-9]{8}):(?:[A-F0-9]{8}) ` + + `(?:[A-F0-9]{2}):(?:[A-F0-9]{8}) (?:[A-F0-9]{8}) \s+(?:\d+) \s+(?:\d+) (?P\d+)` +) + +var ( + socketRegex = regexp.MustCompile(socketPattern) + tcpRegexp = regexp.MustCompile(tcpPattern) +) + +func getLocalAddr(pid, fd int) (addr uint32, port uint16, err error) { + var ( + socket string + match []string + inode int + tcpFile *os.File + scanner *bufio.Scanner + candidate int + port64 int64 + addr64 int64 + ) + + socket, err = os.Readlink(fmt.Sprintf("/proc/%d/fd/%d", pid, fd)) + if err != nil { + return + } + + match = socketRegex.FindStringSubmatch(socket) + if match == nil { + err = fmt.Errorf("Fd %d not a socket", fd) + return + } + + inode, err = strconv.Atoi(match[1]) + if err != nil { + return + } + + tcpFile, err = os.Open(fmt.Sprintf("/proc/%d/net/tcp", pid)) + if err != nil { + return + } + defer tcpFile.Close() + + scanner = bufio.NewScanner(tcpFile) + for scanner.Scan() { + match = tcpRegexp.FindStringSubmatch(scanner.Text()) + if match == nil { + continue + } + + candidate, err = strconv.Atoi(match[6]) + if err != nil { + return + } + if candidate != inode { + continue + } + + addr64, err = strconv.ParseInt(match[2], 16, 32) + if err != nil { + return + } + addr = uint32(addr64) + + // use a 32 bit int for target, at the result is a uint16 + port64, err = strconv.ParseInt(match[3], 16, 32) + if err != nil { + return + } + port = uint16(port64) + + return + } + + if err = scanner.Err(); err != nil { + return + } + + err = fmt.Errorf("Fd %d not found for proc %d", fd, pid) + return +}