diff --git a/experimental/ftrace/ftrace.go b/experimental/ftrace/ftrace.go new file mode 100644 index 0000000000..95952c3bd0 --- /dev/null +++ b/experimental/ftrace/ftrace.go @@ -0,0 +1,204 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "os" + "path" + "regexp" + "strconv" + "strings" +) + +const ( + perms = 0777 +) + +var ( + lineMatcher = regexp.MustCompile(`^\s*[a-z\-]+\-(\d+)\s+\[(\d{3})] (?:\.|1){4} ([\d\.]+): (.*)$`) + enterMatcher = regexp.MustCompile(`^([\w_]+)\((.*)\)$`) + argMatcher = regexp.MustCompile(`(\w+): (\w+)`) + exitMatcher = regexp.MustCompile(`^([\w_]+) -> (\w+)$`) +) + +// Ftrace is a tracer using ftrace... +type Ftrace struct { + ftraceRoot string + root string + outstanding map[int]*syscall // map from pid (readlly tid) to outstanding syscall +} + +type syscall struct { + pid int + cpu int + ts float64 + name string + args map[string]string + returnCode int64 +} + +func findDebugFS() (string, error) { + contents, err := ioutil.ReadFile("/proc/mounts") + if err != nil { + return "", err + } + scanner := bufio.NewScanner(bytes.NewBuffer(contents)) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 3 { + continue + } + if fields[2] == "debugfs" { + return fields[1], nil + } + } + if err := scanner.Err(); err != nil { + return "", err + } + return "", fmt.Errorf("Not found") +} + +// NewFtracer constucts a new Ftrace instance. +func NewFtracer() (*Ftrace, error) { + root, err := findDebugFS() + if err != nil { + return nil, err + } + scopeRoot := path.Join(root, "tracing", "instances", "scope") + if err := os.Mkdir(scopeRoot, perms); err != nil && os.IsExist(err) { + if err := os.Remove(scopeRoot); err != nil { + return nil, err + } + if err := os.Mkdir(scopeRoot, perms); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + return &Ftrace{ + ftraceRoot: root, + root: scopeRoot, + outstanding: map[int]*syscall{}, + }, nil +} + +func (f *Ftrace) destroy() error { + return os.Remove(f.root) +} + +func (f *Ftrace) enableTracing() error { + // need to enable tracing at root to get trace_pipe to block in my instance. Weird + if err := ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms); err != nil { + return err + } + return ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("1"), perms) +} + +func (f *Ftrace) disableTracing() error { + if err := ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("0"), perms); err != nil { + return err + } + + return ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms) +} + +func (f *Ftrace) enableEvent(class, event string) error { + return ioutil.WriteFile(path.Join(f.root, "events", class, event, "enable"), []byte("1"), perms) +} + +func mustAtoi(a string) int { + i, err := strconv.Atoi(a) + if err != nil { + panic(err) + } + return i +} + +func mustAtof(a string) float64 { + i, err := strconv.ParseFloat(a, 64) + if err != nil { + panic(err) + } + return i +} + +func (f *Ftrace) events(out chan<- *syscall) { + file, err := os.Open(path.Join(f.root, "trace_pipe")) + if err != nil { + panic(err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + matches := lineMatcher.FindStringSubmatch(scanner.Text()) + if matches == nil { + continue + } + pid := mustAtoi(matches[1]) + log := matches[4] + + if enterMatches := enterMatcher.FindStringSubmatch(log); enterMatches != nil { + name := enterMatches[1] + args := map[string]string{} + for _, arg := range argMatcher.FindAllStringSubmatch(enterMatches[2], -1) { + args[arg[1]] = arg[2] + } + + s := &syscall{ + pid: pid, + cpu: mustAtoi(matches[2]), + ts: mustAtof(matches[3]), + name: strings.TrimPrefix(name, "sys_"), + args: args, + } + + f.outstanding[pid] = s + + } else if exitMatches := exitMatcher.FindStringSubmatch(log); exitMatches != nil { + s, ok := f.outstanding[pid] + if !ok { + continue + } + delete(f.outstanding, pid) + returnCode, err := strconv.ParseUint(exitMatches[2], 0, 64) + if err != nil { + panic(err) + } + s.returnCode = int64(returnCode) + out <- s + } else { + fmt.Printf("Unmatched: %s", log) + } + } + + if err := scanner.Err(); err != nil { + panic(err) + } +} + +func (f *Ftrace) start() error { + for _, e := range []struct{ class, event string }{ + {"syscalls", "sys_enter_connect"}, + {"syscalls", "sys_exit_connect"}, + {"syscalls", "sys_enter_accept"}, + {"syscalls", "sys_exit_accept"}, + {"syscalls", "sys_enter_accept4"}, + {"syscalls", "sys_exit_accept4"}, + {"syscalls", "sys_enter_close"}, + {"syscalls", "sys_exit_close"}, + } { + if err := f.enableEvent(e.class, e.event); err != nil { + return err + } + } + + return f.enableTracing() +} + +func (f *Ftrace) stop() error { + defer f.destroy() + return f.disableTracing() +} diff --git a/experimental/ftrace/main.go b/experimental/ftrace/main.go new file mode 100644 index 0000000000..fe7789678a --- /dev/null +++ b/experimental/ftrace/main.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "strconv" + "time" + + "github.com/bluele/gcache" +) + +const cacheSize = 500 + +// On every connect and accept, we lookup the local addr +// As this is expensive, we cache the result +var fdAddrCache = gcache.New(cacheSize).LRU().Expiration(15 * time.Second).Build() + +type fdCacheKey struct { + pid int + fd int +} +type fdCacheValue struct { + addr uint32 + port uint16 +} + +func getCachedLocalAddr(pid, fd int) (uint32, uint16, error) { + key := fdCacheKey{pid, fd} + val, err := fdAddrCache.Get(key) + if val != nil { + return val.(fdCacheValue).addr, val.(fdCacheValue).port, nil + } + + addr, port, err := getLocalAddr(pid, fd) + if err != nil { + return 0, 0, err + } + fdAddrCache.Set(key, fdCacheValue{addr, port}) + return addr, port, nil +} + +// On every connect or accept, we cache the syscall that caused +// it for matching with a connection from conntrack +var syscallCache = gcache.New(cacheSize).LRU().Expiration(15 * time.Second).Build() + +type syscallCacheKey struct { + localAddr uint32 + localPort uint16 +} +type syscallCacheValue *syscall + +// One ever conntrack connection, we cache it by local addr, port to match with +// a future syscall +var conntrackCache = gcache.New(cacheSize).LRU().Expiration(15 * time.Second).Build() + +type conntrackCacheKey syscallCacheKey + +// And keep a list of successfully matched connection, for us to close out +// when we get the close syscall + +func main() { + ftrace, err := NewFtracer() + if err != nil { + panic(err) + } + + ftrace.start() + defer ftrace.stop() + + syscalls := make(chan *syscall, 100) + go ftrace.events(syscalls) + + onConnection := func(s *syscall) { + fdStr, ok := s.args["fd"] + if !ok { + panic("no pid") + } + fd64, err := strconv.ParseInt(fdStr, 32, 16) + if err != nil { + panic(err) + } + fd := int(fd64) + + addr, port, err := getCachedLocalAddr(s.pid, fd) + if err != nil { + fmt.Printf("Failed to get local addr for pid=%d fd=%d: %v\n", s.pid, fd, err) + return + } + + fmt.Printf("%+v %d %d\n", s, addr, port) + syscallCache.Set(syscallCacheKey{addr, port}, s) + } + + onAccept := func(s *syscall) { + + } + + onClose := func(s *syscall) { + + } + + fmt.Println("Started") + + for { + select { + case s := <-syscalls: + + switch s.name { + case "connect": + onConnection(s) + case "accept", "accept4": + onAccept(s) + case "close": + onClose(s) + } + } + } +} diff --git a/experimental/ftrace/proc.go b/experimental/ftrace/proc.go new file mode 100644 index 0000000000..15320cdf9b --- /dev/null +++ b/experimental/ftrace/proc.go @@ -0,0 +1,94 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "regexp" + "strconv" +) + +const ( + socketPattern = `^socket:\[(\d+)\]$` + tcpPattern = `^\s*(?P\d+): (?P[A-F0-9]{8}):(?P[A-F0-9]{4}) ` + + `(?P[A-F0-9]{8}):(?P[A-F0-9]{4}) (?:[A-F0-9]{2}) (?:[A-F0-9]{8}):(?:[A-F0-9]{8}) ` + + `(?:[A-F0-9]{2}):(?:[A-F0-9]{8}) (?:[A-F0-9]{8}) \s+(?:\d+) \s+(?:\d+) (?P\d+)` +) + +var ( + socketRegex = regexp.MustCompile(socketPattern) + tcpRegexp = regexp.MustCompile(tcpPattern) +) + +func getLocalAddr(pid, fd int) (addr uint32, port uint16, err error) { + var ( + socket string + match []string + inode int + tcpFile *os.File + scanner *bufio.Scanner + candidate int + port64 int64 + addr64 int64 + ) + + socket, err = os.Readlink(fmt.Sprintf("/proc/%d/fd/%d", pid, fd)) + if err != nil { + return + } + + match = socketRegex.FindStringSubmatch(socket) + if match == nil { + err = fmt.Errorf("Fd %d not a socket", fd) + return + } + + inode, err = strconv.Atoi(match[1]) + if err != nil { + return + } + + tcpFile, err = os.Open(fmt.Sprintf("/proc/%d/net/tcp", pid)) + if err != nil { + return + } + defer tcpFile.Close() + + scanner = bufio.NewScanner(tcpFile) + for scanner.Scan() { + match = tcpRegexp.FindStringSubmatch(scanner.Text()) + if match == nil { + continue + } + + candidate, err = strconv.Atoi(match[6]) + if err != nil { + return + } + if candidate != inode { + continue + } + + addr64, err = strconv.ParseInt(match[2], 16, 32) + if err != nil { + return + } + addr = uint32(addr64) + + // use a 32 bit int for target, at the result is a uint16 + port64, err = strconv.ParseInt(match[3], 16, 32) + if err != nil { + return + } + port = uint16(port64) + + return + } + + if err = scanner.Err(); err != nil { + return + } + + err = fmt.Errorf("Fd %d not found for proc %d", fd, pid) + return +}