init

at-cluster/vendor/github.com/maruel/panicparse/stack/bucket.go

@@ -0,0 +1,105 @@
+// Copyright 2015 Marc-Antoine Ruel. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package stack
+
+import (
+	"sort"
+)
+
+// Similarity is the level at which two call lines arguments must match to be
+// considered similar enough to coalesce them.
+type Similarity int
+
+const (
+	// ExactFlags requires same bits (e.g. Locked).
+	ExactFlags Similarity = iota
+	// ExactLines requests the exact same arguments on the call line.
+	ExactLines
+	// AnyPointer considers different pointers a similar call line.
+	AnyPointer
+	// AnyValue accepts any value as similar call line.
+	AnyValue
+)
+
+// Aggregate merges similar goroutines into buckets.
+//
+// The buckets are ordered in library provided order of relevancy. You can
+// reorder at your chosing.
+func Aggregate(goroutines []*Goroutine, similar Similarity) []*Bucket {
+	type count struct {
+		ids   []int
+		first bool
+	}
+	b := map[*Signature]*count{}
+	// O(n²). Fix eventually.
+	for _, routine := range goroutines {
+		found := false
+		for key, c := range b {
+			// When a match is found, this effectively drops the other goroutine ID.
+			if key.similar(&routine.Signature, similar) {
+				found = true
+				c.ids = append(c.ids, routine.ID)
+				c.first = c.first || routine.First
+				if !key.equal(&routine.Signature) {
+					// Almost but not quite equal. There's different pointers passed
+					// around but the same values. Zap out the different values.
+					newKey := key.merge(&routine.Signature)
+					b[newKey] = c
+					delete(b, key)
+				}
+				break
+			}
+		}
+		if !found {
+			// Create a copy of the Signature, since it will be mutated.
+			key := &Signature{}
+			*key = routine.Signature
+			b[key] = &count{ids: []int{routine.ID}, first: routine.First}
+		}
+	}
+	out := make(buckets, 0, len(b))
+	for signature, c := range b {
+		sort.Ints(c.ids)
+		out = append(out, &Bucket{Signature: *signature, IDs: c.ids, First: c.first})
+	}
+	sort.Sort(out)
+	return out
+}
+
+// Bucket is a stack trace signature and the list of goroutines that fits this
+// signature.
+type Bucket struct {
+	Signature
+	// IDs is the ID of each Goroutine with this Signature.
+	IDs []int
+	// First is true if this Bucket contains the first goroutine, e.g. the one
+	// Signature that likely generated the panic() call, if any.
+	First bool
+}
+
+// less does reverse sort.
+func (b *Bucket) less(r *Bucket) bool {
+	if b.First || r.First {
+		return b.First
+	}
+	return b.Signature.less(&r.Signature)
+}
+
+//
+
+// buckets is a list of Bucket sorted by repeation count.
+type buckets []*Bucket
+
+func (b buckets) Len() int {
+	return len(b)
+}
+
+func (b buckets) Less(i, j int) bool {
+	return b[i].less(b[j])
+}
+
+func (b buckets) Swap(i, j int) {
+	b[j], b[i] = b[i], b[j]
+}

at-cluster/vendor/github.com/maruel/panicparse/stack/context.go

@@ -0,0 +1,750 @@
+// Copyright 2018 Marc-Antoine Ruel. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package stack
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/user"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// Context is a parsing context.
+//
+// It contains the deduced GOROOT and GOPATH, if guesspaths is true.
+type Context struct {
+	// Goroutines is the Goroutines found.
+	//
+	// They are in the order that they were printed.
+	Goroutines []*Goroutine
+
+	// GOROOT is the GOROOT as detected in the traceback, not the on the host.
+	//
+	// It can be empty if no root was determined, for example the traceback
+	// contains only non-stdlib source references.
+	//
+	// Empty is guesspaths was false.
+	GOROOT string
+	// GOPATHs is the GOPATH as detected in the traceback, with the value being
+	// the corresponding path mapped to the host.
+	//
+	// It can be empty if only stdlib code is in the traceback or if no local
+	// sources were matched up. In the general case there is only one entry in
+	// the map.
+	//
+	// Nil is guesspaths was false.
+	GOPATHs map[string]string
+
+	localgoroot  string
+	localgopaths []string
+}
+
+// ParseDump processes the output from runtime.Stack().
+//
+// Returns nil *Context if no stack trace was detected.
+//
+// It pipes anything not detected as a panic stack trace from r into out. It
+// assumes there is junk before the actual stack trace. The junk is streamed to
+// out.
+//
+// If guesspaths is false, no guessing of GOROOT and GOPATH is done, and Call
+// entites do not have LocalSrcPath and IsStdlib filled in.
+func ParseDump(r io.Reader, out io.Writer, guesspaths bool) (*Context, error) {
+	goroutines, err := parseDump(r, out)
+	if len(goroutines) == 0 {
+		return nil, err
+	}
+	c := &Context{
+		Goroutines:   goroutines,
+		localgoroot:  runtime.GOROOT(),
+		localgopaths: getGOPATHs(),
+	}
+	nameArguments(goroutines)
+	// Corresponding local values on the host for Context.
+	if guesspaths {
+		c.findRoots()
+		for _, r := range c.Goroutines {
+			// Note that this is important to call it even if
+			// c.GOROOT == c.localgoroot.
+			r.updateLocations(c.GOROOT, c.localgoroot, c.GOPATHs)
+		}
+	}
+	return c, err
+}
+
+// Private stuff.
+
+const (
+	lockedToThread   = "locked to thread"
+	elided           = "...additional frames elided..."
+	raceHeaderFooter = "=================="
+	raceHeader       = "WARNING: DATA RACE"
+)
+
+// These are effectively constants.
+var (
+	// TODO(maruel): Handle corrupted stack cases:
+	// - missed stack barrier
+	// - found next stack barrier at 0x123; expected
+	// - runtime: unexpected return pc for FUNC_NAME called from 0x123
+
+	reRoutineHeader = regexp.MustCompile("^([ \t]*)goroutine (\\d+) \\[([^\\]]+)\\]\\:$")
+	reMinutes       = regexp.MustCompile("^(\\d+) minutes$")
+	reUnavail       = regexp.MustCompile("^(?:\t| +)goroutine running on other thread; stack unavailable")
+	// See gentraceback() in src/runtime/traceback.go for more information.
+	// - Sometimes the source file comes up as "<autogenerated>". It is the
+	//   compiler than generated these, not the runtime.
+	// - The tab may be replaced with spaces when a user copy-paste it, handle
+	//   this transparently.
+	// - "runtime.gopanic" is explicitly replaced with "panic" by gentraceback().
+	// - The +0x123 byte offset is printed when frame.pc > _func.entry. _func is
+	//   generated by the linker.
+	// - The +0x123 byte offset is not included with generated code, e.g. unnamed
+	//   functions "func·006()" which is generally go func() { ... }()
+	//   statements. Since the _func is generated at runtime, it's probably why
+	//   _func.entry is not set.
+	// - C calls may have fp=0x123 sp=0x123 appended. I think it normally happens
+	//   when a signal is not correctly handled. It is printed with m.throwing>0.
+	//   These are discarded.
+	// - For cgo, the source file may be "??".
+	reFile = regexp.MustCompile("^(?:\t| +)(\\?\\?|\\<autogenerated\\>|.+\\.(?:c|go|s))\\:(\\d+)(?:| \\+0x[0-9a-f]+)(?:| fp=0x[0-9a-f]+ sp=0x[0-9a-f]+(?:| pc=0x[0-9a-f]+))$")
+	// Sadly, it doesn't note the goroutine number so we could cascade them per
+	// parenthood.
+	reCreated = regexp.MustCompile("^created by (.+)$")
+	reFunc    = regexp.MustCompile("^(.+)\\((.*)\\)$")
+
+	// See https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/tsan/rtl/tsan_report.cc
+	// for the code generating these messages. Please note only the block in
+	//   #else  // #if !SANITIZER_GO
+	// is used.
+	// TODO(maruel): "    [failed to restore the stack]\n\n"
+	// TODO(maruel): "Global var %s of size %zu at %p declared at %s:%zu\n"
+	reRaceOperationHeader             = regexp.MustCompile("^(Read|Write) at (0x[0-9a-f]+) by goroutine (\\d+):$")
+	reRacePreviousOperationHeader     = regexp.MustCompile("^Previous (read|write) at (0x[0-9a-f]+) by goroutine (\\d+):$")
+	reRacePreviousOperationMainHeader = regexp.MustCompile("^Previous (read|write) at (0x[0-9a-f]+) by main goroutine:$")
+	reRaceGoroutine                   = regexp.MustCompile("^Goroutine (\\d+) \\((running|finished)\\) created at:$")
+)
+
+func parseDump(r io.Reader, out io.Writer) ([]*Goroutine, error) {
+	scanner := bufio.NewScanner(r)
+	scanner.Split(scanLines)
+	// Do not enable race detection parsing yet, since it cannot be returned in
+	// Context at the moment.
+	s := scanningState{}
+	for scanner.Scan() {
+		line, err := s.scan(scanner.Text())
+		if line != "" {
+			_, _ = io.WriteString(out, line)
+		}
+		if err != nil {
+			return s.goroutines, err
+		}
+	}
+	return s.goroutines, scanner.Err()
+}
+
+// scanLines is similar to bufio.ScanLines except that it:
+//     - doesn't drop '\n'
+//     - doesn't strip '\r'
+//     - returns when the data is bufio.MaxScanTokenSize bytes
+func scanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+	if i := bytes.IndexByte(data, '\n'); i >= 0 {
+		return i + 1, data[0 : i+1], nil
+	}
+	if atEOF {
+		return len(data), data, nil
+	}
+	if len(data) >= bufio.MaxScanTokenSize {
+		// Returns the line even if it is not at EOF nor has a '\n', otherwise the
+		// scanner will return bufio.ErrTooLong which is definitely not what we
+		// want.
+		return len(data), data, nil
+	}
+	return 0, nil, nil
+}
+
+// state is the state of the scan to detect and process a stack trace.
+type state int
+
+// Initial state is normal. Other states are when a stack trace is detected.
+const (
+	// Outside a stack trace.
+	// to: gotRoutineHeader, raceHeader1
+	normal state = iota
+
+	// Panic stack trace:
+
+	// Empty line between goroutines.
+	// from: gotFileCreated, gotFileFunc
+	// to: gotRoutineHeader, normal
+	betweenRoutine
+	// Goroutine header was found, e.g. "goroutine 1 [running]:"
+	// from: normal
+	// to: gotUnavail, gotFunc
+	gotRoutineHeader
+	// Function call line was found, e.g. "main.main()"
+	// from: gotRoutineHeader
+	// to: gotFile
+	gotFunc
+	// Goroutine creation line was found, e.g. "created by main.glob..func4"
+	// from: gotFileFunc
+	// to: gotFileCreated
+	gotCreated
+	// File header was found, e.g. "\t/foo/bar/baz.go:116 +0x35"
+	// from: gotFunc
+	// to: gotFunc, gotCreated, betweenRoutine, normal
+	gotFileFunc
+	// File header was found, e.g. "\t/foo/bar/baz.go:116 +0x35"
+	// from: gotCreated
+	// to: betweenRoutine, normal
+	gotFileCreated
+	// State when the goroutine stack is instead is reUnavail.
+	// from: gotRoutineHeader
+	// to: betweenRoutine, gotCreated
+	gotUnavail
+
+	// Race detector:
+
+	// Got "=================="
+	// from: normal
+	// to: normal, gotRaceHeader
+	gotRaceHeader1
+	// Got "WARNING: DATA RACE"
+	// from: gotRaceHeader1
+	// to: normal, gotRaceOperationHeader
+	gotRaceHeader
+	// A race operation was found, e.g. "Read at 0x00c0000e4030 by goroutine 7:"
+	// from: gotRaceHeader
+	// to: normal, gotRaceOperationFunc
+	gotRaceOperationHeader
+	// Function that caused the race, e.g. "  main.panicRace.func1()"
+	// from: gotRaceOperationHeader
+	// to: normal, gotRaceOperationFile
+	gotRaceOperationFunc
+	// Function that caused the race, e.g. "  main.panicRace.func1()"
+	// from: gotRaceOperationFunc
+	// to: normal, betweenRaces
+	gotRaceOperationFile
+	// Goroutine header, e.g. "Goroutine 7 (running) created at:"
+	// from: betweenRaces
+	// to: normal, gotRaceOperationHeader
+	gotRaceGoroutineHeader
+	// Function that caused the race, e.g. "  main.panicRace.func1()"
+	// from: gotRaceGoroutineHeader
+	// to: normal, gotRaceGoroutineFile
+	gotRaceGoroutineFunc
+	// Function that caused the race, e.g. "  main.panicRace.func1()"
+	// from: gotRaceGoroutineFunc
+	// to: normal, betweenRaces
+	gotRaceGoroutineFile
+	// Empty line between goroutines.
+	// from: gotRaceOperationFile
+	// to: normal, gotRaceOperationHeader
+	betweenRaces
+)
+
+type raceOp struct {
+	write bool
+	addr  uint64
+	id    int
+}
+
+// scanningState is the state of the scan to detect and process a stack trace
+// and stores the traces found.
+type scanningState struct {
+	// Determines if race detection is enabled. Currently false since scan()
+	// would swallow the race detector output, but the data is not part of
+	// Context yet.
+	raceDetectionEnabled bool
+
+	// goroutines contains all the goroutines found.
+	goroutines []*Goroutine
+
+	state  state
+	prefix string
+	races  []raceOp
+}
+
+// scan scans one line, updates goroutines and move to the next state.
+func (s *scanningState) scan(line string) (string, error) {
+	var cur *Goroutine
+	if len(s.goroutines) != 0 {
+		cur = s.goroutines[len(s.goroutines)-1]
+	}
+	trimmed := line
+	if strings.HasSuffix(line, "\r\n") {
+		trimmed = line[:len(line)-2]
+	} else if strings.HasSuffix(line, "\n") {
+		trimmed = line[:len(line)-1]
+	} else {
+		// There's two cases:
+		// - It's the end of the stream and it's not terminating with EOL character.
+		// - The line is longer than bufio.MaxScanTokenSize
+		if s.state == normal {
+			return line, nil
+		}
+		// Let it flow. It's possible the last line was trimmed and we still want to parse it.
+	}
+
+	if trimmed != "" && s.prefix != "" {
+		// This can only be the case if s.state != normal or the line is empty.
+		if !strings.HasPrefix(trimmed, s.prefix) {
+			prefix := s.prefix
+			s.state = normal
+			s.prefix = ""
+			return "", fmt.Errorf("inconsistent indentation: %q, expected %q", trimmed, prefix)
+		}
+		trimmed = trimmed[len(s.prefix):]
+	}
+
+	switch s.state {
+	case normal:
+		// We could look for '^panic:' but this is more risky, there can be a lot
+		// of junk between this and the stack dump.
+		fallthrough
+	case betweenRoutine:
+		// Look for a goroutine header.
+		if match := reRoutineHeader.FindStringSubmatch(trimmed); match != nil {
+			if id, err := strconv.Atoi(match[2]); err == nil {
+				// See runtime/traceback.go.
+				// "<state>, \d+ minutes, locked to thread"
+				items := strings.Split(match[3], ", ")
+				sleep := 0
+				locked := false
+				for i := 1; i < len(items); i++ {
+					if items[i] == lockedToThread {
+						locked = true
+						continue
+					}
+					// Look for duration, if any.
+					if match2 := reMinutes.FindStringSubmatch(items[i]); match2 != nil {
+						sleep, _ = strconv.Atoi(match2[1])
+					}
+				}
+				g := &Goroutine{
+					Signature: Signature{
+						State:    items[0],
+						SleepMin: sleep,
+						SleepMax: sleep,
+						Locked:   locked,
+					},
+					ID:    id,
+					First: len(s.goroutines) == 0,
+				}
+				s.goroutines = append(s.goroutines, g)
+				s.state = gotRoutineHeader
+				s.prefix = match[1]
+				return "", nil
+			}
+		}
+		// Switch to race detection mode.
+		if s.raceDetectionEnabled && trimmed == raceHeaderFooter {
+			s.state = gotRaceHeader1
+			// Send the line to the user.
+			return line, nil
+		}
+		// Fallthrough.
+		s.state = normal
+		s.prefix = ""
+		return line, nil
+
+	case gotRoutineHeader:
+		if reUnavail.MatchString(trimmed) {
+			// Generate a fake stack entry.
+			cur.Stack.Calls = []Call{{SrcPath: "<unavailable>"}}
+			// Next line is expected to be an empty line.
+			s.state = gotUnavail
+			return "", nil
+		}
+		call, err := parseFunc(trimmed)
+		if call != nil {
+			cur.Stack.Calls = append(cur.Stack.Calls, *call)
+			s.state = gotFunc
+			return "", err
+		}
+		return "", fmt.Errorf("expected a function after a goroutine header, got: %q", strings.TrimSpace(trimmed))
+
+	case gotFunc:
+		// Look for a file.
+		if match := reFile.FindStringSubmatch(trimmed); match != nil {
+			num, err := strconv.Atoi(match[2])
+			if err != nil {
+				return "", fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(trimmed))
+			}
+			// cur.Stack.Calls is guaranteed to have at least one item.
+			i := len(cur.Stack.Calls) - 1
+			cur.Stack.Calls[i].SrcPath = match[1]
+			cur.Stack.Calls[i].Line = num
+			s.state = gotFileFunc
+			return "", nil
+		}
+		return "", fmt.Errorf("expected a file after a function, got: %q", strings.TrimSpace(trimmed))
+
+	case gotCreated:
+		// Look for a file.
+		if match := reFile.FindStringSubmatch(trimmed); match != nil {
+			num, err := strconv.Atoi(match[2])
+			if err != nil {
+				return "", fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(trimmed))
+			}
+			cur.CreatedBy.SrcPath = match[1]
+			cur.CreatedBy.Line = num
+			s.state = gotFileCreated
+			return "", nil
+		}
+		return "", fmt.Errorf("expected a file after a created line, got: %q", trimmed)
+
+	case gotFileFunc:
+		if match := reCreated.FindStringSubmatch(trimmed); match != nil {
+			cur.CreatedBy.Func.Raw = match[1]
+			s.state = gotCreated
+			return "", nil
+		}
+		if elided == trimmed {
+			cur.Stack.Elided = true
+			// TODO(maruel): New state.
+			return "", nil
+		}
+		call, err := parseFunc(trimmed)
+		if call != nil {
+			cur.Stack.Calls = append(cur.Stack.Calls, *call)
+			s.state = gotFunc
+			return "", err
+		}
+		if trimmed == "" {
+			s.state = betweenRoutine
+			return "", nil
+		}
+		// Back to normal state.
+		s.state = normal
+		s.prefix = ""
+		return line, nil
+
+	case gotFileCreated:
+		if trimmed == "" {
+			s.state = betweenRoutine
+			return "", nil
+		}
+		s.state = normal
+		s.prefix = ""
+		return line, nil
+
+	case gotUnavail:
+		if trimmed == "" {
+			s.state = betweenRoutine
+			return "", nil
+		}
+		if match := reCreated.FindStringSubmatch(trimmed); match != nil {
+			cur.CreatedBy.Func.Raw = match[1]
+			s.state = gotCreated
+			return "", nil
+		}
+		return "", fmt.Errorf("expected empty line after unavailable stack, got: %q", strings.TrimSpace(trimmed))
+
+	case gotRaceHeader1:
+		if raceHeader == trimmed {
+			s.state = gotRaceHeader
+			// Send the line to the user.
+			return line, nil
+		}
+		s.state = normal
+		return line, nil
+
+	case gotRaceHeader:
+		if match := reRaceOperationHeader.FindStringSubmatch(trimmed); match != nil {
+			w := match[1] == "Write"
+			addr, err := strconv.ParseUint(match[2], 0, 64)
+			if err != nil {
+				return "", fmt.Errorf("failed to parse address on line: %q", strings.TrimSpace(trimmed))
+			}
+			id, err := strconv.Atoi(match[3])
+			if err != nil {
+				return "", fmt.Errorf("failed to parse goroutine id on line: %q", strings.TrimSpace(trimmed))
+			}
+			s.races = append(s.races, raceOp{w, addr, id})
+			s.state = gotRaceOperationHeader
+			return "", nil
+		}
+		s.state = normal
+		return line, nil
+
+	case gotRaceOperationHeader:
+		call, err := parseFunc(trimmed)
+		if call != nil {
+			// TODO(maruel): Figure out.
+			//cur.Stack.Calls = append(cur.Stack.Calls, *call)
+			s.state = gotRaceOperationFunc
+			return "", err
+		}
+		return "", fmt.Errorf("expected a function after a race operation, got: %q", trimmed)
+
+	case gotRaceGoroutineHeader:
+		call, err := parseFunc(strings.TrimLeft(trimmed, "\t "))
+		if call != nil {
+			cur.Stack.Calls = append(cur.Stack.Calls, *call)
+			s.state = gotRaceGoroutineFunc
+			return "", err
+		}
+		return "", fmt.Errorf("expected a function after a race operation, got: %q", trimmed)
+
+	case gotRaceOperationFunc:
+		if match := reFile.FindStringSubmatch(trimmed); match != nil {
+			_, err := strconv.Atoi(match[2])
+			if err != nil {
+				return "", fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(trimmed))
+			}
+			/* TODO(maruel): Figure out.
+			// cur.Stack.Calls is guaranteed to have at least one item.
+			i := len(cur.Stack.Calls) - 1
+			cur.Stack.Calls[i].SrcPath = match[1]
+			cur.Stack.Calls[i].Line = num
+			*/
+			s.state = gotRaceOperationFile
+			return "", nil
+		}
+		return "", fmt.Errorf("expected a file after a race function, got: %q", trimmed)
+
+	case gotRaceGoroutineFunc:
+		if match := reFile.FindStringSubmatch(trimmed); match != nil {
+			num, err := strconv.Atoi(match[2])
+			if err != nil {
+				return "", fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(trimmed))
+			}
+			// cur.Stack.Calls is guaranteed to have at least one item.
+			i := len(cur.Stack.Calls) - 1
+			cur.Stack.Calls[i].SrcPath = match[1]
+			cur.Stack.Calls[i].Line = num
+			s.state = gotRaceGoroutineFile
+			return "", nil
+		}
+		return "", fmt.Errorf("expected a file after a race function, got: %q", trimmed)
+
+	case gotRaceOperationFile:
+		if trimmed == "" {
+			s.state = betweenRaces
+			return "", nil
+		}
+		return "", fmt.Errorf("expected an empty line after a race file, got: %q", trimmed)
+
+	case gotRaceGoroutineFile:
+		if trimmed == "" {
+			s.state = betweenRaces
+			return "", nil
+		}
+		if trimmed == raceHeaderFooter {
+			// Done.
+			s.state = normal
+			return "", nil
+		}
+		call, err := parseFunc(strings.TrimLeft(trimmed, "\t "))
+		if call != nil {
+			// TODO(maruel): Process match.
+			s.state = gotRaceGoroutineFunc
+			return "", err
+		}
+		return "", fmt.Errorf("expected a function or the end after a race file, got: %q", trimmed)
+
+	case betweenRaces:
+		// Either Previous or Goroutine.
+		if match := reRacePreviousOperationHeader.FindStringSubmatch(trimmed); match != nil {
+			w := match[1] == "write"
+			addr, err := strconv.ParseUint(match[2], 0, 64)
+			if err != nil {
+				return "", fmt.Errorf("failed to parse address on line: %q", strings.TrimSpace(trimmed))
+			}
+			id, err := strconv.Atoi(match[3])
+			if err != nil {
+				return "", fmt.Errorf("failed to parse goroutine id on line: %q", strings.TrimSpace(trimmed))
+			}
+			s.races = append(s.races, raceOp{w, addr, id})
+			s.state = gotRaceOperationHeader
+			return "", nil
+		}
+		if match := reRaceGoroutine.FindStringSubmatch(trimmed); match != nil {
+			id, err := strconv.Atoi(match[1])
+			if err != nil {
+				return "", fmt.Errorf("failed to parse goroutine id on line: %q", strings.TrimSpace(trimmed))
+			}
+			g := &Goroutine{
+				Signature: Signature{State: match[2]},
+				ID:        id,
+				First:     len(s.goroutines) == 0,
+			}
+			s.goroutines = append(s.goroutines, g)
+			s.state = gotRaceGoroutineHeader
+			return "", nil
+		}
+		return "", fmt.Errorf("expected an operator or goroutine, got: %q", trimmed)
+
+	default:
+		return "", errors.New("internal error")
+	}
+}
+
+// parseFunc only return an error if also returning a Call.
+func parseFunc(line string) (*Call, error) {
+	if match := reFunc.FindStringSubmatch(line); match != nil {
+		call := &Call{Func: Func{Raw: match[1]}}
+		for _, a := range strings.Split(match[2], ", ") {
+			if a == "..." {
+				call.Args.Elided = true
+				continue
+			}
+			if a == "" {
+				// Remaining values were dropped.
+				break
+			}
+			v, err := strconv.ParseUint(a, 0, 64)
+			if err != nil {
+				return call, fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(line))
+			}
+			call.Args.Values = append(call.Args.Values, Arg{Value: v})
+		}
+		return call, nil
+	}
+	return nil, nil
+}
+
+// hasPathPrefix returns true if any of s is the prefix of p.
+func hasPathPrefix(p string, s map[string]string) bool {
+	for prefix := range s {
+		if strings.HasPrefix(p, prefix+"/") {
+			return true
+		}
+	}
+	return false
+}
+
+// getFiles returns all the source files deduped and ordered.
+func getFiles(goroutines []*Goroutine) []string {
+	files := map[string]struct{}{}
+	for _, g := range goroutines {
+		for _, c := range g.Stack.Calls {
+			files[c.SrcPath] = struct{}{}
+		}
+	}
+	out := make([]string, 0, len(files))
+	for f := range files {
+		out = append(out, f)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// splitPath splits a path into its components.
+//
+// The first item has its initial path separator kept.
+func splitPath(p string) []string {
+	if p == "" {
+		return nil
+	}
+	var out []string
+	s := ""
+	for _, c := range p {
+		if c != '/' || (len(out) == 0 && strings.Count(s, "/") == len(s)) {
+			s += string(c)
+		} else if s != "" {
+			out = append(out, s)
+			s = ""
+		}
+	}
+	if s != "" {
+		out = append(out, s)
+	}
+	return out
+}
+
+// isFile returns true if the path is a valid file.
+func isFile(p string) bool {
+	// TODO(maruel): Is it faster to open the file or to stat it? Worth a perf
+	// test on Windows.
+	i, err := os.Stat(p)
+	return err == nil && !i.IsDir()
+}
+
+// rootedIn returns a root if the file split in parts is rooted in root.
+func rootedIn(root string, parts []string) string {
+	//log.Printf("rootIn(%s, %v)", root, parts)
+	for i := 1; i < len(parts); i++ {
+		suffix := filepath.Join(parts[i:]...)
+		if isFile(filepath.Join(root, suffix)) {
+			return filepath.Join(parts[:i]...)
+		}
+	}
+	return ""
+}
+
+// findRoots sets member GOROOT and GOPATHs.
+func (c *Context) findRoots() {
+	c.GOPATHs = map[string]string{}
+	for _, f := range getFiles(c.Goroutines) {
+		// TODO(maruel): Could a stack dump have mixed cases? I think it's
+		// possible, need to confirm and handle.
+		//log.Printf("  Analyzing %s", f)
+		if c.GOROOT != "" && strings.HasPrefix(f, c.GOROOT+"/") {
+			continue
+		}
+		if hasPathPrefix(f, c.GOPATHs) {
+			continue
+		}
+		parts := splitPath(f)
+		if c.GOROOT == "" {
+			if r := rootedIn(c.localgoroot, parts); r != "" {
+				c.GOROOT = r
+				//log.Printf("Found GOROOT=%s", c.GOROOT)
+				continue
+			}
+		}
+		found := false
+		for _, l := range c.localgopaths {
+			if r := rootedIn(l, parts); r != "" {
+				//log.Printf("Found GOPATH=%s", r)
+				c.GOPATHs[r] = l
+				found = true
+				break
+			}
+		}
+		if !found {
+			// If the source is not found, just too bad.
+			//log.Printf("Failed to find locally: %s / %s", f, goroot)
+		}
+	}
+}
+
+func getGOPATHs() []string {
+	var out []string
+	for _, v := range filepath.SplitList(os.Getenv("GOPATH")) {
+		// Disallow non-absolute paths?
+		if v != "" {
+			out = append(out, v)
+		}
+	}
+	if len(out) == 0 {
+		homeDir := ""
+		u, err := user.Current()
+		if err != nil {
+			homeDir = os.Getenv("HOME")
+			if homeDir == "" {
+				panic(fmt.Sprintf("Could not get current user or $HOME: %s\n", err.Error()))
+			}
+		} else {
+			homeDir = u.HomeDir
+		}
+		out = []string{homeDir + "go"}
+	}
+	return out
+}

at-cluster/vendor/github.com/maruel/panicparse/stack/source.go

@@ -0,0 +1,303 @@
+// Copyright 2015 Marc-Antoine Ruel. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+// This file contains the code to process sources, to be able to deduct the
+// original types.
+
+package stack
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io/ioutil"
+	"log"
+	"math"
+	"strings"
+)
+
+// cache is a cache of sources on the file system.
+type cache struct {
+	files  map[string][]byte
+	parsed map[string]*parsedFile
+}
+
+// Augment processes source files to improve calls to be more descriptive.
+//
+// It modifies goroutines in place. It requires calling ParseDump() with
+// guesspaths set to true to work properly.
+func Augment(goroutines []*Goroutine) {
+	c := &cache{}
+	for _, g := range goroutines {
+		c.augmentGoroutine(g)
+	}
+}
+
+// augmentGoroutine processes source files to improve call to be more
+// descriptive.
+//
+// It modifies the routine.
+func (c *cache) augmentGoroutine(goroutine *Goroutine) {
+	if c.files == nil {
+		c.files = map[string][]byte{}
+	}
+	if c.parsed == nil {
+		c.parsed = map[string]*parsedFile{}
+	}
+	// For each call site, look at the next call and populate it. Then we can
+	// walk back and reformat things.
+	for i := range goroutine.Stack.Calls {
+		c.load(goroutine.Stack.Calls[i].LocalSrcPath)
+	}
+
+	// Once all loaded, we can look at the next call when available.
+	for i := 0; i < len(goroutine.Stack.Calls)-1; i++ {
+		// Get the AST from the previous call and process the call line with it.
+		if f := c.getFuncAST(&goroutine.Stack.Calls[i]); f != nil {
+			processCall(&goroutine.Stack.Calls[i], f)
+		}
+	}
+}
+
+// Private stuff.
+
+// load loads a source file and parses the AST tree. Failures are ignored.
+func (c *cache) load(fileName string) {
+	if _, ok := c.parsed[fileName]; ok {
+		return
+	}
+	c.parsed[fileName] = nil
+	if !strings.HasSuffix(fileName, ".go") {
+		// Ignore C and assembly.
+		c.files[fileName] = nil
+		return
+	}
+	log.Printf("load(%s)", fileName)
+	if _, ok := c.files[fileName]; !ok {
+		var err error
+		if c.files[fileName], err = ioutil.ReadFile(fileName); err != nil {
+			log.Printf("Failed to read %s: %s", fileName, err)
+			c.files[fileName] = nil
+			return
+		}
+	}
+	fset := token.NewFileSet()
+	src := c.files[fileName]
+	parsed, err := parser.ParseFile(fset, fileName, src, 0)
+	if err != nil {
+		log.Printf("Failed to parse %s: %s", fileName, err)
+		return
+	}
+	// Convert the line number into raw file offset.
+	offsets := []int{0, 0}
+	start := 0
+	for l := 1; start < len(src); l++ {
+		start += bytes.IndexByte(src[start:], '\n') + 1
+		offsets = append(offsets, start)
+	}
+	c.parsed[fileName] = &parsedFile{offsets, parsed}
+}
+
+func (c *cache) getFuncAST(call *Call) *ast.FuncDecl {
+	if p := c.parsed[call.LocalSrcPath]; p != nil {
+		return p.getFuncAST(call.Func.Name(), call.Line)
+	}
+	return nil
+}
+
+type parsedFile struct {
+	lineToByteOffset []int
+	parsed           *ast.File
+}
+
+// getFuncAST gets the callee site function AST representation for the code
+// inside the function f at line l.
+func (p *parsedFile) getFuncAST(f string, l int) (d *ast.FuncDecl) {
+	if len(p.lineToByteOffset) <= l {
+		// The line number in the stack trace line does not exist in the file. That
+		// can only mean that the sources on disk do not match the sources used to
+		// build the binary.
+		// TODO(maruel): This should be surfaced, so that source parsing is
+		// completely ignored.
+		return
+	}
+
+	// Walk the AST to find the lineToByteOffset that fits the line number.
+	var lastFunc *ast.FuncDecl
+	var found ast.Node
+	// Inspect() goes depth first. This means for example that a function like:
+	// func a() {
+	//   b := func() {}
+	//   c()
+	// }
+	//
+	// Were we are looking at the c() call can return confused values. It is
+	// important to look at the actual ast.Node hierarchy.
+	ast.Inspect(p.parsed, func(n ast.Node) bool {
+		if d != nil {
+			return false
+		}
+		if n == nil {
+			return true
+		}
+		if found != nil {
+			// We are walking up.
+		}
+		if int(n.Pos()) >= p.lineToByteOffset[l] {
+			// We are expecting a ast.CallExpr node. It can be harder to figure out
+			// when there are multiple calls on a single line, as the stack trace
+			// doesn't have file byte offset information, only line based.
+			// gofmt will always format to one function call per line but there can
+			// be edge cases, like:
+			//   a = A{Foo(), Bar()}
+			d = lastFunc
+			//p.processNode(call, n)
+			return false
+		} else if f, ok := n.(*ast.FuncDecl); ok {
+			lastFunc = f
+		}
+		return true
+	})
+	return
+}
+
+func name(n ast.Node) string {
+	switch t := n.(type) {
+	case *ast.InterfaceType:
+		return "interface{}"
+	case *ast.Ident:
+		return t.Name
+	case *ast.SelectorExpr:
+		return t.Sel.Name
+	case *ast.StarExpr:
+		return "*" + name(t.X)
+	default:
+		return "<unknown>"
+	}
+}
+
+// fieldToType returns the type name and whether if it's an ellipsis.
+func fieldToType(f *ast.Field) (string, bool) {
+	switch arg := f.Type.(type) {
+	case *ast.ArrayType:
+		return "[]" + name(arg.Elt), false
+	case *ast.Ellipsis:
+		return name(arg.Elt), true
+	case *ast.FuncType:
+		// Do not print the function signature to not overload the trace.
+		return "func", false
+	case *ast.Ident:
+		return arg.Name, false
+	case *ast.InterfaceType:
+		return "interface{}", false
+	case *ast.SelectorExpr:
+		return arg.Sel.Name, false
+	case *ast.StarExpr:
+		return "*" + name(arg.X), false
+	case *ast.MapType:
+		return fmt.Sprintf("map[%s]%s", name(arg.Key), name(arg.Value)), false
+	case *ast.ChanType:
+		return fmt.Sprintf("chan %s", name(arg.Value)), false
+	default:
+		// TODO(maruel): Implement anything missing.
+		return "<unknown>", false
+	}
+}
+
+// extractArgumentsType returns the name of the type of each input argument.
+func extractArgumentsType(f *ast.FuncDecl) ([]string, bool) {
+	var fields []*ast.Field
+	if f.Recv != nil {
+		if len(f.Recv.List) != 1 {
+			panic("Expect only one receiver; please fix panicparse's code")
+		}
+		// If it is an object receiver (vs a pointer receiver), its address is not
+		// printed in the stack trace so it needs to be ignored.
+		if _, ok := f.Recv.List[0].Type.(*ast.StarExpr); ok {
+			fields = append(fields, f.Recv.List[0])
+		}
+	}
+	var types []string
+	extra := false
+	for _, arg := range append(fields, f.Type.Params.List...) {
+		// Assert that extra is only set on the last item of fields?
+		var t string
+		t, extra = fieldToType(arg)
+		mult := len(arg.Names)
+		if mult == 0 {
+			mult = 1
+		}
+		for i := 0; i < mult; i++ {
+			types = append(types, t)
+		}
+	}
+	return types, extra
+}
+
+// processCall walks the function and populate call accordingly.
+func processCall(call *Call, f *ast.FuncDecl) {
+	values := make([]uint64, len(call.Args.Values))
+	for i := range call.Args.Values {
+		values[i] = call.Args.Values[i].Value
+	}
+	index := 0
+	pop := func() uint64 {
+		if len(values) != 0 {
+			x := values[0]
+			values = values[1:]
+			index++
+			return x
+		}
+		return 0
+	}
+	popName := func() string {
+		n := call.Args.Values[index].Name
+		v := pop()
+		if len(n) == 0 {
+			return fmt.Sprintf("0x%x", v)
+		}
+		return n
+	}
+
+	types, extra := extractArgumentsType(f)
+	for i := 0; len(values) != 0; i++ {
+		var t string
+		if i >= len(types) {
+			if !extra {
+				// These are unexpected value! Print them as hex.
+				call.Args.Processed = append(call.Args.Processed, popName())
+				continue
+			}
+			t = types[len(types)-1]
+		} else {
+			t = types[i]
+		}
+		switch t {
+		case "float32":
+			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%g", math.Float32frombits(uint32(pop()))))
+		case "float64":
+			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%g", math.Float64frombits(pop())))
+		case "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", "uint64":
+			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%d", pop()))
+		case "string":
+			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s, len=%d)", t, popName(), pop()))
+		default:
+			if strings.HasPrefix(t, "*") {
+				call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s)", t, popName()))
+			} else if strings.HasPrefix(t, "[]") {
+				call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s len=%d cap=%d)", t, popName(), pop(), pop()))
+			} else {
+				// Assumes it's an interface. For now, discard the object value, which
+				// is probably not a good idea.
+				call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s)", t, popName()))
+				pop()
+			}
+		}
+		if len(values) == 0 && call.Args.Elided {
+			return
+		}
+	}
+}

at-cluster/vendor/github.com/maruel/panicparse/stack/stack.go

@@ -0,0 +1,588 @@
+// Copyright 2015 Marc-Antoine Ruel. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+// Package stack analyzes stack dump of Go processes and simplifies it.
+//
+// It is mostly useful on servers will large number of identical goroutines,
+// making the crash dump harder to read than strictly necessary.
+package stack
+
+import (
+	"fmt"
+	"math"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// Func is a function call.
+//
+// Go stack traces print a mangled function call, this wrapper unmangle the
+// string before printing and adds other filtering methods.
+type Func struct {
+	Raw string
+}
+
+// String is the fully qualified function name.
+//
+// Sadly Go is a bit confused when the package name doesn't match the directory
+// containing the source file and will use the directory name instead of the
+// real package name.
+func (f *Func) String() string {
+	s, _ := url.QueryUnescape(f.Raw)
+	return s
+}
+
+// Name is the naked function name.
+func (f *Func) Name() string {
+	parts := strings.SplitN(filepath.Base(f.Raw), ".", 2)
+	if len(parts) == 1 {
+		return parts[0]
+	}
+	return parts[1]
+}
+
+// PkgName is the package name for this function reference.
+func (f *Func) PkgName() string {
+	parts := strings.SplitN(filepath.Base(f.Raw), ".", 2)
+	if len(parts) == 1 {
+		return ""
+	}
+	s, _ := url.QueryUnescape(parts[0])
+	return s
+}
+
+// PkgDotName returns "<package>.<func>" format.
+func (f *Func) PkgDotName() string {
+	parts := strings.SplitN(filepath.Base(f.Raw), ".", 2)
+	s, _ := url.QueryUnescape(parts[0])
+	if len(parts) == 1 {
+		return parts[0]
+	}
+	if s != "" || parts[1] != "" {
+		return s + "." + parts[1]
+	}
+	return ""
+}
+
+// IsExported returns true if the function is exported.
+func (f *Func) IsExported() bool {
+	name := f.Name()
+	parts := strings.Split(name, ".")
+	r, _ := utf8.DecodeRuneInString(parts[len(parts)-1])
+	if unicode.ToUpper(r) == r {
+		return true
+	}
+	return f.PkgName() == "main" && name == "main"
+}
+
+// Arg is an argument on a Call.
+type Arg struct {
+	Value uint64 // Value is the raw value as found in the stack trace
+	Name  string // Name is a pseudo name given to the argument
+}
+
+// IsPtr returns true if we guess it's a pointer. It's only a guess, it can be
+// easily be confused by a bitmask.
+func (a *Arg) IsPtr() bool {
+	// Assumes all pointers are above 16Mb and positive.
+	return a.Value > 16*1024*1024 && a.Value < math.MaxInt64
+}
+
+func (a *Arg) String() string {
+	if a.Name != "" {
+		return a.Name
+	}
+	if a.Value == 0 {
+		return "0"
+	}
+	return fmt.Sprintf("0x%x", a.Value)
+}
+
+// Args is a series of function call arguments.
+type Args struct {
+	Values    []Arg    // Values is the arguments as shown on the stack trace. They are mangled via simplification.
+	Processed []string // Processed is the arguments generated from processing the source files. It can have a length lower than Values.
+	Elided    bool     // If set, it means there was a trailing ", ..."
+}
+
+func (a *Args) String() string {
+	var v []string
+	if len(a.Processed) != 0 {
+		v = make([]string, 0, len(a.Processed))
+		for _, item := range a.Processed {
+			v = append(v, item)
+		}
+	} else {
+		v = make([]string, 0, len(a.Values))
+		for _, item := range a.Values {
+			v = append(v, item.String())
+		}
+	}
+	if a.Elided {
+		v = append(v, "...")
+	}
+	return strings.Join(v, ", ")
+}
+
+// equal returns true only if both arguments are exactly equal.
+func (a *Args) equal(r *Args) bool {
+	if a.Elided != r.Elided || len(a.Values) != len(r.Values) {
+		return false
+	}
+	for i, l := range a.Values {
+		if l != r.Values[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// similar returns true if the two Args are equal or almost but not quite
+// equal.
+func (a *Args) similar(r *Args, similar Similarity) bool {
+	if a.Elided != r.Elided || len(a.Values) != len(r.Values) {
+		return false
+	}
+	if similar == AnyValue {
+		return true
+	}
+	for i, l := range a.Values {
+		switch similar {
+		case ExactFlags, ExactLines:
+			if l != r.Values[i] {
+				return false
+			}
+		default:
+			if l.IsPtr() != r.Values[i].IsPtr() || (!l.IsPtr() && l != r.Values[i]) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// merge merges two similar Args, zapping out differences.
+func (a *Args) merge(r *Args) Args {
+	out := Args{
+		Values: make([]Arg, len(a.Values)),
+		Elided: a.Elided,
+	}
+	for i, l := range a.Values {
+		if l != r.Values[i] {
+			out.Values[i].Name = "*"
+			out.Values[i].Value = l.Value
+		} else {
+			out.Values[i] = l
+		}
+	}
+	return out
+}
+
+// Call is an item in the stack trace.
+type Call struct {
+	SrcPath      string // Full path name of the source file as seen in the trace
+	LocalSrcPath string // Full path name of the source file as seen in the host.
+	Line         int    // Line number
+	Func         Func   // Fully qualified function name (encoded).
+	Args         Args   // Call arguments
+	IsStdlib     bool   // true if it is a Go standard library function. This includes the 'go test' generated main executable.
+}
+
+// equal returns true only if both calls are exactly equal.
+func (c *Call) equal(r *Call) bool {
+	return c.SrcPath == r.SrcPath && c.Line == r.Line && c.Func == r.Func && c.Args.equal(&r.Args)
+}
+
+// similar returns true if the two Call are equal or almost but not quite
+// equal.
+func (c *Call) similar(r *Call, similar Similarity) bool {
+	return c.SrcPath == r.SrcPath && c.Line == r.Line && c.Func == r.Func && c.Args.similar(&r.Args, similar)
+}
+
+// merge merges two similar Call, zapping out differences.
+func (c *Call) merge(r *Call) Call {
+	return Call{
+		SrcPath:      c.SrcPath,
+		Line:         c.Line,
+		Func:         c.Func,
+		Args:         c.Args.merge(&r.Args),
+		LocalSrcPath: c.LocalSrcPath,
+		IsStdlib:     c.IsStdlib,
+	}
+}
+
+// SrcName returns the base file name of the source file.
+func (c *Call) SrcName() string {
+	return filepath.Base(c.SrcPath)
+}
+
+// SrcLine returns "source.go:line", including only the base file name.
+func (c *Call) SrcLine() string {
+	return fmt.Sprintf("%s:%d", c.SrcName(), c.Line)
+}
+
+// FullSrcLine returns "/path/to/source.go:line".
+//
+// This file path is mutated to look like the local path.
+func (c *Call) FullSrcLine() string {
+	return fmt.Sprintf("%s:%d", c.SrcPath, c.Line)
+}
+
+// PkgSrc is one directory plus the file name of the source file.
+func (c *Call) PkgSrc() string {
+	return filepath.Join(filepath.Base(filepath.Dir(c.SrcPath)), c.SrcName())
+}
+
+// IsPkgMain returns true if it is in the main package.
+func (c *Call) IsPkgMain() bool {
+	return c.Func.PkgName() == "main"
+}
+
+const testMainSrc = "_test" + string(os.PathSeparator) + "_testmain.go"
+
+// updateLocations initializes LocalSrcPath and IsStdlib.
+func (c *Call) updateLocations(goroot, localgoroot string, gopaths map[string]string) {
+	if c.SrcPath != "" {
+		// Always check GOROOT first, then GOPATH.
+		if strings.HasPrefix(c.SrcPath, goroot) {
+			// Replace remote GOROOT with local GOROOT.
+			c.LocalSrcPath = filepath.Join(localgoroot, c.SrcPath[len(goroot):])
+		} else {
+			// Replace remote GOPATH with local GOPATH.
+			c.LocalSrcPath = c.SrcPath
+			// TODO(maruel): Sort for deterministic behavior?
+			for prefix, dest := range gopaths {
+				if strings.HasPrefix(c.SrcPath, prefix) {
+					c.LocalSrcPath = filepath.Join(dest, c.SrcPath[len(prefix):])
+					break
+				}
+			}
+		}
+	}
+	// Consider _test/_testmain.go as stdlib since it's injected by "go test".
+	c.IsStdlib = (goroot != "" && strings.HasPrefix(c.SrcPath, goroot)) || c.PkgSrc() == testMainSrc
+}
+
+// Stack is a call stack.
+type Stack struct {
+	Calls  []Call // Call stack. First is original function, last is leaf function.
+	Elided bool   // Happens when there's >100 items in Stack, currently hardcoded in package runtime.
+}
+
+// equal returns true on if both call stacks are exactly equal.
+func (s *Stack) equal(r *Stack) bool {
+	if len(s.Calls) != len(r.Calls) || s.Elided != r.Elided {
+		return false
+	}
+	for i := range s.Calls {
+		if !s.Calls[i].equal(&r.Calls[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+// similar returns true if the two Stack are equal or almost but not quite
+// equal.
+func (s *Stack) similar(r *Stack, similar Similarity) bool {
+	if len(s.Calls) != len(r.Calls) || s.Elided != r.Elided {
+		return false
+	}
+	for i := range s.Calls {
+		if !s.Calls[i].similar(&r.Calls[i], similar) {
+			return false
+		}
+	}
+	return true
+}
+
+// merge merges two similar Stack, zapping out differences.
+func (s *Stack) merge(r *Stack) *Stack {
+	// Assumes similar stacks have the same length.
+	out := &Stack{
+		Calls:  make([]Call, len(s.Calls)),
+		Elided: s.Elided,
+	}
+	for i := range s.Calls {
+		out.Calls[i] = s.Calls[i].merge(&r.Calls[i])
+	}
+	return out
+}
+
+// less compares two Stack, where the ones that are less are more
+// important, so they come up front.
+//
+// A Stack with more private functions is 'less' so it is at the top.
+// Inversely, a Stack with only public functions is 'more' so it is at the
+// bottom.
+func (s *Stack) less(r *Stack) bool {
+	lStdlib := 0
+	lPrivate := 0
+	for _, c := range s.Calls {
+		if c.IsStdlib {
+			lStdlib++
+		} else {
+			lPrivate++
+		}
+	}
+	rStdlib := 0
+	rPrivate := 0
+	for _, s := range r.Calls {
+		if s.IsStdlib {
+			rStdlib++
+		} else {
+			rPrivate++
+		}
+	}
+	if lPrivate > rPrivate {
+		return true
+	}
+	if lPrivate < rPrivate {
+		return false
+	}
+	if lStdlib > rStdlib {
+		return false
+	}
+	if lStdlib < rStdlib {
+		return true
+	}
+
+	// Stack lengths are the same.
+	for x := range s.Calls {
+		if s.Calls[x].Func.Raw < r.Calls[x].Func.Raw {
+			return true
+		}
+		if s.Calls[x].Func.Raw > r.Calls[x].Func.Raw {
+			return true
+		}
+		if s.Calls[x].PkgSrc() < r.Calls[x].PkgSrc() {
+			return true
+		}
+		if s.Calls[x].PkgSrc() > r.Calls[x].PkgSrc() {
+			return true
+		}
+		if s.Calls[x].Line < r.Calls[x].Line {
+			return true
+		}
+		if s.Calls[x].Line > r.Calls[x].Line {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *Stack) updateLocations(goroot, localgoroot string, gopaths map[string]string) {
+	for i := range s.Calls {
+		s.Calls[i].updateLocations(goroot, localgoroot, gopaths)
+	}
+}
+
+// Signature represents the signature of one or multiple goroutines.
+//
+// It is effectively the stack trace plus the goroutine internal bits, like
+// it's state, if it is thread locked, which call site created this goroutine,
+// etc.
+type Signature struct {
+	// Use git grep 'gopark(|unlock)\(' to find them all plus everything listed
+	// in runtime/traceback.go. Valid values includes:
+	//     - chan send, chan receive, select
+	//     - finalizer wait, mark wait (idle),
+	//     - Concurrent GC wait, GC sweep wait, force gc (idle)
+	//     - IO wait, panicwait
+	//     - semacquire, semarelease
+	//     - sleep, timer goroutine (idle)
+	//     - trace reader (blocked)
+	// Stuck cases:
+	//     - chan send (nil chan), chan receive (nil chan), select (no cases)
+	// Runnable states:
+	//    - idle, runnable, running, syscall, waiting, dead, enqueue, copystack,
+	// Scan states:
+	//    - scan, scanrunnable, scanrunning, scansyscall, scanwaiting, scandead,
+	//      scanenqueue
+	State     string
+	CreatedBy Call // Which other goroutine which created this one.
+	SleepMin  int  // Wait time in minutes, if applicable.
+	SleepMax  int  // Wait time in minutes, if applicable.
+	Stack     Stack
+	Locked    bool // Locked to an OS thread.
+}
+
+// equal returns true only if both signatures are exactly equal.
+func (s *Signature) equal(r *Signature) bool {
+	if s.State != r.State || !s.CreatedBy.equal(&r.CreatedBy) || s.Locked != r.Locked || s.SleepMin != r.SleepMin || s.SleepMax != r.SleepMax {
+		return false
+	}
+	return s.Stack.equal(&r.Stack)
+}
+
+// similar returns true if the two Signature are equal or almost but not quite
+// equal.
+func (s *Signature) similar(r *Signature, similar Similarity) bool {
+	if s.State != r.State || !s.CreatedBy.similar(&r.CreatedBy, similar) {
+		return false
+	}
+	if similar == ExactFlags && s.Locked != r.Locked {
+		return false
+	}
+	return s.Stack.similar(&r.Stack, similar)
+}
+
+// merge merges two similar Signature, zapping out differences.
+func (s *Signature) merge(r *Signature) *Signature {
+	min := s.SleepMin
+	if r.SleepMin < min {
+		min = r.SleepMin
+	}
+	max := s.SleepMax
+	if r.SleepMax > max {
+		max = r.SleepMax
+	}
+	return &Signature{
+		State:     s.State,     // Drop right side.
+		CreatedBy: s.CreatedBy, // Drop right side.
+		SleepMin:  min,
+		SleepMax:  max,
+		Stack:     *s.Stack.merge(&r.Stack),
+		Locked:    s.Locked || r.Locked, // TODO(maruel): This is weirdo.
+	}
+}
+
+// less compares two Signature, where the ones that are less are more
+// important, so they come up front. A Signature with more private functions is
+// 'less' so it is at the top. Inversely, a Signature with only public
+// functions is 'more' so it is at the bottom.
+func (s *Signature) less(r *Signature) bool {
+	if s.Stack.less(&r.Stack) {
+		return true
+	}
+	if r.Stack.less(&s.Stack) {
+		return false
+	}
+	if s.Locked && !r.Locked {
+		return true
+	}
+	if r.Locked && !s.Locked {
+		return false
+	}
+	if s.State < r.State {
+		return true
+	}
+	if s.State > r.State {
+		return false
+	}
+	return false
+}
+
+// SleepString returns a string "N-M minutes" if the goroutine(s) slept for a
+// long time.
+//
+// Returns an empty string otherwise.
+func (s *Signature) SleepString() string {
+	if s.SleepMax == 0 {
+		return ""
+	}
+	if s.SleepMin != s.SleepMax {
+		return fmt.Sprintf("%d~%d minutes", s.SleepMin, s.SleepMax)
+	}
+	return fmt.Sprintf("%d minutes", s.SleepMax)
+}
+
+// CreatedByString return a short context about the origin of this goroutine
+// signature.
+func (s *Signature) CreatedByString(fullPath bool) string {
+	created := s.CreatedBy.Func.PkgDotName()
+	if created == "" {
+		return ""
+	}
+	created += " @ "
+	if fullPath {
+		created += s.CreatedBy.FullSrcLine()
+	} else {
+		created += s.CreatedBy.SrcLine()
+	}
+	return created
+}
+
+func (s *Signature) updateLocations(goroot, localgoroot string, gopaths map[string]string) {
+	s.CreatedBy.updateLocations(goroot, localgoroot, gopaths)
+	s.Stack.updateLocations(goroot, localgoroot, gopaths)
+}
+
+// Goroutine represents the state of one goroutine, including the stack trace.
+type Goroutine struct {
+	Signature      // It's stack trace, internal bits, state, which call site created it, etc.
+	ID        int  // Goroutine ID.
+	First     bool // First is the goroutine first printed, normally the one that crashed.
+}
+
+// Private stuff.
+
+// nameArguments is a post-processing step where Args are 'named' with numbers.
+func nameArguments(goroutines []*Goroutine) {
+	// Set a name for any pointer occurring more than once.
+	type object struct {
+		args      []*Arg
+		inPrimary bool
+		id        int
+	}
+	objects := map[uint64]object{}
+	// Enumerate all the arguments.
+	for i := range goroutines {
+		for j := range goroutines[i].Stack.Calls {
+			for k := range goroutines[i].Stack.Calls[j].Args.Values {
+				arg := goroutines[i].Stack.Calls[j].Args.Values[k]
+				if arg.IsPtr() {
+					objects[arg.Value] = object{
+						args:      append(objects[arg.Value].args, &goroutines[i].Stack.Calls[j].Args.Values[k]),
+						inPrimary: objects[arg.Value].inPrimary || i == 0,
+					}
+				}
+			}
+		}
+		// CreatedBy.Args is never set.
+	}
+	order := make(uint64Slice, 0, len(objects)/2)
+	for k, obj := range objects {
+		if len(obj.args) > 1 && obj.inPrimary {
+			order = append(order, k)
+		}
+	}
+	sort.Sort(order)
+	nextID := 1
+	for _, k := range order {
+		for _, arg := range objects[k].args {
+			arg.Name = fmt.Sprintf("#%d", nextID)
+		}
+		nextID++
+	}
+
+	// Now do the rest. This is done so the output is deterministic.
+	order = make(uint64Slice, 0, len(objects))
+	for k := range objects {
+		order = append(order, k)
+	}
+	sort.Sort(order)
+	for _, k := range order {
+		// Process the remaining pointers, they were not referenced by primary
+		// thread so will have higher IDs.
+		if objects[k].inPrimary {
+			continue
+		}
+		for _, arg := range objects[k].args {
+			arg.Name = fmt.Sprintf("#%d", nextID)
+		}
+		nextID++
+	}
+}
+
+type uint64Slice []uint64
+
+func (a uint64Slice) Len() int           { return len(a) }
+func (a uint64Slice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] }