init

2019-08-08 17:13:34 +08:00
parent e8e5966a8c
commit f0bf2d5fec
2885 changed files with 1195993 additions and 12 deletions
@@ -0,0 +1,18 @@
+Copyright (c) 2013, Space Monkey, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,89 @@
+gls
+===
+
+Goroutine local storage
+
+### IMPORTANT NOTE ###
+
+It is my duty to point you to https://blog.golang.org/context, which is how 
+Google solves all of the problems you'd perhaps consider using this package
+for at scale. 
+
+One downside to Google's approach is that *all* of your functions must have
+a new first argument, but after clearing that hurdle everything else is much
+better.
+
+If you aren't interested in this warning, read on.
+
+### Huhwaht? Why? ###
+
+Every so often, a thread shows up on the
+[golang-nuts](https://groups.google.com/d/forum/golang-nuts) asking for some
+form of goroutine-local-storage, or some kind of goroutine id, or some kind of
+context. There are a few valid use cases for goroutine-local-storage, one of
+the most prominent being log line context. One poster was interested in being
+able to log an HTTP request context id in every log line in the same goroutine
+as the incoming HTTP request, without having to change every library and
+function call he was interested in logging.
+
+This would be pretty useful. Provided that you could get some kind of
+goroutine-local-storage, you could call
+[log.SetOutput](http://golang.org/pkg/log/#SetOutput) with your own logging
+writer that checks goroutine-local-storage for some context information and
+adds that context to your log lines.
+
+But alas, Andrew Gerrand's typically diplomatic answer to the question of
+goroutine-local variables was:
+
+> We wouldn't even be having this discussion if thread local storage wasn't
+> useful. But every feature comes at a cost, and in my opinion the cost of
+> threadlocals far outweighs their benefits. They're just not a good fit for
+> Go.
+
+So, yeah, that makes sense. That's a pretty good reason for why the language
+won't support a specific and (relatively) unuseful feature that requires some
+runtime changes, just for the sake of a little bit of log improvement.
+
+But does Go require runtime changes?
+
+### How it works ###
+
+Go has pretty fantastic introspective and reflective features, but one thing Go
+doesn't give you is any kind of access to the stack pointer, or frame pointer,
+or goroutine id, or anything contextual about your current stack. It gives you
+access to your list of callers, but only along with program counters, which are
+fixed at compile time.
+
+But it does give you the stack.
+
+So, we define 16 special functions and embed base-16 tags into the stack using
+the call order of those 16 functions. Then, we can read our tags back out of
+the stack looking at the callers list.
+
+We then use these tags as an index into a traditional map for implementing
+this library.
+
+### What are people saying? ###
+
+"Wow, that's horrifying."
+
+"This is the most terrible thing I have seen in a very long time."
+
+"Where is it getting a context from? Is this serializing all the requests? 
+What the heck is the client being bound to? What are these tags? Why does he 
+need callers? Oh god no. No no no."
+
+### Docs ###
+
+Please see the docs at http://godoc.org/github.com/jtolds/gls
+
+### Related ###
+
+If you're okay relying on the string format of the current runtime stacktrace 
+including a unique goroutine id (not guaranteed by the spec or anything, but 
+very unlikely to change within a Go release), you might be able to squeeze 
+out a bit more performance by using this similar library, inspired by some 
+code Brad Fitzpatrick wrote for debugging his HTTP/2 library: 
+https://github.com/tylerb/gls (in contrast, jtolds/gls doesn't require 
+any knowledge of the string format of the runtime stacktrace, which 
+probably adds unnecessary overhead).
@@ -0,0 +1,153 @@
+// Package gls implements goroutine-local storage.
+package gls
+
+import (
+	"sync"
+)
+
+var (
+	mgrRegistry    = make(map[*ContextManager]bool)
+	mgrRegistryMtx sync.RWMutex
+)
+
+// Values is simply a map of key types to value types. Used by SetValues to
+// set multiple values at once.
+type Values map[interface{}]interface{}
+
+// ContextManager is the main entrypoint for interacting with
+// Goroutine-local-storage. You can have multiple independent ContextManagers
+// at any given time. ContextManagers are usually declared globally for a given
+// class of context variables. You should use NewContextManager for
+// construction.
+type ContextManager struct {
+	mtx    sync.Mutex
+	values map[uint]Values
+}
+
+// NewContextManager returns a brand new ContextManager. It also registers the
+// new ContextManager in the ContextManager registry which is used by the Go
+// method. ContextManagers are typically defined globally at package scope.
+func NewContextManager() *ContextManager {
+	mgr := &ContextManager{values: make(map[uint]Values)}
+	mgrRegistryMtx.Lock()
+	defer mgrRegistryMtx.Unlock()
+	mgrRegistry[mgr] = true
+	return mgr
+}
+
+// Unregister removes a ContextManager from the global registry, used by the
+// Go method. Only intended for use when you're completely done with a
+// ContextManager. Use of Unregister at all is rare.
+func (m *ContextManager) Unregister() {
+	mgrRegistryMtx.Lock()
+	defer mgrRegistryMtx.Unlock()
+	delete(mgrRegistry, m)
+}
+
+// SetValues takes a collection of values and a function to call for those
+// values to be set in. Anything further down the stack will have the set
+// values available through GetValue. SetValues will add new values or replace
+// existing values of the same key and will not mutate or change values for
+// previous stack frames.
+// SetValues is slow (makes a copy of all current and new values for the new
+// gls-context) in order to reduce the amount of lookups GetValue requires.
+func (m *ContextManager) SetValues(new_values Values, context_call func()) {
+	if len(new_values) == 0 {
+		context_call()
+		return
+	}
+
+	mutated_keys := make([]interface{}, 0, len(new_values))
+	mutated_vals := make(Values, len(new_values))
+
+	EnsureGoroutineId(func(gid uint) {
+		m.mtx.Lock()
+		state, found := m.values[gid]
+		if !found {
+			state = make(Values, len(new_values))
+			m.values[gid] = state
+		}
+		m.mtx.Unlock()
+
+		for key, new_val := range new_values {
+			mutated_keys = append(mutated_keys, key)
+			if old_val, ok := state[key]; ok {
+				mutated_vals[key] = old_val
+			}
+			state[key] = new_val
+		}
+
+		defer func() {
+			if !found {
+				m.mtx.Lock()
+				delete(m.values, gid)
+				m.mtx.Unlock()
+				return
+			}
+
+			for _, key := range mutated_keys {
+				if val, ok := mutated_vals[key]; ok {
+					state[key] = val
+				} else {
+					delete(state, key)
+				}
+			}
+		}()
+
+		context_call()
+	})
+}
+
+// GetValue will return a previously set value, provided that the value was set
+// by SetValues somewhere higher up the stack. If the value is not found, ok
+// will be false.
+func (m *ContextManager) GetValue(key interface{}) (
+	value interface{}, ok bool) {
+	gid, ok := GetGoroutineId()
+	if !ok {
+		return nil, false
+	}
+
+	m.mtx.Lock()
+	state, found := m.values[gid]
+	m.mtx.Unlock()
+
+	if !found {
+		return nil, false
+	}
+	value, ok = state[key]
+	return value, ok
+}
+
+func (m *ContextManager) getValues() Values {
+	gid, ok := GetGoroutineId()
+	if !ok {
+		return nil
+	}
+	m.mtx.Lock()
+	state, _ := m.values[gid]
+	m.mtx.Unlock()
+	return state
+}
+
+// Go preserves ContextManager values and Goroutine-local-storage across new
+// goroutine invocations. The Go method makes a copy of all existing values on
+// all registered context managers and makes sure they are still set after
+// kicking off the provided function in a new goroutine. If you don't use this
+// Go method instead of the standard 'go' keyword, you will lose values in
+// ContextManagers, as goroutines have brand new stacks.
+func Go(cb func()) {
+	mgrRegistryMtx.RLock()
+	defer mgrRegistryMtx.RUnlock()
+
+	for mgr := range mgrRegistry {
+		values := mgr.getValues()
+		if len(values) > 0 {
+			cb = func(mgr *ContextManager, cb func()) func() {
+				return func() { mgr.SetValues(values, cb) }
+			}(mgr, cb)
+		}
+	}
+
+	go cb()
+}
@@ -0,0 +1,21 @@
+package gls
+
+import (
+	"sync"
+)
+
+var (
+	keyMtx     sync.Mutex
+	keyCounter uint64
+)
+
+// ContextKey is a throwaway value you can use as a key to a ContextManager
+type ContextKey struct{ id uint64 }
+
+// GenSym will return a brand new, never-before-used ContextKey
+func GenSym() ContextKey {
+	keyMtx.Lock()
+	defer keyMtx.Unlock()
+	keyCounter += 1
+	return ContextKey{id: keyCounter}
+}
@@ -0,0 +1,25 @@
+package gls
+
+var (
+	stackTagPool = &idPool{}
+)
+
+// Will return this goroutine's identifier if set. If you always need a
+// goroutine identifier, you should use EnsureGoroutineId which will make one
+// if there isn't one already.
+func GetGoroutineId() (gid uint, ok bool) {
+	return readStackTag()
+}
+
+// Will call cb with the current goroutine identifier. If one hasn't already
+// been generated, one will be created and set first. The goroutine identifier
+// might be invalid after cb returns.
+func EnsureGoroutineId(cb func(gid uint)) {
+	if gid, ok := readStackTag(); ok {
+		cb(gid)
+		return
+	}
+	gid := stackTagPool.Acquire()
+	defer stackTagPool.Release(gid)
+	addStackTag(gid, func() { cb(gid) })
+}
@@ -0,0 +1,34 @@
+package gls
+
+// though this could probably be better at keeping ids smaller, the goal of
+// this class is to keep a registry of the smallest unique integer ids
+// per-process possible
+
+import (
+	"sync"
+)
+
+type idPool struct {
+	mtx      sync.Mutex
+	released []uint
+	max_id   uint
+}
+
+func (p *idPool) Acquire() (id uint) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	if len(p.released) > 0 {
+		id = p.released[len(p.released)-1]
+		p.released = p.released[:len(p.released)-1]
+		return id
+	}
+	id = p.max_id
+	p.max_id++
+	return id
+}
+
+func (p *idPool) Release(id uint) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	p.released = append(p.released, id)
+}
@@ -0,0 +1,147 @@
+package gls
+
+// so, basically, we're going to encode integer tags in base-16 on the stack
+
+const (
+	bitWidth       = 4
+	stackBatchSize = 16
+)
+
+var (
+	pc_lookup   = make(map[uintptr]int8, 17)
+	mark_lookup [16]func(uint, func())
+)
+
+func init() {
+	setEntries := func(f func(uint, func()), v int8) {
+		var ptr uintptr
+		f(0, func() {
+			ptr = findPtr()
+		})
+		pc_lookup[ptr] = v
+		if v >= 0 {
+			mark_lookup[v] = f
+		}
+	}
+	setEntries(github_com_jtolds_gls_markS, -0x1)
+	setEntries(github_com_jtolds_gls_mark0, 0x0)
+	setEntries(github_com_jtolds_gls_mark1, 0x1)
+	setEntries(github_com_jtolds_gls_mark2, 0x2)
+	setEntries(github_com_jtolds_gls_mark3, 0x3)
+	setEntries(github_com_jtolds_gls_mark4, 0x4)
+	setEntries(github_com_jtolds_gls_mark5, 0x5)
+	setEntries(github_com_jtolds_gls_mark6, 0x6)
+	setEntries(github_com_jtolds_gls_mark7, 0x7)
+	setEntries(github_com_jtolds_gls_mark8, 0x8)
+	setEntries(github_com_jtolds_gls_mark9, 0x9)
+	setEntries(github_com_jtolds_gls_markA, 0xa)
+	setEntries(github_com_jtolds_gls_markB, 0xb)
+	setEntries(github_com_jtolds_gls_markC, 0xc)
+	setEntries(github_com_jtolds_gls_markD, 0xd)
+	setEntries(github_com_jtolds_gls_markE, 0xe)
+	setEntries(github_com_jtolds_gls_markF, 0xf)
+}
+
+func addStackTag(tag uint, context_call func()) {
+	if context_call == nil {
+		return
+	}
+	github_com_jtolds_gls_markS(tag, context_call)
+}
+
+// these private methods are named this horrendous name so gopherjs support
+// is easier. it shouldn't add any runtime cost in non-js builds.
+
+//go:noinline
+func github_com_jtolds_gls_markS(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark0(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark1(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark2(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark3(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark4(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark5(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark6(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark7(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark8(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_mark9(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_markA(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_markB(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_markC(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_markD(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_markE(tag uint, cb func()) { _m(tag, cb) }
+
+//go:noinline
+func github_com_jtolds_gls_markF(tag uint, cb func()) { _m(tag, cb) }
+
+func _m(tag_remainder uint, cb func()) {
+	if tag_remainder == 0 {
+		cb()
+	} else {
+		mark_lookup[tag_remainder&0xf](tag_remainder>>bitWidth, cb)
+	}
+}
+
+func readStackTag() (tag uint, ok bool) {
+	var current_tag uint
+	offset := 0
+	for {
+		batch, next_offset := getStack(offset, stackBatchSize)
+		for _, pc := range batch {
+			val, ok := pc_lookup[pc]
+			if !ok {
+				continue
+			}
+			if val < 0 {
+				return current_tag, true
+			}
+			current_tag <<= bitWidth
+			current_tag += uint(val)
+		}
+		if next_offset == 0 {
+			break
+		}
+		offset = next_offset
+	}
+	return 0, false
+}
+
+func (m *ContextManager) preventInlining() {
+	// dunno if findPtr or getStack are likely to get inlined in a future release
+	// of go, but if they are inlined and their callers are inlined, that could
+	// hork some things. let's do our best to explain to the compiler that we
+	// really don't want those two functions inlined by saying they could change
+	// at any time. assumes preventInlining doesn't get compiled out.
+	// this whole thing is probably overkill.
+	findPtr = m.values[0][0].(func() uintptr)
+	getStack = m.values[0][1].(func(int, int) ([]uintptr, int))
+}
@@ -0,0 +1,75 @@
+// +build js
+
+package gls
+
+// This file is used for GopherJS builds, which don't have normal runtime
+// stack trace support
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/gopherjs/gopherjs/js"
+)
+
+const (
+	jsFuncNamePrefix = "github_com_jtolds_gls_mark"
+)
+
+func jsMarkStack() (f []uintptr) {
+	lines := strings.Split(
+		js.Global.Get("Error").New().Get("stack").String(), "\n")
+	f = make([]uintptr, 0, len(lines))
+	for i, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		if i == 0 {
+			if line != "Error" {
+				panic("didn't understand js stack trace")
+			}
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 2 || fields[0] != "at" {
+			panic("didn't understand js stack trace")
+		}
+
+		pos := strings.Index(fields[1], jsFuncNamePrefix)
+		if pos < 0 {
+			continue
+		}
+		pos += len(jsFuncNamePrefix)
+		if pos >= len(fields[1]) {
+			panic("didn't understand js stack trace")
+		}
+		char := string(fields[1][pos])
+		switch char {
+		case "S":
+			f = append(f, uintptr(0))
+		default:
+			val, err := strconv.ParseUint(char, 16, 8)
+			if err != nil {
+				panic("didn't understand js stack trace")
+			}
+			f = append(f, uintptr(val)+1)
+		}
+	}
+	return f
+}
+
+// variables to prevent inlining
+var (
+	findPtr = func() uintptr {
+		funcs := jsMarkStack()
+		if len(funcs) == 0 {
+			panic("failed to find function pointer")
+		}
+		return funcs[0]
+	}
+
+	getStack = func(offset, amount int) (stack []uintptr, next_offset int) {
+		return jsMarkStack(), 0
+	}
+)
@@ -0,0 +1,30 @@
+// +build !js
+
+package gls
+
+// This file is used for standard Go builds, which have the expected runtime
+// support
+
+import (
+	"runtime"
+)
+
+var (
+	findPtr = func() uintptr {
+		var pc [1]uintptr
+		n := runtime.Callers(4, pc[:])
+		if n != 1 {
+			panic("failed to find function pointer")
+		}
+		return pc[0]
+	}
+
+	getStack = func(offset, amount int) (stack []uintptr, next_offset int) {
+		stack = make([]uintptr, amount)
+		stack = stack[:runtime.Callers(offset, stack)]
+		if len(stack) < amount {
+			return stack, 0
+		}
+		return stack, offset + len(stack)
+	}
+)