lib/sync: Add option for sasha-s/go-deadlock

lib/sync/debug.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/logger"
+	"github.com/sasha-s/go-deadlock"
 )
 
 var (
@@ -22,14 +23,15 @@ var (
 	// We make an exception in this package and have an actual "if debug { ...
 	// }" variable, as it may be rather performance critical and does
 	// nonstandard things (from a debug logging PoV).
-	debug = strings.Contains(os.Getenv("STTRACE"), "sync") || os.Getenv("STTRACE") == "all"
+	debug       = strings.Contains(os.Getenv("STTRACE"), "sync") || os.Getenv("STTRACE") == "all"
+	useDeadlock = os.Getenv("STDEADLOCK") != ""
 )
 
 func init() {
 	l.SetDebug("sync", strings.Contains(os.Getenv("STTRACE"), "sync") || os.Getenv("STTRACE") == "all")
 
 	if n, err := strconv.Atoi(os.Getenv("STLOCKTHRESHOLD")); err == nil {
-		threshold = time.Duration(n) * time.Millisecond
+		deadlock.Opts.DeadlockTimeout = threshold = time.Duration(n) * time.Millisecond
 	}
 	l.Debugf("Enabling lock logging at %v threshold", threshold)
 }

lib/sync/sync.go

@@ -15,6 +15,8 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+
+	"github.com/sasha-s/go-deadlock"
 )
 
 type Mutex interface {
@@ -35,6 +37,9 @@ type WaitGroup interface {
 }
 
 func NewMutex() Mutex {
+	if useDeadlock {
+		return deadlock.Mutex{}
+	}
 	if debug {
 		mutex := &loggedMutex{}
 		mutex.holder.Store(holder{})
@@ -44,6 +49,9 @@ func NewMutex() Mutex {
 }
 
 func NewRWMutex() RWMutex {
+	if useDeadlock {
+		return deadlock.RWMutex{}
+	}
 	if debug {
 		mutex := &loggedRWMutex{
 			readHolders: make(map[int][]holder),

vendor/github.com/sasha-s/go-deadlock/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/sasha-s/go-deadlock/deadlock.go

@@ -0,0 +1,314 @@
+package deadlock
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// Opts control how deadlock detection behaves.
+// Options are supposed to be set once at a startup (say, when parsing flags).
+var Opts = struct {
+	// Mutex/RWMutex would work exactly as their sync counterparts
+	// -- almost no runtime penalty, no deadlock detection if Disable == true.
+	Disable bool
+	// Would disable lock order based deadlock detection if DisableLockOrderDetection == true.
+	DisableLockOrderDetection bool
+	// Waiting for a lock for longer than DeadlockTimeout is considered a deadlock.
+	// Ignored is DeadlockTimeout <= 0.
+	DeadlockTimeout time.Duration
+	// OnPotentialDeadlock is called each time a potential deadlock is deetcted -- either based on
+	// lock order or on lock wait time.
+	OnPotentialDeadlock func()
+	// Will keep MaxMapSize lock pairs (happens before // happens after) in the map.
+	// The map resets once the threshold is reached.
+	MaxMapSize int
+	// Will print to deadlock info to log buffer.
+	LogBuf io.Writer
+}{
+	DeadlockTimeout: time.Second * 30,
+	OnPotentialDeadlock: func() {
+		os.Exit(2)
+	},
+	MaxMapSize: 1024 * 64,
+	LogBuf:     os.Stderr,
+}
+
+// A Mutex is a drop-in replacement for sync.Mutex.
+// Performs deadlock detection unless disabled in Opts.
+type Mutex struct {
+	mu sync.Mutex
+}
+
+// Lock locks the mutex.
+// If the lock is already in use, the calling goroutine
+// blocks until the mutex is available.
+//
+// Unless deadlock detection is disabled, logs potential deadlocks to stderr,
+// calling Opts.OnPotentialDeadlock on each occasion.
+func (m *Mutex) Lock() {
+	lock(m.mu.Lock, m)
+}
+
+// Unlock unlocks the mutex.
+// It is a run-time error if m is not locked on entry to Unlock.
+//
+// A locked Mutex is not associated with a particular goroutine.
+// It is allowed for one goroutine to lock a Mutex and then
+// arrange for another goroutine to unlock it.
+func (m *Mutex) Unlock() {
+	if Opts.Disable {
+		m.mu.Unlock()
+		return
+	}
+	m.mu.Unlock()
+	PostUnlock(m)
+}
+
+// An RWMutex is a drop-in replacement for sync.RWMutex.
+// Performs deadlock detection unless disabled in Opts.
+type RWMutex struct {
+	mu sync.RWMutex
+}
+
+// Lock locks rw for writing.
+// If the lock is already locked for reading or writing,
+// Lock blocks until the lock is available.
+// To ensure that the lock eventually becomes available,
+// a blocked Lock call excludes new readers from acquiring
+// the lock.
+//
+// Unless deadlock detection is disabled, logs potential deadlocks to stderr,
+// calling Opts.OnPotentialDeadlock on each occasion.
+func (m *RWMutex) Lock() {
+	lock(m.mu.Lock, m)
+}
+
+// Unlock unlocks the mutex for writing.  It is a run-time error if rw is
+// not locked for writing on entry to Unlock.
+//
+// As with Mutexes, a locked RWMutex is not associated with a particular
+// goroutine.  One goroutine may RLock (Lock) an RWMutex and then
+// arrange for another goroutine to RUnlock (Unlock) it.
+func (m *RWMutex) Unlock() {
+	if Opts.Disable {
+		m.mu.Unlock()
+		return
+	}
+	m.mu.Unlock()
+	PostUnlock(m)
+}
+
+// RLock locks the mutex for reading.
+//
+// Unless deadlock detection is disabled, logs potential deadlocks to stderr,
+// calling Opts.OnPotentialDeadlock on each occasion.
+func (m *RWMutex) RLock() {
+	lock(m.mu.RLock, m)
+}
+
+// RUnlock undoes a single RLock call;
+// it does not affect other simultaneous readers.
+// It is a run-time error if rw is not locked for reading
+// on entry to RUnlock.
+func (m *RWMutex) RUnlock() {
+	if Opts.Disable {
+		m.mu.RUnlock()
+		return
+	}
+	m.mu.RUnlock()
+	PostUnlock(m)
+}
+
+// RLocker returns a Locker interface that implements
+// the Lock and Unlock methods by calling RLock and RUnlock.
+func (m *RWMutex) RLocker() sync.Locker {
+	return (*rlocker)(m)
+}
+
+func PreLock(skip int, p interface{}) {
+	lo.PreLock(skip, p)
+}
+
+func PostLock(skip int, p interface{}) {
+	lo.PostLock(skip, p)
+}
+
+func PostUnlock(p interface{}) {
+	lo.PostUnlock(p)
+}
+
+func lock(lockFn func(), ptr interface{}) {
+	if Opts.Disable {
+		lockFn()
+		return
+	}
+	PreLock(4, ptr)
+	if Opts.DeadlockTimeout <= 0 {
+		lockFn()
+	} else {
+		ch := make(chan struct{})
+		go func() {
+			lockFn()
+			close(ch)
+		}()
+		for {
+			t := time.NewTimer(Opts.DeadlockTimeout)
+			defer t.Stop()
+			select {
+			case <-t.C:
+				lo.mu.Lock()
+				prev, ok := lo.cur[ptr]
+				if !ok {
+					lo.mu.Unlock()
+					break // Nobody seems to be holding a lock, try again.
+				}
+				fmt.Fprintln(Opts.LogBuf, header)
+				fmt.Fprintln(Opts.LogBuf, "Previous place where the lock was grabbed")
+				fmt.Fprintf(Opts.LogBuf, "goroutine %v lock %p\n", prev.gid, ptr)
+				printStack(Opts.LogBuf, prev.stack)
+				fmt.Fprintln(Opts.LogBuf, "Have been trying to lock it again for more than", Opts.DeadlockTimeout)
+				fmt.Fprintf(Opts.LogBuf, "goroutine %v lock %p\n", getGID(), ptr)
+				printStack(Opts.LogBuf, callers(2))
+				fmt.Fprintln(Opts.LogBuf)
+				stacks := stacks()
+				grs := bytes.Split(stacks, []byte("\n\n"))
+				for _, g := range grs {
+					if extractGID(g) == prev.gid {
+						fmt.Fprintln(Opts.LogBuf, "Here is what goroutine", prev.gid, "doing now")
+						Opts.LogBuf.Write(g)
+						fmt.Fprintln(Opts.LogBuf)
+					}
+				}
+				lo.other(ptr)
+				fmt.Fprintln(Opts.LogBuf, "All current goroutines:")
+				Opts.LogBuf.Write(stacks)
+				lo.mu.Unlock()
+				Opts.OnPotentialDeadlock()
+				<-ch
+				PostLock(4, ptr)
+				return
+			case <-ch:
+				PostLock(4, ptr)
+				return
+			}
+		}
+	}
+	PostLock(4, ptr)
+}
+
+type lockOrder struct {
+	mu    sync.Mutex
+	cur   map[interface{}]stackGID // stacktraces + gids for the locks currently taken.
+	order map[beforeAfter]ss       // expected order of locks.
+}
+
+type stackGID struct {
+	stack []uintptr
+	gid   uint64
+}
+
+type beforeAfter struct {
+	before interface{}
+	after  interface{}
+}
+
+type ss struct {
+	before []uintptr
+	after  []uintptr
+}
+
+var lo = newLockOrder()
+
+func newLockOrder() *lockOrder {
+	return &lockOrder{
+		cur:   map[interface{}]stackGID{},
+		order: map[beforeAfter]ss{},
+	}
+}
+
+func (l *lockOrder) PostLock(skip int, p interface{}) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.cur[p] = stackGID{callers(skip), getGID()}
+}
+
+func (l *lockOrder) PreLock(skip int, p interface{}) {
+	if Opts.DisableLockOrderDetection {
+		return
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	stack := callers(skip)
+	gid := getGID()
+	for b, bs := range l.cur {
+		if b == p {
+			continue
+		}
+		if bs.gid != gid { // We want locks taken in the same goroutine only.
+			continue
+		}
+		if s, ok := l.order[beforeAfter{p, b}]; ok {
+			fmt.Fprintln(Opts.LogBuf, header, "Inconsistent locking. saw this ordering in one goroutine:")
+			fmt.Fprintln(Opts.LogBuf, "happened before")
+			printStack(Opts.LogBuf, s.before)
+			fmt.Fprintln(Opts.LogBuf, "happened after")
+			printStack(Opts.LogBuf, s.after)
+			fmt.Fprintln(Opts.LogBuf, "in another goroutine: happened before")
+			printStack(Opts.LogBuf, bs.stack)
+			fmt.Fprintln(Opts.LogBuf, "happend after")
+			printStack(Opts.LogBuf, stack)
+			l.other(p)
+			Opts.OnPotentialDeadlock()
+		}
+		l.order[beforeAfter{b, p}] = ss{bs.stack, stack}
+		if len(l.order) == Opts.MaxMapSize { // Reset the map to keep memory footprint bounded.
+			l.order = map[beforeAfter]ss{}
+		}
+	}
+	l.cur[p] = stackGID{stack, gid}
+}
+
+func (l *lockOrder) PostUnlock(p interface{}) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	delete(l.cur, p)
+}
+
+type rlocker RWMutex
+
+func (r *rlocker) Lock()   { (*RWMutex)(r).RLock() }
+func (r *rlocker) Unlock() { (*RWMutex)(r).RUnlock() }
+
+// Under lo.mu Locked.
+func (l *lockOrder) other(ptr interface{}) {
+	fmt.Fprintln(Opts.LogBuf, "\nOther goroutines holding locks:")
+	for k, pp := range l.cur {
+		if k == ptr {
+			continue
+		}
+		fmt.Fprintf(Opts.LogBuf, "goroutine %v lock %p\n", pp.gid, k)
+		printStack(Opts.LogBuf, pp.stack)
+	}
+	fmt.Fprintln(Opts.LogBuf)
+}
+
+// Hacky way of getting a goroutine ID.
+func getGID() uint64 {
+	b := make([]byte, 64)
+	return extractGID(b[:runtime.Stack(b, false)])
+}
+
+func extractGID(stack []byte) uint64 {
+	b := bytes.TrimPrefix(stack, []byte("goroutine "))
+	b = b[:bytes.IndexByte(b, ' ')]
+	gid, _ := strconv.ParseUint(string(b), 10, 64)
+	return gid
+}
+
+const header = "POTENTIAL DEADLOCK:"

vendor/github.com/sasha-s/go-deadlock/deadlock_test.go

@@ -0,0 +1,127 @@
+package deadlock
+
+import (
+	"log"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestNoDeadlocks(t *testing.T) {
+	defer restore()()
+	Opts.DeadlockTimeout = time.Millisecond * 5000
+	var a RWMutex
+	var b Mutex
+	var c RWMutex
+	var wg sync.WaitGroup
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for k := 0; k < 5; k++ {
+				func() {
+					a.Lock()
+					defer a.Unlock()
+					func() {
+						b.Lock()
+						defer b.Unlock()
+						func() {
+							c.RLock()
+							defer c.RUnlock()
+							time.Sleep(time.Duration((1000 + rand.Intn(1000))) * time.Millisecond / 200)
+						}()
+					}()
+				}()
+			}
+		}()
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for k := 0; k < 5; k++ {
+				func() {
+					a.RLock()
+					defer a.RUnlock()
+					func() {
+						b.Lock()
+						defer b.Unlock()
+						func() {
+							c.Lock()
+							defer c.Unlock()
+							time.Sleep(time.Duration((1000 + rand.Intn(1000))) * time.Millisecond / 200)
+						}()
+					}()
+				}()
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func TestLockOrder(t *testing.T) {
+	defer restore()()
+	Opts.DeadlockTimeout = 0
+	var deadlocks uint32
+	Opts.OnPotentialDeadlock = func() {
+		atomic.AddUint32(&deadlocks, 1)
+	}
+	var a RWMutex
+	var b Mutex
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		a.Lock()
+		b.Lock()
+		b.Unlock()
+		a.Unlock()
+	}()
+	wg.Wait()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		b.Lock()
+		a.RLock()
+		a.RUnlock()
+		b.Unlock()
+	}()
+	wg.Wait()
+	if deadlocks != 1 {
+		t.Fatalf("expected 1 deadlock, detected %d", deadlocks)
+	}
+}
+
+func TestHardDeadlock(t *testing.T) {
+	defer restore()()
+	Opts.DisableLockOrderDetection = true
+	Opts.DeadlockTimeout = time.Millisecond * 20
+	var deadlocks uint32
+	Opts.OnPotentialDeadlock = func() {
+		atomic.AddUint32(&deadlocks, 1)
+	}
+	var mu Mutex
+	mu.Lock()
+	ch := make(chan struct{})
+	go func() {
+		defer close(ch)
+		mu.Lock()
+		defer mu.Unlock()
+	}()
+	select {
+	case <-ch:
+	case <-time.After(time.Millisecond * 100):
+	}
+	if deadlocks != 1 {
+		t.Fatalf("expected 1 deadlock, detected %d", deadlocks)
+	}
+	log.Println("****************")
+	mu.Unlock()
+}
+
+func restore() func() {
+	opts := Opts
+	return func() {
+		Opts = opts
+	}
+}

vendor/github.com/sasha-s/go-deadlock/stacktraces.go

@@ -0,0 +1,107 @@
+package deadlock
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+func callers(skip int) []uintptr {
+	s := make([]uintptr, 50) // Most relevant context seem to appear near the top of the stack.
+	return s[:runtime.Callers(2+skip, s)]
+}
+
+func printStack(w io.Writer, stack []uintptr) {
+	home := os.Getenv("HOME")
+	usr, err := user.Current()
+	if err == nil {
+		home = usr.HomeDir
+	}
+	cwd, _ := os.Getwd()
+
+	for i, pc := range stack {
+		f := runtime.FuncForPC(pc)
+		name := f.Name()
+		pkg := ""
+		if pos := strings.LastIndex(name, "/"); pos >= 0 {
+			name = name[pos+1:]
+		}
+		if pos := strings.Index(name, "."); pos >= 0 {
+			pkg = name[:pos]
+			name = name[pos+1:]
+		}
+		file, line := f.FileLine(pc - 1)
+		if (pkg == "runtime" && name == "goexit") || (pkg == "testing" && name == "tRunner") {
+			fmt.Fprintln(w)
+			return
+		}
+		tail := ""
+		if i == 0 {
+			tail = " <<<<<" // Make the line performing a lock prominent.
+		}
+		// Shorten the file name.
+		clean := file
+		if cwd != "" {
+			cl, err := filepath.Rel(cwd, file)
+			if err == nil {
+				clean = cl
+			}
+		}
+		if home != "" {
+			s2 := strings.Replace(file, home, "~", 1)
+			if len(clean) > len(s2) {
+				clean = s2
+			}
+		}
+		fmt.Fprintf(w, "%s:%d %s.%s %s%s\n", clean, line, pkg, name, code(file, line), tail)
+	}
+	fmt.Fprintln(w)
+}
+
+var fileSources struct {
+	sync.Mutex
+	lines map[string][][]byte
+}
+
+// Reads souce file lines from disk if not cached already.
+func getSourceLines(file string) [][]byte {
+	fileSources.Lock()
+	defer fileSources.Unlock()
+	if fileSources.lines == nil {
+		fileSources.lines = map[string][][]byte{}
+	}
+	if lines, ok := fileSources.lines[file]; ok {
+		return lines
+	}
+	text, _ := ioutil.ReadFile(file)
+	fileSources.lines[file] = bytes.Split(text, []byte{'\n'})
+	return fileSources.lines[file]
+}
+
+func code(file string, line int) string {
+	lines := getSourceLines(file)
+	// lines are 1 based.
+	if line >= len(lines) || line <= 0 {
+		return "???"
+	}
+	return "{ " + string(bytes.TrimSpace(lines[line-1])) + " }"
+}
+
+// Stacktraces for all goroutines.
+func stacks() []byte {
+	buf := make([]byte, 1024*16)
+	for {
+		n := runtime.Stack(buf, true)
+		if n < len(buf) {
+			return buf[:n]
+		}
+		buf = make([]byte, 2*len(buf))
+	}
+}

vendor/manifest

@@ -4,247 +4,270 @@
 		{
 			"importpath": "github.com/AudriusButkevicius/go-nat-pmp",
 			"repository": "https://github.com/AudriusButkevicius/go-nat-pmp",
+			"vcs": "",
 			"revision": "452c97607362b2ab5a7839b8d1704f0396b640ca",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/bkaradzic/go-lz4",
 			"repository": "https://github.com/bkaradzic/go-lz4",
+			"vcs": "",
 			"revision": "74ddf82598bc4745b965729e9c6a463bedd33049",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/calmh/du",
 			"repository": "https://github.com/calmh/du",
+			"vcs": "",
 			"revision": "3c0690cca16228b97741327b1b6781397afbdb24",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/calmh/luhn",
 			"repository": "https://github.com/calmh/luhn",
+			"vcs": "",
 			"revision": "0c8388ff95fa92d4094011e5a04fc99dea3d1632",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/calmh/xdr",
 			"repository": "https://github.com/calmh/xdr",
+			"vcs": "",
 			"revision": "f9b9f8f7aa27725f5cabb699bd9099ca7ce09143",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/cznic/b",
 			"repository": "https://github.com/cznic/b",
+			"vcs": "",
 			"revision": "bcff30a622dbdcb425aba904792de1df606dab7c",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/cznic/bufs",
 			"repository": "https://github.com/cznic/bufs",
+			"vcs": "",
 			"revision": "3dcccbd7064a1689f9c093a988ea11ac00e21f51",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/cznic/fileutil",
 			"repository": "https://github.com/cznic/fileutil",
+			"vcs": "",
 			"revision": "1c9c88fbf552b3737c7b97e1f243860359687976",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/cznic/internal/buffer",
 			"repository": "https://github.com/cznic/internal",
+			"vcs": "",
 			"revision": "cef02a853c3a93623c42eacd574e7ea05f55531b",
 			"branch": "master",
-			"path": "/buffer",
-			"notests": true
+			"path": "/buffer"
 		},
 		{
 			"importpath": "github.com/cznic/internal/file",
 			"repository": "https://github.com/cznic/internal",
+			"vcs": "",
 			"revision": "cef02a853c3a93623c42eacd574e7ea05f55531b",
 			"branch": "master",
-			"path": "/file",
-			"notests": true
+			"path": "/file"
 		},
 		{
 			"importpath": "github.com/cznic/internal/slice",
 			"repository": "https://github.com/cznic/internal",
+			"vcs": "",
 			"revision": "cef02a853c3a93623c42eacd574e7ea05f55531b",
 			"branch": "master",
-			"path": "/slice",
-			"notests": true
+			"path": "/slice"
 		},
 		{
 			"importpath": "github.com/cznic/lldb",
 			"repository": "https://github.com/cznic/lldb",
+			"vcs": "",
 			"revision": "7376b3bed3d27a7b640e264bfaf278d6d5232550",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/cznic/mathutil",
 			"repository": "https://github.com/cznic/mathutil",
+			"vcs": "",
 			"revision": "78ad7f262603437f0ecfebc835d80094f89c8f54",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/cznic/ql",
 			"repository": "https://github.com/cznic/ql",
+			"vcs": "",
 			"revision": "c81467d34c630800dd4ba81033e234a8159ff2e3",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/cznic/sortutil",
 			"repository": "https://github.com/cznic/sortutil",
+			"vcs": "",
 			"revision": "4c7342852e65c2088c981288f2c5610d10b9f7f4",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/cznic/strutil",
 			"repository": "https://github.com/cznic/strutil",
+			"vcs": "",
 			"revision": "1eb03e3cc9d345307a45ec82bd3016cde4bd4464",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/cznic/zappy",
 			"repository": "https://github.com/cznic/zappy",
+			"vcs": "",
 			"revision": "2533cb5b45cc6c07421468ce262899ddc9d53fb7",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/d4l3k/messagediff",
 			"repository": "https://github.com/d4l3k/messagediff",
+			"vcs": "",
 			"revision": "7b706999d935b04cf2dbc71a5a5afcbd288aeb48",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/edsrzf/mmap-go",
 			"repository": "https://github.com/edsrzf/mmap-go",
+			"vcs": "",
 			"revision": "935e0e8a636ca4ba70b713f3e38a19e1b77739e8",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/gobwas/glob",
 			"repository": "https://github.com/gobwas/glob",
+			"vcs": "",
 			"revision": "0354991b92587e2742549d3036f3b5bae5ab03f2",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/gogo/protobuf",
 			"repository": "https://github.com/gogo/protobuf",
+			"vcs": "",
 			"revision": "7883e1468d48d969e1c3ce4bcde89b6a7dd4adc4",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/golang/groupcache/lru",
 			"repository": "https://github.com/golang/groupcache",
+			"vcs": "",
 			"revision": "02826c3e79038b59d737d3b1c0a1d937f71a4433",
 			"branch": "master",
-			"path": "/lru",
-			"notests": true
+			"path": "/lru"
 		},
 		{
 			"importpath": "github.com/golang/snappy",
 			"repository": "https://github.com/golang/snappy",
+			"vcs": "",
 			"revision": "5f1c01d9f64b941dd9582c638279d046eda6ca31",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/jackpal/gateway",
 			"repository": "https://github.com/jackpal/gateway",
+			"vcs": "",
 			"revision": "3e333950771011fed13be63e62b9f473c5e0d9bf",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/juju/ratelimit",
 			"repository": "https://github.com/juju/ratelimit",
+			"vcs": "",
 			"revision": "77ed1c8a01217656d2080ad51981f6e99adaa177",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/kardianos/osext",
 			"repository": "https://github.com/kardianos/osext",
+			"vcs": "",
 			"revision": "29ae4ffbc9a6fe9fb2bc5029050ce6996ea1d3bc",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/lib/pq",
 			"repository": "https://github.com/lib/pq",
+			"vcs": "",
 			"revision": "ee1442bda7bd1b6a84e913bdb421cb1874ec629d",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/minio/sha256-simd",
 			"repository": "https://github.com/minio/sha256-simd",
+			"vcs": "",
 			"revision": "672e7bc9f3482375df73741cf57a157fe187ec26",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/onsi/ginkgo",
 			"repository": "https://github.com/onsi/ginkgo",
+			"vcs": "",
 			"revision": "ac3d45ddd7ef5c4d7fc4d037b615a81f4d67981e",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/onsi/gomega",
 			"repository": "https://github.com/onsi/gomega",
+			"vcs": "",
 			"revision": "a1094b2db2d4845621602c667bd4ccf09834544e",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/oschwald/geoip2-golang",
 			"repository": "https://github.com/oschwald/geoip2-golang",
+			"vcs": "",
 			"revision": "51714a0e79df40e00a94ae5086ec2a5532c9ee57",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/oschwald/maxminddb-golang",
 			"repository": "https://github.com/oschwald/maxminddb-golang",
+			"vcs": "",
 			"revision": "a26428e0305b837e06fe69221489819126a346c8",
-			"branch": "master",
-			"notests": true
+			"branch": "master"
 		},
 		{
 			"importpath": "github.com/rcrowley/go-metrics",
 			"repository": "https://github.com/rcrowley/go-metrics",
+			"vcs": "",
 			"revision": "eeba7bd0dd01ace6e690fa833b3f22aaec29af43",
 			"branch": "master"
 		},
+		{
+			"importpath": "github.com/sasha-s/go-deadlock",
+			"repository": "https://github.com/sasha-s/go-deadlock",
+			"vcs": "git",
+			"revision": "09aefc0ac06ad74d91ca31acdb11fc981c159149",
+			"branch": "master"
+		},
 		{
 			"importpath": "github.com/stathat/go",
 			"repository": "https://github.com/stathat/go",
+			"vcs": "",
 			"revision": "91dfa3a59c5b233fef9a346a1460f6e2bc889d93",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/syndtr/goleveldb",
 			"repository": "https://github.com/syndtr/goleveldb",
+			"vcs": "",
 			"revision": "6ae1797c0b42b9323fc27ff7dcf568df88f2f33d",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/thejerf/suture",
 			"repository": "https://github.com/thejerf/suture",
+			"vcs": "",
 			"revision": "fe1c0d795ff7a7e54fc54ef6afb36ee0adf0c276",
 			"branch": "master"
 		},
 		{
 			"importpath": "github.com/vitrun/qart/coding",
 			"repository": "https://github.com/vitrun/qart",
+			"vcs": "",
 			"revision": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0",
 			"branch": "master",
 			"path": "/coding"
@@ -252,6 +275,7 @@
 		{
 			"importpath": "github.com/vitrun/qart/gf256",
 			"repository": "https://github.com/vitrun/qart",
+			"vcs": "",
 			"revision": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0",
 			"branch": "master",
 			"path": "/gf256"
@@ -259,6 +283,7 @@
 		{
 			"importpath": "github.com/vitrun/qart/qr",
 			"repository": "https://github.com/vitrun/qart",
+			"vcs": "",
 			"revision": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0",
 			"branch": "master",
 			"path": "/qr"
@@ -266,6 +291,7 @@
 		{
 			"importpath": "golang.org/x/crypto/bcrypt",
 			"repository": "https://go.googlesource.com/crypto",
+			"vcs": "",
 			"revision": "e311231e83195f401421a286060d65643f9c9d40",
 			"branch": "master",
 			"path": "/bcrypt"
@@ -273,6 +299,7 @@
 		{
 			"importpath": "golang.org/x/crypto/blowfish",
 			"repository": "https://go.googlesource.com/crypto",
+			"vcs": "",
 			"revision": "e311231e83195f401421a286060d65643f9c9d40",
 			"branch": "master",
 			"path": "/blowfish"
@@ -280,22 +307,23 @@
 		{
 			"importpath": "golang.org/x/net/bpf",
 			"repository": "https://go.googlesource.com/net",
+			"vcs": "",
 			"revision": "749a502dd1eaf3e5bfd4f8956748c502357c0bbe",
 			"branch": "master",
-			"path": "/bpf",
-			"notests": true
+			"path": "/bpf"
 		},
 		{
 			"importpath": "golang.org/x/net/context",
 			"repository": "https://go.googlesource.com/net",
+			"vcs": "",
 			"revision": "749a502dd1eaf3e5bfd4f8956748c502357c0bbe",
 			"branch": "master",
-			"path": "/context",
-			"notests": true
+			"path": "/context"
 		},
 		{
 			"importpath": "golang.org/x/net/internal/iana",
 			"repository": "https://go.googlesource.com/net",
+			"vcs": "",
 			"revision": "08f168e593b5aab61849054b77981de812666697",
 			"branch": "master",
 			"path": "/internal/iana"
@@ -303,14 +331,15 @@
 		{
 			"importpath": "golang.org/x/net/internal/netreflect",
 			"repository": "https://go.googlesource.com/net",
+			"vcs": "",
 			"revision": "749a502dd1eaf3e5bfd4f8956748c502357c0bbe",
 			"branch": "master",
-			"path": "/internal/netreflect",
-			"notests": true
+			"path": "/internal/netreflect"
 		},
 		{
 			"importpath": "golang.org/x/net/ipv6",
 			"repository": "https://go.googlesource.com/net",
+			"vcs": "",
 			"revision": "749a502dd1eaf3e5bfd4f8956748c502357c0bbe",
 			"branch": "master",
 			"path": "/ipv6"
@@ -318,6 +347,7 @@
 		{
 			"importpath": "golang.org/x/net/proxy",
 			"repository": "https://go.googlesource.com/net",
+			"vcs": "",
 			"revision": "749a502dd1eaf3e5bfd4f8956748c502357c0bbe",
 			"branch": "master",
 			"path": "/proxy"
@@ -325,30 +355,31 @@
 		{
 			"importpath": "golang.org/x/net/route",
 			"repository": "https://go.googlesource.com/net",
+			"vcs": "",
 			"revision": "749a502dd1eaf3e5bfd4f8956748c502357c0bbe",
 			"branch": "master",
-			"path": "/route",
-			"notests": true
+			"path": "/route"
 		},
 		{
 			"importpath": "golang.org/x/sys/unix",
 			"repository": "https://go.googlesource.com/sys",
+			"vcs": "",
 			"revision": "a408501be4d17ee978c04a618e7a1b22af058c0e",
 			"branch": "master",
-			"path": "/unix",
-			"notests": true
+			"path": "/unix"
 		},
 		{
 			"importpath": "golang.org/x/sys/windows",
 			"repository": "https://go.googlesource.com/sys",
+			"vcs": "",
 			"revision": "a408501be4d17ee978c04a618e7a1b22af058c0e",
 			"branch": "master",
-			"path": "/windows",
-			"notests": true
+			"path": "/windows"
 		},
 		{
 			"importpath": "golang.org/x/text/transform",
 			"repository": "https://go.googlesource.com/text",
+			"vcs": "",
 			"revision": "a71fd10341b064c10f4a81ceac72bcf70f26ea34",
 			"branch": "master",
 			"path": "/transform"
@@ -356,6 +387,7 @@
 		{
 			"importpath": "golang.org/x/text/unicode/norm",
 			"repository": "https://go.googlesource.com/text",
+			"vcs": "",
 			"revision": "a71fd10341b064c10f4a81ceac72bcf70f26ea34",
 			"branch": "master",
 			"path": "/unicode/norm"