Performance: make filequeue not suck

integration/test.sh

@@ -30,7 +30,7 @@ EOT
 
 	mkdir files-$i
 	pushd files-$i >/dev/null
-	../genfiles -maxexp 21 -files 400
+	../genfiles -maxexp 21 -files 4000
 	../md5r > ../md5-$i
 	popd >/dev/null
 done

model/filequeue.go

@@ -18,6 +18,7 @@ type FileQueue struct {
 	fmut         sync.Mutex // protects files and sorted
 	availability map[string][]string
 	amut         sync.Mutex // protects availability
+	queued       map[string]bool
 }
 
 type queuedFile struct {
@@ -60,6 +61,7 @@ type queuedBlock struct {
 func NewFileQueue() *FileQueue {
 	return &FileQueue{
 		availability: make(map[string][]string),
+		queued:       make(map[string]bool),
 	}
 }
 
@@ -67,10 +69,8 @@ func (q *FileQueue) Add(name string, blocks []Block, monitor Monitor) {
 	q.fmut.Lock()
 	defer q.fmut.Unlock()
 
-	for _, f := range q.files {
-		if f.name == name {
-			panic("re-adding added file " + f.name)
-		}
+	if q.queued[name] {
+		return
 	}
 
 	q.files = append(q.files, queuedFile{
@@ -81,6 +81,7 @@ func (q *FileQueue) Add(name string, blocks []Block, monitor Monitor) {
 		channel:      make(chan content),
 		monitor:      monitor,
 	})
+	q.queued[name] = true
 	q.sorted = false
 }
 
@@ -116,6 +117,7 @@ func (q *FileQueue) Get(nodeID string) (queuedBlock, bool) {
 					mon.FileDone()
 				}
 			}
+			delete(q.queued, qf.name)
 			q.deleteAt(i)
 			return queuedBlock{}, false
 		}
@@ -159,6 +161,7 @@ func (q *FileQueue) Done(file string, offset int64, data []byte) {
 				err := qf.monitor.FileBegins(qf.channel)
 				if err != nil {
 					log.Printf("WARNING: %s: %v (not synced)", qf.name, err)
+					delete(q.queued, qf.name)
 					q.deleteAt(i)
 					return
 				}
@@ -175,6 +178,7 @@ func (q *FileQueue) Done(file string, offset int64, data []byte) {
 						log.Printf("WARNING: %s: %v", qf.name, err)
 					}
 				}
+				delete(q.queued, qf.name)
 				q.deleteAt(i)
 			}
 			return
@@ -183,18 +187,6 @@ func (q *FileQueue) Done(file string, offset int64, data []byte) {
 	panic("unreachable")
 }
 
-func (q *FileQueue) Queued(file string) bool {
-	q.fmut.Lock()
-	defer q.fmut.Unlock()
-
-	for _, qf := range q.files {
-		if qf.name == file {
-			return true
-		}
-	}
-	return false
-}
-
 func (q *FileQueue) QueuedFiles() (files []string) {
 	q.fmut.Lock()
 	defer q.fmut.Unlock()
@@ -213,6 +205,7 @@ func (q *FileQueue) deleteFile(n string) {
 	for i, file := range q.files {
 		if n == file.name {
 			q.deleteAt(i)
+			delete(q.queued, file.name)
 			return
 		}
 	}

model/model.go

@@ -755,9 +755,7 @@ func (m *Model) recomputeNeed() {
 	m.gmut.RUnlock()
 
 	for _, ao := range toAdd {
-		if !m.fq.Queued(ao.n) {
-			m.fq.Add(ao.n, ao.remote, ao.fm)
-		}
+		m.fq.Add(ao.n, ao.remote, ao.fm)
 	}
 	for _, gf := range toDelete {
 		m.dq <- gf

model/model_test.go

@@ -385,10 +385,7 @@ func TestIgnoreWithUnknownFlags(t *testing.T) {
 	}
 }
 
-func prepareModel(n int, m *Model) []protocol.FileInfo {
-	fs, _ := m.Walk(false)
-	m.ReplaceLocal(fs)
-
+func genFiles(n int) []protocol.FileInfo {
 	files := make([]protocol.FileInfo, n)
 	t := time.Now().Unix()
 	for i := 0; i < n; i++ {
@@ -399,33 +396,39 @@ func prepareModel(n int, m *Model) []protocol.FileInfo {
 		}
 	}
 
-	m.Index("42", files)
 	return files
 }
 
-func BenchmarkRecomputeGlobal10k(b *testing.B) {
+func BenchmarkIndex10000(b *testing.B) {
 	m := NewModel("testdata", 1e6)
-	prepareModel(10000, m)
+	fs, _ := m.Walk(false)
+	m.ReplaceLocal(fs)
+	files := genFiles(10000)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.recomputeGlobal()
+		m.Index("42", files)
 	}
 }
 
-func BenchmarkRecomputeNeed10K(b *testing.B) {
+func BenchmarkIndex00100(b *testing.B) {
 	m := NewModel("testdata", 1e6)
-	prepareModel(10000, m)
+	fs, _ := m.Walk(false)
+	m.ReplaceLocal(fs)
+	files := genFiles(100)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.recomputeNeed()
+		m.Index("42", files)
 	}
 }
 
-func BenchmarkIndexUpdate10000(b *testing.B) {
+func BenchmarkIndexUpdate10000f10000(b *testing.B) {
 	m := NewModel("testdata", 1e6)
-	files := prepareModel(10000, m)
+	fs, _ := m.Walk(false)
+	m.ReplaceLocal(fs)
+	files := genFiles(10000)
+	m.Index("42", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -433,6 +436,34 @@ func BenchmarkIndexUpdate10000(b *testing.B) {
 	}
 }
 
+func BenchmarkIndexUpdate10000f00100(b *testing.B) {
+	m := NewModel("testdata", 1e6)
+	fs, _ := m.Walk(false)
+	m.ReplaceLocal(fs)
+	files := genFiles(10000)
+	m.Index("42", files)
+
+	ufiles := genFiles(100)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.IndexUpdate("42", ufiles)
+	}
+}
+
+func BenchmarkIndexUpdate10000f00001(b *testing.B) {
+	m := NewModel("testdata", 1e6)
+	fs, _ := m.Walk(false)
+	m.ReplaceLocal(fs)
+	files := genFiles(10000)
+	m.Index("42", files)
+
+	ufiles := genFiles(1)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.IndexUpdate("42", ufiles)
+	}
+}
+
 type FakeConnection struct {
 	id          string
 	requestData []byte