goroutine 217065 [running]:
runtime/pprof.writeGoroutineStacks(0x9feb80, 0xc4485342a0, 0x30, 0xc442ad4960)
/usr/lib/go-1.8/src/runtime/pprof/pprof.go:603 +0x79
runtime/pprof.writeGoroutine(0x9feb80, 0xc4485342a0, 0x2, 0xc420275a90, 0x411798)
/usr/lib/go-1.8/src/runtime/pprof/pprof.go:592 +0x44
runtime/pprof.(*Profile).WriteTo(0xa2dc00, 0x9feb80, 0xc4485342a0, 0x2, 0xc4485342a0, 0xc420275cc0)
/usr/lib/go-1.8/src/runtime/pprof/pprof.go:302 +0x3b5
net/http/pprof.handler.ServeHTTP(0xc442ad4821, 0x9, 0xa039c0, 0xc4485342a0, 0xc430e80700)
/usr/lib/go-1.8/src/net/http/pprof/pprof.go:209 +0x1d1
net/http/pprof.Index(0xa039c0, 0xc4485342a0, 0xc430e80700)
/usr/lib/go-1.8/src/net/http/pprof/pprof.go:221 +0x1e3
net/http.HandlerFunc.ServeHTTP(0x85ba50, 0xa039c0, 0xc4485342a0, 0xc430e80700)
/usr/lib/go-1.8/src/net/http/server.go:1942 +0x44
net/http.(*ServeMux).ServeHTTP(0xa39e60, 0xa039c0, 0xc4485342a0, 0xc430e80700)
/usr/lib/go-1.8/src/net/http/server.go:2238 +0x130
net/http.serverHandler.ServeHTTP(0xc42025c000, 0xa039c0, 0xc4485342a0, 0xc430e80700)
/usr/lib/go-1.8/src/net/http/server.go:2568 +0x92
net/http.(*conn).serve(0xc44d6940a0, 0xa04100, 0xc449bf8800)
/usr/lib/go-1.8/src/net/http/server.go:1825 +0x612
created by net/http.(*Server).Serve
/usr/lib/go-1.8/src/net/http/server.go:2668 +0x2ce
goroutine 1 [select, 30032 minutes]:
github.com/google/mtail/mtail.(*Mtail).shutdownHandler(0xc420018300)
/home/godog/.go/src/github.com/google/mtail/mtail/mtail.go:255 +0x2b7
github.com/google/mtail/mtail.(*Mtail).Serve(0xc420018300)
/home/godog/.go/src/github.com/google/mtail/mtail/mtail.go:238 +0x28a
github.com/google/mtail/mtail.(*Mtail).Run(0xc420018300)
/home/godog/.go/src/github.com/google/mtail/mtail/mtail.go:289 +0x54
main.main()
/home/godog/.go/src/github.com/google/mtail/main.go:80 +0x5e7
goroutine 17 [syscall, 30032 minutes, locked to thread]:
runtime.goexit()
/usr/lib/go-1.8/src/runtime/asm_amd64.s:2197 +0x1
goroutine 5 [chan receive]:
github.com/golang/glog.(*loggingT).flushDaemon(0xa3a3c0)
/home/godog/.go/src/github.com/golang/glog/glog.go:882 +0x7a
created by github.com/golang/glog.init.1
/home/godog/.go/src/github.com/golang/glog/glog.go:410 +0x21d
goroutine 6 [syscall, 30032 minutes]:
os/signal.signal_recv(0x0)
/usr/lib/go-1.8/src/runtime/sigqueue.go:116 +0x104
os/signal.loop()
/usr/lib/go-1.8/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
/usr/lib/go-1.8/src/os/signal/signal_unix.go:28 +0x41
goroutine 8 [syscall, 30032 minutes]:
syscall.Syscall6(0xe8, 0x4, 0xc42017dcb4, 0x7, 0xffffffffffffffff, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go-1.8/src/syscall/asm_linux_amd64.s:44 +0x5
golang.org/x/sys/unix.EpollWait(0x4, 0xc42017dcb4, 0x7, 0x7, 0xffffffffffffffff, 0x3, 0xc4200eeb60, 0xc42008c100)
/home/godog/.go/src/golang.org/x/sys/unix/zsyscall_linux_amd64.go:1492 +0x77
gopkg.in/fsnotify%2ev1.(*fdPoller).wait(0xc4200d5200, 0x85b400, 0xc4200d5200, 0x0)
/home/godog/.go/src/gopkg.in/fsnotify.v1/inotify_poller.go:86 +0x91
gopkg.in/fsnotify%2ev1.(*Watcher).readEvents(0xc420016b90)
/home/godog/.go/src/gopkg.in/fsnotify.v1/inotify.go:192 +0x196
created by gopkg.in/fsnotify%2ev1.NewWatcher
/home/godog/.go/src/gopkg.in/fsnotify.v1/inotify.go:61 +0x2d1
goroutine 9 [chan receive, 30032 minutes]:
github.com/google/mtail/watcher.(*LogWatcher).run(0xc420015cf0)
/home/godog/.go/src/github.com/google/mtail/watcher/log_watcher.go:46 +0xac
created by github.com/google/mtail/watcher.NewLogWatcher
/home/godog/.go/src/github.com/google/mtail/watcher/log_watcher.go:29 +0xc7
goroutine 10 [chan receive, 30032 minutes]:
github.com/google/mtail/vm.(*Loader).processEvents(0xc420070d80)
/home/godog/.go/src/github.com/google/mtail/vm/loader.go:219 +0xc1
created by github.com/google/mtail/vm.NewLoader
/home/godog/.go/src/github.com/google/mtail/vm/loader.go:205 +0x238
goroutine 11 [chan send, 10262 minutes]:
github.com/google/mtail/vm.(*Loader).processLines(0xc420070d80, 0xc420070a20)
/home/godog/.go/src/github.com/google/mtail/vm/loader.go:243 +0x1ad
created by github.com/google/mtail/vm.NewLoader
/home/godog/.go/src/github.com/google/mtail/vm/loader.go:206 +0x267
goroutine 34 [chan receive, 30032 minutes]:
github.com/google/mtail/watcher.(*LogWatcher).run.func1(0xc420015cf0)
/home/godog/.go/src/github.com/google/mtail/watcher/log_watcher.go:41 +0x63
created by github.com/google/mtail/watcher.(*LogWatcher).run
/home/godog/.go/src/github.com/google/mtail/watcher/log_watcher.go:45 +0x54
goroutine 12 [semacquire, 10262 minutes]:
sync.runtime_Semacquire(0xc4201d2288)
/usr/lib/go-1.8/src/runtime/sema.go:47 +0x34
sync.(*RWMutex).Lock(0xc4201d2280)
/usr/lib/go-1.8/src/sync/rwmutex.go:91 +0x6e
github.com/google/mtail/metrics.(*Metric).GetDatum(0xc4201d2280, 0xc42dafe530, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0)
/home/godog/.go/src/github.com/google/mtail/metrics/metric.go:121 +0x9d
github.com/google/mtail/vm.(*VM).execute(0xc4201da500, 0xc4582c6730, 0x1a, 0x7b1300, 0xc420162c48)
/home/godog/.go/src/github.com/google/mtail/vm/vm.go:493 +0x206
github.com/google/mtail/vm.(*VM).processLine(0xc4201da500, 0xc449248a80, 0x7e)
/home/godog/.go/src/github.com/google/mtail/vm/vm.go:563 +0x174
github.com/google/mtail/vm.(*VM).Run(0xc4201da500, 0xc42016c360, 0xc42016c3c0)
/home/godog/.go/src/github.com/google/mtail/vm/vm.go:579 +0x135
created by github.com/google/mtail/vm.(*Loader).CompileAndRun
/home/godog/.go/src/github.com/google/mtail/vm/loader.go:132 +0x4b0
goroutine 13 [select, 10262 minutes]:
gopkg.in/fsnotify%2ev1.(*Watcher).readEvents(0xc4201669b0)
/home/godog/.go/src/gopkg.in/fsnotify.v1/inotify.go:266 +0x5e2
created by gopkg.in/fsnotify%2ev1.NewWatcher
/home/godog/.go/src/gopkg.in/fsnotify.v1/inotify.go:61 +0x2d1
goroutine 14 [chan send, 10262 minutes]:
github.com/google/mtail/watcher.(*LogWatcher).run(0xc420162e40)
/home/godog/.go/src/github.com/google/mtail/watcher/log_watcher.go:52 +0x267
created by github.com/google/mtail/watcher.NewLogWatcher
/home/godog/.go/src/github.com/google/mtail/watcher/log_watcher.go:29 +0xc7
goroutine 15 [chan send, 10262 minutes]:
github.com/google/mtail/tailer.(*Tailer).read(0xc4201d9420, 0xa06f00, 0xc4202fa008, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc420038f00)
/home/godog/.go/src/github.com/google/mtail/tailer/tail.go:169 +0x1fe
github.com/google/mtail/tailer.(*Tailer).handleLogUpdate(0xc4201d9420, 0xc4479a7ce0, 0x15)
/home/godog/.go/src/github.com/google/mtail/tailer/tail.go:141 +0x17b
github.com/google/mtail/tailer.(*Tailer).run(0xc4201d9420)
/home/godog/.go/src/github.com/google/mtail/tailer/tail.go:328 +0x1dd
created by github.com/google/mtail/tailer.New
/home/godog/.go/src/github.com/google/mtail/tailer/tail.go:88 +0x210
goroutine 35 [chan receive, 30032 minutes]:
github.com/google/mtail/watcher.(*LogWatcher).run.func1(0xc420162e40)
/home/godog/.go/src/github.com/google/mtail/watcher/log_watcher.go:41 +0x63
created by github.com/google/mtail/watcher.(*LogWatcher).run
/home/godog/.go/src/github.com/google/mtail/watcher/log_watcher.go:45 +0x54
goroutine 16 [semacquire, 10262 minutes]:
sync.runtime_Semacquire(0xc4201d228c)
/usr/lib/go-1.8/src/runtime/sema.go:47 +0x34
sync.(*RWMutex).RLock(0xc4201d2280)
/usr/lib/go-1.8/src/sync/rwmutex.go:43 +0x49
github.com/google/mtail/exporter.metricToGraphite(0xc420162e08, 0x7, 0xc4201d2280, 0xc449ee2ac0, 0x0, 0x0)
/home/godog/.go/src/github.com/google/mtail/exporter/graphite.go:25 +0x5a
github.com/google/mtail/exporter.(*Exporter).writeSocketMetrics(0xc420155e00, 0xa06400, 0xc42000e008, 0x85b358, 0xc420015010, 0xc420015018, 0x0, 0x0)
/home/godog/.go/src/github.com/google/mtail/exporter/export.go:102 +0x1f5
github.com/google/mtail/exporter.(*Exporter).WriteMetrics(0xc420155e00)
/home/godog/.go/src/github.com/google/mtail/exporter/export.go:125 +0x2b3
github.com/google/mtail/exporter.(*Exporter).StartMetricPush.func1(0xc4201e40c0, 0xc420155e00)
/home/godog/.go/src/github.com/google/mtail/exporter/export.go:142 +0x56
created by github.com/google/mtail/exporter.(*Exporter).StartMetricPush
/home/godog/.go/src/github.com/google/mtail/exporter/export.go:145 +0xf4
goroutine 50 [IO wait]:
net.runtime_pollWait(0x7f4f1c2b5f00, 0x72, 0x0)
/usr/lib/go-1.8/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc4202f8068, 0x72, 0x0, 0xc42612a380)
/usr/lib/go-1.8/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc4202f8068, 0xffffffffffffffff, 0x0)
/usr/lib/go-1.8/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).accept(0xc4202f8000, 0x0, 0x9fee00, 0xc42612a380)
/usr/lib/go-1.8/src/net/fd_unix.go:430 +0x1e5
net.(*TCPListener).accept(0xc4202fa000, 0xc44d694120, 0x7c8bc0, 0xffffffffffffffff)
/usr/lib/go-1.8/src/net/tcpsock_posix.go:136 +0x2e
net.(*TCPListener).AcceptTCP(0xc4202fa000, 0xc420272e50, 0xc420272e58, 0xc420272e48)
/usr/lib/go-1.8/src/net/tcpsock.go:215 +0x49
net/http.tcpKeepAliveListener.Accept(0xc4202fa000, 0x85b8a0, 0xc44d6940a0, 0xa041c0, 0xc4202ea120)
/usr/lib/go-1.8/src/net/http/server.go:3044 +0x2f
net/http.(*Server).Serve(0xc42025c000, 0xa03b80, 0xc4202fa000, 0x0, 0x0)
/usr/lib/go-1.8/src/net/http/server.go:2643 +0x228
net/http.(*Server).ListenAndServe(0xc42025c000, 0xc42025c000, 0xc420206018)
/usr/lib/go-1.8/src/net/http/server.go:2585 +0xb0
net/http.ListenAndServe(0xc420206018, 0x5, 0x0, 0x0, 0x4, 0xc420206018)
/usr/lib/go-1.8/src/net/http/server.go:2787 +0x7f
github.com/google/mtail/mtail.(*Mtail).Serve.func1(0xc420018300)
/home/godog/.go/src/github.com/google/mtail/mtail/mtail.go:233 +0x108
created by github.com/google/mtail/mtail.(*Mtail).Serve
/home/godog/.go/src/github.com/google/mtail/mtail/mtail.go:237 +0x27c
goroutine 51 [select, 30032 minutes, locked to thread]:
runtime.gopark(0x85be68, 0x0, 0x84225f, 0x6, 0x18, 0x2)
/usr/lib/go-1.8/src/runtime/proc.go:271 +0x13a
runtime.selectgoImpl(0xc42003df50, 0x0, 0x18)
/usr/lib/go-1.8/src/runtime/select.go:423 +0x1364
runtime.selectgo(0xc42003df50)
/usr/lib/go-1.8/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
/usr/lib/go-1.8/src/runtime/signal_unix.go:434 +0x2dd
runtime.goexit()
/usr/lib/go-1.8/src/runtime/asm_amd64.s:2197 +0x1
goroutine 217493 [semacquire, 5378 minutes]:
sync.runtime_Semacquire(0xc4201d228c)
/usr/lib/go-1.8/src/runtime/sema.go:47 +0x34
sync.(*RWMutex).RLock(0xc4201d2280)
/usr/lib/go-1.8/src/sync/rwmutex.go:43 +0x49
github.com/google/mtail/exporter.(*Exporter).HandlePrometheusMetrics(0xc420155e00, 0xa039c0, 0xc448534000, 0xc430e80000)
/home/godog/.go/src/github.com/google/mtail/exporter/prometheus.go:37 +0x110
github.com/google/mtail/exporter.(*Exporter).HandlePrometheusMetrics-fm(0xa039c0, 0xc448534000, 0xc430e80000)
/home/godog/.go/src/github.com/google/mtail/mtail/mtail.go:226 +0x48
net/http.HandlerFunc.ServeHTTP(0xc420162ed0, 0xa039c0, 0xc448534000, 0xc430e80000)
/usr/lib/go-1.8/src/net/http/server.go:1942 +0x44
net/http.(*ServeMux).ServeHTTP(0xa39e60, 0xa039c0, 0xc448534000, 0xc430e80000)
/usr/lib/go-1.8/src/net/http/server.go:2238 +0x130
net/http.serverHandler.ServeHTTP(0xc42025c000, 0xa039c0, 0xc448534000, 0xc430e80000)
/usr/lib/go-1.8/src/net/http/server.go:2568 +0x92
net/http.(*conn).serve(0xc423216960, 0xa04100, 0xc449bf8080)
/usr/lib/go-1.8/src/net/http/server.go:1825 +0x612
created by net/http.(*Server).Serve
/usr/lib/go-1.8/src/net/http/server.go:2668 +0x2ce
goroutine 217066 [runnable]:
net/http.(*connReader).backgroundRead(0xc449bf8840)
/usr/lib/go-1.8/src/net/http/server.go:655
created by net/http.(*connReader).startBackgroundRead
/usr/lib/go-1.8/src/net/http/server.go:652 +0xdf
goroutine 217452 [chan send, 10262 minutes]:
github.com/google/mtail/metrics.(*Metric).EmitLabelSets(0xc4201d2280, 0xc42db79380)
/home/godog/.go/src/github.com/google/mtail/metrics/metric.go:177 +0x115
created by github.com/google/mtail/exporter.(*Exporter).writeSocketMetrics
/home/godog/.go/src/github.com/google/mtail/exporter/export.go:100 +0x16c
goroutine 217500 [semacquire, 3 minutes]:
sync.runtime_Semacquire(0xc4201d228c)
/usr/lib/go-1.8/src/runtime/sema.go:47 +0x34
sync.(*RWMutex).RLock(0xc4201d2280)
/usr/lib/go-1.8/src/sync/rwmutex.go:43 +0x49
github.com/google/mtail/exporter.(*Exporter).HandlePrometheusMetrics(0xc420155e00, 0xa039c0, 0xc4485340e0, 0xc430e80100)
/home/godog/.go/src/github.com/google/mtail/exporter/prometheus.go:37 +0x110
github.com/google/mtail/exporter.(*Exporter).HandlePrometheusMetrics-fm(0xa039c0, 0xc4485340e0, 0xc430e80100)
/home/godog/.go/src/github.com/google/mtail/mtail/mtail.go:226 +0x48
net/http.HandlerFunc.ServeHTTP(0xc420162ed0, 0xa039c0, 0xc4485340e0, 0xc430e80100)
/usr/lib/go-1.8/src/net/http/server.go:1942 +0x44
net/http.(*ServeMux).ServeHTTP(0xa39e60, 0xa039c0, 0xc4485340e0, 0xc430e80100)
/usr/lib/go-1.8/src/net/http/server.go:2238 +0x130
net/http.serverHandler.ServeHTTP(0xc42025c000, 0xa039c0, 0xc4485340e0, 0xc430e80100)
/usr/lib/go-1.8/src/net/http/server.go:2568 +0x92
net/http.(*conn).serve(0xc423217d60, 0xa04100, 0xc449bf8280)
/usr/lib/go-1.8/src/net/http/server.go:1825 +0x612
created by net/http.(*Server).Serve
/usr/lib/go-1.8/src/net/http/server.go:2668 +0x2ce