goroutine profile: total 51
16 @ 0x42e60a 0x42e6be 0x40542b 0x4051c5 0x6eb0ec 0x45b901
# 0x6eb0eb github.com/google/mtail/metrics.(*Metric).EmitLabelSets+0x5b /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/metrics/metric.go:179
13 @ 0x42e60a 0x42992a 0x428fa7 0x49ff9b 0x4a008d 0x4a1987 0x50a15f 0x51b02a 0x6c1436 0x473bae 0x473e37 0x6b52ab 0x473bae 0x473e37 0x6bacf2 0x6baad6 0x4ca485 0x6eebce 0x7546d8 0x6bcc14 0x6be880 0x6bf92c 0x6bbc31 0x45b901
# 0x428fa6 internal/poll.runtime_pollWait+0x56 /usr/lib/go-1.10/src/runtime/netpoll.go:173
# 0x49ff9a internal/poll.(*pollDesc).wait+0x9a /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:85
# 0x4a008c internal/poll.(*pollDesc).waitWrite+0x3c /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:94
# 0x4a1986 internal/poll.(*FD).Write+0x236 /usr/lib/go-1.10/src/internal/poll/fd_unix.go:264
# 0x50a15e net.(*netFD).Write+0x4e /usr/lib/go-1.10/src/net/fd_unix.go:220
# 0x51b029 net.(*conn).Write+0x69 /usr/lib/go-1.10/src/net/net.go:188
# 0x6c1435 net/http.checkConnErrorWriter.Write+0x55 /usr/lib/go-1.10/src/net/http/server.go:3313
# 0x473bad bufio.(*Writer).Flush+0x7d /usr/lib/go-1.10/src/bufio/bufio.go:573
# 0x473e36 bufio.(*Writer).Write+0x106 /usr/lib/go-1.10/src/bufio/bufio.go:609
# 0x6b52aa net/http.(*chunkWriter).Write+0xaa /usr/lib/go-1.10/src/net/http/server.go:372
# 0x473bad bufio.(*Writer).Flush+0x7d /usr/lib/go-1.10/src/bufio/bufio.go:573
# 0x473e36 bufio.(*Writer).Write+0x106 /usr/lib/go-1.10/src/bufio/bufio.go:609
# 0x6bacf1 net/http.(*response).write+0x141 /usr/lib/go-1.10/src/net/http/server.go:1529
# 0x6baad5 net/http.(*response).Write+0x55 /usr/lib/go-1.10/src/net/http/server.go:1499
# 0x4ca484 fmt.Fprintf+0xa4 /usr/lib/go-1.10/src/fmt/print.go:189
# 0x6eebcd github.com/google/mtail/exporter.(*Exporter).HandlePrometheusMetrics+0x3ad /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/exporter/prometheus.go:54
# 0x7546d7 github.com/google/mtail/exporter.(*Exporter).HandlePrometheusMetrics-fm+0x47 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/mtail/mtail.go:210
# 0x6bcc13 net/http.HandlerFunc.ServeHTTP+0x43 /usr/lib/go-1.10/src/net/http/server.go:1947
# 0x6be87f net/http.(*ServeMux).ServeHTTP+0x12f /usr/lib/go-1.10/src/net/http/server.go:2337
# 0x6bf92b net/http.serverHandler.ServeHTTP+0xbb /usr/lib/go-1.10/src/net/http/server.go:2694
# 0x6bbc30 net/http.(*conn).serve+0x650 /usr/lib/go-1.10/src/net/http/server.go:1830
3 @ 0x42e60a 0x42992a 0x428fa7 0x49ff9b 0x4a008d 0x4a1987 0x50a15f 0x51b02a 0x6c1436 0x473bae 0x473e37 0x6b52ab 0x473bae 0x473e37 0x6bacf2 0x6baad6 0x4ca7fb 0x6eea4a 0x7546d8 0x6bcc14 0x6be880 0x6bf92c 0x6bbc31 0x45b901
# 0x428fa6 internal/poll.runtime_pollWait+0x56 /usr/lib/go-1.10/src/runtime/netpoll.go:173
# 0x49ff9a internal/poll.(*pollDesc).wait+0x9a /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:85
# 0x4a008c internal/poll.(*pollDesc).waitWrite+0x3c /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:94
# 0x4a1986 internal/poll.(*FD).Write+0x236 /usr/lib/go-1.10/src/internal/poll/fd_unix.go:264
# 0x50a15e net.(*netFD).Write+0x4e /usr/lib/go-1.10/src/net/fd_unix.go:220
# 0x51b029 net.(*conn).Write+0x69 /usr/lib/go-1.10/src/net/net.go:188
# 0x6c1435 net/http.checkConnErrorWriter.Write+0x55 /usr/lib/go-1.10/src/net/http/server.go:3313
# 0x473bad bufio.(*Writer).Flush+0x7d /usr/lib/go-1.10/src/bufio/bufio.go:573
# 0x473e36 bufio.(*Writer).Write+0x106 /usr/lib/go-1.10/src/bufio/bufio.go:609
# 0x6b52aa net/http.(*chunkWriter).Write+0xaa /usr/lib/go-1.10/src/net/http/server.go:372
# 0x473bad bufio.(*Writer).Flush+0x7d /usr/lib/go-1.10/src/bufio/bufio.go:573
# 0x473e36 bufio.(*Writer).Write+0x106 /usr/lib/go-1.10/src/bufio/bufio.go:609
# 0x6bacf1 net/http.(*response).write+0x141 /usr/lib/go-1.10/src/net/http/server.go:1529
# 0x6baad5 net/http.(*response).Write+0x55 /usr/lib/go-1.10/src/net/http/server.go:1499
# 0x4ca7fa fmt.Fprint+0x8a /usr/lib/go-1.10/src/fmt/print.go:223
# 0x6eea49 github.com/google/mtail/exporter.(*Exporter).HandlePrometheusMetrics+0x229 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/exporter/prometheus.go:57
# 0x7546d7 github.com/google/mtail/exporter.(*Exporter).HandlePrometheusMetrics-fm+0x47 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/mtail/mtail.go:210
# 0x6bcc13 net/http.HandlerFunc.ServeHTTP+0x43 /usr/lib/go-1.10/src/net/http/server.go:1947
# 0x6be87f net/http.(*ServeMux).ServeHTTP+0x12f /usr/lib/go-1.10/src/net/http/server.go:2337
# 0x6bf92b net/http.serverHandler.ServeHTTP+0xbb /usr/lib/go-1.10/src/net/http/server.go:2694
# 0x6bbc30 net/http.(*conn).serve+0x650 /usr/lib/go-1.10/src/net/http/server.go:1830
2 @ 0x42e60a 0x42992a 0x428fa7 0x49ff9b 0x4a001d 0x4a0e7d 0x509c9f 0x51adaa 0x6b6b18 0x47206e 0x472d4c 0x472f64 0x64ad20 0x64ab2b 0x6b150c 0x6b7dbf 0x6bbabc 0x45b901
# 0x428fa6 internal/poll.runtime_pollWait+0x56 /usr/lib/go-1.10/src/runtime/netpoll.go:173
# 0x49ff9a internal/poll.(*pollDesc).wait+0x9a /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:85
# 0x4a001c internal/poll.(*pollDesc).waitRead+0x3c /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:90
# 0x4a0e7c internal/poll.(*FD).Read+0x17c /usr/lib/go-1.10/src/internal/poll/fd_unix.go:157
# 0x509c9e net.(*netFD).Read+0x4e /usr/lib/go-1.10/src/net/fd_unix.go:202
# 0x51ada9 net.(*conn).Read+0x69 /usr/lib/go-1.10/src/net/net.go:176
# 0x6b6b17 net/http.(*connReader).Read+0xf7 /usr/lib/go-1.10/src/net/http/server.go:764
# 0x47206d bufio.(*Reader).fill+0x11d /usr/lib/go-1.10/src/bufio/bufio.go:100
# 0x472d4b bufio.(*Reader).ReadSlice+0x2b /usr/lib/go-1.10/src/bufio/bufio.go:341
# 0x472f63 bufio.(*Reader).ReadLine+0x33 /usr/lib/go-1.10/src/bufio/bufio.go:370
# 0x64ad1f net/textproto.(*Reader).readLineSlice+0x6f /usr/lib/go-1.10/src/net/textproto/reader.go:55
# 0x64ab2a net/textproto.(*Reader).ReadLine+0x2a /usr/lib/go-1.10/src/net/textproto/reader.go:36
# 0x6b150b net/http.readRequest+0x8b /usr/lib/go-1.10/src/net/http/request.go:929
# 0x6b7dbe net/http.(*conn).readRequest+0x16e /usr/lib/go-1.10/src/net/http/server.go:944
# 0x6bbabb net/http.(*conn).serve+0x4db /usr/lib/go-1.10/src/net/http/server.go:1768
2 @ 0x42e60a 0x42e6be 0x406082 0x405d7b 0x6f4d48 0x45b901
# 0x6f4d47 github.com/google/mtail/watcher.(*LogWatcher).run.func1+0x97 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/watcher/log_watcher.go:58
1 @ 0x411152 0x442e46 0x6e4a82 0x45b901
# 0x442e45 os/signal.signal_recv+0xa5 /usr/lib/go-1.10/src/runtime/sigqueue.go:139
# 0x6e4a81 os/signal.loop+0x21 /usr/lib/go-1.10/src/os/signal/signal_unix.go:22
1 @ 0x42e60a 0x42992a 0x428fa7 0x49ff9b 0x4a001d 0x4a0e7d 0x509c9f 0x51adaa 0x6b666a 0x45b901
# 0x428fa6 internal/poll.runtime_pollWait+0x56 /usr/lib/go-1.10/src/runtime/netpoll.go:173
# 0x49ff9a internal/poll.(*pollDesc).wait+0x9a /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:85
# 0x4a001c internal/poll.(*pollDesc).waitRead+0x3c /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:90
# 0x4a0e7c internal/poll.(*FD).Read+0x17c /usr/lib/go-1.10/src/internal/poll/fd_unix.go:157
# 0x509c9e net.(*netFD).Read+0x4e /usr/lib/go-1.10/src/net/fd_unix.go:202
# 0x51ada9 net.(*conn).Read+0x69 /usr/lib/go-1.10/src/net/net.go:176
# 0x6b6669 net/http.(*connReader).backgroundRead+0x59 /usr/lib/go-1.10/src/net/http/server.go:668
1 @ 0x42e60a 0x42992a 0x428fa7 0x49ff9b 0x4a001d 0x4a2418 0x50a5b2 0x52472e 0x522d09 0x6c0f5f 0x6bfcf5 0x6bfa49 0x6c0a8a 0x75434e 0x45b901
# 0x428fa6 internal/poll.runtime_pollWait+0x56 /usr/lib/go-1.10/src/runtime/netpoll.go:173
# 0x49ff9a internal/poll.(*pollDesc).wait+0x9a /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:85
# 0x4a001c internal/poll.(*pollDesc).waitRead+0x3c /usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:90
# 0x4a2417 internal/poll.(*FD).Accept+0x1a7 /usr/lib/go-1.10/src/internal/poll/fd_unix.go:372
# 0x50a5b1 net.(*netFD).accept+0x41 /usr/lib/go-1.10/src/net/fd_unix.go:238
# 0x52472d net.(*TCPListener).accept+0x2d /usr/lib/go-1.10/src/net/tcpsock_posix.go:136
# 0x522d08 net.(*TCPListener).AcceptTCP+0x48 /usr/lib/go-1.10/src/net/tcpsock.go:246
# 0x6c0f5e net/http.tcpKeepAliveListener.Accept+0x2e /usr/lib/go-1.10/src/net/http/server.go:3216
# 0x6bfcf4 net/http.(*Server).Serve+0x1a4 /usr/lib/go-1.10/src/net/http/server.go:2770
# 0x6bfa48 net/http.(*Server).ListenAndServe+0xa8 /usr/lib/go-1.10/src/net/http/server.go:2711
# 0x6c0a89 net/http.ListenAndServe+0x79 /usr/lib/go-1.10/src/net/http/server.go:2969
# 0x75434d github.com/google/mtail/mtail.(*MtailServer).Serve.func1+0xbd /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/mtail/mtail.go:217
1 @ 0x42e60a 0x42e6be 0x40542b 0x4051c5 0x6f4965 0x6f4b8a 0x45b901
# 0x6f4964 github.com/google/mtail/watcher.(*LogWatcher).sendEvent+0x84 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/watcher/log_watcher.go:50
# 0x6f4b89 github.com/google/mtail/watcher.(*LogWatcher).run+0x1e9 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/watcher/log_watcher.go:69
1 @ 0x42e60a 0x42e6be 0x40542b 0x4051c5 0x717db0 0x71786a 0x719cc5 0x45b901
# 0x717daf github.com/google/mtail/tailer.(*Tailer).read+0x21f /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/tailer/tail.go:244
# 0x717869 github.com/google/mtail/tailer.(*Tailer).handleLogUpdate+0x1e9 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/tailer/tail.go:183
# 0x719cc4 github.com/google/mtail/tailer.(*Tailer).run+0x1d4 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/tailer/tail.go:429
1 @ 0x42e60a 0x42e6be 0x40542b 0x4051c5 0x7356aa 0x45b901
# 0x7356a9 github.com/google/mtail/vm.(*Loader).processLines+0x99 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/vm/loader.go:360
1 @ 0x42e60a 0x42e6be 0x406082 0x405d7b 0x4f05bb 0x45b901
# 0x4f05ba github.com/golang/glog.(*loggingT).flushDaemon+0x8a /build/mtail-3.0.0~rc5/build/src/github.com/golang/glog/glog.go:882
1 @ 0x42e60a 0x42e6be 0x406082 0x405d7b 0x6f4a3a 0x45b901
# 0x6f4a39 github.com/google/mtail/watcher.(*LogWatcher).run+0x99 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/watcher/log_watcher.go:63
1 @ 0x42e60a 0x42e6be 0x406082 0x405d7b 0x73541f 0x45b901
# 0x73541e github.com/google/mtail/vm.(*Loader).processEvents+0x6e /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/vm/loader.go:338
1 @ 0x42e60a 0x42e6be 0x43eda4 0x43e9c9 0x4649ae 0x6ea778 0x74658d 0x74b533 0x74b6a5 0x45b901
# 0x43e9c8 sync.runtime_Semacquire+0x38 /usr/lib/go-1.10/src/runtime/sema.go:56
# 0x4649ad sync.(*RWMutex).Lock+0x6d /usr/lib/go-1.10/src/sync/rwmutex.go:98
# 0x6ea777 github.com/google/mtail/metrics.(*Metric).GetDatum+0x67 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/metrics/metric.go:123
# 0x74658c github.com/google/mtail/vm.(*VM).execute+0x24c /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/vm/vm.go:644
# 0x74b532 github.com/google/mtail/vm.(*VM).processLine+0x152 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/vm/vm.go:756
# 0x74b6a4 github.com/google/mtail/vm.(*VM).Run+0xf4 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/vm/vm.go:774
1 @ 0x42e60a 0x43e120 0x4584d4 0x45b901
# 0x42e609 runtime.gopark+0x119 /usr/lib/go-1.10/src/runtime/proc.go:291
# 0x43e11f runtime.selectgo+0xe4f /usr/lib/go-1.10/src/runtime/select.go:392
# 0x4584d3 runtime.ensureSigM.func1+0x1f3 /usr/lib/go-1.10/src/runtime/signal_unix.go:549
1 @ 0x42e60a 0x43e120 0x6f33d0 0x45b901
# 0x6f33cf github.com/fsnotify/fsnotify.(*Watcher).readEvents+0x5af /build/mtail-3.0.0~rc5/build/src/github.com/fsnotify/fsnotify/inotify.go:284
1 @ 0x42e60a 0x43e120 0x753fcd 0x753cda 0x75426c 0x7635c7 0x42e1b2 0x45b901
# 0x753fcc github.com/google/mtail/mtail.(*MtailServer).WaitForShutdown+0x14c /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/mtail/mtail.go:239
# 0x753cd9 github.com/google/mtail/mtail.(*MtailServer).Serve+0x1f9 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/mtail/mtail.go:222
# 0x75426b github.com/google/mtail/mtail.(*MtailServer).Run+0x16b /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/mtail/mtail.go:290
# 0x7635c6 main.main+0x516 /build/mtail-3.0.0~rc5/build/src/github.com/google/mtail/main.go:144
# 0x42e1b1 runtime.main+0x211 /usr/lib/go-1.10/src/runtime/proc.go:198
1 @ 0x48aff5 0x6f1cfa 0x6f3d81 0x6f2fb4 0x45b901
# 0x48aff4 syscall.Syscall6+0x4 /usr/lib/go-1.10/src/syscall/asm_linux_amd64.s:44
# 0x6f1cf9 golang.org/x/sys/unix.EpollWait+0x79 /build/mtail-3.0.0~rc5/build/src/golang.org/x/sys/unix/zsyscall_linux_amd64.go:1555
# 0x6f3d80 github.com/fsnotify/fsnotify.(*fdPoller).wait+0x90 /build/mtail-3.0.0~rc5/build/src/github.com/fsnotify/fsnotify/inotify_poller.go:86
# 0x6f2fb3 github.com/fsnotify/fsnotify.(*Watcher).readEvents+0x193 /build/mtail-3.0.0~rc5/build/src/github.com/fsnotify/fsnotify/inotify.go:192
1 @ 0x75a2a8 0x75a0b0 0x756bf4 0x7623bd 0x762741 0x6bcc14 0x6be880 0x6bf92c 0x6bbc31 0x45b901
# 0x75a2a7 runtime/pprof.writeRuntimeProfile+0x97 /usr/lib/go-1.10/src/runtime/pprof/pprof.go:679
# 0x75a0af runtime/pprof.writeGoroutine+0x9f /usr/lib/go-1.10/src/runtime/pprof/pprof.go:641
# 0x756bf3 runtime/pprof.(*Profile).WriteTo+0x3e3 /usr/lib/go-1.10/src/runtime/pprof/pprof.go:310
# 0x7623bc net/http/pprof.handler.ServeHTTP+0x20c /usr/lib/go-1.10/src/net/http/pprof/pprof.go:243
# 0x762740 net/http/pprof.Index+0x1d0 /usr/lib/go-1.10/src/net/http/pprof/pprof.go:254
# 0x6bcc13 net/http.HandlerFunc.ServeHTTP+0x43 /usr/lib/go-1.10/src/net/http/server.go:1947
# 0x6be87f net/http.(*ServeMux).ServeHTTP+0x12f /usr/lib/go-1.10/src/net/http/server.go:2337
# 0x6bf92b net/http.serverHandler.ServeHTTP+0xbb /usr/lib/go-1.10/src/net/http/server.go:2694
# 0x6bbc30 net/http.(*conn).serve+0x650 /usr/lib/go-1.10/src/net/http/server.go:1830