Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

panic: runtime error: index out of range #781

Closed
klyngbaek opened this issue Jul 25, 2014 · 3 comments
Closed

panic: runtime error: index out of range #781

klyngbaek opened this issue Jul 25, 2014 · 3 comments
Milestone
0.8.0

Comments

@klyngbaek
Copy link

I have two crash reports indicating a panic: runtime error: index out of range error.

  | |_   _|      / _| |          |  __ \|  _ \  |
  |   | |  _ __ | |_| |_   ___  _| |  | | |_) | |
  |   | | | '_ \|  _| | | | \ \/ / |  | |  _ <  |
  |  _| |_| | | | | | | |_| |>  <| |__| | |_) | |
  | |_____|_| |_|_| |_|\__,_/_/\_\_____/|____/  |
  +---------------------------------------------+

  panic: runtime error: index out of range

  goroutine 32 [running]:
  runtime.panic(0x8a29e0, 0x1042d57)
         /home/vagrant/bin/go/src/pkg/runtime/panic.c:266 +0xb6
  engine.GetValue(0xc21001edc0, 0xc21020d140, 0x4, 0x4, 0xc21020d180, ...)
         /home/vagrant/influxdb/src/engine/arithmetic.go:31 +0x377
  engine.PlusOperator(0xc210124750, 0x2, 0x2, 0xc21020d140, 0x4, ...)
         /home/vagrant/influxdb/src/engine/arithmetic.go:54 +0x116
  engine.GetValue(0xc21001e9b0, 0xc21020d140, 0x4, 0x4, 0xc21020d180, ...)
         /home/vagrant/influxdb/src/engine/arithmetic.go:37 +0x538
  engine.func·014(0xc21010fcd0, 0xc21010fcd0, 0xc21010fcd0)
         /home/vagrant/influxdb/src/engine/engine.go:666 +0x376
  engine.func·016(0xc21010fc80, 0xc21020c0c8, 0x2)
         /home/vagrant/influxdb/src/engine/merge.go:57 +0x8cb
  engine.(*mergeState).yieldNextPoints(0xc210165180, 0xc210128050, 0x0, 0x0)
         /home/vagrant/influxdb/src/engine/merge.go:193 +0x23d
  engine.func·018(0xc210167410, 0x7de840, 0xc2101650f0)
         /home/vagrant/influxdb/src/engine/merge.go:279 +0x9c
  engine.(*QueryEngine).yieldSeriesData(0xc210150000, 0xc210167410, 0xc210188cf0)
         /home/vagrant/influxdb/src/engine/engine.go:176 +0x30
  engine.(*QueryEngine).YieldSeries(0xc210150000, 0xc210167410, 0x7fb60d392f38)
         /home/vagrant/influxdb/src/engine/engine.go:172 +0x122
  coordinator.(*CoordinatorImpl).readFromResponseChannels(0xc2100ad8c0, 0x7fb6146dd690, 0xc210150000, 0x7fb6146dd550, 0xc2101262c0, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:367 +0x2f7
  created by coordinator.(*CoordinatorImpl).runQuerySpec
         /home/vagrant/influxdb/src/coordinator/coordinator.go:441 +0x3df

  goroutine 1 [IO wait]:
  net.runtime_pollWait(0x7fb6146d73f0, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc2100b1920, 0x72, 0x7fb6146cef88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc2100b1920, 0xb, 0x7fb6146cef88)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc2100b18c0, 0xa1d1f8, 0x0, 0x7fb6146cef88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc2101aeae8, 0x5d2c1b, 0x7fb61453f960, 0x5d2c1b)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc2101aeae8, 0x7fb6146dd118, 0xc2101ba6e8, 0xc210154d80, 0x0)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  net/http.(*Server).Serve(0xc210128730, 0x7fb6146d65a0, 0xc2101aeae8, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1622 +0x91
  api/http.(*HttpServer).serveListener(0xc2100c4500, 0x7fb6146d65a0, 0xc2101aeae8, 0xc2101aeaf0)
         /home/vagrant/influxdb/src/api/http/api.go:187 +0x98
  api/http.(*HttpServer).Serve(0xc2100c4500, 0x7fb6146d65a0, 0xc2101aeae8)
         /home/vagrant/influxdb/src/api/http/api.go:157 +0xce4
  api/http.(*HttpServer).ListenAndServe(0xc2100c4500)
         /home/vagrant/influxdb/src/api/http/api.go:79 +0x16c
  server.(*Server).ListenAndServe(0xc210043900, 0xc210043900, 0x0)
         /home/vagrant/influxdb/src/server/server.go:170 +0xbb0
  main.main()
         /home/vagrant/influxdb/src/daemon/influxd.go:159 +0xd2a

  goroutine 3 [syscall]:
  os/signal.loop()
         /home/vagrant/bin/go/src/pkg/os/signal/signal_unix.go:21 +0x1e
  created by os/signal.init·1
         /home/vagrant/bin/go/src/pkg/os/signal/signal_unix.go:27 +0x31

  goroutine 4 [chan receive]:
  code.google.com/p/log4go.ConsoleLogWriter.run(0xc210069000, 0x7fb6146cf110, 0xc210000008)
  /home/vagrant/influxdb/src/code.google.com/p/log4go/termlog.go:27 +0x60
  created by code.google.com/p/log4go.NewConsoleLogWriter
  /home/vagrant/influxdb/src/code.google.com/p/log4go/termlog.go:19 +0x67

  goroutine 5 [syscall]:
  runtime.goexit()
         /home/vagrant/bin/go/src/pkg/runtime/proc.c:1394

  goroutine 6 [select]:
  code.google.com/p/log4go.func·002()
  /home/vagrant/influxdb/src/code.google.com/p/log4go/filelog.go:84 +0x84c
  created by code.google.com/p/log4go.NewFileLogWriter
  /home/vagrant/influxdb/src/code.google.com/p/log4go/filelog.go:116 +0x2d1

  goroutine 7 [finalizer wait]:
  runtime.park(0x451a20, 0x1059c78, 0x10446a8)
         /home/vagrant/bin/go/src/pkg/runtime/proc.c:1342 +0x66
  runfinq()
         /home/vagrant/bin/go/src/pkg/runtime/mgc0.c:2279 +0x84
  runtime.goexit()
         /home/vagrant/bin/go/src/pkg/runtime/proc.c:1394

  goroutine 8 [chan receive]:
  wal.(*WAL).processEntries(0xc210071e00)
         /home/vagrant/influxdb/src/wal/wal.go:252 +0x3f
  created by wal.NewWAL
         /home/vagrant/influxdb/src/wal/wal.go:103 +0x9f3

  goroutine 9 [sleep]:
  time.Sleep(0x8bb2c97000)
         /tmp/makerelease886106415/go/src/pkg/runtime/time.goc:31 +0x31
  cluster.func·001()
  /home/vagrant/influxdb/src/cluster/cluster_configuration.go:133 +0x35
  created by cluster.(*ClusterConfiguration).CreateFutureShardsAutomaticallyBeforeTimeComes
  /home/vagrant/influxdb/src/cluster/cluster_configuration.go:138 +0x63

  goroutine 10 [chan receive]:
  main.waitForSignals(0x7fb6146d6520, 0xc210043900)
         /home/vagrant/influxdb/src/daemon/null_profiler.go:23 +0x126
  created by main.startProfiler
         /home/vagrant/influxdb/src/daemon/null_profiler.go:15 +0x38

  goroutine 12 [IO wait]:
  net.runtime_pollWait(0x7fb6146d7540, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc210051680, 0x72, 0x7fb6146cef88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc210051680, 0xb, 0x7fb6146cef88)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc210051620, 0xa1d1f8, 0x0, 0x7fb6146cef88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc2100bf088, 0x18, 0xc2100de810, 0x5d29f3)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc2100bf088, 0x0, 0x0, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  net/http.(*Server).Serve(0xc21010f5f0, 0x7fb6146d65a0, 0xc2100bf088, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1622 +0x91
  coordinator.func·009()
         /home/vagrant/influxdb/src/coordinator/raft_server.go:527 +0x3a
  created by coordinator.(*RaftServer).Serve
         /home/vagrant/influxdb/src/coordinator/raft_server.go:531 +0x4d9

  goroutine 24 [chan receive]:
  server.(*Server).startReportingLoop(0xc210043900, 0xc210195800)
         /home/vagrant/influxdb/src/server/server.go:182 +0xbf
  created by server.(*Server).ListenAndServe
         /home/vagrant/influxdb/src/server/server.go:163 +0xaae

  goroutine 14 [select]:
  github.com/goraft/raft.(*server).leaderLoop(0xc210149120)
  /home/vagrant/influxdb/src/github.com/goraft/raft/server.go:765 +0x5fe
  github.com/goraft/raft.(*server).loop(0xc210149120)
  /home/vagrant/influxdb/src/github.com/goraft/raft/server.go:568 +0x33f
  created by github.com/goraft/raft.(*server).Start
  /home/vagrant/influxdb/src/github.com/goraft/raft/server.go:472 +0x7af

  goroutine 15 [select]:
  coordinator.(*RaftServer).CompactLog(0xc2100d89a0)
         /home/vagrant/influxdb/src/coordinator/raft_server.go:317 +0x2ef
  created by coordinator.(*RaftServer).startRaft
         /home/vagrant/influxdb/src/coordinator/raft_server.go:371 +0x375

  goroutine 17 [select]:
  coordinator.(*RaftServer).raftLeaderLoop(0xc2100d89a0, 0xc2101c0b00)
         /home/vagrant/influxdb/src/coordinator/raft_server.go:427 +0x29c
  created by coordinator.(*RaftServer).raftEventHandler
         /home/vagrant/influxdb/src/coordinator/raft_server.go:416 +0x1d0

  goroutine 19 [IO wait]:
  net.runtime_pollWait(0x7fb6146d7498, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc21014c290, 0x72, 0x7fb6146cef88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc21014c290, 0xb, 0x7fb6146cef88)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc21014c230, 0xa1d1f8, 0x0, 0x7fb6146cef88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc2100bf058, 0xc2100e9690, 0x0, 0x7fb6146d6570)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc2100bf058, 0xc2100e9690, 0x7fb614535f38, 0x1, 0x1)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  coordinator.(*ProtobufServer).ListenAndServe(0xc2101328c0)
         /home/vagrant/influxdb/src/coordinator/protobuf_server.go:64 +0x1c7
  created by server.(*Server).ListenAndServe
         /home/vagrant/influxdb/src/server/server.go:118 +0x218

  goroutine 20 [select]:
  cluster.(*WriteBuffer).handleWrites(0xc21007ea80)
         /home/vagrant/influxdb/src/cluster/write_buffer.go:75 +0xca
  created by cluster.NewWriteBuffer
         /home/vagrant/influxdb/src/cluster/write_buffer.go:43 +0x24f

  goroutine 23 [IO wait]:
  net.runtime_pollWait(0x7fb6146d7348, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc210051f40, 0x72, 0x7fb6146cef88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc210051f40, 0xb, 0x7fb6146cef88)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc210051ee0, 0xa1d1f8, 0x0, 0x7fb6146cef88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc2102740c0, 0x18, 0xc210178010, 0x5d29f3)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc2102740c0, 0x50, 0x1065b00, 0x18, 0x0)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  net/http.(*Server).Serve(0xc210167a00, 0x7fb6146d65a0, 0xc2102740c0, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1622 +0x91
  net/http.Serve(0x7fb6146d65a0, 0xc2102740c0, 0x7fb6146dd098, 0xc2101ebe20, 0x1b, ...)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1561 +0x70
  admin.(*HttpServer).ListenAndServe(0xc210132940)
         /home/vagrant/influxdb/src/admin/http_server.go:35 +0x170
  created by server.(*Server).ListenAndServe
         /home/vagrant/influxdb/src/server/server.go:132 +0x46c

  goroutine 29 [runnable]:
  coordinator.(*CoordinatorImpl).runQuerySpec(0xc2100ad8c0, 0xc21000f500, 0x7fb6146dd550, 0xc2101262c0, 0x0, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:446 +0x47e
  coordinator.(*CoordinatorImpl).runQuery(0xc2100ad8c0, 0xc21000f500, 0x7fb6146dd550, 0xc2101262c0, 0x0, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:157 +0x45
  coordinator.(*CoordinatorImpl).RunQuery(0xc2100ad8c0, 0x7fb6146dd4c8, 0xc210176200, 0xc210200006, 0xa, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:135 +0x8d8
  api/http.func·004(0x7fb6146dd4c8, 0xc210176200, 0xc210200013, 0x4, 0xc21020001a)
         /home/vagrant/influxdb/src/api/http/api.go:310 +0x2e2
  api/http.yieldUser(0x7fb6146dd4c8, 0xc210176200, 0x7fb60d376c10, 0xc21020001a, 0x4, ...)
         /home/vagrant/influxdb/src/api/http/api.go:477 +0x3b
  api/http.(*HttpServer).tryAsClusterAdmin(0xc2100c4500, 0x7fb6146dd468, 0xc21000f460, 0xc2101ec000, 0x7fb60d376c10)
         /home/vagrant/influxdb/src/api/http/api.go:539 +0x34a
  api/http.(*HttpServer).tryAsDbUserAndClusterAdmin(0xc2100c4500, 0x7fb6146dd468, 0xc21000f460, 0xc2101ec000, 0x7fb60d376c10)
         /home/vagrant/influxdb/src/api/http/api.go:709 +0x268
  api/http.(*HttpServer).query(0xc2100c4500, 0x7fb6146dd468, 0xc21000f460, 0xc2101ec000)
         /home/vagrant/influxdb/src/api/http/api.go:320 +0x14d
  api/http.*HttpServer.(api/http.query)·fm(0x7fb6146dd468, 0xc21000f460, 0xc2101ec000)
         /home/vagrant/influxdb/src/api/http/api.go:103 +0x44
  api/http.func·032(0x7fb6146dd468, 0xc21000f460, 0xc2101ec000)
         /home/vagrant/influxdb/src/api/http/cors.go:14 +0x1d0
  net/http.HandlerFunc.ServeHTTP(0xc2100bdd20, 0x7fb6146dd468, 0xc21000f460, 0xc2101ec000)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1220 +0x40
  github.com/bmizerany/pat.(*PatternServeMux).ServeHTTP(0xc2101aeaf0, 0x7fb6146dd468, 0xc21000f460, 0xc2101ec000)
  /home/vagrant/influxdb/src/github.com/bmizerany/pat/mux.go:109 +0x1be
  net/http.serverHandler.ServeHTTP(0xc210128730, 0x7fb6146dd468, 0xc21000f460, 0xc2101ec000)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1597 +0x16e
  net/http.(*conn).serve(0xc210154d00)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1167 +0x7b7
  created by net/http.(*Server).Serve
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1644 +0x28b

  goroutine 30 [IO wait]:
  net.runtime_pollWait(0x7fb6146d71f8, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc21019c840, 0x72, 0x7fb6146cef88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc21019c840, 0xb, 0x7fb6146cef88)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).Read(0xc21019c7e0, 0xc2101ed000, 0x1000, 0x1000, 0x0, ...)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:204 +0x2a0
  net.(*conn).Read(0xc2101ba6e8, 0xc2101ed000, 0x1000, 0x1000, 0x0, ...)
         /home/vagrant/bin/go/src/pkg/net/net.go:122 +0xc5
  net/http.(*liveSwitchReader).Read(0xc210154da8, 0xc2101ed000, 0x1000, 0x1000, 0xf6, ...)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:204 +0xa5
  io.(*LimitedReader).Read(0xc2101b99a0, 0xc2101ed000, 0x1000, 0x1000, 0xf6, ...)
         /home/vagrant/bin/go/src/pkg/io/io.go:398 +0xbb
  bufio.(*Reader).fill(0xc2101c22a0)
         /home/vagrant/bin/go/src/pkg/bufio/bufio.go:91 +0x110
  bufio.(*Reader).ReadSlice(0xc2101c22a0, 0xc21015b00a, 0x0, 0x0, 0x0, ...)
         /home/vagrant/bin/go/src/pkg/bufio/bufio.go:274 +0x204
  bufio.(*Reader).ReadLine(0xc2101c22a0, 0x0, 0x0, 0x0, 0x0, ...)
         /home/vagrant/bin/go/src/pkg/bufio/bufio.go:305 +0x63
  net/textproto.(*Reader).readLineSlice(0xc2100a82a0, 0x7fb6146c6000, 0x7ce9a0, 0x7fb614553ce8, 0x466eb2, ...)
         /home/vagrant/bin/go/src/pkg/net/textproto/reader.go:55 +0x61
  net/textproto.(*Reader).ReadLine(0xc2100a82a0, 0xc2100d65b0, 0x0, 0x589de5, 0xc21019c7e0)
         /home/vagrant/bin/go/src/pkg/net/textproto/reader.go:36 +0x27
  net/http.ReadRequest(0xc2101c22a0, 0xc2100d65b0, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/request.go:526 +0x88
  net/http.(*conn).readRequest(0xc210154d80, 0x0, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:575 +0x1bb
  net/http.(*conn).serve(0xc210154d80)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1123 +0x3b4
  created by net/http.(*Server).Serve
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1644 +0x28b

  goroutine 31 [chan receive]:
  coordinator.func·002()
         /home/vagrant/influxdb/src/coordinator/coordinator.go:313 +0x57
  created by coordinator.(*CoordinatorImpl).getShardsAndProcessor
         /home/vagrant/influxdb/src/coordinator/coordinator.go:326 +0x36b
  root@rts7:~# ^C
  root@rts7:~# cd /mnt/d1
  root@rts7:/mnt/d1# cd influxdb/
  root@rts7:/mnt/d1/influxdb# l
  total 6228
  lrwxrwxrwx 1 root     root          14 Jul 18 21:19 current -> versions/0.7.3
  -rw-r--r-- 1 root     root     6367936 Jun 13 15:33 influxdb_latest_amd64.deb
  drwxr-xr-x 4 influxdb influxdb    4096 Jul 24 11:50 shared
  drwxrwxr-x 3 influxdb influxdb    4096 Jul 18 21:16 versions
  root@rts7:/mnt/d1/influxdb# cd shared/
  root@rts7:/mnt/d1/influxdb/shared# l
  total 39272
  -rw-r--r-- 1 influxdb influxdb     5080 Jul 18 21:16 benchmark_config.toml
  -rw-r--r-- 1 influxdb influxdb     8223 Jul 20 09:15 config.toml
  drwxr--r-- 5 influxdb influxdb     4096 Jul 20 09:18 data
  -rw-r--r-- 1 influxdb influxdb        5 Jul 24 11:54 influxdb.pid
  -rw-r----- 1 influxdb influxdb 27026209 Jul 24 11:55 log.txt
  -rw-r--r-- 1 influxdb influxdb    37932 Jul 18 23:57 log.txt.001
  -rw-r----- 1 influxdb influxdb   355194 Jul 19 23:58 log.txt.002
  -rw-r----- 1 influxdb influxdb   660649 Jul 20 23:59 log.txt.003
  -rw-r----- 1 influxdb influxdb   430946 Jul 21 23:59 log.txt.004
  -rw-r----- 1 influxdb influxdb  1552319 Jul 22 23:57 log.txt.005
  -rw-r----- 1 influxdb influxdb 10076114 Jul 23 23:59 log.txt.006
  drwxr--r-- 5 influxdb influxdb     4096 Jul 18 21:17 xx
  root@rts7:/mnt/d1/influxdb/shared# tail log.txt
  [2014/07/24 11:55:14 PDT] [INFO] (cluster.func·004:1048) Recovered local server
  [2014/07/24 11:55:14 PDT] [INFO] (server.(*Server).ListenAndServe:125) recovered
  [2014/07/24 11:55:14 PDT] [INFO] (coordinator.(*CoordinatorImpl).ConnectToProtobufServers:1014) Connecting to other nodes in the cluster
  [2014/07/24 11:55:14 PDT] [INFO] (server.(*Server).ListenAndServe:131) Starting admin interface on port 8083
  [2014/07/24 11:55:14 PDT] [WARN] (server.(*Server).ListenAndServe:148) Cannot start udp server. please check your configuration
  [2014/07/24 11:55:14 PDT] [WARN] (server.(*Server).ListenAndServe:148) Cannot start udp server. please check your configuration
  [2014/07/24 11:55:14 PDT] [INFO] (server.(*Server).ListenAndServe:169) Starting Http Api server on port 8086
  [2014/07/24 11:55:14 PDT] [INFO] (server.(*Server).reportStats:207) Reporting stats: &influxdb.Series{Name:"reports", Columns:[]string{"os", "arch", "id", "version"}, Points:[][]interface {}{[]interface {}{"linux", "amd64", "41e3bb229d21bcca", "0.7.3"}}}
  [2014/07/24 11:55:26 PDT] [INFO] (coordinator.(*CoordinatorImpl).RunQuery:69) Start Query: db: test_stats, u: root, q: select 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e.iops_read) from 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e inner join 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e
  [2014/07/24 11:55:26 PDT] [INFO] (coordinator.(*CoordinatorImpl).RunQuery:69) Start Query: db: test_stats, u: root, q: select (feefaf1d-c77a-0010-9ebc-60eb6982134c.memFree + 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e.iops_read) from feefaf1d-c77a-0010-9ebc-60eb6982134c inner join 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e
  root@rts7:/mnt/d1/influxdb/shared# tail -f log.txt
  [2014/07/24 11:55:14 PDT] [INFO] (cluster.func·004:1048) Recovered local server
  [2014/07/24 11:55:14 PDT] [INFO] (server.(*Server).ListenAndServe:125) recovered
  [2014/07/24 11:55:14 PDT] [INFO] (coordinator.(*CoordinatorImpl).ConnectToProtobufServers:1014) Connecting to other nodes in the cluster
  [2014/07/24 11:55:14 PDT] [INFO] (server.(*Server).ListenAndServe:131) Starting admin interface on port 8083
  [2014/07/24 11:55:14 PDT] [WARN] (server.(*Server).ListenAndServe:148) Cannot start udp server. please check your configuration
  [2014/07/24 11:55:14 PDT] [WARN] (server.(*Server).ListenAndServe:148) Cannot start udp server. please check your configuration
  [2014/07/24 11:55:14 PDT] [INFO] (server.(*Server).ListenAndServe:169) Starting Http Api server on port 8086
  [2014/07/24 11:55:14 PDT] [INFO] (server.(*Server).reportStats:207) Reporting stats: &influxdb.Series{Name:"reports", Columns:[]string{"os", "arch", "id", "version"}, Points:[][]interface {}{[]interface {}{"linux", "amd64", "41e3bb229d21bcca", "0.7.3"}}}
  [2014/07/24 11:55:26 PDT] [INFO] (coordinator.(*CoordinatorImpl).RunQuery:69) Start Query: db: test_stats, u: root, q: select 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e.iops_read) from 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e inner join 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e
  [2014/07/24 11:55:26 PDT] [INFO] (coordinator.(*CoordinatorImpl).RunQuery:69) Start Query: db: test_stats, u: root, q: select (feefaf1d-c77a-0010-9ebc-60eb6982134c.memFree + 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e.iops_read) from feefaf1d-c77a-0010-9ebc-60eb6982134c inner join 4d5188fc-f4a5-4e21-b52c-88299b0f2a4e
  runtime.park(0x451a20, 0x1059c78, 0x10446a8)
         /home/vagrant/bin/go/src/pkg/runtime/proc.c:1342 +0x66
  runfinq()
         /home/vagrant/bin/go/src/pkg/runtime/mgc0.c:2279 +0x84
  runtime.goexit()
         /home/vagrant/bin/go/src/pkg/runtime/proc.c:1394

  goroutine 8 [chan receive]:
  wal.(*WAL).processEntries(0xc210070e00)
         /home/vagrant/influxdb/src/wal/wal.go:252 +0x3f
  created by wal.NewWAL
         /home/vagrant/influxdb/src/wal/wal.go:103 +0x9f3

  goroutine 9 [sleep]:
  time.Sleep(0x8bb2c97000)
         /tmp/makerelease886106415/go/src/pkg/runtime/time.goc:31 +0x31
  cluster.func·001()
  /home/vagrant/influxdb/src/cluster/cluster_configuration.go:133 +0x35
  created by cluster.(*ClusterConfiguration).CreateFutureShardsAutomaticallyBeforeTimeComes
  /home/vagrant/influxdb/src/cluster/cluster_configuration.go:138 +0x63

  goroutine 11 [chan receive]:
  main.waitForSignals(0x7f0e5eaac728, 0xc210042900)
         /home/vagrant/influxdb/src/daemon/null_profiler.go:23 +0x126
  created by main.startProfiler
         /home/vagrant/influxdb/src/daemon/null_profiler.go:15 +0x38

  goroutine 12 [IO wait]:
  net.runtime_pollWait(0x7f0e5eaad748, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc210050220, 0x72, 0x7f0e5eaa4f88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc210050220, 0xb, 0x7f0e5eaa4f88)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc2100501c0, 0xa1d1f8, 0x0, 0x7f0e5eaa4f88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc21010ec18, 0x18, 0xc2100dc810, 0x5d29f3)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc21010ec18, 0x0, 0x0, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  net/http.(*Server).Serve(0xc2101019b0, 0x7f0e5eaac7a8, 0xc21010ec18, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1622 +0x91
  coordinator.func·009()
        /home/vagrant/influxdb/src/coordinator/raft_server.go:527 +0x3a
  created by coordinator.(*RaftServer).Serve
         /home/vagrant/influxdb/src/coordinator/raft_server.go:531 +0x4d9

  goroutine 14 [select]:
  github.com/goraft/raft.(*server).leaderLoop(0xc210051240)
  /home/vagrant/influxdb/src/github.com/goraft/raft/server.go:765 +0x5fe
  github.com/goraft/raft.(*server).loop(0xc210051240)
  /home/vagrant/influxdb/src/github.com/goraft/raft/server.go:568 +0x33f
  created by github.com/goraft/raft.(*server).Start
  /home/vagrant/influxdb/src/github.com/goraft/raft/server.go:472 +0x7af

  goroutine 15 [select]:
  coordinator.(*RaftServer).CompactLog(0xc2100d6b00)
         /home/vagrant/influxdb/src/coordinator/raft_server.go:317 +0x2ef
  created by coordinator.(*RaftServer).startRaft
         /home/vagrant/influxdb/src/coordinator/raft_server.go:371 +0x375

  goroutine 17 [select]:
  coordinator.(*RaftServer).raftLeaderLoop(0xc2100d6b00, 0xc2101b46c0)
         /home/vagrant/influxdb/src/coordinator/raft_server.go:427 +0x29c
  created by coordinator.(*RaftServer).raftEventHandler
         /home/vagrant/influxdb/src/coordinator/raft_server.go:416 +0x1d0

  goroutine 19 [IO wait]:
  net.runtime_pollWait(0x7f0e5eaad6a0, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc21007f300, 0x72, 0x7f0e5eaa4f88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc21007f300, 0xb, 0x7f0e5eaa4f88)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc21007f2a0, 0xa1d1f8, 0x0, 0x7f0e5eaa4f88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc2100001f0, 0xc210217d90, 0x0, 0x7f0e5eaac778)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc2100001f0, 0xc210217d90, 0x7f0e58a7ef38, 0x1, 0x1)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  coordinator.(*ProtobufServer).ListenAndServe(0xc2101039c0)
         /home/vagrant/influxdb/src/coordinator/protobuf_server.go:64 +0x1c7
  created by server.(*Server).ListenAndServe
         /home/vagrant/influxdb/src/server/server.go:118 +0x218

  goroutine 20 [select]:
  cluster.(*WriteBuffer).handleWrites(0xc21007f700)
         /home/vagrant/influxdb/src/cluster/write_buffer.go:75 +0xca
  created by cluster.NewWriteBuffer
         /home/vagrant/influxdb/src/cluster/write_buffer.go:43 +0x24f

  goroutine 29 [runnable]:
  coordinator.(*CoordinatorImpl).runQuerySpec(0xc2100b1a40, 0xc2101d2320, 0x7f0e5eab3810, 0xc210000cc8, 0x0, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:446 +0x47e
  coordinator.(*CoordinatorImpl).runQuery(0xc2100b1a40, 0xc2101d2320, 0x7f0e5eab3810, 0xc210000cc8, 0x0, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:157 +0x45
  coordinator.(*CoordinatorImpl).RunQuery(0xc2100b1a40, 0x7f0e5eab3788, 0xc21014e140, 0xc2101d1906, 0xa, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:135 +0x8d8
  api/http.func·004(0x7f0e5eab3788, 0xc21014e140, 0xc2101d1913, 0x4, 0xc2101d191a)
        /home/vagrant/influxdb/src/api/http/api.go:310 +0x2e2
  api/http.yieldUser(0x7f0e5eab3788, 0xc21014e140, 0x7f0e58a7ac10, 0xc2101d191a, 0x4, ...)
         /home/vagrant/influxdb/src/api/http/api.go:477 +0x3b
  api/http.(*HttpServer).tryAsClusterAdmin(0xc2100c1500, 0x7f0e5eab36d8, 0xc2101d2280, 0xc210203c30, 0x7f0e58a7ac10)
         /home/vagrant/influxdb/src/api/http/api.go:539 +0x34a
  api/http.(*HttpServer).tryAsDbUserAndClusterAdmin(0xc2100c1500, 0x7f0e5eab36d8, 0xc2101d2280, 0xc210203c30, 0x7f0e58a7ac10)
         /home/vagrant/influxdb/src/api/http/api.go:709 +0x268
  api/http.(*HttpServer).query(0xc2100c1500, 0x7f0e5eab36d8, 0xc2101d2280, 0xc210203c30)
         /home/vagrant/influxdb/src/api/http/api.go:320 +0x14d
  api/http.*HttpServer.(api/http.query)·fm(0x7f0e5eab36d8, 0xc2101d2280, 0xc210203c30)
        /home/vagrant/influxdb/src/api/http/api.go:103 +0x44
  api/http.func·032(0x7f0e5eab36d8, 0xc2101d2280, 0xc210203c30)
        /home/vagrant/influxdb/src/api/http/cors.go:14 +0x1d0
  net/http.HandlerFunc.ServeHTTP(0xc21018e3e0, 0x7f0e5eab36d8, 0xc2101d2280, 0xc210203c30)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1220 +0x40
  github.com/bmizerany/pat.(*PatternServeMux).ServeHTTP(0xc21024a320, 0x7f0e5eab36d8, 0xc2101d2280, 0xc210203c30)
  /home/vagrant/influxdb/src/github.com/bmizerany/pat/mux.go:109 +0x1be
  net/http.serverHandler.ServeHTTP(0xc2101b0320, 0x7f0e5eab36d8, 0xc2101d2280, 0xc210203c30)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1597 +0x16e
  net/http.(*conn).serve(0xc2101b1c80)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1167 +0x7b7
  created by net/http.(*Server).Serve
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1644 +0x28b

  goroutine 23 [IO wait]:
  net.runtime_pollWait(0x7f0e5eaad5f8, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc2101bf1b0, 0x72, 0x7f0e5eaa4f88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc2101bf1b0, 0xb, 0x7f0e5eaa4f88)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc2101bf150, 0xa1d1f8, 0x0, 0x7f0e5eaa4f88, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc2101e7740, 0x18, 0xc21018b810, 0x5d29f3)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc2101e7740, 0x50, 0x1065b00, 0x18, 0x0)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  net/http.(*Server).Serve(0xc2100ba190, 0x7f0e5eaac7a8, 0xc2101e7740, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1622 +0x91
  net/http.Serve(0x7f0e5eaac7a8, 0xc2101e7740, 0x7f0e5eab3218, 0xc2101ba210, 0x1b, ...)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1561 +0x70
  admin.(*HttpServer).ListenAndServe(0xc210103a40)
         /home/vagrant/influxdb/src/admin/http_server.go:35 +0x170
  created by server.(*Server).ListenAndServe
         /home/vagrant/influxdb/src/server/server.go:132 +0x46c

  goroutine 24 [chan receive]:
  server.(*Server).startReportingLoop(0xc210042900, 0xc2100ae040)
         /home/vagrant/influxdb/src/server/server.go:182 +0xbf
  created by server.(*Server).ListenAndServe
         /home/vagrant/influxdb/src/server/server.go:163 +0xaae

  goroutine 33 [chan receive]:
  coordinator.func·002()
        /home/vagrant/influxdb/src/coordinator/coordinator.go:313 +0x57
  created by coordinator.(*CoordinatorImpl).getShardsAndProcessor
         /home/vagrant/influxdb/src/coordinator/coordinator.go:326 +0x36b

 +---------------------------------------------+
  |  _____        __ _            _____  ____   |
  | |_   _|      / _| |          |  __ \|  _ \  |
  |   | |  _ __ | |_| |_   ___  _| |  | | |_) | |
  |   | | | '_ \|  _| | | | \ \/ / |  | |  _ <  |
  |  _| |_| | | | | | | |_| |>  <| |__| | |_) | |
  | |_____|_| |_|_| |_|\__,_/_/\_\_____/|____/  |
  +---------------------------------------------+

  panic: runtime error: index out of range

  goroutine 38 [running]:
  runtime.panic(0x8a29e0, 0x1042d57)
         /home/vagrant/bin/go/src/pkg/runtime/panic.c:266 +0xb6
  engine.GetValue(0xc2102870a0, 0xc210143d40, 0x4, 0x4, 0xc210143d80, ...)
         /home/vagrant/influxdb/src/engine/arithmetic.go:31 +0x377
  engine.PlusOperator(0xc210284530, 0x2, 0x2, 0xc210143d40, 0x4, ...)
         /home/vagrant/influxdb/src/engine/arithmetic.go:54 +0x116
  engine.GetValue(0xc210287000, 0xc210143d40, 0x4, 0x4, 0xc210143d80, ...)
         /home/vagrant/influxdb/src/engine/arithmetic.go:37 +0x538
  engine.func·014(0xc2102874b0, 0xc2102874b0, 0xc2102874b0)
         /home/vagrant/influxdb/src/engine/engine.go:666 +0x376
  engine.func·016(0xc210287460, 0xc21028a038, 0x2)
         /home/vagrant/influxdb/src/engine/merge.go:57 +0x8cb
  engine.(*mergeState).yieldNextPoints(0xc2102881b0, 0xc2102870f0, 0x0, 0x0)
         /home/vagrant/influxdb/src/engine/merge.go:193 +0x23d
  engine.func·018(0xc210148050, 0x7de840, 0xc210288120)
         /home/vagrant/influxdb/src/engine/merge.go:279 +0x9c
  engine.(*QueryEngine).yieldSeriesData(0xc2101ad460, 0xc210148050, 0xc210284520)
         /home/vagrant/influxdb/src/engine/engine.go:176 +0x30
  engine.(*QueryEngine).YieldSeries(0xc2101ad460, 0xc210148050, 0x7f2a59405f38)
         /home/vagrant/influxdb/src/engine/engine.go:172 +0x122
  coordinator.(*CoordinatorImpl).readFromResponseChannels(0xc21012eaa0, 0x7f2a595afff8, 0xc2101ad460, 0x7f2a595aff68, 0xc21024af90, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:367 +0x2f7
  created by coordinator.(*CoordinatorImpl).runQuerySpec
         /home/vagrant/influxdb/src/coordinator/coordinator.go:441 +0x3df

  goroutine 1 [IO wait]:
  net.runtime_pollWait(0x7f2a595a9740, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc2101cb370, 0x72, 0x7f2a595a1050, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc2101cb370, 0xb, 0x7f2a595a1050)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc2101cb310, 0xa1d1f8, 0x0, 0x7f2a595a1050, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc21022d210, 0x5d2c1b, 0x7f2a59411960, 0x5d2c1b)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc21022d210, 0x7f2a595af8f8, 0xc2101a1578, 0xc210194780, 0x0)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  net/http.(*Server).Serve(0xc210148640, 0x7f2a595a88f0, 0xc21022d210, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1622 +0x91
  api/http.(*HttpServer).serveListener(0xc2100c35a0, 0x7f2a595a88f0, 0xc21022d210, 0xc21022d218)
         /home/vagrant/influxdb/src/api/http/api.go:187 +0x98
  api/http.(*HttpServer).Serve(0xc2100c35a0, 0x7f2a595a88f0, 0xc21022d210)
         /home/vagrant/influxdb/src/api/http/api.go:157 +0xce4
  api/http.(*HttpServer).ListenAndServe(0xc2100c35a0)
         /home/vagrant/influxdb/src/api/http/api.go:79 +0x16c
  server.(*Server).ListenAndServe(0xc210042990, 0xc210042990, 0x0)
         /home/vagrant/influxdb/src/server/server.go:170 +0xbb0
  main.main()
         /home/vagrant/influxdb/src/daemon/influxd.go:159 +0xd2a

  goroutine 3 [syscall]:
  os/signal.loop()
         /home/vagrant/bin/go/src/pkg/os/signal/signal_unix.go:21 +0x1e
  created by os/signal.init·1
         /home/vagrant/bin/go/src/pkg/os/signal/signal_unix.go:27 +0x31

  goroutine 4 [chan receive]:
  code.google.com/p/log4go.ConsoleLogWriter.run(0xc210068000, 0x7f2a595a11d8, 0xc210000008)
  /home/vagrant/influxdb/src/code.google.com/p/log4go/termlog.go:27 +0x60
  created by code.google.com/p/log4go.NewConsoleLogWriter
  /home/vagrant/influxdb/src/code.google.com/p/log4go/termlog.go:19 +0x67

  goroutine 5 [syscall]:
  runtime.goexit()
         /home/vagrant/bin/go/src/pkg/runtime/proc.c:1394

  goroutine 6 [select]:
  code.google.com/p/log4go.func·002()
  /home/vagrant/influxdb/src/code.google.com/p/log4go/filelog.go:84 +0x84c
  created by code.google.com/p/log4go.NewFileLogWriter
  /home/vagrant/influxdb/src/code.google.com/p/log4go/filelog.go:116 +0x2d1

  goroutine 7 [finalizer wait]:
  runtime.park(0x451a20, 0x1059c78, 0x10446a8)
         /home/vagrant/bin/go/src/pkg/runtime/proc.c:1342 +0x66
  runfinq()
         /home/vagrant/bin/go/src/pkg/runtime/mgc0.c:2279 +0x84
  runtime.goexit()
         /home/vagrant/bin/go/src/pkg/runtime/proc.c:1394

  goroutine 8 [chan receive]:
  wal.(*WAL).processEntries(0xc210070e00)
         /home/vagrant/influxdb/src/wal/wal.go:252 +0x3f
  created by wal.NewWAL
         /home/vagrant/influxdb/src/wal/wal.go:103 +0x9f3

  goroutine 9 [sleep]:
  time.Sleep(0x8bb2c97000)
         /tmp/makerelease886106415/go/src/pkg/runtime/time.goc:31 +0x31
  cluster.func·001()
  /home/vagrant/influxdb/src/cluster/cluster_configuration.go:133 +0x35
  created by cluster.(*ClusterConfiguration).CreateFutureShardsAutomaticallyBeforeTimeComes
  /home/vagrant/influxdb/src/cluster/cluster_configuration.go:138 +0x63

  goroutine 10 [chan receive]:
  main.waitForSignals(0x7f2a595a8870, 0xc210042990)
         /home/vagrant/influxdb/src/daemon/null_profiler.go:23 +0x126
  created by main.startProfiler
         /home/vagrant/influxdb/src/daemon/null_profiler.go:15 +0x38

  coordinator.(*CoordinatorImpl).runQuerySpec(0xc210152ee0, 0xc2100b8460, 0x7f1d95e11c98, 0xc210283a60, 0x0, ...)
  goroutine 12 [IO wait]:
  net.runtime_pollWait(0x7f2a595a9890, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc210050680, 0x72, 0x7f2a595a1050, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc210050680, 0xb, 0x7f2a595a1050)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc210050620, 0xa1d1f8, 0x0, 0x7f2a595a1050, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc21013ff20, 0x18, 0xc2100de810, 0x5d29f3)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc21013ff20, 0x0, 0x0, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  net/http.(*Server).Serve(0xc21017c5f0, 0x7f2a595a88f0, 0xc21013ff20, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1622 +0x91
  coordinator.func·009()
        /home/vagrant/influxdb/src/coordinator/raft_server.go:527 +0x3a
  created by coordinator.(*RaftServer).Serve
         /home/vagrant/influxdb/src/coordinator/raft_server.go:531 +0x4d9

  goroutine 14 [select]:
  github.com/goraft/raft.(*server).leaderLoop(0xc2100697e0)
  /home/vagrant/influxdb/src/github.com/goraft/raft/server.go:765 +0x5fe
  github.com/goraft/raft.(*server).loop(0xc2100697e0)
  /home/vagrant/influxdb/src/github.com/goraft/raft/server.go:568 +0x33f
  created by github.com/goraft/raft.(*server).Start
  /home/vagrant/influxdb/src/github.com/goraft/raft/server.go:472 +0x7af

  goroutine 15 [select]:
  coordinator.(*RaftServer).CompactLog(0xc2100d7bb0)
         /home/vagrant/influxdb/src/coordinator/raft_server.go:317 +0x2ef
  created by coordinator.(*RaftServer).startRaft
         /home/vagrant/influxdb/src/coordinator/raft_server.go:371 +0x375

  goroutine 17 [select]:
  coordinator.(*RaftServer).raftLeaderLoop(0xc2100d7bb0, 0xc2101ac740)
         /home/vagrant/influxdb/src/coordinator/raft_server.go:427 +0x29c
  created by coordinator.(*RaftServer).raftEventHandler
         /home/vagrant/influxdb/src/coordinator/raft_server.go:416 +0x1d0

  goroutine 19 [IO wait]:
  net.runtime_pollWait(0x7f2a595a97e8, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc210190990, 0x72, 0x7f2a595a1050, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc210190990, 0xb, 0x7f2a595a1050)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc210190930, 0xa1d1f8, 0x0, 0x7f2a595a1050, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc210162048, 0xc2101a9a40, 0x0, 0x7f2a595a88c0)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc210162048, 0xc2101a9a40, 0x7f2a51c1ff38, 0x1, 0x1)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  coordinator.(*ProtobufServer).ListenAndServe(0xc210188880)
         /home/vagrant/influxdb/src/coordinator/protobuf_server.go:64 +0x1c7
  created by server.(*Server).ListenAndServe
         /home/vagrant/influxdb/src/server/server.go:118 +0x218

  goroutine 20 [select]:
  cluster.(*WriteBuffer).handleWrites(0xc210190310)
         /home/vagrant/influxdb/src/cluster/write_buffer.go:75 +0xca
  created by cluster.NewWriteBuffer
         /home/vagrant/influxdb/src/cluster/write_buffer.go:43 +0x24f

  goroutine 36 [runnable]:
  coordinator.(*CoordinatorImpl).queryShards(0xc21012eaa0, 0xc21000f1e0, 0xc2100c00d0, 0x2, 0x2, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:394 +0x8b
  coordinator.(*CoordinatorImpl).runQuerySpec(0xc21012eaa0, 0xc21000f1e0, 0x7f2a595aff68, 0xc21024af90, 0x0, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:443 +0x440
  coordinator.(*CoordinatorImpl).runQuery(0xc21012eaa0, 0xc21000f1e0, 0x7f2a595aff68, 0xc21024af90, 0x0, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:157 +0x45
  coordinator.(*CoordinatorImpl).RunQuery(0xc21012eaa0, 0x7f2a595afcd8, 0xc21007a9c0, 0xc2101360c6, 0xa, ...)
         /home/vagrant/influxdb/src/coordinator/coordinator.go:135 +0x8d8
  api/http.func·004(0x7f2a595afcd8, 0xc21007a9c0, 0xc2101360d3, 0x4, 0xc2101360da)
        /home/vagrant/influxdb/src/api/http/api.go:310 +0x2e2
  api/http.yieldUser(0x7f2a595afcd8, 0xc21007a9c0, 0x7f2a51c1bc10, 0xc2101360da, 0x4, ...)
         /home/vagrant/influxdb/src/api/http/api.go:477 +0x3b
  api/http.(*HttpServer).tryAsClusterAdmin(0xc2100c35a0, 0x7f2a595afc78, 0xc21000f0a0, 0xc210285270, 0x7f2a51c1bc10)
         /home/vagrant/influxdb/src/api/http/api.go:539 +0x34a
  api/http.(*HttpServer).tryAsDbUserAndClusterAdmin(0xc2100c35a0, 0x7f2a595afc78, 0xc21000f0a0, 0xc210285270, 0x7f2a51c1bc10)
         /home/vagrant/influxdb/src/api/http/api.go:709 +0x268
  api/http.(*HttpServer).query(0xc2100c35a0, 0x7f2a595afc78, 0xc21000f0a0, 0xc210285270)
         /home/vagrant/influxdb/src/api/http/api.go:320 +0x14d
  api/http.*HttpServer.(api/http.query)·fm(0x7f2a595afc78, 0xc21000f0a0, 0xc210285270)
        /home/vagrant/influxdb/src/api/http/api.go:103 +0x44
  api/http.func·032(0x7f2a595afc78, 0xc21000f0a0, 0xc210285270)
        /home/vagrant/influxdb/src/api/http/cors.go:14 +0x1d0
  net/http.HandlerFunc.ServeHTTP(0xc21027ac60, 0x7f2a595afc78, 0xc21000f0a0, 0xc210285270)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1220 +0x40
  github.com/bmizerany/pat.(*PatternServeMux).ServeHTTP(0xc21022d218, 0x7f2a595afc78, 0xc21000f0a0, 0xc210285270)
  /home/vagrant/influxdb/src/github.com/bmizerany/pat/mux.go:109 +0x1be
  net/http.serverHandler.ServeHTTP(0xc210148640, 0x7f2a595afc78, 0xc21000f0a0, 0xc210285270)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1597 +0x16e
  net/http.(*conn).serve(0xc210194780)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1167 +0x7b7
  created by net/http.(*Server).Serve
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1644 +0x28b

  goroutine 23 [IO wait]:
  net.runtime_pollWait(0x7f2a595a9698, 0x72, 0x0)
         /tmp/makerelease886106415/go/src/pkg/runtime/netpoll.goc:116 +0x6a
  net.(*pollDesc).Wait(0xc2101deed0, 0x72, 0x7f2a595a1050, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
  net.(*pollDesc).WaitRead(0xc2101deed0, 0xb, 0x7f2a595a1050)
         /home/vagrant/bin/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
  net.(*netFD).accept(0xc2101dee70, 0xa1d1f8, 0x0, 0x7f2a595a1050, 0xb)
         /home/vagrant/bin/go/src/pkg/net/fd_unix.go:382 +0x2c2
  net.(*TCPListener).AcceptTCP(0xc2101e1510, 0x18, 0xc210134810, 0x5d29f3)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:233 +0x47
  net.(*TCPListener).Accept(0xc2101e1510, 0x50, 0x1065b00, 0x18, 0x0)
         /home/vagrant/bin/go/src/pkg/net/tcpsock_posix.go:243 +0x27
  net/http.(*Server).Serve(0xc21001ec30, 0x7f2a595a88f0, 0xc2101e1510, 0x0, 0x0)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1622 +0x91
  net/http.Serve(0x7f2a595a88f0, 0xc2101e1510, 0x7f2a595af878, 0xc2101dab30, 0x1b, ...)
         /home/vagrant/bin/go/src/pkg/net/http/server.go:1561 +0x70
  admin.(*HttpServer).ListenAndServe(0xc210188900)
         /home/vagrant/influxdb/src/admin/http_server.go:35 +0x170
  created by server.(*Server).ListenAndServe
         /home/vagrant/influxdb/src/server/server.go:132 +0x46c

  goroutine 24 [chan receive]:
  server.(*Server).startReportingLoop(0xc210042990, 0xc2102799d0)
         /home/vagrant/influxdb/src/server/server.go:182 +0xbf
  created by server.(*Server).ListenAndServe
         /home/vagrant/influxdb/src/server/server.go:163 +0xaae

  goroutine 37 [chan receive]:
  coordinator.func·002()
        /home/vagrant/influxdb/src/coordinator/coordinator.go:313 +0x57
  created by coordinator.(*CoordinatorImpl).getShardsAndProcessor
         /home/vagrant/influxdb/src/coordinator/coordinator.go:326 +0x36b

  goroutine 39 [runnable]:
  datastore.(*LevelDbShard).executeQueryForSeries(0xc21007ab40, 0xc21000f1e0, 0xc210284520, 0xa, 0xc210253720, ...)
         /home/vagrant/influxdb/src/datastore/leveldb_shard.go:294 +0x1460
  datastore.(*LevelDbShard).Query(0xc21007ab40, 0xc21000f1e0, 0x7f2a595b0040, 0xc210195480, 0x0, ...)
         /home/vagrant/influxdb/src/datastore/leveldb_shard.go:144 +0x4ba
  cluster.(*ShardData).Query(0xc2100b00e0, 0xc21000f1e0, 0xc2101c7c00)
         /home/vagrant/influxdb/src/cluster/shard.go:285 +0x648
  created by coordinator.(*CoordinatorImpl).queryShards
         /home/vagrant/influxdb/src/coordinator/coordinator.go:406 +0x279
  [07/25/14 11:09:07] [INFO] Loading configuration file /opt/influxdb/shared/config.toml
@xthexder
Copy link

I just ran into (I'm assuming the related problem) myself.
I can consistently crash my database running the following query:

SELECT mean(bytes_sent.value + bytes_rcvd.value), bytes_sent.hostname FROM system.net.bytes_rcvd as bytes_rcvd inner join system.net.bytes_sent as bytes_sent group by bytes_sent.hostname where time > now() - 1h LIMIT 10

Here's my stack trace running on 0.8.0rc3

panic: runtime error: index out of range

goroutine 56 [running]:
runtime.panic(0xb12000, 0x1440d57)
        /home/vagrant/.gvm/gos/go1.2.1/src/pkg/runtime/panic.c:266 +0xb6
github.com/influxdb/influxdb/engine.GetValue(0xc210a80460, 0xc210a610c0, 0x4, 0x4, 0xc210a61100, ...)
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/engine/arithmetic.go:32 +0x377
github.com/influxdb/influxdb/engine.(*QueryEngine).aggregateValuesForSeries(0xc210a7e8c0, 0xc210a5fcb0, 0xb43620, 0x3)
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/engine/engine.go:460 +0x37a
github.com/influxdb/influxdb/engine.func·009(0xc210a5fcb0, 0xc210a5fcb0, 0xc210a5fcb0)
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/engine/engine.go:366 +0x54
github.com/influxdb/influxdb/engine.func·016(0xc210a5fc40, 0xc210066a58, 0x2)
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/engine/merge.go:57 +0x8cb
github.com/influxdb/influxdb/engine.(*mergeState).yieldNextPoints(0xc210a47a20, 0xc210a80500, 0x0, 0x0)
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/engine/merge.go:193 +0x23d
github.com/influxdb/influxdb/engine.func·018(0xc21069f0e0, 0xa3e840, 0xc210a479c0)
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/engine/merge.go:279 +0x9c
github.com/influxdb/influxdb/engine.(*QueryEngine).yieldSeriesData(0xc210a7e8c0, 0xc21069f0e0, 0xc210a568b0)
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/engine/engine.go:176 +0x30
github.com/influxdb/influxdb/engine.(*QueryEngine).YieldSeries(0xc210a7e8c0, 0xc21069f0e0, 0x7f5d7a8c0f28)
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/engine/engine.go:172 +0x122
github.com/influxdb/influxdb/coordinator.(*CoordinatorImpl).readFromResponseChannels(0xc2100be210, 0x7f5d802a6590, 0xc210a7e8c0, 0x7f5d802a6388, 0xc210a77bf0, ...)
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/coordinator/coordinator.go:362 +0x358
created by github.com/influxdb/influxdb/coordinator.(*CoordinatorImpl).runQuerySpec
        /home/vagrant/gocodez/src/github.com/influxdb/influxdb/coordinator/coordinator.go:440 +0x42e

@klyngbaek
Copy link
Author

Yes, it looks like it. My panic was caused by a very similar query with an 'inner join' as well.

@cboggs
Copy link

cboggs commented Jul 28, 2014

+1. I've seen similar behavior when including a text field alongside some "math'd" fields in an inner join. Select multiple fields by themselves is fine, but trying to perform an operation on two numerical fields while also selecting (outside of the math operation) a text field causes a panic. Will snag output shortly to verify that it looks similar to the output above, but seems like the same problem so far.

I logged issue #740 on this as well. Can close as duplicate if that's preferred.

@jvshahid jvshahid added this to the 0.8.0 milestone Jul 28, 2014
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
0.8.0
Development

No branches or pull requests
4 participants
@jvshahid @klyngbaek @xthexder @cboggs