trace.go 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059
  1. // Copyright 2015 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. /*
  5. Package trace implements tracing of requests and long-lived objects.
  6. It exports HTTP interfaces on /debug/requests and /debug/events.
  7. A trace.Trace provides tracing for short-lived objects, usually requests.
  8. A request handler might be implemented like this:
  9. func fooHandler(w http.ResponseWriter, req *http.Request) {
  10. tr := trace.New("mypkg.Foo", req.URL.Path)
  11. defer tr.Finish()
  12. ...
  13. tr.LazyPrintf("some event %q happened", str)
  14. ...
  15. if err := somethingImportant(); err != nil {
  16. tr.LazyPrintf("somethingImportant failed: %v", err)
  17. tr.SetError()
  18. }
  19. }
  20. The /debug/requests HTTP endpoint organizes the traces by family,
  21. errors, and duration. It also provides histogram of request duration
  22. for each family.
  23. A trace.EventLog provides tracing for long-lived objects, such as RPC
  24. connections.
  25. // A Fetcher fetches URL paths for a single domain.
  26. type Fetcher struct {
  27. domain string
  28. events trace.EventLog
  29. }
  30. func NewFetcher(domain string) *Fetcher {
  31. return &Fetcher{
  32. domain,
  33. trace.NewEventLog("mypkg.Fetcher", domain),
  34. }
  35. }
  36. func (f *Fetcher) Fetch(path string) (string, error) {
  37. resp, err := http.Get("http://" + f.domain + "/" + path)
  38. if err != nil {
  39. f.events.Errorf("Get(%q) = %v", path, err)
  40. return "", err
  41. }
  42. f.events.Printf("Get(%q) = %s", path, resp.Status)
  43. ...
  44. }
  45. func (f *Fetcher) Close() error {
  46. f.events.Finish()
  47. return nil
  48. }
  49. The /debug/events HTTP endpoint organizes the event logs by family and
  50. by time since the last error. The expanded view displays recent log
  51. entries and the log's call stack.
  52. */
  53. package trace
  54. import (
  55. "bytes"
  56. "fmt"
  57. "html/template"
  58. "io"
  59. "log"
  60. "net"
  61. "net/http"
  62. "runtime"
  63. "sort"
  64. "strconv"
  65. "sync"
  66. "sync/atomic"
  67. "time"
  68. "golang.org/x/net/context"
  69. "golang.org/x/net/internal/timeseries"
  70. )
  71. // DebugUseAfterFinish controls whether to debug uses of Trace values after finishing.
  72. // FOR DEBUGGING ONLY. This will slow down the program.
  73. var DebugUseAfterFinish = false
  74. // AuthRequest determines whether a specific request is permitted to load the
  75. // /debug/requests or /debug/events pages.
  76. //
  77. // It returns two bools; the first indicates whether the page may be viewed at all,
  78. // and the second indicates whether sensitive events will be shown.
  79. //
  80. // AuthRequest may be replaced by a program to customise its authorisation requirements.
  81. //
  82. // The default AuthRequest function returns (true, true) iff the request comes from localhost/127.0.0.1/[::1].
  83. var AuthRequest = func(req *http.Request) (any, sensitive bool) {
  84. host, _, err := net.SplitHostPort(req.RemoteAddr)
  85. switch {
  86. case err != nil: // Badly formed address; fail closed.
  87. return false, false
  88. case host == "localhost" || host == "127.0.0.1" || host == "::1":
  89. return true, true
  90. default:
  91. return false, false
  92. }
  93. }
  94. func init() {
  95. http.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
  96. any, sensitive := AuthRequest(req)
  97. if !any {
  98. http.Error(w, "not allowed", http.StatusUnauthorized)
  99. return
  100. }
  101. w.Header().Set("Content-Type", "text/html; charset=utf-8")
  102. Render(w, req, sensitive)
  103. })
  104. http.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
  105. any, sensitive := AuthRequest(req)
  106. if !any {
  107. http.Error(w, "not allowed", http.StatusUnauthorized)
  108. return
  109. }
  110. w.Header().Set("Content-Type", "text/html; charset=utf-8")
  111. RenderEvents(w, req, sensitive)
  112. })
  113. }
  114. // Render renders the HTML page typically served at /debug/requests.
  115. // It does not do any auth checking; see AuthRequest for the default auth check
  116. // used by the handler registered on http.DefaultServeMux.
  117. // req may be nil.
  118. func Render(w io.Writer, req *http.Request, sensitive bool) {
  119. data := &struct {
  120. Families []string
  121. ActiveTraceCount map[string]int
  122. CompletedTraces map[string]*family
  123. // Set when a bucket has been selected.
  124. Traces traceList
  125. Family string
  126. Bucket int
  127. Expanded bool
  128. Traced bool
  129. Active bool
  130. ShowSensitive bool // whether to show sensitive events
  131. Histogram template.HTML
  132. HistogramWindow string // e.g. "last minute", "last hour", "all time"
  133. // If non-zero, the set of traces is a partial set,
  134. // and this is the total number.
  135. Total int
  136. }{
  137. CompletedTraces: completedTraces,
  138. }
  139. data.ShowSensitive = sensitive
  140. if req != nil {
  141. // Allow show_sensitive=0 to force hiding of sensitive data for testing.
  142. // This only goes one way; you can't use show_sensitive=1 to see things.
  143. if req.FormValue("show_sensitive") == "0" {
  144. data.ShowSensitive = false
  145. }
  146. if exp, err := strconv.ParseBool(req.FormValue("exp")); err == nil {
  147. data.Expanded = exp
  148. }
  149. if exp, err := strconv.ParseBool(req.FormValue("rtraced")); err == nil {
  150. data.Traced = exp
  151. }
  152. }
  153. completedMu.RLock()
  154. data.Families = make([]string, 0, len(completedTraces))
  155. for fam := range completedTraces {
  156. data.Families = append(data.Families, fam)
  157. }
  158. completedMu.RUnlock()
  159. sort.Strings(data.Families)
  160. // We are careful here to minimize the time spent locking activeMu,
  161. // since that lock is required every time an RPC starts and finishes.
  162. data.ActiveTraceCount = make(map[string]int, len(data.Families))
  163. activeMu.RLock()
  164. for fam, s := range activeTraces {
  165. data.ActiveTraceCount[fam] = s.Len()
  166. }
  167. activeMu.RUnlock()
  168. var ok bool
  169. data.Family, data.Bucket, ok = parseArgs(req)
  170. switch {
  171. case !ok:
  172. // No-op
  173. case data.Bucket == -1:
  174. data.Active = true
  175. n := data.ActiveTraceCount[data.Family]
  176. data.Traces = getActiveTraces(data.Family)
  177. if len(data.Traces) < n {
  178. data.Total = n
  179. }
  180. case data.Bucket < bucketsPerFamily:
  181. if b := lookupBucket(data.Family, data.Bucket); b != nil {
  182. data.Traces = b.Copy(data.Traced)
  183. }
  184. default:
  185. if f := getFamily(data.Family, false); f != nil {
  186. var obs timeseries.Observable
  187. f.LatencyMu.RLock()
  188. switch o := data.Bucket - bucketsPerFamily; o {
  189. case 0:
  190. obs = f.Latency.Minute()
  191. data.HistogramWindow = "last minute"
  192. case 1:
  193. obs = f.Latency.Hour()
  194. data.HistogramWindow = "last hour"
  195. case 2:
  196. obs = f.Latency.Total()
  197. data.HistogramWindow = "all time"
  198. }
  199. f.LatencyMu.RUnlock()
  200. if obs != nil {
  201. data.Histogram = obs.(*histogram).html()
  202. }
  203. }
  204. }
  205. if data.Traces != nil {
  206. defer data.Traces.Free()
  207. sort.Sort(data.Traces)
  208. }
  209. completedMu.RLock()
  210. defer completedMu.RUnlock()
  211. if err := pageTmpl.ExecuteTemplate(w, "Page", data); err != nil {
  212. log.Printf("net/trace: Failed executing template: %v", err)
  213. }
  214. }
  215. func parseArgs(req *http.Request) (fam string, b int, ok bool) {
  216. if req == nil {
  217. return "", 0, false
  218. }
  219. fam, bStr := req.FormValue("fam"), req.FormValue("b")
  220. if fam == "" || bStr == "" {
  221. return "", 0, false
  222. }
  223. b, err := strconv.Atoi(bStr)
  224. if err != nil || b < -1 {
  225. return "", 0, false
  226. }
  227. return fam, b, true
  228. }
  229. func lookupBucket(fam string, b int) *traceBucket {
  230. f := getFamily(fam, false)
  231. if f == nil || b < 0 || b >= len(f.Buckets) {
  232. return nil
  233. }
  234. return f.Buckets[b]
  235. }
  236. type contextKeyT string
  237. var contextKey = contextKeyT("golang.org/x/net/trace.Trace")
  238. // NewContext returns a copy of the parent context
  239. // and associates it with a Trace.
  240. func NewContext(ctx context.Context, tr Trace) context.Context {
  241. return context.WithValue(ctx, contextKey, tr)
  242. }
  243. // FromContext returns the Trace bound to the context, if any.
  244. func FromContext(ctx context.Context) (tr Trace, ok bool) {
  245. tr, ok = ctx.Value(contextKey).(Trace)
  246. return
  247. }
  248. // Trace represents an active request.
  249. type Trace interface {
  250. // LazyLog adds x to the event log. It will be evaluated each time the
  251. // /debug/requests page is rendered. Any memory referenced by x will be
  252. // pinned until the trace is finished and later discarded.
  253. LazyLog(x fmt.Stringer, sensitive bool)
  254. // LazyPrintf evaluates its arguments with fmt.Sprintf each time the
  255. // /debug/requests page is rendered. Any memory referenced by a will be
  256. // pinned until the trace is finished and later discarded.
  257. LazyPrintf(format string, a ...interface{})
  258. // SetError declares that this trace resulted in an error.
  259. SetError()
  260. // SetRecycler sets a recycler for the trace.
  261. // f will be called for each event passed to LazyLog at a time when
  262. // it is no longer required, whether while the trace is still active
  263. // and the event is discarded, or when a completed trace is discarded.
  264. SetRecycler(f func(interface{}))
  265. // SetTraceInfo sets the trace info for the trace.
  266. // This is currently unused.
  267. SetTraceInfo(traceID, spanID uint64)
  268. // SetMaxEvents sets the maximum number of events that will be stored
  269. // in the trace. This has no effect if any events have already been
  270. // added to the trace.
  271. SetMaxEvents(m int)
  272. // Finish declares that this trace is complete.
  273. // The trace should not be used after calling this method.
  274. Finish()
  275. }
  276. type lazySprintf struct {
  277. format string
  278. a []interface{}
  279. }
  280. func (l *lazySprintf) String() string {
  281. return fmt.Sprintf(l.format, l.a...)
  282. }
  283. // New returns a new Trace with the specified family and title.
  284. func New(family, title string) Trace {
  285. tr := newTrace()
  286. tr.ref()
  287. tr.Family, tr.Title = family, title
  288. tr.Start = time.Now()
  289. tr.events = make([]event, 0, maxEventsPerTrace)
  290. activeMu.RLock()
  291. s := activeTraces[tr.Family]
  292. activeMu.RUnlock()
  293. if s == nil {
  294. activeMu.Lock()
  295. s = activeTraces[tr.Family] // check again
  296. if s == nil {
  297. s = new(traceSet)
  298. activeTraces[tr.Family] = s
  299. }
  300. activeMu.Unlock()
  301. }
  302. s.Add(tr)
  303. // Trigger allocation of the completed trace structure for this family.
  304. // This will cause the family to be present in the request page during
  305. // the first trace of this family. We don't care about the return value,
  306. // nor is there any need for this to run inline, so we execute it in its
  307. // own goroutine, but only if the family isn't allocated yet.
  308. completedMu.RLock()
  309. if _, ok := completedTraces[tr.Family]; !ok {
  310. go allocFamily(tr.Family)
  311. }
  312. completedMu.RUnlock()
  313. return tr
  314. }
  315. func (tr *trace) Finish() {
  316. tr.Elapsed = time.Now().Sub(tr.Start)
  317. if DebugUseAfterFinish {
  318. buf := make([]byte, 4<<10) // 4 KB should be enough
  319. n := runtime.Stack(buf, false)
  320. tr.finishStack = buf[:n]
  321. }
  322. activeMu.RLock()
  323. m := activeTraces[tr.Family]
  324. activeMu.RUnlock()
  325. m.Remove(tr)
  326. f := getFamily(tr.Family, true)
  327. for _, b := range f.Buckets {
  328. if b.Cond.match(tr) {
  329. b.Add(tr)
  330. }
  331. }
  332. // Add a sample of elapsed time as microseconds to the family's timeseries
  333. h := new(histogram)
  334. h.addMeasurement(tr.Elapsed.Nanoseconds() / 1e3)
  335. f.LatencyMu.Lock()
  336. f.Latency.Add(h)
  337. f.LatencyMu.Unlock()
  338. tr.unref() // matches ref in New
  339. }
  340. const (
  341. bucketsPerFamily = 9
  342. tracesPerBucket = 10
  343. maxActiveTraces = 20 // Maximum number of active traces to show.
  344. maxEventsPerTrace = 10
  345. numHistogramBuckets = 38
  346. )
  347. var (
  348. // The active traces.
  349. activeMu sync.RWMutex
  350. activeTraces = make(map[string]*traceSet) // family -> traces
  351. // Families of completed traces.
  352. completedMu sync.RWMutex
  353. completedTraces = make(map[string]*family) // family -> traces
  354. )
  355. type traceSet struct {
  356. mu sync.RWMutex
  357. m map[*trace]bool
  358. // We could avoid the entire map scan in FirstN by having a slice of all the traces
  359. // ordered by start time, and an index into that from the trace struct, with a periodic
  360. // repack of the slice after enough traces finish; we could also use a skip list or similar.
  361. // However, that would shift some of the expense from /debug/requests time to RPC time,
  362. // which is probably the wrong trade-off.
  363. }
  364. func (ts *traceSet) Len() int {
  365. ts.mu.RLock()
  366. defer ts.mu.RUnlock()
  367. return len(ts.m)
  368. }
  369. func (ts *traceSet) Add(tr *trace) {
  370. ts.mu.Lock()
  371. if ts.m == nil {
  372. ts.m = make(map[*trace]bool)
  373. }
  374. ts.m[tr] = true
  375. ts.mu.Unlock()
  376. }
  377. func (ts *traceSet) Remove(tr *trace) {
  378. ts.mu.Lock()
  379. delete(ts.m, tr)
  380. ts.mu.Unlock()
  381. }
  382. // FirstN returns the first n traces ordered by time.
  383. func (ts *traceSet) FirstN(n int) traceList {
  384. ts.mu.RLock()
  385. defer ts.mu.RUnlock()
  386. if n > len(ts.m) {
  387. n = len(ts.m)
  388. }
  389. trl := make(traceList, 0, n)
  390. // Fast path for when no selectivity is needed.
  391. if n == len(ts.m) {
  392. for tr := range ts.m {
  393. tr.ref()
  394. trl = append(trl, tr)
  395. }
  396. sort.Sort(trl)
  397. return trl
  398. }
  399. // Pick the oldest n traces.
  400. // This is inefficient. See the comment in the traceSet struct.
  401. for tr := range ts.m {
  402. // Put the first n traces into trl in the order they occur.
  403. // When we have n, sort trl, and thereafter maintain its order.
  404. if len(trl) < n {
  405. tr.ref()
  406. trl = append(trl, tr)
  407. if len(trl) == n {
  408. // This is guaranteed to happen exactly once during this loop.
  409. sort.Sort(trl)
  410. }
  411. continue
  412. }
  413. if tr.Start.After(trl[n-1].Start) {
  414. continue
  415. }
  416. // Find where to insert this one.
  417. tr.ref()
  418. i := sort.Search(n, func(i int) bool { return trl[i].Start.After(tr.Start) })
  419. trl[n-1].unref()
  420. copy(trl[i+1:], trl[i:])
  421. trl[i] = tr
  422. }
  423. return trl
  424. }
  425. func getActiveTraces(fam string) traceList {
  426. activeMu.RLock()
  427. s := activeTraces[fam]
  428. activeMu.RUnlock()
  429. if s == nil {
  430. return nil
  431. }
  432. return s.FirstN(maxActiveTraces)
  433. }
  434. func getFamily(fam string, allocNew bool) *family {
  435. completedMu.RLock()
  436. f := completedTraces[fam]
  437. completedMu.RUnlock()
  438. if f == nil && allocNew {
  439. f = allocFamily(fam)
  440. }
  441. return f
  442. }
  443. func allocFamily(fam string) *family {
  444. completedMu.Lock()
  445. defer completedMu.Unlock()
  446. f := completedTraces[fam]
  447. if f == nil {
  448. f = newFamily()
  449. completedTraces[fam] = f
  450. }
  451. return f
  452. }
  453. // family represents a set of trace buckets and associated latency information.
  454. type family struct {
  455. // traces may occur in multiple buckets.
  456. Buckets [bucketsPerFamily]*traceBucket
  457. // latency time series
  458. LatencyMu sync.RWMutex
  459. Latency *timeseries.MinuteHourSeries
  460. }
  461. func newFamily() *family {
  462. return &family{
  463. Buckets: [bucketsPerFamily]*traceBucket{
  464. {Cond: minCond(0)},
  465. {Cond: minCond(50 * time.Millisecond)},
  466. {Cond: minCond(100 * time.Millisecond)},
  467. {Cond: minCond(200 * time.Millisecond)},
  468. {Cond: minCond(500 * time.Millisecond)},
  469. {Cond: minCond(1 * time.Second)},
  470. {Cond: minCond(10 * time.Second)},
  471. {Cond: minCond(100 * time.Second)},
  472. {Cond: errorCond{}},
  473. },
  474. Latency: timeseries.NewMinuteHourSeries(func() timeseries.Observable { return new(histogram) }),
  475. }
  476. }
  477. // traceBucket represents a size-capped bucket of historic traces,
  478. // along with a condition for a trace to belong to the bucket.
  479. type traceBucket struct {
  480. Cond cond
  481. // Ring buffer implementation of a fixed-size FIFO queue.
  482. mu sync.RWMutex
  483. buf [tracesPerBucket]*trace
  484. start int // < tracesPerBucket
  485. length int // <= tracesPerBucket
  486. }
  487. func (b *traceBucket) Add(tr *trace) {
  488. b.mu.Lock()
  489. defer b.mu.Unlock()
  490. i := b.start + b.length
  491. if i >= tracesPerBucket {
  492. i -= tracesPerBucket
  493. }
  494. if b.length == tracesPerBucket {
  495. // "Remove" an element from the bucket.
  496. b.buf[i].unref()
  497. b.start++
  498. if b.start == tracesPerBucket {
  499. b.start = 0
  500. }
  501. }
  502. b.buf[i] = tr
  503. if b.length < tracesPerBucket {
  504. b.length++
  505. }
  506. tr.ref()
  507. }
  508. // Copy returns a copy of the traces in the bucket.
  509. // If tracedOnly is true, only the traces with trace information will be returned.
  510. // The logs will be ref'd before returning; the caller should call
  511. // the Free method when it is done with them.
  512. // TODO(dsymonds): keep track of traced requests in separate buckets.
  513. func (b *traceBucket) Copy(tracedOnly bool) traceList {
  514. b.mu.RLock()
  515. defer b.mu.RUnlock()
  516. trl := make(traceList, 0, b.length)
  517. for i, x := 0, b.start; i < b.length; i++ {
  518. tr := b.buf[x]
  519. if !tracedOnly || tr.spanID != 0 {
  520. tr.ref()
  521. trl = append(trl, tr)
  522. }
  523. x++
  524. if x == b.length {
  525. x = 0
  526. }
  527. }
  528. return trl
  529. }
  530. func (b *traceBucket) Empty() bool {
  531. b.mu.RLock()
  532. defer b.mu.RUnlock()
  533. return b.length == 0
  534. }
  535. // cond represents a condition on a trace.
  536. type cond interface {
  537. match(t *trace) bool
  538. String() string
  539. }
  540. type minCond time.Duration
  541. func (m minCond) match(t *trace) bool { return t.Elapsed >= time.Duration(m) }
  542. func (m minCond) String() string { return fmt.Sprintf("≥%gs", time.Duration(m).Seconds()) }
  543. type errorCond struct{}
  544. func (e errorCond) match(t *trace) bool { return t.IsError }
  545. func (e errorCond) String() string { return "errors" }
  546. type traceList []*trace
  547. // Free calls unref on each element of the list.
  548. func (trl traceList) Free() {
  549. for _, t := range trl {
  550. t.unref()
  551. }
  552. }
  553. // traceList may be sorted in reverse chronological order.
  554. func (trl traceList) Len() int { return len(trl) }
  555. func (trl traceList) Less(i, j int) bool { return trl[i].Start.After(trl[j].Start) }
  556. func (trl traceList) Swap(i, j int) { trl[i], trl[j] = trl[j], trl[i] }
  557. // An event is a timestamped log entry in a trace.
  558. type event struct {
  559. When time.Time
  560. Elapsed time.Duration // since previous event in trace
  561. NewDay bool // whether this event is on a different day to the previous event
  562. Recyclable bool // whether this event was passed via LazyLog
  563. What interface{} // string or fmt.Stringer
  564. Sensitive bool // whether this event contains sensitive information
  565. }
  566. // WhenString returns a string representation of the elapsed time of the event.
  567. // It will include the date if midnight was crossed.
  568. func (e event) WhenString() string {
  569. if e.NewDay {
  570. return e.When.Format("2006/01/02 15:04:05.000000")
  571. }
  572. return e.When.Format("15:04:05.000000")
  573. }
  574. // discarded represents a number of discarded events.
  575. // It is stored as *discarded to make it easier to update in-place.
  576. type discarded int
  577. func (d *discarded) String() string {
  578. return fmt.Sprintf("(%d events discarded)", int(*d))
  579. }
  580. // trace represents an active or complete request,
  581. // either sent or received by this program.
  582. type trace struct {
  583. // Family is the top-level grouping of traces to which this belongs.
  584. Family string
  585. // Title is the title of this trace.
  586. Title string
  587. // Timing information.
  588. Start time.Time
  589. Elapsed time.Duration // zero while active
  590. // Trace information if non-zero.
  591. traceID uint64
  592. spanID uint64
  593. // Whether this trace resulted in an error.
  594. IsError bool
  595. // Append-only sequence of events (modulo discards).
  596. mu sync.RWMutex
  597. events []event
  598. refs int32 // how many buckets this is in
  599. recycler func(interface{})
  600. disc discarded // scratch space to avoid allocation
  601. finishStack []byte // where finish was called, if DebugUseAfterFinish is set
  602. }
  603. func (tr *trace) reset() {
  604. // Clear all but the mutex. Mutexes may not be copied, even when unlocked.
  605. tr.Family = ""
  606. tr.Title = ""
  607. tr.Start = time.Time{}
  608. tr.Elapsed = 0
  609. tr.traceID = 0
  610. tr.spanID = 0
  611. tr.IsError = false
  612. tr.events = nil
  613. tr.refs = 0
  614. tr.recycler = nil
  615. tr.disc = 0
  616. tr.finishStack = nil
  617. }
  618. // delta returns the elapsed time since the last event or the trace start,
  619. // and whether it spans midnight.
  620. // L >= tr.mu
  621. func (tr *trace) delta(t time.Time) (time.Duration, bool) {
  622. if len(tr.events) == 0 {
  623. return t.Sub(tr.Start), false
  624. }
  625. prev := tr.events[len(tr.events)-1].When
  626. return t.Sub(prev), prev.Day() != t.Day()
  627. }
  628. func (tr *trace) addEvent(x interface{}, recyclable, sensitive bool) {
  629. if DebugUseAfterFinish && tr.finishStack != nil {
  630. buf := make([]byte, 4<<10) // 4 KB should be enough
  631. n := runtime.Stack(buf, false)
  632. log.Printf("net/trace: trace used after finish:\nFinished at:\n%s\nUsed at:\n%s", tr.finishStack, buf[:n])
  633. }
  634. /*
  635. NOTE TO DEBUGGERS
  636. If you are here because your program panicked in this code,
  637. it is almost definitely the fault of code using this package,
  638. and very unlikely to be the fault of this code.
  639. The most likely scenario is that some code elsewhere is using
  640. a requestz.Trace after its Finish method is called.
  641. You can temporarily set the DebugUseAfterFinish var
  642. to help discover where that is; do not leave that var set,
  643. since it makes this package much less efficient.
  644. */
  645. e := event{When: time.Now(), What: x, Recyclable: recyclable, Sensitive: sensitive}
  646. tr.mu.Lock()
  647. e.Elapsed, e.NewDay = tr.delta(e.When)
  648. if len(tr.events) < cap(tr.events) {
  649. tr.events = append(tr.events, e)
  650. } else {
  651. // Discard the middle events.
  652. di := int((cap(tr.events) - 1) / 2)
  653. if d, ok := tr.events[di].What.(*discarded); ok {
  654. (*d)++
  655. } else {
  656. // disc starts at two to count for the event it is replacing,
  657. // plus the next one that we are about to drop.
  658. tr.disc = 2
  659. if tr.recycler != nil && tr.events[di].Recyclable {
  660. go tr.recycler(tr.events[di].What)
  661. }
  662. tr.events[di].What = &tr.disc
  663. }
  664. // The timestamp of the discarded meta-event should be
  665. // the time of the last event it is representing.
  666. tr.events[di].When = tr.events[di+1].When
  667. if tr.recycler != nil && tr.events[di+1].Recyclable {
  668. go tr.recycler(tr.events[di+1].What)
  669. }
  670. copy(tr.events[di+1:], tr.events[di+2:])
  671. tr.events[cap(tr.events)-1] = e
  672. }
  673. tr.mu.Unlock()
  674. }
  675. func (tr *trace) LazyLog(x fmt.Stringer, sensitive bool) {
  676. tr.addEvent(x, true, sensitive)
  677. }
  678. func (tr *trace) LazyPrintf(format string, a ...interface{}) {
  679. tr.addEvent(&lazySprintf{format, a}, false, false)
  680. }
  681. func (tr *trace) SetError() { tr.IsError = true }
  682. func (tr *trace) SetRecycler(f func(interface{})) {
  683. tr.recycler = f
  684. }
  685. func (tr *trace) SetTraceInfo(traceID, spanID uint64) {
  686. tr.traceID, tr.spanID = traceID, spanID
  687. }
  688. func (tr *trace) SetMaxEvents(m int) {
  689. // Always keep at least three events: first, discarded count, last.
  690. if len(tr.events) == 0 && m > 3 {
  691. tr.events = make([]event, 0, m)
  692. }
  693. }
  694. func (tr *trace) ref() {
  695. atomic.AddInt32(&tr.refs, 1)
  696. }
  697. func (tr *trace) unref() {
  698. if atomic.AddInt32(&tr.refs, -1) == 0 {
  699. if tr.recycler != nil {
  700. // freeTrace clears tr, so we hold tr.recycler and tr.events here.
  701. go func(f func(interface{}), es []event) {
  702. for _, e := range es {
  703. if e.Recyclable {
  704. f(e.What)
  705. }
  706. }
  707. }(tr.recycler, tr.events)
  708. }
  709. freeTrace(tr)
  710. }
  711. }
  712. func (tr *trace) When() string {
  713. return tr.Start.Format("2006/01/02 15:04:05.000000")
  714. }
  715. func (tr *trace) ElapsedTime() string {
  716. t := tr.Elapsed
  717. if t == 0 {
  718. // Active trace.
  719. t = time.Since(tr.Start)
  720. }
  721. return fmt.Sprintf("%.6f", t.Seconds())
  722. }
  723. func (tr *trace) Events() []event {
  724. tr.mu.RLock()
  725. defer tr.mu.RUnlock()
  726. return tr.events
  727. }
  728. var traceFreeList = make(chan *trace, 1000) // TODO(dsymonds): Use sync.Pool?
  729. // newTrace returns a trace ready to use.
  730. func newTrace() *trace {
  731. select {
  732. case tr := <-traceFreeList:
  733. return tr
  734. default:
  735. return new(trace)
  736. }
  737. }
  738. // freeTrace adds tr to traceFreeList if there's room.
  739. // This is non-blocking.
  740. func freeTrace(tr *trace) {
  741. if DebugUseAfterFinish {
  742. return // never reuse
  743. }
  744. tr.reset()
  745. select {
  746. case traceFreeList <- tr:
  747. default:
  748. }
  749. }
  750. func elapsed(d time.Duration) string {
  751. b := []byte(fmt.Sprintf("%.6f", d.Seconds()))
  752. // For subsecond durations, blank all zeros before decimal point,
  753. // and all zeros between the decimal point and the first non-zero digit.
  754. if d < time.Second {
  755. dot := bytes.IndexByte(b, '.')
  756. for i := 0; i < dot; i++ {
  757. b[i] = ' '
  758. }
  759. for i := dot + 1; i < len(b); i++ {
  760. if b[i] == '0' {
  761. b[i] = ' '
  762. } else {
  763. break
  764. }
  765. }
  766. }
  767. return string(b)
  768. }
  769. var pageTmpl = template.Must(template.New("Page").Funcs(template.FuncMap{
  770. "elapsed": elapsed,
  771. "add": func(a, b int) int { return a + b },
  772. }).Parse(pageHTML))
  773. const pageHTML = `
  774. {{template "Prolog" .}}
  775. {{template "StatusTable" .}}
  776. {{template "Epilog" .}}
  777. {{define "Prolog"}}
  778. <html>
  779. <head>
  780. <title>/debug/requests</title>
  781. <style type="text/css">
  782. body {
  783. font-family: sans-serif;
  784. }
  785. table#tr-status td.family {
  786. padding-right: 2em;
  787. }
  788. table#tr-status td.active {
  789. padding-right: 1em;
  790. }
  791. table#tr-status td.latency-first {
  792. padding-left: 1em;
  793. }
  794. table#tr-status td.empty {
  795. color: #aaa;
  796. }
  797. table#reqs {
  798. margin-top: 1em;
  799. }
  800. table#reqs tr.first {
  801. {{if $.Expanded}}font-weight: bold;{{end}}
  802. }
  803. table#reqs td {
  804. font-family: monospace;
  805. }
  806. table#reqs td.when {
  807. text-align: right;
  808. white-space: nowrap;
  809. }
  810. table#reqs td.elapsed {
  811. padding: 0 0.5em;
  812. text-align: right;
  813. white-space: pre;
  814. width: 10em;
  815. }
  816. address {
  817. font-size: smaller;
  818. margin-top: 5em;
  819. }
  820. </style>
  821. </head>
  822. <body>
  823. <h1>/debug/requests</h1>
  824. {{end}} {{/* end of Prolog */}}
  825. {{define "StatusTable"}}
  826. <table id="tr-status">
  827. {{range $fam := .Families}}
  828. <tr>
  829. <td class="family">{{$fam}}</td>
  830. {{$n := index $.ActiveTraceCount $fam}}
  831. <td class="active {{if not $n}}empty{{end}}">
  832. {{if $n}}<a href="?fam={{$fam}}&b=-1{{if $.Expanded}}&exp=1{{end}}">{{end}}
  833. [{{$n}} active]
  834. {{if $n}}</a>{{end}}
  835. </td>
  836. {{$f := index $.CompletedTraces $fam}}
  837. {{range $i, $b := $f.Buckets}}
  838. {{$empty := $b.Empty}}
  839. <td {{if $empty}}class="empty"{{end}}>
  840. {{if not $empty}}<a href="?fam={{$fam}}&b={{$i}}{{if $.Expanded}}&exp=1{{end}}">{{end}}
  841. [{{.Cond}}]
  842. {{if not $empty}}</a>{{end}}
  843. </td>
  844. {{end}}
  845. {{$nb := len $f.Buckets}}
  846. <td class="latency-first">
  847. <a href="?fam={{$fam}}&b={{$nb}}">[minute]</a>
  848. </td>
  849. <td>
  850. <a href="?fam={{$fam}}&b={{add $nb 1}}">[hour]</a>
  851. </td>
  852. <td>
  853. <a href="?fam={{$fam}}&b={{add $nb 2}}">[total]</a>
  854. </td>
  855. </tr>
  856. {{end}}
  857. </table>
  858. {{end}} {{/* end of StatusTable */}}
  859. {{define "Epilog"}}
  860. {{if $.Traces}}
  861. <hr />
  862. <h3>Family: {{$.Family}}</h3>
  863. {{if or $.Expanded $.Traced}}
  864. <a href="?fam={{$.Family}}&b={{$.Bucket}}">[Normal/Summary]</a>
  865. {{else}}
  866. [Normal/Summary]
  867. {{end}}
  868. {{if or (not $.Expanded) $.Traced}}
  869. <a href="?fam={{$.Family}}&b={{$.Bucket}}&exp=1">[Normal/Expanded]</a>
  870. {{else}}
  871. [Normal/Expanded]
  872. {{end}}
  873. {{if not $.Active}}
  874. {{if or $.Expanded (not $.Traced)}}
  875. <a href="?fam={{$.Family}}&b={{$.Bucket}}&rtraced=1">[Traced/Summary]</a>
  876. {{else}}
  877. [Traced/Summary]
  878. {{end}}
  879. {{if or (not $.Expanded) (not $.Traced)}}
  880. <a href="?fam={{$.Family}}&b={{$.Bucket}}&exp=1&rtraced=1">[Traced/Expanded]</a>
  881. {{else}}
  882. [Traced/Expanded]
  883. {{end}}
  884. {{end}}
  885. {{if $.Total}}
  886. <p><em>Showing <b>{{len $.Traces}}</b> of <b>{{$.Total}}</b> traces.</em></p>
  887. {{end}}
  888. <table id="reqs">
  889. <caption>
  890. {{if $.Active}}Active{{else}}Completed{{end}} Requests
  891. </caption>
  892. <tr><th>When</th><th>Elapsed&nbsp;(s)</th></tr>
  893. {{range $tr := $.Traces}}
  894. <tr class="first">
  895. <td class="when">{{$tr.When}}</td>
  896. <td class="elapsed">{{$tr.ElapsedTime}}</td>
  897. <td>{{$tr.Title}}</td>
  898. {{/* TODO: include traceID/spanID */}}
  899. </tr>
  900. {{if $.Expanded}}
  901. {{range $tr.Events}}
  902. <tr>
  903. <td class="when">{{.WhenString}}</td>
  904. <td class="elapsed">{{elapsed .Elapsed}}</td>
  905. <td>{{if or $.ShowSensitive (not .Sensitive)}}... {{.What}}{{else}}<em>[redacted]</em>{{end}}</td>
  906. </tr>
  907. {{end}}
  908. {{end}}
  909. {{end}}
  910. </table>
  911. {{end}} {{/* if $.Traces */}}
  912. {{if $.Histogram}}
  913. <h4>Latency (&micro;s) of {{$.Family}} over {{$.HistogramWindow}}</h4>
  914. {{$.Histogram}}
  915. {{end}} {{/* if $.Histogram */}}
  916. </body>
  917. </html>
  918. {{end}} {{/* end of Epilog */}}
  919. `