OpenCensus

Planet scale observability with OpenCensus

18 July 2018

Emmanuel Odeke

Orijtech, Inc.

About myself

What is OpenCensus?

OpenCensus

Dapper —> Census —> OpenCensus

Why OpenCensus?

Planet scale observability?

Go version

Concepts

Tracing

Tracing

Trace integrations

What does a trace look like?

Given an app

Tracing code

package main

import (
    "log"
    "net/http"

    "go.opencensus.io/plugin/ochttp"
    "go.opencensus.io/trace"
)

func main() {
    h := &ochttp.Handler{Handler: http.HandlerFunc(search)}
    if err := http.ListenAndServe(":8080", h); err != nil {
        log.Fatalf("Failed to listen and serve: %v", err)
    }
}

func search(w http.ResponseWriter, r *http.Request) {
    ctx, span := trace.StartSpan(r.Context(), "Search")
    defer span.End()

    // Use the context and the rest of the code goes below
    _ = ctx
}

Trace dissection

How did we get that?

package main

import (
    "log"

    "contrib.go.opencensus.io/exporter/stackdriver"
    "go.opencensus.io/trace"
)

func main() {
    sd, err := stackdriver.NewExporter(stackdriver.Options{ProjectID: "census-demos"})
    if err != nil {
        log.Fatalf("Failed to register Stackdriver Trace exporter: %v", err)
    }
    trace.RegisterExporter(sd)
}

Available trace exporters

Creating a custom trace exporter -- this could be yours!

package main

import (
    "context"
    "fmt"
    "time"

    "go.opencensus.io/trace"
)

type customPrintExporter int

func (ce *customPrintExporter) ExportSpan(sd *trace.SpanData) {
    fmt.Printf("Name: %s\nTraceID: %x\nSpanID: %x\nParentSpanID: %x\nStartTime: %s\nEndTime: %s\nAnnotations: %+v\n",
        sd.Name, sd.TraceID, sd.SpanID, sd.ParentSpanID, sd.StartTime, sd.EndTime, sd.Annotations)
}

func main() {
    trace.ApplyConfig(trace.Config{DefaultSampler: trace.ProbabilitySampler(0.999)})
    trace.RegisterExporter(new(customPrintExporter))
    _, span := trace.StartSpan(context.Background(), "sample")
    span.Annotate([]trace.Attribute{trace.Int64Attribute("invocations", 1)}, "Invoked it")
    span.End()
    <-time.After(500 * time.Millisecond)
}

Name: sample
TraceID: 3934363535316338633035643463373533333963623135363234316233363964
SpanID: 35366562303561373638363336303331
ParentSpanID: 30303030303030303030303030303030
StartTime: 2018-07-19 03:45:03.081946 -0700 PDT m=+0.000799779
EndTime: 2018-07-19 03:45:03.081971099 -0700 PDT m=+0.000824878
Annotations: [{Time:2018-07-19 03:45:03.081952 -0700 PDT m=+0.000805604 Message:Invoked it Attributes:map[invocations:1]}]

Program exited.

Metrics/Stats

Metrics/Stats

Metrics/stats

package main

import (
    "context"
    "time"

    "go.opencensus.io/stats"
    "go.opencensus.io/tag"
)

var MRequestsIn = stats.Int64("demo/requests", "The number of requests received", "1")
var MLatency = stats.Float64("demo/latency", "The number of seconds to process an entity", "ms")
var KeyMethod, _ = tag.NewKey("method")

func search(w http.ResponseWriter, r *http.Request) {
    ctx, _ = tag.New(ctx, tag.Insert(KeyMethod, "search"))
    startTime := time.Now()
    defer func() {
        endTimeMs := float64(time.Since(startTime).Nanoseconds()) / 1e6
        // Record the stats here
        stats.Record(ctx, MRequestsIn.M(1), MLatency.M(endTimeMs))
    }()
    // Search logic below...
}

Metrics/stats: Introspection and aggregation of metrics with views

import (
    "go.opencensus.io/stats"
    "go.opencensus.io/view"
)

var LatencyView = &view.View{
    Measure: MLatency, Name: "demo/latency", Description: "Latency per call",
    TagKeys:     []tag.Key{KeyMethod},
    Aggregation: view.Distribution(0, 5, 10, 20, 50, 100, 200, 300, 500, 800),
}
var RequestsView = &view.View{Measure: MRequestsIn, Name: "demo/requests",
    Description: "Number of requests", Aggregation: view.Count(),
}

func init() {
    if err := view.Register(LatencyView, RequestsView); err != nil {
        log.Fatalf("Failed to register custom views: %v", err)
    }
}

Metrics dissection

How did we get that?

package main

import (
    "log"
    "net/http"

    "go.opencensus.io/exporter/prometheus"
)

func main() {
    pe, err := prometheus.NewExporter(prometheus.Options{Namespace: "excavate"})
    if err != nil {
        log.Fatalf("Failed to create Prometheus exporter: %v", err)
    }
    view.RegisterExporter(pe)
    go func() {
        if err := http.ListenAndServe(":9888", pe); err != nil {
            log.Fatalf("Failed to create Prometheus exporter: %v", err)
        }
    }()
}

Available metrics/stats exporters

Creating a custom metrics/stats exporter

package main

import (
    "context"
    "fmt"
    "time"

    "go.opencensus.io/stats"
    "go.opencensus.io/stats/view"
    "go.opencensus.io/tag"
)

type customPrintExporter struct{}
func (ce *customPrintExporter) ExportView(vd *view.Data) { fmt.Printf("Send me to your metrics backend:\n%+v\n", vd)}

func main() {
    view.RegisterExporter(new(customPrintExporter)); view.SetReportingPeriod(time.Millisecond)
    _ = view.Register(RequestsView)
    ctx, _ := tag.New(context.Background(), tag.Insert(KeyMethod, "search"))
    stats.Record(ctx, MRequestsIn.M(23))
    <-time.After(2 * time.Millisecond)
}

var KeyMethod, _ = tag.NewKey("method")
var MRequestsIn, RequestsView = stats.Int64("demo/requests", "The number of requests received", "1"), &view.View{Measure: MRequestsIn, Name: "demo/requests", Description: "Number of requests", Aggregation: view.Count(), TagKeys: []tag.Key{KeyMethod}}

zPages

zPages

https://godoc.org/go.opencensus.io/zpages 

import (
    "log"
    "net/http"

    "go.opencensus.io/zpages"
)

func main() {
    mux := http.NewServeMux()
    zpages.Handle(mux, "/debug")
    log.Fatal(http.ListenAndServe(":7788", mux))
}

zPages /debug/rpcz

zPages /debug/tracez

If I'd like to "examine mongodb pool.Get latencies and see if we've got traces with latencies > 10 us"?

HTTP and gRPC support

HTTP support

package main

import (
    "net/http"

    "go.opencensus.io/plugin/ochttp"
    "go.opencensus.io/stats/view"
)

func main() {
    // Start of metrics/stats enabling by registering views
    _ = view.Register(ochttp.DefaultServerViews...)
    _ = view.Register(ochttp.DefaultClientViews...)

    // Start of trace propagation configuration for HTTP
    // For a trace enabled HTTP client/transport
    _ = &http.Client{Transport: &ochttp.Transport{Base: http.DefaultTransport}}
    // For a trace enabled HTTP server/handler
    _ = &ochttp.Handler{Handler: http.DefaultServeMux}
}

gRPC support

package main

import (
    "go.opencensus.io/plugin/ocgrpc"
    "go.opencensus.io/stats/view"
    "google.golang.org/grpc"
)

func main() {
    _ = view.Register(ocgrpc.DefaultServerViews...)
    _ = view.Register(ocgrpc.DefaultClientViews...)
    _ = grpc.NewServer(grpc.StatsHandler(new(ocgrpc.ServerHandler)))
}

Framework and driver integrations/instrumentation work

Demo

Demo: consumer media search app

Community

Questions?

Thank you

Emmanuel Odeke

Orijtech, Inc.

emmanuel@orijtech.com

https://orijtech.com/

@odeke_et