Skip to content

Fix client metrics recording on round trip error #7146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 35 additions & 17 deletions instrumentation/net/http/otelhttp/transport.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,34 @@ func (t *Transport) RoundTrip(r *http.Request) (*http.Response, error) {
t.propagators.Inject(ctx, propagation.HeaderCarrier(r.Header))

res, err := t.rt.RoundTrip(r)

metricAttributes := semconv.MetricAttributes{
Req: r,
AdditionalAttributes: append(labeler.Get(), t.metricAttributesFromRequest(r)...),
}

// defer metrics recording function to record the metrics on error or no error
defer func() {
// metrics
metricOpts := t.semconv.MetricOptions(metricAttributes)

metricData := semconv.MetricData{
RequestSize: bw.BytesRead(),
}

if err == nil {
// For handling response bytes we leverage a callback when the client reads the http response
readRecordFunc := func(n int64) {
t.semconv.RecordResponseSize(ctx, n, metricOpts)
}

res.Body = newWrappedBody(span, readRecordFunc, res.Body)
}

// set transport record metrics
t.recordMetrics(ctx, requestStartTime, metricData, metricOpts)
}()

if err != nil {
// set error type attribute if the error is part of the predefined
// error types.
Expand All @@ -141,36 +169,26 @@ func (t *Transport) RoundTrip(r *http.Request) (*http.Response, error) {

span.SetStatus(codes.Error, err.Error())
span.End()

return res, err
}

// metrics
metricOpts := t.semconv.MetricOptions(semconv.MetricAttributes{
Req: r,
StatusCode: res.StatusCode,
AdditionalAttributes: append(labeler.Get(), t.metricAttributesFromRequest(r)...),
})

// For handling response bytes we leverage a callback when the client reads the http response
readRecordFunc := func(n int64) {
t.semconv.RecordResponseSize(ctx, n, metricOpts)
}
metricAttributes.StatusCode = res.StatusCode

// traces
span.SetAttributes(t.semconv.ResponseTraceAttrs(res)...)
span.SetStatus(t.semconv.Status(res.StatusCode))

res.Body = newWrappedBody(span, readRecordFunc, res.Body)
return res, nil
}

func (t *Transport) recordMetrics(ctx context.Context, requestStartTime time.Time, metricData semconv.MetricData, metricOpts map[string]semconv.MetricOpts) {
// Use floating point division here for higher precision (instead of Millisecond method).
elapsedTime := float64(time.Since(requestStartTime)) / float64(time.Millisecond)

t.semconv.RecordMetrics(ctx, semconv.MetricData{
RequestSize: bw.BytesRead(),
ElapsedTime: elapsedTime,
}, metricOpts)
metricData.ElapsedTime = elapsedTime

return res, nil
t.semconv.RecordMetrics(ctx, metricData, metricOpts)
}

func (t *Transport) metricAttributesFromRequest(r *http.Request) []attribute.KeyValue {
Expand Down
56 changes: 56 additions & 0 deletions instrumentation/net/http/otelhttp/transport_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -958,6 +958,62 @@
}
}

func TestMetricsExistenceOnRequestError(t *testing.T) {
var rm metricdata.ResourceMetrics
const (
clientRequestSize = "http.client.request.size"
clientDuration = "http.client.duration"
)
ctx := context.TODO()
reader := sdkmetric.NewManualReader()
provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
defer func() {
err := provider.Shutdown(ctx)
if err != nil {
t.Errorf("Error shutting down provider: %v", err)
}
}()

transport := NewTransport(http.DefaultTransport, WithMeterProvider(provider))
client := http.Client{Transport: transport}

// simulate an error by closing the server
// before the request is made
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
ts.Close()

r, err := http.NewRequest(http.MethodGet, ts.URL, nil)
require.NoError(t, err)

_, err = client.Do(r)
require.Error(t, err)

err = reader.Collect(ctx, &rm)
assert.NoError(t, err)

assert.Len(t, rm.ScopeMetrics[0].Metrics, 2)
metricsFound := 0
for _, m := range rm.ScopeMetrics[0].Metrics {
switch m.Name {

Check failure on line 999 in instrumentation/net/http/otelhttp/transport_test.go

View workflow job for this annotation

GitHub Actions / lint

response body must be closed (bodyclose)
case clientRequestSize:
d, ok := m.Data.(metricdata.Sum[int64])
assert.True(t, ok)
assert.Len(t, d.DataPoints, 1)
metricsFound++
case clientDuration:
d, ok := m.Data.(metricdata.Histogram[float64])
assert.True(t, ok)
assert.Len(t, d.DataPoints, 1)
metricsFound++
}
}

// make sure client request size and duration metrics is recorded
assert.Equal(t, 2, metricsFound, "Expected both metrics to be recorded")
}

func TestDefaultAttributesHandling(t *testing.T) {
var rm metricdata.ResourceMetrics
const (
Expand Down
Loading