The example demonstrates explicit tracing instrumentation with otel4s and OpenTelemetry.

README.md

otel4s - tracing instrumentation example

This gist contains the complete code used in section 7 of the article https://ochenashko.com/practical-observability-distributed-tracing.

The example demonstrates explicit tracing instrumentation with otel4s and OpenTelemetry.

Running the example

Launch Grafana LGTM as a Docker container:

$ docker run -p 3000:3000 -p 4317:4317 -p 4318:4318 --rm -ti grafana/otel-lgtm

Start a service:

$ scala-cli run Main.scala

Send a request to the service:

$ curl localhost:8080/weather/Kyiv

You can then explore the generated traces in Grafana at http://localhost:3000.

ForecastModule.scala

import cats.effect.IO
import org.http4s.client.Client
import org.typelevel.otel4s.Attribute
import org.typelevel.otel4s.trace.{StatusCode, Tracer, TracerProvider}

class ForecastModule(client: Client[IO])(using Tracer[IO]) {
  def checkWeather(location: String): IO[String] =
    Tracer[IO].span("checkWeather", Attribute("location", location)).use { span =>
      getForecast(location) <* span.addEvent("forecast-received")
    }

  private def getForecast(location: String): IO[String] =
    Tracer[IO].span("getForecast").use { span =>
      for {
        _        <- IO.println(s"Getting forecast for $location")
        forecast <- client.expect[String](s"https://wttr.in/$location?format=3")
        _        <- span.setStatus(StatusCode.Ok)
      } yield forecast
    }
}

object ForecastModule {
  def create(client: Client[IO])(using TracerProvider[IO]): IO[ForecastModule] =
    for {
      given Tracer[IO] <- TracerProvider[IO].tracer("module.forecast").withVersion("1.2.3").get
    } yield new ForecastModule(client)
}

HttpClient.scala

import cats.effect.{IO, Resource}
import org.http4s.*
import org.http4s.client.Client
import org.http4s.client.middleware.Logger
import org.http4s.ember.client.EmberClientBuilder
import org.typelevel.otel4s.context.propagation.TextMapUpdater
import org.typelevel.otel4s.semconv.attributes.HttpAttributes
import org.typelevel.otel4s.trace.*

object HttpClient {

  def create(using TracerProvider[IO]): Resource[IO, Client[IO]] =
    EmberClientBuilder
      .default[IO]
      .build
      .map(client => Logger[IO](logHeaders = true, logBody = true)(client))
      .evalMap(client => HttpClient.traced(client))

  private def traced(client: Client[IO])(using TracerProvider[IO]): IO[Client[IO]] =
    for {
      given Tracer[IO] <- TracerProvider[IO].get("org.http4s.client")
    } yield tracingMiddleware(client)

  private def tracingMiddleware(client: Client[IO])(using Tracer[IO]): Client[IO] =
    Client { request =>
      Resource.uncancelable { poll =>
        for {
          spanOps <- Tracer[IO]
            .spanBuilder(s"${request.method} ${request.uri}")
            .withSpanKind(SpanKind.Client)
            .build
            .resource

          headers  <- Resource.eval(Tracer[IO].propagate(Headers.empty))
          response <- poll(client.run(request.withHeaders(headers)).mapK(spanOps.trace))

          _ <- Resource.eval(
            spanOps.span.addAttribute(HttpAttributes.HttpResponseStatusCode(response.status.code))
          )
        } yield response
      }
    }

  private given TextMapUpdater[Headers] with {
    def updated(carrier: Headers, key: String, value: String): Headers =
      carrier.put(key -> value)
  }

}

HttpServer.scala

import cats.data.Kleisli
import cats.effect.{IO, Resource}
import org.http4s.dsl.io.*
import org.http4s.*
import org.http4s.server.Server
import org.http4s.ember.server.EmberServerBuilder
import org.typelevel.ci.CIString
import org.typelevel.otel4s.Attribute
import org.typelevel.otel4s.context.propagation.TextMapGetter
import org.typelevel.otel4s.semconv.attributes.HttpAttributes
import org.typelevel.otel4s.trace.*

class HttpServer(forecastModule: ForecastModule)(using Tracer[IO]) {
  import HttpServer.given

  def start: Resource[IO, Server] =
    EmberServerBuilder.default[IO].withHttpApp(httpApp).build

  def httpApp: HttpApp[IO] =
    tracingMiddleware(routes.orNotFound)

  private def routes: HttpRoutes[IO] = HttpRoutes.of { case req @ GET -> Root / "weather" / location =>
    for {
      forecast <- forecastModule.checkWeather(location)
    } yield Response[IO]().withEntity(forecast)
  }

  private def tracingMiddleware(httpApp: HttpApp[IO]): HttpApp[IO] =
    Kleisli { (req: Request[IO]) =>
      IO.uncancelable { poll =>
        Tracer[IO].joinOrRoot(req.headers) {
          Tracer[IO]
            .spanBuilder(s"${req.method} ${req.uri}")
            .withSpanKind(SpanKind.Server)
            .build
            .use { span =>
              poll(httpApp.run(req)).flatTap { response =>
                span.addAttribute(HttpAttributes.HttpResponseStatusCode(response.status.code))
              }
            }
        }
      }
    }

}

object HttpServer {
  def create(forecastModule: ForecastModule)(using TracerProvider[IO]): IO[HttpServer] =
    for {
      given Tracer[IO] <- TracerProvider[IO].get("org.http4s.server")
    } yield new HttpServer(forecastModule)

  private given TextMapGetter[Headers] with {
    def get(carrier: Headers, key: String): Option[String] =
      carrier.get(CIString(key)).map(_.head.value)

    def keys(carrier: Headers): Iterable[String] =
      carrier.headers.view.map(_.name).distinct.map(_.toString).toSeq
  }

}

Main.scala

//> using scala "3.7.4"
//> using dep "org.http4s::http4s-ember-client:0.23.33"
//> using dep "org.http4s::http4s-ember-server:0.23.33"
//> using dep "org.http4s::http4s-dsl:0.23.33"
//> using dep "org.typelevel::otel4s-oteljava:0.14.0"
//> using dep "org.typelevel::otel4s-semconv:0.14.0"
//> using dep "io.opentelemetry:opentelemetry-exporter-otlp:1.58.0"
//> using dep "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.58.0"
//> using dep "ch.qos.logback:logback-classic:1.5.24"
//> using javaOpt "-Dotel.service.name=weather-service"
//> using javaOpt "-Dotel.java.global-autoconfigure.enabled=true"

//> using file "HttpClient.scala"
//> using file "HttpServer.scala"
//> using file "ForecastModule.scala"

import cats.effect.{IO, IOApp, Resource}
import org.typelevel.otel4s.oteljava.OtelJava
import org.typelevel.otel4s.trace.*

object Main extends IOApp.Simple {

  def run: IO[Unit] = {
    for {
      otel4s                   <- OtelJava.autoConfigured[IO]()
      given TracerProvider[IO] <- Resource.pure(otel4s.tracerProvider)
      client                   <- HttpClient.create
      forecastModule           <- Resource.eval(ForecastModule.create(client))
      httpServer               <- Resource.eval(HttpServer.create(forecastModule))
      _                        <- httpServer.start
    } yield ()
  }.useForever

}