feat(server): add OpenTelemetry observability stack (#1154)

Co-authored-by: Claude <noreply@anthropic.com>

apps/server/.env

@@ -15,3 +15,4 @@ BACKEND_LLM_BASE_URL="change-me"
 
 CLIENT_URL="change-me"
 
+# OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"

apps/server/otel/loki/loki.yaml

@@ -16,7 +16,7 @@ common:
 
 schema_config:
   configs:
-    - from: '2024-01-01'
+    - from: "2024-01-01"
       store: tsdb
       object_store: filesystem
       schema: v13

apps/server/package.json

@@ -38,10 +38,10 @@
     "drizzle-valibot": "catalog:",
     "hono": "catalog:",
     "injeca": "catalog:",
-    "ioredis": "^5.6.1",
+    "ioredis": "^5.10.0",
     "pg": "^8.20.0",
     "postgres": "^3.4.8",
-    "stripe": "^20.3.0",
+    "stripe": "^20.4.0",
     "tsx": "^4.21.0",
     "valibot": "catalog:"
   },

apps/server/src/app.ts

@@ -14,6 +14,7 @@ import { createLoggLogger, injeca, lifecycle } from 'injeca'
 import { createAuth } from './libs/auth'
 import { createDrizzle, migrateDatabase } from './libs/db'
 import { parsedEnv } from './libs/env'
+import { initOtel } from './libs/otel'
 import { createRedis } from './libs/redis'
 import { sessionMiddleware } from './middlewares/auth'
 import { otelMiddleware } from './middlewares/otel'
@@ -51,9 +52,20 @@ interface AppDeps {
   stripeService: StripeDBService
   configKV: ConfigKVService
   env: Env
+  otel: OtelMetrics | null
 }
 
-function buildApp({ auth, characterService, chatService, providerService, fluxService, stripeService, configKV, env }: AppDeps) {
+function buildApp({
+  auth,
+  characterService,
+  chatService,
+  providerService,
+  fluxService,
+  stripeService,
+  configKV,
+  env,
+  otel,
+}: AppDeps) {
   const logger = useLogger('app').useGlobalConfig()
 
   const app = new Hono<HonoEnv>()
@@ -148,6 +160,7 @@ async function createApp() {
       if (!o)
         return null
 
+      o.start()
       dependsOn.lifecycle.appHooks.onStop(() => o.shutdown())
       return o
     },
@@ -213,7 +226,17 @@ async function createApp() {
   })
 
   await injeca.start()
-  const resolved = await injeca.resolve({ auth, characterService, chatService, providerService, fluxService, stripeService, configKV, env: parsedEnv })
+  const resolved = await injeca.resolve({
+    auth,
+    characterService,
+    chatService,
+    providerService,
+    fluxService,
+    stripeService,
+    configKV,
+    otel,
+    env: parsedEnv,
+  })
   const app = buildApp({
     auth: resolved.auth,
     characterService: resolved.characterService,
@@ -223,6 +246,7 @@ async function createApp() {
     stripeService: resolved.stripeService,
     configKV: resolved.configKV,
     env: resolved.env,
+    otel: resolved.otel,
   })
 
   logger.withFields({ port: 3000 }).log('Server started')

apps/server/src/libs/env.ts

@@ -23,6 +23,14 @@ const EnvSchema = object({
 
   BACKEND_LLM_BASE_URL: optional(string()),
   BACKEND_LLM_API_KEY: optional(string()),
+
+  // OpenTelemetry
+  OTEL_SERVICE_NAMESPACE: optional(string(), 'airi'),
+  OTEL_SERVICE_NAME: optional(string(), 'server'),
+  OTEL_TRACES_SAMPLING_RATIO: optional(string(), '1.0'),
+  OTEL_EXPORTER_OTLP_ENDPOINT: optional(string()),
+  OTEL_EXPORTER_OTLP_HEADERS: optional(string()),
+  OTEL_DEBUG: optional(string()),
 })
 
 export type Env = InferOutput<typeof EnvSchema>

apps/server/src/libs/otel.ts

@@ -80,11 +80,10 @@ export function initOtel(env: Env) {
     resource,
     sampler,
     spanProcessors: [new BatchSpanProcessor(traceExporter)],
-    metricReaders: [new PeriodicExportingMetricReader({
+    metricReader: new PeriodicExportingMetricReader({
       exporter: metricExporter,
-      exportIntervalMillis: 15_000,
-      exportTimeoutMillis: 10_000,
-    })],
+      exportIntervalMillis: 15000,
+    }),
     logRecordProcessors: [new BatchLogRecordProcessor(logExporter)],
     instrumentations: [
       new HttpInstrumentation({
@@ -101,11 +100,10 @@ export function initOtel(env: Env) {
     ],
   })
 
-  // SDK must start BEFORE metrics.getMeter() — the metrics API does NOT
-  // have a proxy mechanism like traces. getMeter() called before start()
-  // returns a permanent NoopMeter that never upgrades.
-  sdk.start()
-  logger.log(`OpenTelemetry initialized, exporting to ${otlpEndpoint}, sampling ratio: ${samplingRatio}`)
+  const start = () => {
+    sdk.start()
+    logger.log(`OpenTelemetry initialized, exporting to ${otlpEndpoint}, sampling ratio: ${samplingRatio}`)
+  }
 
   const meter = metrics.getMeter(serviceName)
 
@@ -163,6 +161,7 @@ export function initOtel(env: Env) {
     authFailures,
     stripeEvents,
 
+    start,
     shutdown,
   }
 }

apps/server/src/middlewares/otel.ts

@@ -17,32 +17,31 @@ export function otelMiddleware(otelMetrics: {
   return async (c, next) => {
     const startTime = performance.now()
     const method = c.req.method
-    const path = c.req.path
+    const path = c.req.routePath || c.req.path
+    const attributes = {
+      'http.method': method,
+      'http.route': path,
+      'http.url': c.req.url,
+    }
 
-    otelMetrics.httpActiveRequests.add(1, { 'http.request.method': method, 'http.route': path })
+    otelMetrics.httpActiveRequests.add(1, { 'http.method': method, 'http.route': path })
 
-    const span = tracer.startSpan(`${method} ${path}`, {
-      attributes: {
-        'http.request.method': method,
-        'http.route': path,
-        'url.full': c.req.url,
-      },
-    })
+    const span = tracer.startSpan(`${method} ${path}`, { attributes })
 
     try {
       await context.with(trace.setSpan(context.active(), span), () => next())
 
       const status = c.res.status
-      span.setAttribute('http.response.status_code', status)
+      span.setAttribute('http.status_code', status)
 
       if (status >= 500) {
         span.setStatus({ code: SpanStatusCode.ERROR, message: `HTTP ${status}` })
       }
 
       otelMetrics.httpRequestDuration.record(performance.now() - startTime, {
-        'http.request.method': method,
+        'http.method': method,
         'http.route': path,
-        'http.response.status_code': status,
+        'http.status_code': status,
       })
     }
     catch (err) {
@@ -51,7 +50,7 @@ export function otelMiddleware(otelMetrics: {
       throw err
     }
     finally {
-      otelMetrics.httpActiveRequests.add(-1, { 'http.request.method': method, 'http.route': path })
+      otelMetrics.httpActiveRequests.add(-1, { 'http.method': method, 'http.route': path })
       span.end()
     }
   }