From 963306944ecf9bc30ac7b7525d9126569b20a0ce Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Fri, 6 Mar 2026 02:51:54 +0800 Subject: [PATCH] feat(server): add OpenTelemetry observability stack (#1154) Co-authored-by: Claude --- apps/server/.env | 1 + apps/server/otel/loki/loki.yaml | 2 +- apps/server/package.json | 4 ++-- apps/server/src/app.ts | 28 ++++++++++++++++++++++++++-- apps/server/src/libs/env.ts | 8 ++++++++ apps/server/src/libs/otel.ts | 17 ++++++++--------- apps/server/src/middlewares/otel.ts | 25 ++++++++++++------------- 7 files changed, 58 insertions(+), 27 deletions(-) diff --git a/apps/server/.env b/apps/server/.env index 48c07d6e4..77dd9b6dc 100644 --- a/apps/server/.env +++ b/apps/server/.env @@ -15,3 +15,4 @@ BACKEND_LLM_BASE_URL="change-me" CLIENT_URL="change-me" +# OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" diff --git a/apps/server/otel/loki/loki.yaml b/apps/server/otel/loki/loki.yaml index 7d1fedf54..0a1e348e1 100644 --- a/apps/server/otel/loki/loki.yaml +++ b/apps/server/otel/loki/loki.yaml @@ -16,7 +16,7 @@ common: schema_config: configs: - - from: '2024-01-01' + - from: "2024-01-01" store: tsdb object_store: filesystem schema: v13 diff --git a/apps/server/package.json b/apps/server/package.json index 1be4c6f1f..73b474a50 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -38,10 +38,10 @@ "drizzle-valibot": "catalog:", "hono": "catalog:", "injeca": "catalog:", - "ioredis": "^5.6.1", + "ioredis": "^5.10.0", "pg": "^8.20.0", "postgres": "^3.4.8", - "stripe": "^20.3.0", + "stripe": "^20.4.0", "tsx": "^4.21.0", "valibot": "catalog:" }, diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index c0d7c6b89..41f4212c4 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -14,6 +14,7 @@ import { createLoggLogger, injeca, lifecycle } from 'injeca' import { createAuth } from './libs/auth' import { createDrizzle, migrateDatabase } from './libs/db' import { parsedEnv } from './libs/env' +import { initOtel } from './libs/otel' import { createRedis } from './libs/redis' import { sessionMiddleware } from './middlewares/auth' import { otelMiddleware } from './middlewares/otel' @@ -51,9 +52,20 @@ interface AppDeps { stripeService: StripeDBService configKV: ConfigKVService env: Env + otel: OtelMetrics | null } -function buildApp({ auth, characterService, chatService, providerService, fluxService, stripeService, configKV, env }: AppDeps) { +function buildApp({ + auth, + characterService, + chatService, + providerService, + fluxService, + stripeService, + configKV, + env, + otel, +}: AppDeps) { const logger = useLogger('app').useGlobalConfig() const app = new Hono() @@ -148,6 +160,7 @@ async function createApp() { if (!o) return null + o.start() dependsOn.lifecycle.appHooks.onStop(() => o.shutdown()) return o }, @@ -213,7 +226,17 @@ async function createApp() { }) await injeca.start() - const resolved = await injeca.resolve({ auth, characterService, chatService, providerService, fluxService, stripeService, configKV, env: parsedEnv }) + const resolved = await injeca.resolve({ + auth, + characterService, + chatService, + providerService, + fluxService, + stripeService, + configKV, + otel, + env: parsedEnv, + }) const app = buildApp({ auth: resolved.auth, characterService: resolved.characterService, @@ -223,6 +246,7 @@ async function createApp() { stripeService: resolved.stripeService, configKV: resolved.configKV, env: resolved.env, + otel: resolved.otel, }) logger.withFields({ port: 3000 }).log('Server started') diff --git a/apps/server/src/libs/env.ts b/apps/server/src/libs/env.ts index de0edd947..74668d279 100644 --- a/apps/server/src/libs/env.ts +++ b/apps/server/src/libs/env.ts @@ -23,6 +23,14 @@ const EnvSchema = object({ BACKEND_LLM_BASE_URL: optional(string()), BACKEND_LLM_API_KEY: optional(string()), + + // OpenTelemetry + OTEL_SERVICE_NAMESPACE: optional(string(), 'airi'), + OTEL_SERVICE_NAME: optional(string(), 'server'), + OTEL_TRACES_SAMPLING_RATIO: optional(string(), '1.0'), + OTEL_EXPORTER_OTLP_ENDPOINT: optional(string()), + OTEL_EXPORTER_OTLP_HEADERS: optional(string()), + OTEL_DEBUG: optional(string()), }) export type Env = InferOutput diff --git a/apps/server/src/libs/otel.ts b/apps/server/src/libs/otel.ts index db4a59f01..f9f42e4aa 100644 --- a/apps/server/src/libs/otel.ts +++ b/apps/server/src/libs/otel.ts @@ -80,11 +80,10 @@ export function initOtel(env: Env) { resource, sampler, spanProcessors: [new BatchSpanProcessor(traceExporter)], - metricReaders: [new PeriodicExportingMetricReader({ + metricReader: new PeriodicExportingMetricReader({ exporter: metricExporter, - exportIntervalMillis: 15_000, - exportTimeoutMillis: 10_000, - })], + exportIntervalMillis: 15000, + }), logRecordProcessors: [new BatchLogRecordProcessor(logExporter)], instrumentations: [ new HttpInstrumentation({ @@ -101,11 +100,10 @@ export function initOtel(env: Env) { ], }) - // SDK must start BEFORE metrics.getMeter() — the metrics API does NOT - // have a proxy mechanism like traces. getMeter() called before start() - // returns a permanent NoopMeter that never upgrades. - sdk.start() - logger.log(`OpenTelemetry initialized, exporting to ${otlpEndpoint}, sampling ratio: ${samplingRatio}`) + const start = () => { + sdk.start() + logger.log(`OpenTelemetry initialized, exporting to ${otlpEndpoint}, sampling ratio: ${samplingRatio}`) + } const meter = metrics.getMeter(serviceName) @@ -163,6 +161,7 @@ export function initOtel(env: Env) { authFailures, stripeEvents, + start, shutdown, } } diff --git a/apps/server/src/middlewares/otel.ts b/apps/server/src/middlewares/otel.ts index f12bfc1c6..d7f94a596 100644 --- a/apps/server/src/middlewares/otel.ts +++ b/apps/server/src/middlewares/otel.ts @@ -17,32 +17,31 @@ export function otelMiddleware(otelMetrics: { return async (c, next) => { const startTime = performance.now() const method = c.req.method - const path = c.req.path + const path = c.req.routePath || c.req.path + const attributes = { + 'http.method': method, + 'http.route': path, + 'http.url': c.req.url, + } - otelMetrics.httpActiveRequests.add(1, { 'http.request.method': method, 'http.route': path }) + otelMetrics.httpActiveRequests.add(1, { 'http.method': method, 'http.route': path }) - const span = tracer.startSpan(`${method} ${path}`, { - attributes: { - 'http.request.method': method, - 'http.route': path, - 'url.full': c.req.url, - }, - }) + const span = tracer.startSpan(`${method} ${path}`, { attributes }) try { await context.with(trace.setSpan(context.active(), span), () => next()) const status = c.res.status - span.setAttribute('http.response.status_code', status) + span.setAttribute('http.status_code', status) if (status >= 500) { span.setStatus({ code: SpanStatusCode.ERROR, message: `HTTP ${status}` }) } otelMetrics.httpRequestDuration.record(performance.now() - startTime, { - 'http.request.method': method, + 'http.method': method, 'http.route': path, - 'http.response.status_code': status, + 'http.status_code': status, }) } catch (err) { @@ -51,7 +50,7 @@ export function otelMiddleware(otelMetrics: { throw err } finally { - otelMetrics.httpActiveRequests.add(-1, { 'http.request.method': method, 'http.route': path }) + otelMetrics.httpActiveRequests.add(-1, { 'http.method': method, 'http.route': path }) span.end() } }