diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 6cbd28f60..f18f3fb7e 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -18,6 +18,7 @@ import { createLoggLogger, injeca, lifecycle } from 'injeca' import { createAuth } from './libs/auth' import { createDrizzle, migrateDatabase } from './libs/db' import { parsedEnv } from './libs/env' +import { initializeExternalDependency } from './libs/external-dependency' import { initOtel } from './libs/otel' import { createRedis } from './libs/redis' import { sessionMiddleware } from './middlewares/auth' @@ -213,11 +214,25 @@ export async function createApp() { const db = injeca.provide('datastore:db', { dependsOn: { env: parsedEnv, lifecycle }, build: async ({ dependsOn }) => { - const { db: dbInstance, pool } = createDrizzle(dependsOn.env.DATABASE_URL) - await dbInstance.execute('SELECT 1') - logger.log('Connected to database') - await migrateDatabase(dbInstance) - logger.log('Applied schema') + const { db: dbInstance, pool } = await initializeExternalDependency( + 'Database', + logger, + async (attempt) => { + const connection = createDrizzle(dependsOn.env.DATABASE_URL) + + try { + await connection.db.execute('SELECT 1') + logger.log(`Connected to database on attempt ${attempt}`) + await migrateDatabase(connection.db) + logger.log(`Applied schema on attempt ${attempt}`) + return connection + } + catch (error) { + await connection.pool.end() + throw error + } + }, + ) dependsOn.lifecycle.appHooks.onStop(() => pool.end()) return dbInstance @@ -225,11 +240,29 @@ export async function createApp() { }) const redis = injeca.provide('datastore:redis', { - dependsOn: { env: parsedEnv }, + dependsOn: { env: parsedEnv, lifecycle }, build: async ({ dependsOn }) => { - const redisInstance = createRedis(dependsOn.env.REDIS_URL) - await redisInstance.connect() - logger.log('Connected to Redis') + const redisInstance = await initializeExternalDependency( + 'Redis', + logger, + async (attempt) => { + const instance = createRedis(dependsOn.env.REDIS_URL) + + try { + await instance.connect() + logger.log(`Connected to Redis on attempt ${attempt}`) + return instance + } + catch (error) { + instance.disconnect() + throw error + } + }, + ) + + dependsOn.lifecycle.appHooks.onStop(async () => { + await redisInstance.quit() + }) return redisInstance }, }) diff --git a/apps/server/src/bin/run-billing-events-consumer.ts b/apps/server/src/bin/run-billing-events-consumer.ts index 4feb5f264..c6e2d0742 100644 --- a/apps/server/src/bin/run-billing-events-consumer.ts +++ b/apps/server/src/bin/run-billing-events-consumer.ts @@ -5,6 +5,7 @@ import process, { pid } from 'node:process' import { initLogger, LoggerFormat, LoggerLevel, useLogger } from '@guiiai/logg' import { parseEnv } from '../libs/env' +import { initializeExternalDependency } from '../libs/external-dependency' import { createRedis } from '../libs/redis' import { createBillingMqService } from '../services/billing-mq' import { createBillingMqWorker } from '../services/billing-mq-worker' @@ -30,9 +31,23 @@ export async function runBillingEventsConsumer(options: RunBillingEventsConsumer const env = parseEnv(process.env) const logger = useLogger(options.loggerName).useGlobalConfig() - const redis = createRedis(env.REDIS_URL) + const redis = await initializeExternalDependency( + 'Redis', + logger, + async (attempt) => { + const instance = createRedis(env.REDIS_URL) - await redis.connect() + try { + await instance.connect() + logger.log(`Connected to Redis on attempt ${attempt}`) + return instance + } + catch (error) { + instance.disconnect() + throw error + } + }, + ) const abortController = new AbortController() const consumer = env.BILLING_EVENTS_CONSUMER_NAME ?? `${options.group}-${pid}` diff --git a/apps/server/src/bin/run-outbox-dispatcher.ts b/apps/server/src/bin/run-outbox-dispatcher.ts index 7ad3385e9..8d65ccabc 100644 --- a/apps/server/src/bin/run-outbox-dispatcher.ts +++ b/apps/server/src/bin/run-outbox-dispatcher.ts @@ -4,6 +4,7 @@ import { initLogger, LoggerFormat, LoggerLevel, useLogger } from '@guiiai/logg' import { createDrizzle, migrateDatabase } from '../libs/db' import { parseEnv } from '../libs/env' +import { initializeExternalDependency } from '../libs/external-dependency' import { createRedis } from '../libs/redis' import { createBillingMqService } from '../services/billing-mq' import { createOutboxDispatcher } from '../services/outbox-dispatcher' @@ -23,12 +24,42 @@ export async function runOutboxDispatcher(): Promise { const env = parseEnv(process.env) const logger = useLogger('outbox-dispatcher').useGlobalConfig() - const { db, pool } = createDrizzle(env.DATABASE_URL) - const redis = createRedis(env.REDIS_URL) + const { db, pool } = await initializeExternalDependency( + 'Database', + logger, + async (attempt) => { + const connection = createDrizzle(env.DATABASE_URL) - await db.execute('SELECT 1') - await migrateDatabase(db) - await redis.connect() + try { + await connection.db.execute('SELECT 1') + logger.log(`Connected to database on attempt ${attempt}`) + await migrateDatabase(connection.db) + logger.log(`Applied schema on attempt ${attempt}`) + return connection + } + catch (error) { + await connection.pool.end() + throw error + } + }, + ) + const redis = await initializeExternalDependency( + 'Redis', + logger, + async (attempt) => { + const instance = createRedis(env.REDIS_URL) + + try { + await instance.connect() + logger.log(`Connected to Redis on attempt ${attempt}`) + return instance + } + catch (error) { + instance.disconnect() + throw error + } + }, + ) const abortController = new AbortController() const claimedBy = env.OUTBOX_DISPATCHER_NAME ?? `outbox-dispatcher-${pid}` diff --git a/apps/server/src/libs/external-dependency.ts b/apps/server/src/libs/external-dependency.ts new file mode 100644 index 000000000..f11de609a --- /dev/null +++ b/apps/server/src/libs/external-dependency.ts @@ -0,0 +1,35 @@ +import { withRetry } from '@moeru/std' + +const EXTERNAL_DEPENDENCY_INIT_MAX_ATTEMPTS = 5 +const EXTERNAL_DEPENDENCY_INIT_BASE_DELAY_MS = 5000 + +interface ExternalDependencyLogger { + log: (message: string) => unknown + withError: (error: unknown) => { + warn: (message: string) => unknown + } +} + +export async function initializeExternalDependency( + dependencyName: string, + logger: ExternalDependencyLogger, + initialize: (attempt: number) => Promise, +): Promise { + let attempt = 0 + + return await withRetry( + async () => { + attempt += 1 + return await initialize(attempt) + }, + { + retry: EXTERNAL_DEPENDENCY_INIT_MAX_ATTEMPTS - 1, + retryDelay: EXTERNAL_DEPENDENCY_INIT_BASE_DELAY_MS, + retryDelayFactor: 2, + retryDelayMax: EXTERNAL_DEPENDENCY_INIT_BASE_DELAY_MS * 2 ** (EXTERNAL_DEPENDENCY_INIT_MAX_ATTEMPTS - 1), + onError: (error) => { + logger.withError(error).warn(`${dependencyName} initialization failed on attempt ${attempt}/${EXTERNAL_DEPENDENCY_INIT_MAX_ATTEMPTS}`) + }, + }, + )() +}