feat(server): retry when booting error

apps/server/src/app.ts

@@ -18,6 +18,7 @@ import { createLoggLogger, injeca, lifecycle } from 'injeca'
 import { createAuth } from './libs/auth'
 import { createDrizzle, migrateDatabase } from './libs/db'
 import { parsedEnv } from './libs/env'
+import { initializeExternalDependency } from './libs/external-dependency'
 import { initOtel } from './libs/otel'
 import { createRedis } from './libs/redis'
 import { sessionMiddleware } from './middlewares/auth'
@@ -213,11 +214,25 @@ export async function createApp() {
   const db = injeca.provide('datastore:db', {
     dependsOn: { env: parsedEnv, lifecycle },
     build: async ({ dependsOn }) => {
-      const { db: dbInstance, pool } = createDrizzle(dependsOn.env.DATABASE_URL)
-      await dbInstance.execute('SELECT 1')
-      logger.log('Connected to database')
-      await migrateDatabase(dbInstance)
-      logger.log('Applied schema')
+      const { db: dbInstance, pool } = await initializeExternalDependency(
+        'Database',
+        logger,
+        async (attempt) => {
+          const connection = createDrizzle(dependsOn.env.DATABASE_URL)
+
+          try {
+            await connection.db.execute('SELECT 1')
+            logger.log(`Connected to database on attempt ${attempt}`)
+            await migrateDatabase(connection.db)
+            logger.log(`Applied schema on attempt ${attempt}`)
+            return connection
+          }
+          catch (error) {
+            await connection.pool.end()
+            throw error
+          }
+        },
+      )
 
       dependsOn.lifecycle.appHooks.onStop(() => pool.end())
       return dbInstance
@@ -225,11 +240,29 @@ export async function createApp() {
   })
 
   const redis = injeca.provide('datastore:redis', {
-    dependsOn: { env: parsedEnv },
+    dependsOn: { env: parsedEnv, lifecycle },
     build: async ({ dependsOn }) => {
-      const redisInstance = createRedis(dependsOn.env.REDIS_URL)
-      await redisInstance.connect()
-      logger.log('Connected to Redis')
+      const redisInstance = await initializeExternalDependency(
+        'Redis',
+        logger,
+        async (attempt) => {
+          const instance = createRedis(dependsOn.env.REDIS_URL)
+
+          try {
+            await instance.connect()
+            logger.log(`Connected to Redis on attempt ${attempt}`)
+            return instance
+          }
+          catch (error) {
+            instance.disconnect()
+            throw error
+          }
+        },
+      )
+
+      dependsOn.lifecycle.appHooks.onStop(async () => {
+        await redisInstance.quit()
+      })
       return redisInstance
     },
   })

apps/server/src/bin/run-billing-events-consumer.ts

@@ -5,6 +5,7 @@ import process, { pid } from 'node:process'
 import { initLogger, LoggerFormat, LoggerLevel, useLogger } from '@guiiai/logg'
 
 import { parseEnv } from '../libs/env'
+import { initializeExternalDependency } from '../libs/external-dependency'
 import { createRedis } from '../libs/redis'
 import { createBillingMqService } from '../services/billing-mq'
 import { createBillingMqWorker } from '../services/billing-mq-worker'
@@ -30,9 +31,23 @@ export async function runBillingEventsConsumer(options: RunBillingEventsConsumer
 
   const env = parseEnv(process.env)
   const logger = useLogger(options.loggerName).useGlobalConfig()
-  const redis = createRedis(env.REDIS_URL)
+  const redis = await initializeExternalDependency(
+    'Redis',
+    logger,
+    async (attempt) => {
+      const instance = createRedis(env.REDIS_URL)
 
-  await redis.connect()
+      try {
+        await instance.connect()
+        logger.log(`Connected to Redis on attempt ${attempt}`)
+        return instance
+      }
+      catch (error) {
+        instance.disconnect()
+        throw error
+      }
+    },
+  )
 
   const abortController = new AbortController()
   const consumer = env.BILLING_EVENTS_CONSUMER_NAME ?? `${options.group}-${pid}`

apps/server/src/bin/run-outbox-dispatcher.ts

@@ -4,6 +4,7 @@ import { initLogger, LoggerFormat, LoggerLevel, useLogger } from '@guiiai/logg'
 
 import { createDrizzle, migrateDatabase } from '../libs/db'
 import { parseEnv } from '../libs/env'
+import { initializeExternalDependency } from '../libs/external-dependency'
 import { createRedis } from '../libs/redis'
 import { createBillingMqService } from '../services/billing-mq'
 import { createOutboxDispatcher } from '../services/outbox-dispatcher'
@@ -23,12 +24,42 @@ export async function runOutboxDispatcher(): Promise<void> {
 
   const env = parseEnv(process.env)
   const logger = useLogger('outbox-dispatcher').useGlobalConfig()
-  const { db, pool } = createDrizzle(env.DATABASE_URL)
-  const redis = createRedis(env.REDIS_URL)
+  const { db, pool } = await initializeExternalDependency(
+    'Database',
+    logger,
+    async (attempt) => {
+      const connection = createDrizzle(env.DATABASE_URL)
 
-  await db.execute('SELECT 1')
-  await migrateDatabase(db)
-  await redis.connect()
+      try {
+        await connection.db.execute('SELECT 1')
+        logger.log(`Connected to database on attempt ${attempt}`)
+        await migrateDatabase(connection.db)
+        logger.log(`Applied schema on attempt ${attempt}`)
+        return connection
+      }
+      catch (error) {
+        await connection.pool.end()
+        throw error
+      }
+    },
+  )
+  const redis = await initializeExternalDependency(
+    'Redis',
+    logger,
+    async (attempt) => {
+      const instance = createRedis(env.REDIS_URL)
+
+      try {
+        await instance.connect()
+        logger.log(`Connected to Redis on attempt ${attempt}`)
+        return instance
+      }
+      catch (error) {
+        instance.disconnect()
+        throw error
+      }
+    },
+  )
 
   const abortController = new AbortController()
   const claimedBy = env.OUTBOX_DISPATCHER_NAME ?? `outbox-dispatcher-${pid}`

apps/server/src/libs/external-dependency.ts

@@ -0,0 +1,35 @@
+import { withRetry } from '@moeru/std'
+
+const EXTERNAL_DEPENDENCY_INIT_MAX_ATTEMPTS = 5
+const EXTERNAL_DEPENDENCY_INIT_BASE_DELAY_MS = 5000
+
+interface ExternalDependencyLogger {
+  log: (message: string) => unknown
+  withError: (error: unknown) => {
+    warn: (message: string) => unknown
+  }
+}
+
+export async function initializeExternalDependency<T>(
+  dependencyName: string,
+  logger: ExternalDependencyLogger,
+  initialize: (attempt: number) => Promise<T>,
+): Promise<T> {
+  let attempt = 0
+
+  return await withRetry(
+    async () => {
+      attempt += 1
+      return await initialize(attempt)
+    },
+    {
+      retry: EXTERNAL_DEPENDENCY_INIT_MAX_ATTEMPTS - 1,
+      retryDelay: EXTERNAL_DEPENDENCY_INIT_BASE_DELAY_MS,
+      retryDelayFactor: 2,
+      retryDelayMax: EXTERNAL_DEPENDENCY_INIT_BASE_DELAY_MS * 2 ** (EXTERNAL_DEPENDENCY_INIT_MAX_ATTEMPTS - 1),
+      onError: (error) => {
+        logger.withError(error).warn(`${dependencyName} initialization failed on attempt ${attempt}/${EXTERNAL_DEPENDENCY_INIT_MAX_ATTEMPTS}`)
+      },
+    },
+  )()
+}