Produktionsgeschichten: Debugging von Notification-Delivery im großen Maßstab

// Das ist, was unsere Alerts uns sagten
const alertTimeline = [
  { time: '06:15', service: 'email', metric: 'delivery_rate', value: 60, threshold: 95 },
  { time: '06:16', service: 'push', metric: 'timeout_rate', value: 25, threshold: 5 },
  { time: '06:18', service: 'websocket', metric: 'connection_count', value: 85000, threshold: 50000 },
  { time: '06:20', service: 'database', metric: 'connection_pool', value: 95, threshold: 80 },
  { time: '06:22', service: 'redis', metric: 'memory_usage', value: 92, threshold: 85 }
];

// Aber das war, was tatsächlich unter der Haube passierte
const realityCheck = {
  emailProvider: 'Rate-Limiting uns wegen Reputation-Score-Abfall',
  pushService: 'Apple APNS lehnt fehlerhafte Payloads von Template-Bug ab',
  websockets: 'Connection-Storm von mobiler App, die fehlgeschlagene Push-Registrierungen wiederholt',
  database: 'Deadlocks von gleichzeitigen Notification-Preference-Updates',
  redis: 'Speichererschöpfung durch unbegrenzte Connection-Metadata-Speicherung'
};

class EmergencyCircuitBreaker {
  private isOpen = false;
  private openedAt?: Date;
  private failureCount = 0;
  private readonly failureThreshold = 10;
  private readonly recoveryTimeoutMs = 30000;

  async executeWithBreaker<T>(
    operation: () => Promise<T>,
    fallback?: () => Promise<T>
  ): Promise<T> {
    if (this.isOpen) {
      if (this.shouldAttemptReset()) {
        console.log('Circuit Breaker versucht Reset');
        this.isOpen = false;
        this.failureCount = 0;
      } else {
        if (fallback) {
          return await fallback();
        }
        throw new Error('Circuit Breaker ist offen');
      }
    }

    try {
      const result = await operation();
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      if (fallback && this.isOpen) {
        return await fallback();
      }
      throw error;
    }
  }

  private onSuccess(): void {
    this.failureCount = 0;
  }

  private onFailure(): void {
    this.failureCount++;
    if (this.failureCount >= this.failureThreshold) {
      this.isOpen = true;
      this.openedAt = new Date();
      console.warn(`Circuit Breaker nach ${this.failureCount} Fehlern geöffnet`);
    }
  }
}

// Notfall-Notification-Service mit Circuit Breakern
class EmergencyNotificationService {
  private emailBreaker = new EmergencyCircuitBreaker();
  private pushBreaker = new EmergencyCircuitBreaker();
  private websocketBreaker = new EmergencyCircuitBreaker();

  async processNotification(event: NotificationEvent): Promise<void> {
    // Primäre Kanäle mit Circuit Breakern und Fallbacks versuchen
    await Promise.allSettled([
      this.emailBreaker.executeWithBreaker(
        () => this.sendEmail(event),
        () => this.queueForLaterDelivery(event, 'email')
      ),
      this.pushBreaker.executeWithBreaker(
        () => this.sendPush(event),
        () => this.sendWebSocketFallback(event)
      ),
      this.websocketBreaker.executeWithBreaker(
        () => this.sendWebSocket(event),
        () => this.storeForPolling(event)
      )
    ]);
  }
}

// Unser Tracing offenbarte die Kaskadensequenz
const traceAnalysis = {
  '06:14:45': 'Neue App-Version mit fehlerhafter Push-Token-Registrierung deployed',
  '06:15:00': 'Fehlerhafte Push-Payloads führen dazu, dass APNS ablehnt und Verbindungen schließt',
  '06:15:30': 'Mobile App wiederholt Push-Registrierung, erzeugt WebSocket-Connection-Storm',
  '06:16:00': 'Datenbankverbindungspool durch Preference-Update-Queries erschöpft',
  '06:16:30': 'E-Mail-Service wechselt zu Backup-Provider, triggert Rate-Limits',
  '06:17:00': 'Redis-Speicher füllt sich mit verwaisten Connection-Metadaten',
  '06:17:30': 'System tritt in vollständigen Kaskadenausfall-Modus ein'
};

// Die Debugging-Query, die das Muster offenbarte
const debugQuery = `
  SELECT 
    ne.correlation_id,
    ne.notification_type,
    nd.channel,
    nd.status,
    nd.error_message,
    nd.created_at
  FROM notification_events ne
  JOIN notification_deliveries nd ON ne.id = nd.event_id
  WHERE ne.created_at > '2024-11-29 06:14:00'
    AND nd.status IN ('failed', 'timeout')
  ORDER BY nd.created_at DESC
  LIMIT 1000;
`;

interface ObservabilityHierarchy {
  // Level 1: Benutzerauswirkung (Was Kunden sehen)
  userImpact: {
    notificationsReceived: number;
    averageDeliveryTime: number;
    userComplaints: number;
  };
  
  // Level 2: Service-Gesundheit (Wie unsere Systeme performen)  
  serviceHealth: {
    deliveryRates: Record<NotificationChannel, number>;
    errorRates: Record<string, number>;
    responseTimes: Record<string, number>;
  };
  
  // Level 3: Infrastruktur (Was unter der Haube passiert)
  infrastructure: {
    databaseConnections: number;
    redisMemory: number;
    queueDepths: Record<string, number>;
  };
  
  // Level 4: Externe Abhängigkeiten (Dinge, die wir nicht kontrollieren)
  externalDeps: {
    emailProviderStatus: string;
    pushProviderLatency: number;
    cloudServiceHealth: string;
  };
}

class HierarchicalMonitoring {
  async assessSystemHealth(): Promise<SystemHealthSnapshot> {
    // Mit Benutzerauswirkung beginnen - das ist, was wirklich zählt
    const userImpact = await this.getUserImpactMetrics();
    
    if (userImpact.isHealthy) {
      return { status: 'healthy', details: userImpact };
    }
    
    // Wenn Benutzerauswirkung schlecht ist, durch die Hierarchie nach unten bohren
    const serviceHealth = await this.getServiceHealthMetrics();
    const infrastructure = await this.getInfrastructureMetrics(); 
    const externalDeps = await this.getExternalDepMetrics();
    
    // Probleme über Hierarchieebenen korrelieren
    const rootCause = this.correlateIssues({
      userImpact,
      serviceHealth, 
      infrastructure,
      externalDeps
    });
    
    return {
      status: 'degraded',
      rootCause,
      remediationSteps: this.generateRemediationPlan(rootCause)
    };
  }
}

{{#each user.recentActivities}}
  <div class="activity-item">
    <span>{{formatDate this.createdAt}}</span>
    <span>{{this.description}}</span>
    {{#if this.projectName}}
      <span>in {{getProjectDetails this.projectId}}</span>
    {{/if}}
  </div>
{{/each}}

class TemplatePerformanceProfiler {
  private renderTimes: Map<string, number[]> = new Map();
  private queryCount: Map<string, number> = new Map();
  private activeRenders: Map<string, Date> = new Map();

  async profileRender(
    templateId: string,
    templateContent: string,
    data: any
  ): Promise<ProfiledRenderResult> {
    const renderId = `${templateId}-${Date.now()}`;
    this.activeRenders.set(renderId, new Date());
    
    // Datenbankaufrufe wrappen, um Abfragen pro Template zu zählen
    const originalQuery = this.db.query;
    let queryCount = 0;
    
    this.db.query = (...args) => {
      queryCount++;
      return originalQuery.apply(this.db, args);
    };
    
    try {
      const startTime = Date.now();
      const result = await this.templateEngine.render(templateContent, data);
      const renderTime = Date.now() - startTime;
      
      // Performance-Metriken speichern
      if (!this.renderTimes.has(templateId)) {
        this.renderTimes.set(templateId, []);
      }
      this.renderTimes.get(templateId)!.push(renderTime);
      this.queryCount.set(renderId, queryCount);
      
      // Bei verdächtigen Mustern alarmieren
      if (queryCount > 10) {
        console.warn(`Template ${templateId} führte ${queryCount} DB-Abfragen während Render aus`);
      }
      
      if (renderTime > 1000) {
        console.warn(`Template ${templateId} brauchte ${renderTime}ms zum Rendern`);
      }
      
      return {
        content: result,
        renderTime,
        queryCount,
        metrics: this.calculateMetrics(templateId)
      };
      
    } finally {
      // Original Query-Methode wiederherstellen
      this.db.query = originalQuery;
      this.activeRenders.delete(renderId);
    }
  }

  private calculateMetrics(templateId: string): TemplateMetrics {
    const times = this.renderTimes.get(templateId) || [];
    const recentTimes = times.slice(-100); // Letzte 100 Renders
    
    return {
      averageRenderTime: recentTimes.reduce((a, b) => a + b, 0) / recentTimes.length,
      p95RenderTime: this.percentile(recentTimes, 0.95),
      p99RenderTime: this.percentile(recentTimes, 0.99),
      renderCount: recentTimes.length,
      suspiciousPatterns: this.detectPatterns(recentTimes)
    };
  }

  // Empfehlungen basierend auf Performance-Mustern generieren
  generateOptimizationSuggestions(templateId: string): string[] {
    const metrics = this.calculateMetrics(templateId);
    const suggestions: string[] = [];
    
    if (metrics.averageRenderTime > 500) {
      suggestions.push('Erwäge Caching häufig verwendeter Daten');
    }
    
    if (metrics.p99RenderTime > 2000) {
      suggestions.push('Template hat hohe Tail-Latenz - untersuche langsame Pfade');
    }
    
    const avgQueries = Array.from(this.queryCount.values())
      .reduce((a, b) => a + b, 0) / this.queryCount.size;
    
    if (avgQueries > 5) {
      suggestions.push('Zu viele Datenbankabfragen - erwäge Daten-Preloading');
    }
    
    return suggestions;
  }
}

class SafeTemplateRenderer {
  private readonly MAX_RENDER_TIME = 2000; // 2 Sekunden
  private readonly MAX_DB_QUERIES = 10;
  private readonly CACHE_TTL = 300; // 5 Minuten

  async renderWithGuardrails(
    templateId: string,
    userId: string,
    data: any
  ): Promise<string> {
    // Häufig benötigte Daten vorladen, um N+1-Queries zu verhindern
    const enhancedData = await this.preloadTemplateData(userId, data);
    
    // Render-Beschränkungen einrichten
    const renderPromise = this.templateEngine.render(
      templateId, 
      enhancedData,
      {
        timeout: this.MAX_RENDER_TIME,
        maxQueries: this.MAX_DB_QUERIES,
        enableCache: true
      }
    );
    
    try {
      return await Promise.race([
        renderPromise,
        this.createTimeoutPromise(this.MAX_RENDER_TIME)
      ]);
    } catch (error) {
      if (error instanceof TimeoutError) {
        // Fallback zu gecachter Version oder einfachem Template
        return await this.renderFallbackTemplate(templateId, userId, data);
      }
      throw error;
    }
  }

  private async preloadTemplateData(userId: string, data: any): Promise<any> {
    // Template analysieren, um zu bestimmen, welche Daten es benötigt
    const requiredData = this.analyzeTemplateDataNeeds(data.templateContent);
    
    // Alle erforderlichen Daten in einzelnen Abfragen batch-laden
    const preloadedData = await Promise.all([
      requiredData.needsProjects ? this.loadUserProjects(userId) : null,
      requiredData.needsActivities ? this.loadUserActivities(userId, 10) : null,
      requiredData.needsTeamInfo ? this.loadUserTeamInfo(userId) : null
    ]);
    
    return {
      ...data,
      projects: preloadedData[0],
      recentActivities: preloadedData[1], 
      teamInfo: preloadedData[2]
    };
  }

  private async renderFallbackTemplate(
    templateId: string, 
    userId: string, 
    data: any
  ): Promise<string> {
    // Vereinfachte Template-Version verwenden, die keine komplexen Daten benötigt
    const fallbackTemplate = await this.getFallbackTemplate(templateId);
    return await this.templateEngine.render(fallbackTemplate, {
      user: data.user,
      basicData: this.extractBasicData(data)
    });
  }
}

// Die "verbesserte" Retry-Logik der mobilen App - tu das nicht
class NotificationConnectionManager {
  connect() {
    this.ws = new WebSocket(this.endpoint);
    
    this.ws.onclose = () => {
      // Exponentieller Backoff... dachten sie
      this.retryDelay = Math.min(this.retryDelay * 2, 30000);
      setTimeout(() => this.connect(), this.retryDelay);
    };
    
    this.ws.onerror = () => {
      // Bei Fehler sofort wiederholen - das war das Problem
      this.connect();
    };
  }
}

class DefensiveWebSocketServer {
  private connectionCounts: Map<string, number> = new Map();
  private rateLimiter: Map<string, Date[]> = new Map();
  private readonly MAX_CONNECTIONS_PER_USER = 5;
  private readonly RATE_LIMIT_WINDOW = 60000; // 1 Minute
  private readonly RATE_LIMIT_MAX = 10; // 10 Verbindungen pro Minute

  async handleConnection(socket: WebSocket, request: IncomingMessage): Promise<void> {
    const clientId = this.getClientIdentifier(request);
    const userId = await this.authenticateConnection(request);
    
    // Rate-Limiting-Prüfung
    if (!this.checkRateLimit(clientId)) {
      socket.close(1008, 'Rate-Limit überschritten');
      this.logSecurityEvent('rate_limit_exceeded', clientId);
      return;
    }
    
    // Verbindungsanzahl-Prüfung pro Benutzer
    const userConnections = this.connectionCounts.get(userId) || 0;
    if (userConnections >= this.MAX_CONNECTIONS_PER_USER) {
      socket.close(1008, 'Zu viele Verbindungen');
      this.logSecurityEvent('connection_limit_exceeded', userId);
      return;
    }
    
    // Server-Last-Schutz
    const serverLoad = await this.getCurrentServerLoad();
    if (serverLoad > 0.9) {
      // Nur hochprioritäre Verbindungen unter Last akzeptieren
      if (!this.isHighPriorityUser(userId)) {
        socket.close(1013, 'Server überlastet - bitte später wiederholen');
        return;
      }
    }
    
    this.setupConnection(socket, userId, clientId);
  }

  private checkRateLimit(clientId: string): boolean {
    const now = new Date();
    const windowStart = new Date(now.getTime() - this.RATE_LIMIT_WINDOW);
    
    if (!this.rateLimiter.has(clientId)) {
      this.rateLimiter.set(clientId, []);
    }
    
    const connections = this.rateLimiter.get(clientId)!;
    
    // Alte Verbindungsversuche entfernen
    const recentConnections = connections.filter(date => date > windowStart);
    this.rateLimiter.set(clientId, recentConnections);
    
    // Prüfen, ob unter Rate-Limit
    if (recentConnections.length >= this.RATE_LIMIT_MAX) {
      return false;
    }
    
    // Diesen Verbindungsversuch aufzeichnen
    recentConnections.push(now);
    return true;
  }

  private async getCurrentServerLoad(): Promise<number> {
    const metrics = await Promise.all([
      this.getCPUUsage(),
      this.getMemoryUsage(),
      this.getConnectionCount(),
      this.getEventQueueDepth()
    ]);
    
    // Gewichteter Durchschnitt verschiedener Last-Indikatoren
    return (
      metrics[0] * 0.3 + // CPU
      metrics[1] * 0.2 + // Memory  
      metrics[2] * 0.3 + // Connections
      metrics[3] * 0.2   // Queue depth
    );
  }

  // Graceful Degradation unter Last
  private async handleConnectionUnderLoad(
    socket: WebSocket, 
    userId: string
  ): Promise<void> {
    // Update-Häufigkeit für nicht-kritische Notifications reduzieren
    const updateInterval = this.getAdaptiveUpdateInterval();
    
    // Kritische Notification-Typen priorisieren
    const allowedNotificationTypes = this.getCriticalNotificationTypes();
    
    socket.send(JSON.stringify({
      type: 'connection_degraded',
      message: 'Reduzierter Service aufgrund hoher Last',
      updateInterval,
      allowedTypes: allowedNotificationTypes
    }));
  }
}

class IncidentDashboard {
  async getCurrentSystemState(): Promise<SystemSnapshot> {
    const timestamp = new Date();
    
    // Metriken parallel für Geschwindigkeit sammeln
    const [
      deliveryMetrics,
      errorMetrics, 
      performanceMetrics,
      externalServiceStatus
    ] = await Promise.all([
      this.getDeliveryMetrics(),
      this.getErrorMetrics(),
      this.getPerformanceMetrics(),
      this.checkExternalServices()
    ]);
    
    return {
      timestamp,
      overall: this.calculateOverallHealth(deliveryMetrics, errorMetrics),
      deliveryMetrics: {
        email: deliveryMetrics.email,
        push: deliveryMetrics.push,
        websocket: deliveryMetrics.websocket,
        sms: deliveryMetrics.sms
      },
      errors: {
        byChannel: errorMetrics.byChannel,
        byType: errorMetrics.byType,
        trending: errorMetrics.trending
      },
      performance: {
        avgDeliveryTime: performanceMetrics.avgDeliveryTime,
        p95DeliveryTime: performanceMetrics.p95DeliveryTime,
        queueDepths: performanceMetrics.queueDepths
      },
      externalServices: externalServiceStatus,
      recommendations: this.generateRecommendations(deliveryMetrics, errorMetrics)
    };
  }

  private generateRecommendations(
    delivery: any, 
    errors: any
  ): string[] {
    const recommendations: string[] = [];
    
    // E-Mail-Zustellungsprobleme
    if (delivery.email.successRate &lt;0.95) {
      recommendations.push('E-Mail-Provider-Status und Reputation-Score prüfen');
    }
    
    // Push-Notification-Probleme  
    if (delivery.push.successRate &lt;0.9) {
      recommendations.push('Push-Zertifikate und Payload-Format verifizieren');
    }
    
    // Hohe Fehlerrate
    if (errors.overall.rate > 0.05) {
      recommendations.push('Häufigste Fehlermuster untersuchen');
    }
    
    return recommendations;
  }
}

class NotificationTracer {
  async traceNotificationJourney(correlationId: string): Promise<NotificationTrace> {
    // Die vollständige Reise einer Notification durch das System abrufen
    const events = await this.db.query(`
      SELECT 
        ne.id as event_id,
        ne.notification_type,
        ne.created_at,
        ne.data,
        nd.channel,
        nd.status,
        nd.attempt_count,
        nd.error_message,
        nd.sent_at,
        nd.delivered_at
      FROM notification_events ne
      LEFT JOIN notification_deliveries nd ON ne.id = nd.event_id  
      WHERE ne.correlation_id = $1
      ORDER BY ne.created_at, nd.created_at
    `, [correlationId]);
    
    // Auch Logs von externen Services abrufen
    const externalLogs = await Promise.all([
      this.getEmailProviderLogs(correlationId),
      this.getPushProviderLogs(correlationId),
      this.getWebSocketLogs(correlationId)
    ]);
    
    return {
      correlationId,
      timeline: this.buildTimeline(events, externalLogs),
      status: this.determineOverallStatus(events),
      failurePoints: this.identifyFailures(events, externalLogs),
      recommendations: this.generateTraceRecommendations(events)
    };
  }

  private buildTimeline(events: any[], externalLogs: any[]): TimelineEvent[] {
    const allEvents = [
      ...events.map(e => ({ 
        timestamp: e.created_at, 
        type: 'internal', 
        details: e 
      })),
      ...externalLogs.flat().map(e => ({ 
        timestamp: e.timestamp, 
        type: 'external', 
        details: e 
      }))
    ];
    
    return allEvents.sort((a, b) => 
      new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
    );
  }
}

class PredictiveAlerting {
  async checkPredictiveMetrics(): Promise<Alert[]> {
    const alerts: Alert[] = [];
    
    // Zustellungsraten-Trends prüfen (nicht nur aktuelle Rate)
    const deliveryTrend = await this.calculateDeliveryTrend('1h');
    if (deliveryTrend.slope < -0.1) { // Um 10%+ pro Stunde fallend
      alerts.push({
        level: 'warning',
        message: 'Zustellungsrate tendiert nach unten',
        details: `Rate fällt um ${deliveryTrend.slope * 100}% pro Stunde`,
        predictedImpact: 'Systemausfall in ~2 Stunden, falls Trend anhält'
      });
    }
    
    // Queue-Depth-Wachstum prüfen
    const queueGrowth = await this.calculateQueueGrowthRate('30m');
    if (queueGrowth > 1000) { // Wächst um 1000+ Items pro 30min
      alerts.push({
        level: 'critical',
        message: 'Notification-Queue wächst nicht nachhaltig',
        details: `Queue wächst um ${queueGrowth} Items pro 30min`,
        predictedImpact: 'Queue-Overflow in ~45 Minuten'
      });
    }
    
    // Entstehung von Fehlermustern prüfen
    const errorPatterns = await this.detectEmergingErrorPatterns();
    for (const pattern of errorPatterns) {
      if (pattern.confidence > 0.8) {
        alerts.push({
          level: 'warning',
          message: `Neues Fehlermuster erkannt: ${pattern.type}`,
          details: pattern.description,
          predictedImpact: `Potentielle Systemauswirkung: ${pattern.impact}`
        });
      }
    }
    
    return alerts;
  }
}