DynamoDB Toolbox: How It Saved Me From 6 Months of Type Hell and Schema Disasters

// The old way - this haunts my dreams
const params = {
  TableName: 'Users',
  Key: {
    'PK': { S: `USER#${userId}` },
    'SK': { S: `PROFILE#${userId}` }
  },
  UpdateExpression: 'SET #email = :email, #updatedAt = :updatedAt, #version = #version + :inc',
  ExpressionAttributeNames: {
    '#email': 'email',
    '#updatedAt': 'updatedAt',
    '#version': 'version'
  },
  ExpressionAttributeValues: {
    ':email': { S: newEmail },
    ':updatedAt': { S: new Date().toISOString() },
    ':inc': { N: '1' }
  },
  ConditionExpression: 'attribute_exists(PK) AND #version = :currentVersion',
  ReturnValues: 'ALL_NEW'
};

const result = await dynamodb.updateItem(params).promise();

// What he intended
const updateParams = {
  UpdateExpression: 'SET preferences = :prefs',
  ExpressionAttributeValues: {
    ':prefs': { M: { theme: { S: 'dark' } } }
  }
};

// What actually happened (copy-paste error)
const updateParams = {
  UpdateExpression: 'SET preferences = :prefs',
  ExpressionAttributeValues: {
    ':prefs': { S: JSON.stringify({ theme: 'dark' }) } // Wrong type!
  }
};

// In user-service.ts
'SET #email = :email, #updatedAt = :updatedAt'

// In profile-service.ts
'SET email = :email, updatedAt = :updatedAt' // Missing #

// In preferences-service.ts
'SET #email = :e, #updated = :u' // Different attribute names

// In admin-service.ts
'SET email = :email, #updatedAt = :updatedAt' // Mixed style

// lib/database/entities.ts - The foundation that saved our sanity
import { Entity, Table } from 'dynamodb-toolbox';
import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';

// Single DynamoDB client for the entire application
const dynamoClient = new DynamoDBClient({
  region: process.env.AWS_REGION,
  // Connection reuse settings that reduced costs by 15%
  maxAttempts: 3,
  requestHandler: {
    connectionTimeout: 1000,
    socketTimeout: 1000,
  },
});

const docClient = DynamoDBDocumentClient.from(dynamoClient, {
  marshallOptions: {
    removeUndefinedValues: true,
    convertEmptyValues: false,
  },
  unmarshallOptions: {
    wrapNumbers: false,
  },
});

// Our single table that handles everything
export const MainTable = new Table({
  name: process.env.MAIN_TABLE_NAME!,
  partitionKey: 'PK',
  sortKey: 'SK',
  DocumentClient: docClient,
  // Indexes that actually get used in production
  indexes: {
    GSI1: {
      partitionKey: 'GSI1PK',
      sortKey: 'GSI1SK',
    },
    GSI2: {
      partitionKey: 'GSI2PK',
      sortKey: 'GSI2SK',
    },
  },
});

// User entity with full type safety
export const UserEntity = new Entity({
  name: 'User',
  attributes: {
    // Primary keys
    PK: { partitionKey: true, hidden: true, default: (data: any) => `USER#${data.userId}` },
    SK: { sortKey: true, hidden: true, default: (data: any) => `USER#${data.userId}` },

    // User attributes with validation
    userId: { type: 'string', required: true },
    email: {
      type: 'string',
      required: true,
      // Custom validation that prevented the email incident
      validate: (email: string) => {
        const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
        if (!emailRegex.test(email)) {
          throw new Error('Invalid email format');
        }
        return email.toLowerCase();
      }
    },
    username: {
      type: 'string',
      required: true,
      validate: (username: string) => {
        if (username.length <3 || username.length > 30) {
          throw new Error('Username must be 3-30 characters');
        }
        return username;
      }
    },

    // Profile data
    firstName: { type: 'string' },
    lastName: { type: 'string' },
    avatar: { type: 'string' },
    bio: { type: 'string' },

    // Preferences with default values
    preferences: {
      type: 'map',
      default: {},
      properties: {
        theme: { type: 'string', default: 'light' },
        notifications: { type: 'boolean', default: true },
        language: { type: 'string', default: 'en' },
      }
    },

    // Metadata
    createdAt: { type: 'string', default: () => new Date().toISOString() },
    updatedAt: { type: 'string', default: () => new Date().toISOString() },
    version: { type: 'number', default: 1 },

    // GSI attributes for different access patterns
    GSI1PK: { default: (data: any) => `EMAIL#${data.email}` },
    GSI1SK: { default: (data: any) => `USER#${data.userId}` },
    GSI2PK: { default: (data: any) => `USERNAME#${data.username}` },
    GSI2SK: { default: (data: any) => `USER#${data.userId}` },
  },
  table: MainTable,
} as const);

// Organization entity for multi-tenant support
export const OrganizationEntity = new Entity({
  name: 'Organization',
  attributes: {
    PK: { partitionKey: true, hidden: true, default: (data: any) => `ORG#${data.orgId}` },
    SK: { sortKey: true, hidden: true, default: (data: any) => `ORG#${data.orgId}` },

    orgId: { type: 'string', required: true },
    name: { type: 'string', required: true },
    domain: { type: 'string' },
    plan: { type: 'string', default: 'free' },

    // Settings with nested validation
    settings: {
      type: 'map',
      default: {},
      properties: {
        maxUsers: { type: 'number', default: 10 },
        features: { type: 'set', default: new Set(['basic']) },
        billing: {
          type: 'map',
          properties: {
            customerId: { type: 'string' },
            subscriptionId: { type: 'string' },
          }
        }
      }
    },

    createdAt: { type: 'string', default: () => new Date().toISOString() },
    updatedAt: { type: 'string', default: () => new Date().toISOString() },

    // GSI for domain lookups
    GSI1PK: { default: (data: any) => `DOMAIN#${data.domain}` },
    GSI1SK: { default: (data: any) => `ORG#${data.orgId}` },
  },
  table: MainTable,
} as const);

// Membership entity for user-organization relationships
export const MembershipEntity = new Entity({
  name: 'Membership',
  attributes: {
    PK: { partitionKey: true, hidden: true, default: (data: any) => `ORG#${data.orgId}` },
    SK: { sortKey: true, hidden: true, default: (data: any) => `USER#${data.userId}` },

    userId: { type: 'string', required: true },
    orgId: { type: 'string', required: true },
    role: { type: 'string', required: true, default: 'member' },
    permissions: { type: 'set', default: new Set() },

    joinedAt: { type: 'string', default: () => new Date().toISOString() },
    invitedBy: { type: 'string' },
    status: { type: 'string', default: 'active' },

    // Reverse lookup GSI
    GSI1PK: { default: (data: any) => `USER#${data.userId}` },
    GSI1SK: { default: (data: any) => `ORG#${data.orgId}` },
  },
  table: MainTable,
} as const);

// TypeScript types derived from entities
export type User = typeof UserEntity extends Entity<any, any, any, infer T> ? T : never;
export type Organization = typeof OrganizationEntity extends Entity<any, any, any, infer T> ? T : never;
export type Membership = typeof MembershipEntity extends Entity<any, any, any, infer T> ? T : never;

// services/user-service.ts - The service layer that handles complexity
import { UserEntity, OrganizationEntity, MembershipEntity } from '../database/entities';
import { ConditionalCheckFailedException } from '@aws-sdk/client-dynamodb';

export class UserService {
  // Create user with validation and error handling
  async createUser(userData: {
    userId: string;
    email: string;
    username: string;
    firstName?: string;
    lastName?: string;
    orgId?: string;
  }): Promise<User> {
    try {
      // Check if user already exists
      const existingUser = await this.getUserById(userData.userId);
      if (existingUser) {
        throw new Error('User already exists');
      }

      // Check if email is already taken (using GSI1)
      const existingEmail = await this.getUserByEmail(userData.email);
      if (existingEmail) {
        throw new Error('Email already registered');
      }

      // Check if username is taken (using GSI2)
      const existingUsername = await this.getUserByUsername(userData.username);
      if (existingUsername) {
        throw new Error('Username already taken');
      }

      // Create the user
      const result = await UserEntity.put({
        ...userData,
        version: 1,
      }, {
        conditions: { attr: 'PK', exists: false } // Prevent overwrites
      });

      // If user is joining an organization, create membership
      if (userData.orgId) {
        await MembershipEntity.put({
          userId: userData.userId,
          orgId: userData.orgId,
          role: 'member',
          status: 'active',
        });
      }

      return result.Item;
    } catch (error) {
      if (error instanceof ConditionalCheckFailedException) {
        throw new Error('User creation failed: user may already exist');
      }
      throw error;
    }
  }

  // Get user by ID with error handling
  async getUserById(userId: string): Promise<User | null> {
    try {
      const result = await UserEntity.get({
        userId,
      });

      return result.Item || null;
    } catch (error) {
      console.error('Error getting user by ID:', error);
      throw new Error('Failed to retrieve user');
    }
  }

  // Get user by email using GSI1
  async getUserByEmail(email: string): Promise<User | null> {
    try {
      const result = await UserEntity.query('GSI1PK', {
        eq: `EMAIL#${email.toLowerCase()}`,
      }, {
        index: 'GSI1',
        limit: 1,
      });

      return result.Items?.[0] || null;
    } catch (error) {
      console.error('Error getting user by email:', error);
      throw new Error('Failed to retrieve user by email');
    }
  }

  // Get user by username using GSI2
  async getUserByUsername(username: string): Promise<User | null> {
    try {
      const result = await UserEntity.query('GSI2PK', {
        eq: `USERNAME#${username}`,
      }, {
        index: 'GSI2',
        limit: 1,
      });

      return result.Items?.[0] || null;
    } catch (error) {
      console.error('Error getting user by username:', error);
      throw new Error('Failed to retrieve user by username');
    }
  }

  // Update user with optimistic locking
  async updateUser(
    userId: string,
    updates: Partial<User>,
    expectedVersion?: number
  ): Promise<User> {
    try {
      const conditions: any[] = [
        { attr: 'PK', exists: true }
      ];

      // Optimistic locking to prevent concurrent updates
      if (expectedVersion !== undefined) {
        conditions.push({ attr: 'version', eq: expectedVersion });
      }

      const result = await UserEntity.update({
        userId,
        ...updates,
        updatedAt: new Date().toISOString(),
        // Increment version for optimistic locking
        version: { $add: 1 },
      }, {
        conditions,
        returnValues: 'ALL_NEW',
      });

      return result.Item;
    } catch (error) {
      if (error instanceof ConditionalCheckFailedException) {
        throw new Error('Update failed: user was modified by another process');
      }
      throw error;
    }
  }

  // Update user preferences with validation
  async updateUserPreferences(
    userId: string,
    preferences: Partial<User['preferences']>
  ): Promise<User> {
    try {
      // Get current user to merge preferences
      const currentUser = await this.getUserById(userId);
      if (!currentUser) {
        throw new Error('User not found');
      }

      const mergedPreferences = {
        ...currentUser.preferences,
        ...preferences,
      };

      return await this.updateUser(userId, {
        preferences: mergedPreferences,
      }, currentUser.version);
    } catch (error) {
      console.error('Error updating user preferences:', error);
      throw error;
    }
  }

  // Get user's organizations
  async getUserOrganizations(userId: string): Promise<Array<Organization & { role: string }>> {
    try {
      // Query memberships for this user
      const membershipResult = await MembershipEntity.query('GSI1PK', {
        eq: `USER#${userId}`,
      }, {
        index: 'GSI1',
      });

      if (!membershipResult.Items || membershipResult.Items.length === 0) {
        return [];
      }

      // Get organization details for each membership
      const organizations = await Promise.all(
        membershipResult.Items.map(async (membership) => {
          const orgResult = await OrganizationEntity.get({
            orgId: membership.orgId,
          });

          return {
            ...orgResult.Item!,
            role: membership.role,
          };
        })
      );

      return organizations.filter(org => org !== null);
    } catch (error) {
      console.error('Error getting user organizations:', error);
      throw new Error('Failed to retrieve user organizations');
    }
  }

  // Soft delete user
  async deleteUser(userId: string): Promise<void> {
    try {
      // First, remove from all organizations
      const memberships = await MembershipEntity.query('GSI1PK', {
        eq: `USER#${userId}`,
      }, {
        index: 'GSI1',
      });

      if (memberships.Items) {
        await Promise.all(
          memberships.Items.map(membership =>
            MembershipEntity.delete({
              orgId: membership.orgId,
              userId: membership.userId,
            })
          )
        );
      }

      // Mark user as deleted instead of hard delete
      await UserEntity.update({
        userId,
        status: 'deleted',
        deletedAt: new Date().toISOString(),
        // Clear sensitive data
        email: `deleted-${userId}@deleted.com`,
        username: `deleted-${userId}`,
        firstName: undefined,
        lastName: undefined,
        avatar: undefined,
        bio: undefined,
      });

    } catch (error) {
      console.error('Error deleting user:', error);
      throw new Error('Failed to delete user');
    }
  }
}

export const userService = new UserService();

// handlers/users/create.ts - Lambda handler that actually works
import { APIGatewayProxyEvent, APIGatewayProxyResult } from 'aws-lambda';
import { userService } from '../../services/user-service';
import { z } from 'zod';

// Input validation schema
const CreateUserSchema = z.object({
  userId: z.string().min(1).max(50),
  email: z.string().email(),
  username: z.string().min(3).max(30).regex(/^[a-zA-Z0-9_-]+$/),
  firstName: z.string().optional(),
  lastName: z.string().optional(),
  orgId: z.string().optional(),
});

export const handler = async (event: APIGatewayProxyEvent): Promise<APIGatewayProxyResult> => {
  console.log('Create user request:', {
    requestId: event.requestContext.requestId,
    sourceIp: event.requestContext.identity.sourceIp,
  });

  try {
    // Parse and validate input
    if (!event.body) {
      return {
        statusCode: 400,
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          error: 'Request body is required',
          code: 'MISSING_BODY',
        }),
      };
    }

    let requestData;
    try {
      requestData = JSON.parse(event.body);
    } catch (error) {
      return {
        statusCode: 400,
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          error: 'Invalid JSON in request body',
          code: 'INVALID_JSON',
        }),
      };
    }

    // Validate with Zod
    const validationResult = CreateUserSchema.safeParse(requestData);
    if (!validationResult.success) {
      return {
        statusCode: 400,
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          error: 'Validation failed',
          code: 'VALIDATION_ERROR',
          details: validationResult.error.errors,
        }),
      };
    }

    // Create user
    const user = await userService.createUser(validationResult.data);

    // Remove sensitive fields from response
    const { preferences, ...safeUser } = user;

    return {
      statusCode: 201,
      headers: {
        'Content-Type': 'application/json',
        'X-Request-ID': event.requestContext.requestId,
      },
      body: JSON.stringify({
        message: 'User created successfully',
        user: safeUser,
      }),
    };

  } catch (error) {
    console.error('Error creating user:', {
      error: error.message,
      stack: error.stack,
      requestId: event.requestContext.requestId,
    });

    // Handle known business errors
    if (error.message.includes('already exists') ||
        error.message.includes('already registered') ||
        error.message.includes('already taken')) {
      return {
        statusCode: 409,
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          error: error.message,
          code: 'CONFLICT',
        }),
      };
    }

    // Generic error response
    return {
      statusCode: 500,
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        error: 'Internal server error',
        code: 'INTERNAL_ERROR',
        requestId: event.requestContext.requestId,
      }),
    };
  }
};

// patterns/optimistic-locking.ts - Prevent race conditions
export async function updateWithOptimisticLocking<T extends { version: number }>(
  entity: any,
  itemKey: any,
  updates: Partial<T>,
  maxRetries = 3
): Promise<T> {
  let retries = 0;

  while (retries < maxRetries) {
    try {
      // Get current item with version
      const currentItem = await entity.get(itemKey);
      if (!currentItem.Item) {
        throw new Error('Item not found');
      }

      const currentVersion = currentItem.Item.version;

      // Attempt update with version check
      const result = await entity.update({
        ...itemKey,
        ...updates,
        updatedAt: new Date().toISOString(),
        version: { $add: 1 },
      }, {
        conditions: [
          { attr: 'version', eq: currentVersion }
        ],
        returnValues: 'ALL_NEW',
      });

      return result.Item;

    } catch (error) {
      if (error instanceof ConditionalCheckFailedException && retries < maxRetries - 1) {
        retries++;
        // Exponential backoff
        await new Promise(resolve => setTimeout(resolve, Math.pow(2, retries) * 100));
        continue;
      }
      throw error;
    }
  }

  throw new Error('Max retries exceeded for optimistic locking');
}

// patterns/batch-operations.ts - Handle large datasets efficiently
export class BatchOperations {
  static async batchWrite<T>(
    entity: any,
    items: T[],
    operation: 'put' | 'delete' = 'put',
    batchSize = 25 // DynamoDB limit
  ): Promise<void> {
    const batches = this.chunkArray(items, batchSize);

    for (const batch of batches) {
      const batchRequests = batch.map(item => {
        if (operation === 'put') {
          return { PutRequest: { Item: item } };
        } else {
          return { DeleteRequest: { Key: item } };
        }
      });

      await entity.table.batchWrite({
        RequestItems: {
          [entity.table.name]: batchRequests
        }
      });

      // Rate limiting to avoid throttling
      await new Promise(resolve => setTimeout(resolve, 100));
    }
  }

  static async batchGet<T>(
    entity: any,
    keys: any[],
    batchSize = 100 // DynamoDB limit
  ): Promise<T[]> {
    const batches = this.chunkArray(keys, batchSize);
    const results: T[] = [];

    for (const batch of batches) {
      const response = await entity.table.batchGet({
        RequestItems: {
          [entity.table.name]: {
            Keys: batch
          }
        }
      });

      const items = response.Responses?.[entity.table.name] || [];
      results.push(...items);
    }

    return results;
  }

  private static chunkArray<T>(array: T[], chunkSize: number): T[][] {
    const chunks: T[][] = [];
    for (let i = 0; i < array.length; i += chunkSize) {
      chunks.push(array.slice(i, i + chunkSize));
    }
    return chunks;
  }
}

// patterns/transactions.ts - Ensure data consistency
import { TransactWriteCommand } from '@aws-sdk/lib-dynamodb';
import { MainTable } from '../database/entities';

export class TransactionService {
  // Create user and organization in a single transaction
  async createUserWithOrganization(userData: any, orgData: any): Promise<void> {
    const transactItems = [
      {
        Put: {
          TableName: MainTable.name,
          Item: {
            PK: `USER#${userData.userId}`,
            SK: `USER#${userData.userId}`,
            ...userData,
            createdAt: new Date().toISOString(),
            version: 1,
          },
          ConditionExpression: 'attribute_not_exists(PK)',
        },
      },
      {
        Put: {
          TableName: MainTable.name,
          Item: {
            PK: `ORG#${orgData.orgId}`,
            SK: `ORG#${orgData.orgId}`,
            ...orgData,
            createdAt: new Date().toISOString(),
            version: 1,
          },
          ConditionExpression: 'attribute_not_exists(PK)',
        },
      },
      {
        Put: {
          TableName: MainTable.name,
          Item: {
            PK: `ORG#${orgData.orgId}`,
            SK: `USER#${userData.userId}`,
            userId: userData.userId,
            orgId: orgData.orgId,
            role: 'owner',
            joinedAt: new Date().toISOString(),
          },
        },
      },
    ];

    const command = new TransactWriteCommand({
      TransactItems: transactItems,
    });

    await MainTable.DocumentClient.send(command);
  }

  // Transfer organization ownership atomically
  async transferOwnership(orgId: string, fromUserId: string, toUserId: string): Promise<void> {
    const transactItems = [
      {
        Update: {
          TableName: MainTable.name,
          Key: {
            PK: `ORG#${orgId}`,
            SK: `USER#${fromUserId}`,
          },
          UpdateExpression: 'SET #role = :memberRole',
          ExpressionAttributeNames: {
            '#role': 'role',
          },
          ExpressionAttributeValues: {
            ':memberRole': 'member',
          },
          ConditionExpression: '#role = :ownerRole',
          ExpressionAttributeNames: {
            '#role': 'role',
          },
          ExpressionAttributeValues: {
            ':ownerRole': 'owner',
          },
        },
      },
      {
        Update: {
          TableName: MainTable.name,
          Key: {
            PK: `ORG#${orgId}`,
            SK: `USER#${toUserId}`,
          },
          UpdateExpression: 'SET #role = :ownerRole',
          ExpressionAttributeNames: {
            '#role': 'role',
          },
          ExpressionAttributeValues: {
            ':ownerRole': 'owner',
          },
          ConditionExpression: '#role = :memberRole',
          ExpressionAttributeNames: {
            '#role': 'role',
          },
          ExpressionAttributeValues: {
            ':memberRole': 'member',
          },
        },
      },
    ];

    const command = new TransactWriteCommand({
      TransactItems: transactItems,
    });

    await MainTable.DocumentClient.send(command);
  }
}

// config/dynamodb-config.ts - Configuration that reduces costs
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';

// Singleton pattern for connection reuse
class DynamoDBManager {
  private static instance: DynamoDBManager;
  private client: DynamoDBClient;
  private docClient: DynamoDBDocumentClient;

  private constructor() {
    this.client = new DynamoDBClient({
      region: process.env.AWS_REGION,
      // Connection settings that reduced our Lambda costs by 15%
      maxAttempts: 3,
      requestHandler: {
        connectionTimeout: 1000,
        socketTimeout: 1000,
      },
      // Connection reuse
      requestHandler: {
        keepAlive: true,
        keepAliveMsecs: 1000,
        maxSockets: 50,
      },
    });

    this.docClient = DynamoDBDocumentClient.from(this.client, {
      marshallOptions: {
        removeUndefinedValues: true,
        convertEmptyValues: false,
        convertClassInstanceToMap: true,
      },
      unmarshallOptions: {
        wrapNumbers: false,
      },
    });
  }

  static getInstance(): DynamoDBManager {
    if (!DynamoDBManager.instance) {
      DynamoDBManager.instance = new DynamoDBManager();
    }
    return DynamoDBManager.instance;
  }

  getClient(): DynamoDBClient {
    return this.client;
  }

  getDocClient(): DynamoDBDocumentClient {
    return this.docClient;
  }
}

export const dynamoManager = DynamoDBManager.getInstance();
export const docClient = dynamoManager.getDocClient();

// patterns/query-optimization.ts - Patterns that improved performance 10x
export class QueryOptimizer {
  // Efficient pagination with cursor-based approach
  static async paginatedQuery<T>(
    entity: any,
    partitionKey: string,
    partitionValue: string,
    options: {
      limit?: number;
      cursor?: string;
      sortKeyCondition?: any;
      filters?: any;
      index?: string;
    } = {}
  ): Promise<{
    items: T[];
    nextCursor?: string;
    hasMore: boolean;
  }> {
    const queryParams: any = {
      [partitionKey]: { eq: partitionValue },
    };

    if (options.sortKeyCondition) {
      Object.assign(queryParams, options.sortKeyCondition);
    }

    const queryOptions: any = {
      limit: options.limit || 20,
      index: options.index,
    };

    // Cursor-based pagination
    if (options.cursor) {
      queryOptions.startKey = JSON.parse(Buffer.from(options.cursor, 'base64').toString());
    }

    // Add filters
    if (options.filters) {
      queryOptions.filters = options.filters;
    }

    const result = await entity.query(partitionKey, queryParams, queryOptions);

    const items = result.Items || [];
    const hasMore = !!result.LastEvaluatedKey;
    let nextCursor: string | undefined;

    if (hasMore && result.LastEvaluatedKey) {
      nextCursor = Buffer.from(JSON.stringify(result.LastEvaluatedKey)).toString('base64');
    }

    return {
      items,
      nextCursor,
      hasMore,
    };
  }

  // Parallel queries for multiple partition keys
  static async parallelQuery<T>(
    entity: any,
    queries: Array<{
      partitionKey: string;
      partitionValue: string;
      sortKeyCondition?: any;
      index?: string;
    }>
  ): Promise<T[]> {
    const queryPromises = queries.map(query =>
      entity.query(query.partitionKey, {
        eq: query.partitionValue,
        ...query.sortKeyCondition,
      }, {
        index: query.index,
      })
    );

    const results = await Promise.all(queryPromises);
    return results.flatMap(result => result.Items || []);
  }

  // Efficient count queries without retrieving items
  static async getCount(
    entity: any,
    partitionKey: string,
    partitionValue: string,
    options: {
      sortKeyCondition?: any;
      filters?: any;
      index?: string;
    } = {}
  ): Promise<number> {
    const queryParams: any = {
      [partitionKey]: { eq: partitionValue },
    };

    if (options.sortKeyCondition) {
      Object.assign(queryParams, options.sortKeyCondition);
    }

    const result = await entity.query(partitionKey, queryParams, {
      select: 'COUNT',
      index: options.index,
      filters: options.filters,
    });

    return result.Count || 0;
  }
}

// tests/setup/dynamodb-local.ts - Testing setup that caught bugs before production
import { spawn, ChildProcess } from 'child_process';
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
import { CreateTableCommand, DeleteTableCommand } from '@aws-sdk/client-dynamodb';

export class DynamoDBLocalTestEnvironment {
  private dynamoProcess: ChildProcess | null = null;
  private client: DynamoDBClient;

  constructor() {
    this.client = new DynamoDBClient({
      region: 'us-east-1',
      endpoint: 'http://localhost:8000',
      credentials: {
        accessKeyId: 'fake',
        secretAccessKey: 'fake',
      },
    });
  }

  async start(): Promise<void> {
    return new Promise((resolve, reject) => {
      // Start DynamoDB Local
      this.dynamoProcess = spawn('java', [
        '-Djava.library.path=./DynamoDBLocal_lib',
        '-jar', 'DynamoDBLocal.jar',
        '-sharedDb',
        '-port', '8000'
      ], {
        cwd: './dynamodb-local',
        stdio: 'pipe',
      });

      this.dynamoProcess.stdout?.on('data', (data) => {
        if (data.toString().includes('Initializing DynamoDB Local')) {
          resolve();
        }
      });

      this.dynamoProcess.on('error', reject);

      // Timeout after 10 seconds
      setTimeout(() => reject(new Error('DynamoDB Local startup timeout')), 10000);
    });
  }

  async createTable(): Promise<void> {
    const createTableCommand = new CreateTableCommand({
      TableName: 'TestTable',
      KeySchema: [
        { AttributeName: 'PK', KeyType: 'HASH' },
        { AttributeName: 'SK', KeyType: 'RANGE' },
      ],
      AttributeDefinitions: [
        { AttributeName: 'PK', AttributeType: 'S' },
        { AttributeName: 'SK', AttributeType: 'S' },
        { AttributeName: 'GSI1PK', AttributeType: 'S' },
        { AttributeName: 'GSI1SK', AttributeType: 'S' },
      ],
      GlobalSecondaryIndexes: [
        {
          IndexName: 'GSI1',
          KeySchema: [
            { AttributeName: 'GSI1PK', KeyType: 'HASH' },
            { AttributeName: 'GSI1SK', KeyType: 'RANGE' },
          ],
          Projection: { ProjectionType: 'ALL' },
          BillingMode: 'PAY_PER_REQUEST',
        },
      ],
      BillingMode: 'PAY_PER_REQUEST',
    });

    await this.client.send(createTableCommand);
  }

  async cleanup(): Promise<void> {
    try {
      await this.client.send(new DeleteTableCommand({
        TableName: 'TestTable',
      }));
    } catch (error) {
      // Table might not exist
    }

    if (this.dynamoProcess) {
      this.dynamoProcess.kill();
      this.dynamoProcess = null;
    }
  }
}

// tests/integration/user-service.test.ts - Tests that actually matter
import { describe, beforeAll, afterAll, beforeEach, test, expect } from '@jest/globals';
import { DynamoDBLocalTestEnvironment } from '../setup/dynamodb-local';
import { UserService } from '../../services/user-service';

describe('UserService Integration Tests', () => {
  let testEnv: DynamoDBLocalTestEnvironment;
  let userService: UserService;

  beforeAll(async () => {
    testEnv = new DynamoDBLocalTestEnvironment();
    await testEnv.start();
    await testEnv.createTable();
    userService = new UserService();
  });

  afterAll(async () => {
    await testEnv.cleanup();
  });

  beforeEach(async () => {
    // Clean up between tests
    // Implementation depends on your cleanup strategy
  });

  test('should create user with validation', async () => {
    const userData = {
      userId: 'test-user-1',
      email: 'test@example.com',
      username: 'testuser',
      firstName: 'Test',
      lastName: 'User',
    };

    const user = await userService.createUser(userData);

    expect(user).toBeDefined();
    expect(user.userId).toBe(userData.userId);
    expect(user.email).toBe(userData.email);
    expect(user.version).toBe(1);
    expect(user.createdAt).toBeDefined();
  });

  test('should prevent duplicate email registration', async () => {
    const userData1 = {
      userId: 'user1',
      email: 'duplicate@example.com',
      username: 'user1',
    };

    const userData2 = {
      userId: 'user2',
      email: 'duplicate@example.com', // Same email
      username: 'user2',
    };

    await userService.createUser(userData1);

    await expect(userService.createUser(userData2))
      .rejects.toThrow('Email already registered');
  });

  test('should handle concurrent updates with optimistic locking', async () => {
    // Create user
    const user = await userService.createUser({
      userId: 'concurrent-test',
      email: 'concurrent@example.com',
      username: 'concurrent',
    });

    // Simulate concurrent updates
    const update1Promise = userService.updateUser(user.userId, {
      firstName: 'Update1',
    }, user.version);

    const update2Promise = userService.updateUser(user.userId, {
      firstName: 'Update2',
    }, user.version);

    // One should succeed, one should fail
    const results = await Promise.allSettled([update1Promise, update2Promise]);

    const successes = results.filter(r => r.status === 'fulfilled');
    const failures = results.filter(r => r.status === 'rejected');

    expect(successes).toHaveLength(1);
    expect(failures).toHaveLength(1);
    expect(failures[0].reason.message).toContain('modified by another process');
  });

  test('should query users by email efficiently', async () => {
    const userData = {
      userId: 'query-test',
      email: 'query@example.com',
      username: 'queryuser',
    };

    await userService.createUser(userData);

    const foundUser = await userService.getUserByEmail('query@example.com');

    expect(foundUser).toBeDefined();
    expect(foundUser!.userId).toBe(userData.userId);
  });
});

// Implement new operations alongside existing ones
class UserRepository {
  // Old method (keep for now)
  async getUserOld(userId: string) {
    const params = {
      TableName: 'Users',
      Key: { PK: { S: `USER#${userId}` }, SK: { S: `USER#${userId}` } }
    };
    return await this.dynamoClient.getItem(params).promise();
  }

  // New method with DynamoDB Toolbox
  async getUser(userId: string) {
    return await UserEntity.get({ userId });
  }
}

// Use feature flags to gradually switch
const useNewRepository = process.env.USE_DYNAMODB_TOOLBOX === 'true';

const user = useNewRepository
  ? await userRepo.getUser(userId)
  : await userRepo.getUserOld(userId);