AWS Lambda + S3 Signed URLs: Büyük Dosya Yükleme için Pratik Çözüm

// Bütçemizi ve kullanıcı deneyimini öldüren Lambdaexport const uploadHandler = async (event: APIGatewayEvent) => {  // Bu HER upload için 15 DAKİKAYA kadar çalışıyordu  const file = parseMultipartFormData(event.body);
  // Memory kullanımı büyük dosyalar için 3GB+'a çıkıyordu  const processedFile = await processVideo(file);
  // S3 upload 10+ dakika sürebiliyordu  const result = await s3.upload({    Bucket: 'my-videos',    Key: `uploads/${uuidv4()}`,    Body: processedFile,  }).promise();
  return { statusCode: 200, body: JSON.stringify(result) };};

// lib/file-upload-stack.tsimport * as cdk from 'aws-cdk-lib';import { Construct } from 'constructs';import * as s3 from 'aws-cdk-lib/aws-s3';import * as lambda from 'aws-cdk-lib/aws-lambda';import * as apigateway from 'aws-cdk-lib/aws-apigateway';import * as s3n from 'aws-cdk-lib/aws-s3-notifications';import * as iam from 'aws-cdk-lib/aws-iam';import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs';
export class FileUploadStack extends cdk.Stack {  constructor(scope: Construct, id: string, props?: cdk.StackProps) {    super(scope, id, props);
    // Maliyet optimizasyonu için lifecycle policy'leri olan S3 bucket    const uploadBucket = new s3.Bucket(this, 'UploadBucket', {      bucketName: `${this.stackName}-uploads-${this.account}`,      cors: [        {          allowedOrigins: ['*'],          allowedMethods: [            s3.HttpMethods.PUT,            s3.HttpMethods.POST,            s3.HttpMethods.GET,            s3.HttpMethods.HEAD,          ],          allowedHeaders: ['*'],          exposedHeaders: ['ETag'],          maxAge: 3600,        },      ],      // Tamamlanmamış multipart upload'ları 7 gün sonra otomatik sil      lifecycleRules: [        {          id: 'AbortIncompleteMultipartUploads',          enabled: true,          abortIncompleteMultipartUploadsAfter: cdk.Duration.days(7),        },        {          id: 'TransitionToIA',          enabled: true,          transitions: [            {              storageClass: s3.StorageClass.INFREQUENT_ACCESS,              transitionAfter: cdk.Duration.days(30),            },            {              storageClass: s3.StorageClass.GLACIER,              transitionAfter: cdk.Duration.days(90),            },          ],        },      ],      // Güvenlik için public access'i engelle      blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,      encryption: s3.BucketEncryption.S3_MANAGED,    });
    // Signed URL generate eden Lambda - <200ms'de çalışır    const signedUrlGenerator = new NodejsFunction(this, 'SignedUrlGenerator', {      entry: 'src/handlers/generate-signed-url.ts',      runtime: lambda.Runtime.NODEJS_20_X,      architecture: lambda.Architecture.ARM_64,      memorySize: 512, // Küçük memory footprint      timeout: cdk.Duration.seconds(30),      environment: {        UPLOAD_BUCKET: uploadBucket.bucketName,        ALLOWED_FILE_TYPES: 'video/mp4,video/quicktime,video/x-msvideo,image/jpeg,image/png',        MAX_FILE_SIZE: '10737418240', // 10GB byte cinsinden        SIGNED_URL_EXPIRY: '3600', // 1 saat      },      bundling: {        minify: true,        sourceMap: true,        target: 'es2022',      },    });
    // Signed URL generator'a signed URL oluşturma yetkisi ver    uploadBucket.grantReadWrite(signedUrlGenerator);    signedUrlGenerator.addToRolePolicy(      new iam.PolicyStatement({        effect: iam.Effect.ALLOW,        actions: ['s3:PutObjectAcl', 's3:GetObject'],        resources: [uploadBucket.arnForObjects('*')],      })    );
    // Post-upload processing için Lambda    const fileProcessor = new NodejsFunction(this, 'FileProcessor', {      entry: 'src/handlers/process-file.ts',      runtime: lambda.Runtime.NODEJS_20_X,      architecture: lambda.Architecture.ARM_64,      memorySize: 2048, // Processing için daha yüksek memory      timeout: cdk.Duration.minutes(5),      environment: {        UPLOAD_BUCKET: uploadBucket.bucketName,      },      bundling: {        minify: true,        sourceMap: true,        target: 'es2022',        // Video processing için gerekirse ffmpeg dahil et        nodeModules: ['fluent-ffmpeg'],      },    });
    uploadBucket.grantReadWrite(fileProcessor);
    // Processing tetiklemek için S3 event notification    uploadBucket.addEventNotification(      s3.EventType.OBJECT_CREATED,      new s3n.LambdaDestination(fileProcessor),      { prefix: 'uploads/' } // Sadece uploads/ prefix'indeki dosyaları işle    );
    // Signed URL generation için API Gateway    const api = new apigateway.RestApi(this, 'FileUploadApi', {      restApiName: 'File Upload API',      description: 'S3 signed URL generate etme API\'si',      defaultCorsPreflightOptions: {        allowOrigins: apigateway.Cors.ALL_ORIGINS,        allowMethods: ['GET', 'POST', 'OPTIONS'],        allowHeaders: ['Content-Type', 'Authorization'],      },    });
    const uploads = api.root.addResource('uploads');    const signedUrl = uploads.addResource('signed-url');
    signedUrl.addMethod(      'POST',      new apigateway.LambdaIntegration(signedUrlGenerator, {        requestTemplates: {          'application/json': '{"body": $input.json("$")}',        },      })    );
    // Outputlar    new cdk.CfnOutput(this, 'ApiUrl', {      value: api.url,      description: 'API Gateway URL',    });
    new cdk.CfnOutput(this, 'BucketName', {      value: uploadBucket.bucketName,      description: 'S3 Upload Bucket Name',    });  }}

// src/handlers/generate-signed-url.tsimport { APIGatewayProxyEvent, APIGatewayProxyResult } from 'aws-lambda';import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';import { getSignedUrl } from '@aws-sdk/s3-request-presigner';import { z } from 'zod';
// Input validation şemasıconst SignedUrlRequestSchema = z.object({  fileName: z.string().min(1).max(255),  fileSize: z.number().int().min(1).max(10737418240), // 10GB max  fileType: z.string().regex(/^(video|image|audio)\/[a-zA-Z0-9][a-zA-Z0-9\!\-\_]*[a-zA-Z0-9]*$/),  uploadId: z.string().uuid().optional(), // Takip için});
const s3Client = new S3Client({ region: process.env.AWS_REGION });
export const handler = async (  event: APIGatewayProxyEvent): Promise<APIGatewayProxyResult> => {  console.log('Signed URL generate isteği:', {    body: event.body,    headers: event.headers  });
  try {    // Request'i parse et ve validate et    const body = JSON.parse(event.body || '{}');    const request = SignedUrlRequestSchema.parse(body);
    // Güvenlik kontrolleri    const allowedTypes = process.env.ALLOWED_FILE_TYPES?.split(',') || [];    if (!allowedTypes.includes(request.fileType)) {      return {        statusCode: 400,        headers: { 'Content-Type': 'application/json' },        body: JSON.stringify({          error: 'Dosya tipi izin verilmeyen',          allowedTypes,        }),      };    }
    const maxSize = parseInt(process.env.MAX_FILE_SIZE || '5368709120'); // 5GB default    if (request.fileSize > maxSize) {      return {        statusCode: 400,        headers: { 'Content-Type': 'application/json' },        body: JSON.stringify({          error: 'Dosya çok büyük',          maxSize,          receivedSize: request.fileSize,        }),      };    }
    // Timestamp ve sanitized filename ile unique key generate et    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');    const sanitizedFileName = request.fileName.replace(/[^a-zA-Z0-9.-]/g, '_');    const key = `uploads/${timestamp}-${sanitizedFileName}`;
    // PUT request için signed URL oluştur    const putObjectCommand = new PutObjectCommand({      Bucket: process.env.UPLOAD_BUCKET!,      Key: key,      ContentType: request.fileType,      ContentLength: request.fileSize,      // Processing için metadata ekle      Metadata: {        'original-filename': request.fileName,        'upload-id': request.uploadId || 'direct-upload',        'file-size': request.fileSize.toString(),        'uploaded-at': new Date().toISOString(),      },      // Güvenlik header'ları      ServerSideEncryption: 'AES256',    });
    const signedUrl = await getSignedUrl(s3Client, putObjectCommand, {      expiresIn: parseInt(process.env.SIGNED_URL_EXPIRY || '3600'), // 1 saat default    });
    console.log('Signed URL başarıyla generate edildi:', {      key,      fileSize: request.fileSize,      fileType: request.fileType,      expiresIn: process.env.SIGNED_URL_EXPIRY,    });
    return {      statusCode: 200,      headers: {        'Content-Type': 'application/json',        'Access-Control-Allow-Origin': '*',        'Cache-Control': 'no-cache',      },      body: JSON.stringify({        signedUrl,        key,        method: 'PUT',        headers: {          'Content-Type': request.fileType,          'Content-Length': request.fileSize.toString(),        },        expiresAt: new Date(Date.now() + parseInt(process.env.SIGNED_URL_EXPIRY || '3600') * 1000).toISOString(),      }),    };
  } catch (error) {    console.error('Signed URL generation hatası:', error);
    if (error instanceof z.ZodError) {      return {        statusCode: 400,        headers: { 'Content-Type': 'application/json' },        body: JSON.stringify({          error: 'Geçersiz request',          details: error.errors,        }),      };    }
    return {      statusCode: 500,      headers: { 'Content-Type': 'application/json' },      body: JSON.stringify({        error: 'Signed URL generate edilemedi',      }),    };  }};

// hooks/useFileUpload.tsimport { useState, useCallback } from 'react';
interface UploadProgress {  loaded: number;  total: number;  percentage: number;}
interface UseFileUploadReturn {  upload: (file: File) => Promise<string>;  progress: UploadProgress | null;  isUploading: boolean;  error: string | null;}
export const useFileUpload = (): UseFileUploadReturn => {  const [progress, setProgress] = useState<UploadProgress | null>(null);  const [isUploading, setIsUploading] = useState(false);  const [error, setError] = useState<string | null>(null);
  const upload = useCallback(async (file: File): Promise<string> => {    setIsUploading(true);    setError(null);    setProgress(null);
    try {      console.log('Dosya için upload başlatılıyor:', {        name: file.name,        size: file.size,        type: file.type,      });
      // Adım 1: Signed URL iste      const signedUrlResponse = await fetch('/api/uploads/signed-url', {        method: 'POST',        headers: {          'Content-Type': 'application/json',        },        body: JSON.stringify({          fileName: file.name,          fileSize: file.size,          fileType: file.type,          uploadId: crypto.randomUUID(),        }),      });
      if (!signedUrlResponse.ok) {        const errorData = await signedUrlResponse.json();        throw new Error(errorData.error || 'Signed URL alınamadı');      }
      const { signedUrl, key, headers } = await signedUrlResponse.json();
      console.log('Signed URL alındı, direkt S3 upload başlatılıyor');
      // Adım 2: Progress tracking ile direkt S3'e upload      const uploadResponse = await fetch(signedUrl, {        method: 'PUT',        headers: {          'Content-Type': file.type,          'Content-Length': file.size.toString(),          ...headers,        },        body: file,      });
      if (!uploadResponse.ok) {        throw new Error(`Upload başarısız: ${uploadResponse.status} ${uploadResponse.statusText}`);      }
      console.log('Upload başarıyla tamamlandı');
      return key; // Referans için S3 object key'i döndür
    } catch (err) {      const errorMessage = err instanceof Error ? err.message : 'Upload başarısız';      setError(errorMessage);      console.error('Upload hatası:', err);      throw err;    } finally {      setIsUploading(false);      setProgress(null);    }  }, []);
  return {    upload,    progress,    isUploading,    error,  };};

// Client-side validation'a asla tek başına güvenmeconst ALLOWED_MIME_TYPES = {  'image/jpeg': [0xFF, 0xD8, 0xFF],  'image/png': [0x89, 0x50, 0x4E, 0x47],  'video/mp4': [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70],} as const;
function validateFileType(buffer: Buffer, declaredType: string): boolean {  const signature = ALLOWED_MIME_TYPES[declaredType as keyof typeof ALLOWED_MIME_TYPES];  if (!signature) return false;
  return signature.every((byte, index) => buffer[index] === byte);}
// Processing Lambda'nızdaconst fileBuffer = await s3Client.send(new GetObjectCommand({  Bucket: bucketName,  Key: objectKey,  Range: 'bytes=0-10', // Signature kontrolü için sadece ilk birkaç byte'ı al}));
const isValidType = validateFileType(fileBuffer.Body as Buffer, contentType);if (!isValidType) {  throw new Error('Dosya tipi validation başarısız');}

// Metadata'lı genişletilmiş request şemasıconst FileUploadWithMetadataSchema = z.object({  // Dosya bilgisi  fileName: z.string().min(1).max(255),  fileSize: z.number().int().min(1).max(10737418240),  fileType: z.string().regex(/^(video|image|audio)\/[a-zA-Z0-9][a-zA-Z0-9\!\-\_]*[a-zA-Z0-9]*$/),
  // İş metadata'sı  title: z.string().min(1).max(200),  description: z.string().max(1000).optional(),  category: z.enum(['education', 'entertainment', 'business', 'other']),  tags: z.array(z.string()).max(10),  isPublic: z.boolean(),
  // Upload metadata'sı  uploadId: z.string().uuid(),  userId: z.string().uuid(),  organizationId: z.string().uuid().optional(),});
// S3 objesine metadata dahil etmek için değiştirilmiş signed URL generatorconst putObjectCommand = new PutObjectCommand({  Bucket: process.env.UPLOAD_BUCKET!,  Key: key,  ContentType: request.fileType,  ContentLength: request.fileSize,  Metadata: {    'original-filename': request.fileName,    'upload-id': request.uploadId,    'user-id': request.userId,    'title': request.title,    'description': request.description || '',    'category': request.category,    'tags': JSON.stringify(request.tags),    'is-public': request.isPublic.toString(),    'uploaded-at': new Date().toISOString(),  },  // Daha iyi organizasyon ve faturalandırma için object tag'leri ekle  Tagging: `Category=${request.category}&IsPublic=${request.isPublic}&UserId=${request.userId}`,});

// Kapsamlı lifecycle management ile genişletilmiş CDK stackconst uploadBucket = new s3.Bucket(this, 'UploadBucket', {  lifecycleRules: [    // Kural 1: Başarısız multipart upload'ları temizle    {      id: 'CleanupFailedUploads',      enabled: true,      abortIncompleteMultipartUploadsAfter: cdk.Duration.days(1),    },
    // Kural 2: Erişim patterns'a göre transition    {      id: 'StorageClassTransitions',      enabled: true,      transitions: [        {          storageClass: s3.StorageClass.INFREQUENT_ACCESS,          transitionAfter: cdk.Duration.days(30),        },        {          storageClass: s3.StorageClass.GLACIER,          transitionAfter: cdk.Duration.days(90),        },        {          storageClass: s3.StorageClass.DEEP_ARCHIVE,          transitionAfter: cdk.Duration.days(365),        },      ],    },
    // Kural 3: Geçici/processing dosyalarını sil    {      id: 'CleanupTempFiles',      enabled: true,      filter: s3.LifecycleFilter.prefix('temp/'),      expiration: cdk.Duration.days(7),    },
    // Kural 4: Kullanıcı-spesifik retention (örnek: ücretsiz tier kullanıcıları)    {      id: 'FreeTierRetention',      enabled: true,      filter: s3.LifecycleFilter.tag('UserTier', 'free'),      expiration: cdk.Duration.days(90),    },  ],
  // Versioning'i yanlışlıkla silinme koruması için etkinleştir  versioned: true,});

// Kullanıcı silme isteklerini işleyen Lambdaexport const deleteFileHandler = async (event: APIGatewayProxyEvent): Promise<APIGatewayProxyResult> => {  try {    const { fileId } = JSON.parse(event.body || '{}');    const userId = getUserIdFromJWT(event.headers.authorization);
    // Sahiplik doğrula    const fileRecord = await dynamoClient.send(new GetItemCommand({      TableName: process.env.UPLOAD_RECORDS_TABLE!,      Key: marshall({ id: fileId }),    }));
    if (!fileRecord.Item) {      return { statusCode: 404, body: JSON.stringify({ error: 'Dosya bulunamadı' }) };    }
    const file = unmarshall(fileRecord.Item);    if (file.userId !== userId) {      return { statusCode: 403, body: JSON.stringify({ error: 'Erişim reddedildi' }) };    }
    // Önce soft delete (silinmiş olarak işaretle, ama S3'den hemen kaldırma)    await dynamoClient.send(new UpdateItemCommand({      TableName: process.env.UPLOAD_RECORDS_TABLE!,      Key: marshall({ id: fileId }),      UpdateExpression: 'SET #status = :status, #deletedAt = :deletedAt',      ExpressionAttributeNames: {        '#status': 'status',        '#deletedAt': 'deletedAt',      },      ExpressionAttributeValues: marshall({        ':status': 'deleted',        ':deletedAt': new Date().toISOString(),      }),    }));
    // Grace period'dan sonra lifecycle cleanup için S3 delete tag'i ekle    await s3Client.send(new PutObjectTaggingCommand({      Bucket: process.env.UPLOAD_BUCKET!,      Key: file.s3Key,      Tagging: {        TagSet: [          { Key: 'Status', Value: 'deleted' },          { Key: 'DeletedAt', Value: new Date().toISOString() },          { Key: 'GracePeriodDays', Value: '30' },        ],      },    }));
    return {      statusCode: 200,      body: JSON.stringify({        message: 'Dosya silme için planlandı',        gracePeriod: '30 gün',      }),    };
  } catch (error) {    console.error('Dosya silme hatası:', error);    return { statusCode: 500, body: JSON.stringify({ error: 'Silme başarısız' }) };  }};

// Kullanıcı veri export'u için Lambda (GDPR Madde 20)export const exportUserDataHandler = async (event: APIGatewayProxyEvent) => {  const userId = getUserIdFromJWT(event.headers.authorization);
  // Kullanıcının tüm dosyalarını al  const userFiles = await dynamoClient.send(new ScanCommand({    TableName: process.env.UPLOAD_RECORDS_TABLE!,    FilterExpression: '#userId = :userId',    ExpressionAttributeNames: { '#userId': 'userId' },    ExpressionAttributeValues: marshall({ ':userId': userId }),  }));
  // Tüm dosyalar için download link'leri generate et  const fileExports = await Promise.all(    (userFiles.Items || []).map(async (item) => {      const file = unmarshall(item);
      if (file.status !== 'completed') return null;
      // Geçici download URL generate et      const downloadUrl = await getSignedUrl(        s3Client,        new GetObjectCommand({          Bucket: process.env.UPLOAD_BUCKET!,          Key: file.s3Key,        }),        { expiresIn: 3600 * 24 } // 24 saat      );
      return {        fileId: file.id,        originalFileName: file.fileName,        title: file.title,        description: file.description,        category: file.category,        tags: file.tags,        uploadedAt: file.createdAt,        fileSize: file.fileSize,        downloadUrl,      };    })  );
  const exportData = {    userId,    exportedAt: new Date().toISOString(),    files: fileExports.filter(Boolean),    summary: {      totalFiles: fileExports.filter(Boolean).length,      totalSize: fileExports.reduce((sum, file) => sum + (file?.fileSize || 0), 0),    },  };
  return {    statusCode: 200,    body: JSON.stringify({      exportUrl: exportDownloadUrl,      expiresAt: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString(),      summary: exportData.summary,    }),  };};

// Kullanım patterns'ı analiz edip storage class'ları optimize eden Lambdaexport const optimizeStorageHandler = async (event: ScheduledEvent) => {  const s3Inventory = await getS3Inventory(); // S3 inventory report'larından
  for (const object of s3Inventory) {    const lastAccessed = await getObjectAccessTime(object.key);    const daysSinceAccess = (Date.now() - lastAccessed) / (1000 * 60 * 60 * 24);
    // Erişim patterns'a göre otomatik transition    if (daysSinceAccess > 90 && object.storageClass === 'STANDARD') {      await s3Client.send(new CopyObjectCommand({        CopySource: `${object.bucket}/${object.key}`,        Bucket: object.bucket,        Key: object.key,        StorageClass: 'GLACIER',        MetadataDirective: 'COPY',      }));
      console.log('Glacier\'a geçirildi:', { key: object.key, daysSinceAccess });    }
    // Çok eski dosyalar için deep archive    if (daysSinceAccess > 365 && object.storageClass === 'GLACIER') {      await s3Client.send(new CopyObjectCommand({        CopySource: `${object.bucket}/${object.key}`,        Bucket: object.bucket,        Key: object.key,        StorageClass: 'DEEP_ARCHIVE',        MetadataDirective: 'COPY',      }));
      console.log('Deep Archive\'a geçirildi:', { key: object.key, daysSinceAccess });    }    }};