Large File Upload Optimization Strategies

// Frontend - Chunked Upload
async function uploadLargeFile(file, chunkSize = 2 * 1024 * 1024) { // 2MB chunks
  const totalChunks = Math.ceil(file.size / chunkSize);
  const uploadId = generateUniqueId();
  
  for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
    const start = chunkIndex * chunkSize;
    const end = Math.min(start + chunkSize, file.size);
    const chunk = file.slice(start, end);
    
    await uploadChunk(chunk, chunkIndex, totalChunks, uploadId);
    updateProgress((chunkIndex + 1) / totalChunks * 100);
  }
  
  // Finalize upload
  const fileUrl = await finalizeUpload(uploadId, file.name);
  return fileUrl;
}

async function uploadChunk(chunk, chunkIndex, totalChunks, uploadId) {
  const formData = new FormData();
  formData.append('chunk', chunk);
  formData.append('chunkIndex', chunkIndex);
  formData.append('totalChunks', totalChunks);
  formData.append('uploadId', uploadId);
  
  const response = await fetch('/api/upload-chunk', {
    method: 'POST',
    body: formData
  });
  
  if (!response.ok) {
    throw new Error(`Chunk ${chunkIndex} upload failed`);
  }
}

// Start upload immediately on file selection
document.getElementById('fileInput').addEventListener('change', (e) => {
  const file = e.target.files[0];
  if (file) {
    // Start background upload
    const uploadPromise = startBackgroundUpload(file);
    
    // Store upload promise reference
    window.pendingUpload = uploadPromise;
    
    // Show upload progress
    showUploadProgress(uploadPromise);
  }
});

// Check upload status on page B
async function checkUploadStatus() {
  const uploadId = localStorage.getItem('currentUploadId');
  if (uploadId) {
    const status = await fetch(`/api/upload-status/${uploadId}`);
    const { isComplete, progress, fileUrl } = await status.json();
    
    if (isComplete) {
      return fileUrl;
    } else {
      // Still uploading, show progress
      showUploadProgress(progress);
      return null;
    }
  }
}

// Using JSZip for compression
async function compressAndUpload(file) {
  const zip = new JSZip();
  zip.file(file.name, file);
  
  const compressedBlob = await zip.generateAsync({
    type: "blob",
    compression: "DEFLATE",
    compressionOptions: { level: 6 }
  });
  
  console.log(`Original: ${file.size} bytes, Compressed: ${compressedBlob.size} bytes`);
  
  return await uploadFile(compressedBlob, file.name + '.zip');
}

// Get presigned URL from your server
async function getPresignedUrl(fileName, fileType) {
  const response = await fetch('/api/presigned-url', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ fileName, fileType })
  });
  return await response.json();
}

// Upload directly to S3
async function uploadToS3(file) {
  const { uploadUrl, fileKey } = await getPresignedUrl(file.name, file.type);
  
  const response = await fetch(uploadUrl, {
    method: 'PUT',
    body: file,
    headers: {
      'Content-Type': file.type
    }
  });
  
  if (response.ok) {
    return fileKey; // Use this to retrieve file later
  }
}

class UploadQueue {
  constructor() {
    this.queue = [];
    this.activeUploads = 0;
    this.maxConcurrent = 2;
  }
  
  async addFile(file) {
    const uploadTask = {
      file,
      id: generateUniqueId(),
      status: 'pending',
      progress: 0,
      retries: 0
    };
    
    this.queue.push(uploadTask);
    this.processQueue();
    
    return uploadTask.id;
  }
  
  async processQueue() {
    if (this.activeUploads >= this.maxConcurrent) return;
    
    const task = this.queue.find(t => t.status === 'pending');
    if (!task) return;
    
    task.status = 'uploading';
    this.activeUploads++;
    
    try {
      await this.uploadWithRetry(task);
      task.status = 'completed';
    } catch (error) {
      task.status = 'failed';
      console.error('Upload failed:', error);
    } finally {
      this.activeUploads--;
      this.processQueue(); // Process next item
    }
  }
  
  async uploadWithRetry(task, maxRetries = 3) {
    while (task.retries < maxRetries) {
      try {
        return await this.uploadFile(task);
      } catch (error) {
        task.retries++;
        if (task.retries >= maxRetries) throw error;
        
        // Exponential backoff
        await new Promise(resolve => 
          setTimeout(resolve, Math.pow(2, task.retries) * 1000)
        );
      }
    }
  }
}

class SmartFileUploader {
  constructor() {
    this.chunkSize = 2 * 1024 * 1024; // 2MB
    this.maxRetries = 3;
    this.compressionThreshold = 10 * 1024 * 1024; // 10MB
  }
  
  async uploadFile(file) {
    // Step 1: Check if compression would help
    const shouldCompress = file.size > this.compressionThreshold && 
                          this.canCompress(file.type);
    
    let fileToUpload = file;
    if (shouldCompress) {
      fileToUpload = await this.compressFile(file);
    }
    
    // Step 2: Choose upload strategy based on file size
    if (fileToUpload.size > 5 * 1024 * 1024) { // 5MB
      return await this.chunkedUpload(fileToUpload, file.name);
    } else {
      return await this.simpleUpload(fileToUpload);
    }
  }
  
  async chunkedUpload(file, originalName) {
    const uploadId = generateUniqueId();
    const totalChunks = Math.ceil(file.size / this.chunkSize);
    
    // Store upload metadata
    localStorage.setItem('uploadProgress', JSON.stringify({
      uploadId,
      totalChunks,
      completedChunks: 0,
      originalName
    }));
    
    for (let i = 0; i < totalChunks; i++) {
      await this.uploadChunkWithRetry(file, i, totalChunks, uploadId);
      
      // Update progress
      const progress = JSON.parse(localStorage.getItem('uploadProgress'));
      progress.completedChunks = i + 1;
      localStorage.setItem('uploadProgress', JSON.stringify(progress));
      
      // Emit progress event
      this.onProgress && this.onProgress((i + 1) / totalChunks * 100);
    }
    
    // Finalize upload
    const result = await this.finalizeUpload(uploadId, originalName);
    localStorage.removeItem('uploadProgress');
    
    return result;
  }
  
  // Resume interrupted upload
  async resumeUpload() {
    const progress = JSON.parse(localStorage.getItem('uploadProgress'));
    if (!progress) return null;
    
    const { uploadId, totalChunks, completedChunks, originalName } = progress;
    
    // Check server status
    const serverStatus = await this.checkUploadStatus(uploadId);
    if (serverStatus.completed) {
      localStorage.removeItem('uploadProgress');
      return serverStatus.fileUrl;
    }
    
    // Resume from where we left off
    // Implementation continues...
  }
}