crm.clientright.ru/crm_extensions/file_storage/auto_migrate_to_s3.php

<?php
/**
 * Auto Migrate Local Files to S3
 * 
 * Этот скрипт автоматически находит новые локальные файлы и переносит их в S3
 * Можно запускать по cron или вызывать после создания файлов
 */

ini_set('memory_limit', '512M');
set_time_limit(300); // 5 минут максимум
date_default_timezone_set('Europe/Moscow');

$ROOT = '/var/www/fastuser/data/www/crm.clientright.ru/';
require_once $ROOT . 'config.inc.php';
require_once $ROOT . 'crm_extensions/file_storage/FilePathManager.php';

// CLI options
$opts = getopt('', [
    'limit::',
    'age-minutes::',
    'dry-run::',
    'force::'
]);

$limit = isset($opts['limit']) ? (int)$opts['limit'] : 50;
$ageMinutes = isset($opts['age-minutes']) ? (int)$opts['age-minutes'] : 5; // Файлы младше 5 минут
$dryRun = isset($opts['dry-run']) ? (int)$opts['dry-run'] !== 0 : false;
$force = isset($opts['force']) ? (int)$opts['force'] !== 0 : false;

// Database connection
$mysqli = new mysqli($dbconfig['db_server'], $dbconfig['db_username'], $dbconfig['db_password'], $dbconfig['db_name']);
if ($mysqli->connect_error) {
    die("Connection failed: " . $mysqli->connect_error);
}
$mysqli->set_charset("utf8");

// Logging
$logDir = $ROOT . 'logs/';
if (!is_dir($logDir)) {
    mkdir($logDir, 0777, true);
}
$logFile = $logDir . 'auto_s3_migration.log';

function logMessage($message, $toConsole = true) {
    global $logFile;
    $logLine = date('[Y-m-d H:i:s] ') . $message . "\n";
    file_put_contents($logFile, $logLine, FILE_APPEND);
    if ($toConsole) {
        echo $logLine;
    }
}

logMessage("=== Auto S3 Migration Started ===");
logMessage("Mode: " . ($dryRun ? "DRY RUN" : "LIVE"));
logMessage("Limit: $limit files");
logMessage("Age filter: files created in last $ageMinutes minutes");

// Find recent local files that need migration
$ageTimestamp = date('Y-m-d H:i:s', time() - ($ageMinutes * 60));

$query = "SELECT n.notesid, n.filename, n.filesize, n.filetype, c.createdtime
          FROM vtiger_notes n
          LEFT JOIN vtiger_crmentity c ON n.notesid = c.crmid
          WHERE n.filelocationtype = 'I'
          AND (n.s3_key IS NULL OR n.s3_key = '')
          AND n.filesize > 0
          AND c.createdtime >= ?
          ORDER BY c.createdtime DESC
          LIMIT ?";

$stmt = $mysqli->prepare($query);
$stmt->bind_param('si', $ageTimestamp, $limit);
$stmt->execute();
$result = $stmt->get_result();

$totalFiles = $result->num_rows;
logMessage("Found $totalFiles local files for migration");

if ($totalFiles === 0) {
    logMessage("No files to migrate. Exiting.");
    exit(0);
}

// Load S3 service
require_once $ROOT . 'include/Storage/S3StorageService.php';
$s3Service = new S3StorageService();
$s3Bucket = 'f9825c87-4e3558f6-f9b6-405c-ad3d-d1535c49b61c';

$migrated = 0;
$failed = 0;
$skipped = 0;

while ($row = $result->fetch_assoc()) {
    $notesid = $row['notesid'];
    $filename = $row['filename'];
    $filesize = $row['filesize'];
    $filetype = $row['filetype'];
    $createdtime = $row['createdtime'];
    
    logMessage("Processing file ID $notesid: $filename ($filesize bytes)");
    
    // Try to find the physical file
    $localFilePath = null;
    $possiblePaths = [
        $ROOT . 'storage/' . $filename,
        $ROOT . $filename,
    ];
    
    // Try structured storage paths (year/month/week)
    for ($year = 2023; $year <= date('Y'); $year++) {
        $yearDir = $ROOT . 'storage/' . $year . '/';
        if (is_dir($yearDir)) {
            $months = glob($yearDir . '*', GLOB_ONLYDIR);
            foreach ($months as $monthDir) {
                $weeks = glob($monthDir . '/*', GLOB_ONLYDIR);
                foreach ($weeks as $weekDir) {
                    $possiblePaths[] = $weekDir . '/' . $filename;
                }
                $possiblePaths[] = $monthDir . '/' . $filename;
            }
            $possiblePaths[] = $yearDir . $filename;
        }
    }
    
    // Find the actual file
    foreach ($possiblePaths as $path) {
        if (file_exists($path) && is_readable($path)) {
            $localFilePath = $path;
            break;
        }
    }
    
    if (!$localFilePath) {
        logMessage("  ❌ Physical file not found. Skipping.");
        $skipped++;
        continue;
    }
    
    logMessage("  📁 Found at: $localFilePath");
    
    // Verify file size matches
    $actualSize = filesize($localFilePath);
    if ($actualSize != $filesize) {
        logMessage("  ⚠️  Size mismatch: DB=$filesize, Actual=$actualSize");
    }
    
    if (!$dryRun) {
        try {
            // Upload to S3
            $s3Key = 'crm2/CRM_Active_Files/Documents/' . $notesid . '/' . $filename;
            
            logMessage("  📤 Uploading to S3: $s3Key");
            $s3Result = $s3Service->put($s3Key, $localFilePath);
            
            if ($s3Result && isset($s3Result['ObjectURL'])) {
                $s3Url = $s3Result['ObjectURL'];
                $s3Etag = isset($s3Result['ETag']) ? trim($s3Result['ETag'], '"') : '';
                
                // Create backup before updating DB
                $backupDir = $ROOT . 'crm_extensions/file_storage/backups/auto_migration/';
                if (!is_dir($backupDir)) {
                    mkdir($backupDir, 0777, true);
                }
                $backupFile = $backupDir . "auto_migrate_backup_{$notesid}_" . date('Ymd_His') . ".json";
                file_put_contents($backupFile, json_encode($row, JSON_PRETTY_PRINT));
                
                // Update database
                $updateQuery = "UPDATE vtiger_notes SET 
                               filename = ?, 
                               filelocationtype = 'E',
                               s3_key = ?,
                               s3_bucket = ?,
                               s3_etag = ?
                               WHERE notesid = ?";
                
                $updateStmt = $mysqli->prepare($updateQuery);
                $updateStmt->bind_param('ssssi', $s3Url, $s3Key, $s3Bucket, $s3Etag, $notesid);
                
                if ($updateStmt->execute()) {
                    logMessage("  ✅ Database updated");
                    
                    // Remove local file after successful migration
                    if (unlink($localFilePath)) {
                        logMessage("  🗑️  Local file removed");
                    } else {
                        logMessage("  ⚠️  Could not remove local file");
                    }
                    
                    $migrated++;
                } else {
                    logMessage("  ❌ Database update failed: " . $updateStmt->error);
                    $failed++;
                }
                $updateStmt->close();
                
            } else {
                logMessage("  ❌ S3 upload failed");
                $failed++;
            }
            
        } catch (Exception $e) {
            logMessage("  ❌ Migration error: " . $e->getMessage());
            $failed++;
        }
    } else {
        logMessage("  [DRY RUN] Would migrate to S3");
        $migrated++; // Count for dry run
    }
    
    // Small delay to avoid overwhelming the system
    usleep(100000); // 0.1 second
}

$stmt->close();
$mysqli->close();

logMessage("=== Migration Summary ===");
logMessage("Total files processed: $totalFiles");
logMessage("Successfully migrated: $migrated");
logMessage("Failed: $failed");
logMessage("Skipped (file not found): $skipped");
logMessage("Dry run: " . ($dryRun ? "YES" : "NO"));

if (!$dryRun && $migrated > 0) {
    logMessage("🎉 Auto migration completed successfully!");
}

logMessage("=== Auto S3 Migration Finished ===");
?>