Changeset 3420528
- Timestamp:
- 12/15/2025 10:04:54 PM (3 months ago)
- Location:
- siteskite/trunk
- Files:
-
- 1 added
- 11 edited
-
includes/API/RestAPI.php (modified) (2 diffs)
-
includes/Backup/BackupManager.php (modified) (44 diffs)
-
includes/Backup/Incremental/Impl/BackblazeB2ObjectStore.php (modified) (4 diffs)
-
includes/Backup/Incremental/ProviderManifests.php (modified) (1 diff)
-
includes/Cloud/BackblazeB2Manager.php (modified) (1 diff)
-
includes/Core/Plugin.php (modified) (7 diffs)
-
includes/Core/Utils.php (modified) (1 diff)
-
includes/Cron/CronTriggerHandler.php (modified) (8 diffs)
-
includes/Cron/ExternalCronManager.php (modified) (11 diffs)
-
includes/Cron/HybridCronManager.php (added)
-
readme.txt (modified) (3 diffs)
-
siteskite-link.php (modified) (2 diffs)
Legend:
- Unmodified
- Added
- Removed
-
siteskite/trunk/includes/API/RestAPI.php
r3418578 r3420528 632 632 'methods' => WP_REST_Server::CREATABLE, 633 633 'callback' => [$this->cron_trigger_handler, 'handle_trigger'], 634 'permission_callback' => '__return_true', // Authentication via secret token in header 634 'permission_callback' => function(\WP_REST_Request $request) { 635 // Enhanced security: Verify secret token before allowing access 636 // This is checked in handle_trigger, but we add an extra layer here 637 $secret = $request->get_header('X-SiteSkite-Cron-Secret'); 638 if (empty($secret)) { 639 // Also check query parameter as fallback (for services that can't set headers) 640 $secret = $request->get_param('siteskite_key'); 641 } 642 643 // If no secret provided, deny access immediately 644 if (empty($secret)) { 645 return false; 646 } 647 648 // Actual verification happens in handle_trigger for proper error handling 649 // This permission callback just ensures some form of authentication is attempted 650 return true; 651 }, 635 652 // Don't validate args here - we handle validation in the callback 636 653 // This allows cron-job.org to send JSON body without WordPress REST API validation issues 637 654 'args' => [] 655 ] 656 ); 657 658 // Cron continuity check endpoint (for external cron service to check if WordPress cron is running) 659 register_rest_route( 660 self::API_NAMESPACE, 661 '/cron/check-continuity', 662 [ 663 'methods' => WP_REST_Server::READABLE, 664 'callback' => [$this->cron_trigger_handler, 'handle_continuity_check'], 665 'permission_callback' => function(\WP_REST_Request $request) { 666 // Enhanced security: Verify secret token before allowing access 667 $secret = $request->get_header('X-SiteSkite-Cron-Secret'); 668 if (empty($secret)) { 669 $secret = $request->get_param('siteskite_key'); 670 } 671 672 if (empty($secret)) { 673 return false; 674 } 675 676 return true; 677 }, 678 'args' => [ 679 'action' => [ 680 'required' => true, 681 'type' => 'string', 682 'sanitize_callback' => 'sanitize_text_field' 683 ], 684 'args' => [ 685 'required' => false, 686 'type' => 'array', 687 'default' => [] 688 ], 689 'siteskite_key' => [ 690 'required' => false, 691 'type' => 'string', 692 'sanitize_callback' => 'sanitize_text_field', 693 'description' => 'Alternative authentication method if header cannot be set' 694 ] 695 ] 638 696 ] 639 697 ); … … 703 761 // Log what we received for debugging 704 762 $this->logger->debug('Backup start request parameters check', [ 705 'backup_id' => $backup_id,706 'provider' => $provider,707 'has_provider_config' => is_array($request->get_param('provider_config')),708 'provider_config_type' => gettype($request->get_param('provider_config')),709 'provider_config_keys' => is_array($request->get_param('provider_config')) ? array_keys($request->get_param('provider_config')) : null,710 'received_credential_keys' => is_array($provider_credentials) ? array_keys($provider_credentials) : null,711 'all_param_keys' => array_keys($all_params),712 'has_credentials' => is_array($provider_credentials) && !empty($provider_credentials)763 // 'backup_id' => $backup_id, 764 // 'provider' => $provider, 765 // 'has_provider_config' => is_array($request->get_param('provider_config')), 766 // 'provider_config_type' => gettype($request->get_param('provider_config')), 767 // 'provider_config_keys' => is_array($request->get_param('provider_config')) ? array_keys($request->get_param('provider_config')) : null, 768 // 'received_credential_keys' => is_array($provider_credentials) ? array_keys($provider_credentials) : null, 769 // 'all_param_keys' => array_keys($all_params), 770 // 'has_credentials' => is_array($provider_credentials) && !empty($provider_credentials) 713 771 ]); 714 772 -
siteskite/trunk/includes/Backup/BackupManager.php
r3418578 r3420528 137 137 private ?ExternalCronManager $external_cron_manager = null; 138 138 139 /** 140 * @var \SiteSkite\Cron\HybridCronManager|null Hybrid cron manager instance 141 */ 142 private ?\SiteSkite\Cron\HybridCronManager $hybrid_cron_manager = null; 143 139 144 // Backup directory is now accessed via SITESKITE_BACKUP_PATH constant 140 145 … … 154 159 'litespeed', 155 160 'backup-migration-*', 161 162 156 163 'wp-content/cache', 157 164 'wp-content/uploads/siteskite-backups', … … 170 177 'wp-content/wpvividbackups', 171 178 172 //temporary test 173 'wp-content/plugins', 174 'wp-admin', 175 'wp-includes', 176 179 180 181 /* --- excluded paths for testing --- */ 182 // 'wp-content', 183 // 'wp-admin', 184 // 'wp-includes', 177 185 178 186 ]; … … 198 206 * @param \SiteSkite\Backup\Incremental\IncrementalStatus $incremental_status Incremental status manager instance 199 207 * @param ExternalCronManager|null $external_cron_manager External cron manager instance 208 * @param \SiteSkite\Cron\HybridCronManager|null $hybrid_cron_manager Hybrid cron manager instance 200 209 */ 201 210 public function __construct( … … 211 220 AWSManager $aws_manager, 212 221 \SiteSkite\Backup\Incremental\IncrementalStatus $incremental_status, 213 ?ExternalCronManager $external_cron_manager = null 222 ?ExternalCronManager $external_cron_manager = null, 223 ?\SiteSkite\Cron\HybridCronManager $hybrid_cron_manager = null 214 224 ) { 215 225 $this->s3_manager = $s3_manager; … … 225 235 $this->incremental_status = $incremental_status; 226 236 $this->external_cron_manager = $external_cron_manager; 237 $this->hybrid_cron_manager = $hybrid_cron_manager; 227 238 $this->init_backup_directory(); 228 239 … … 278 289 279 290 /** 280 * Schedule cron event (uses external cron if enabled, otherwiseWordPress cron)291 * Schedule cron event (uses hybrid cron if available, otherwise external cron, fallback to WordPress cron) 281 292 */ 282 293 private function schedule_cron_event(string $hook, array $args, int $delay_seconds = 0, string $title = ''): void 283 294 { 284 // Try external cron first if enabled 295 // Determine WordPress cron interval based on hook type 296 $wp_cron_interval = 60; // Default 1 minute 297 if ($hook === 'siteskite_incremental_continue') { 298 $wp_cron_interval = 60; // 1 minute for incremental continuation 299 } elseif (strpos($hook, 'backup') !== false || strpos($hook, 'restore') !== false) { 300 $wp_cron_interval = 300; // 5 minutes for backup/restore operations 301 } 302 303 // Try hybrid cron first if available 304 if ($this->hybrid_cron_manager) { 305 $scheduled = $this->hybrid_cron_manager->schedule_hybrid_event( 306 $hook, 307 $args, 308 $delay_seconds, 309 $title, 310 $wp_cron_interval 311 ); 312 if ($scheduled) { 313 $this->logger->debug('Scheduled hybrid cron event', [ 314 'hook' => $hook, 315 'delay' => $delay_seconds, 316 'wp_cron_interval' => $wp_cron_interval 317 ]); 318 return; 319 } 320 } 321 322 // Fallback to original external cron logic 285 323 if ($this->external_cron_manager && $this->external_cron_manager->is_enabled()) { 286 324 // Check if job already exists to prevent duplicates … … 333 371 // Fallback to WordPress cron 334 372 wp_schedule_single_event(time() + $delay_seconds, $hook, $args); 373 374 // Notify hybrid cron manager if WordPress cron was scheduled 375 if ($this->hybrid_cron_manager) { 376 $this->hybrid_cron_manager->on_wp_cron_run($hook, $args); 377 } 378 } 379 380 /** 381 * Ensure only a single active incremental-continue job per site. 382 * Removes any WP-Cron and external-cron jobs for other incremental backup IDs. 383 */ 384 private function cleanup_other_incremental_jobs(string $active_backup_id): void 385 { 386 // Clean up external cron jobs for siteskite_incremental_continue on other backup IDs 387 if ($this->external_cron_manager && $this->external_cron_manager->is_enabled()) { 388 $jobs = get_option('siteskite_external_cron_jobs', []); 389 if (is_array($jobs)) { 390 foreach ($jobs as $job) { 391 if (($job['action'] ?? '') !== 'siteskite_incremental_continue') { 392 continue; 393 } 394 395 $args = $job['args'] ?? []; 396 $job_backup_id = $args[0] ?? ($args['backup_id'] ?? ''); 397 398 if (!empty($job_backup_id) && $job_backup_id !== $active_backup_id && isset($job['job_id'])) { 399 $this->external_cron_manager->delete_job((int) $job['job_id']); 400 } 401 } 402 } 403 } 404 405 // Clean up WordPress cron events for siteskite_incremental_continue on other backup IDs 406 if (function_exists('_get_cron_array')) { 407 $crons = _get_cron_array(); 408 if (is_array($crons)) { 409 foreach ($crons as $timestamp => $hooks) { 410 if (empty($hooks['siteskite_incremental_continue'])) { 411 continue; 412 } 413 414 foreach ($hooks['siteskite_incremental_continue'] as $event) { 415 $args = $event['args'] ?? []; 416 $event_backup_id = $args[0] ?? ($args['backup_id'] ?? ''); 417 418 if (!empty($event_backup_id) && $event_backup_id !== $active_backup_id) { 419 // Remove this scheduled event for the other backup ID 420 wp_unschedule_event((int) $timestamp, 'siteskite_incremental_continue', $args); 421 } 422 } 423 } 424 } 425 } 335 426 } 336 427 … … 384 475 385 476 update_option(self::OPTION_PREFIX . $backup_id, $backup_info); 477 478 // If incremental mode is enabled, ensure there are no stray incremental-continue jobs 479 // for previous backups on this site. Only one incremental chain should be active at a time. 480 if ($this->is_incremental_enabled()) { 481 $this->cleanup_other_incremental_jobs($backup_id); 482 } 386 483 387 484 // Schedule the actual backup process to run immediately after this request … … 2848 2945 $this->update_backup_status($backup_id, ['provider' => 'backblaze_b2']); 2849 2946 2947 // Ensure a persistent external cron job exists for incremental continuation of this backup 2948 if ($this->external_cron_manager && $this->external_cron_manager->is_enabled()) { 2949 $continueArgs = [$backup_id]; 2950 $existingJobId = $this->external_cron_manager->get_job_id_by_action('siteskite_incremental_continue', $continueArgs); 2951 if (!$existingJobId) { 2952 $jobId = $this->external_cron_manager->create_recurring_job( 2953 'siteskite_incremental_continue', 2954 $continueArgs, 2955 'every_minute', 2956 [], 2957 'SiteSkite Incremental Continue' 2958 ); 2959 2960 if ($jobId) { 2961 $this->logger->info('Scheduled persistent external cron for incremental continue at upload start', [ 2962 'backup_id' => $backup_id, 2963 'job_id' => $jobId 2964 ]); 2965 } else { 2966 $this->logger->warning('Failed to schedule persistent external cron for incremental continue at upload start', [ 2967 'backup_id' => $backup_id 2968 ]); 2969 } 2970 } else { 2971 $this->logger->debug('Persistent external cron for incremental continue already exists at upload start', [ 2972 'backup_id' => $backup_id, 2973 'job_id' => $existingJobId 2974 ]); 2975 } 2976 } 2977 2850 2978 try { 2851 2979 // Send notification … … 3593 3721 $bytesSinceSave += $chunkSize; 3594 3722 3595 // Save progress every 200 uploads 3596 if ($uploadedSinceSave >= 200) { 3723 // Save progress every 200 uploads (or every 50 uploads to reduce race conditions) 3724 // More frequent saves help prevent duplicate uploads when multiple processes run concurrently 3725 $saveInterval = 50; // Reduced from 200 to save more frequently 3726 if ($uploadedSinceSave >= $saveInterval) { 3597 3727 $coordinator->saveRemoteIndex($remoteIndex, $provider); 3598 3728 try { … … 3731 3861 if ($hash === '' || $size <= 0) { unset($deferred[$hash]); continue; } 3732 3862 if (isset($processedThisRun[$hash])) { unset($deferred[$hash]); continue; } 3863 // Check if chunk exists on cloud before retrying (prevents duplicate uploads) 3864 if (isset($remoteIndex[$hash])) { 3865 unset($deferred[$hash]); 3866 continue; 3867 } 3868 // Check if chunk exists on cloud even if not in remoteIndex (handles race conditions) 3869 if ($objectStore->exists($hash)) { 3870 $remoteIndex[$hash] = time(); 3871 $processedThisRun[$hash] = true; 3872 unset($deferred[$hash]); 3873 $this->logger->debug('Deferred chunk already exists on cloud, skipping upload', [ 3874 'backup_id' => $backup_id, 3875 'hash' => $hash 3876 ]); 3877 continue; 3878 } 3733 3879 if (!isset($remoteIndex[$hash])) { 3734 3880 // Find the file for this chunk and read the data … … 3796 3942 $objectStore->put($hash, $mem, strlen($chunkData)); 3797 3943 $chunkSize = strlen($chunkData); 3798 $uploadedBytes += $chunkSize; 3799 $uploadedSinceSave++; 3800 $bytesSinceSave += $chunkSize; 3944 // Only count bytes if chunk wasn't already uploaded (prevent double-counting) 3945 // Check remoteIndex to see if it was already uploaded in a previous cycle 3946 if (!isset($remoteIndex[$hash])) { 3947 $uploadedBytes += $chunkSize; 3948 $uploadedSinceSave++; 3949 $bytesSinceSave += $chunkSize; 3950 } 3801 3951 $remoteIndex[$hash] = time(); 3802 3952 $processedThisRun[$hash] = true; … … 3810 3960 } else { unset($deferred[$hash]); } 3811 3961 } 3812 if ($uploadedSinceSave >= 200) {3962 if ($uploadedSinceSave >= 50) { 3813 3963 $coordinator->saveRemoteIndex($remoteIndex, $provider); 3814 3964 try { if (function_exists('set_transient')) { set_transient($processedTransientKey, array_keys($processedThisRun), HOUR_IN_SECONDS); } } catch (\Throwable $e) { /* ignore */ } … … 3963 4113 $bytesSinceSave += $chunkSize; 3964 4114 4115 // Track database uploaded bytes separately for accurate size reporting 4116 $progress['db_uploaded_bytes'] = (int)($progress['db_uploaded_bytes'] ?? 0) + $chunkSize; 4117 3965 4118 // Track successfully uploaded chunk for cleanup 3966 4119 // For S3ObjectStore (async), we'll delete after waitForUploadsToComplete() … … 3982 4135 } 3983 4136 3984 // Save progress every 200 uploads3985 if ($uploadedSinceSave >= 200) {4137 // Save progress every 50 uploads (reduced from 200 to prevent duplicates) 4138 if ($uploadedSinceSave >= 50) { 3986 4139 $coordinator->saveRemoteIndex($remoteIndex, $provider); 3987 4140 try { … … 3993 4146 $progress = $this->get_incremental_progress($backup_id); 3994 4147 $progress['uploaded_bytes'] = (int) ($progress['uploaded_bytes'] ?? 0) + $bytesSinceSave; 4148 // Save db_uploaded_bytes if it was tracked during this batch 4149 if ($dbUploadedBytes > 0) { 4150 $progress['db_uploaded_bytes'] = (int)($progress['db_uploaded_bytes'] ?? 0) + $dbUploadedBytes; 4151 } 3995 4152 $this->update_incremental_progress($backup_id, $progress); 3996 4153 } catch (\Throwable $e) { /* ignore progress persisting */ } 3997 4154 $uploadedSinceSave = 0; 3998 4155 $bytesSinceSave = 0; 4156 $dbUploadedBytes = 0; // Reset for next batch 3999 4157 } 4000 4158 … … 4004 4162 'chunk_size' => $chunkSize 4005 4163 ]); 4164 } catch (\RuntimeException $e) { 4165 $message = $e->getMessage(); 4166 $this->logger->debug('Database chunk upload failed', [ 4167 'backup_id' => $backup_id, 4168 'provider' => $provider, 4169 'chunk_hash' => $chunkHash, 4170 'error' => $message 4171 ]); 4172 if (isset($mem) && is_resource($mem)) { 4173 fclose($mem); 4174 } 4175 // Centralized retry classification 4176 $action = \SiteSkite\Backup\Incremental\RetryPolicy::classify($provider, $e); 4177 if ($action === 'refresh') { 4178 // Request fresh credentials, persist current progress, and schedule a retry soon 4179 $this->refresh_credentials_for_incremental($backup_id, $provider); 4180 try { $coordinator->saveRemoteIndex($remoteIndex, $provider); } catch (\Throwable $t) { /* ignore */ } 4181 try { $this->update_incremental_progress($backup_id, $progress); } catch (\Throwable $t) { /* ignore */ } 4182 $this->schedule_cron_event('siteskite_incremental_continue', [$backup_id], 60, "SiteSkite Incremental Continue"); 4183 // Don't return here - continue processing other chunks 4184 } elseif ($action === 'mark_present') { 4185 // Chunk already exists on cloud, mark as present 4186 $remoteIndex[$chunkHash] = time(); 4187 $processedThisRun[$chunkHash] = true; 4188 $dbSkippedCount++; 4189 // Delete local chunk since it's already on cloud 4190 if (file_exists($chunkPath)) { 4191 wp_delete_file($chunkPath); 4192 } 4193 } elseif ($action === 'retry') { 4194 // Defer for retry - add to deferred_chunks in progress 4195 if (!isset($progress['deferred_db_chunks'])) { 4196 $progress['deferred_db_chunks'] = []; 4197 } 4198 $progress['deferred_db_chunks'][$chunkHash] = [ 4199 'chunk_path' => $chunkPath, 4200 'size' => strlen($chunkData), 4201 'retry_count' => (int)($progress['deferred_db_chunks'][$chunkHash]['retry_count'] ?? 0) + 1 4202 ]; 4203 $this->logger->debug('Deferred database chunk for retry', [ 4204 'backup_id' => $backup_id, 4205 'chunk_hash' => $chunkHash, 4206 'retry_count' => $progress['deferred_db_chunks'][$chunkHash]['retry_count'] 4207 ]); 4208 } else { 4209 // For fatal errors or skip, log and continue 4210 $this->logger->warning('Database chunk upload failed, skipping', [ 4211 'backup_id' => $backup_id, 4212 'chunk_hash' => $chunkHash, 4213 'error' => $message, 4214 'action' => $action 4215 ]); 4216 } 4006 4217 } catch (\Throwable $e) { 4007 $this->logger->error(' Failed to uploaddatabase chunk', [4218 $this->logger->error('Unexpected error uploading database chunk', [ 4008 4219 'backup_id' => $backup_id, 4009 4220 'chunk_hash' => $chunkHash, 4010 'error' => $e->getMessage() 4221 'error' => $e->getMessage(), 4222 'type' => get_class($e) 4011 4223 ]); 4224 // Defer for retry on unexpected errors 4225 if (!isset($progress['deferred_db_chunks'])) { 4226 $progress['deferred_db_chunks'] = []; 4227 } 4228 $progress['deferred_db_chunks'][$chunkHash] = [ 4229 'chunk_path' => $chunkPath, 4230 'size' => strlen($chunkData), 4231 'retry_count' => (int)($progress['deferred_db_chunks'][$chunkHash]['retry_count'] ?? 0) + 1 4232 ]; 4012 4233 } 4013 4234 } … … 4024 4245 $progress = $this->get_incremental_progress($backup_id); 4025 4246 $progress['uploaded_bytes'] = (int) ($progress['uploaded_bytes'] ?? 0) + $bytesSinceSave; 4247 // Save db_uploaded_bytes if it was tracked during this batch 4248 if ($dbUploadedBytes > 0) { 4249 $progress['db_uploaded_bytes'] = (int)($progress['db_uploaded_bytes'] ?? 0) + $dbUploadedBytes; 4250 } 4026 4251 $this->update_incremental_progress($backup_id, $progress); 4027 4252 } catch (\Throwable $e) { /* ignore progress persisting */ } 4028 4253 $bytesSinceSave = 0; 4254 $dbUploadedBytes = 0; // Reset for next batch 4029 4255 } 4030 4256 … … 4051 4277 } 4052 4278 4279 // Process deferred database chunks with retry logic 4280 $deferredDbChunks = $progress['deferred_db_chunks'] ?? []; 4281 if (!empty($deferredDbChunks)) { 4282 $this->logger->info('Processing deferred database chunks', [ 4283 'backup_id' => $backup_id, 4284 'deferred_count' => count($deferredDbChunks) 4285 ]); 4286 4287 $dbDeferredRetries = 0; 4288 while (!empty($deferredDbChunks) && $dbDeferredRetries < 3) { 4289 if (function_exists('get_option') && get_option('siteskite_disable_network_calls')) { break; } 4290 $dbDeferredRetries++; 4291 // Backoff grows: 2s, 4s, 8s 4292 sleep($dbDeferredRetries === 1 ? 2 : ($dbDeferredRetries === 2 ? 4 : 8)); 4293 4294 foreach ($deferredDbChunks as $chunkHash => $meta) { 4295 // Skip if already processed or in remote index 4296 if (isset($processedThisRun[$chunkHash]) || isset($remoteIndex[$chunkHash])) { 4297 unset($deferredDbChunks[$chunkHash]); 4298 continue; 4299 } 4300 4301 // Check if chunk exists on cloud before retrying 4302 if ($objectStore->exists($chunkHash)) { 4303 $remoteIndex[$chunkHash] = time(); 4304 $processedThisRun[$chunkHash] = true; 4305 $dbSkippedCount++; 4306 // Delete local chunk since it's already on cloud 4307 $chunkPath = $meta['chunk_path'] ?? $this->backup_config->getChunkPath($chunkHash); 4308 if (file_exists($chunkPath)) { 4309 wp_delete_file($chunkPath); 4310 } 4311 unset($deferredDbChunks[$chunkHash]); 4312 $this->logger->debug('Deferred database chunk already exists on cloud, skipping upload', [ 4313 'backup_id' => $backup_id, 4314 'chunk_hash' => $chunkHash 4315 ]); 4316 continue; 4317 } 4318 4319 $chunkPath = $meta['chunk_path'] ?? $this->backup_config->getChunkPath($chunkHash); 4320 $chunkSize = (int)($meta['size'] ?? 0); 4321 $retryCount = (int)($meta['retry_count'] ?? 0); 4322 4323 if (!file_exists($chunkPath) || $chunkSize <= 0) { 4324 $this->logger->warning('Deferred database chunk file missing or invalid, removing from deferred', [ 4325 'backup_id' => $backup_id, 4326 'chunk_hash' => $chunkHash, 4327 'chunk_path' => $chunkPath 4328 ]); 4329 unset($deferredDbChunks[$chunkHash]); 4330 continue; 4331 } 4332 4333 // Read chunk data 4334 $chunkData = file_get_contents($chunkPath); 4335 if ($chunkData === false || strlen($chunkData) !== $chunkSize) { 4336 $this->logger->warning('Failed to read deferred database chunk or size mismatch', [ 4337 'backup_id' => $backup_id, 4338 'chunk_hash' => $chunkHash, 4339 'expected_size' => $chunkSize, 4340 'actual_size' => $chunkData !== false ? strlen($chunkData) : 0 4341 ]); 4342 unset($deferredDbChunks[$chunkHash]); 4343 continue; 4344 } 4345 4346 // Retry upload 4347 try { 4348 $mem = fopen('php://temp', 'rb+'); 4349 fwrite($mem, $chunkData); 4350 rewind($mem); 4351 4352 $objectStore->put($chunkHash, $mem, strlen($chunkData)); 4353 4354 // Only count bytes if chunk wasn't already uploaded (prevent double-counting) 4355 // Check remoteIndex to see if it was already uploaded in a previous cycle 4356 $wasAlreadyUploaded = isset($remoteIndex[$chunkHash]); 4357 $remoteIndex[$chunkHash] = time(); 4358 $processedThisRun[$chunkHash] = true; 4359 4360 if (!$wasAlreadyUploaded) { 4361 $dbUploadedBytes += $chunkSize; 4362 $uploadedBytes += $chunkSize; 4363 $dbUploadedCount++; 4364 $uploadedSinceSave++; 4365 $bytesSinceSave += $chunkSize; 4366 4367 // Track database uploaded bytes (only count once) 4368 $progress['db_uploaded_bytes'] = (int)($progress['db_uploaded_bytes'] ?? 0) + $chunkSize; 4369 } else { 4370 // Chunk was already uploaded, just mark as processed 4371 $this->logger->debug('Deferred database chunk was already uploaded, skipping byte count', [ 4372 'backup_id' => $backup_id, 4373 'chunk_hash' => $chunkHash 4374 ]); 4375 } 4376 4377 // Delete local chunk after successful upload 4378 if (file_exists($chunkPath)) { 4379 wp_delete_file($chunkPath); 4380 } 4381 4382 unset($deferredDbChunks[$chunkHash]); 4383 4384 $this->logger->debug('Successfully retried deferred database chunk', [ 4385 'backup_id' => $backup_id, 4386 'chunk_hash' => $chunkHash, 4387 'retry_count' => $retryCount 4388 ]); 4389 } catch (\Throwable $e) { 4390 $this->logger->debug('Deferred database chunk retry failed', [ 4391 'backup_id' => $backup_id, 4392 'chunk_hash' => $chunkHash, 4393 'retry_count' => $retryCount, 4394 'error' => $e->getMessage() 4395 ]); 4396 // Keep in deferred for next retry attempt 4397 $deferredDbChunks[$chunkHash]['retry_count'] = $retryCount + 1; 4398 } 4399 } 4400 4401 // Save progress during retry loop 4402 if ($uploadedSinceSave >= 50) { 4403 $coordinator->saveRemoteIndex($remoteIndex, $provider); 4404 try { 4405 if (function_exists('set_transient')) { 4406 set_transient($processedTransientKey, array_keys($processedThisRun), HOUR_IN_SECONDS); 4407 } 4408 } catch (\Throwable $e) { /* ignore */ } 4409 try { 4410 $progress = $this->get_incremental_progress($backup_id); 4411 $progress['uploaded_bytes'] = (int) ($progress['uploaded_bytes'] ?? 0) + $bytesSinceSave; 4412 if ($dbUploadedBytes > 0) { 4413 $progress['db_uploaded_bytes'] = (int)($progress['db_uploaded_bytes'] ?? 0) + $dbUploadedBytes; 4414 } 4415 $progress['deferred_db_chunks'] = $deferredDbChunks; 4416 $this->update_incremental_progress($backup_id, $progress); 4417 } catch (\Throwable $e) { /* ignore progress persisting */ } 4418 $uploadedSinceSave = 0; 4419 $bytesSinceSave = 0; 4420 $dbUploadedBytes = 0; 4421 } 4422 } 4423 4424 // Update progress with remaining deferred chunks 4425 $progress['deferred_db_chunks'] = $deferredDbChunks; 4426 if (!empty($deferredDbChunks)) { 4427 $this->logger->info('Some deferred database chunks remain after retries', [ 4428 'backup_id' => $backup_id, 4429 'remaining_count' => count($deferredDbChunks) 4430 ]); 4431 } else { 4432 unset($progress['deferred_db_chunks']); 4433 } 4434 } 4435 4436 // Save final database uploaded bytes to progress 4437 if ($dbUploadedBytes > 0) { 4438 try { 4439 $progress = $this->get_incremental_progress($backup_id); 4440 $progress['db_uploaded_bytes'] = (int)($progress['db_uploaded_bytes'] ?? 0) + $dbUploadedBytes; 4441 $this->update_incremental_progress($backup_id, $progress); 4442 } catch (\Throwable $e) { /* ignore */ } 4443 } 4444 4445 // Comprehensive scan: Check for any remaining chunks in the chunks folder that need upload 4446 // This catches chunks that may have been created but not included in the manifest 4447 // or chunks from duplicate backup runs 4448 $this->logger->info('Scanning chunks folder for any remaining chunks to upload', [ 4449 'backup_id' => $backup_id, 4450 'provider' => $provider 4451 ]); 4452 4453 $chunksBaseDir = SITESKITE_BACKUP_PATH . '/chunks'; 4454 $remainingChunksUploaded = 0; 4455 $remainingChunksSkipped = 0; 4456 $remainingChunksBytes = 0; 4457 4458 if (is_dir($chunksBaseDir)) { 4459 // Scan all subdirectories in chunks folder (organized by first 2 chars of hash) 4460 $subdirs = glob($chunksBaseDir . '/*', GLOB_ONLYDIR); 4461 foreach ($subdirs as $subdir) { 4462 // Find all .blob files in this subdirectory 4463 $blobFiles = glob($subdir . '/*.blob'); 4464 foreach ($blobFiles as $blobFile) { 4465 // Extract hash from filename (filename is hash.blob) 4466 $filename = basename($blobFile, '.blob'); 4467 4468 // Validate hash format (should be 64 hex characters for SHA256) 4469 if (strlen($filename) !== 64 || !ctype_xdigit($filename)) { 4470 continue; 4471 } 4472 4473 $chunkHash = $filename; 4474 4475 // Skip if already processed or in remote index 4476 if (isset($processedThisRun[$chunkHash]) || isset($remoteIndex[$chunkHash])) { 4477 $remainingChunksSkipped++; 4478 continue; 4479 } 4480 4481 // Check if chunk exists on cloud 4482 try { 4483 if ($objectStore->exists($chunkHash)) { 4484 $remoteIndex[$chunkHash] = time(); 4485 $processedThisRun[$chunkHash] = true; 4486 $remainingChunksSkipped++; 4487 4488 // Delete local chunk since it's already on cloud 4489 if (file_exists($blobFile)) { 4490 wp_delete_file($blobFile); 4491 } 4492 continue; 4493 } 4494 } catch (\Throwable $e) { 4495 $this->logger->debug('Error checking if remaining chunk exists on cloud', [ 4496 'backup_id' => $backup_id, 4497 'chunk_hash' => $chunkHash, 4498 'error' => $e->getMessage() 4499 ]); 4500 continue; 4501 } 4502 4503 // Chunk exists locally but not on cloud - upload it 4504 try { 4505 $chunkData = file_get_contents($blobFile); 4506 if ($chunkData === false) { 4507 $this->logger->warning('Failed to read remaining chunk file', [ 4508 'backup_id' => $backup_id, 4509 'chunk_hash' => $chunkHash, 4510 'chunk_path' => $blobFile 4511 ]); 4512 continue; 4513 } 4514 4515 $mem = fopen('php://temp', 'rb+'); 4516 fwrite($mem, $chunkData); 4517 rewind($mem); 4518 4519 $objectStore->put($chunkHash, $mem, strlen($chunkData)); 4520 fclose($mem); 4521 4522 $remoteIndex[$chunkHash] = time(); 4523 $processedThisRun[$chunkHash] = true; 4524 $chunkSize = strlen($chunkData); 4525 $dbUploadedBytes += $chunkSize; 4526 $uploadedBytes += $chunkSize; 4527 $remainingChunksBytes += $chunkSize; 4528 $remainingChunksUploaded++; 4529 4530 // Track database uploaded bytes 4531 try { 4532 $progress = $this->get_incremental_progress($backup_id); 4533 $progress['db_uploaded_bytes'] = (int)($progress['db_uploaded_bytes'] ?? 0) + $chunkSize; 4534 $this->update_incremental_progress($backup_id, $progress); 4535 } catch (\Throwable $e) { /* ignore */ } 4536 4537 // Delete local chunk after successful upload 4538 if (file_exists($blobFile)) { 4539 wp_delete_file($blobFile); 4540 $this->logger->debug('Uploaded and deleted remaining chunk', [ 4541 'backup_id' => $backup_id, 4542 'chunk_hash' => $chunkHash, 4543 'chunk_size' => $chunkSize 4544 ]); 4545 } 4546 } catch (\Throwable $e) { 4547 $this->logger->warning('Failed to upload remaining chunk', [ 4548 'backup_id' => $backup_id, 4549 'chunk_hash' => $chunkHash, 4550 'chunk_path' => $blobFile, 4551 'error' => $e->getMessage() 4552 ]); 4553 } 4554 } 4555 } 4556 4557 // Save remote index after scanning 4558 if ($remainingChunksUploaded > 0 || $remainingChunksSkipped > 0) { 4559 $coordinator->saveRemoteIndex($remoteIndex, $provider); 4560 $this->logger->info('Completed scan of chunks folder', [ 4561 'backup_id' => $backup_id, 4562 'provider' => $provider, 4563 'remaining_chunks_uploaded' => $remainingChunksUploaded, 4564 'remaining_chunks_skipped' => $remainingChunksSkipped, 4565 'remaining_chunks_bytes' => $remainingChunksBytes 4566 ]); 4567 } 4568 } 4569 4053 4570 $this->logger->info('Database chunks processing completed', [ 4054 4571 'backup_id' => $backup_id, 4055 4572 'provider' => $provider, 4056 'uploaded_count' => $dbUploadedCount, 4057 'skipped_count' => $dbSkippedCount, 4573 'uploaded_count' => $dbUploadedCount + $remainingChunksUploaded, 4574 'skipped_count' => $dbSkippedCount + $remainingChunksSkipped, 4575 'remaining_chunks_uploaded' => $remainingChunksUploaded, 4058 4576 'uploaded_bytes' => $dbUploadedBytes 4059 4577 ]); … … 4069 4587 'manifest_id' => $dbManifestId 4070 4588 ]); 4071 } 4072 4589 4590 // Even without a manifest, scan for any remaining chunks that need upload 4591 // This handles cases where chunks were created but manifest wasn't saved 4592 $this->logger->info('Scanning chunks folder for orphaned chunks (no manifest found)', [ 4593 'backup_id' => $backup_id, 4594 'provider' => $provider 4595 ]); 4596 4597 $chunksBaseDir = SITESKITE_BACKUP_PATH . '/chunks'; 4598 $orphanedChunksUploaded = 0; 4599 $orphanedChunksSkipped = 0; 4600 4601 if (is_dir($chunksBaseDir)) { 4602 $subdirs = glob($chunksBaseDir . '/*', GLOB_ONLYDIR); 4603 foreach ($subdirs as $subdir) { 4604 $blobFiles = glob($subdir . '/*.blob'); 4605 foreach ($blobFiles as $blobFile) { 4606 $filename = basename($blobFile, '.blob'); 4607 if (strlen($filename) !== 64 || !ctype_xdigit($filename)) { 4608 continue; 4609 } 4610 4611 $chunkHash = $filename; 4612 4613 // Skip if already in remote index 4614 if (isset($remoteIndex[$chunkHash])) { 4615 $orphanedChunksSkipped++; 4616 continue; 4617 } 4618 4619 // Check if chunk exists on cloud 4620 try { 4621 if ($objectStore->exists($chunkHash)) { 4622 $remoteIndex[$chunkHash] = time(); 4623 $orphanedChunksSkipped++; 4624 if (file_exists($blobFile)) { 4625 wp_delete_file($blobFile); 4626 } 4627 continue; 4628 } 4629 } catch (\Throwable $e) { 4630 continue; 4631 } 4632 4633 // Upload orphaned chunk 4634 try { 4635 $chunkData = file_get_contents($blobFile); 4636 if ($chunkData === false) { 4637 continue; 4638 } 4639 4640 $mem = fopen('php://temp', 'rb+'); 4641 fwrite($mem, $chunkData); 4642 rewind($mem); 4643 4644 $objectStore->put($chunkHash, $mem, strlen($chunkData)); 4645 fclose($mem); 4646 4647 $remoteIndex[$chunkHash] = time(); 4648 $chunkSize = strlen($chunkData); 4649 $uploadedBytes += $chunkSize; 4650 $orphanedChunksUploaded++; 4651 4652 // Track database uploaded bytes 4653 try { 4654 $progress = $this->get_incremental_progress($backup_id); 4655 $progress['db_uploaded_bytes'] = (int)($progress['db_uploaded_bytes'] ?? 0) + $chunkSize; 4656 $this->update_incremental_progress($backup_id, $progress); 4657 } catch (\Throwable $e) { /* ignore */ } 4658 4659 if (file_exists($blobFile)) { 4660 wp_delete_file($blobFile); 4661 } 4662 } catch (\Throwable $e) { 4663 // Log but continue 4664 } 4665 } 4666 } 4667 4668 if ($orphanedChunksUploaded > 0 || $orphanedChunksSkipped > 0) { 4669 $coordinator->saveRemoteIndex($remoteIndex, $provider); 4670 $this->logger->info('Completed scan for orphaned chunks', [ 4671 'backup_id' => $backup_id, 4672 'provider' => $provider, 4673 'orphaned_chunks_uploaded' => $orphanedChunksUploaded, 4674 'orphaned_chunks_skipped' => $orphanedChunksSkipped 4675 ]); 4676 } 4677 } 4678 } 4679 4680 // Final comprehensive scan: Always check for any remaining chunks in the chunks folder 4681 // This ensures we catch chunks that may have been created after manifest processing 4682 // or from duplicate backup runs 4683 $this->logger->info('Performing final scan of chunks folder for any remaining chunks', [ 4684 'backup_id' => $backup_id, 4685 'provider' => $provider 4686 ]); 4687 4688 $chunksBaseDir = SITESKITE_BACKUP_PATH . '/chunks'; 4689 $finalScanUploaded = 0; 4690 $finalScanSkipped = 0; 4691 $finalScanBytes = 0; 4692 4693 if (is_dir($chunksBaseDir)) { 4694 $subdirs = glob($chunksBaseDir . '/*', GLOB_ONLYDIR); 4695 foreach ($subdirs as $subdir) { 4696 $blobFiles = glob($subdir . '/*.blob'); 4697 foreach ($blobFiles as $blobFile) { 4698 $filename = basename($blobFile, '.blob'); 4699 if (strlen($filename) !== 64 || !ctype_xdigit($filename)) { 4700 continue; 4701 } 4702 4703 $chunkHash = $filename; 4704 4705 // Skip if already processed or in remote index 4706 if (isset($processedThisRun[$chunkHash]) || isset($remoteIndex[$chunkHash])) { 4707 $finalScanSkipped++; 4708 continue; 4709 } 4710 4711 // Check if chunk exists on cloud 4712 try { 4713 if ($objectStore->exists($chunkHash)) { 4714 $remoteIndex[$chunkHash] = time(); 4715 $processedThisRun[$chunkHash] = true; 4716 $finalScanSkipped++; 4717 if (file_exists($blobFile)) { 4718 wp_delete_file($blobFile); 4719 } 4720 continue; 4721 } 4722 } catch (\Throwable $e) { 4723 $this->logger->debug('Error checking chunk in final scan', [ 4724 'backup_id' => $backup_id, 4725 'chunk_hash' => $chunkHash, 4726 'error' => $e->getMessage() 4727 ]); 4728 continue; 4729 } 4730 4731 // Chunk exists locally but not on cloud - upload it 4732 try { 4733 $chunkData = file_get_contents($blobFile); 4734 if ($chunkData === false) { 4735 continue; 4736 } 4737 4738 $mem = fopen('php://temp', 'rb+'); 4739 fwrite($mem, $chunkData); 4740 rewind($mem); 4741 4742 $objectStore->put($chunkHash, $mem, strlen($chunkData)); 4743 fclose($mem); 4744 4745 $remoteIndex[$chunkHash] = time(); 4746 $processedThisRun[$chunkHash] = true; 4747 $chunkSize = strlen($chunkData); 4748 $uploadedBytes += $chunkSize; 4749 $finalScanBytes += $chunkSize; 4750 $finalScanUploaded++; 4751 4752 // Track database uploaded bytes 4753 try { 4754 $progress = $this->get_incremental_progress($backup_id); 4755 $progress['db_uploaded_bytes'] = (int)($progress['db_uploaded_bytes'] ?? 0) + $chunkSize; 4756 $this->update_incremental_progress($backup_id, $progress); 4757 } catch (\Throwable $e) { /* ignore */ } 4758 4759 if (file_exists($blobFile)) { 4760 wp_delete_file($blobFile); 4761 $this->logger->info('Uploaded remaining chunk from final scan', [ 4762 'backup_id' => $backup_id, 4763 'chunk_hash' => $chunkHash, 4764 'chunk_size' => $chunkSize 4765 ]); 4766 } 4767 } catch (\Throwable $e) { 4768 $this->logger->warning('Failed to upload chunk in final scan', [ 4769 'backup_id' => $backup_id, 4770 'chunk_hash' => $chunkHash, 4771 'error' => $e->getMessage() 4772 ]); 4773 } 4774 } 4775 } 4776 4777 // Save remote index after final scan 4778 if ($finalScanUploaded > 0 || $finalScanSkipped > 0) { 4779 $coordinator->saveRemoteIndex($remoteIndex, $provider); 4780 $this->logger->info('Final scan of chunks folder completed', [ 4781 'backup_id' => $backup_id, 4782 'provider' => $provider, 4783 'final_scan_uploaded' => $finalScanUploaded, 4784 'final_scan_skipped' => $finalScanSkipped, 4785 'final_scan_bytes' => $finalScanBytes 4786 ]); 4787 } 4788 } 4789 4073 4790 $this->logger->info('Chunk upload completed', [ 4074 4791 'backup_id' => $backup_id, … … 4077 4794 'files_processed' => count($filesToProcess), 4078 4795 'total_files' => $progress['total_files'], 4079 'processed_files_count' => count($progress['processed_files']) 4796 'processed_files_count' => count($progress['processed_files']), 4797 'final_scan_uploaded' => $finalScanUploaded 4080 4798 ]); 4081 4799 … … 4125 4843 4126 4844 // Schedule next chunk if not completed and more files to process OR deferred chunks remain 4127 if ($processed 150Files || $hasDeferredChunks) {4845 if ($processedCount < $totalFiles || $hasDeferredChunks) { 4128 4846 $this->logger->info('Scheduling next incremental continue chunk', [ 4129 4847 'backup_id' => $backup_id, … … 4136 4854 ]); 4137 4855 4138 // Use external cron if enabled and we processed 150 files (or have deferred chunks) 4856 // If we've hit a full batch or have deferred work, use a long-lived external cron 4857 // so that continuation is resilient even if WordPress cron stops running. 4139 4858 if ($this->external_cron_manager && $this->external_cron_manager->is_enabled() && ($processed150Files || $hasDeferredChunks)) { 4140 // Check if job already exists to prevent duplicates4859 // Check if a recurring incremental-continue job already exists 4141 4860 $existing_job_id = $this->external_cron_manager->get_job_id_by_action('siteskite_incremental_continue', [$backup_id]); 4142 4861 4143 4862 if (!$existing_job_id) { 4144 // Schedule recurring "every_minute" job that continues until all files are processed 4145 // Do NOT mark for auto-deletion - it will run every minute until completion 4863 // Schedule a recurring "every_minute" job that stays visible until we explicitly clean it up 4146 4864 $job_id = $this->external_cron_manager->create_recurring_job( 4147 4865 'siteskite_incremental_continue', … … 4153 4871 4154 4872 if ($job_id) { 4155 $this->logger->info('Scheduled external cron for incremental continue (recurring every minute)', [4873 $this->logger->info('Scheduled persistent external cron for incremental continue', [ 4156 4874 'backup_id' => $backup_id, 4157 4875 'job_id' => $job_id, … … 4162 4880 ]); 4163 4881 } else { 4164 // Fallback to coordinator scheduling if external cron fails4165 $this->logger->warning('Failed to schedule external cron, falling back tocoordinator', [4882 // Fallback to coordinator scheduling if external cron creation fails 4883 $this->logger->warning('Failed to schedule persistent external cron, falling back to WordPress cron coordinator', [ 4166 4884 'backup_id' => $backup_id 4167 4885 ]); … … 4170 4888 } 4171 4889 } else { 4172 $this->logger->debug(' External cron job already exists for incremental continue, skipping creation', [4890 $this->logger->debug('Persistent external cron job for incremental continue already exists, skipping creation', [ 4173 4891 'backup_id' => $backup_id, 4174 4892 'existing_job_id' => $existing_job_id … … 4176 4894 } 4177 4895 } else { 4178 // Use coordinator scheduling if external cron is disabled or conditions not met4896 // Use coordinator / WordPress cron scheduling if external cron is not available 4179 4897 $delay = $hasDeferredChunks ? 10 : 15; 4180 4898 $coordinator->scheduleNext($backup_id, $delay); 4181 4899 } 4182 4900 } else { 4183 // Not enough files processed and no deferred chunks - use normal scheduling 4184 $delay = $hasDeferredChunks ? 10 : 15; 4185 $coordinator->scheduleNext($backup_id, $delay); 4901 // No additional work detected; nothing to schedule 4902 $this->logger->info('No additional work detected for incremental continue', [ 4903 'backup_id' => $backup_id, 4904 'processed_files' => $processedCount, 4905 'total_files' => $totalFiles 4906 ]); 4186 4907 } 4187 4908 } else { … … 4238 4959 { 4239 4960 try { 4961 // Check if backup is already completed to prevent duplicate runs 4962 $backup_info = get_option(self::OPTION_PREFIX . $backup_id); 4963 if ($backup_info && isset($backup_info['status']) && $backup_info['status'] === 'completed') { 4964 $this->logger->info('Backup already completed, skipping duplicate run', [ 4965 'backup_id' => $backup_id, 4966 'status' => $backup_info['status'] 4967 ]); 4968 return; 4969 } 4970 4971 // Prevent duplicate concurrent executions with a lightweight lock 4972 $lock_key = 'siteskite_full_backup_lock_' . $backup_id; 4973 $lock_value = time(); 4974 if (!add_option($lock_key, $lock_value, '', 'no')) { 4975 $existing_lock = get_option($lock_key); 4976 // If another run started within the last 2 minutes, skip this trigger 4977 if ($existing_lock && (time() - (int) $existing_lock) < 120) { 4978 $this->logger->info('Full backup already in progress, skipping duplicate trigger', [ 4979 'backup_id' => $backup_id, 4980 'lock_age' => time() - (int) $existing_lock 4981 ]); 4982 return; 4983 } 4984 // Stale lock: remove and attempt to acquire once more 4985 delete_option($lock_key); 4986 if (!add_option($lock_key, $lock_value, '', 'no')) { 4987 $this->logger->info('Failed to acquire full backup lock, skipping trigger', [ 4988 'backup_id' => $backup_id 4989 ]); 4990 return; 4991 } 4992 } 4993 4240 4994 $this->logger->info('Processing full backup cron', [ 4241 'backup_id' => $backup_id 4242 ]); 4243 4244 $this->process_full_backup($backup_id); 4995 'backup_id' => $backup_id 4996 ]); 4997 4998 try { 4999 $this->process_full_backup($backup_id); 5000 } finally { 5001 // Release lock after processing attempt 5002 delete_option($lock_key); 5003 } 4245 5004 } catch (\Exception $e) { 4246 5005 $this->logger->error('Full backup cron failed', [ … … 4268 5027 return; 4269 5028 } 5029 5030 // Ensure no other incremental-continue jobs are active for different backup IDs 5031 // This enforces the "only one incremental job at a time" rule. 5032 $this->cleanup_other_incremental_jobs($backup_id); 4270 5033 4271 5034 // Check if backup is still valid … … 4276 5039 'backup_id' => $backup_id 4277 5040 ]); 4278 return; 4279 } 4280 4281 // If backup is already completed, no need to continue processing 4282 $backup_status = $backup_info['status'] ?? ''; 4283 if ($backup_status === 'completed') { 4284 $this->logger->debug('Backup already completed, skipping incremental continue', [ 4285 'backup_id' => $backup_id, 4286 'status' => $backup_status 4287 ]); 4288 4289 // Clean up any pending incremental continuation jobs for this backup 5041 5042 // Clean up external cron jobs for non-existent backup 4290 5043 if ($this->external_cron_manager && $this->external_cron_manager->is_enabled()) { 4291 5044 $deleted_count = $this->external_cron_manager->delete_jobs_by_backup_id('siteskite_incremental_continue', $backup_id); 4292 5045 if ($deleted_count > 0) { 4293 $this->logger->info('Cleaned up pending incremental continuation jobs for completedbackup', [5046 $this->logger->info('Cleaned up external cron jobs for non-existent backup', [ 4294 5047 'backup_id' => $backup_id, 4295 5048 'deleted_count' => $deleted_count … … 4297 5050 } 4298 5051 } 4299 4300 // Also clear WordPress cron events for this backup4301 5052 wp_clear_scheduled_hook('siteskite_incremental_continue', [$backup_id]); 4302 4303 // Ensure logs are cleared once more on any stray continue invocations post-completion4304 try { $this->logger->delete_todays_log(); } catch (\Throwable $t) { /* ignore */ }4305 try { $this->logger->cleanup_old_logs(30); } catch (\Throwable $t) { /* ignore */ }4306 5053 return; 5054 } 5055 5056 // Early check: If backup is already completed, clean up and return immediately 5057 $backup_status = $backup_info['status'] ?? ''; 5058 if ($backup_status === 'completed') { 5059 // Check if there are any remaining database chunks that need to be uploaded 5060 // This can happen if a duplicate backup run created new chunks after completion 5061 $dbManifestId = 'database:' . $backup_id; 5062 $dbManifestJson = $this->manifest_store->get($dbManifestId); 5063 $hasRemainingChunks = false; 5064 5065 if ($dbManifestJson) { 5066 $dbManifest = json_decode($dbManifestJson, true); 5067 if (json_last_error() === JSON_ERROR_NONE && is_array($dbManifest)) { 5068 $newDbChunks = $dbManifest['changes']['new_chunks'] ?? []; 5069 if (!empty($newDbChunks)) { 5070 // Check if any of these chunks exist locally but haven't been uploaded 5071 $status_data = $this->get_backup_status_data($backup_id); 5072 $provider = is_array($status_data) ? ($status_data['provider'] ?? '') : ''; 5073 5074 if ($provider) { 5075 try { 5076 $objectStore = \SiteSkite\Backup\Incremental\ProviderFactory::build($this->logger, $provider, $status_data); 5077 if ($objectStore) { 5078 $coordinator = new \SiteSkite\Backup\Incremental\RemoteIndexCoordinator($this->logger, $this->backup_config); 5079 $remoteIndex = $coordinator->loadRemoteIndex($backup_id, $provider); 5080 5081 foreach ($newDbChunks as $chunkHash) { 5082 $chunkPath = $this->backup_config->getChunkPath($chunkHash); 5083 if (file_exists($chunkPath) && !isset($remoteIndex[$chunkHash]) && !$objectStore->exists($chunkHash)) { 5084 $hasRemainingChunks = true; 5085 $this->logger->warning('Found remaining database chunk that needs upload after backup completion', [ 5086 'backup_id' => $backup_id, 5087 'chunk_hash' => $chunkHash, 5088 'chunk_path' => $chunkPath 5089 ]); 5090 break; 5091 } 5092 } 5093 } 5094 } catch (\Throwable $t) { 5095 $this->logger->debug('Error checking for remaining chunks', [ 5096 'backup_id' => $backup_id, 5097 'error' => $t->getMessage() 5098 ]); 5099 } 5100 } 5101 } 5102 } 5103 } 5104 5105 if (!$hasRemainingChunks) { 5106 $this->logger->debug('Backup already completed, skipping incremental continue', [ 5107 'backup_id' => $backup_id, 5108 'status' => $backup_status 5109 ]); 5110 5111 // Clean up any pending incremental continuation jobs for this backup 5112 if ($this->external_cron_manager && $this->external_cron_manager->is_enabled()) { 5113 $deleted_count = $this->external_cron_manager->delete_jobs_by_backup_id('siteskite_incremental_continue', $backup_id); 5114 if ($deleted_count > 0) { 5115 $this->logger->info('Cleaned up pending incremental continuation jobs for completed backup', [ 5116 'backup_id' => $backup_id, 5117 'deleted_count' => $deleted_count 5118 ]); 5119 } 5120 } 5121 5122 // Also clear WordPress cron events for this backup 5123 wp_clear_scheduled_hook('siteskite_incremental_continue', [$backup_id]); 5124 5125 // Ensure logs are cleared once more on any stray continue invocations post-completion 5126 try { $this->logger->delete_todays_log(); } catch (\Throwable $t) { /* ignore */ } 5127 try { $this->logger->cleanup_old_logs(30); } catch (\Throwable $t) { /* ignore */ } 5128 return; 5129 } else { 5130 // There are remaining chunks, continue processing to upload them 5131 $this->logger->info('Backup completed but found remaining database chunks, continuing to upload', [ 5132 'backup_id' => $backup_id 5133 ]); 5134 } 4307 5135 } 4308 5136 … … 5225 6053 $provider = $status_data['provider'] ?? 'unknown'; 5226 6054 5227 // Guard against duplicate completion calls 6055 // Guard against duplicate completion calls using atomic lock mechanism 5228 6056 $completion_lock_key = 'siteskite_completing_' . $backup_id; 5229 if (function_exists('get_transient') && get_transient($completion_lock_key)) { 6057 $lock_timeout = 300; // 5 minutes 6058 6059 // Use WordPress options for more atomic lock checking 6060 if (function_exists('get_option')) { 6061 $existing_lock = get_option($completion_lock_key, false); 6062 if ($existing_lock) { 6063 $lock_timestamp = (int)$existing_lock; 6064 $lock_age = time() - $lock_timestamp; 6065 6066 // If lock is still valid (less than timeout), skip 6067 if ($lock_age < $lock_timeout) { 6068 $this->logger->debug('Incremental backup completion already in progress, skipping duplicate call', [ 6069 'backup_id' => $backup_id, 6070 'provider' => $provider, 6071 'lock_age_seconds' => $lock_age 6072 ]); 6073 return; 6074 } else { 6075 // Lock expired, log and continue (will set new lock below) 6076 $this->logger->debug('Completion lock expired, proceeding with new completion attempt', [ 6077 'backup_id' => $backup_id, 6078 'lock_age_seconds' => $lock_age 6079 ]); 6080 } 6081 } 6082 6083 // Set completion lock atomically using current timestamp 6084 update_option($completion_lock_key, time(), false); 6085 } elseif (function_exists('get_transient') && get_transient($completion_lock_key)) { 6086 // Fallback to transient if options not available 5230 6087 $this->logger->debug('Incremental backup completion already in progress, skipping duplicate call', [ 5231 6088 'backup_id' => $backup_id, … … 5233 6090 ]); 5234 6091 return; 5235 } 5236 5237 // Set completion lock (expires in 5 minutes to prevent permanent locks) 5238 if (function_exists('set_transient')) { 5239 set_transient($completion_lock_key, true, 300); 6092 } elseif (function_exists('set_transient')) { 6093 // Fallback: Set completion lock (expires in 5 minutes to prevent permanent locks) 6094 set_transient($completion_lock_key, true, $lock_timeout); 5240 6095 } 5241 6096 … … 5269 6124 $this->update_incremental_progress($backup_id, $progress); 5270 6125 6126 // Release completion lock on verification failure to allow retry 6127 if (function_exists('delete_option')) { 6128 delete_option($completion_lock_key); 6129 } elseif (function_exists('delete_transient')) { 6130 delete_transient($completion_lock_key); 6131 } 6132 5271 6133 return; 5272 6134 } 5273 6135 5274 6136 // Calculate file info for completion - use tracked uploaded_bytes instead of manifest stats 6137 // This ensures we only report actually uploaded bytes, not total bytes including skipped chunks 5275 6138 $db_manifestId = $this->manifest_store->getLatestId('database'); 5276 6139 $db_manifestJson = $db_manifestId ? $this->manifest_store->get('database:' . $db_manifestId) : null; 5277 6140 $db_manifest = $db_manifestJson ? json_decode($db_manifestJson, true) : []; 5278 $db_size = (int)($db_manifest['stats']['bytes_processed'] ?? 0);5279 6141 5280 6142 // Get actual uploaded bytes from progress tracker (tracks chunk uploads accurately) … … 5282 6144 $uploadedBytes = (int)($progressNow['uploaded_bytes'] ?? 0); 5283 6145 $totalFilesPlanned = (int)($progressNow['total_files'] ?? 0); 6146 6147 // Log current progress state for debugging 6148 $this->logger->debug('Size calculation from progress', [ 6149 'backup_id' => $backup_id, 6150 'uploaded_bytes' => $uploadedBytes, 6151 'db_uploaded_bytes' => $progressNow['db_uploaded_bytes'] ?? 'not set', 6152 'total_files' => $totalFilesPlanned 6153 ]); 6154 6155 // Calculate database size: only count actually uploaded chunks, not all chunks 6156 // Get database uploaded bytes from the upload process (stored separately) 6157 // For database, we need to check if chunks were uploaded or skipped 6158 $db_uploaded_bytes = 0; 6159 if (isset($progressNow['db_uploaded_bytes'])) { 6160 $db_uploaded_bytes = (int)($progressNow['db_uploaded_bytes']); 6161 $this->logger->debug('Using tracked db_uploaded_bytes from progress', [ 6162 'backup_id' => $backup_id, 6163 'db_uploaded_bytes' => $db_uploaded_bytes 6164 ]); 6165 } else { 6166 // Fallback: if tracking is missing, we need to calculate from actually uploaded chunks 6167 // This is more accurate than using bytes_processed which includes reused chunks 6168 $new_db_chunks = $db_manifest['changes']['new_chunks'] ?? []; 6169 $reused_db_chunks = $db_manifest['changes']['reused_chunks'] ?? []; 6170 $total_db_chunks = count($new_db_chunks) + count($reused_db_chunks); 6171 6172 if ($total_db_chunks > 0 && count($new_db_chunks) > 0) { 6173 // Calculate size based on new chunks only (reused chunks weren't uploaded) 6174 $total_bytes_processed = (int)($db_manifest['stats']['bytes_processed'] ?? 0); 6175 // Estimate: only count new chunks proportionally 6176 $db_uploaded_bytes = (int)(($total_bytes_processed * count($new_db_chunks)) / $total_db_chunks); 6177 6178 $this->logger->debug('Calculated db_uploaded_bytes from manifest (new chunks only)', [ 6179 'backup_id' => $backup_id, 6180 'db_uploaded_bytes' => $db_uploaded_bytes, 6181 'new_chunks' => count($new_db_chunks), 6182 'reused_chunks' => count($reused_db_chunks), 6183 'total_bytes_processed' => $total_bytes_processed 6184 ]); 6185 } else { 6186 // No new chunks means nothing was uploaded 6187 $db_uploaded_bytes = 0; 6188 $this->logger->debug('No new database chunks, db_uploaded_bytes is 0', [ 6189 'backup_id' => $backup_id 6190 ]); 6191 } 6192 } 5284 6193 5285 6194 // Use tracked uploaded_bytes when files were actually uploaded; fallback to manifest stats if tracking is missing … … 5295 6204 $db_manifest_bytes = $db_manifestJson ? strlen($db_manifestJson) : 0; 5296 6205 $files_manifest_bytes = $files_manifestJson ? strlen($files_manifestJson) : 0; 5297 $db_ size= $db_manifest_bytes;6206 $db_uploaded_bytes = $db_manifest_bytes; 5298 6207 $files_size = $files_manifest_bytes; 5299 6208 $this->logger->info('Reporting manifest upload sizes since no content needed upload', [ … … 5314 6223 $file_info = [ 5315 6224 'database' => [ 5316 'size' => $db_ size,6225 'size' => $db_uploaded_bytes, 5317 6226 'path' => 'incremental_database_objects', 5318 6227 'download_url' => null // No direct download for incremental … … 5322 6231 'path' => 'incremental_objects' 5323 6232 ], 5324 'total_size' => $db_ size+ $files_size6233 'total_size' => $db_uploaded_bytes + $files_size 5325 6234 ]; 5326 6235 … … 5364 6273 $this->cleanup_incremental_records($backup_id, $provider); 5365 6274 6275 // Cleanup chunks folder - remove database chunks that were uploaded in this backup 6276 try { 6277 $this->cleanup_uploaded_chunks($backup_id, $provider); 6278 } catch (\Throwable $t) { 6279 $this->logger->warning('Failed to cleanup chunks folder', [ 6280 'backup_id' => $backup_id, 6281 'error' => $t->getMessage() 6282 ]); 6283 } 6284 5366 6285 // Clean up external cron recurring job for incremental continue 5367 6286 if ($this->external_cron_manager && $this->external_cron_manager->is_enabled()) { … … 5378 6297 wp_clear_scheduled_hook('siteskite_incremental_continue', [$backup_id]); 5379 6298 5380 $this->logger->delete_todays_log();6299 // $this->logger->delete_todays_log(); 5381 6300 5382 6301 // Immediately cleanup old logs (mirroring restore behavior). Does not touch manifests or indexes. 5383 6302 try { 5384 $this->logger->cleanup_old_logs(30);6303 // $this->logger->cleanup_old_logs(30); 5385 6304 $this->logger->info('Old logs cleanup completed (immediate post-incremental)', [ 'backup_id' => $backup_id ]); 5386 6305 } catch (\Throwable $t) { … … 5389 6308 5390 6309 // Release completion lock on success 5391 if (function_exists('delete_transient')) { 6310 if (function_exists('delete_option')) { 6311 delete_option($completion_lock_key); 6312 } elseif (function_exists('delete_transient')) { 5392 6313 delete_transient($completion_lock_key); 5393 6314 } … … 5422 6343 5423 6344 // Release completion lock on error 5424 if (function_exists('delete_transient')) { 6345 if (function_exists('delete_option')) { 6346 delete_option($completion_lock_key); 6347 } elseif (function_exists('delete_transient')) { 5425 6348 delete_transient($completion_lock_key); 5426 6349 } … … 5560 6483 'backup_id' => $backup_id, 5561 6484 'provider' => $provider, 6485 'error' => $t->getMessage() 6486 ]); 6487 } 6488 } 6489 6490 /** 6491 * Cleanup chunks folder - remove database chunks that were uploaded in this backup 6492 * Only removes chunks from this specific backup to avoid deleting shared chunks 6493 * Checks if chunks exist on cloud before deleting to prevent data loss 6494 */ 6495 private function cleanup_uploaded_chunks(string $backup_id, string $provider): void 6496 { 6497 try { 6498 // Get object store to check if chunks exist on cloud 6499 $objectStore = null; 6500 try { 6501 // Get status data for the provider to build object store 6502 $status_data = $this->get_backup_status_data($backup_id); 6503 if (is_array($status_data) && !empty($status_data)) { 6504 $objectStore = \SiteSkite\Backup\Incremental\ProviderFactory::build($this->logger, $provider, $status_data); 6505 } else { 6506 // Try to get status data from provider-specific option 6507 $provider_keys = [ 6508 'dropbox' => 'siteskite_dropbox_status_' . $backup_id, 6509 'google_drive' => 'siteskite_gdrive_status_' . $backup_id, 6510 'backblaze_b2' => 'siteskite_backblaze_b2_status_' . $backup_id, 6511 'pcloud' => 'siteskite_pcloud_status_' . $backup_id, 6512 'aws_s3' => 'siteskite_aws_status_' . $backup_id, 6513 'aws' => 'siteskite_aws_status_' . $backup_id, 6514 ]; 6515 $optKey = $provider_keys[$provider] ?? null; 6516 if ($optKey) { 6517 $status_data = get_option($optKey); 6518 if (is_array($status_data) && !empty($status_data)) { 6519 $objectStore = \SiteSkite\Backup\Incremental\ProviderFactory::build($this->logger, $provider, $status_data); 6520 } 6521 } 6522 } 6523 } catch (\Throwable $e) { 6524 $this->logger->warning('Failed to build object store for cleanup verification', [ 6525 'backup_id' => $backup_id, 6526 'provider' => $provider, 6527 'error' => $e->getMessage() 6528 ]); 6529 // Continue without verification if object store can't be built 6530 } 6531 6532 // Get database manifest to find chunks that were uploaded in this backup 6533 $db_manifestId = 'database:' . $backup_id; 6534 $db_manifestJson = $this->manifest_store->get($db_manifestId); 6535 6536 if (!$db_manifestJson) { 6537 $this->logger->debug('No database manifest found for chunks cleanup', [ 6538 'backup_id' => $backup_id 6539 ]); 6540 return; 6541 } 6542 6543 $db_manifest = json_decode($db_manifestJson, true); 6544 if (json_last_error() !== JSON_ERROR_NONE || !is_array($db_manifest)) { 6545 return; 6546 } 6547 6548 // Only cleanup new chunks that were uploaded (not reused chunks which may be needed by other backups) 6549 $new_chunks = $db_manifest['changes']['new_chunks'] ?? []; 6550 6551 // Also check deferred_db_chunks from progress - these may have been uploaded in retries 6552 $progress = $this->get_incremental_progress($backup_id); 6553 $deferred_db_chunks = $progress['deferred_db_chunks'] ?? []; 6554 $chunksToCheck = array_unique(array_merge($new_chunks, array_keys($deferred_db_chunks))); 6555 6556 if (empty($chunksToCheck)) { 6557 $this->logger->debug('No chunks to cleanup', [ 6558 'backup_id' => $backup_id 6559 ]); 6560 return; 6561 } 6562 6563 $chunksBaseDir = SITESKITE_BACKUP_PATH . '/chunks'; 6564 if (!is_dir($chunksBaseDir)) { 6565 return; 6566 } 6567 6568 $deletedCount = 0; 6569 $failedCount = 0; 6570 $skippedNotOnCloud = 0; 6571 $verifiedOnCloud = 0; 6572 6573 foreach ($chunksToCheck as $chunkHash) { 6574 $chunkPath = $this->backup_config->getChunkPath($chunkHash); 6575 6576 // Only delete if file exists 6577 if (!file_exists($chunkPath)) { 6578 continue; 6579 } 6580 6581 // Verify chunk exists on cloud before deleting (safety check) 6582 $shouldDelete = true; 6583 if ($objectStore !== null) { 6584 try { 6585 if ($objectStore->exists($chunkHash)) { 6586 $verifiedOnCloud++; 6587 $shouldDelete = true; 6588 } else { 6589 // Chunk not on cloud - don't delete, it may need to be uploaded 6590 $skippedNotOnCloud++; 6591 $shouldDelete = false; 6592 $this->logger->warning('Chunk not found on cloud, skipping deletion', [ 6593 'backup_id' => $backup_id, 6594 'chunk_hash' => substr($chunkHash, 0, 16) . '...', 6595 'chunk_path' => $chunkPath 6596 ]); 6597 } 6598 } catch (\Throwable $e) { 6599 // If verification fails, be conservative and don't delete 6600 $this->logger->warning('Failed to verify chunk on cloud, skipping deletion', [ 6601 'backup_id' => $backup_id, 6602 'chunk_hash' => substr($chunkHash, 0, 16) . '...', 6603 'error' => $e->getMessage() 6604 ]); 6605 $shouldDelete = false; 6606 } 6607 } 6608 6609 if ($shouldDelete) { 6610 if (wp_delete_file($chunkPath)) { 6611 $deletedCount++; 6612 } else { 6613 $failedCount++; 6614 $this->logger->warning('Failed to delete chunk file', [ 6615 'backup_id' => $backup_id, 6616 'chunk_hash' => substr($chunkHash, 0, 16) . '...', 6617 'chunk_path' => $chunkPath 6618 ]); 6619 } 6620 } 6621 } 6622 6623 if ($deletedCount > 0 || $skippedNotOnCloud > 0) { 6624 $this->logger->info('Cleaned up uploaded chunks after backup completion', [ 6625 'backup_id' => $backup_id, 6626 'deleted_count' => $deletedCount, 6627 'failed_count' => $failedCount, 6628 'skipped_not_on_cloud' => $skippedNotOnCloud, 6629 'verified_on_cloud' => $verifiedOnCloud, 6630 'total_chunks_checked' => count($chunksToCheck) 6631 ]); 6632 } 6633 } catch (\Throwable $t) { 6634 $this->logger->error('Error during chunks cleanup', [ 6635 'backup_id' => $backup_id, 5562 6636 'error' => $t->getMessage() 5563 6637 ]); -
siteskite/trunk/includes/Backup/Incremental/Impl/BackblazeB2ObjectStore.php
r3418578 r3420528 48 48 public function exists(string $hash): bool 49 49 { 50 // Use cached authentication instead of authenticating every time 51 $this->ensureAuthenticated(); 52 50 53 $fileName = $this->keyFor($hash); 51 $resp = \call_user_func('\\wp_remote_post', 'https://api.backblazeb2.com/b2api/v2/b2_authorize_account', [ 'headers' => [ 'Authorization' => 'Basic ' . base64_encode($this->keyId . ':' . $this->applicationKey) ], 'body' => '{}', 'timeout' => 30 ]); 52 if (\call_user_func('\\is_wp_error', $resp)) { return false; } 53 $auth = json_decode((string) \call_user_func('\\wp_remote_retrieve_body', $resp), true); 54 $apiUrl = $auth['apiUrl'] ?? ''; 55 $token = $auth['authorizationToken'] ?? ''; 56 if (!$apiUrl || !$token) { return false; } 57 $bucketResp = \call_user_func('\\wp_remote_post', $apiUrl . '/b2api/v2/b2_list_buckets', [ 'headers' => [ 'Authorization' => $token, 'Content-Type' => 'application/json' ], 'body' => \call_user_func('\\wp_json_encode', [ 'accountId' => $auth['accountId'] ?? '', 'bucketName' => $this->bucketName ]) ]); 58 if (\call_user_func('\\is_wp_error', $bucketResp)) { return false; } 59 $bucketData = json_decode((string) \call_user_func('\\wp_remote_retrieve_body', $bucketResp), true); 60 $bucketId = null; 61 foreach (($bucketData['buckets'] ?? []) as $b) { if (($b['bucketName'] ?? '') === $this->bucketName) { $bucketId = $b['bucketId'] ?? null; break; } } 62 if (!$bucketId) { return false; } 63 $list = \call_user_func('\\wp_remote_post', $apiUrl . '/b2api/v2/b2_list_file_names', [ 'headers' => [ 'Authorization' => $token, 'Content-Type' => 'application/json' ], 'body' => \call_user_func('\\wp_json_encode', [ 'bucketId' => $bucketId, 'prefix' => $fileName, 'maxFileCount' => 1 ]) ]); 64 if (\call_user_func('\\is_wp_error', $list)) { return false; } 54 $bucketId = $this->getBucketId(); 55 56 // Use cached API URL and auth token 57 $list = \call_user_func('\\wp_remote_post', $this->apiUrl . '/b2api/v2/b2_list_file_names', [ 58 'headers' => [ 59 'Authorization' => $this->authToken, 60 'Content-Type' => 'application/json' 61 ], 62 'body' => \call_user_func('\\wp_json_encode', [ 63 'bucketId' => $bucketId, 64 'prefix' => $fileName, 65 'maxFileCount' => 1 66 ]), 67 'timeout' => 30 68 ]); 69 70 if (\call_user_func('\\is_wp_error', $list)) { 71 return false; 72 } 73 65 74 $listData = json_decode((string) \call_user_func('\\wp_remote_retrieve_body', $list), true); 66 foreach (($listData['files'] ?? []) as $f) { if (($f['fileName'] ?? '') === $fileName) { return true; } } 75 foreach (($listData['files'] ?? []) as $f) { 76 if (($f['fileName'] ?? '') === $fileName) { 77 return true; 78 } 79 } 67 80 return false; 68 81 } … … 88 101 elseif ($this->tokenExpiresAt && ($this->tokenExpiresAt - $currentTime) < 3600) { 89 102 $needsRefresh = true; 90 $this->logger->debug('Backblaze B2 token expires within 1 hour, proactively refreshing credentials', [91 'expires_in' => $this->tokenExpiresAt - $currentTime92 ]);103 // $this->logger->debug('Backblaze B2 token expires within 1 hour, proactively refreshing credentials', [ 104 // 'expires_in' => $this->tokenExpiresAt - $currentTime 105 // ]); 93 106 } 94 107 // Check if we've uploaded too many files (refresh after ~1500 files) … … 108 121 $this->clearCredentials(); 109 122 110 $this->logger->debug('Authenticating with Backblaze B2', [111 'key_id' => substr($this->keyId, 0, 8) . '...'112 ]);123 // $this->logger->debug('Authenticating with Backblaze B2', [ 124 // 'key_id' => substr($this->keyId, 0, 8) . '...' 125 // ]); 113 126 114 127 $resp = \call_user_func('\\wp_remote_post', 'https://api.backblazeb2.com/b2api/v2/b2_authorize_account', [ … … 144 157 } 145 158 146 $this->logger->debug('Backblaze B2 authentication successful', [147 'api_url' => $this->apiUrl,148 'upload_count_reset' => true149 ]);159 // $this->logger->debug('Backblaze B2 authentication successful', [ 160 // 'api_url' => $this->apiUrl, 161 // 'upload_count_reset' => true 162 // ]); 150 163 } 151 164 -
siteskite/trunk/includes/Backup/Incremental/ProviderManifests.php
r3389896 r3420528 448 448 return [ 'success' => false, 'error' => 'Missing Backblaze B2 credentials' ]; 449 449 } 450 451 // Check if manifest already exists to prevent duplicate uploads 452 $existing_file = $this->b2->get_file_by_name($manifest_path, $key_id, $application_key, $bucket_name); 453 if (($existing_file['success'] ?? false) && !empty($existing_file['file_id'])) { 454 $this->logger->debug('Manifest already exists on cloud, skipping upload', [ 455 'manifest_path' => $manifest_path, 456 'file_id' => $existing_file['file_id'] ?? 'unknown' 457 ]); 458 return [ 'success' => true, 'skipped' => true, 'file_id' => $existing_file['file_id'] ?? null ]; 459 } 460 450 461 $temp_file = $this->createTempFile('manifest_'); 451 462 file_put_contents($temp_file, $content); -
siteskite/trunk/includes/Cloud/BackblazeB2Manager.php
r3400025 r3420528 653 653 654 654 655 $this->logger->debug('Backblaze B2 authentication successful', [656 'api_url' => $this->api_url,657 'account_id' => $this->account_id ? 'present' : 'missing'658 ]);655 // $this->logger->debug('Backblaze B2 authentication successful', [ 656 // 'api_url' => $this->api_url, 657 // 'account_id' => $this->account_id ? 'present' : 'missing' 658 // ]); 659 659 660 660 return ['success' => true]; -
siteskite/trunk/includes/Core/Plugin.php
r3418578 r3420528 27 27 use SiteSkite\Cron\ExternalCronManager; 28 28 use SiteSkite\Cron\CronTriggerHandler; 29 use SiteSkite\Cron\HybridCronManager; 29 30 30 31 use function add_action; … … 148 149 149 150 /** 151 * @var HybridCronManager Hybrid cron manager instance 152 */ 153 private ?HybridCronManager $hybrid_cron_manager = null; 154 155 /** 150 156 * @var CronTriggerHandler Cron trigger handler instance 151 157 */ … … 186 192 // Initialize external cron manager 187 193 $this->external_cron_manager = new ExternalCronManager($this->logger); 194 195 // Initialize hybrid cron manager 196 $this->hybrid_cron_manager = new HybridCronManager($this->logger, $this->external_cron_manager); 188 197 189 198 // Initialize cloud storage managers … … 231 240 $this->aws_manager, 232 241 $incremental_status, 233 $this->external_cron_manager 242 $this->external_cron_manager, 243 $this->hybrid_cron_manager 234 244 ); 235 245 … … 249 259 $this->backup_manager, 250 260 $this->schedule_manager, 251 $this->restore_manager 261 $this->restore_manager, 262 $this->hybrid_cron_manager 252 263 ); 253 264 … … 325 336 'handle_incremental_continue' 326 337 ], 10, 1); 338 339 // Hook into WordPress cron actions to notify hybrid cron manager 340 if ($this->hybrid_cron_manager) { 341 // List of cron hooks that should use hybrid cron 342 $hybrid_cron_hooks = [ 343 'siteskite_incremental_continue', 344 'process_database_backup_cron', 345 'process_files_backup_cron', 346 'process_full_backup_cron', 347 'siteskite_process_restore', 348 'siteskite_process_incremental_restore' 349 ]; 350 351 foreach ($hybrid_cron_hooks as $hook) { 352 add_action($hook, function(...$args) use ($hook) { 353 if ($this->hybrid_cron_manager) { 354 $this->hybrid_cron_manager->on_wp_cron_run($hook, $args); 355 } 356 }, 999); // High priority to run after the actual action 357 } 358 } 359 360 // Verify SiteSkite cron token when wp-cron.php is called with siteskite_key 361 // This adds security layer for SiteSkite-triggered cron 362 if (defined('DOING_CRON') && DOING_CRON && isset($_GET['siteskite_key'])) { 363 add_action('init', function() { 364 if ($this->hybrid_cron_manager) { 365 $token = sanitize_text_field($_GET['siteskite_key'] ?? ''); 366 if (!$this->hybrid_cron_manager->verify_siteskite_cron_token($token)) { 367 // Invalid token - log and exit gracefully 368 if ($this->logger) { 369 $this->logger->warning('Invalid SiteSkite cron token in wp-cron.php', [ 370 'token_length' => strlen($token), 371 'ip' => $_SERVER['REMOTE_ADDR'] ?? 'unknown' 372 ]); 373 } 374 // Don't block cron execution, but log the security issue 375 } else { 376 // Valid token - log for telemetry 377 if ($this->logger) { 378 $this->logger->debug('SiteSkite-triggered cron verified', [ 379 'ip' => $_SERVER['REMOTE_ADDR'] ?? 'unknown' 380 ]); 381 } 382 } 383 } 384 }, 1); // Early priority 385 } 327 386 } 328 387 … … 467 526 if (strpos($route, '/cron/trigger') !== false) { 468 527 $this->logger->debug('REST API request received', [ 469 'route' => $route,470 'method' => $request->get_method(),471 'headers' => $request->get_headers(),472 'body' => substr($request->get_body(), 0, 500) // First 500 chars528 // 'route' => $route, 529 // 'method' => $request->get_method(), 530 // 'headers' => $request->get_headers(), 531 // 'body' => substr($request->get_body(), 0, 500) // First 500 chars 473 532 ]); 474 533 } -
siteskite/trunk/includes/Core/Utils.php
r3418578 r3420528 47 47 return (string)$query_params['userID']; 48 48 } 49 50 /** 51 * Check if current cron execution was triggered by SiteSkite 52 * This allows identification of SiteSkite-triggered cron vs normal WordPress cron 53 * 54 * Usage: 55 * if (Utils::is_siteskite_triggered_cron()) { 56 * // Run SiteSkite-specific logic 57 * // Skip heavy jobs for normal cron 58 * // Add telemetry 59 * } 60 * 61 * @return bool True if cron was triggered by SiteSkite (wp-cron.php?doing_wp_cron&siteskite_key=...) 62 */ 63 public static function is_siteskite_triggered_cron(): bool 64 { 65 return defined('DOING_CRON') && DOING_CRON && 66 isset($_GET['siteskite_key']) && 67 !empty($_GET['siteskite_key']); 68 } 49 69 } -
siteskite/trunk/includes/Cron/CronTriggerHandler.php
r3418578 r3420528 21 21 private Logger $logger; 22 22 private ExternalCronManager $cron_manager; 23 private ?HybridCronManager $hybrid_cron_manager; 23 24 private BackupManager $backup_manager; 24 25 private ScheduleManager $schedule_manager; … … 30 31 BackupManager $backup_manager, 31 32 ScheduleManager $schedule_manager, 32 RestoreManager $restore_manager 33 RestoreManager $restore_manager, 34 ?HybridCronManager $hybrid_cron_manager = null 33 35 ) { 34 36 $this->logger = $logger; 35 37 $this->cron_manager = $cron_manager; 38 $this->hybrid_cron_manager = $hybrid_cron_manager; 36 39 $this->backup_manager = $backup_manager; 37 40 $this->schedule_manager = $schedule_manager; … … 46 49 // Log that we received the request (for debugging) 47 50 $this->logger->info('Cron trigger endpoint called', [ 48 'method' => $request->get_method(),49 'route' => $request->get_route(),50 'headers' => $request->get_headers(),51 'has_body' => !empty($request->get_body())51 // 'method' => $request->get_method(), 52 // 'route' => $request->get_route(), 53 // 'headers' => $request->get_headers(), 54 // 'has_body' => !empty($request->get_body()) 52 55 ]); 53 56 54 57 try { 55 // Verify secret token58 // Enhanced security: Verify secret token from header or query parameter 56 59 $secret = $request->get_header('X-SiteSkite-Cron-Secret'); 60 if (empty($secret)) { 61 // Fallback to query parameter for services that can't set custom headers 62 $secret = $request->get_param('siteskite_key'); 63 } 64 57 65 $expected_secret = $this->cron_manager->get_secret_token(); 58 66 59 if (empty($secret) || $secret !== $expected_secret) { 67 // Use hash_equals for timing-safe comparison 68 if (empty($secret) || !hash_equals($expected_secret, $secret)) { 60 69 $this->logger->warning('Invalid or missing secret token in cron trigger', [ 61 ' received' => substr($secret ?? '', 0, 10) . '...',62 ' expected' => substr($expected_secret, 0, 10) . '...',63 ' headers' => $request->get_headers(),64 ' all_headers' => $request->get_headers()->getAll()70 'has_header' => !empty($request->get_header('X-SiteSkite-Cron-Secret')), 71 'has_query_param' => !empty($request->get_param('siteskite_key')), 72 'ip' => $_SERVER['REMOTE_ADDR'] ?? 'unknown', 73 'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? 'unknown' 65 74 ]); 66 75 return new WP_Error( 67 76 'invalid_secret', 68 'Invalid secrettoken',77 'Invalid or missing authentication token', 69 78 ['status' => 401] 70 79 ); … … 98 107 // Log the received body for debugging 99 108 $this->logger->info('Received cron trigger request', [ 100 'body' => $body,101 'body_type' => gettype($body),102 'raw_body' => substr($request->get_body(), 0, 500),103 'content_type' => $request->get_header('Content-Type')109 // 'body' => $body, 110 // 'body_type' => gettype($body), 111 // 'raw_body' => substr($request->get_body(), 0, 500), 112 // 'content_type' => $request->get_header('Content-Type') 104 113 ]); 105 114 … … 141 150 $result = $this->execute_action($action, $args); 142 151 152 // Notify hybrid cron manager that external trigger occurred 153 // This will schedule WordPress cron for next run 154 // Only do this if the action was actually executed (not skipped) 155 if ($this->hybrid_cron_manager && ($result['status'] ?? '') !== 'skipped') { 156 $this->hybrid_cron_manager->on_external_trigger($action, $args); 157 } 158 143 159 // Check if job should be deleted after successful run 144 160 if ($this->cron_manager->should_delete_after_first_run($action, $args)) { … … 148 164 if ($deleted) { 149 165 $this->logger->info('Deleted recurring job after first successful execution', [ 150 'job_id' => $job_id,151 'action' => $action,152 'args' => $args166 // 'job_id' => $job_id, 167 // 'action' => $action, 168 // 'args' => $args 153 169 ]); 154 170 } else { … … 301 317 } 302 318 319 // Early check: If backup doesn't exist or is already completed, skip execution and clean up jobs 320 $backup_info = get_option('siteskite_backup_' . $backup_id); 321 if (!$backup_info) { 322 $this->logger->info('Backup info not found, skipping incremental continue trigger and cleaning up jobs', [ 323 'backup_id' => $backup_id 324 ]); 325 326 // Clean up external cron jobs immediately 327 if ($this->cron_manager && $this->cron_manager->is_enabled()) { 328 $deleted_count = $this->cron_manager->delete_jobs_by_backup_id('siteskite_incremental_continue', $backup_id); 329 if ($deleted_count > 0) { 330 $this->logger->info('Cleaned up external cron jobs for non-existent backup', [ 331 'backup_id' => $backup_id, 332 'deleted_count' => $deleted_count 333 ]); 334 } 335 } 336 337 // Clear WordPress cron events 338 wp_clear_scheduled_hook('siteskite_incremental_continue', [$backup_id]); 339 340 // Clean up hybrid cron state if it exists 341 if ($this->hybrid_cron_manager) { 342 $this->hybrid_cron_manager->cleanup_job('siteskite_incremental_continue', [$backup_id]); 343 } 344 345 return [ 346 'status' => 'skipped', 347 'backup_id' => $backup_id, 348 'reason' => 'backup_not_found' 349 ]; 350 } 351 352 if (isset($backup_info['status']) && $backup_info['status'] === 'completed') { 353 $this->logger->info('Backup already completed, skipping incremental continue trigger and cleaning up jobs', [ 354 'backup_id' => $backup_id 355 ]); 356 357 // Clean up external cron jobs immediately 358 if ($this->cron_manager && $this->cron_manager->is_enabled()) { 359 $deleted_count = $this->cron_manager->delete_jobs_by_backup_id('siteskite_incremental_continue', $backup_id); 360 if ($deleted_count > 0) { 361 $this->logger->info('Cleaned up external cron jobs for completed backup', [ 362 'backup_id' => $backup_id, 363 'deleted_count' => $deleted_count 364 ]); 365 } 366 } 367 368 // Clear WordPress cron events 369 wp_clear_scheduled_hook('siteskite_incremental_continue', [$backup_id]); 370 371 // Clean up hybrid cron state if it exists 372 if ($this->hybrid_cron_manager) { 373 $this->hybrid_cron_manager->cleanup_job('siteskite_incremental_continue', [$backup_id]); 374 } 375 376 return [ 377 'status' => 'skipped', 378 'backup_id' => $backup_id, 379 'reason' => 'backup_already_completed' 380 ]; 381 } 382 303 383 $this->backup_manager->handle_incremental_continue($backup_id); 304 384 … … 391 471 ]; 392 472 } 473 474 /** 475 * Handle continuity check request from external cron service 476 * External service calls this to check if WordPress cron is still running 477 */ 478 public function handle_continuity_check(WP_REST_Request $request): WP_REST_Response|WP_Error 479 { 480 try { 481 // Enhanced security: Verify secret token from header or query parameter 482 $secret = $request->get_header('X-SiteSkite-Cron-Secret'); 483 if (empty($secret)) { 484 $secret = $request->get_param('siteskite_key'); 485 } 486 487 $expected_secret = $this->cron_manager->get_secret_token(); 488 489 // Use hash_equals for timing-safe comparison 490 if (empty($secret) || !hash_equals($expected_secret, $secret)) { 491 $this->logger->warning('Invalid secret token in continuity check', [ 492 'ip' => $_SERVER['REMOTE_ADDR'] ?? 'unknown' 493 ]); 494 return new WP_Error( 495 'invalid_secret', 496 'Invalid or missing authentication token', 497 ['status' => 401] 498 ); 499 } 500 501 // Get action and args from request 502 $action = $request->get_param('action') ?? ''; 503 $args = $request->get_param('args') ?? []; 504 505 if (empty($action)) { 506 return new WP_Error( 507 'missing_action', 508 'Action is required', 509 ['status' => 400] 510 ); 511 } 512 513 if (!is_array($args)) { 514 $args = []; 515 } 516 517 // Check continuity using hybrid cron manager 518 if (!$this->hybrid_cron_manager) { 519 return rest_ensure_response([ 520 'success' => false, 521 'message' => 'Hybrid cron manager not available', 522 'should_trigger' => true // Default to triggering if we can't check 523 ]); 524 } 525 526 $continuity_maintained = $this->hybrid_cron_manager->check_continuity($action, $args); 527 528 return rest_ensure_response([ 529 'success' => true, 530 'continuity_maintained' => $continuity_maintained, 531 'should_trigger' => !$continuity_maintained, 532 'action' => $action, 533 'checked_at' => time() 534 ]); 535 } catch (\Exception $e) { 536 $this->logger->error('Error checking cron continuity', [ 537 'error' => $e->getMessage(), 538 'trace' => $e->getTraceAsString() 539 ]); 540 return new WP_Error( 541 'check_error', 542 $e->getMessage(), 543 ['status' => 500] 544 ); 545 } 546 } 393 547 } 394 548 -
siteskite/trunk/includes/Cron/ExternalCronManager.php
r3418578 r3420528 18 18 use function wp_remote_get; 19 19 use function add_query_arg; 20 use function site_url; 20 21 21 22 /** … … 238 239 239 240 $this->logger->debug('Preparing to create recurring job', [ 240 'action' => $action, 241 'url' => $url, 242 'schedule' => $schedule, 243 'body' => $body 244 ]); 245 241 // 'action' => $action, 242 // 'url' => $url, 243 // 'schedule' => $schedule, 244 // 'body' => $body 245 ]); 246 247 // For wp-cron trigger, use GET request, no body/headers needed 248 $is_wp_cron_trigger = ($action === 'siteskite_trigger_wp_cron'); 249 246 250 $job_data = [ 247 251 'job' => [ … … 249 253 'enabled' => true, 250 254 'title' => $title ?: "SiteSkite: {$action}", 251 'requestMethod' => 1, // POST 252 'schedule' => $schedule, 253 'extendedData' => [ 254 'headers' => [ 255 'X-SiteSkite-Cron-Secret' => $this->secret_token, 256 'Content-Type' => 'application/json' 257 ], 258 'body' => wp_json_encode($body) 259 ] 255 'requestMethod' => $is_wp_cron_trigger ? 0 : 1, // 0 = GET, 1 = POST 256 'schedule' => $schedule 260 257 ] 261 258 ]; 259 260 // Only add headers and body for non-wp-cron triggers 261 if (!$is_wp_cron_trigger) { 262 $job_data['job']['extendedData'] = [ 263 'headers' => [ 264 'X-SiteSkite-Cron-Secret' => $this->secret_token, 265 'Content-Type' => 'application/json' 266 ], 267 'body' => wp_json_encode($body) 268 ]; 269 } 262 270 263 271 $this->logger->debug('Sending API request to create recurring job', [ … … 286 294 287 295 $this->logger->debug('API response received', [ 288 'action' => $action,289 'response_code' => $response_code,290 'response_body_length' => strlen($response_body),291 'response_data' => $data,292 'json_error' => json_last_error() !== JSON_ERROR_NONE ? json_last_error_msg() : null296 // 'action' => $action, 297 // 'response_code' => $response_code, 298 // 'response_body_length' => strlen($response_body), 299 // 'response_data' => $data, 300 // 'json_error' => json_last_error() !== JSON_ERROR_NONE ? json_last_error_msg() : null 293 301 ]); 294 302 … … 302 310 303 311 $this->logger->info('Created external cron job', [ 304 'job_id' => $job_id,305 'action' => $action,306 'frequency' => $frequency312 //'job_id' => $job_id, 313 //'action' => $action, 314 //'frequency' => $frequency 307 315 ]); 308 316 … … 402 410 $body = $this->build_request_body($action, $args); 403 411 412 // For wp-cron trigger, use GET request, no body/headers needed 413 $is_wp_cron_trigger = ($action === 'siteskite_trigger_wp_cron'); 414 404 415 $job_data = [ 405 416 'job' => [ … … 407 418 'enabled' => true, 408 419 'title' => $title ?: "SiteSkite: {$action}", 409 'requestMethod' => 1, // POST 410 'schedule' => $schedule, // Simple single-event schedule (runs once) 411 'extendedData' => [ 412 'headers' => [ 413 'X-SiteSkite-Cron-Secret' => $this->secret_token, 414 'Content-Type' => 'application/json' 415 ], 416 'body' => wp_json_encode($body) 417 ] 420 'requestMethod' => $is_wp_cron_trigger ? 0 : 1, // 0 = GET, 1 = POST 421 'schedule' => $schedule // Simple single-event schedule (runs once) 418 422 ] 419 423 ]; 424 425 // Only add headers and body for non-wp-cron triggers 426 if (!$is_wp_cron_trigger) { 427 $job_data['job']['extendedData'] = [ 428 'headers' => [ 429 'X-SiteSkite-Cron-Secret' => $this->secret_token, 430 'Content-Type' => 'application/json' 431 ], 432 'body' => wp_json_encode($body) 433 ]; 434 } 420 435 421 436 $response = $this->make_api_request('PUT', '/jobs', $job_data); … … 443 458 444 459 $this->logger->info('Created external cron single job', [ 445 'job_id' => $job_id,446 'action' => $action,447 'timestamp' => $timestamp,448 'runs_at' => gmdate('Y-m-d H:i:s', $timestamp),449 'delete_after_first_run' => $delete_after_first_run460 //'job_id' => $job_id, 461 //'action' => $action, 462 //'timestamp' => $timestamp, 463 //'runs_at' => gmdate('Y-m-d H:i:s', $timestamp), 464 //'delete_after_first_run' => $delete_after_first_run 450 465 ]); 451 466 … … 494 509 $this->logger->info('Deleted external cron job', ['job_id' => $job_id]); 495 510 return true; 496 } else { 497 $this->logger->error('Failed to delete external cron job', [ 511 } 512 513 // Treat 404 as already deleted: clean local state and stop retrying 514 if ($response_code === 404) { 515 $this->remove_stored_job($job_id); 516 $this->logger->info('External cron job already deleted remotely, removed locally', [ 498 517 'job_id' => $job_id, 499 518 'response_code' => $response_code 500 519 ]); 501 return false; 502 } 520 return true; 521 } 522 523 $this->logger->error('Failed to delete external cron job', [ 524 'job_id' => $job_id, 525 'response_code' => $response_code 526 ]); 527 return false; 503 528 } catch (\Exception $e) { 504 529 $this->logger->error('Exception deleting external cron job', [ … … 560 585 private function get_trigger_url(string $action): string 561 586 { 587 // Special case: wp-cron trigger should call wp-cron.php directly 588 if ($action === 'siteskite_trigger_wp_cron') { 589 return site_url('wp-cron.php?doing_wp_cron'); 590 } 591 562 592 return rest_url('siteskite/v1/cron/trigger'); 563 593 } … … 568 598 private function build_request_body(string $action, array $args): array 569 599 { 600 // For wp-cron trigger, no body needed - just call the URL 601 if ($action === 'siteskite_trigger_wp_cron') { 602 return []; 603 } 604 570 605 return [ 571 606 'action' => $action, -
siteskite/trunk/readme.txt
r3418578 r3420528 5 5 Tested up to: 6.9 6 6 Requires PHP: 7.4 7 Stable tag: 1.0. 77 Stable tag: 1.0.8 8 8 License: GPLv2 or later 9 9 License URI: https://www.gnu.org/licenses/gpl-2.0.html … … 189 189 == Changelog == 190 190 191 = 1.0.8 (16 December 2025) = 192 * Improved: Backup performance 193 191 194 = 1.0.7 (13 December 2025) = 192 195 * Added: Better Backup management … … 217 220 == Upgrade Notice == 218 221 222 = 1.0.8 (16 December 2025) = 223 * Improved: Backup performance 224 219 225 = 1.0.7 (13 December 2025) = 220 226 * Added: Better Backup management -
siteskite/trunk/siteskite-link.php
r3418578 r3420528 4 4 * Plugin URI: https://siteskite.com 5 5 * Description: Link your WordPress site with SiteSkite for effortless updates, backups, monitoring, and maintenance—everything in one place. 6 * Version: 1.0. 76 * Version: 1.0.8 7 7 * Requires at least: 5.3 8 8 * Requires PHP: 7.4 … … 29 29 30 30 // Plugin version 31 define('SITESKITE_VERSION', '1.0. 7');31 define('SITESKITE_VERSION', '1.0.8'); 32 32 33 33 // Plugin file, path, and URL
Note: See TracChangeset
for help on using the changeset viewer.