From cf9f5f0bab48bc68cc01a206010028fa46066332 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 17 Dec 2022 19:43:40 -0700 Subject: [PATCH 1/5] Add migration --- ...352_add_status_id_index_to_media_table.php | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 database/migrations/2022_12_18_012352_add_status_id_index_to_media_table.php diff --git a/database/migrations/2022_12_18_012352_add_status_id_index_to_media_table.php b/database/migrations/2022_12_18_012352_add_status_id_index_to_media_table.php new file mode 100644 index 000000000..6f400ead2 --- /dev/null +++ b/database/migrations/2022_12_18_012352_add_status_id_index_to_media_table.php @@ -0,0 +1,36 @@ +index('status_id'); + $table->index('replicated_at'); + $table->index('version'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('media', function (Blueprint $table) { + $table->dropIndex('media_status_id_index'); + $table->dropIndex('media_replicated_at_index'); + $table->dropIndex('media_version_index'); + }); + } +}; From 0bc5f1306dbf1cdf36bc4364012d06c24258c36e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 17 Dec 2022 20:28:23 -0700 Subject: [PATCH 2/5] Update configs --- .../Commands/CatchUnoptimizedMedia.php | 32 +++++++++---------- .../Commands/MediaGarbageCollector.php | 6 ++-- app/Console/Kernel.php | 4 +-- config/logging.php | 6 ++++ config/media.php | 2 ++ 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/app/Console/Commands/CatchUnoptimizedMedia.php b/app/Console/Commands/CatchUnoptimizedMedia.php index 9038c12fb..a62bd8651 100644 --- a/app/Console/Commands/CatchUnoptimizedMedia.php +++ b/app/Console/Commands/CatchUnoptimizedMedia.php @@ -40,22 +40,20 @@ class CatchUnoptimizedMedia extends Command */ public function handle() { - DB::transaction(function() { - Media::whereNull('processed_at') - ->where('skip_optimize', '!=', true) - ->whereNull('remote_url') - ->whereNotNull('status_id') - ->whereNotNull('media_path') - ->where('created_at', '>', now()->subHours(1)) - ->whereIn('mime', [ - 'image/jpeg', - 'image/png', - ]) - ->chunk(50, function($medias) { - foreach ($medias as $media) { - ImageOptimize::dispatch($media); - } - }); - }); + Media::whereNull('processed_at') + ->where('created_at', '>', now()->subHours(1)) + ->where('skip_optimize', '!=', true) + ->whereNull('remote_url') + ->whereNotNull('status_id') + ->whereNotNull('media_path') + ->whereIn('mime', [ + 'image/jpeg', + 'image/png', + ]) + ->chunk(50, function($medias) { + foreach ($medias as $media) { + ImageOptimize::dispatch($media); + } + }); } } diff --git a/app/Console/Commands/MediaGarbageCollector.php b/app/Console/Commands/MediaGarbageCollector.php index c8fce9199..3c266a0d6 100644 --- a/app/Console/Commands/MediaGarbageCollector.php +++ b/app/Console/Commands/MediaGarbageCollector.php @@ -4,7 +4,6 @@ namespace App\Console\Commands; use Illuminate\Console\Command; use App\{Media, Status}; -use Carbon\Carbon; use App\Services\MediaStorageService; class MediaGarbageCollector extends Command @@ -40,11 +39,10 @@ class MediaGarbageCollector extends Command */ public function handle() { - $limit = 20000; + $limit = 500; $gc = Media::whereNull('status_id') - ->where('created_at', '<', Carbon::now()->subHours(12)->toDateTimeString()) - ->orderBy('created_at','asc') + ->where('created_at', '<', now()->subHours(2)->toDateTimeString()) ->take($limit) ->get(); diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index dd1bae944..cada75351 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -25,8 +25,8 @@ class Kernel extends ConsoleKernel */ protected function schedule(Schedule $schedule) { - $schedule->command('media:optimize')->hourly(); - $schedule->command('media:gc')->hourly(); + $schedule->command('media:optimize')->hourlyAt(40); + $schedule->command('media:gc')->hourlyAt(5); $schedule->command('horizon:snapshot')->everyFiveMinutes(); $schedule->command('story:gc')->everyFiveMinutes(); $schedule->command('gc:failedjobs')->dailyAt(3); diff --git a/config/logging.php b/config/logging.php index 34034449e..ae21f72ac 100644 --- a/config/logging.php +++ b/config/logging.php @@ -89,6 +89,12 @@ return [ 'emergency' => [ 'path' => storage_path('logs/laravel.log'), ], + + 'media' => [ + 'driver' => 'single', + 'path' => storage_path('logs/media.log'), + 'level' => 'debug' + ], ], ]; diff --git a/config/media.php b/config/media.php index bd392998a..c73b26798 100644 --- a/config/media.php +++ b/config/media.php @@ -1,6 +1,8 @@ env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true), + 'exif' => [ 'database' => env('MEDIA_EXIF_DATABASE', false), ], From b8c3f153c49e61a54eac741352f7afbb1e548de7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 17 Dec 2022 20:38:11 -0700 Subject: [PATCH 3/5] Add MediaS3GarbageCollector command to clear local media after uploaded to S3 disks after 12 hours --- .../Commands/MediaS3GarbageCollector.php | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 app/Console/Commands/MediaS3GarbageCollector.php diff --git a/app/Console/Commands/MediaS3GarbageCollector.php b/app/Console/Commands/MediaS3GarbageCollector.php new file mode 100644 index 000000000..6eed4fbda --- /dev/null +++ b/app/Console/Commands/MediaS3GarbageCollector.php @@ -0,0 +1,98 @@ +error('Cloud storage not enabled. Exiting...'); + return; + } + + $deleteEnabled = config('media.delete_local_after_cloud'); + if(!$deleteEnabled) { + $this->error('Delete local storage after cloud upload is not enabled'); + return; + } + + $limit = $this->option('limit'); + $minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first()->id; + + $gc = Media::whereNotNull(['status_id', 'cdn_url', 'replicated_at']) + ->whereNot('version', '4') + ->where('id', '<', $minId) + ->inRandomOrder() + ->take($limit) + ->get(); + + $totalSize = 0; + $bar = $this->output->createProgressBar($gc->count()); + $bar->start(); + $cloudDisk = Storage::disk(config('filesystems.cloud')); + $localDisk = Storage::disk('local'); + + foreach($gc as $media) { + if( + $cloudDisk->exists($media->media_path) + ) { + if( $localDisk->exists($media->media_path)) { + $localDisk->delete($media->media_path); + $media->version = 4; + $media->save(); + $totalSize = $totalSize + $media->size; + } else { + $media->version = 4; + $media->save(); + } + } else { + Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]); + } + $bar->advance(); + } + $bar->finish(); + $this->line(' '); + $this->info('Finished!'); + if($totalSize) { + $this->info('Cleared ' . $totalSize . ' bytes of media from local disk!'); + } + return 0; + } +} From 30e08e676deedc79ebc6efb0b58cbb124841d5cf Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 17 Dec 2022 20:38:42 -0700 Subject: [PATCH 4/5] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fa3a6d0c..b1ea43c7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ - Update ApiV1Controller, remove min avatar size limit, fixes #3715 ([2b0db812](https://github.com/pixelfed/pixelfed/commit/2b0db812)) - Update InboxPipeline, add inbox job queue and separate http sig validation from activity handling ([e6c1604d](https://github.com/pixelfed/pixelfed/commit/e6c1604d)) - Update InboxPipeline, dispatch Follow/Accept Follow jobs to follow queue ([f62d2494](https://github.com/pixelfed/pixelfed/commit/f62d2494)) +- Add MediaS3GarbageCollector command to clear local media after uploaded to S3 disks after 12 hours ([b8c3f153](https://github.com/pixelfed/pixelfed/commit/b8c3f153)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4) From fe51054122e58dc198533f718e67dd2cb5e96a8f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 17 Dec 2022 20:40:32 -0700 Subject: [PATCH 5/5] Update console kernel --- app/Console/Kernel.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index cada75351..f0a955f8d 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -32,6 +32,10 @@ class Kernel extends ConsoleKernel $schedule->command('gc:failedjobs')->dailyAt(3); $schedule->command('gc:passwordreset')->dailyAt('09:41'); $schedule->command('gc:sessions')->twiceDaily(13, 23); + + if(config('pixelfed.cloud_storage') && config('media.delete_local_after_cloud')) { + $schedule->command('media:s3gc')->hourlyAt(15); + } } /**