From a14af93b020839a79b3060cd5e4388d8310d48dc Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 17 Dec 2022 21:44:01 -0700 Subject: [PATCH 1/2] Update MediaS3GarbageCollector commmand, disable logging by default and optimize huge invocations --- .../Commands/MediaS3GarbageCollector.php | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/app/Console/Commands/MediaS3GarbageCollector.php b/app/Console/Commands/MediaS3GarbageCollector.php index 81250e46a..f7d84f2bf 100644 --- a/app/Console/Commands/MediaS3GarbageCollector.php +++ b/app/Console/Commands/MediaS3GarbageCollector.php @@ -15,7 +15,7 @@ class MediaS3GarbageCollector extends Command * * @var string */ - protected $signature = 'media:s3gc {--limit=200}'; + protected $signature = 'media:s3gc {--limit=200} {--huge} {--log-errors}'; /** * The console command description. @@ -54,8 +54,23 @@ class MediaS3GarbageCollector extends Command } $limit = $this->option('limit'); + $hugeMode = $this->option('huge'); + $log = $this->option('log-errors'); + + if($limit > 2000 && !$hugeMode) { + $this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag'); + return; + } + $minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first()->id; + return $hugeMode ? + $this->hugeMode($minId, $limit, $log) : + $this->regularMode($minId, $limit, $log); + } + + protected function regularMode($minId, $limit, $log) + { $gc = Media::whereRemoteMedia(false) ->whereNotNull(['status_id', 'cdn_url', 'replicated_at']) ->whereNot('version', '4') @@ -84,7 +99,9 @@ class MediaS3GarbageCollector extends Command $media->save(); } } else { - Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]); + if($log) { + Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]); + } } $bar->advance(); } @@ -96,4 +113,42 @@ class MediaS3GarbageCollector extends Command } return 0; } + + protected function hugeMode($minId, $limit, $log) + { + $cloudDisk = Storage::disk(config('filesystems.cloud')); + $localDisk = Storage::disk('local'); + + $bar = $this->output->createProgressBar($limit); + $bar->start(); + + Media::whereRemoteMedia(false) + ->whereNotNull(['status_id', 'cdn_url', 'replicated_at']) + ->whereNot('version', '4') + ->where('id', '<', $minId) + ->chunk(50, function($medias) use($cloudDisk, $localDisk, $bar, $log) { + foreach($medias as $media) { + if($cloudDisk->exists($media->media_path)) { + if( $localDisk->exists($media->media_path)) { + $localDisk->delete($media->media_path); + $media->version = 4; + $media->save(); + $totalSize = $totalSize + $media->size; + } else { + $media->version = 4; + $media->save(); + } + } else { + if($log) { + Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]); + } + } + $bar->advance(); + } + }); + + $bar->finish(); + $this->line(' '); + $this->info('Finished!'); + } } From b3c4d8c47c8a102ae65a243a385cd45483d609d0 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 17 Dec 2022 21:44:43 -0700 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1ea43c7b..20b5ebab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ - Update InboxPipeline, add inbox job queue and separate http sig validation from activity handling ([e6c1604d](https://github.com/pixelfed/pixelfed/commit/e6c1604d)) - Update InboxPipeline, dispatch Follow/Accept Follow jobs to follow queue ([f62d2494](https://github.com/pixelfed/pixelfed/commit/f62d2494)) - Add MediaS3GarbageCollector command to clear local media after uploaded to S3 disks after 12 hours ([b8c3f153](https://github.com/pixelfed/pixelfed/commit/b8c3f153)) +- Update MediaS3GarbageCollector command, disable logging by default and optimize huge invocations ([a14af93b](https://github.com/pixelfed/pixelfed/commit/a14af93b)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)