From a7613baee658003f25d13daf96710be01a98c94f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 15 Jun 2022 03:32:04 -0600 Subject: [PATCH 1/9] Update ApiV1Controller, fix public timeline min/max id pagination --- app/Http/Controllers/Api/ApiV1Controller.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index 8723dfd7d..6478284b8 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -2015,6 +2015,19 @@ class ApiV1Controller extends Controller } $res = collect($feed) + ->filter(function($k) use($min, $max) { + if(!$min && !$max) { + return true; + } + + if($min) { + return $min != $k; + } + + if($max) { + return $max != $k; + } + }) ->map(function($k) use($user) { $status = StatusService::getMastodon($k); if(!$status || !isset($status['account']) || !isset($status['account']['id'])) { @@ -2032,7 +2045,6 @@ class ApiV1Controller extends Controller }) ->take($limit) ->values(); - // ->toArray(); $baseUrl = config('app.url') . '/api/v1/timelines/public?limit=' . $limit . '&'; if($remote) { From 9860b4a7359daa8f246af9a1c54055efdef132e7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 15 Jun 2022 03:32:43 -0600 Subject: [PATCH 2/9] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 884a62f8a..213839add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ - Update follower counts on follow_request approval ([e97900a0](https://github.com/pixelfed/pixelfed/commit/e97900a0)) - Update ApiV1Controller, improve local/remote logic in public timeline endpoint ([4ff179ad](https://github.com/pixelfed/pixelfed/commit/4ff179ad)) - Update ApiV1Controller, fix network timeline ([11e99d78](https://github.com/pixelfed/pixelfed/commit/11e99d78)) +- Update ApiV1Controller, fix public timeline min/max id pagination ([a7613bae](https://github.com/pixelfed/pixelfed/commit/a7613bae)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.3 (2022-05-09)](https://github.com/pixelfed/pixelfed/compare/v0.11.2...v0.11.3) From 44f4a9edd9ea248d3d82df37b6fed2b62ecee0df Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 18 Jun 2022 23:47:16 -0600 Subject: [PATCH 3/9] Improve CollectionService cache invalidation, fixes #3548 --- .editorconfig | 1 - app/Http/Controllers/CollectionController.php | 12 ++++++++++++ app/Jobs/StatusPipeline/StatusDelete.php | 15 +++++++++++++++ app/Services/CollectionService.php | 8 ++++++-- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.editorconfig b/.editorconfig index 3c44241cc..6b67635e7 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,7 +1,6 @@ root = true [*] -indent_style = space indent_size = 4 end_of_line = lf charset = utf-8 diff --git a/app/Http/Controllers/CollectionController.php b/app/Http/Controllers/CollectionController.php index 7a753d6aa..4e6272f04 100644 --- a/app/Http/Controllers/CollectionController.php +++ b/app/Http/Controllers/CollectionController.php @@ -132,6 +132,18 @@ class CollectionController extends Controller $collection = Collection::whereProfileId($profileId)->findOrFail($collectionId); $count = $collection->items()->count(); + if($count) { + CollectionItem::whereCollectionId($collection->id) + ->get() + ->filter(function($col) { + return StatusService::get($col->object_id, false) == null; + }) + ->each(function($col) use($collectionId) { + CollectionService::removeItem($collectionId, $col->object_id); + $col->delete(); + }); + } + $max = config('pixelfed.max_collection_length'); if($count >= $max) { abort(400, 'You can only add '.$max.' posts per collection'); diff --git a/app/Jobs/StatusPipeline/StatusDelete.php b/app/Jobs/StatusPipeline/StatusDelete.php index e0dc2a71c..64f5a1cba 100644 --- a/app/Jobs/StatusPipeline/StatusDelete.php +++ b/app/Jobs/StatusPipeline/StatusDelete.php @@ -5,6 +5,7 @@ namespace App\Jobs\StatusPipeline; use DB, Storage; use App\{ AccountInterstitial, + CollectionItem, MediaTag, Notification, Report, @@ -25,6 +26,7 @@ use GuzzleHttp\Pool; use GuzzleHttp\Client; use GuzzleHttp\Promise; use App\Util\ActivityPub\HttpSignature; +use App\Services\CollectionService; use App\Services\StatusService; use App\Services\MediaStorageService; @@ -89,6 +91,19 @@ class StatusDelete implements ShouldQueue $parent->save(); }); } + + DB::transaction(function() use($status) { + CollectionItem::whereObjectType('App\Status') + ->whereObjectId($status->id) + ->get() + ->each(function($col) { + $id = $col->collection_id; + $sid = $col->object_id; + $col->delete(); + CollectionService::removeItem($id, $sid); + }); + }); + DB::transaction(function() use($status) { $comments = Status::where('in_reply_to_id', $status->id)->get(); foreach ($comments as $comment) { diff --git a/app/Services/CollectionService.php b/app/Services/CollectionService.php index 33a3303ff..215e8cf46 100644 --- a/app/Services/CollectionService.php +++ b/app/Services/CollectionService.php @@ -9,7 +9,7 @@ use Illuminate\Support\Facades\Redis; class CollectionService { - const CACHE_KEY = 'pf:services:collections:'; + const CACHE_KEY = 'pf:services:collections-v1:'; public static function getItems($id, $start = 0, $stop = 10) { @@ -41,6 +41,9 @@ class CollectionService return CollectionItem::whereCollectionId($id) ->orderBy('order') ->get() + ->filter(function($item) use($id) { + return StatusService::get($item->object_id) != null; + }) ->each(function($item) use ($id) { self::addItem($id, $item->object_id, $item->order); }) @@ -80,13 +83,14 @@ class CollectionService 'visibility' => $collection->visibility, 'title' => $collection->title, 'description' => $collection->description, - 'thumb' => self::getThumb($id), + 'thumb' => '/storage/no-preview.png', 'url' => $collection->url(), 'published_at' => $collection->published_at ]; }); if($collection) { + $collection['thumb'] = self::getThumb($id); $collection['post_count'] = self::count($id); } From 8018c46f1f9e5d6bf4d2036179bd4fa8f8469bb9 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 18 Jun 2022 23:48:17 -0600 Subject: [PATCH 4/9] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 213839add..da951a980 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - Update ApiV1Controller, improve local/remote logic in public timeline endpoint ([4ff179ad](https://github.com/pixelfed/pixelfed/commit/4ff179ad)) - Update ApiV1Controller, fix network timeline ([11e99d78](https://github.com/pixelfed/pixelfed/commit/11e99d78)) - Update ApiV1Controller, fix public timeline min/max id pagination ([a7613bae](https://github.com/pixelfed/pixelfed/commit/a7613bae)) +- Improve CollectionService cache invalidation, fixes [#3548](https://github.com/pixelfed/pixelfed/issues/3548) ([44f4a9ed](https://github.com/pixelfed/pixelfed/commit/44f4a9ed)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.3 (2022-05-09)](https://github.com/pixelfed/pixelfed/compare/v0.11.2...v0.11.3) From 1eba7f810a798db61fe45d12ea282078a2a18140 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 19 Jun 2022 00:14:05 -0600 Subject: [PATCH 5/9] Update Inbox, improve status deletion cache invalidation --- app/Util/ActivityPub/Inbox.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index 2d48cb97f..e75a5228b 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -36,6 +36,7 @@ use App\Util\ActivityPub\Validator\UndoFollow as UndoFollowValidator; use App\Services\PollService; use App\Services\FollowerService; +use App\Services\StatusService; use App\Models\Conversation; class Inbox @@ -644,6 +645,7 @@ class Inbox if(!$status) { return; } + StatusService::del($status->id, true); Notification::whereActorId($profile->id) ->whereItemType('App\Status') ->whereItemId($status->id) From bb1cccbe030314d2d33c04d2c7cab05a0a556508 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 19 Jun 2022 06:52:06 -0600 Subject: [PATCH 6/9] Update MediaDeletePipeline, fix async media deletion --- app/Console/Commands/MediaGarbageCollector.php | 4 ++-- app/Http/Controllers/ComposeController.php | 2 -- app/Jobs/MediaPipeline/MediaDeletePipeline.php | 4 +++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/Console/Commands/MediaGarbageCollector.php b/app/Console/Commands/MediaGarbageCollector.php index 11d330ee8..c8fce9199 100644 --- a/app/Console/Commands/MediaGarbageCollector.php +++ b/app/Console/Commands/MediaGarbageCollector.php @@ -51,10 +51,10 @@ class MediaGarbageCollector extends Command $bar = $this->output->createProgressBar($gc->count()); $bar->start(); foreach($gc as $media) { - MediaStorageService::delete($media); - $media->forceDelete(); + MediaStorageService::delete($media, true); $bar->advance(); } $bar->finish(); + $this->line(''); } } diff --git a/app/Http/Controllers/ComposeController.php b/app/Http/Controllers/ComposeController.php index b6b40bc6e..4ce4ed03a 100644 --- a/app/Http/Controllers/ComposeController.php +++ b/app/Http/Controllers/ComposeController.php @@ -228,8 +228,6 @@ class ComposeController extends Controller MediaStorageService::delete($media, true); - $media->forceDelete(); - return response()->json([ 'msg' => 'Successfully deleted', 'code' => 200 diff --git a/app/Jobs/MediaPipeline/MediaDeletePipeline.php b/app/Jobs/MediaPipeline/MediaDeletePipeline.php index 3c486a9cc..df91c8316 100644 --- a/app/Jobs/MediaPipeline/MediaDeletePipeline.php +++ b/app/Jobs/MediaPipeline/MediaDeletePipeline.php @@ -57,7 +57,9 @@ class MediaDeletePipeline implements ShouldQueue $disk->deleteDirectory($i); } - return 1; + $media->forceDelete(); + + return; } } From 03a85460af9dbb55c412c427e77b88ab2a762cd9 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 19 Jun 2022 07:05:07 -0600 Subject: [PATCH 7/9] Fix timeline infinite scroll --- app/Http/Controllers/PublicApiController.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/Http/Controllers/PublicApiController.php b/app/Http/Controllers/PublicApiController.php index 5171509d3..8f9507feb 100644 --- a/app/Http/Controllers/PublicApiController.php +++ b/app/Http/Controllers/PublicApiController.php @@ -401,6 +401,7 @@ class PublicApiController extends Controller } $res = collect($feed) + ->take($limit) ->map(function($k) use($user) { $status = StatusService::get($k); if($status && isset($status['account']) && $user) { @@ -680,6 +681,7 @@ class PublicApiController extends Controller } $res = collect($feed) + ->take($limit) ->map(function($k) use($user) { $status = StatusService::get($k); if($status && isset($status['account']) && $user) { From 9ef998513373a19a6f6d39ff198f2bfb4412dbc2 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 19 Jun 2022 07:06:08 -0600 Subject: [PATCH 8/9] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da951a980..cb491c957 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,9 @@ - Update ApiV1Controller, fix network timeline ([11e99d78](https://github.com/pixelfed/pixelfed/commit/11e99d78)) - Update ApiV1Controller, fix public timeline min/max id pagination ([a7613bae](https://github.com/pixelfed/pixelfed/commit/a7613bae)) - Improve CollectionService cache invalidation, fixes [#3548](https://github.com/pixelfed/pixelfed/issues/3548) ([44f4a9ed](https://github.com/pixelfed/pixelfed/commit/44f4a9ed)) +- Improve inbox status deletion cache invalidation ([1eba7f81](https://github.com/pixelfed/pixelfed/commit/1eba7f81)) +- Update MediaDeletePipeline, fix async media deletion ([bb1cccbe](https://github.com/pixelfed/pixelfed/commit/bb1cccbe)) +- Fix timeline infinite scroll ([03a85460](https://github.com/pixelfed/pixelfed/commit/03a85460)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.3 (2022-05-09)](https://github.com/pixelfed/pixelfed/compare/v0.11.2...v0.11.3) From c32bb55b8a70d744f9400c519756334c3a4b0dba Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 20 Jun 2022 21:48:37 -0600 Subject: [PATCH 9/9] Update AP helpers, fix last_fetched_at condition --- app/Util/ActivityPub/Helpers.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index b72eeb32b..b6fea3b29 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -695,7 +695,7 @@ class Helpers { } if($profile = Profile::whereRemoteUrl($url)->first()) { - if($profile->last_fetched_at->lt(now()->subHours(24))) { + if($profile->last_fetched_at && $profile->last_fetched_at->lt(now()->subHours(24))) { return self::profileUpdateOrCreate($url); } return $profile;