From a0da6ec39adebd360bb10c8a8e964b3a31b84a5b Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 15 Jan 2023 02:35:10 -0700 Subject: [PATCH 1/4] Update InstanceService, improve unlisted/banned network post filtering --- app/Services/InstanceService.php | 16 +++++++++++++++- app/Services/NetworkTimelineService.php | 13 ++++++++++++- app/Util/ActivityPub/Helpers.php | 9 ++++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/app/Services/InstanceService.php b/app/Services/InstanceService.php index 3f051d66d..b53e8ac00 100644 --- a/app/Services/InstanceService.php +++ b/app/Services/InstanceService.php @@ -7,13 +7,14 @@ use App\Instance; class InstanceService { + const CACHE_KEY_BY_DOMAIN = 'pf:services:instance:by_domain:'; const CACHE_KEY_BANNED_DOMAINS = 'instances:banned:domains'; const CACHE_KEY_UNLISTED_DOMAINS = 'instances:unlisted:domains'; const CACHE_KEY_NSFW_DOMAINS = 'instances:auto_cw:domains'; public static function getByDomain($domain) { - return Cache::remember('pf:services:instance:by_domain:'.$domain, 3600, function() use($domain) { + return Cache::remember(self::CACHE_KEY_BY_DOMAIN.$domain, 3600, function() use($domain) { return Instance::whereDomain($domain)->first(); }); } @@ -50,4 +51,17 @@ class InstanceService return $instance->software; }); } + + public static function refresh() + { + Cache::forget(self::CACHE_KEY_BANNED_DOMAINS); + Cache::forget(self::CACHE_KEY_UNLISTED_DOMAINS); + Cache::forget(self::CACHE_KEY_NSFW_DOMAINS); + + self::getBannedDomains(); + self::getUnlistedDomains(); + self::getNsfwDomains(); + + return true; + } } diff --git a/app/Services/NetworkTimelineService.php b/app/Services/NetworkTimelineService.php index a25f6712c..52fc9b0bd 100644 --- a/app/Services/NetworkTimelineService.php +++ b/app/Services/NetworkTimelineService.php @@ -77,6 +77,11 @@ class NetworkTimelineService if(self::count() == 0 || $force == true) { $hideNsfw = config('instance.hide_nsfw_on_public_feeds'); Redis::del(self::CACHE_KEY); + $filteredDomains = collect(InstanceService::getBannedDomains()) + ->merge(InstanceService::getUnlistedDomains()) + ->unique() + ->values() + ->toArray(); $ids = Status::whereNotNull('uri') ->whereScope('public') ->when($hideNsfw, function($q, $hideNsfw) { @@ -88,7 +93,13 @@ class NetworkTimelineService ->where('created_at', '>', now()->subHours(config('instance.timeline.network.max_hours_old'))) ->orderByDesc('created_at') ->limit($limit) - ->pluck('id'); + ->pluck('uri', 'id'); + $ids = $ids->filter(function($k, $v) use($filteredDomains) { + $domain = parse_url($k, PHP_URL_HOST); + return !in_array($domain, $filteredDomains); + })->map(function($k, $v) { + return $v; + })->flatten(); foreach($ids as $id) { self::add($id); } diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index b5603666c..39e7a8bf2 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -510,7 +510,14 @@ class Helpers { $status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old'))) && (config('instance.hide_nsfw_on_public_feeds') == true ? $status->is_nsfw == false : true) ) { - NetworkTimelineService::add($status->id); + $filteredDomains = collect(InstanceService::getBannedDomains()) + ->merge(InstanceService::getUnlistedDomains()) + ->unique() + ->values() + ->toArray(); + if(!in_array($urlDomain, $filteredDomains)) { + NetworkTimelineService::add($status->id); + } } IncrementPostCount::dispatch($pid)->onQueue('low'); From 042036847708f7dd0ba389c0f22811b7c423ada2 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 15 Jan 2023 02:35:50 -0700 Subject: [PATCH 2/4] Add AdminInstance resource --- app/Http/Resources/AdminInstance.php | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 app/Http/Resources/AdminInstance.php diff --git a/app/Http/Resources/AdminInstance.php b/app/Http/Resources/AdminInstance.php new file mode 100644 index 000000000..4a52e3481 --- /dev/null +++ b/app/Http/Resources/AdminInstance.php @@ -0,0 +1,31 @@ + $this->id, + 'domain' => $this->domain, + 'software' => $this->software, + 'unlisted' => (bool) $this->unlisted, + 'auto_cw' => (bool) $this->auto_cw, + 'banned' => (bool) $this->banned, + 'user_count' => $this->user_count, + 'status_count' => $this->status_count, + 'last_crawled_at' => $this->last_crawled_at, + 'actors_last_synced_at' => $this->actors_last_synced_at, + 'created_at' => $this->created_at, + ]; + } +} From d5dc92822407689dc4b2307b025d6871811696a2 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 15 Jan 2023 02:36:55 -0700 Subject: [PATCH 3/4] Update admin api --- .../Controllers/Api/AdminApiController.php | 94 +++++++++++++++++++ routes/api.php | 4 + 2 files changed, 98 insertions(+) diff --git a/app/Http/Controllers/Api/AdminApiController.php b/app/Http/Controllers/Api/AdminApiController.php index ad587c756..94707ca8d 100644 --- a/app/Http/Controllers/Api/AdminApiController.php +++ b/app/Http/Controllers/Api/AdminApiController.php @@ -9,6 +9,7 @@ use Auth, Cache, DB; use Carbon\Carbon; use App\{ AccountInterstitial, + Instance, Like, Media, Profile, @@ -19,9 +20,12 @@ use App\{ use App\Services\AccountService; use App\Services\AdminStatsService; use App\Services\ConfigCacheService; +use App\Services\InstanceService; use App\Services\ModLogService; use App\Services\StatusService; +use App\Services\NetworkTimelineService; use App\Services\NotificationService; +use App\Http\Resources\AdminInstance; use App\Http\Resources\AdminUser; class AdminApiController extends Controller @@ -486,4 +490,94 @@ class AdminApiController extends Controller ] ]]); } + + public function instances(Request $request) + { + abort_if(!$request->user(), 404); + abort_unless($request->user()->is_admin === 1, 404); + $this->validate($request, [ + 'q' => 'sometimes', + 'sort' => 'sometimes|in:asc,desc', + 'sort_by' => 'sometimes|in:id,status_count,user_count,domain', + 'filter' => 'sometimes|in:all,unlisted,auto_cw,banned', + ]); + + $q = $request->input('q'); + $sort = $request->input('sort', 'desc') === 'asc' ? 'asc' : 'desc'; + $sortBy = $request->input('sort_by', 'id'); + $filter = $request->input('filter'); + + $res = Instance::when($q, function($query, $q) { + return $query->where('domain', 'like', '%' . $q . '%'); + }) + ->when($filter, function($query, $filter) { + if($filter === 'all') { + return $query; + } else { + return $query->where($filter, true); + } + }) + ->when($sortBy, function($query, $sortBy) use($sort) { + return $query->orderBy($sortBy, $sort); + }, function($query) { + return $query->orderBy('id', 'desc'); + }) + ->cursorPaginate(10) + ->withQueryString(); + + return AdminInstance::collection($res); + } + + public function getInstance(Request $request) + { + abort_if(!$request->user(), 404); + abort_unless($request->user()->is_admin === 1, 404); + + $id = $request->input('id'); + $res = Instance::findOrFail($id); + + return new AdminInstance($res); + } + + public function moderateInstance(Request $request) + { + abort_if(!$request->user(), 404); + abort_unless($request->user()->is_admin === 1, 404); + + $this->validate($request, [ + 'id' => 'required', + 'key' => 'required|in:unlisted,auto_cw,banned', + 'value' => 'required' + ]); + + $id = $request->input('id'); + $key = $request->input('key'); + $value = (bool) filter_var($request->input('value'), FILTER_VALIDATE_BOOLEAN); + $res = Instance::findOrFail($id); + $res->{$key} = $value; + $res->save(); + + InstanceService::refresh(); + NetworkTimelineService::warmCache(true); + + return new AdminInstance($res); + } + + public function refreshInstanceStats(Request $request) + { + abort_if(!$request->user(), 404); + abort_unless($request->user()->is_admin === 1, 404); + + $this->validate($request, [ + 'id' => 'required', + ]); + + $id = $request->input('id'); + $instance = Instance::findOrFail($id); + $instance->user_count = Profile::whereDomain($instance->domain)->count(); + $instance->status_count = Profile::whereDomain($instance->domain)->leftJoin('statuses', 'profiles.id', '=', 'statuses.profile_id')->count(); + $instance->save(); + + return new AdminInstance($instance); + } } diff --git a/routes/api.php b/routes/api.php index f3ab0c31f..72fbf53c1 100644 --- a/routes/api.php +++ b/routes/api.php @@ -193,5 +193,9 @@ Route::group(['prefix' => 'api'], function() use($middleware) { Route::get('users/list', 'Api\AdminApiController@getUsers')->middleware($middleware); Route::get('users/get', 'Api\AdminApiController@getUser')->middleware($middleware); Route::post('users/action', 'Api\AdminApiController@userAdminAction')->middleware($middleware); + Route::get('instances/list', 'Api\AdminApiController@instances')->middleware($middleware); + Route::get('instances/get', 'Api\AdminApiController@getInstance')->middleware($middleware); + Route::post('instances/moderate', 'Api\AdminApiController@moderateInstance')->middleware($middleware); + Route::post('instances/refresh-stats', 'Api\AdminApiController@refreshInstanceStats')->middleware($middleware); }); }); From 8bfdf2c29ec06f839393ab96c713f5c24a4dc485 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 15 Jan 2023 02:38:45 -0700 Subject: [PATCH 4/4] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec7abf287..1ff807a5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ - Update database config, add sticky flag https://laravel.com/docs/9.x/database#the-sticky-option ([10b65980](https://github.com/pixelfed/pixelfed/commit/10b65980)) - Update profile audience to filter blocked instances ([e0c3dae3](https://github.com/pixelfed/pixelfed/commit/e0c3dae3)) - Update SearchApiV2Service, improve query performance ([4d1f2811](https://github.com/pixelfed/pixelfed/commit/4d1f2811)) +- Update InstanceService, improve unlisted/banned network post filtering ([a0da6ec3](https://github.com/pixelfed/pixelfed/commit/a0da6ec3)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)