Merge pull request #12130 from nanaya/beatmap-es-index-speedup

Speed up beatmapset es indexing

Author: notbakaneko

.env.example

@@ -326,6 +326,7 @@ CLIENT_CHECK_VERSION=false
 
 # TAGS_CACHE_DURATION=60
 # BEATMAP_TAGS_CACHE_DURATION=60
+# BEATMAP_TOP_TAG_COUNT=50
 
 # OSU_SENTRY_MIN_LOG_DURATION_MS=500
 # SENTRY_TRACES_SAMPLE_RATE=

app/Models/Beatmap.php

@@ -307,6 +307,7 @@ public function getAttribute($key)
             'baseMaxCombo',
             'beatmapDiscussions',
             'beatmapOwners',
+            'beatmapTags',
             'beatmapset',
             'difficulty',
             'difficultyAttribs',
@@ -392,6 +393,22 @@ public function maxCombo()
         return $maxCombo?->value;
     }
 
+    public function slowTopTagIds(): array
+    {
+        return $this->memoize(__FUNCTION__, function () {
+            $countById = [];
+            foreach ($this->beatmapTags as $vote) {
+                $countById[$vote->tag_id] ??= ['tag_id' => $vote->tag_id, 'count' => 0];
+                $countById[$vote->tag_id]['count']++;
+            }
+            usort($countById, fn ($a, $b) => $a['count'] === $b['count']
+                ? $a['tag_id'] - $b['tag_id']
+                : $b['count'] - $a['count']);
+
+            return array_slice($countById, 0, $GLOBALS['cfg']['osu']['tags']['top_tag_count']);
+        });
+    }
+
     public function status()
     {
         return array_search($this->approved, Beatmapset::STATES, true);
@@ -405,7 +422,7 @@ public function topTagIds()
             fn () => \Cache::remember(
                 "beatmap_top_tag_ids:{$this->getKey()}",
                 $GLOBALS['cfg']['osu']['tags']['beatmap_tags_cache_duration'],
-                fn () => $this->beatmapTags()->topTagIds()->limit(50)->get()->toArray(),
+                fn () => $this->beatmapTags()->topTagIds()->limit($GLOBALS['cfg']['osu']['tags']['top_tag_count'])->get()->toArray(),
             ),
         );
     }

app/Models/BeatmapTag.php

@@ -25,9 +25,14 @@ class BeatmapTag extends Model
     protected $primaryKey = ':composite';
     protected $primaryKeys = ['beatmap_id', 'tag_id', 'user_id'];
 
+    public function scopeDefault(Builder $query): Builder
+    {
+        return $query->whereHas('user', fn ($userQuery) => $userQuery->default());
+    }
+
     public function scopeTopTagIds(Builder $query)
     {
-        return $query->whereHas('user', fn ($userQuery) => $userQuery->default())
+        return $query->default()
             ->groupBy('tag_id')
             ->select('tag_id')
             ->selectRaw('COUNT(*) as count')

app/Models/Traits/Es/BeatmapsetSearch.php

@@ -22,6 +22,7 @@ public static function esIndexingQuery()
         return static::withoutGlobalScopes()
             ->active()
             ->with('beatmaps') // note that the with query will run with the default scopes.
+            ->with(['beatmaps.beatmapTags' => fn ($q) => $q->default()])
             ->with('beatmaps.beatmapOwners')
             ->with('beatmaps.baseDifficultyRatings');
     }
@@ -38,7 +39,7 @@ private static function esBeatmapTags(Beatmap $beatmap): array
         return array_reject_null(
             array_map(
                 fn ($tagId) => $tags->get($tagId['tag_id'])?->name,
-                $beatmap->topTagIds()
+                $beatmap->slowTopTagIds()
             )
         );
     }

config/osu.php

@@ -205,8 +205,9 @@
         'notice' => presence(str_replace('\n', "\n", env('STORE_NOTICE') ?? '')),
     ],
     'tags' => [
-        'tags_cache_duration' => 60 * (get_int(env('TAGS_CACHE_DURATION')) ?? 60), // in minutes, converted to seconds
         'beatmap_tags_cache_duration' => 60 * (get_int(env('BEATMAP_TAGS_CACHE_DURATION')) ?? 60), // in minutes, converted to seconds
+        'tags_cache_duration' => 60 * (get_int(env('TAGS_CACHE_DURATION')) ?? 60), // in minutes, converted to seconds
+        'top_tag_count' => get_int(env('BEATMAP_TOP_TAG_COUNT')) ?? 50,
     ],
     'team' => [
         'create_require_supporter' => get_bool(env('TEAM_CREATE_REQUIRE_SUPPORTER')) ?? false,