refactor(cache): optimize caching and dispatching behaviour (#1039)

* refactor: just refactor

1. Moved getting object ID into separated method.
2. Add method to get object ids. Needs for `message_delete_bulk`, `emoji_update` and `stickers_update`
3. Cache `GuildMember` like `Member`
4. Deleting messages from cache on `message_delete_bulk` event

* chore: run pre-commit

* refactor: merge conditions into one

* refactor: add emojis to guild cache in attrs_post_init

* fix: typo

* refactor: move message cache control to method

- Remove message from cache on `message_delete` event

* ci: correct from checks.

* fix the fix of bug

* docs: capitalize return and ` to '

* refactor: leave only `message`

* ref: move `__modify_guild_cache` to own method

* ci: correct from checks.

* docs: add and update docstrings

* refactor: properly caching members and roles

* ci: correct from checks.

* refactor: refuse from raw remove events and ...

put `obj` to `__modify_guild_cache` if guild_obj is None

* ci: correct from checks.

* fix: `guild_obj` referenced before assignment

* fix: help me

There are only two models which can be converted to model without `Guild` prefix.

* refactor: use walrus operator

Co-authored-by: EdVraz <88881326+EdVraz@users.noreply.github.com>

* fix: add missed obj to dispatch

* refactor: remove extra dispatch

* refactor: message already removed from cache?

* refactor: remove `__delete_message_cache`

* refactor: dispatch `guild_emojis_update` without `raw`

* ci: correct from checks.

* refactor: change condition for `ids`

* refactor: change condition in getting ids

* feat: add support for `_execution` events

* refactor: dispatch update event if there are no id & ids

* revert: add support for `_execution` event

* refactor: some stuff

* refactor: use `cache.merge` to resolve guild overloading

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: EdVraz <88881326+EdVraz@users.noreply.github.com>

Author: 3 people

interactions/api/gateway/client.py

@@ -32,9 +32,12 @@
 from ..http.client import HTTPClient
 from ..models.flags import Intents
 from ..models.guild import Guild
+from ..models.gw import GuildMember, GuildRole
 from ..models.member import Member
+from ..models.message import Message
 from ..models.misc import Snowflake
 from ..models.presence import ClientPresence
+from ..models.role import Role
 from .heartbeat import _Heartbeat
 from .ratelimit import WSRateLimit
 
@@ -437,87 +440,64 @@ def _dispatch_event(self, event: str, data: dict) -> None:
         elif event not in {"TYPING_START", "VOICE_STATE_UPDATE", "VOICE_SERVER_UPDATE"}:
             name: str = event.lower()
             try:
+                data["_client"] = self._http
 
                 _event_path: list = [section.capitalize() for section in name.split("_")]
                 _name: str = _event_path[0] if len(_event_path) < 3 else "".join(_event_path[:-1])
                 model = getattr(__import__(path), _name)
-
-                data["_client"] = self._http
                 obj = model(**data)
 
-                _cache: "Storage" = self._http.cache[model]
+                guild_obj = guild_model = None
+                if model is GuildRole:
+                    guild_obj = Role(**role_data) if (role_data := data.get("role")) else None
+                    guild_model = Role
+                elif model is GuildMember:
+                    guild_obj = Member(**data)
+                    guild_model = Member
 
-                if isinstance(obj, Member):
-                    id = (Snowflake(data["guild_id"]), obj.id)
-                else:
-                    id = getattr(obj, "id", None)
+                _cache: "Storage" = self._http.cache[model]
+                _guild_cache: "Storage" = self._http.cache[guild_model]
 
+                ids = None
+                id = self.__get_object_id(data, obj, model)
                 if id is None:
-                    if model.__name__ == "GuildScheduledEventUser":
-                        id = model.guild_scheduled_event_id
-                    elif model.__name__ == "Presence":
-                        id = obj.user.id
-                    elif model.__name__ in [
-                        "Invite",
-                        "GuildBan",
-                        "ChannelPins",
-                        "MessageReaction",
-                        "MessageReactionRemove",
-                        "MessageDelete",
-                        # Extend this for everything that should not be cached
-                    ]:
-                        id = None
-                    elif model.__name__.startswith("Guild"):
-                        model_name = model.__name__[5:]
-                        if _data := getattr(obj, model_name, None):
-                            id = (
-                                getattr(_data, "id")
-                                if not isinstance(_data, dict)
-                                else Snowflake(_data["id"])
-                            )
-                        elif hasattr(obj, f"{model_name}_id"):
-                            id = getattr(obj, f"{model_name}_id", None)
-
-                def __modify_guild_cache():
-                    if not (
-                        (guild_id := data.get("guild_id"))
-                        and not isinstance(obj, Guild)
-                        and "message" not in name
-                        and id is not None
-                    ):
-                        return
-                    if guild := self._http.cache[Guild].get(Snowflake(guild_id)):
-                        model_name: str = model.__name__
-                        if "guild" in model_name:
-                            model_name = model_name[5:]
-                        elif model_name == "threadmembers":
-                            return
-                        _obj = getattr(guild, f"{model_name.lower()}s", None)
-                        if _obj is not None and isinstance(_obj, list):
-                            if "_create" in name or "_add" in name:
-                                _obj.append(obj)
-                            for index, __obj in enumerate(_obj):
-                                if __obj.id == id:
-                                    if "_remove" in name or "_delete" in name:
-                                        _obj.remove(__obj)
-
-                                    elif "_update" in name and hasattr(obj, "id"):
-                                        _obj[index] = obj
-                                    break
-                            setattr(guild, f"{model_name}s", _obj)
-                        self._http.cache[Guild].add(guild)
+                    ids = self.__get_object_ids(obj, model)
 
                 if "_create" in name or "_add" in name:
+                    self._dispatch.dispatch(f"on_{name}", obj)
+
                     if id:
                         _cache.merge(obj, id)
-                    self._dispatch.dispatch(f"on_{name}", obj)
-                    __modify_guild_cache()
+                        if guild_obj:
+                            _guild_cache.add(guild_obj, id)
+
+                    self.__modify_guild_cache(
+                        name, data, guild_model or model, guild_obj or obj, id, ids
+                    )
 
                 elif "_update" in name:
                     self._dispatch.dispatch(f"on_raw_{name}", obj)
-                    if not id:
+
+                    if not id and ids is None:
+                        return self._dispatch.dispatch(f"on_{name}", obj)
+
+                    self.__modify_guild_cache(
+                        name, data, guild_model or model, guild_obj or obj, id, ids
+                    )
+                    if ids is not None:
+                        # Not cached but it needed for guild_emojis_update and guild_stickers_update events
+                        return self._dispatch.dispatch(f"on_{name}", obj)
+                    if id is None:
                         return
-                    old_obj = self._http.cache[model].get(id)
+
+                    if guild_obj:
+                        old_guild_obj = _guild_cache.get(id)
+                        if old_guild_obj:
+                            old_guild_obj.update(**guild_obj._json)
+                        else:
+                            _guild_cache.add(guild_obj, id)
+
+                    old_obj = _cache.get(id)
                     if old_obj:
                         before = model(**old_obj._json)
                         old_obj.update(**obj._json)
@@ -526,27 +506,163 @@ def __modify_guild_cache():
                         old_obj = obj
 
                     _cache.add(old_obj, id)
-                    __modify_guild_cache()
-
                     self._dispatch.dispatch(
                         f"on_{name}", before, old_obj
                     )  # give previously stored and new one
 
                 elif "_remove" in name or "_delete" in name:
-                    self._dispatch.dispatch(f"on_raw_{name}", obj)
-                    __modify_guild_cache()
+                    self._dispatch.dispatch(
+                        f"on_raw_{name}", obj
+                    )  # Deprecated. Remove this in the future.
+
+                    old_obj = None
                     if id:
+                        _guild_cache.pop(id)
+                        self.__modify_guild_cache(
+                            name, data, guild_model or model, guild_obj or obj, id, ids
+                        )
                         old_obj = _cache.pop(id)
-                        self._dispatch.dispatch(f"on_{name}", old_obj)
-                    elif "_delete_bulk" in name:
-                        self._dispatch.dispatch(f"on_{name}", obj)
+
+                    elif ids is not None and "message" in name:
+                        # currently only message has '_delete_bulk' event but ig better keep this condition for future.
+                        _message_cache: "Storage" = self._http.cache[Message]
+                        for message_id in ids:
+                            _message_cache.pop(message_id)
+
+                    self._dispatch.dispatch(f"on_{name}", old_obj or obj)
 
                 else:
                     self._dispatch.dispatch(f"on_{name}", obj)
 
             except AttributeError as error:
                 log.warning(f"An error occurred dispatching {name}: {error}")
 
+    def __get_object_id(
+        self, data: dict, obj: Any, model: Any
+    ) -> Optional[Union[Snowflake, Tuple[Snowflake, Snowflake]]]:
+        """
+        Gets an ID from object.
+
+        :param data: The data for the event.
+        :type data: dict
+        :param obj: The object of the event.
+        :type obj: Any
+        :param model: The model of the event.
+        :type model: Any
+        :return: Object ID
+        :rtype: Optional[Union[Snowflake, Tuple[Snowflake, Snowflake]]]
+        """
+        if isinstance(obj, (Member, GuildMember)):
+            id = (Snowflake(data["guild_id"]), obj.id)
+        else:
+            id = getattr(obj, "id", None)
+        if id is not None:
+            return id
+
+        if model.__name__ == "GuildScheduledEventUser":
+            id = obj.guild_scheduled_event_id
+        elif model.__name__ == "Presence":
+            id = obj.user.id
+        elif model.__name__ in [
+            "GuildBan",
+            # Extend this for everything that starts with 'Guild' and should not be cached
+        ]:
+            id = None
+        elif model.__name__.startswith("Guild"):
+            model_name = model.__name__[5:].lower()
+            if (_data := getattr(obj, model_name, None)) and not isinstance(_data, list):
+                id = getattr(_data, "id") if not isinstance(_data, dict) else Snowflake(_data["id"])
+            elif hasattr(obj, f"{model_name}_id"):
+                id = getattr(obj, f"{model_name}_id", None)
+
+        return id
+
+    def __get_object_ids(self, obj: Any, model: Any) -> Optional[List[Snowflake]]:
+        """
+        Gets a list of ids of object.
+
+        :param obj: The object of the event.
+        :type obj: Any
+        :param model: The model of the event.
+        :type model: Any
+        :return: Object IDs
+        :rtype: Optional[Union[Snowflake, Tuple[Snowflake, Snowflake]]]
+        """
+        ids = getattr(obj, "ids", None)
+        if ids is not None:
+            return ids
+
+        if model.__name__.startswith("Guild"):
+            model_name = model.__name__[5:].lower()
+            if (_data := getattr(obj, model_name, None)) is not None:
+                ids = [
+                    getattr(_obj, "id") if not isinstance(_obj, dict) else Snowflake(_obj["id"])
+                    for _obj in _data
+                ]
+
+        return ids
+
+    def __modify_guild_cache(
+        self,
+        name: str,
+        data: dict,
+        model: Any,
+        obj: Any,
+        id: Optional[Snowflake] = None,
+        ids: Optional[List[Snowflake]] = None,
+    ):
+        """
+        Modifies guild cache.
+
+        :param event: The name of the event.
+        :type event: str
+        :param data: The data for the event.
+        :type data: dict
+        :param obj: The object of the event.
+        :type obj: Any
+        :param model: The model of the event.
+        :type model: Any
+        """
+        if not (
+            (guild_id := data.get("guild_id"))
+            and not isinstance(obj, Guild)
+            and "message" not in name
+            and (id is not None or ids is not None)
+            and (guild := self._http.cache[Guild].get(Snowflake(guild_id)))
+        ):
+            return
+
+        attr: str = model.__name__.lower()
+
+        if attr.startswith("guild"):
+            attr = attr[5:]
+        if attr == "threadmembers":  # TODO: Figure out why this here
+            return
+        if not attr.endswith("s"):
+            attr = f"{attr}s"
+        iterable = getattr(guild, attr, None)
+        if iterable is not None and isinstance(iterable, list):
+            if "_create" in name or "_add" in name:
+                iterable.append(obj)
+            if id:
+                _id = id[1] if isinstance(id, tuple) else id
+                for index, __obj in enumerate(iterable):
+                    if __obj.id == _id:
+                        if "_remove" in name or "_delete" in name:
+                            iterable.remove(__obj)
+
+                        elif "_update" in name and hasattr(obj, "id"):
+                            iterable[index] = obj
+                        break
+            elif ids is not None and "_update" in name:
+                objs = getattr(obj, attr, None)
+                if objs is not None:
+                    iterable.clear()
+                    iterable.extend(objs)
+            setattr(guild, attr, iterable)
+
+        self._http.cache[Guild].add(guild)
+
     def __contextualize(self, data: dict) -> "_Context":
         """
         Takes raw data given back from the Gateway

interactions/api/models/guild.py

@@ -471,6 +471,8 @@ def __attrs_post_init__(self):  # sourcery skip: last-if-guard
                     self.member_count = guild.member_count
                 if not self.presences:
                     self.presences = guild.presences
+                if not self.emojis:
+                    self.emojis = guild.emojis
 
         if self.members:
             for member in self.members: