Optimize HTTP connection pooling and config I/O in high-frequency event handlers

activity_stats/activity_stats.py

@@ -81,27 +81,29 @@ async def _stop_tracking_game(self, guild_config, user_id: int, game_name: str,
             duration = timestamp - start_time
 
             if duration > 0:
-                # Update global game stats
-                async with guild_config.game_stats() as game_stats:
+                # Batch all config updates in a single context manager access
+                async with guild_config.all() as all_data:
+                    # Update global game stats
+                    game_stats = all_data.setdefault("game_stats", {})
                     if game_name not in game_stats:
                         game_stats[game_name] = 0
                     game_stats[game_name] += duration
 
-                # Update user game stats
-                async with guild_config.user_game_stats() as user_game_stats:
+                    # Update user game stats
+                    user_game_stats = all_data.setdefault("user_game_stats", {})
                     if user_id_str not in user_game_stats:
                         user_game_stats[user_id_str] = {}
                     if game_name not in user_game_stats[user_id_str]:
                         user_game_stats[user_id_str][game_name] = 0
                     user_game_stats[user_id_str][game_name] += duration
 
-            # Remove from last_activity
-            async with guild_config.last_activity() as last_activity_update:
-                if user_id_str in last_activity_update and game_name in last_activity_update[user_id_str]:
-                    del last_activity_update[user_id_str][game_name]
-                    # Clean up empty user entries
-                    if not last_activity_update[user_id_str]:
-                        del last_activity_update[user_id_str]
+                    # Remove from last_activity
+                    last_activity_update = all_data.setdefault("last_activity", {})
+                    if user_id_str in last_activity_update and game_name in last_activity_update[user_id_str]:
+                        del last_activity_update[user_id_str][game_name]
+                        # Clean up empty user entries
+                        if not last_activity_update[user_id_str]:
+                            del last_activity_update[user_id_str]
 
     @commands.guild_only()
     @commands.group(name="activity", invoke_without_command=True)

albion_auth/auth.py

@@ -10,32 +10,54 @@
 log = logging.getLogger("red.cogs.albion_auth")
 
 
-async def http_get(url, params=None):
-    """Make HTTP GET request with retries"""
+async def http_get(url, params=None, client=None):
+    """Make HTTP GET request with retries
+
+    Args:
+        url: URL to fetch
+        params: Query parameters
+        client: Optional httpx.AsyncClient to reuse. If None, creates a new one.
+    """
     max_attempts = 3
     attempt = 0
     log.info(f"Making HTTP GET request to {url} with params: {params}")
-    while attempt < max_attempts:
-        try:
-            async with httpx.AsyncClient() as client:
+
+    # Create client if not provided
+    should_close = client is None
+    if should_close:
+        client = httpx.AsyncClient()
+
+    try:
+        while attempt < max_attempts:
+            try:
                 r = await client.get(url, params=params, timeout=10.0)
 
-            if r.status_code == 200:
-                response_data = r.json()
-                log.info(f"HTTP GET successful for {url} - Status: {r.status_code}")
-                log.debug(f"Response data: {response_data}")
-                return response_data
-            else:
+                if r.status_code == 200:
+                    response_data = r.json()
+                    log.info(f"HTTP GET successful for {url} - Status: {r.status_code}")
+                    log.debug(f"Response data: {response_data}")
+                    return response_data
+                else:
+                    attempt += 1
+                    log.warning(
+                        f"HTTP GET failed for {url} - Status: {r.status_code}, "
+                        f"Attempt {attempt}/{max_attempts}"
+                    )
+                    await asyncio.sleep(2)
+            except (httpx.ConnectTimeout, httpx.RequestError) as e:
                 attempt += 1
-                log.warning(f"HTTP GET failed for {url} - Status: {r.status_code}, Attempt {attempt}/{max_attempts}")
+                log.warning(
+                    f"HTTP GET error for {url}: {type(e).__name__}: {str(e)}, "
+                    f"Attempt {attempt}/{max_attempts}"
+                )
                 await asyncio.sleep(2)
-        except (httpx.ConnectTimeout, httpx.RequestError) as e:
-            attempt += 1
-            log.warning(f"HTTP GET error for {url}: {type(e).__name__}: {str(e)}, Attempt {attempt}/{max_attempts}")
-            await asyncio.sleep(2)
 
-    log.error(f"HTTP GET failed after {max_attempts} attempts for {url}")
-    return None
+        log.error(f"HTTP GET failed after {max_attempts} attempts for {url}")
+        return None
+    finally:
+        # Only close if we created it
+        if should_close:
+            await client.aclose()
 
 
 class AlbionAuth(commands.Cog):
@@ -51,6 +73,7 @@ def __init__(self, bot):
             enable_daily_check=True
         )
         self._check_task = None
+        self._http_client = httpx.AsyncClient()
 
     async def cog_load(self):
         """Start the background task when cog loads"""
@@ -62,12 +85,15 @@ async def cog_unload(self):
         if self._check_task:
             self._check_task.cancel()
             log.info("Cancelled daily name check task")
+        if self._http_client:
+            await self._http_client.aclose()
+            log.info("Closed HTTP client")
 
     async def search_player_in_region(self, name, region_url, region_name):
         """Search for a player by name in a specific region"""
         log.info(f"Searching for player '{name}' in {region_name} region")
         params = {"q": name}
-        result = await http_get(region_url, params)
+        result = await http_get(region_url, params, client=self._http_client)
 
         if result and result.get("players"):
             player = result["players"][0]

albion_regear/regear.py

@@ -8,39 +8,67 @@
 log = logging.getLogger("red.cogs.albion_regear")
 
 
-async def http_get(url, params=None):
-    """Make HTTP GET request with retries"""
+async def http_get(url, params=None, client=None):
+    """Make HTTP GET request with retries
+
+    Args:
+        url: URL to fetch
+        params: Query parameters
+        client: Optional httpx.AsyncClient to reuse. If None, creates a new one.
+    """
     max_attempts = 3
     attempt = 0
     log.info(f"Making HTTP GET request to {url} with params: {params}")
-    while attempt < max_attempts:
-        try:
-            async with httpx.AsyncClient() as client:
+
+    # Create client if not provided
+    should_close = client is None
+    if should_close:
+        client = httpx.AsyncClient()
+
+    try:
+        while attempt < max_attempts:
+            try:
                 r = await client.get(url, params=params, timeout=10.0)
 
-            if r.status_code == 200:
-                response_data = r.json()
-                log.info(f"HTTP GET successful for {url} - Status: {r.status_code}")
-                log.debug(f"Response data: {response_data}")
-                return response_data
-            else:
+                if r.status_code == 200:
+                    response_data = r.json()
+                    log.info(f"HTTP GET successful for {url} - Status: {r.status_code}")
+                    log.debug(f"Response data: {response_data}")
+                    return response_data
+                else:
+                    attempt += 1
+                    log.warning(
+                        f"HTTP GET failed for {url} - Status: {r.status_code}, "
+                        f"Attempt {attempt}/{max_attempts}"
+                    )
+                    await asyncio.sleep(2)
+            except (httpx.ConnectTimeout, httpx.RequestError) as e:
                 attempt += 1
-                log.warning(f"HTTP GET failed for {url} - Status: {r.status_code}, Attempt {attempt}/{max_attempts}")
+                log.warning(
+                    f"HTTP GET error for {url}: {type(e).__name__}: {str(e)}, "
+                    f"Attempt {attempt}/{max_attempts}"
+                )
                 await asyncio.sleep(2)
-        except (httpx.ConnectTimeout, httpx.RequestError) as e:
-            attempt += 1
-            log.warning(f"HTTP GET error for {url}: {type(e).__name__}: {str(e)}, Attempt {attempt}/{max_attempts}")
-            await asyncio.sleep(2)
 
-    log.error(f"HTTP GET failed after {max_attempts} attempts for {url}")
-    return None
+        log.error(f"HTTP GET failed after {max_attempts} attempts for {url}")
+        return None
+    finally:
+        # Only close if we created it
+        if should_close:
+            await client.aclose()
 
 
 class AlbionRegear(commands.Cog):
     """Calculates regear costs for Albion Online deaths"""
 
     def __init__(self, bot):
         self.bot = bot
+        self._http_client = httpx.AsyncClient()
+
+    async def cog_unload(self):
+        """Close HTTP client when cog unloads"""
+        if self._http_client:
+            await self._http_client.aclose()
 
     def normalize_quality(self, quality):
         """Normalize quality value for price lookups
@@ -81,7 +109,7 @@ async def search_player(self, name):
         log.info(f"Searching for player: {name}")
         url = "https://gameinfo-ams.albiononline.com/api/gameinfo/search"
         params = {"q": name}
-        result = await http_get(url, params)
+        result = await http_get(url, params, client=self._http_client)
 
         if result and result.get("players"):
             player = result["players"][0]
@@ -95,7 +123,7 @@ async def get_latest_death(self, player_id):
         """Get the latest death event for a player"""
         log.info(f"Fetching latest death for player ID: {player_id}")
         url = f"https://gameinfo-ams.albiononline.com/api/gameinfo/players/{player_id}/deaths"
-        result = await http_get(url)
+        result = await http_get(url, client=self._http_client)
 
         if result and len(result) > 0:
             death = result[0]
@@ -127,7 +155,7 @@ async def get_item_prices(self, items_with_quality):
         item_list = ",".join(items_with_quality.keys())
         log.info(f"Fetching prices for {len(items_with_quality)} items: {list(items_with_quality.keys())}")
         url = f"https://europe.albion-online-data.com/api/v2/stats/prices/{item_list}?locations=Bridgewatch"
-        result = await http_get(url)
+        result = await http_get(url, client=self._http_client)
 
         if not result:
             log.error("Failed to fetch item prices - API returned no data")

movie_vote/movie_vote.py

@@ -31,6 +31,12 @@ def __init__(self, bot):
             "notify_episode": [],
         }
         self.config.register_guild(**default_guild)
+        self._http_client = httpx.AsyncClient()
+
+    async def cog_unload(self):
+        """Close HTTP client when cog unloads"""
+        if self._http_client:
+            await self._http_client.aclose()
 
     async def red_delete_data_for_user(self, **kwargs):
         """Nothing to delete."""
@@ -39,7 +45,8 @@ async def red_delete_data_for_user(self, **kwargs):
     async def get_latest_episodes(self, imdb_id: str) -> Union[Dict[str, Any], None]:
         """Get the latest episodes from vidsrc"""
         response = await http_get(
-                "https://vidsrc.me/episodes/latest/page-1.json"
+                "https://vidsrc.me/episodes/latest/page-1.json",
+                client=self._http_client
             )
         if not response:
             log.info("Response was empty. %s", response)
@@ -421,14 +428,28 @@ async def on_raw_reaction_add(self, payload):
         if not await self._is_movie_channel(payload):
             return
 
-        channel = await self.bot.fetch_channel(payload.channel_id)
-        message = await channel.fetch_message(payload.message_id)
-        user = await self.bot.fetch_user(payload.user_id)
-        emoji = payload.emoji
+        # Ignore bot's own reactions
+        if payload.user_id == self.bot.user.id:
+            return
 
-        if user.id == self.bot.user.id:
+        # Use get methods first (cached), fallback to fetch only if needed
+        channel = self.bot.get_channel(payload.channel_id)
+        if not channel:
+            channel = await self.bot.fetch_channel(payload.channel_id)
+
+        if not channel:
+            log.warning(f"Channel {payload.channel_id} not found for reaction add event")
             return
 
+        # Fetch message to get reactions
+        message = await channel.fetch_message(payload.message_id)
+
+        user = self.bot.get_user(payload.user_id)
+        if not user:
+            user = await self.bot.fetch_user(payload.user_id)
+
+        emoji = payload.emoji
+
         log.info(f"Reaction added. {user.name} on '{message.clean_content}'")
         await self.count_votes(message, emoji)
 
@@ -438,14 +459,28 @@ async def on_raw_reaction_remove(self, payload):
         if not await self._is_movie_channel(payload):
             return
 
-        channel = await self.bot.fetch_channel(payload.channel_id)
-        message = await channel.fetch_message(payload.message_id)
-        user = await self.bot.fetch_user(payload.user_id)
-        emoji = payload.emoji
+        # Ignore bot's own reactions
+        if payload.user_id == self.bot.user.id:
+            return
 
-        if user.id == self.bot.user.id:
+        # Use get methods first (cached), fallback to fetch only if needed
+        channel = self.bot.get_channel(payload.channel_id)
+        if not channel:
+            channel = await self.bot.fetch_channel(payload.channel_id)
+
+        if not channel:
+            log.warning(f"Channel {payload.channel_id} not found for reaction remove event")
             return
 
+        # Fetch message to get reactions
+        message = await channel.fetch_message(payload.message_id)
+
+        user = self.bot.get_user(payload.user_id)
+        if not user:
+            user = await self.bot.fetch_user(payload.user_id)
+
+        emoji = payload.emoji
+
         log.info(f"Reaction removed. {user.name} on '{message.clean_content}'")
         await self.count_votes(message, emoji)
 
@@ -578,20 +613,35 @@ def fix_custom_emoji(self, emoji):
         return None
 
 
-async def http_get(url):
+async def http_get(url, client=None):
+    """Make HTTP GET request with retries
+
+    Args:
+        url: URL to fetch
+        client: Optional httpx.AsyncClient to reuse. If None, creates a new one.
+    """
     max_attempts = 3
     attempt = 0
-    while (
-        max_attempts > attempt
-    ):  # httpx doesn't support retries, so we'll build our own basic loop for that
-        try:
-            async with httpx.AsyncClient() as client:
+
+    # Create client if not provided
+    should_close = client is None
+    if should_close:
+        client = httpx.AsyncClient()
+
+    try:
+        while max_attempts > attempt:
+            try:
                 r = await client.get(url, headers={"user-agent": "psykzz-cogs/1.0.0"})
-            if r.status_code == 200:
-                return r.json()
-            else:
+                if r.status_code == 200:
+                    return r.json()
+                else:
+                    attempt += 1
+                    await asyncio.sleep(1)
+            except (httpx.ConnectTimeout, httpx.RequestError):
                 attempt += 1
-            await asyncio.sleep(1)
-        except (httpx._exceptions.ConnectTimeout, httpx._exceptions.HTTPError):
-            attempt += 1
-            await asyncio.sleep(1)
+                await asyncio.sleep(1)
+        return None
+    finally:
+        # Only close if we created it
+        if should_close:
+            await client.aclose()