From 1968bcf79524e9663264a8be0b0da57e8a349c08 Mon Sep 17 00:00:00 2001 From: Mitch Woollatt <21175347+MitchellAW@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:12:30 +0930 Subject: [PATCH 1/7] Add headers to logs for non-2xx responses --- compuglobal/api/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compuglobal/api/client.py b/compuglobal/api/client.py index 927401a..5c17103 100644 --- a/compuglobal/api/client.py +++ b/compuglobal/api/client.py @@ -54,7 +54,7 @@ async def get(self, url: str, params: dict[str, Any] | None = None) -> dict[str, log.debug("Response %s | GET %s", response.status, url) return await response.json() - log.error("Non-2xx response %s | GET %s", response.status, url) + log.error("Non-2xx response %s | POST %s | Headers %s", response.status, url, response.headers) raise APIPageStatusError(response.status, self.base_url) async def post_data(self, url: str, json: dict[str, Any] | list[Any] | None) -> str: @@ -83,7 +83,7 @@ async def post_data(self, url: str, json: dict[str, Any] | list[Any] | None) -> log.debug("Response %s | POST %s", response.status, url) return await response.text() - log.error("Non-2xx response %s | POST %s") + log.error("Non-2xx response %s | POST %s | Headers %s", response.status, url, response.headers) raise APIPageStatusError(response.status, self.base_url) async def handle_request(self, request: PreparedRequest) -> str | dict[str, Any] | list[Any]: From 890ae15e7bb45fc27a86a3bd5d3f5c7b6a2256c8 Mon Sep 17 00:00:00 2001 From: Mitch Woollatt <21175347+MitchellAW@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:31:35 +0930 Subject: [PATCH 2/7] Add a retries error & retry_after to status errors --- compuglobal/__init__.py | 3 ++- compuglobal/errors.py | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/compuglobal/__init__.py b/compuglobal/__init__.py index 9df559a..4e91e53 100644 --- a/compuglobal/__init__.py +++ b/compuglobal/__init__.py @@ -8,7 +8,7 @@ MasterOfAllScience, Morbotron, ) -from compuglobal.errors import APIPageStatusError, NoSearchResultsFoundError +from compuglobal.errors import APIPageStatusError, MaximumRetriesExceededError, NoSearchResultsFoundError from compuglobal.models.comic import ComicLayout, ComicOverlay, ComicPanel, ComicStrip from compuglobal.models.episode import Episode, EpisodeMetadata, EpisodeSummary from compuglobal.models.font import FontAlignment, FontColor, FontFamily @@ -42,6 +42,7 @@ "FrameResult", "Frinkiac", "MasterOfAllScience", + "MaximumRetriesExceededError", "Morbotron", "NoSearchResultsFoundError", "OverlayFormat", diff --git a/compuglobal/errors.py b/compuglobal/errors.py index b595a9f..6f3e99e 100644 --- a/compuglobal/errors.py +++ b/compuglobal/errors.py @@ -1,5 +1,7 @@ """All errors/exceptions returned by CompuGlobal APIs.""" +from http import HTTPStatus + class NoSearchResultsFoundError(Exception): """Raised when no search results are found.""" @@ -8,6 +10,13 @@ def __init__(self, message: str = "No search results found.") -> None: super().__init__(message) +class MaximumRetriesExceededError(Exception): + """Raised when the maximum number of retries is exceeded.""" + + def __init__(self, message: str = "Maximum number of retries exceeded.") -> None: + super().__init__(message) + + class APIPageStatusError(Exception): """Raised when the status code for the API is not 200. @@ -17,8 +26,17 @@ class APIPageStatusError(Exception): The page status code returned by the API. url : str The base url that raised the error. + retry_after : int | None + Number of seconds to wait before retrying if error was a 429. """ - def __init__(self, page_status: int, url: str) -> None: - super().__init__(f"Error {page_status}. {url} may be down.") + def __init__(self, page_status: int, url: str, retry_after: int | None = None) -> None: + self.page_status = page_status + self.url = url + self.retry_after = retry_after + + msg = f"Error {page_status}. {url} may be down." + if page_status == HTTPStatus.TOO_MANY_REQUESTS and self.retry_after is not None: + msg += f" Retry after {retry_after} seconds." + super().__init__(msg) From 943b814f5e384994501d2dd005449d38e3e790b7 Mon Sep 17 00:00:00 2001 From: Mitch Woollatt <21175347+MitchellAW@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:32:56 +0930 Subject: [PATCH 3/7] Retry 429 requests in HTTP client with max_retries Retries requests up to max_retries for all 429 page status errors. --- compuglobal/api/client.py | 62 ++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/compuglobal/api/client.py b/compuglobal/api/client.py index 5c17103..624f23f 100644 --- a/compuglobal/api/client.py +++ b/compuglobal/api/client.py @@ -1,5 +1,6 @@ """Module for handling API requests to CGHMC APIs.""" +import asyncio import logging from http import HTTPStatus from typing import Any @@ -7,7 +8,7 @@ from aiohttp import ClientSession from compuglobal.api.endpoint import PreparedRequest, RequestMethod -from compuglobal.errors import APIPageStatusError +from compuglobal.errors import APIPageStatusError, MaximumRetriesExceededError log = logging.getLogger(__name__) @@ -21,12 +22,15 @@ class CompuGlobalAPIClient: The base URL of the API (e.g. https://frinkiac.com) session : ClientSession The client session to use for all API requests + max_retries : int, optional + The maximum number of retries for each request before raising an :class:`APIPageStatusError` """ - def __init__(self, base_url: str, session: ClientSession) -> None: + def __init__(self, base_url: str, session: ClientSession, max_retries: int = 0) -> None: self.base_url = base_url self.session = session + self.max_retries = max_retries async def get(self, url: str, params: dict[str, Any] | None = None) -> dict[str, Any] | list[Any]: """Get the JSON response from the API using the given url, and params. @@ -55,6 +59,11 @@ async def get(self, url: str, params: dict[str, Any] | None = None) -> dict[str, return await response.json() log.error("Non-2xx response %s | POST %s | Headers %s", response.status, url, response.headers) + + if response.status == HTTPStatus.TOO_MANY_REQUESTS: + retry_after = int(response.headers.get("Retry-After", 60)) + raise APIPageStatusError(response.status, self.base_url, retry_after=retry_after) + raise APIPageStatusError(response.status, self.base_url) async def post_data(self, url: str, json: dict[str, Any] | list[Any] | None) -> str: @@ -84,6 +93,10 @@ async def post_data(self, url: str, json: dict[str, Any] | list[Any] | None) -> return await response.text() log.error("Non-2xx response %s | POST %s | Headers %s", response.status, url, response.headers) + if response.status == HTTPStatus.TOO_MANY_REQUESTS: + retry_after = int(response.headers.get("Retry-After", 60)) + raise APIPageStatusError(response.status, self.base_url, retry_after=retry_after) + raise APIPageStatusError(response.status, self.base_url) async def handle_request(self, request: PreparedRequest) -> str | dict[str, Any] | list[Any]: @@ -99,9 +112,44 @@ async def handle_request(self, request: PreparedRequest) -> str | dict[str, Any] str | dict[str, Any] | list[Any] The json response from the API - """ - log.debug("%s %s | params=%s | body=%s", request.method.value, request.url, request.params, request.body) - if request.method == RequestMethod.POST: - return await self.post_data(request.url, json=request.body) + Raises + ------ + APIPageStatusError + Raised immediately for non-2xx responses that are not retryable (e.g. anything other than a 429). + MaximumRetriesExceededError + Raised when the maximum number of retries is exceeded. - return await self.get(request.url, params=request.params) + """ + for attempt in range(self.max_retries + 1): + try: + log.debug( + "%s %s | params=%s | body=%s", + request.method.value, + request.url, + request.params, + request.body, + ) + if request.method == RequestMethod.POST: + return await self.post_data(request.url, json=request.body) + + return await self.get(request.url, params=request.params) + + except APIPageStatusError as error: + if error.page_status != HTTPStatus.TOO_MANY_REQUESTS or attempt >= self.max_retries: + raise + + log.warning( + "Rate limited (429) | %s %s | retrying in %.2fs (attempt %d/%d)", + request.method.value, + request.url, + error.retry_after, + attempt + 1, + self.max_retries + 1, + ) + + if error.retry_after is not None: + # Add a buffer of 50-1000ms depending on retry amount + buffer = max(0.05, min(1, error.retry_after * 0.02)) + await asyncio.sleep(error.retry_after + buffer) + + raise MaximumRetriesExceededError From 2df3f934f4bf2ccd799e39e410a580dfba0c1b3d Mon Sep 17 00:00:00 2001 From: Mitch Woollatt <21175347+MitchellAW@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:34:25 +0930 Subject: [PATCH 4/7] Add max_retries parameter to API classes --- compuglobal/aio.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/compuglobal/aio.py b/compuglobal/aio.py index 04141f2..08e87ba 100644 --- a/compuglobal/aio.py +++ b/compuglobal/aio.py @@ -37,6 +37,8 @@ class AsyncCompuGlobalAPI: The client session to use for all API calls default_format : OverlayFormat | None, optional The default overlay format to use for all overlays/subtitles + max_retries : int, optional + The maximum number of retries for all API requests Attributes ---------- @@ -64,7 +66,12 @@ class AsyncCompuGlobalAPI: media: MediaAPI = MediaAPI() metadata: MetadataAPI = MetadataAPI() - def __init__(self, session: aiohttp.ClientSession, default_format: OverlayFormat | None = None) -> None: + def __init__( + self, + session: aiohttp.ClientSession, + default_format: OverlayFormat | None = None, + max_retries: int = 0, + ) -> None: extra_fonts = list(self.EXTRA_FONTS) if default_format is None: chosen_font = extra_fonts[0] if len(extra_fonts) > 0 else FontFamily.IMPACT @@ -77,7 +84,7 @@ def __init__(self, session: aiohttp.ClientSession, default_format: OverlayFormat allowed_fonts=allowed_fonts, default_format=default_format, ) - self.client = CompuGlobalAPIClient(base_url=self.BASE_URL, session=session) + self.client = CompuGlobalAPIClient(base_url=self.BASE_URL, session=session, max_retries=max_retries) async def get_screencap( self, From 247e909f11401e7c4f39da92fb1ed2290a39c69d Mon Sep 17 00:00:00 2001 From: Mitch Woollatt <21175347+MitchellAW@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:34:57 +0930 Subject: [PATCH 5/7] Include logged warnings in integration tests If an integration test is held up by a long sleep (~60s) from 429s, ensure that warning is logged. --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index cb41b72..429847c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -15,7 +15,7 @@ jobs: uses: ./.github/actions/setup - name: Run integration tests - run: uv run pytest -m "integration" + run: uv run pytest -m "integration" --log-cli-level=WARNING -v - name: Upload integration test report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 From 09295c34b5a6737057af08752af6df64ede44a6a Mon Sep 17 00:00:00 2001 From: Mitch Woollatt <21175347+MitchellAW@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:52:06 +0930 Subject: [PATCH 6/7] Only log statuses as errors for non-429s --- compuglobal/api/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compuglobal/api/client.py b/compuglobal/api/client.py index 624f23f..e293e4d 100644 --- a/compuglobal/api/client.py +++ b/compuglobal/api/client.py @@ -58,12 +58,11 @@ async def get(self, url: str, params: dict[str, Any] | None = None) -> dict[str, log.debug("Response %s | GET %s", response.status, url) return await response.json() - log.error("Non-2xx response %s | POST %s | Headers %s", response.status, url, response.headers) - if response.status == HTTPStatus.TOO_MANY_REQUESTS: retry_after = int(response.headers.get("Retry-After", 60)) raise APIPageStatusError(response.status, self.base_url, retry_after=retry_after) + log.error("Non-2xx response %s | POST %s | Headers %s", response.status, url, response.headers) raise APIPageStatusError(response.status, self.base_url) async def post_data(self, url: str, json: dict[str, Any] | list[Any] | None) -> str: From a13ccec94ea2aa49f90ab99c24a252f64bd4641d Mon Sep 17 00:00:00 2001 From: Mitch Woollatt <21175347+MitchellAW@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:52:28 +0930 Subject: [PATCH 7/7] Retry integration tests once for 429s --- tests/integration/test_aio_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_aio_integration.py b/tests/integration/test_aio_integration.py index fcb18ef..3b9cabd 100644 --- a/tests/integration/test_aio_integration.py +++ b/tests/integration/test_aio_integration.py @@ -28,7 +28,7 @@ async def api(request: pytest.FixtureRequest) -> AsyncGenerator[AsyncCompuGlobalAPI]: api_class = request.param async with aiohttp.ClientSession() as session: - yield api_class(session=session) + yield api_class(session=session, max_retries=1) @pytest_asyncio.fixture