mirror of
https://github.com/beetbox/beets.git
synced 2026-07-22 03:17:26 -04:00
TimeoutAndRetrySession: fold in RateLimitAdapter and 429 retry
This commit is contained in:
@@ -71,22 +71,22 @@ class TimeoutAndRetrySession(requests.Session, metaclass=SingletonMeta):
|
||||
* raises exceptions for HTTP error status codes
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
def __init__(self, rate_limit: float = 0.25, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.headers["User-Agent"] = f"beets/{__version__} https://beets.io/"
|
||||
|
||||
retry = Retry(
|
||||
total=6,
|
||||
backoff_factor=0.5,
|
||||
# Retry on server errors
|
||||
status_forcelist=[
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
HTTPStatus.BAD_GATEWAY,
|
||||
HTTPStatus.SERVICE_UNAVAILABLE,
|
||||
HTTPStatus.GATEWAY_TIMEOUT,
|
||||
HTTPStatus.TOO_MANY_REQUESTS,
|
||||
],
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
adapter = RateLimitAdapter(rate_limit=rate_limit, max_retries=retry)
|
||||
self.mount("https://", adapter)
|
||||
self.mount("http://", adapter)
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ from contextlib import contextmanager, suppress
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property, partial, total_ordering
|
||||
from html import unescape
|
||||
from http import HTTPStatus
|
||||
from itertools import filterfalse, groupby
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, ClassVar, NamedTuple
|
||||
@@ -33,7 +32,6 @@ from urllib.parse import quote, quote_plus, urlencode, urlparse
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from unidecode import unidecode
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from beets import plugins, ui
|
||||
from beets.autotag import string_dist
|
||||
@@ -45,7 +43,6 @@ from beets.util.lyrics import INSTRUMENTAL_LYRICS, Lyrics
|
||||
|
||||
from ._utils.requests import (
|
||||
HTTPNotFoundError,
|
||||
RateLimitAdapter,
|
||||
RequestHandler,
|
||||
TimeoutAndRetrySession,
|
||||
)
|
||||
@@ -172,45 +169,12 @@ def slug(text: str) -> str:
|
||||
return re.sub(r"\W+", "-", unidecode(text).lower().strip()).strip("-")
|
||||
|
||||
|
||||
class LyricsSession(TimeoutAndRetrySession):
|
||||
"""HTTP session for lyrics backends with rate limiting and exponential backoff.
|
||||
|
||||
Builds on :class:`TimeoutAndRetrySession` — inherits User-Agent header,
|
||||
default timeout, and response status checking. Adds a :class:`RateLimitAdapter`
|
||||
to enforce a minimum interval between requests, and configures urllib3 Retry
|
||||
with exponential backoff on rate limit (429) responses.
|
||||
"""
|
||||
|
||||
def __init__(self, rate_limit: float = 0.25):
|
||||
"""Initialize session with rate limiting and retry/backoff.
|
||||
|
||||
Args:
|
||||
rate_limit: Minimum interval in seconds between consecutive
|
||||
requests (default 0.25s = 4 requests/sec).
|
||||
"""
|
||||
super().__init__()
|
||||
retry = Retry(
|
||||
total=6,
|
||||
backoff_factor=1.0,
|
||||
status_forcelist=[
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
HTTPStatus.BAD_GATEWAY,
|
||||
HTTPStatus.SERVICE_UNAVAILABLE,
|
||||
HTTPStatus.GATEWAY_TIMEOUT,
|
||||
HTTPStatus.TOO_MANY_REQUESTS,
|
||||
],
|
||||
)
|
||||
adapter = RateLimitAdapter(rate_limit=rate_limit, max_retries=retry)
|
||||
self.mount("https://", adapter)
|
||||
self.mount("http://", adapter)
|
||||
|
||||
|
||||
class LyricsRequestHandler(RequestHandler):
|
||||
_log: Logger
|
||||
|
||||
def create_session(self) -> TimeoutAndRetrySession:
|
||||
"""Return a rate-limited session for lyrics HTTP requests."""
|
||||
return LyricsSession()
|
||||
return TimeoutAndRetrySession()
|
||||
|
||||
def status_to_error(self, code: int) -> type[requests.HTTPError] | None:
|
||||
if err := super().status_to_error(code):
|
||||
|
||||
@@ -96,6 +96,10 @@ For plugin developers
|
||||
Other changes
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- :doc:`plugins/lyrics`: Fold rate limiting and 429 retry from the
|
||||
lyrics-specific ``LyricsSession`` into the shared
|
||||
:class:`~beetsplug._utils.requests.TimeoutAndRetrySession` so all plugins
|
||||
benefit. The standalone ``LyricsSession`` class has been removed.
|
||||
- :doc:`plugins/spotify`: ``spotifysync`` now batches its SQLite commit for a
|
||||
sync run, follows the standard beets write-before-store pattern, and logs
|
||||
audio-features API unavailability only once per run.
|
||||
|
||||
Reference in New Issue
Block a user