Diamond75/zotify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Changed internal rate limiter library used to implement adjustable rate limit modes

Browse source
This commit is contained in:
DraftKinner 2025-02-13 15:08:56 -05:00
parent caae869d48
commit ba9590d8f8
5 changed files with 104 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
requirements.txt
View file

@ -5,4 +5,4 @@ Pillow
pkce
requests
tqdm
ratelimit
limits

2
setup.cfg
View file

@ -26,7 +26,7 @@ install_requires =
pkce
requests
tqdm
ratelimit
limits
[options.entry_points]
console_scripts =

47
zotify/__init__.py
View file

@ -5,9 +5,9 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
from pathlib import Path
from threading import Thread
from typing import Any
from time import time_ns
from time import time_ns, sleep
from urllib.parse import urlencode, urlparse, parse_qs
from ratelimit import limits, sleep_and_retry
from limits import storage, strategies, RateLimitItemPerSecond
from librespot.audio import AudioKeyManager, CdnManager
from librespot.audio.decoders import VorbisOnlyAudioQuality
@ -31,7 +31,7 @@ from requests import HTTPError, get, post
from zotify.loader import Loader
from zotify.playable import Episode, Track
from zotify.utils import Quality
from zotify.utils import Quality, RateLimitMode
API_URL = "https://api.sp" + "otify.com/v1/"
AUTH_URL = "https://accounts.sp" + "otify.com/"
@ -66,8 +66,12 @@ SCOPES = [
"user-top-read",
]
RATE_LIMIT_API = "rate_limit_api"
RATE_LIMIT_MAX_CONSECUTIVE_HITS = 10
RATE_LIMIT_RESTORE_CONDITION = 15
RATE_LIMIT_INTERVAL_SECS = 30
RATE_LIMIT_CALLS_PER_INTERVAL = 9
RATE_LIMIT_CALLS_NORMAL = 9
RATE_LIMIT_CALLS_REDUCED = 3
class Session(LibrespotSession):
@ -98,6 +102,7 @@ class Session(LibrespotSession):
self.__language = language
self.connect()
self.authenticate(session_builder.login_credentials)
self.rate_limiter = RateLimiter()
@staticmethod
def from_file(cred_file: Path | str, language: str = "en") -> Session:
@ -225,9 +230,12 @@ class Session(LibrespotSession):
self.__auth_lock.notify_all()
self.mercury().interested_in("sp" + "otify:user:attributes:update", self)
@sleep_and_retry
@limits(calls=RATE_LIMIT_CALLS_PER_INTERVAL, period=RATE_LIMIT_INTERVAL_SECS)
def api(self) -> ApiClient:
# Check rate limiter before making calls to api
while not self.rate_limiter.check():
sleep(1)
self.rate_limiter.hit()
return super().api()
@ -433,3 +441,30 @@ class OAuth:
self.end_headers()
self.wfile.write(b"Authorization code not found.")
Thread(target=self.server.shutdown).start()
class RateLimiter:
rate_limits = {
RateLimitMode.NORMAL: RateLimitItemPerSecond(
RATE_LIMIT_CALLS_NORMAL, RATE_LIMIT_INTERVAL_SECS
),
RateLimitMode.REDUCED: RateLimitItemPerSecond(
RATE_LIMIT_CALLS_REDUCED, RATE_LIMIT_INTERVAL_SECS
),
}
def __init__(self):
self.storage = storage.MemoryStorage()
self.moving_window = strategies.MovingWindowRateLimiter(self.storage)
self.mode = RateLimitMode.NORMAL
self.rate_limit = RateLimiter.rate_limits[self.mode]
def check(self):
return self.moving_window.test(self.rate_limit, RATE_LIMIT_API)
def hit(self):
self.moving_window.hit(self.rate_limit, RATE_LIMIT_API)
def set_mode(self, mode: RateLimitMode):
self.mode = mode
self.rate_limit = RateLimiter.rate_limits[self.mode]

70
zotify/app.py
View file

@ -3,13 +3,19 @@ from pathlib import Path
from typing import Any
from time import sleep
from zotify import OAuth, Session, RATE_LIMIT_INTERVAL_SECS
from zotify import (
OAuth,
Session,
RATE_LIMIT_INTERVAL_SECS,
RATE_LIMIT_MAX_CONSECUTIVE_HITS,
RATE_LIMIT_RESTORE_CONDITION,
)
from zotify.collections import Album, Artist, Collection, Episode, Playlist, Show, Track
from zotify.config import Config
from zotify.file import TranscodingError
from zotify.loader import Loader
from zotify.logger import LogChannel, Logger
from zotify.utils import AudioFormat, PlayableType
from zotify.utils import AudioFormat, PlayableType, RateLimitMode
class ParseError(ValueError): ...
@ -261,9 +267,11 @@ class App:
self.__duplicates.update(duplicates)
def download_all(self, collections: list[Collection]) -> None:
self.rate_limit_hits = 0
self.last_rate_limit_hit = 0
count = 0
total = sum(len(c.playables) for c in collections)
rate_limit_hits = 0
for collection in collections:
for playable in collection.playables:
count += 1
@ -285,23 +293,22 @@ class App:
# Get track data
if playable.type == PlayableType.TRACK:
try:
self.restore_rate_limit(count)
with Loader("Fetching track..."):
track = self.__session.get_track(
playable.id, self.__config.download_quality
)
except RuntimeError as err:
Logger.log(LogChannel.SKIPS, f"Skipping track #{count}: {err}")
if "audio key" in str(err).lower():
rate_limit_hits += 1
sleep_time = RATE_LIMIT_INTERVAL_SECS * rate_limit_hits
with Loader(
f"Rate limit hit! Sleeping for {sleep_time}s..."
):
sleep(sleep_time)
self.handle_runtime_error(err, playable.type, count)
continue
elif playable.type == PlayableType.EPISODE:
with Loader("Fetching episode..."):
track = self.__session.get_episode(playable.id)
try:
self.restore_rate_limit(count)
with Loader("Fetching episode..."):
track = self.__session.get_episode(playable.id)
except RuntimeError as err:
self.handle_runtime_error(err, playable.type, count)
continue
else:
Logger.log(
LogChannel.SKIPS,
@ -377,4 +384,39 @@ class App:
)
# Reset rate limit counter for every successful download
rate_limit_hits = 0
self.rate_limit_hits = 0
def restore_rate_limit(self, count: int) -> None:
if (
self.__session.rate_limiter.mode == RateLimitMode.REDUCED
and (count - self.last_rate_limit_hit) > RATE_LIMIT_RESTORE_CONDITION
):
with Loader("Restoring rate limit to normal..."):
self.__session.rate_limiter.set_mode(RateLimitMode.NORMAL)
sleep(RATE_LIMIT_INTERVAL_SECS)
def handle_runtime_error(
self, err: str, playable_type: PlayableType, count: int
) -> None:
Logger.log(LogChannel.SKIPS, f"Skipping {playable_type.value} #{count}: {err}")
if "audio key" in str(err).lower():
self.handle_rate_limit_hit(count)
def handle_rate_limit_hit(self, count: int) -> None:
self.rate_limit_hits += 1
self.last_rate_limit_hit = count
# Exit program if rate limit hit cutoff is reached
if self.rate_limit_hits > RATE_LIMIT_MAX_CONSECUTIVE_HITS:
Logger.log(LogChannel.ERRORS, "Server too busy or down. Try again later")
exit(1)
# Reduce internal rate limiter
if self.__session.rate_limiter.mode == RateLimitMode.NORMAL:
self.__session.rate_limiter.set_mode(RateLimitMode.REDUCED)
# Sleep for one interval
with Loader(
f"Server rate limit hit! Sleeping for {RATE_LIMIT_INTERVAL_SECS}s..."
):
sleep(RATE_LIMIT_INTERVAL_SECS)

5
zotify/utils.py
View file

@ -122,6 +122,11 @@ class PlayableData:
duplicate: bool = False
class RateLimitMode(Enum):
NORMAL = "normal"
REDUCED = "reduced"
class OptionalOrFalse(Action):
def __init__(
self,