Changed internal rate limiter library used to implement adjustable rate limit modes

requirements.txt

@@ -5,4 +5,4 @@ Pillow
 pkce
 requests
 tqdm
-ratelimit
+limits

setup.cfg

@@ -26,7 +26,7 @@ install_requires =
     pkce
     requests
     tqdm
-    ratelimit
+    limits
 
 [options.entry_points]
 console_scripts =

zotify/__init__.py

@@ -5,9 +5,9 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
 from pathlib import Path
 from threading import Thread
 from typing import Any
-from time import time_ns
+from time import time_ns, sleep
 from urllib.parse import urlencode, urlparse, parse_qs
-from ratelimit import limits, sleep_and_retry
+from limits import storage, strategies, RateLimitItemPerSecond
 
 from librespot.audio import AudioKeyManager, CdnManager
 from librespot.audio.decoders import VorbisOnlyAudioQuality
@@ -31,7 +31,7 @@ from requests import HTTPError, get, post
 
 from zotify.loader import Loader
 from zotify.playable import Episode, Track
-from zotify.utils import Quality
+from zotify.utils import Quality, RateLimitMode
 
 API_URL = "https://api.sp" + "otify.com/v1/"
 AUTH_URL = "https://accounts.sp" + "otify.com/"
@@ -66,8 +66,12 @@ SCOPES = [
     "user-top-read",
 ]
 
+RATE_LIMIT_API = "rate_limit_api"
+RATE_LIMIT_MAX_CONSECUTIVE_HITS = 10
+RATE_LIMIT_RESTORE_CONDITION = 15
 RATE_LIMIT_INTERVAL_SECS = 30
-RATE_LIMIT_CALLS_PER_INTERVAL = 9
+RATE_LIMIT_CALLS_NORMAL = 9
+RATE_LIMIT_CALLS_REDUCED = 3
 
 
 class Session(LibrespotSession):
@@ -98,6 +102,7 @@ class Session(LibrespotSession):
             self.__language = language
             self.connect()
             self.authenticate(session_builder.login_credentials)
+        self.rate_limiter = RateLimiter()
 
     @staticmethod
     def from_file(cred_file: Path | str, language: str = "en") -> Session:
@@ -225,9 +230,12 @@ class Session(LibrespotSession):
             self.__auth_lock.notify_all()
         self.mercury().interested_in("sp" + "otify:user:attributes:update", self)
 
-    @sleep_and_retry
-    @limits(calls=RATE_LIMIT_CALLS_PER_INTERVAL, period=RATE_LIMIT_INTERVAL_SECS)
     def api(self) -> ApiClient:
+        # Check rate limiter before making calls to api
+        while not self.rate_limiter.check():
+            sleep(1)
+
+        self.rate_limiter.hit()
         return super().api()
 
 
@@ -433,3 +441,30 @@ class OAuth:
                 self.end_headers()
                 self.wfile.write(b"Authorization code not found.")
                 Thread(target=self.server.shutdown).start()
+
+
+class RateLimiter:
+    rate_limits = {
+        RateLimitMode.NORMAL: RateLimitItemPerSecond(
+            RATE_LIMIT_CALLS_NORMAL, RATE_LIMIT_INTERVAL_SECS
+        ),
+        RateLimitMode.REDUCED: RateLimitItemPerSecond(
+            RATE_LIMIT_CALLS_REDUCED, RATE_LIMIT_INTERVAL_SECS
+        ),
+    }
+
+    def __init__(self):
+        self.storage = storage.MemoryStorage()
+        self.moving_window = strategies.MovingWindowRateLimiter(self.storage)
+        self.mode = RateLimitMode.NORMAL
+        self.rate_limit = RateLimiter.rate_limits[self.mode]
+
+    def check(self):
+        return self.moving_window.test(self.rate_limit, RATE_LIMIT_API)
+
+    def hit(self):
+        self.moving_window.hit(self.rate_limit, RATE_LIMIT_API)
+
+    def set_mode(self, mode: RateLimitMode):
+        self.mode = mode
+        self.rate_limit = RateLimiter.rate_limits[self.mode]

zotify/app.py

@@ -3,13 +3,19 @@ from pathlib import Path
 from typing import Any
 from time import sleep
 
-from zotify import OAuth, Session, RATE_LIMIT_INTERVAL_SECS
+from zotify import (
+    OAuth,
+    Session,
+    RATE_LIMIT_INTERVAL_SECS,
+    RATE_LIMIT_MAX_CONSECUTIVE_HITS,
+    RATE_LIMIT_RESTORE_CONDITION,
+)
 from zotify.collections import Album, Artist, Collection, Episode, Playlist, Show, Track
 from zotify.config import Config
 from zotify.file import TranscodingError
 from zotify.loader import Loader
 from zotify.logger import LogChannel, Logger
-from zotify.utils import AudioFormat, PlayableType
+from zotify.utils import AudioFormat, PlayableType, RateLimitMode
 
 
 class ParseError(ValueError): ...
@@ -261,9 +267,11 @@ class App:
                 self.__duplicates.update(duplicates)
 
     def download_all(self, collections: list[Collection]) -> None:
+        self.rate_limit_hits = 0
+        self.last_rate_limit_hit = 0
+
         count = 0
         total = sum(len(c.playables) for c in collections)
-        rate_limit_hits = 0
         for collection in collections:
             for playable in collection.playables:
                 count += 1
@@ -285,23 +293,22 @@ class App:
                 # Get track data
                 if playable.type == PlayableType.TRACK:
                     try:
+                        self.restore_rate_limit(count)
                         with Loader("Fetching track..."):
                             track = self.__session.get_track(
                                 playable.id, self.__config.download_quality
                             )
                     except RuntimeError as err:
-                        Logger.log(LogChannel.SKIPS, f"Skipping track #{count}: {err}")
+                        self.handle_runtime_error(err, playable.type, count)
-                        if "audio key" in str(err).lower():
-                            rate_limit_hits += 1
-                            sleep_time = RATE_LIMIT_INTERVAL_SECS * rate_limit_hits
-                            with Loader(
-                                f"Rate limit hit! Sleeping for {sleep_time}s..."
-                            ):
-                                sleep(sleep_time)
                         continue
                 elif playable.type == PlayableType.EPISODE:
-                    with Loader("Fetching episode..."):
+                    try:
-                        track = self.__session.get_episode(playable.id)
+                        self.restore_rate_limit(count)
+                        with Loader("Fetching episode..."):
+                            track = self.__session.get_episode(playable.id)
+                    except RuntimeError as err:
+                        self.handle_runtime_error(err, playable.type, count)
+                        continue
                 else:
                     Logger.log(
                         LogChannel.SKIPS,
@@ -377,4 +384,39 @@ class App:
                         )
 
                 # Reset rate limit counter for every successful download
-                rate_limit_hits = 0
+                self.rate_limit_hits = 0
+
+    def restore_rate_limit(self, count: int) -> None:
+        if (
+            self.__session.rate_limiter.mode == RateLimitMode.REDUCED
+            and (count - self.last_rate_limit_hit) > RATE_LIMIT_RESTORE_CONDITION
+        ):
+            with Loader("Restoring rate limit to normal..."):
+                self.__session.rate_limiter.set_mode(RateLimitMode.NORMAL)
+                sleep(RATE_LIMIT_INTERVAL_SECS)
+
+    def handle_runtime_error(
+        self, err: str, playable_type: PlayableType, count: int
+    ) -> None:
+        Logger.log(LogChannel.SKIPS, f"Skipping {playable_type.value} #{count}: {err}")
+        if "audio key" in str(err).lower():
+            self.handle_rate_limit_hit(count)
+
+    def handle_rate_limit_hit(self, count: int) -> None:
+        self.rate_limit_hits += 1
+        self.last_rate_limit_hit = count
+
+        # Exit program if rate limit hit cutoff is reached
+        if self.rate_limit_hits > RATE_LIMIT_MAX_CONSECUTIVE_HITS:
+            Logger.log(LogChannel.ERRORS, "Server too busy or down. Try again later")
+            exit(1)
+
+        # Reduce internal rate limiter
+        if self.__session.rate_limiter.mode == RateLimitMode.NORMAL:
+            self.__session.rate_limiter.set_mode(RateLimitMode.REDUCED)
+
+        # Sleep for one interval
+        with Loader(
+            f"Server rate limit hit! Sleeping for {RATE_LIMIT_INTERVAL_SECS}s..."
+        ):
+            sleep(RATE_LIMIT_INTERVAL_SECS)

zotify/utils.py

@@ -122,6 +122,11 @@ class PlayableData:
     duplicate: bool = False
 
 
+class RateLimitMode(Enum):
+    NORMAL = "normal"
+    REDUCED = "reduced"
+
+
 class OptionalOrFalse(Action):
     def __init__(
         self,