Skip to content

Commit 48ef15d

Browse files
mayhemmayhem
authored
Merge pull request #151 from metabrainz/lb-radio-429-fix
LB-1628: LB radio 429 fix
2 parents 2f3a152 + d931c4d commit 48ef15d

File tree

13 files changed

+135
-54
lines changed

13 files changed

+135
-54
lines changed

‎troi/content_resolver/artist_search.py‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from collections import defaultdict
33
import datetime
44
import sys
5+
from time import sleep
56

67
import peewee
78
import requests
@@ -37,15 +38,21 @@ def __init__(self):
3738
def get_similar_artists(self, artist_mbid):
3839
""" Fetch similar artists, given an artist_mbid. Returns a sored plist of artists. """
3940

40-
r = requests.post("https://labs.api.listenbrainz.org/similar-artists/json",
41-
json=[{
42-
'artist_mbids':
43-
[artist_mbid],
44-
'algorithm':
45-
"session_based_days_7500_session_300_contribution_5_threshold_10_limit_100_filter_True_skip_30"
46-
}])
47-
if r.status_code != 200:
48-
raise RuntimeError(f"Cannot fetch similar artists: {r.status_code} ({r.text})")
41+
while True:
42+
r = requests.post("https://labs.api.listenbrainz.org/similar-artists/json",
43+
json=[{
44+
'artist_mbids':
45+
[artist_mbid],
46+
'algorithm':
47+
"session_based_days_7500_session_300_contribution_5_threshold_10_limit_100_filter_True_skip_30"
48+
}])
49+
if r.status_code == 429:
50+
sleep(2)
51+
continue
52+
if r.status_code != 200:
53+
raise RuntimeError(f"Cannot fetch similar artists: {r.status_code} ({r.text})")
54+
55+
break
4956

5057
artists = r.json()
5158

‎troi/content_resolver/unresolved_recording.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,14 +100,14 @@ def get_releases(self):
100100
params = {"recording_mbids": args, "inc": "artist release"}
101101
while True:
102102
r = requests.get("https://api.listenbrainz.org/1/metadata/recording", params=params)
103-
if r.status_code != 200:
104-
logger.info("Failed to fetch metadata for recordings: ", r.text)
105-
return []
106-
107103
if r.status_code == 429:
108104
sleep(1)
109105
continue
110106

107+
if r.status_code != 200:
108+
logger.info("Failed to fetch metadata for recordings: ", r.text)
109+
return []
110+
111111
break
112112
recording_data.update(dict(r.json()))
113113

‎troi/musicbrainz/mbid_mapping.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import requests
22
import ujson
3+
from time import sleep
34

45
from troi import Element, Artist, ArtistCredit, Recording, Release, PipelineError
56

@@ -36,9 +37,17 @@ def read(self, inputs):
3637
if not params:
3738
return []
3839

39-
r = requests.post(self.SERVER_URL, json=params)
40-
if r.status_code != 200:
41-
raise PipelineError("Cannot fetch MBID mapping rows from ListenBrainz: HTTP code %d (%s)" % (r.status_code, r.text))
40+
while True:
41+
r = requests.post(self.SERVER_URL, json=params)
42+
if r.status_code == 429:
43+
sleep(2)
44+
continue
45+
46+
if r.status_code != 200:
47+
raise PipelineError("Cannot fetch MBID mapping rows from ListenBrainz: HTTP code %d (%s)" % (r.status_code, r.text))
48+
49+
break
50+
4251

4352
entities = []
4453
for row in r.json():

‎troi/musicbrainz/recording_lookup.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ class RecordingLookupElement(Element):
1515
'''
1616

1717
SERVER_URL = "https://api.listenbrainz.org/1/metadata/recording"
18+
MAX_RECORDINGS = 1000
1819

1920
def __init__(self, skip_not_found=True, lookup_tags=False, tag_threshold=None):
2021
Element.__init__(self)
@@ -43,6 +44,9 @@ def read(self, inputs):
4344
for r in recordings:
4445
recording_mbids.append(r.mbid)
4546

47+
if len(recording_mbids) > self.MAX_RECORDINGS:
48+
raise PipelineError("Cannot fetch more than %d recordings from ListenBrainz." % self.MAX_RECORDINGS)
49+
4650
inc = "artist release"
4751
if self.lookup_tags:
4852
inc += " tag"

‎troi/musicbrainz/related_artist_credits.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import copy
22
from collections import defaultdict
3+
from time import sleep
34

45
import requests
56

@@ -31,9 +32,18 @@ def read(self, inputs):
3132
ac_ids = ",".join([ str(a.artist_credit_id) for a in artists ])
3233
params = {"[artist_credit_id]": ac_ids,
3334
"threshold": self.threshold}
34-
r = requests.get(self.SERVER_URL, params=params)
35-
if r.status_code != 200:
36-
raise PipelineError("Cannot fetch related artist credits from ListenBrainz: HTTP code %d" % r.status_code)
35+
36+
while True:
37+
r = requests.get(self.SERVER_URL, params=params)
38+
if r.status_code == 429:
39+
sleep(2)
40+
continue
41+
42+
if r.status_code != 200:
43+
raise PipelineError("Cannot fetch related artist credits from ListenBrainz: HTTP code %d" % r.status_code)
44+
45+
break
46+
3747

3848
try:
3949
relations = r.text

‎troi/patches/lb_radio.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ def lookup_artist_from_mbid(self, artist_mbid):
119119
if r.status_code == 404:
120120
raise RuntimeError(f"Could not resolve artist mbid {artist_mbid}. Error {r.status_code} {r.text}")
121121

122-
if r.status_code == 429:
122+
if r.status_code in (429, 503):
123123
sleep(2)
124124
continue
125125

‎troi/patches/lb_radio_classes/artist.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import requests
1+
from time import sleep
22

33
import troi
44
from troi import Recording, Artist
@@ -16,6 +16,7 @@ class LBRadioArtistRecordingElement(troi.Element):
1616

1717
MAX_TOP_RECORDINGS_PER_ARTIST = 35 # should lower this when other sources of data get added
1818
MAX_NUM_SIMILAR_ARTISTS = 8
19+
MAX_NUM_RECORDINGS = 1000
1920

2021
def __init__(self, artist_mbid, artist_name, mode="easy", include_similar_artists=True):
2122
troi.Element.__init__(self)
@@ -80,4 +81,4 @@ def read(self, entities):
8081
self.local_storage["user_feedback"].append(msg)
8182
self.data_cache["element-descriptions"].append("artist %s" % self.artist_name)
8283

83-
return interleave([artist_recordings[mbid] for mbid in artist_recordings])
84+
return interleave([artist_recordings[mbid] for mbid in artist_recordings])[:self.MAX_NUM_RECORDINGS]

‎troi/patches/lb_radio_classes/collection.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
import troi
21
from random import shuffle
2+
from time import sleep
33

44
import requests
55

6+
import troi
67
from troi import Recording
78
from troi import TARGET_NUMBER_OF_RECORDINGS
89

@@ -30,11 +31,20 @@ def read(self, entities):
3031

3132
# Fetch collection recordings
3233
params = {"collection": self.mbid, "fmt": "json"}
33-
r = requests.get("https://musicbrainz.org/ws/2/recording", params=params)
34-
if r.status_code == 404:
35-
raise RuntimeError(f"Cannot find collection {self.mbid}.")
36-
if r.status_code != 200:
37-
raise RuntimeError(f"Cannot fetch collection {self.mbid}. {r.text}")
34+
35+
while True:
36+
r = requests.get("https://musicbrainz.org/ws/2/recording", params=params)
37+
if r.status_code == 404:
38+
raise RuntimeError(f"Cannot find collection {self.mbid}.")
39+
40+
if r.status_code == 429:
41+
sleep(2)
42+
continue
43+
44+
if r.status_code != 200:
45+
raise RuntimeError(f"Cannot fetch collection {self.mbid}. {r.text}")
46+
47+
break
3848

3949
# Give feedback about what we collected
4050
self.local_storage["data_cache"]["element-descriptions"].append(f"collection {self.mbid}")

‎troi/patches/lb_radio_classes/country.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def lookup_area_by_name(self, area_name):
3737

3838
while True:
3939
r = requests.get("https://musicbrainz.org/ws/2/area?query=%s&fmt=json" % area_name)
40-
if r.status_code == 503:
40+
if r.status_code in (503, 429):
4141
sleep(1)
4242
continue
4343

@@ -58,7 +58,7 @@ def lookup_area_by_mbid(self, area_mbid):
5858

5959
while True:
6060
r = requests.get("https://musicbrainz.org/ws/2/area/%s?fmt=json" % area_mbid)
61-
if r.status_code == 503:
61+
if r.status_code in (503, 429):
6262
sleep(1)
6363
continue
6464

‎troi/patches/lb_radio_classes/playlist.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import troi
2+
from time import sleep
23
from random import shuffle
34

45
import requests
@@ -29,11 +30,17 @@ def outputs(self):
2930
def read(self, entities):
3031

3132
# Fetch the playlist
32-
r = requests.get(f"https://api.listenbrainz.org/1/playlist/{self.mbid}")
33-
if r.status_code == 404:
34-
raise RuntimeError(f"Cannot find playlist {self.mbid}.")
35-
if r.status_code != 200:
36-
raise RuntimeError(f"Cannot fetch playlist {self.mbid}. {r.text}")
33+
while True:
34+
r = requests.get(f"https://api.listenbrainz.org/1/playlist/{self.mbid}")
35+
if r.status_code == 404:
36+
raise RuntimeError(f"Cannot find playlist {self.mbid}.")
37+
if r.status_code == 429:
38+
sleep(2)
39+
continue
40+
if r.status_code != 200:
41+
raise RuntimeError(f"Cannot fetch playlist {self.mbid}. {r.text}")
42+
43+
break
3744

3845
# Give feedback about the playlist
3946
self.local_storage["data_cache"]["element-descriptions"].append(f"playlist {self.mbid}")

0 commit comments

Comments
 (0)