Use same ThreadPool for all SteamAPI Queries

1 files changed, 13 insertions, 10 deletions

diff --git a/SteamAPI.py b/SteamAPI.py
index c71f438..a992e68 100644
--- a/SteamAPI.py
+++ b/SteamAPI.py
@@ -12,9 +12,10 @@ from Caching import Caching
 ##################################################
 # Directory for a local cache & time to live for cache items in seconds
 CACHE = True
-
 ##################################################
 
+executor = ThreadPoolExecutor(max_workers=25)
+
 class SteamAPI():
     def __init__(self, token):
         self.token = token
@@ -71,12 +72,17 @@ class SteamAPI():
             for steamid in cachedids:
                 steamids_copy.remove(steamid)
 
-        if steamids_copy:
-            steamidlist = ','.join([str(x) for x in steamids_copy])
+        responses = []
+        while steamids_copy:
+            steamidlist = ','.join([str(x) for x in steamids_copy[:50]])
+            steamids_copy = steamids_copy[50:]
             url = 'https://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?format=json&key=%s&steamids=%s' % (self.token, steamidlist)
-            response = urlopen(url)
+            responses.append(executor.submit(urlopen, url))
+
+        for responseF in responses:
+            response = responseF.result()
             if response.status != 200:
-                return []
+                continue
             data = response.read().decode('utf-8')
             jsondata = json.loads(data)
 
@@ -87,13 +93,13 @@ class SteamAPI():
         if CACHE:
             # save newly loaded profiles
             for steamid in steamids_copy:
+                if steamid not in profile:
+                    continue
                 Caching.writeCache('profile', steamid, json.dumps(profile[steamid]))
 
         return profile
 
     def getMultipleFriends(self, steamids):
-        executor = ThreadPoolExecutor(max_workers=10)
-        # Ask steam about friends for each
         results = dict()
         for steamid in steamids:
             results[steamid] = executor.submit(self.getFriends, steamid)
@@ -228,7 +234,6 @@ class SteamAPI():
         return None
 
     def getMultipleUserUserstatsForGame(self, steamids, gameid):
-        executor = ThreadPoolExecutor(max_workers=10)
         futures = dict()
         for steamid in steamids:
             futures[steamid] = executor.submit(self.getUserstatsForGame, steamid, gameid)
@@ -307,7 +312,6 @@ class SteamAPI():
         return None
 
     def getMultipleUserOwnedGames(self, steamids):
-        executor = ThreadPoolExecutor(max_workers=10)
         futures = dict()
         for steamid in steamids:
             futures[steamid] = executor.submit(self.getOwnedGames, steamid)
@@ -319,7 +323,6 @@ class SteamAPI():
         return result
 
     def getDataForPremadefinder(self, steamids):
-        executor = ThreadPoolExecutor(max_workers=20)
 
         futures = dict()
         futures['profiles'] = executor.submit(self.getProfiles, steamids)