Improve Game and DLC Scanning Performance / Reduced Steam API Retires

- SteamCMD: Changed output reading from one character at a time to buffered reads, fixing a stderr deadlock issue and making the process much faster and more responsive to cancellation.
- DLC detection: Eliminated a busy-wait loop that was polling every 200ms on a background thread while waiting for game data to arrive. Now uses proper async signaling so the thread is freed up entirely.
- HTTP retries: Previously every Steam API error was retried up to 10 times with delays, even for "403 Forbidden" errors that will never succeed. The scanner now gives up immediately on permanent errors, preventing up to 90 seconds of wasted retries per failed app, and avoids triggering rate limits by hammering the API on doomed requests
- Replaced fixed 1-second waits with smarter retry timing that waits longer after each failed attempt and adds a small random delay to reduce API requests. Also reduced the maximum number of retry attempts from 10 to 3 for temporary network errors.

CreamInstaller/Forms/SelectForm.cs

@@ -177,6 +177,7 @@ internal sealed partial class SelectForm : CustomForm
         }
 
         int steamGamesToCheck;
+        TaskCompletionSource gameQueriesDone = new();
         if (uninstallAll || programsToScan.Any(c => c.platform is Platform.Steam))
         {
             Stopwatch steamLibTimer = Stopwatch.StartNew();
@@ -244,6 +245,8 @@ internal sealed partial class SelectForm : CustomForm
                         return;
                     StoreAppData storeAppData = await SteamStore.QueryStoreAPI(appId);
                     _ = Interlocked.Decrement(ref steamGamesToCheck);
+                    if (Volatile.Read(ref steamGamesToCheck) == 0)
+                        gameQueriesDone.TrySetResult();
                     CmdAppData cmdAppData = await WithTimeout(SteamCMD.GetAppInfo(appId, branch, buildId), 16000);
                     if (storeAppData is null && cmdAppData is null)
                     {
@@ -273,9 +276,12 @@ internal sealed partial class SelectForm : CustomForm
                             {
                                 if (Program.Canceled)
                                     return;
-                                do // give games steam store api limit priority
+                                while (!Program.Canceled)
-                                    Thread.Sleep(200);
+                                {
-                                while (!Program.Canceled && steamGamesToCheck > 0);
+                                    Task completed = await Task.WhenAny(gameQueriesDone.Task, Task.Delay(250));
+                                    if (completed == gameQueriesDone.Task)
+                                        break;
+                                }
                                 if (Program.Canceled)
                                     return;
                                 string fullGameAppId = null;
@@ -374,7 +380,7 @@ internal sealed partial class SelectForm : CustomForm
                         await task;
                     }
 
-                    steamGamesToCheck = 0;
+                    gameQueriesDone.TrySetResult();
                     if (dlc.IsEmpty)
                     {
                         ProgramData.Log($"[Steam] Skipping {name} ({appId}): no DLCs remained after processing");
@@ -653,7 +659,7 @@ internal sealed partial class SelectForm : CustomForm
         }
         gameDlcTimer.Stop();
 
-        steamGamesToCheck = 0;
+        gameQueriesDone.TrySetResult();
 
         scanTimer.Stop();
         ProgramData.Log($"[Scan] Library scan total: {totalLibraryScanSeconds:F1}s across all platforms");

CreamInstaller/Forms/UpdateForm.cs

@@ -62,7 +62,7 @@ internal sealed partial class UpdateForm : CustomForm
             Version currentVersion = new(Program.Version);
 #endif
             List<ProgramRelease> releases = null;
-            string response =
+            (string response, _) =
                 await HttpClientManager.EnsureGet(
                     $"https://api.github.com/repos/{Program.RepositoryOwner}/{Program.RepositoryName}/releases");
             if (response is not null)

CreamInstaller/Platforms/Steam/SteamCMD.WebAPI.cs

@@ -21,8 +21,15 @@ internal static partial class SteamCMD
             bool cachedExists = cacheFile.FileExists();
             if (!cachedExists || ProgramData.CheckCooldown(appId + ".cmd", isDlc ? CooldownDlc : CooldownGame))
             {
-                string response =
+                (string response, bool permanentFailure) =
                     await HttpClientManager.EnsureGet($"https://api.steamcmd.net/v1/info/{appId}");
+                if (permanentFailure)
+                {
+                    ProgramData.LogSteam("[SteamAPI] SteamCMD web API query failed on attempt #" + attempts + " for " +
+                                            appId + (isDlc ? " (DLC)" : "") +
+                                            ": Permanent failure, aborting retries");
+                    return null;
+                }
                 if (response is not null)
                 {
                     try
@@ -83,13 +90,14 @@ internal static partial class SteamCMD
 
             if (isDlc)
                 break;
-            if (attempts > 10)
+            if (attempts > 3)
             {
-                ProgramData.LogSteam("[SteamAPI] Failed to query SteamCMD web API after 10 tries: " + appId);
+                ProgramData.LogSteam("[SteamAPI] Failed to query SteamCMD web API after 3 tries: " + appId);
                 break;
             }
 
-            Thread.Sleep(1000);
+            int delayMs = Math.Min(1000 * (int)Math.Pow(2, attempts - 1), 10000);
+            await Task.Delay(delayMs + Random.Shared.Next(0, 1000));
         }
 
         return null;

CreamInstaller/Platforms/Steam/SteamCMD.cs

@@ -41,7 +41,7 @@ internal static partial class SteamCMD
             : $"+login anonymous +app_info_print {appId} +quit";
 
     private static async Task<string> Run(string appId)
-        => await Task.Run(() =>
+        => await Task.Run(async () =>
         {
             while (true)
             {
@@ -72,10 +72,14 @@ internal static partial class SteamCMD
                         StandardErrorEncoding = Encoding.UTF8
                     };
                     Process process = Process.Start(processStartInfo);
+                    // Drain stderr asynchronously to prevent pipe deadlock
+                    process.BeginErrorReadLine();
                     StringBuilder output = new();
                     StringBuilder appInfo = new();
                     bool appInfoStarted = false;
                     DateTime lastOutput = DateTime.UtcNow;
+                    const int bufferSize = 4096;
+                    char[] buffer = new char[bufferSize];
                     while (process != null)
                     {
                         if (Program.Canceled)
@@ -85,21 +89,30 @@ internal static partial class SteamCMD
                             break;
                         }
 
-                        int c = process.StandardOutput.Read();
+                        // Buffered read: ReadAsync returns up to bufferSize chars per call,
-                        if (c != -1)
+                        // with Task.WhenAny providing a 5s idle timeout
+                        Task<int> readTask = process.StandardOutput.ReadAsync(buffer, 0, bufferSize);
+                        if (await Task.WhenAny(readTask, Task.Delay(5000)) == readTask)
                         {
-                            lastOutput = DateTime.UtcNow;
+                            int charsRead = await readTask;
-                            char ch = (char)c;
+                            if (charsRead > 0)
-                            if (ch == '{')
+                            {
-                                appInfoStarted = true;
+                                lastOutput = DateTime.UtcNow;
-                            _ = appInfoStarted ? appInfo.Append(ch) : output.Append(ch);
+                                for (int j = 0; j < charsRead; j++)
+                                {
+                                    char ch = buffer[j];
+                                    if (ch == '{')
+                                        appInfoStarted = true;
+                                    _ = appInfoStarted ? appInfo.Append(ch) : output.Append(ch);
+                                }
+                                continue;
+                            }
+                            // charsRead == 0: stream closed, process exited naturally
                         }
+                        // else: timeout — 5 seconds without any output
 
-                        DateTime now = DateTime.UtcNow;
+                        if (!process.HasExited)
-                        TimeSpan timeDiff = now - lastOutput;
+                            process.Kill(true);
-                        if (!(timeDiff.TotalSeconds > 0.1))
-                            continue;
-                        process.Kill(true);
                         process.Close();
                         if (appId != null &&
                             output.ToString().Contains($"No app info for AppID {appId} found, requesting..."))
@@ -107,6 +120,7 @@ internal static partial class SteamCMD
                             AttemptCount[appId]++;
                             processStartInfo.Arguments = GetArguments(appId);
                             process = Process.Start(processStartInfo);
+                            process.BeginErrorReadLine();
                             appInfoStarted = false;
                             _ = output.Clear();
                             _ = appInfo.Clear();
@@ -119,7 +133,7 @@ internal static partial class SteamCMD
                     return appInfo.ToString();
                 }
 
-                Thread.Sleep(200);
+                await Task.Delay(200);
             }
         });
 

CreamInstaller/Platforms/Steam/SteamStore.cs

@@ -55,8 +55,14 @@ internal static class SteamStore
             bool cachedExists = cacheFile.FileExists();
             if (!cachedExists || ProgramData.CheckCooldown(appId, isDlc ? CooldownDlc : CooldownGame))
             {
-                string response =
+                (string response, bool permanentFailure) =
                     await HttpClientManager.EnsureGet($"https://store.steampowered.com/api/appdetails?appids={appId}");
+                if (permanentFailure)
+                {
+                    ProgramData.LogSteam(
+                        "[SteamAPI] " + FormatErrorLog(attempts, appId, gameName, isDlc, "Permanent failure, aborting retries", parentGameName, parentGameAppId));
+                    return null;
+                }
                 if (response is not null)
                 {
                     Dictionary<string, JToken> apps =
@@ -133,14 +139,15 @@ internal static class SteamStore
 
             if (isDlc)
                 break;
-            if (attempts > 10)
+            if (attempts > 3)
             {
                 ProgramData.LogSteam(
-                    "[SteamAPI] " + FormatErrorLog(attempts, appId, gameName, isDlc, "Maximum retry attempts exceeded (10)", parentGameName, parentGameAppId));
+                    "[SteamAPI] " + FormatErrorLog(attempts, appId, gameName, isDlc, "Maximum retry attempts exceeded (3)", parentGameName, parentGameAppId));
                 break;
             }
 
-            Thread.Sleep(1000);
+            int delayMs = Math.Min(1000 * (int)Math.Pow(2, attempts - 1), 10000);
+            await Task.Delay(delayMs + Random.Shared.Next(0, 1000));
         }
 
         return null;

CreamInstaller/Utility/HttpClientManager.cs

@@ -66,7 +66,7 @@ internal static class HttpClientManager
         }
     }
 
-    internal static async Task<string> EnsureGet(string url)
+    internal static async Task<(string content, bool permanentFailure)> EnsureGet(string url)
     {
         try
         {
@@ -75,37 +75,40 @@ internal static class HttpClientManager
                 await HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
             if (response.StatusCode is HttpStatusCode.NotModified &&
                 HttpContentCache.TryGetValue(url, out string content))
-                return content;
+                return (content, false);
             _ = response.EnsureSuccessStatusCode();
             content = await response.Content.ReadAsStringAsync();
             HttpContentCache[url] = content;
-            return content;
+            return (content, false);
         }
         catch (HttpRequestException e)
         {
-            if (e.StatusCode != HttpStatusCode.TooManyRequests)
+            if (e.StatusCode is not null)
             {
-                string statusInfo = e.StatusCode.HasValue ? $" (HTTP {(int)e.StatusCode.Value})" : "";
+                int code = (int)e.StatusCode.Value;
-                ProgramData.LogSteam($"[SteamAPI] Get request failed to {url}{statusInfo}: {e.Message}");
+                bool permanent = code is >= 400 and < 500 and not 429;
-                return null;
+                string label = permanent ? "Permanent failure" : code == 429 ? "Too many requests" : "Get request failed";
+                string statusInfo = $" (HTTP {code}{(permanent ? " - Permanent" : code == 429 ? " - Rate Limited" : "")})";
+                ProgramData.LogSteam($"[SteamAPI] {label} to {url}{statusInfo}: {e.Message}");
+                return (null, permanent);
             }
-            ProgramData.LogSteam($"[SteamAPI] Too many requests to {url} (HTTP 429 - Rate Limited)");
+            ProgramData.LogSteam($"[SteamAPI] Get request failed to {url}: {e.Message}");
-            return null;
+            return (null, false);
         }
         catch (TaskCanceledException)
         {
             ProgramData.LogSteam("[SteamAPI] Get request timed out for " + url);
-            return null;
+            return (null, false);
         }
         catch (OperationCanceledException)
         {
             ProgramData.LogSteam("[SteamAPI] Get request was cancelled for " + url);
-            return null;
+            return (null, false);
         }
         catch (Exception e)
         {
             ProgramData.LogSteam("[SteamAPI] Get request failed to " + url + ": " + e.Message);
-            return null;
+            return (null, false);
         }
     }