gitea-mirror/CreamInstaller
1
0
Fork 0
mirror of https://github.com/FroggMaster/CreamInstaller.git synced 2026-07-28 14:47:04 -07:00
Code Issues Packages Projects Releases Wiki Activity

Improve Game and DLC Scanning Performance / Reduced Steam API Retires

Browse Source 
- SteamCMD: Changed output reading from one character at a time to buffered reads, fixing a stderr deadlock issue and making the process much faster and more responsive to cancellation.
- DLC detection: Eliminated a busy-wait loop that was polling every 200ms on a background thread while waiting for game data to arrive. Now uses proper async signaling so the thread is freed up entirely.
- HTTP retries: Previously every Steam API error was retried up to 10 times with delays, even for "403 Forbidden" errors that will never succeed. The scanner now gives up immediately on permanent errors, preventing up to 90 seconds of wasted retries per failed app, and avoids triggering rate limits by hammering the API on doomed requests
- Replaced fixed 1-second waits with smarter retry timing that waits longer after each failed attempt and adds a small random delay to reduce API requests. Also reduced the maximum number of retry attempts from 10 to 3 for temporary network errors.
This commit is contained in:
Frog
2026-06-17 13:44:26 -07:00
parent 6ecab7a4d3
commit 36623eec09
6 changed files with 78 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CreamInstaller/Forms/SelectForm.cs
+11 -5
View File
@@ -177,6 +177,7 @@ internal sealed partial class SelectForm : CustomForm
}
int steamGamesToCheck;
TaskCompletionSource gameQueriesDone = new();
if (uninstallAll || programsToScan.Any(c => c.platform is Platform.Steam))
{
Stopwatch steamLibTimer = Stopwatch.StartNew();
@@ -244,6 +245,8 @@ internal sealed partial class SelectForm : CustomForm
return;
StoreAppData storeAppData = await SteamStore.QueryStoreAPI(appId);
_ = Interlocked.Decrement(ref steamGamesToCheck);
if (Volatile.Read(ref steamGamesToCheck) == 0)
gameQueriesDone.TrySetResult();
CmdAppData cmdAppData = await WithTimeout(SteamCMD.GetAppInfo(appId, branch, buildId), 16000);
if (storeAppData is null && cmdAppData is null)
{
@@ -273,9 +276,12 @@ internal sealed partial class SelectForm : CustomForm
{
if (Program.Canceled)
return;
do // give games steam store api limit priority
Thread.Sleep(200);
while (!Program.Canceled && steamGamesToCheck > 0);
while (!Program.Canceled)
{
Task completed = await Task.WhenAny(gameQueriesDone.Task, Task.Delay(250));
if (completed == gameQueriesDone.Task)
break;
}
if (Program.Canceled)
return;
string fullGameAppId = null;
@@ -374,7 +380,7 @@ internal sealed partial class SelectForm : CustomForm
await task;
}
steamGamesToCheck = 0;
gameQueriesDone.TrySetResult();
if (dlc.IsEmpty)
{
ProgramData.Log($"[Steam] Skipping {name} ({appId}): no DLCs remained after processing");
@@ -653,7 +659,7 @@ internal sealed partial class SelectForm : CustomForm
}
gameDlcTimer.Stop();
steamGamesToCheck = 0;
gameQueriesDone.TrySetResult();
scanTimer.Stop();
ProgramData.Log($"[Scan] Library scan total: {totalLibraryScanSeconds:F1}s across all platforms");
CreamInstaller/Forms/UpdateForm.cs
+1 -1
View File
@@ -62,7 +62,7 @@ internal sealed partial class UpdateForm : CustomForm
Version currentVersion = new(Program.Version);
#endif
List<ProgramRelease> releases = null;
string response =
(string response, _) =
await HttpClientManager.EnsureGet(
$"https://api.github.com/repos/{Program.RepositoryOwner}/{Program.RepositoryName}/releases");
if (response is not null)
CreamInstaller/Platforms/Steam/SteamCMD.WebAPI.cs
+12 -4
View File
@@ -21,8 +21,15 @@ internal static partial class SteamCMD
bool cachedExists = cacheFile.FileExists();
if (!cachedExists || ProgramData.CheckCooldown(appId + ".cmd", isDlc ? CooldownDlc : CooldownGame))
{
string response =
(string response, bool permanentFailure) =
await HttpClientManager.EnsureGet($"https://api.steamcmd.net/v1/info/{appId}");
if (permanentFailure)
{
ProgramData.LogSteam("[SteamAPI] SteamCMD web API query failed on attempt #" + attempts + " for " +
appId + (isDlc ? " (DLC)" : "") +
": Permanent failure, aborting retries");
return null;
}
if (response is not null)
{
try
@@ -83,13 +90,14 @@ internal static partial class SteamCMD
if (isDlc)
break;
if (attempts > 10)
if (attempts > 3)
{
ProgramData.LogSteam("[SteamAPI] Failed to query SteamCMD web API after 10 tries: " + appId);
ProgramData.LogSteam("[SteamAPI] Failed to query SteamCMD web API after 3 tries: " + appId);
break;
}
Thread.Sleep(1000);
int delayMs = Math.Min(1000 * (int)Math.Pow(2, attempts - 1), 10000);
await Task.Delay(delayMs + Random.Shared.Next(0, 1000));
}
return null;
CreamInstaller/Platforms/Steam/SteamCMD.cs
+28 -14
View File
@@ -41,7 +41,7 @@ internal static partial class SteamCMD
: $"+login anonymous +app_info_print {appId} +quit";
private static async Task<string> Run(string appId)
=> await Task.Run(() =>
=> await Task.Run(async () =>
{
while (true)
{
@@ -72,10 +72,14 @@ internal static partial class SteamCMD
StandardErrorEncoding = Encoding.UTF8
};
Process process = Process.Start(processStartInfo);
// Drain stderr asynchronously to prevent pipe deadlock
process.BeginErrorReadLine();
StringBuilder output = new();
StringBuilder appInfo = new();
bool appInfoStarted = false;
DateTime lastOutput = DateTime.UtcNow;
const int bufferSize = 4096;
char[] buffer = new char[bufferSize];
while (process != null)
{
if (Program.Canceled)
@@ -85,21 +89,30 @@ internal static partial class SteamCMD
break;
}
int c = process.StandardOutput.Read();
if (c != -1)
// Buffered read: ReadAsync returns up to bufferSize chars per call,
// with Task.WhenAny providing a 5s idle timeout
Task<int> readTask = process.StandardOutput.ReadAsync(buffer, 0, bufferSize);
if (await Task.WhenAny(readTask, Task.Delay(5000)) == readTask)
{
lastOutput = DateTime.UtcNow;
char ch = (char)c;
if (ch == '{')
appInfoStarted = true;
_ = appInfoStarted ? appInfo.Append(ch) : output.Append(ch);
int charsRead = await readTask;
if (charsRead > 0)
{
lastOutput = DateTime.UtcNow;
for (int j = 0; j < charsRead; j++)
{
char ch = buffer[j];
if (ch == '{')
appInfoStarted = true;
_ = appInfoStarted ? appInfo.Append(ch) : output.Append(ch);
}
continue;
}
// charsRead == 0: stream closed, process exited naturally
}
// else: timeout — 5 seconds without any output
DateTime now = DateTime.UtcNow;
TimeSpan timeDiff = now - lastOutput;
if (!(timeDiff.TotalSeconds > 0.1))
continue;
process.Kill(true);
if (!process.HasExited)
process.Kill(true);
process.Close();
if (appId != null &&
output.ToString().Contains($"No app info for AppID {appId} found, requesting..."))
@@ -107,6 +120,7 @@ internal static partial class SteamCMD
AttemptCount[appId]++;
processStartInfo.Arguments = GetArguments(appId);
process = Process.Start(processStartInfo);
process.BeginErrorReadLine();
appInfoStarted = false;
_ = output.Clear();
_ = appInfo.Clear();
@@ -119,7 +133,7 @@ internal static partial class SteamCMD
return appInfo.ToString();
}
Thread.Sleep(200);
await Task.Delay(200);
}
});
CreamInstaller/Platforms/Steam/SteamStore.cs
+11 -4
View File
@@ -55,8 +55,14 @@ internal static class SteamStore
bool cachedExists = cacheFile.FileExists();
if (!cachedExists || ProgramData.CheckCooldown(appId, isDlc ? CooldownDlc : CooldownGame))
{
string response =
(string response, bool permanentFailure) =
await HttpClientManager.EnsureGet($"https://store.steampowered.com/api/appdetails?appids={appId}");
if (permanentFailure)
{
ProgramData.LogSteam(
"[SteamAPI] " + FormatErrorLog(attempts, appId, gameName, isDlc, "Permanent failure, aborting retries", parentGameName, parentGameAppId));
return null;
}
if (response is not null)
{
Dictionary<string, JToken> apps =
@@ -133,14 +139,15 @@ internal static class SteamStore
if (isDlc)
break;
if (attempts > 10)
if (attempts > 3)
{
ProgramData.LogSteam(
"[SteamAPI] " + FormatErrorLog(attempts, appId, gameName, isDlc, "Maximum retry attempts exceeded (10)", parentGameName, parentGameAppId));
"[SteamAPI] " + FormatErrorLog(attempts, appId, gameName, isDlc, "Maximum retry attempts exceeded (3)", parentGameName, parentGameAppId));
break;
}
Thread.Sleep(1000);
int delayMs = Math.Min(1000 * (int)Math.Pow(2, attempts - 1), 10000);
await Task.Delay(delayMs + Random.Shared.Next(0, 1000));
}
return null;
CreamInstaller/Utility/HttpClientManager.cs
+15 -12
View File
@@ -66,7 +66,7 @@ internal static class HttpClientManager
}
}
internal static async Task<string> EnsureGet(string url)
internal static async Task<(string content, bool permanentFailure)> EnsureGet(string url)
{
try
{
@@ -75,37 +75,40 @@ internal static class HttpClientManager
await HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
if (response.StatusCode is HttpStatusCode.NotModified &&
HttpContentCache.TryGetValue(url, out string content))
return content;
return (content, false);
_ = response.EnsureSuccessStatusCode();
content = await response.Content.ReadAsStringAsync();
HttpContentCache[url] = content;
return content;
return (content, false);
}
catch (HttpRequestException e)
{
if (e.StatusCode != HttpStatusCode.TooManyRequests)
if (e.StatusCode is not null)
{
string statusInfo = e.StatusCode.HasValue ? $" (HTTP {(int)e.StatusCode.Value})" : "";
ProgramData.LogSteam($"[SteamAPI] Get request failed to {url}{statusInfo}: {e.Message}");
return null;
int code = (int)e.StatusCode.Value;
bool permanent = code is >= 400 and < 500 and not 429;
string label = permanent ? "Permanent failure" : code == 429 ? "Too many requests" : "Get request failed";
string statusInfo = $" (HTTP {code}{(permanent ? " - Permanent" : code == 429 ? " - Rate Limited" : "")})";
ProgramData.LogSteam($"[SteamAPI] {label} to {url}{statusInfo}: {e.Message}");
return (null, permanent);
}
ProgramData.LogSteam($"[SteamAPI] Too many requests to {url} (HTTP 429 - Rate Limited)");
return null;
ProgramData.LogSteam($"[SteamAPI] Get request failed to {url}: {e.Message}");
return (null, false);
}
catch (TaskCanceledException)
{
ProgramData.LogSteam("[SteamAPI] Get request timed out for " + url);
return null;
return (null, false);
}
catch (OperationCanceledException)
{
ProgramData.LogSteam("[SteamAPI] Get request was cancelled for " + url);
return null;
return (null, false);
}
catch (Exception e)
{
ProgramData.LogSteam("[SteamAPI] Get request failed to " + url + ": " + e.Message);
return null;
return (null, false);
}
}