From 04780884423bb773c03d70d90bfcac6ae45ea476 Mon Sep 17 00:00:00 2001 From: Frog Date: Sun, 21 Jun 2026 00:30:04 -0700 Subject: [PATCH] Scan and Logging Performance Improvements - Parallelized Steam ACF and DLC scanning - Replaced lock-based logging with async channel to avoid UI locks during parallel thread calls. - Added MD5 cache during file checks to reduce redundant I/O calls --- CreamInstaller/Forms/SelectForm.cs | 7 +- .../Platforms/Steam/SteamLibrary.cs | 81 +++++++------- CreamInstaller/Resources/Resources.cs | 24 ++--- CreamInstaller/Utility/ProgramData.cs | 100 ++++++++++++++---- 4 files changed, 134 insertions(+), 78 deletions(-) diff --git a/CreamInstaller/Forms/SelectForm.cs b/CreamInstaller/Forms/SelectForm.cs index 2ccec04..eeb24b7 100644 --- a/CreamInstaller/Forms/SelectForm.cs +++ b/CreamInstaller/Forms/SelectForm.cs @@ -657,12 +657,7 @@ internal sealed partial class SelectForm : CustomForm } Stopwatch gameDlcTimer = Stopwatch.StartNew(); - foreach (Task task in appTasks) - { - if (Program.Canceled) - return; - await task; - } + await Task.WhenAll(appTasks); gameDlcTimer.Stop(); gameQueriesDone.TrySetResult(); diff --git a/CreamInstaller/Platforms/Steam/SteamLibrary.cs b/CreamInstaller/Platforms/Steam/SteamLibrary.cs index 4e128e8..2928dfd 100644 --- a/CreamInstaller/Platforms/Steam/SteamLibrary.cs +++ b/CreamInstaller/Platforms/Steam/SteamLibrary.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; @@ -29,52 +30,57 @@ internal static class SteamLibrary internal static async Task> GetGames() - => await Task.Run(async () => + { + Stopwatch timer = Stopwatch.StartNew(); + List<(string appId, string name, string branch, int buildId, string gameDirectory)> games = new(); + HashSet seenAppIds = new(); + HashSet gameLibraryDirectories = await GetLibraryDirectories(); + ProgramData.Log($"[Steam] Found {gameLibraryDirectories.Count} library folder(s)."); + foreach (string libraryDirectory in gameLibraryDirectories) { - Stopwatch timer = Stopwatch.StartNew(); - List<(string appId, string name, string branch, int buildId, string gameDirectory)> games = new(); - HashSet gameLibraryDirectories = await GetLibraryDirectories(); - ProgramData.Log($"[Steam] Found {gameLibraryDirectories.Count} library folder(s)."); - foreach (string libraryDirectory in gameLibraryDirectories) + if (Program.Canceled) + return games; + ProgramData.Log($"[Steam] Scanning library: {libraryDirectory}"); + foreach ((string appId, string name, string branch, int buildId, string gameDirectory) game in + await GetGamesFromLibraryDirectory(libraryDirectory)) { - if (Program.Canceled) - return games; - ProgramData.Log($"[Steam] Scanning library: {libraryDirectory}"); - foreach ((string appId, string name, string branch, int buildId, string gameDirectory) game in (await - GetGamesFromLibraryDirectory( - libraryDirectory)).Where(game => games.All(_game => _game.appId != game.appId))) + if (seenAppIds.Add(game.appId)) games.Add(game); } + } - foreach ((string appId, string name, string branch, int buildId, string gameDirectory) testGame in - TestGames.Where(t => games.All(g => g.appId != t.appId))) - games.Add(testGame); - if (TestGames.Count > 0) - ProgramData.Log($"[Steam] Injected {TestGames.Count} test game(s)."); - timer.Stop(); - ProgramData.Log($"[Steam] Total games detected: {games.Count} in {(timer.Elapsed.TotalSeconds >= 60 ? $"{timer.Elapsed.TotalSeconds / 60:F1} minutes" : $"{timer.Elapsed.TotalSeconds:F1}s")}"); - return games; - }); + foreach ((string appId, string name, string branch, int buildId, string gameDirectory) testGame in + TestGames.Where(t => !seenAppIds.Contains(t.appId))) + games.Add(testGame); + if (TestGames.Count > 0) + ProgramData.Log($"[Steam] Injected {TestGames.Count} test game(s)."); + timer.Stop(); + ProgramData.Log($"[Steam] Total games detected: {games.Count} in {(timer.Elapsed.TotalSeconds >= 60 ? $"{timer.Elapsed.TotalSeconds / 60:F1} minutes" : $"{timer.Elapsed.TotalSeconds:F1}s")}"); + return games; + } private static async Task> GetGamesFromLibraryDirectory(string libraryDirectory) => await Task.Run(() => { - List<(string appId, string name, string branch, int buildId, string gameDirectory)> games = new(); if (Program.Canceled || !libraryDirectory.DirectoryExists()) { ProgramData.Log($"[Steam] Skipping library (not found or canceled): {libraryDirectory}"); - return games; + return []; } - foreach (string file in libraryDirectory.EnumerateDirectory("*.acf")) + ConcurrentDictionary gamesDict = new(); + Parallel.ForEach(libraryDirectory.EnumerateDirectory("*.acf"), (file, state) => { if (Program.Canceled) - return games; + { + state.Stop(); + return; + } if (!ValveDataFile.TryDeserialize(file.ReadFile(), out VProperty result)) { ProgramData.Log($"[Steam] Failed to deserialize ACF: {file}"); - continue; + return; } string appId = result.Value.GetChild("appid")?.ToString(); @@ -82,11 +88,10 @@ internal static class SteamLibrary string name = result.Value.GetChild("name")?.ToString(); string buildId = result.Value.GetChild("buildid")?.ToString(); if (string.IsNullOrWhiteSpace(appId) || string.IsNullOrWhiteSpace(installdir) || - string.IsNullOrWhiteSpace(name) - || string.IsNullOrWhiteSpace(buildId)) + string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(buildId)) { ProgramData.Log($"[Steam] Skipping ACF with missing fields: {file}"); - continue; + return; } string rawGameDirectory = libraryDirectory + @"\common\" + installdir; @@ -94,12 +99,11 @@ internal static class SteamLibrary if (gameDirectory is null) { ProgramData.Log($"[Steam] Game directory not found (drive may be slow or disconnected): {rawGameDirectory} | App: {name} ({appId})"); - continue; + return; } - if (!int.TryParse(appId, out int _) || !int.TryParse(buildId, out int buildIdInt) || - games.Any(g => g.appId == appId)) - continue; + if (!int.TryParse(appId, out int _) || !int.TryParse(buildId, out int buildIdInt)) + return; VToken userConfig = result.Value.GetChild("UserConfig"); string branch = userConfig?.GetChild("BetaKey")?.ToString(); @@ -114,10 +118,13 @@ internal static class SteamLibrary if (string.IsNullOrWhiteSpace(branch)) branch = "public"; - ProgramData.Log($"[Steam] Detected game: {name} ({appId}) | Branch: {branch} | Dir: {gameDirectory}"); - games.Add((appId, name, branch, buildIdInt, gameDirectory)); - } + if (gamesDict.TryAdd(appId, (name, branch, buildIdInt, gameDirectory))) + ProgramData.Log($"[Steam] Detected game: {name} ({appId}) | Branch: {branch} | Dir: {gameDirectory}"); + }); + List<(string appId, string name, string branch, int buildId, string gameDirectory)> games = new(gamesDict.Count); + foreach (KeyValuePair kv in gamesDict) + games.Add((kv.Key, kv.Value.name, kv.Value.branch, kv.Value.buildId, kv.Value.gameDirectory)); return games; }); diff --git a/CreamInstaller/Resources/Resources.cs b/CreamInstaller/Resources/Resources.cs index ac56a9e..0b14617 100644 --- a/CreamInstaller/Resources/Resources.cs +++ b/CreamInstaller/Resources/Resources.cs @@ -111,19 +111,7 @@ internal static class Resources !rootDirectory.IsCommonIncorrectExecutable(path)) && (validFunc is null || validFunc(path)) && path.TryGetFileBinaryType(out BinaryType binaryType) && - binaryType is BinaryType.BIT64) - executables.Add((path, binaryType)); - } - - foreach (string path in rootDirectory.EnumerateDirectory("*.exe", true)) - { - if (Program.Canceled) - return null; - if (executables.All(e => e.path != path) && (!filterCommon || - !rootDirectory.IsCommonIncorrectExecutable(path)) - && (validFunc is null || validFunc(path)) && - path.TryGetFileBinaryType(out BinaryType binaryType) && - binaryType is BinaryType.BIT32) + binaryType is BinaryType.BIT64 or BinaryType.BIT32) executables.Add((path, binaryType)); } @@ -202,11 +190,19 @@ internal static class Resources return dllDirectories.Count > 0 ? dllDirectories : null; }); + private static readonly System.Collections.Concurrent.ConcurrentDictionary Md5HashCache = new(); + #pragma warning disable CA5351 private static string ComputeMD5(this string filePath) - => filePath.FileExists() && filePath.ReadFileBytes(true) is { } bytes + { + if (Md5HashCache.TryGetValue(filePath, out string cached)) + return cached; + string hash = filePath.FileExists() && filePath.ReadFileBytes(true) is { } bytes ? BitConverter.ToString(MD5.HashData(bytes)).Replace("-", "").ToUpperInvariant() : null; + Md5HashCache[filePath] = hash; + return hash; + } #pragma warning restore CA5351 internal static bool IsResourceFile(this string filePath, ResourceIdentifier identifier) diff --git a/CreamInstaller/Utility/ProgramData.cs b/CreamInstaller/Utility/ProgramData.cs index 3183594..68424ea 100644 --- a/CreamInstaller/Utility/ProgramData.cs +++ b/CreamInstaller/Utility/ProgramData.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.IO; using System.Linq; using System.Text; +using System.Threading.Channels; using System.Threading.Tasks; using System.Windows.Forms; using CreamInstaller; @@ -81,16 +82,69 @@ internal static event Action OnLogSteam; internal static event Action OnLogWarning; internal static event Action OnLogError; - private static readonly object LogLock = new(); + private static string FormatLogEntry(string message) + { + string timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture); + return $"[{timestamp}] {message}{Environment.NewLine}"; + } + + private static string FormatLogErrorEntry(string message, Exception ex) + { + string timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture); + return $"[{timestamp}] [ERROR] {message}{Environment.NewLine}[{timestamp}] [ERROR] Exception: {ex}{Environment.NewLine}"; + } + + private readonly record struct LogEntry(string Path, string Text); + + private static readonly Channel LogChannel = Channel.CreateUnbounded(); + private static readonly Task LogConsumer; + + static ProgramData() + { + LogConsumer = ConsumeLogAsync(); + } + + private static async Task ConsumeLogAsync() + { + ChannelReader reader = LogChannel.Reader; + List batch = new(64); + while (await reader.WaitToReadAsync()) + { + batch.Clear(); + while (reader.TryRead(out LogEntry entry)) + { + batch.Add(entry); + if (batch.Count >= 64) + break; + } + if (batch.Count > 0) + FlushLogBatch(batch); + } + } + + private static void FlushLogBatch(List entries) + { + try + { + foreach (IGrouping group in entries.GroupBy(e => e.Path)) + { + StringBuilder combined = new(group.Sum(e => e.Text.Length)); + foreach (LogEntry entry in group) + _ = combined.Append(entry.Text); + File.AppendAllText(group.Key, combined.ToString(), Encoding.UTF8); + } + } + catch + { + // ignored; logging must never crash the application + } + } internal static void Log(string message) { try { - string timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture); - string entry = $"[{timestamp}] {message}{Environment.NewLine}"; - lock (LogLock) - File.AppendAllText(ScanLogPath, entry, Encoding.UTF8); + LogChannel.Writer.TryWrite(new LogEntry(ScanLogPath, FormatLogEntry(message))); } catch { @@ -103,10 +157,7 @@ internal static event Action OnLogError; { try { - string timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture); - string entry = $"[{timestamp}] {message}{Environment.NewLine}"; - lock (LogLock) - File.AppendAllText(SteamLogPath, entry, Encoding.UTF8); + LogChannel.Writer.TryWrite(new LogEntry(SteamLogPath, FormatLogEntry(message))); } catch { @@ -119,10 +170,7 @@ internal static event Action OnLogError; { try { - string timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture); - string entry = $"[{timestamp}] [WARN] {message}{Environment.NewLine}"; - lock (LogLock) - File.AppendAllText(AppLogPath, entry, Encoding.UTF8); + LogChannel.Writer.TryWrite(new LogEntry(AppLogPath, FormatLogEntry($"[WARN] {message}"))); } catch { @@ -135,12 +183,10 @@ internal static event Action OnLogError; { try { - string timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture); string entry = ex is not null - ? $"[{timestamp}] [ERROR] {message}{Environment.NewLine}[{timestamp}] [ERROR] Exception: {ex}{Environment.NewLine}" - : $"[{timestamp}] [ERROR] {message}{Environment.NewLine}"; - lock (LogLock) - File.AppendAllText(AppLogPath, entry, Encoding.UTF8); + ? FormatLogErrorEntry(message, ex) + : FormatLogEntry($"[ERROR] {message}"); + LogChannel.Writer.TryWrite(new LogEntry(AppLogPath, entry)); } catch { @@ -188,13 +234,25 @@ internal static event Action OnLogError; OldProgramChoicesPath.DeleteFile(); }); + private static readonly System.Collections.Concurrent.ConcurrentDictionary CooldownCache = new(); + internal static bool CheckCooldown(string identifier, int cooldown) { DateTime now = DateTime.UtcNow; - DateTime lastCheck = GetCooldown(identifier) ?? now; - bool cooldownOver = (now - lastCheck).TotalSeconds > cooldown; - if (cooldownOver || now == lastCheck) + + if (CooldownCache.TryGetValue(identifier, out DateTime cached) && + (now - cached).TotalSeconds <= cooldown) + return false; + + DateTime? lastCheck = GetCooldown(identifier); + DateTime effective = lastCheck ?? now; + bool cooldownOver = (now - effective).TotalSeconds > cooldown; + + CooldownCache[identifier] = now; + + if (cooldownOver || now == effective) SetCooldown(identifier, now); + return cooldownOver; }