gitea-mirror/basic-computer-games
1
0
Fork 0
mirror of https://github.com/coding-horror/basic-computer-games.git synced 2025-12-23 07:29:02 -08:00
Code Issues Packages Projects Releases Wiki Activity

Add token reader

Browse Source
This commit is contained in:
Andrew Cooper
2022-02-10 22:40:12 +11:00
parent 25c8dad512
commit 3b42ffd18d
4 changed files with 177 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
00_Common/dotnet/Games.Common/IO/Tokenizer.cs
View File

@@ -12,14 +12,14 @@ namespace Games.Common.IO
private Tokenizer(string input) => _characters = new Queue<char>(input);
public static IEnumerable<string> ParseTokens(string input)
public static IEnumerable<Token> ParseTokens(string input)
{
if (input is null) { throw new ArgumentNullException(nameof(input)); }
return new Tokenizer(input).ParseTokens();
}
private IEnumerable<string> ParseTokens()
private IEnumerable<Token> ParseTokens()
{
while (true)
{
@@ -72,13 +72,18 @@ namespace Games.Common.IO
private struct InQuotedTokenState : ITokenizerState
{
public (ITokenizerState, Token) Consume(char character, Token token) =>
character == Quote ? (new LookForSeparatorState(), token) : (this, token.Append(character));
character == Quote ? (new ExpectSeparatorState(), token) : (this, token.Append(character));
}
private struct LookForSeparatorState : ITokenizerState
private struct ExpectSeparatorState : ITokenizerState
{
public (ITokenizerState, Token) Consume(char character, Token token) =>
(character == Separator ? new AtEndOfTokenState() : this, token);
character == Separator ? (new AtEndOfTokenState(), token) : (new IgnoreRestOfLineState(), token);
}
private struct IgnoreRestOfLineState : ITokenizerState
{
public (ITokenizerState, Token) Consume(char character, Token token) => (this, token);
}
private struct AtEndOfTokenState : ITokenizerState