gitea-mirror/basic-computer-games
1
0
Fork 0
mirror of https://github.com/coding-horror/basic-computer-games.git synced 2025-12-22 23:26:40 -08:00
Code Issues Packages Projects Releases Wiki Activity

Change to file-scoped namespaces

Browse Source
This commit is contained in:
Andrew Cooper
2022-02-15 22:33:44 +11:00
parent fc92500074
commit 35f68dcf72
11 changed files with 642 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
00_Common/dotnet/Games.Common/IO/Tokenizer.cs
View File

@@ -1,102 +1,101 @@
using System;
using System.Collections.Generic;
namespace Games.Common.IO
namespace Games.Common.IO;
internal class Tokenizer
{
internal class Tokenizer
private const char Quote = '"';
private const char Separator = ',';
private readonly Queue<char> _characters;
private Tokenizer(string input) => _characters = new Queue<char>(input);
public static IEnumerable<Token> ParseTokens(string input)
{
private const char Quote = '"';
private const char Separator = ',';
if (input is null) { throw new ArgumentNullException(nameof(input)); }
private readonly Queue<char> _characters;
return new Tokenizer(input).ParseTokens();
}
private Tokenizer(string input) => _characters = new Queue<char>(input);
public static IEnumerable<Token> ParseTokens(string input)
private IEnumerable<Token> ParseTokens()
{
while (true)
{
if (input is null) { throw new ArgumentNullException(nameof(input)); }
var (token, isLastToken) = Consume(_characters);
yield return token;
return new Tokenizer(input).ParseTokens();
}
private IEnumerable<Token> ParseTokens()
{
while (true)
{
var (token, isLastToken) = Consume(_characters);
yield return token;
if (isLastToken) { break; }
}
}
public (Token, bool) Consume(Queue<char> characters)
{
var tokenBuilder = new Token.Builder();
var state = ITokenizerState.LookForStartOfToken;
while (characters.TryDequeue(out var character))
{
(state, tokenBuilder) = state.Consume(character, tokenBuilder);
if (state is AtEndOfTokenState) { return (tokenBuilder.Build(), false); }
}
return (tokenBuilder.Build(), true);
}
private interface ITokenizerState
{
public static ITokenizerState LookForStartOfToken { get; } = new LookForStartOfTokenState();
(ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder);
}
private struct LookForStartOfTokenState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character switch
{
Separator => (new AtEndOfTokenState(), tokenBuilder),
Quote => (new InQuotedTokenState(), tokenBuilder.SetIsQuoted()),
_ when char.IsWhiteSpace(character) => (this, tokenBuilder),
_ => (new InTokenState(), tokenBuilder.Append(character))
};
}
private struct InTokenState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character == Separator
? (new AtEndOfTokenState(), tokenBuilder)
: (this, tokenBuilder.Append(character));
}
private struct InQuotedTokenState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character == Quote
? (new ExpectSeparatorState(), tokenBuilder)
: (this, tokenBuilder.Append(character));
}
private struct ExpectSeparatorState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character == Separator
? (new AtEndOfTokenState(), tokenBuilder)
: (new IgnoreRestOfLineState(), tokenBuilder);
}
private struct IgnoreRestOfLineState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
(this, tokenBuilder);
}
private struct AtEndOfTokenState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
throw new InvalidOperationException();
if (isLastToken) { break; }
}
}
}
public (Token, bool) Consume(Queue<char> characters)
{
var tokenBuilder = new Token.Builder();
var state = ITokenizerState.LookForStartOfToken;
while (characters.TryDequeue(out var character))
{
(state, tokenBuilder) = state.Consume(character, tokenBuilder);
if (state is AtEndOfTokenState) { return (tokenBuilder.Build(), false); }
}
return (tokenBuilder.Build(), true);
}
private interface ITokenizerState
{
public static ITokenizerState LookForStartOfToken { get; } = new LookForStartOfTokenState();
(ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder);
}
private struct LookForStartOfTokenState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character switch
{
Separator => (new AtEndOfTokenState(), tokenBuilder),
Quote => (new InQuotedTokenState(), tokenBuilder.SetIsQuoted()),
_ when char.IsWhiteSpace(character) => (this, tokenBuilder),
_ => (new InTokenState(), tokenBuilder.Append(character))
};
}
private struct InTokenState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character == Separator
? (new AtEndOfTokenState(), tokenBuilder)
: (this, tokenBuilder.Append(character));
}
private struct InQuotedTokenState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character == Quote
? (new ExpectSeparatorState(), tokenBuilder)
: (this, tokenBuilder.Append(character));
}
private struct ExpectSeparatorState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character == Separator
? (new AtEndOfTokenState(), tokenBuilder)
: (new IgnoreRestOfLineState(), tokenBuilder);
}
private struct IgnoreRestOfLineState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
(this, tokenBuilder);
}
private struct AtEndOfTokenState : ITokenizerState
{
public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
throw new InvalidOperationException();
}
}