gitea-mirror/basic-computer-games
1
0
Fork 0
mirror of https://github.com/coding-horror/basic-computer-games.git synced 2025-12-22 07:10:42 -08:00
Code Issues Packages Projects Releases Wiki Activity

Rework token building

Browse Source
This commit is contained in:
Andrew Cooper
2022-02-11 08:19:02 +11:00
parent 3b42ffd18d
commit a7cedfbf7e
2 changed files with 50 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
00_Common/dotnet/Games.Common/IO/Token.cs
View File

@@ -4,34 +4,41 @@ namespace Games.Common.IO
{ {
internal class Token internal class Token
{ {
protected readonly StringBuilder _builder; private readonly string _value;
private int _trailingWhiteSpaceCount;
private Token() private Token(string value)
{ {
_builder = new StringBuilder(); _value = value;
} }
public Token Append(char character) public override string ToString() => _value;
internal class Builder
{ {
_builder.Append(character); private readonly StringBuilder _builder = new();
private bool _isQuoted;
private int _trailingWhiteSpaceCount;
_trailingWhiteSpaceCount = char.IsWhiteSpace(character) ? _trailingWhiteSpaceCount + 1 : 0; public Builder Append(char character)
{
_builder.Append(character);
return this; _trailingWhiteSpaceCount = char.IsWhiteSpace(character) ? _trailingWhiteSpaceCount + 1 : 0;
}
public override string ToString() => _builder.ToString(0, _builder.Length - _trailingWhiteSpaceCount); return this;
}
public static Token Create() => new(); public Builder SetIsQuoted()
{
_isQuoted = true;
return this;
}
public static Token CreateQuoted() => new QuotedToken(); public Token Build()
{
public static implicit operator string(Token token) => token.ToString(); if (!_isQuoted) { _builder.Length -= _trailingWhiteSpaceCount; }
return new Token(_builder.ToString());
internal class QuotedToken : Token }
{
public override string ToString() => _builder.ToString();
} }
} }
} }

43
00_Common/dotnet/Games.Common/IO/Tokenizer.cs
View File

@@ -32,63 +32,70 @@ namespace Games.Common.IO
public (Token, bool) Consume(Queue<char> characters) public (Token, bool) Consume(Queue<char> characters)
{ {
var token = Token.Create(); var tokenBuilder = new Token.Builder();
var state = ITokenizerState.LookForStartOfToken; var state = ITokenizerState.LookForStartOfToken;
while (characters.TryDequeue(out var character)) while (characters.TryDequeue(out var character))
{ {
(state, token) = state.Consume(character, token); (state, tokenBuilder) = state.Consume(character, tokenBuilder);
if (state is AtEndOfTokenState) { return (token, false); } if (state is AtEndOfTokenState) { return (tokenBuilder.Build(), false); }
} }
return (token, true); return (tokenBuilder.Build(), true);
} }
private interface ITokenizerState private interface ITokenizerState
{ {
public static ITokenizerState LookForStartOfToken { get; } = new LookForStartOfTokenState(); public static ITokenizerState LookForStartOfToken { get; } = new LookForStartOfTokenState();
(ITokenizerState, Token) Consume(char character, Token token); (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder);
} }
private struct LookForStartOfTokenState : ITokenizerState private struct LookForStartOfTokenState : ITokenizerState
{ {
public (ITokenizerState, Token) Consume(char character, Token token) => public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character switch character switch
{ {
Separator => (new AtEndOfTokenState(), token), Separator => (new AtEndOfTokenState(), tokenBuilder),
Quote => (new InQuotedTokenState(), Token.CreateQuoted()), Quote => (new InQuotedTokenState(), tokenBuilder.SetIsQuoted()),
_ when char.IsWhiteSpace(character) => (this, token), _ when char.IsWhiteSpace(character) => (this, tokenBuilder),
_ => (new InTokenState(), token.Append(character)) _ => (new InTokenState(), tokenBuilder.Append(character))
}; };
} }
private struct InTokenState : ITokenizerState private struct InTokenState : ITokenizerState
{ {
public (ITokenizerState, Token) Consume(char character, Token token) => public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character == Separator ? (new AtEndOfTokenState(), token) : (this, token.Append(character)); character == Separator
? (new AtEndOfTokenState(), tokenBuilder)
: (this, tokenBuilder.Append(character));
} }
private struct InQuotedTokenState : ITokenizerState private struct InQuotedTokenState : ITokenizerState
{ {
public (ITokenizerState, Token) Consume(char character, Token token) => public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character == Quote ? (new ExpectSeparatorState(), token) : (this, token.Append(character)); character == Quote
? (new ExpectSeparatorState(), tokenBuilder)
: (this, tokenBuilder.Append(character));
} }
private struct ExpectSeparatorState : ITokenizerState private struct ExpectSeparatorState : ITokenizerState
{ {
public (ITokenizerState, Token) Consume(char character, Token token) => public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
character == Separator ? (new AtEndOfTokenState(), token) : (new IgnoreRestOfLineState(), token); character == Separator
? (new AtEndOfTokenState(), tokenBuilder)
: (new IgnoreRestOfLineState(), tokenBuilder);
} }
private struct IgnoreRestOfLineState : ITokenizerState private struct IgnoreRestOfLineState : ITokenizerState
{ {
public (ITokenizerState, Token) Consume(char character, Token token) => (this, token); public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
(this, tokenBuilder);
} }
private struct AtEndOfTokenState : ITokenizerState private struct AtEndOfTokenState : ITokenizerState
{ {
public (ITokenizerState, Token) Consume(char character, Token token) => public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) =>
throw new InvalidOperationException(); throw new InvalidOperationException();
} }
} }