diff --git a/00_Common/dotnet/Games.Common/IO/Token.cs b/00_Common/dotnet/Games.Common/IO/Token.cs index 3f8267cb..4faea1fc 100644 --- a/00_Common/dotnet/Games.Common/IO/Token.cs +++ b/00_Common/dotnet/Games.Common/IO/Token.cs @@ -4,34 +4,41 @@ namespace Games.Common.IO { internal class Token { - protected readonly StringBuilder _builder; - private int _trailingWhiteSpaceCount; + private readonly string _value; - private Token() + private Token(string value) { - _builder = new StringBuilder(); + _value = value; } - public Token Append(char character) + public override string ToString() => _value; + + internal class Builder { - _builder.Append(character); + private readonly StringBuilder _builder = new(); + private bool _isQuoted; + private int _trailingWhiteSpaceCount; - _trailingWhiteSpaceCount = char.IsWhiteSpace(character) ? _trailingWhiteSpaceCount + 1 : 0; + public Builder Append(char character) + { + _builder.Append(character); - return this; - } + _trailingWhiteSpaceCount = char.IsWhiteSpace(character) ? _trailingWhiteSpaceCount + 1 : 0; - public override string ToString() => _builder.ToString(0, _builder.Length - _trailingWhiteSpaceCount); + return this; + } - public static Token Create() => new(); + public Builder SetIsQuoted() + { + _isQuoted = true; + return this; + } - public static Token CreateQuoted() => new QuotedToken(); - - public static implicit operator string(Token token) => token.ToString(); - - internal class QuotedToken : Token - { - public override string ToString() => _builder.ToString(); + public Token Build() + { + if (!_isQuoted) { _builder.Length -= _trailingWhiteSpaceCount; } + return new Token(_builder.ToString()); + } } } } \ No newline at end of file diff --git a/00_Common/dotnet/Games.Common/IO/Tokenizer.cs b/00_Common/dotnet/Games.Common/IO/Tokenizer.cs index 14900eb1..857b3331 100644 --- a/00_Common/dotnet/Games.Common/IO/Tokenizer.cs +++ b/00_Common/dotnet/Games.Common/IO/Tokenizer.cs @@ -32,63 +32,70 @@ namespace Games.Common.IO public (Token, bool) Consume(Queue characters) { - var token = Token.Create(); + var tokenBuilder = new Token.Builder(); var state = ITokenizerState.LookForStartOfToken; while (characters.TryDequeue(out var character)) { - (state, token) = state.Consume(character, token); - if (state is AtEndOfTokenState) { return (token, false); } + (state, tokenBuilder) = state.Consume(character, tokenBuilder); + if (state is AtEndOfTokenState) { return (tokenBuilder.Build(), false); } } - return (token, true); + return (tokenBuilder.Build(), true); } private interface ITokenizerState { public static ITokenizerState LookForStartOfToken { get; } = new LookForStartOfTokenState(); - (ITokenizerState, Token) Consume(char character, Token token); + (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder); } private struct LookForStartOfTokenState : ITokenizerState { - public (ITokenizerState, Token) Consume(char character, Token token) => + public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) => character switch { - Separator => (new AtEndOfTokenState(), token), - Quote => (new InQuotedTokenState(), Token.CreateQuoted()), - _ when char.IsWhiteSpace(character) => (this, token), - _ => (new InTokenState(), token.Append(character)) + Separator => (new AtEndOfTokenState(), tokenBuilder), + Quote => (new InQuotedTokenState(), tokenBuilder.SetIsQuoted()), + _ when char.IsWhiteSpace(character) => (this, tokenBuilder), + _ => (new InTokenState(), tokenBuilder.Append(character)) }; } private struct InTokenState : ITokenizerState { - public (ITokenizerState, Token) Consume(char character, Token token) => - character == Separator ? (new AtEndOfTokenState(), token) : (this, token.Append(character)); + public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) => + character == Separator + ? (new AtEndOfTokenState(), tokenBuilder) + : (this, tokenBuilder.Append(character)); } private struct InQuotedTokenState : ITokenizerState { - public (ITokenizerState, Token) Consume(char character, Token token) => - character == Quote ? (new ExpectSeparatorState(), token) : (this, token.Append(character)); + public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) => + character == Quote + ? (new ExpectSeparatorState(), tokenBuilder) + : (this, tokenBuilder.Append(character)); } private struct ExpectSeparatorState : ITokenizerState { - public (ITokenizerState, Token) Consume(char character, Token token) => - character == Separator ? (new AtEndOfTokenState(), token) : (new IgnoreRestOfLineState(), token); + public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) => + character == Separator + ? (new AtEndOfTokenState(), tokenBuilder) + : (new IgnoreRestOfLineState(), tokenBuilder); } private struct IgnoreRestOfLineState : ITokenizerState { - public (ITokenizerState, Token) Consume(char character, Token token) => (this, token); + public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) => + (this, tokenBuilder); } private struct AtEndOfTokenState : ITokenizerState { - public (ITokenizerState, Token) Consume(char character, Token token) => + public (ITokenizerState, Token.Builder) Consume(char character, Token.Builder tokenBuilder) => throw new InvalidOperationException(); } }