Tokenizer

class Tokenizer {

SourceLines _lineStream;

dchar[] _lineText;

int _line;

int _len;

int _pos;

int _prevLineLength;

uint _state;

WhiteSpaceToken _sharedWhiteSpaceToken;

CommentToken _sharedCommentToken;

StringLiteralToken _sharedStringLiteralToken;

IdentToken _sharedIdentToken;

OpToken _sharedOpToken;

KeywordToken _sharedKeywordToken;

IntegerLiteralToken _sharedIntegerToken;

RealLiteralToken _sharedRealToken;

InvalidToken _sharedInvalidToken;

CharacterLiteralToken _sharedCharacterLiteralToken;

StringAppender _stringLiteralAppender;

StringAppender _commentAppender;

StringAppender _identAppender;

bool _enableCommentText;

bool _errorTolerant;

SysTime buildTime;

dstring VERSION;

dstring VENDOR;

int _startLine;

int _startPos;

}

Members

allTokens Token[] allTokens(): tokenize all
parserError Token parserError(string msg, Token incompleteToken): Either return InvalidToken or throw parser exception depending on current errorTolerant flag
parserError Token parserError(string msg, int startLine, int startPos, TokenType failedTokenType = TokenType.INVALID): Either return InvalidToken or throw parser exception depending on current errorTolerant flag

enableCommentText bool enableCommentText [@property setter]: when false, does not put comment text into comment token - for less allocations
enableCommentText bool enableCommentText [@property getter]: when false, does not put comment text into comment token - for less allocations
errorTolerant bool errorTolerant [@property setter]: when true, returns BadToken instead of throwing exception
errorTolerant bool errorTolerant [@property getter]: when true, returns BadToken instead of throwing exception