Package lu.kbra.pclib.parser

Class Tokenizer

java.lang.Object

lu.kbra.pclib.parser.Tokenizer

Direct Known Subclasses:

ModuleTokenizer

public class Tokenizer
extends Object

Constructor Summary

Constructors

Constructor	Description
Tokenizer(Reader reader)
Tokenizer(String str)

Method Summary

Modifier and Type	Method	Description
char	consume()
protected Token	createToken(TokenType type, int line, int column, String strValue)
protected Token	flushToken()
protected TokenType	getIdentType(String strValue)
List<Token>	getTokens()
boolean	hasNext()
static boolean	isOctalDigit(char digit)
LexerIterator	iterator()
void	lexe()
int	peek()
boolean	peek(char... chars)
int	peek(int offset)
boolean	peek(int offset, char... chars)
boolean	peek(int offset, String s)
boolean	peek(String s)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details

Tokenizer

public Tokenizer(Reader reader)

Tokenizer

public Tokenizer(String str)

Method Details

isOctalDigit

public static boolean isOctalDigit(char digit)

consume

public char consume()

getTokens

public List<Token> getTokens()

hasNext

public boolean hasNext()

iterator

public LexerIterator iterator()

lexe

public void lexe()

peek

public int peek()

peek

public boolean peek(char... chars)

peek

public int peek(int offset)

peek

public boolean peek(int offset,
 char... chars)

peek

public boolean peek(int offset,
 String s)

peek

public boolean peek(String s)

createToken

protected Token createToken(TokenType type,
 int line,
 int column,
 String strValue)

flushToken

protected Token flushToken()

getIdentType

protected TokenType getIdentType(String strValue)