Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   [The "BSD license"]
   Copyright (c) 2005-2009 Terence Parr
   All rights reserved.
  
   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions
   are met:
   1. Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
  2. Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
  3. The name of the author may not be used to endorse or promote products
      derived from this software without specific prior written permission.
 
  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 package org.antlr.runtime;

A lexer is recognizer that draws input symbols from a character stream. lexer grammars result in a subclass of this object. A Lexer object uses simplified match() and error recovery mechanisms in the interest of speed.
 
 public abstract class Lexer extends BaseRecognizer implements TokenSource {
Where is the lexer drawing characters from?
 
 	protected CharStream input;
 
 	public Lexer() {
 	}
 
 	public Lexer(CharStream input) {
 		this. = input;
 	}
 
 	public Lexer(CharStream inputRecognizerSharedState state) {
 		super(state);
 		this. = input;
 	}
 
 	public void reset() {
 		super.reset(); // reset all recognizer state variables
 		// wack Lexer state variables
 		if ( !=null ) {
 			.seek(0); // rewind the input
 		}
 		if ( ==null ) {
 			return// no shared state work to do
 		}
 		. = null;
 		. = null;
 	}

Return a token from this source; i.e., match a token on the char stream.
 
 	public Token nextToken() {
 		while (true) {
 			. = null;
 			. = null;
 			if ( .LA(1)==. ) {
                 Token eof = new CommonToken((CharStream),.,
                                             .,
                                             .index(),.index());
                 eof.setLine(getLine());
                 eof.setCharPositionInLine(getCharPositionInLine());
                 return eof;
 			}
 			try {
 				mTokens();
 				if ( .==null ) {
 					emit();
 				}
 				else if ( .==. ) {
 					continue;
 				}
 				return .;
 			}
 			catch (NoViableAltException nva) {
 				reportError(nva);
				recover(nva); // throw out current char and try again
			}
			catch (RecognitionException re) {
				// match() routine has already called recover()
			}
		}
	}

Instruct the lexer to skip creating a token for current lexer rule and look for another token. nextToken() knows to keep looking when a lexer rule finishes with token set to SKIP_TOKEN. Recall that if token==null at end of any token rule, it creates one for you and emits it.
	public void skip() {
	}

This is the lexer entry point that sets instance var 'token'
	public abstract void mTokens() throws RecognitionException;

Set the char stream and reset the lexer
	public void setCharStream(CharStream input) {
		this. = null;
		this. = input;
	}
		return this.;
	}
	public String getSourceName() {
	}

Currently does not support multiple emits per nextToken invocation for efficiency reasons. Subclass and override this method and nextToken (to push tokens into a list and pull from that list rather than a single variable as this implementation does).
	public void emit(Token token) {
		. = token;
	}

The standard method called to automatically emit a token at the outermost lexical rule. The token object should point into the char buffer start..stop. If there is a text override in 'text', use that to set the token's text. Override this method to emit custom Token objects. If you are building trees, then you should also override Parser or TreeParser.getMissingSymbol().
	public Token emit() {
		emit(t);
		return t;
	}
	public void match(String sthrows MismatchedTokenException {
		int i = 0;
		while ( i<s.length() ) {
			if ( .LA(1)!=s.charAt(i) ) {
				if ( .>0 ) {
					. = true;
					return;
				}
				recover(mte);
				throw mte;
			}
			i++;
			. = false;
		}
	}
	public void matchAny() {
	}
	public void match(int cthrows MismatchedTokenException {
		if ( .LA(1)!=c ) {
			if ( .>0 ) {
				. = true;
				return;
			}
			recover(mte);  // don't really recover; just consume in lexer
			throw mte;
		}
		. = false;
	}
	public void matchRange(int aint b)
	{
		if ( .LA(1)<a || .LA(1)>b ) {
			if ( .>0 ) {
				. = true;
				return;
			}
			recover(mre);
			throw mre;
		}
		. = false;
	}
	public int getLine() {
		return .getLine();
	}
	public int getCharPositionInLine() {
	}

What is the index of the current character of lookahead?
	public int getCharIndex() {
		return .index();
	}

Return the text matched so far for the current token or any text override.
	public String getText() {
		if ( .!=null ) {
			return .;
		}
	}

Set the complete text of this token; it wipes any previous changes to the text.
	public void setText(String text) {
		. = text;
	}
	public void reportError(RecognitionException e) {
TODO: not thought about recovery in lexer yet. // if we've already reported an error and have not matched a token // yet successfully, don't report any errors. if ( errorRecovery ) { //System.err.print("[SPURIOUS] "); return; } errorRecovery = true;
	}
	public String getErrorMessage(RecognitionException eString[] tokenNames) {
		String msg = null;
		if ( e instanceof MismatchedTokenException ) {
			msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting "+getCharErrorDisplay(mte.expecting);
		}
		else if ( e instanceof NoViableAltException ) {
			// for development, can add "decision=<<"+nvae.grammarDecisionDescription+">>"
			// and "(decision="+nvae.decisionNumber+") and
			// "state "+nvae.stateNumber
			msg = "no viable alternative at character "+getCharErrorDisplay(e.c);
		}
		else if ( e instanceof EarlyExitException ) {
			// for development, can add "(decision="+eee.decisionNumber+")"
			msg = "required (...)+ loop did not match anything at character "+getCharErrorDisplay(e.c);
		}
		else if ( e instanceof MismatchedNotSetException ) {
			msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting;
		}
		else if ( e instanceof MismatchedSetException ) {
			msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting;
		}
		else if ( e instanceof MismatchedRangeException ) {
			msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+
		}
		else {
			msg = super.getErrorMessage(etokenNames);
		}
		return msg;
	}
	public String getCharErrorDisplay(int c) {
		String s = String.valueOf((char)c);
		switch ( c ) {
			case . :
				s = "<EOF>";
				break;
			case '\n' :
				s = "\\n";
				break;
			case '\t' :
				s = "\\t";
				break;
			case '\r' :
				s = "\\r";
				break;
		}
		return "'"+s+"'";
	}

Lexers can normally match any char in it's vocabulary after matching a token, so do the easy thing and just kill a character and hope it all works out. You can instead use the rule invocation stack to do sophisticated error recovery if you are in a fragment rule.
	public void recover(RecognitionException re) {
		//System.out.println("consuming char "+(char)input.LA(1)+" during recovery");
		//re.printStackTrace();
	}
	public void traceIn(String ruleNameint ruleIndex)  {
		String inputSymbol = ((char).LT(1))+" line="+getLine()+":"+getCharPositionInLine();
		super.traceIn(ruleNameruleIndexinputSymbol);
	}
	public void traceOut(String ruleNameint ruleIndex)  {
		String inputSymbol = ((char).LT(1))+" line="+getLine()+":"+getCharPositionInLine();
		super.traceOut(ruleNameruleIndexinputSymbol);
	}
New to GrepCode? Check out our FAQ X