Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
  * Conditions Of Use
  *
  * This software was developed by employees of the National Institute of
  * Standards and Technology (NIST), an agency of the Federal Government.
  * Pursuant to title 15 Untied States Code Section 105, works of NIST
  * employees are not subject to copyright protection in the United States
  * and are considered to be in the public domain.  As a result, a formal
  * license is not needed to use the software.
 *
 * This software is provided by NIST as a service and is expressly
 * provided "AS IS."  NIST MAKES NO WARRANTY OF ANY KIND, EXPRESS, IMPLIED
 * OR STATUTORY, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTY OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT
 * AND DATA ACCURACY.  NIST does not warrant or make any representations
 * regarding the use of the software or the results thereof, including but
 * not limited to the correctness, accuracy, reliability or usefulness of
 * the software.
 *
 * Permission to use this software is contingent upon your acceptance
 * of the terms of this agreement
 *
 * .
 *
 */
 package gov.nist.core;
 
A lexical analyzer that is used by all parsers in our implementation.

Author(s):
M. Ranganathan
Version:
1.2
Since:
1.1
 
 public class LexerCore extends StringTokenizer {
 
     // IMPORTANT - All keyword matches should be between START and END
     public static final int START = 2048;
     public static final int END =  + 2048;
     // IMPORTANT -- This should be < END
     public static final int ID =  - 1;
     public static final int SAFE =  - 2;
     // Individial token classes.
     public static final int WHITESPACE =  + 1;
     public static final int DIGIT =  + 2;
     public static final int ALPHA =  + 3;
     public static final int BACKSLASH = (int'\\';
     public static final int QUOTE = (int'\'';
     public static final int AT = (int'@';
     public static final int SP = (int' ';
     public static final int HT = (int'\t';
     public static final int COLON = (int':';
     public static final int STAR = (int'*';
     public static final int DOLLAR = (int'$';
     public static final int PLUS = (int'+';
     public static final int POUND = (int'#';
     public static final int MINUS = (int'-';
     public static final int DOUBLEQUOTE = (int'\"';
     public static final int TILDE = (int'~';
     public static final int BACK_QUOTE = (int'`';
     public static final int NULL = (int'\0';
     public static final int EQUALS = (int'=';
     public static final int SEMICOLON = (int';';
     public static final int SLASH = (int'/';
     public static final int L_SQUARE_BRACKET = (int'[';
     public static final int R_SQUARE_BRACKET = (int']';
     public static final int R_CURLY = (int'}';
     public static final int L_CURLY = (int'{';
     public static final int HAT = (int'^';
     public static final int BAR = (int'|';
     public static final int DOT = (int'.';
     public static final int EXCLAMATION = (int'!';
     public static final int LPAREN = (int'(';
     public static final int RPAREN = (int')';
     public static final int GREATER_THAN = (int'>';
     public static final int LESS_THAN = (int'<';
     public static final int PERCENT = (int'%';
     public static final int QUESTION = (int'?';
     public static final int AND = (int'&';
     public static final int UNDERSCORE = (int'_';
 
     protected static final Hashtable globalSymbolTable;
     protected static final Hashtable lexerTables;
     protected Hashtable currentLexer;
     protected String currentLexerName;
     protected Token currentMatch;
 
     static {
          = new Hashtable();
          = new Hashtable();
     }
 
     protected void addKeyword(String nameint value) {
         // System.out.println("addKeyword " + name + " value = " + value);
         // new Exception().printStackTrace();
         Integer val = Integer.valueOf(value);
        .put(nameval);
        if (!.containsKey(val))
            .put(valname);
    }
    public String lookupToken(int value) {
        if (value > ) {
            return (String.get(Integer.valueOf(value));
        } else {
            Character ch = Character.valueOf((charvalue);
            return ch.toString();
        }
    }
    protected Hashtable addLexer(String lexerName) {
         = (Hashtable.get(lexerName);
        if ( == null) {
             = new Hashtable();
            .put(lexerName);
        }
        return ;
    }
    //public abstract void selectLexer(String lexerName);
    public void selectLexer(String lexerName) {
        this. = lexerName;
    }
    protected LexerCore() {
        this. = new Hashtable();
        this. = "charLexer";
    }

    
Initialize the lexer with a buffer.
    public LexerCore(String lexerNameString buffer) {
        super(buffer);
        this. = lexerName;
    }

    
Peek the next id but dont move the buffer pointer forward.
    public String peekNextId() {
        int oldPtr = ;
        String retval = ttoken();
         = ;
         = oldPtr;
        return retval;
    }

    
Get the next id.
    public String getNextId() {
        return ttoken();
    }
    // call this after you call match
    public Token getNextToken() {
        return this.;
    }

    
Look ahead for one token.
    public Token peekNextToken() throws ParseException {
        return (TokenpeekNextToken(1)[0];
    }
    public Token[] peekNextToken(int ntokensthrows ParseException {
        int old = ;
        Token[] retval = new Token[ntokens];
        for (int i = 0; i < ntokensi++) {
            Token tok = new Token();
            if (startsId()) {
                String id = ttoken();
                tok.tokenValue = id;
                String idUppercase = id.toUpperCase();
                if (.containsKey(idUppercase)) {
                    Integer type = (Integer.get(idUppercase);
                    tok.tokenType = type.intValue();
                } else
                    tok.tokenType = ;
            } else {
                char nextChar = getNextChar();
                tok.tokenValue = String.valueOf(nextChar);
                if (isAlpha(nextChar)) {
                    tok.tokenType = ;
                } else if (isDigit(nextChar)) {
                    tok.tokenType = ;
                } else
                    tok.tokenType = (intnextChar;
            }
            retval[i] = tok;
        }
         = ;
         = old;
        return retval;
    }

    
Match the given token or throw an exception if no such token can be matched.
    public Token match(int tokthrows ParseException {
        if (.) {
            Debug.println("match " + tok);
        }
        if (tok >  && tok < ) {
            if (tok == ) {
                // Generic ID sought.
                if (!startsId())
                    throw new ParseException( + "\nID expected");
                String id = getNextId();
                this. = new Token();
                this.. = id;
                this.. = ;
            } else if (tok == ) {
                if (!startsSafeToken())
                    throw new ParseException( + "\nID expected");
                String id = ttokenSafe();
                this. = new Token();
                this.. = id;
                this.. = ;
            } else {
                String nexttok = getNextId();
                Integer cur = (Integer.get(nexttok.toUpperCase());
                if (cur == null || cur.intValue() != tok)
                    throw new ParseException(
                         + "\nUnexpected Token : " + nexttok,
                        );
                this. = new Token();
                this.. = nexttok;
                this.. = tok;
            }
        } else if (tok > ) {
            // Character classes.
            char next = lookAhead(0);
            if (tok == ) {
                if (!isDigit(next))
                    throw new ParseException( + "\nExpecting DIGIT");
                this. = new Token();
                this.. =
                    String.valueOf(next);
                this.. = tok;
                consume(1);
            } else if (tok == ) {
                if (!isAlpha(next))
                    throw new ParseException( + "\nExpecting ALPHA");
                this. = new Token();
                this.. =
                    String.valueOf(next);
                this.. = tok;
                consume(1);
            }
        } else {
            // This is a direct character spec.
            char ch = (chartok;
            char next = lookAhead(0);
            if (next == ch) {
                /*this.currentMatch = new Token();
                this.currentMatch.tokenValue =
                    String.valueOf(ch);
                this.currentMatch.tokenType = tok;*/
                consume(1);
            } else
                throw new ParseException(
                     + "\nExpecting  >>>" + ch + "<<< got >>>"
                    + next + "<<<");
        }
        return this.;
    }
    public void SPorHT() {
        try {
            char c = lookAhead(0);
            while (c == ' ' || c == '\t') {
                consume(1);
                c = lookAhead(0);
            }
        } catch (ParseException ex) {
            // Ignore
        }
    }

    
JvB: utility function added to validate tokens

Parameters:
c - character to check
Returns:
true iff character c is a valid token character as per RFC3261
See also:
RFC3261 section 25.1: token = 1*(alphanum / "-" / "." / "!" / "%" / "*" / "_" / "+" / "`" / "'" / "~" )
    public static final boolean isTokenCharchar c ) {
        if ( isAlphaDigit(c) ) return true;
        else switch (c)
        {
            case '-':
            case '.':
            case '!':
            case '%':
            case '*':
            case '_':
            case '+':
            case '`':
            case '\'':
            case '~':
                return true;
            default:
                return false;
        }
    }
    public boolean startsId() {
        try {
            char nextChar = lookAhead(0);
            return isTokenChar(nextChar);
        } catch (ParseException ex) {
            return false;
        }
    }
    public boolean startsSafeToken() {
        try {
            char nextChar = lookAhead(0);
            if (isAlphaDigit(nextChar)) {
                return true;
            }
            else {
                switch (nextChar) {
                    case '_':
                    case '+':
                    case '-':
                    case '!':
                    case '`':
                    case '\'':
                    case '.':
                    case '/':
                    case '}':
                    case '{':
                    case ']':
                    case '[':
                    case '^':
                    case '|':
                    case '~':
                    case '%'// bug fix by Bruno Konik, JvB copied here
                    case '#':
                    case '@':
                    case '$':
                    case ':':
                    case ';':
                    case '?':
                    case '\"':
                    case '*':
                    case '='// Issue 155 on java.net
                        return true;
                    default:
                        return false;
                }
            }
        } catch (ParseException ex) {
            return false;
        }
    }
    public String ttoken() {
        int startIdx = ;
        try {
            while (hasMoreChars()) {
                char nextChar = lookAhead(0);
                if ( isTokenChar(nextChar) ) {
                    consume(1);
                } else {
                    break;
                }
            }
            return .substring(startIdx);
        } catch (ParseException ex) {
            return null;
        }
    }
    /* JvB: unreferenced
    public String ttokenAllowSpace() {
        int startIdx = ptr;
        try {
            while (hasMoreChars()) {
                char nextChar = lookAhead(0);
                if (isAlphaDigit(nextChar)) {
                    consume(1);
                }
                else {
                    boolean isValidChar = false;
                    switch (nextChar) {
                        case '_':
                        case '+':
                        case '-':
                        case '!':
                        case '`':
                        case '\'':
                        case '~':
                        case '%': // bug fix by Bruno Konik, JvB copied here
                        case '.':
                        case ' ':
                        case '\t':
                        case '*':
                            isValidChar = true;
                    }
                    if (isValidChar) {
                        consume(1);
                    }
                    else {
                        break;
                    }
                }
            }
            return buffer.substring(startIdx, ptr);
        } catch (ParseException ex) {
            return null;
        }
    }*/
    public String ttokenSafe() {
        int startIdx = ;
        try {
            while (hasMoreChars()) {
                char nextChar = lookAhead(0);
                if (isAlphaDigit(nextChar)) {
                    consume(1);
                }
                else {
                    boolean isValidChar = false;
                    switch (nextChar) {
                        case '_':
                        case '+':
                        case '-':
                        case '!':
                        case '`':
                        case '\'':
                        case '.':
                        case '/':
                        case '}':
                        case '{':
                        case ']':
                        case '[':
                        case '^':
                        case '|':
                        case '~':
                        case '%'// bug fix by Bruno Konik, JvB copied here
                        case '#':
                        case '@':
                        case '$':
                        case ':':
                        case ';':
                        case '?':
                        case '\"':
                        case '*':
                            isValidChar = true;
                    }
                    if (isValidChar) {
                        consume(1);
                    }
                    else {
                        break;
                    }
                }
            }
            return .substring(startIdx);
        } catch (ParseException ex) {
            return null;
        }
    }
    static final char ALPHA_VALID_CHARS = .;
    static final char DIGIT_VALID_CHARS = . - 1;
    static final char ALPHADIGIT_VALID_CHARS = . - 2;
    public void consumeValidChars(char[] validChars) {
        int validCharsLength = validChars.length;
        try {
            while (hasMoreChars()) {
                char nextChar = lookAhead(0);
                boolean isValid = false;
                for (int i = 0; i < validCharsLengthi++) {
                    char validChar = validChars[i];
                    switch(validChar) {
                        case :
                            isValid = isAlpha(nextChar);
                            break;
                        case :
                            isValid = isDigit(nextChar);
                            break;
                        case :
                            isValid = isAlphaDigit(nextChar);
                            break;
                        default:
                            isValid = nextChar == validChar;
                    }
                    if (isValid) {
                        break;
                    }
                }
                if (isValid) {
                    consume(1);
                }
                else {
                    break;
                }
            }
        } catch (ParseException ex) {
        }
    }

    
Parse a comment string cursor is at a ". Leave cursor at closing "

Returns:
the substring containing the quoted string excluding the closing quote.
    public String quotedString() throws ParseException {
        int startIdx =  + 1;
        if (lookAhead(0) != '\"')
            return null;
        consume(1);
        while (true) {
            char next = getNextChar();
            if (next == '\"') {
                // Got to the terminating quote.
                break;
            } else if (next == '\0') {
                throw new ParseException(
                    this. + " :unexpected EOL",
                    this.);
            } else if (next == '\\') {
                consume(1);
            }
        }
        return .substring(startIdx - 1);
    }

    
Parse a comment string cursor is at a "(". Leave cursor at )

Returns:
the substring containing the comment excluding the closing brace.
    public String comment() throws ParseException {
        StringBuffer retval = new StringBuffer();
        if (lookAhead(0) != '(')
            return null;
        consume(1);
        while (true) {
            char next = getNextChar();
            if (next == ')') {
                break;
            } else if (next == '\0') {
                throw new ParseException(
                    this. + " :unexpected EOL",
                    this.);
            } else if (next == '\\') {
                retval.append(next);
                next = getNextChar();
                if (next == '\0')
                    throw new ParseException(
                        this. + " : unexpected EOL",
                        this.);
                retval.append(next);
            } else {
                retval.append(next);
            }
        }
        return retval.toString();
    }

    
Return a substring containing no semicolons.

Returns:
a substring containing no semicolons.
    public String byteStringNoSemicolon() {
        StringBuffer retval = new StringBuffer();
        try {
            while (true) {
                char next = lookAhead(0);
                // bug fix from Ben Evans.
                if (next == '\0' || next == '\n' || next == ';' || next == ',' ) {
                    break;
                } else {
                    consume(1);
                    retval.append(next);
                }
            }
        } catch (ParseException ex) {
            return retval.toString();
        }
        return retval.toString();
    }

    
Scan until you see a slash or an EOL.

Returns:
substring containing no slash.
    public String byteStringNoSlash() {
        StringBuffer retval = new StringBuffer();
        try {
            while (true) {
                char next = lookAhead(0);
                // bug fix from Ben Evans.
                if (next == '\0' || next == '\n' || next == '/'  ) {
                    break;
                } else {
                    consume(1);
                    retval.append(next);
                }
            }
        } catch (ParseException ex) {
            return retval.toString();
        }
        return retval.toString();
    }

    
Return a substring containing no commas

Returns:
a substring containing no commas.
    public String byteStringNoComma() {
        StringBuffer retval = new StringBuffer();
        try {
            while (true) {
                char next = lookAhead(0);
                if (next == '\n' || next == ',') {
                    break;
                } else {
                    consume(1);
                    retval.append(next);
                }
            }
        } catch (ParseException ex) {
        }
        return retval.toString();
    }
    public static String charAsString(char ch) {
        return String.valueOf(ch);
    }

    
Lookahead in the inputBuffer for n chars and return as a string. Do not consume the input.
    public String charAsString(int nchars) {
        return .substring( + nchars);
    }

    
Get and consume the next number.

Returns:
a substring corresponding to a number (i.e. sequence of digits).
    public String number() throws ParseException {
        int startIdx = ;
        try {
            if (!isDigit(lookAhead(0))) {
                throw new ParseException(
                     + ": Unexpected token at " + lookAhead(0),
                    );
            }
            consume(1);
            while (true) {
                char next = lookAhead(0);
                if (isDigit(next)) {
                    consume(1);
                } else
                    break;
            }
            return .substring(startIdx);
        } catch (ParseException ex) {
            return .substring(startIdx);
        }
    }

    
Mark the position for backtracking.

Returns:
the current location of the pointer.
    public int markInputPosition() {
        return ;
    }

    
Rewind the input ptr to the marked position.

Parameters:
position - the position to rewind the parser to.
    public void rewindInputPosition(int position) {
        this. = position;
    }

    
Get the rest of the String

Returns:
rest of the buffer.
    public String getRest() {
        if ( >= .length())
            return null;
        else
            return .substring();
    }

    
Get the sub-String until the character is encountered

Parameters:
c the character to match
Returns:
the substring that matches.
    public String getString(char cthrows ParseException {
        StringBuffer retval = new StringBuffer();
        while (true) {
            char next = lookAhead(0);
            //System.out.println(" next = [" + next + ']' + "ptr = " + ptr);
            //System.out.println(next == '\0');
            if (next == '\0') {
                throw new ParseException(
                    this. + "unexpected EOL",
                    this.);
            } else if (next == c) {
                consume(1);
                break;
            } else if (next == '\\') {
                consume(1);
                char nextchar = lookAhead(0);
                if (nextchar == '\0') {
                    throw new ParseException(
                        this. + "unexpected EOL",
                        this.);
                } else {
                    consume(1);
                    retval.append(nextchar);
                }
            } else {
                consume(1);
                retval.append(next);
            }
        }
        return retval.toString();
    }

    
Get the read pointer.
    public int getPtr() {
        return this.;
    }

    
Get the buffer.
    public String getBuffer() {
        return this.;
    }

    
Create a parse exception.
        return new ParseException(this.this.);
    }
New to GrepCode? Check out our FAQ X