Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
Copyright (c) 2010 itemis AG (http://www.itemis.eu) and others. All rights reserved. This program and the accompanying materials are made available under the terms of the Eclipse Public License v1.0 which accompanies this distribution, and is available at http://www.eclipse.org/legal/epl-v10.html /
  
  package org.eclipse.xtext.ui.editor.syntaxcoloring;
  
 
 import  org.eclipse.jface.text.IDocument;
 import  org.eclipse.jface.text.IRegion;
 import  org.eclipse.jface.text.rules.IToken;
 import  org.eclipse.jface.text.rules.Token;
 
TokenScanner implementation based on XtextDocument.getTokens()

Author(s):
Sven Efftinge - Initial contribution and API
Sebastian Zarnekow - Use reentrant iterator to reduce runtime complexity
 
 public class TokenScanner extends AbstractTokenScanner {
 

A reentrant iterator allows to reuse and filter the elements another iterator and may update its filter criteria continuously.

It is assumed that setRange(IDocument, int, int) will be called with ascending order most of the time thus a sorted iterator may be reused and traversed further after the range has been updated.

 
 		
 		private ILexerTokenRegion current = null;
 		private boolean computedHasNext = false;
 		private boolean hasNext = false;
 		private Predicate<? super IRegion> overlapFilter;
 		private int regionOffset;
 		
 		public boolean hasNext() {
 			// we already know whether we have a next element
 				return ;
 			
 			 = true;
 			while (.hasNext()) {
 					 = true;
 					break;
 				} else {
 					// filter until we leave the valid region - no need to iterate
 					// the delegate until it's done
 					if (.getOffset() > ) {
 						 = false;
 						break;
 					}
 				}
 			} 
 			return ;
 		}
 
 		public ILexerTokenRegion next() {
 			if (!) {
 				// compute hasNext on demand to make sure the internal state is valid
 				hasNext();
 			}
 			if (!) {
 				throw new NoSuchElementException("You should check for #hasNext prior to calling #next");
 			}
 			 = null;
 			 = false;
 			 = false;
 			return result;
 		}
 		
 		public void setRange(IDocument documentfinal int offsetfinal int length) {
 			this. = offset;
 			 = Regions.overlaps(offsetlength);
 			Iterable<ILexerTokenRegionnewIterable = getTokens(document);
 			if ( != null) {
 				if (!.equals(newIterable)) {
 					 = null;
				}
			}
			if ( == null) {
				 = newIterable.iterator();
				 = newIterable;
				 = false;
				 = false;
else {
				if (.getOffset() <= offset && .getOffset() + .getLength() > offset) {
					// offset is inside of current - return current as next
					 = true;
else {
					// restart - use a new delegate
					 = false;
					 = false;
					 = newIterable.iterator();
					 = newIterable;
				}
			}
		}
	}
		this. = tokenIdMapper;
	}
	public void setRange(IDocument documentfinal int offsetfinal int length) {
		.setRange(documentoffsetlength);
	}
	protected Iterable<ILexerTokenRegiongetTokens(IDocument document) {
		XtextDocument doc = (XtextDocumentdocument;
		return doc.getTokens();
	}
	public IToken nextToken() {
			return Token.EOF;
	}
	protected IToken createToken(ILexerTokenRegion currentToken) {
		Token token = new Token(getAttribute(id));
		return token;
	}
	public int getTokenOffset() {
	}
	public int getTokenLength() {
	}
	}
		return ;
	}
	protected void setCurrentToken(ILexerTokenRegion currentToken) {
		this. = currentToken;
	}
	}
New to GrepCode? Check out our FAQ X