Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
Copyright (c) 2010 itemis AG ( and others. All rights reserved. This program and the accompanying materials are made available under the terms of the Eclipse Public License v1.0 which accompanies this distribution, and is available at /
  package org.eclipse.xtext.ui.editor.model;
 import  org.eclipse.jface.text.IDocument;
 import  org.eclipse.jface.text.IRegion;
 import  org.eclipse.jface.text.Region;
 import  org.eclipse.jface.text.TextUtilities;
 import  org.eclipse.jface.text.rules.IPartitionTokenScanner;
 import  org.eclipse.jface.text.rules.IToken;
 import  org.eclipse.jface.text.rules.Token;

Sven Efftinge - Initial contribution and API
Sebastian Zarnekow
 public class PartitionTokenScanner implements IPartitionTokenScanner {

Iterator that is aware of the fact that the tokens are ordered. As soon as the last token in the given overlapRegion was returned, no other tokens are requested from base.

 	protected static class RangedIterator extends AbstractIterator<ILexerTokenRegion> {
 		private final Iterator<ILexerTokenRegiondelegate;
 		private final IRegion overlapRegion;
 		protected RangedIterator(Iterable<ILexerTokenRegionbase, IRegion overlapRegion) {
 			this. = overlapRegion;
 			this. = base.iterator();
 		protected ILexerTokenRegion computeNext() {
 			while(.hasNext()) {
 				ILexerTokenRegion candidate = .next();
 				if (.getOffset() + .getLength() < candidate.getOffset())
 					return endOfData();
 				if (TextUtilities.overlaps(candidate))
 					return candidate;
 			return endOfData();
 	private int currentPartitionOffset = 0;
 	private ILexerTokenRegion nextToken = null;
 	private int currentPartitionLength;
 	public void setMapper(ITokenTypeToPartitionTypeMapper mapper) {
 		this. = mapper;
 	public void setRange(IDocument documentint offsetint length) {
 		setPartialRange(documentoffsetlengthnull, 0);
 	public void setPartialRange(IDocument documentint offsetint lengthString contentTypeint partitionOffset) {
 		int overlapOffset = partitionOffset;
 		int overlapLength = offset + length - partitionOffset;
 		 = new RangedIterator(getTokens(document), new Region(overlapOffsetoverlapLength));
 		this. = overlapOffset;
 		this. = .hasNext()?.next():null;
 	protected Iterable<ILexerTokenRegiongetTokens(IDocument document) {
 		return ((XtextDocumentdocument).getTokens();
 	public IToken nextToken() {
 		if (==null) {
 			return Token.EOF;
 		while (.hasNext()) {
			if (!partitionOfNext.equals(tokenPartition) || !shouldMergePartitions(tokenPartition)) {
				return new Token(tokenPartition);
		if ( != null)
		 = null;
		return new Token(tokenPartition);

	protected boolean shouldMergePartitions(String contentType) {
		return IDocument.DEFAULT_CONTENT_TYPE.equals(contentType);
	public int getTokenOffset() {
	public int getTokenLength() {
New to GrepCode? Check out our FAQ X