Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
   package com.fasterxml.jackson.dataformat.smile.async;
   
   import java.io.*;
   //import java.math.BigDecimal;
   //import java.math.BigInteger;
   import java.util.Arrays;
   
  
  import static com.fasterxml.jackson.dataformat.smile.SmileConstants.BYTE_MARKER_END_OF_STRING;
  
  public class NonBlockingParserImpl
      extends ParserBase
  {
      private final static byte[] NO_BYTES = new byte[0];
      private final static int[] NO_INTS = new int[0];
      private final static String[] NO_STRINGS = new String[0];
  
      /*
      /**********************************************************************
      /* State constants
      /**********************************************************************
       */
  
      // // // Initial bootstrapping states:
      
    
State right after parser has been constructed: waiting for header (which may or may not be mandatory).
  
      protected final static int STATE_INITIAL = 0;


    
State for recognized header marker, either in-feed or initial.
  
      protected final static int STATE_HEADER = 1;
    
    
State in which we are right after decoding a full token.
  
      protected final static int STATE_TOKEN_COMPLETE = 2;
      
      // // // States for decoding numbers:
      protected final static int STATE_NUMBER_INT = 10;
      protected final static int STATE_NUMBER_LONG = 11;
      protected final static int STATE_NUMBER_BIGINT = 12;
      protected final static int STATE_NUMBER_FLOAT = 13;
      protected final static int STATE_NUMBER_DOUBLE = 14;
      protected final static int STATE_NUMBER_BIGDEC = 15;
  
      protected final static int STATE_SHORT_ASCII = 20;
      protected final static int STATE_SHORT_UNICODE = 21;
      protected final static int STATE_LONG_ASCII = 22;
      protected final static int STATE_LONG_UNICODE = 23;
      protected final static int STATE_LONG_SHARED = 24;
      protected final static int STATE_RAW_BINARY = 25;
      protected final static int STATE_QUOTED_BINARY = 26;
      
      /*
      /**********************************************************************
      /* Configuration
      /**********************************************************************
       */
    
    
Codec used for data binding when (if) requested.
  
      protected ObjectCodec _objectCodec;

    
Flag that indicates whether content can legally have raw (unquoted) binary data. Since this information is included both in header and in actual binary data blocks there is redundancy, and we want to ensure settings are compliant. Using application may also want to know this setting in case it does some direct (random) access.
  
      protected boolean _mayContainRawBinary;
  
      protected final boolean _cfgRequireHeader;

    
Helper object used for low-level recycling of Smile-generator specific buffers.
  
      final protected SmileBufferRecycler<String_smileBufferRecycler;
      
      /*
      /**********************************************************************
      /* Input source config
      /**********************************************************************
       */
    
    
This buffer is actually provided via NonBlockingInputFeeder
 
     protected byte[] _inputBuffer = ;
    
    
In addition to current buffer pointer, and end pointer, we will also need to know number of bytes originally contained. This is needed to correctly update location information when the block has been completed.
 
     protected int _origBufferLen;
 
     // And from ParserBase:
 //    protected int _inputPtr;
 //    protected int _inputEnd;
     
     /*
     /**********************************************************************
     /* Additional parsing state
     /**********************************************************************
      */

    
Current main decoding state
 
     protected int _state;

    
Addition indicator within state; contextually relevant for just that state
 
     protected int _substate;
    
    
Flag that indicates that the current token has not yet been fully processed, and needs to be finished for some access (or skipped to obtain the next token)
 
     protected boolean _tokenIncomplete;

    
Specific flag that is set when we encountered a 32-bit floating point value; needed since numeric super classes do not track distinction between float and double, but Smile format does, and we want to retain that separation.
 
     protected boolean _got32BitFloat;

    
For 32-bit values, we may use this for combining values
 
     protected int _pendingInt;

    
For 64-bit values, we may use this for combining values
 
     protected long _pendingLong;
    
    
Flag that is sent when calling application indicates that there will be no more input to parse.
 
     protected boolean _endOfInput = false;
     
     /*
     /**********************************************************************
     /* Symbol handling, decoding
     /**********************************************************************
      */

    
Symbol table that contains field names encountered so far
 
     final protected BytesToNameCanonicalizer _symbols;
    
    
Temporary buffer used for name parsing.
 
     protected int[] _quadBuffer = ;
 
     /*
     /**********************************************************************
     /* Name/entity parsing state
     /**********************************************************************
      */

    
Number of complete quads parsed for current name (quads themselves are stored in _quadBuffer).
 
     protected int _quadCount;

    
Bytes parsed for the current, incomplete, quad
 
     protected int _currQuad;

    
Number of bytes pending/buffered, stored in _currQuad
 
     protected int _currQuadBytes = 0;
     
    
Array of recently seen field names, which may be back referenced by later fields. Defaults set to enable handling even if no header found.
 
     protected String[] _seenNames = ;
 
     protected int _seenNameCount = 0;

    
Array of recently seen field names, which may be back referenced by later fields Defaults set to disable handling if no header found.
 
     protected String[] _seenStringValues = null;
 
     protected int _seenStringValueCount = -1;
     
     /*
     /**********************************************************************
     /* Thread-local recycling
     /**********************************************************************
      */
    
    
ThreadLocal contains a java.lang.ref.SoftReference to a buffer recycler used to provide a low-cost buffer recycling for Smile-specific buffers.
 
         = new ThreadLocal<SoftReference<SmileBufferRecycler<String>>>();
     
     /*
     /**********************************************************************
     /* Life-cycle
     /**********************************************************************
      */
 
     public NonBlockingParserImpl(IOContext ctxtint parserFeaturesint smileFeatures,
             ObjectCodec codecBytesToNameCanonicalizer sym)
     {
         super(ctxtparserFeatures);        
          = codec;
          = sym;
         
          = -1;
          = -1;
 
          = .;
          = ;
          = true;
 
          = (smileFeatures & ...getMask()) != 0;
     }
 
     @Override
     public ObjectCodec getCodec() {
         return ;
     }
 
     @Override
     public void setCodec(ObjectCodec c) {
          = c;
     }

    
Helper method called when it looks like input might contain the signature; and it is necessary to detect and handle signature to get configuration information it might have.

Returns:
True if valid signature was found and handled; false if not
 
     protected boolean handleSignature(boolean consumeFirstByteboolean throwException)
         throws IOExceptionJsonParseException
     {
         if (consumeFirstByte) {
             ++;
         }
         if ( >= ) {
             loadMoreGuaranteed();
         }
         if ([] != .) {
             if (throwException) {
             	_reportError("Malformed content: signature not valid, starts with 0x3a but followed by 0x"
             			+Integer.toHexString([])+", not 0x29");
             }
             return false;
         }
         if (++ >= ) {
             loadMoreGuaranteed();        	
         }
         if ([] != .) {
             if (throwException) {
             	_reportError("Malformed content: signature not valid, starts with 0x3a, 0x29, but followed by 0x"
             			+Integer.toHexString([])+", not 0xA");
             }
             return false;
         }
     	// Good enough; just need version info from 4th byte...
         if (++ >= ) {
             loadMoreGuaranteed();        	
         }
         int ch = [++];
         int versionBits = (ch >> 4) & 0x0F;
         // but failure with version number is fatal, can not ignore
         if (versionBits != .) {
             _reportError("Header version number bits (0x"+Integer.toHexString(versionBits)+") indicate unrecognized version; only 0x0 handled by parser");
         }
 
         // can avoid tracking names, if explicitly disabled
         if ((ch & .) == 0) {
              = null;
              = -1;
         }
         // conversely, shared string values must be explicitly enabled
         if ((ch & .) != 0) {
              = ;
              = 0;
         }
          = ((ch & .) != 0);
         return true;
     }
 
     protected final static SmileBufferRecycler<String_smileBufferRecycler()
     {
         SmileBufferRecycler<Stringbr = (ref == null) ? null : ref.get();
 
         if (br == null) {
             br = new SmileBufferRecycler<String>();
             .set(new SoftReference<SmileBufferRecycler<String>>(br));
         }
         return br;
     }
     
     /*                                                                                       
     /**********************************************************************
     /* Versioned                                                                             
     /**********************************************************************
      */
 
     @Override
     public Version version() {
         return ..version();
     }
     
     /*
     /**********************************************************************
     /* Former StreamBasedParserBase methods
     /**********************************************************************
      */
 
     @Override
     public int releaseBuffered(OutputStream outthrows IOException
     {
         int count =  - ;
         if (count < 1) {
             return 0;
         }
         // let's just advance ptr to end
         int origPtr = ;
         out.write(origPtrcount);
         return count;
     }
     
     @Override
     public Object getInputSource() {
         // since input is "pushed", to traditional source...
         return null;
     }

    
Overridden since we do not really have character-based locations, but we do have byte offset to specify.
 
     @Override
     public JsonLocation getTokenLocation()
     {
         // token location is correctly managed...
         return new JsonLocation(.getSourceReference(),
                 // bytes
                 -1, -1, (int); // char offset, line, column
     }   

    
Overridden since we do not really have character-based locations, but we do have byte offset to specify.
 
     @Override
     public JsonLocation getCurrentLocation()
     {
         final long offset =  + ;
         return new JsonLocation(.getSourceReference(),
                 offset// bytes
                 -1, -1, (intoffset); // char offset, line, column
     }
 
     /*
     /**********************************************************************
     /* Low-level reading, other
     /**********************************************************************
      */
     
     @Override
     protected final boolean loadMore() throws IOException {
         _throwInternal();
         return false;
     }
    
    
Helper method that will try to load at least specified number bytes in input buffer, possible moving existing data around if necessary
 
     protected final boolean _loadToHaveAtLeast(int minAvailablethrows IOException
     {
         _throwInternal();
         return false;
     }
     
     @Override
     protected void _closeInput() throws IOException {
         // nothing to do here
     }
     
     /*
     /**********************************************************************
     /* Overridden methods
     /**********************************************************************
      */
 
     @Override
     protected void _finishString() throws IOExceptionJsonParseException {
         // should never be called; but must be defined for superclass
         _throwInternal();
     }
 
     @Override
     public void close() throws IOException
     {
         super.close();
         // Merge found symbols, if any:
         .release();
     }
 
     @Override
     public boolean hasTextCharacters()
     {
         if ( == .) {
             // yes; is or can be made available efficiently as char[]
             return .hasTextAsCharacters();
         }
         if ( == .) {
             // not necessarily; possible but:
             return ;
         }
         // other types, no benefit from accessing as char[]
         return false;
     }

    
Method called to release internal buffers owned by the base reader. This may be called along with _closeInput() (for example, when explicitly closing this reader instance), or separately (if need be).
 
     @Override
     protected void _releaseBuffers() throws IOException
     {
         super._releaseBuffers();
         {
             String[] nameBuf = ;
             if (nameBuf != null && nameBuf.length > 0) {
                  = null;
                 /* 28-Jun-2011, tatu: With 1.9, caller needs to clear the buffer;
                  *   but we only need to clear up to count as it is not a hash area
                  */
                 if ( > 0) {
                     Arrays.fill(nameBuf, 0, null);
                 }
                 .releaseSeenNamesBuffer(nameBuf);
             }
         }
         {
             String[] valueBuf = ;
             if (valueBuf != null && valueBuf.length > 0) {
                  = null;
                 /* 28-Jun-2011, tatu: With 1.9, caller needs to clear the buffer;
                  *   but we only need to clear up to count as it is not a hash area
                  */
                 if ( > 0) {
                     Arrays.fill(valueBuf, 0, null);
                 }
                 .releaseSeenStringValuesBuffer(valueBuf);
             }
         }
     }
     
     /*
     /**********************************************************************
     /* Extended API
     /**********************************************************************
      */
 
     public boolean mayContainRawBinary() {
         return ;
     }
     
     /*
     /**********************************************************************
     /* JsonParser impl
     /**********************************************************************
      */
 
     @Override
     public JsonToken nextToken() throws IOExceptionJsonParseException
     {
          = ;
         // have we already decoded part of event? If so, continue...
         if () {
             // we might be able to optimize by separate skipping, but for now:
             return _finishToken();
         }
         // also: clear any data retained so far
          = null;
         // Two main modes: values, and field names.
         if (.inObject() &&  != .) {
             return ( = _handleFieldName());
         }
         if ( >= ) {
             return .;
         }
         int ch = [++];
         switch ((ch >> 5) & 0x7) {
         case 0: // short shared string value reference
             if (ch == 0) { // important: this is invalid, don't accept
                 _reportError("Invalid token byte 0x00");
             }
             return _handleSharedString(ch-1);
 
         case 1: // simple literals, numbers
             {
                  = 0;
                 switch (ch & 0x1F) {
                 case 0x00:
                     .resetWithEmpty();
                     return ( = .);
                 case 0x01:
                     return ( = .);
                 case 0x02: // false
                     return ( = .);
                 case 0x03: // 0x03 == true
                     return ( = .);
                 case 0x04:
                      = ;
                     return _nextInt(0, 0);
                 case 0x05:
                      = 0;
                      = ;
                     return _nextLong(0, 0L);
                 case 0x06:
                      = ;
                     return _nextBigInt(0);
                 case 0x07: // illegal
                     break;
                 case 0x08:
                      = 0;
                      = ;
                      = true;
                     return _nextFloat(0, 0);
                 case 0x09:
                      = 0L;
                      = ;
                      = false;
                     return _nextDouble(0, 0L);
                 case 0x0A:
                      = ;
                     return _nextBigDecimal(0);
                 case 0x0B: // illegal
                     break;
                 case 0x1A: // == 0x3A == ':' -> possibly header signature for next chunk?
                     if (!_handleHeader(0)) {
                         return .;
                     }
                     //if (handleSignature(false, false)) {
                     /* Ok, now; end-marker and header both imply doc boundary and a
                      * 'null token'; but if both are seen, they are collapsed.
                      * We can check this by looking at current token; if it's null,
                      * need to get non-null token
                      */
                     if ( == null) {
                         return nextToken();
                     }
                     return ( = null);
                 }
             }
             // and everything else is reserved, for now
             break;
         case 2: // tiny ASCII
             // fall through            
         case 3: // short ASCII
             // fall through
             return _nextShortAscii(0);
 
         case 4: // tiny Unicode
             // fall through
         case 5: // short Unicode
             // No need to decode, unless we have to keep track of back-references (for shared string values)
              = .;
             if ( >= 0) { // shared text values enabled
                 _addSeenStringValue();
             } else {
                  = true;
             }
             return _nextShortUnicode(0);
 
         case 6: // small integers; zigzag encoded
              = SmileUtil.zigzagDecode(ch & 0x1F);
              = ;
             return ( = .);
         case 7: // binary/long-text/long-shared/start-end-markers
             switch (ch & 0x1F) {
             case 0x00: // long variable length ASCII
                 return _nextLongAscii(0);
             case 0x04: // long variable length unicode
                 return _nextLongUnicode(0);
             case 0x08: // binary, 7-bit
                 return _nextQuotedBinary(0);
             case 0x0C: // long shared string
             case 0x0D:
             case 0x0E:
             case 0x0F:
                 return _nextLongSharedString(0);
 //                return _handleSharedString(((ch & 0x3) << 8) + (_inputBuffer[_inputPtr++] & 0xFF));
             case 0x18: // START_ARRAY
                  = .createChildArrayContext(-1, -1);
                 return ( = .);
             case 0x19: // END_ARRAY
                 if (!.inArray()) {
                     _reportMismatchedEndMarker(']''}');
                 }
                  = .getParent();
                 return ( = .);
             case 0x1A: // START_OBJECT
                  = .createChildObjectContext(-1, -1);
                 return ( = .);
             case 0x1B: // not used in this mode; would be END_OBJECT
                 _reportError("Invalid type marker byte 0xFB in value mode (would be END_OBJECT in key mode)");
             case 0x1D: // binary, raw
                 // should we validate this is legal? (as per header)
                 return _nextRawBinary(0);
             case 0x1F: // 0xFF, end of content
                 return ( = null);
             }
             break;
         }
         // If we get this far, type byte is corrupt
         _reportError("Invalid type marker byte 0x"+Integer.toHexString(ch & 0xFF)+" for expected value token");
         return null;
     }
 
     private final JsonToken _handleSharedString(int index)
         throws IOExceptionJsonParseException
     {
         if (index >= ) {
             _reportInvalidSharedStringValue(index);
         }
         return ( = .);
     }
 
     private final void _addSeenStringValue()
         throws IOExceptionJsonParseException
     {
         _finishToken();
         if ( < .) {
             // !!! TODO: actually only store char[], first time around?
             return;
         }
         _expandSeenStringValues();
     }
     
     private final void _expandSeenStringValues()
     {
         String[] oldShared = ;
         int len = oldShared.length;
         String[] newShared;
         if (len == 0) {
             newShared = .allocSeenStringValuesBuffer();
             if (newShared == null) {
                 newShared = new String[.];
             }
         } else if (len == .) { // too many? Just flush...
            newShared = oldShared;
             = 0; // could also clear, but let's not yet bother
         } else {
             int newSize = (len == .) ? 256 : .;
             newShared = new String[newSize];
             System.arraycopy(oldShared, 0, newShared, 0, oldShared.length);
         }
          = newShared;
     }
 
     @Override
     public String getCurrentName() throws IOExceptionJsonParseException
     {
         return .getCurrentName();
     }
 
     @Override
     public NumberType getNumberType()
         throws IOExceptionJsonParseException
     {
     	if () {
     	    return .;
     	}
     	return super.getNumberType();
     }
 
     /*
     /**********************************************************************
     /* AsyncInputFeeder impl
     /**********************************************************************
      */
 
     public final boolean needMoreInput() {
         return ( >=) && !;
     }
 
     public void feedInput(byte[] bufint startint len)
         throws IOException
     {
         // Must not have remaining input
         if ( < ) {
             throw new IOException("Still have "+( - )+" undecoded bytes, should not call 'feedInput'");
         }
         // and shouldn't have been marked as end-of-input
         if () {
             throw new IOException("Already closed, can not feed more input");
         }
         // Time to update pointers first
          += ;
          -= ;
 
         // And then update buffer settings
          = buf;
          = start;
          = start+len;
          = len;
     }
 
     public void endOfInput() {
          = true;
     }
     
     /*
     /**********************************************************************
     /* NonBlockParser impl (except for NonBlockingInputFeeder)
     /**********************************************************************
      */
 
     {
         if (!) {
             return .;
         }
         switch () {
         case // the case if no input has yet been fed
             return .;
         case :
             return .;
         case :
         case :
             return .;
         case :
         case :
         case :
             return .;
         }
         throw new IllegalStateException("Internal error: unknown 'state', "+);
     }
     
     /*
     /**********************************************************************
     /* Internal methods: second-level parsing:
     /**********************************************************************
      */
 
     private final JsonToken _nextInt(int substateint value)
         throws IOExceptionJsonParseException
     {
         while ( < ) {
             int b = [++];
             if (b < 0) { // got it all; these are last 6 bits
                 value = (value << 6) | (b & 0x3F);
                  = SmileUtil.zigzagDecode(value);
                  = ;
                  = false;
                 return ( = .);
             }
             // can't get too big; 5 bytes is max
             if (++substate >= 5 ) {
                 _reportError("Corrupt input; 32-bit VInt extends beyond 5 data bytes");
             }
             value = (value << 7) | b;
         }
         // did not get it all; mark the state so we know where to return:
          = true;
          = substate;
          = value;
          = ;
         return ( = .);
     }
 
     private final JsonToken _nextLong(int substatelong valuethrows IOExceptionJsonParseException
     {
         while ( < ) {
             int b = [++];
             if (b < 0) { // got it all; these are last 6 bits
                 value = (value << 6) | (b & 0x3F);
                  = SmileUtil.zigzagDecode(value);
                  = ;
                  = false;
                 return ( = .);
             }
             // can't get too big; 10 bytes is max
             if (++substate >=  10) {
                 _reportError("Corrupt input; 64-bit VInt extends beyond 10 data bytes");
             }
             value = (value << 7) | b;
         }
         // did not get it all; mark the state so we know where to return:
          = true;
          = substate;
          = value;
          = ;
         return ( = .);
     }
 
     private final JsonToken _nextBigInt(int substatethrows IOExceptionJsonParseException
     {
         // !!! TBI
          = true;
          = substate;
 //        _pendingLong = value;
          = ;
         return ( = .);
     }
 
     /*
     private final boolean _finishBigInteger()
         throws IOException, JsonParseException
     {
         byte[] raw = _read7BitBinaryWithLength();
         if (raw == null) {
             return false;
         }
         _numberBigInt = new BigInteger(raw);
         _numTypesValid = NR_BIGINT;
         return true;
     }
 */
     
     private final JsonToken _nextFloat(int substateint valuethrows IOExceptionJsonParseException
     {
         while ( < ) {
             int b = [++];
             value = (value << 7) + b;
             if (++substate == 5) { // done!
                  = (double) Float.intBitsToFloat(value);
                  = ;
                  = false;
                 return ( = .);
             }
         }
          = true;
          = substate;
          = value;
          = ;
         return ( = .);
     }
 
     private final JsonToken _nextDouble(int substatelong valuethrows IOExceptionJsonParseException
     {
         while ( < ) {
             int b = [++];
             value = (value << 7) + b;
             if (++substate == 10) { // done!
                  = Double.longBitsToDouble(value);
                  = ;
                  = false;
                 return ( = .);
             }
         }
          = true;
          = substate;
          = value;
          = ;
         return ( = .);
     }
 
     private final JsonToken _nextBigDecimal(int substatethrows IOExceptionJsonParseException
     {
         // !!! TBI
          = true;
          = substate;
 //        _pendingLong = value;
          = ;
         return ( = .);
     }
 /*
     private final void _finishBigDecimal()
         throws IOException, JsonParseException
     {
         int scale = SmileUtil.zigzagDecode(_readUnsignedVInt());
         byte[] raw = _read7BitBinaryWithLength();
         _numberBigDecimal = new BigDecimal(new BigInteger(raw), scale);
         _numTypesValid = NR_BIGDECIMAL;
     }
     
  */
 
     /*
     private final int _readUnsignedVInt()
         throws IOException, JsonParseException
     {
         int value = 0;
         while (true) {
             if (_inputPtr >= _inputEnd) {
                 loadMoreGuaranteed();
             }
             int i = _inputBuffer[_inputPtr++];
             if (i < 0) { // last byte
                 value = (value << 6) + (i & 0x3F);
                 return value;
             }
             value = (value << 7) + i;
         }
     }
     */
     
     private final boolean _handleHeader(int substatethrows IOExceptionJsonParseException
     {
         while ( < ) {
             byte b = [++];
             switch (substate) {
             case 0: // after first byte
                 if (b != .) {
                     _reportError("Malformed content: header signature not valid, starts with 0x3a but followed by 0x"
                             +Integer.toHexString([] & 0xFF)+", not 0x29");
                 }
                 break;
             case 1:
                 if (b != .) {
                     _reportError("Malformed content: signature not valid, starts with 0x3a, 0x29, but followed by 0x"
                             +Integer.toHexString([ & 0xFF])+", not 0x0A");
                 }
                 break;
             case 2: // ok, here be the version, config bits...
                 int versionBits = (b >> 4) & 0x0F;
                 // but failure with version number is fatal, can not ignore
                 if (versionBits != .) {
                     _reportError("Header version number bits (0x"+Integer.toHexString(versionBits)+") indicate unrecognized version; only 0x0 handled by parser");
                 }
 
                 // can avoid tracking names, if explicitly disabled
                 if ((b & .) == 0) {
                      = null;
                      = -1;
                 }
                 // conversely, shared string values must be explicitly enabled
                 if ((b & .) != 0) {
                      = ;
                      = 0;
                 }
                  = ((b & .) != 0);
                  = false;
                 return true;
             }
         }
          = true;
          = ;
          = substate;
         return false;
     }
 
     private final JsonToken _nextShortAscii(int substatethrows IOExceptionJsonParseException
     {
          = ;
          = true;
          = substate;
         return ( = .);
     }
 
    private final JsonToken _nextShortUnicode(int substatethrows IOExceptionJsonParseException
    {
         = ;
         = true;
         = substate;
        return ( = .);
    }
    
    /*
    protected final void _decodeShortAsciiValue(int len)
        throws IOException, JsonParseException
    {
        if ((_inputEnd - _inputPtr) < len) {
            _loadToHaveAtLeast(len);
        }
        // Note: we count on fact that buffer must have at least 'len' (<= 64) empty char slots
        final char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
        int outPtr = 0;
        final byte[] inBuf = _inputBuffer;
        int inPtr = _inputPtr;
        // meaning: regular tight loop is no slower, typically faster here:
        for (final int end = inPtr + len; inPtr < end; ++inPtr) {
            outBuf[outPtr++] = (char) inBuf[inPtr];            
        }
        
        _inputPtr = inPtr;
        _textBuffer.setCurrentLength(len);
    }
    protected final void _decodeShortUnicodeValue(int len)
        throws IOException, JsonParseException
    {
        if ((_inputEnd - _inputPtr) < len) {
            _loadToHaveAtLeast(len);
        }
        int outPtr = 0;
        char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
        int inPtr = _inputPtr;
        _inputPtr += len;
        final int[] codes = SmileConstants.sUtf8UnitLengths;
        final byte[] inputBuf = _inputBuffer;
        for (int end = inPtr + len; inPtr < end; ) {
            int i = inputBuf[inPtr++] & 0xFF;
            int code = codes[i];
            if (code != 0) {
                // trickiest one, need surrogate handling
                switch (code) {
                case 1:
                    i = ((i & 0x1F) << 6) | (inputBuf[inPtr++] & 0x3F);
                    break;
                case 2:
                    i = ((i & 0x0F) << 12)
                          | ((inputBuf[inPtr++] & 0x3F) << 6)
                          | (inputBuf[inPtr++] & 0x3F);
                    break;
                case 3:
                    i = ((i & 0x07) << 18)
                        | ((inputBuf[inPtr++] & 0x3F) << 12)
                        | ((inputBuf[inPtr++] & 0x3F) << 6)
                        | (inputBuf[inPtr++] & 0x3F);
                    // note: this is the codepoint value; need to split, too
                    i -= 0x10000;
                    outBuf[outPtr++] = (char) (0xD800 | (i >> 10));
                    i = 0xDC00 | (i & 0x3FF);
                    break;
                default: // invalid
                    _reportError("Invalid byte "+Integer.toHexString(i)+" in short Unicode text block");
                }
            }
            outBuf[outPtr++] = (char) i;
        }        
        _textBuffer.setCurrentLength(outPtr);
    }
     */
    
    private final JsonToken _nextLongAscii(int substatethrows IOExceptionJsonParseException
    {
        // did not get it all; mark the state so we know where to return:
         = ;
         = true;
         = substate;
        return ( = .);
    }
    /*
    private final void _decodeLongAscii()
        throws IOException, JsonParseException
    {
        int outPtr = 0;
        char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
        main_loop:
        while (true) {
            if (_inputPtr >= _inputEnd) {
                loadMoreGuaranteed();
            }
            int inPtr = _inputPtr;
            int left = _inputEnd - inPtr;
            if (outPtr >= outBuf.length) {
                outBuf = _textBuffer.finishCurrentSegment();
                outPtr = 0;
            }
            left = Math.min(left, outBuf.length - outPtr);
            do {
                byte b = _inputBuffer[inPtr++];
                if (b == SmileConstants.BYTE_MARKER_END_OF_STRING) {
                    _inputPtr = inPtr;
                    break main_loop;
                }
                outBuf[outPtr++] = (char) b;                    
            } while (--left > 0);
            _inputPtr = inPtr;
        }
        _textBuffer.setCurrentLength(outPtr);
    }
    */
    private final JsonToken _nextLongUnicode(int substatethrows IOExceptionJsonParseException
    {
        // did not get it all; mark the state so we know where to return:
         = ;
         = true;
         = substate;
        return ( = .);
    }
    
    private final void _decodeLongUnicode()
        throws IOException, JsonParseException
    {
        int outPtr = 0;
        char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
        final int[] codes = SmileConstants.sUtf8UnitLengths;
        int c;
        final byte[] inputBuffer = _inputBuffer;
        main_loop:
        while (true) {
            // First the tight ASCII loop:
            ascii_loop:
            while (true) {
                int ptr = _inputPtr;
                if (ptr >= _inputEnd) {
                    loadMoreGuaranteed();
                    ptr = _inputPtr;
                }
                if (outPtr >= outBuf.length) {
                    outBuf = _textBuffer.finishCurrentSegment();
                    outPtr = 0;
                }
                int max = _inputEnd;
                {
                    int max2 = ptr + (outBuf.length - outPtr);
                    if (max2 < max) {
                        max = max2;
                    }
                }
                while (ptr < max) {
                    c = (int) inputBuffer[ptr++] & 0xFF;
                    if (codes[c] != 0) {
                        _inputPtr = ptr;
                        break ascii_loop;
                    }
                    outBuf[outPtr++] = (char) c;
                }
                _inputPtr = ptr;
            }
            // Ok: end marker, escape or multi-byte?
            if (c == SmileConstants.INT_MARKER_END_OF_STRING) {
                break main_loop;
            }
            switch (codes[c]) {
            case 1: // 2-byte UTF
                c = _decodeUtf8_2(c);
                break;
            case 2: // 3-byte UTF
                if ((_inputEnd - _inputPtr) >= 2) {
                    c = _decodeUtf8_3fast(c);
                } else {
                    c = _decodeUtf8_3(c);
                }
                break;
            case 3: // 4-byte UTF
                c = _decodeUtf8_4(c);
                // Let's add first part right away:
                outBuf[outPtr++] = (char) (0xD800 | (c >> 10));
                if (outPtr >= outBuf.length) {
                    outBuf = _textBuffer.finishCurrentSegment();
                    outPtr = 0;
                }
                c = 0xDC00 | (c & 0x3FF);
                // And let the other char output down below
                break;
            default:
                // Is this good enough error message?
                _reportInvalidChar(c);
            }
            // Need more room?
            if (outPtr >= outBuf.length) {
                outBuf = _textBuffer.finishCurrentSegment();
                outPtr = 0;
            }
            // Ok, let's add char to output:
            outBuf[outPtr++] = (char) c;
        }
        _textBuffer.setCurrentLength(outPtr);
    }
     */
    
    private final JsonToken _nextLongSharedString(int substatethrows IOExceptionJsonParseException
    {
        // did not get it all; mark the state so we know where to return:
         = true;
    &