Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  package org.mapdb;
  
  import java.io.DataInput;
  import java.io.DataOutput;
Custom serializer for BTreeMap keys which enables Delta encoding.

Keys in BTree Nodes are sorted, this enables number of tricks to save disk space. For example for numbers we may store only difference between subsequent numbers, for string we can only take suffix, etc...

Parameters:
<K> type of key
 
 public abstract class BTreeKeySerializer<K>{

    
Serialize keys from single BTree Node.

Parameters:
out output stream where to put ata
start where data start in array. Before this index all keys are null
end where data ends in array (exclusive). From this index all keys are null
keys array of keys for single BTree Node
Throws:
java.io.IOException
 
     public abstract void serialize(DataOutput outint startint endObject[] keysthrows IOException;

    
Deserializes keys for single BTree Node. To

Parameters:
in input stream to read data from
start where data start in array. Before this index all keys are null
end where data ends in array (exclusive). From this index all keys are null
size size of array which should be returned
Returns:
array of keys for single BTree Node
Throws:
java.io.IOException
 
     public abstract Object[] deserialize(DataInput inint startint endint sizethrows IOException;
 
 
     public static final BTreeKeySerializer BASIC = new BTreeKeySerializer.BasicKeySerializer(.);

    
Basic Key Serializer which just writes data without applying any compression. Is used by default if no other Key Serializer is specified.
 
     public static final class BasicKeySerializer extends BTreeKeySerializer<Objectimplements Serializable {
 
         protected final Serializer defaultSerializer;
 
         BasicKeySerializer(Serializer defaultSerializer) {
             this. = defaultSerializer;
         }
 
         @Override
         public void serialize(DataOutput outint startint endObject[] keysthrows IOException {
             for(int i = start;i<end;i++){
                 .serialize(out,keys[i]);
             }
         }
 
         @Override
         public Object[] deserialize(DataInput inint startint endint sizethrows IOException{
             Object[] ret = new Object[size];
             for(int i=starti<endi++){
                 ret[i] = .deserialize(in,-1);
             }
             return ret;
         }
     }


    
Applies delta packing on java.lang.Long. All keys must be non negative. Difference between consequential numbers is also packed itself, so for small diffs it takes only single byte per number.
 
     public static final  BTreeKeySerializer<LongZERO_OR_POSITIVE_LONG = new BTreeKeySerializer<Long>() {
         @Override
         public void serialize(DataOutput outint startint endObject[] keysthrows IOException {
             if(start>=endreturn;
 //            System.out.println(start+" - "+end+" - "+Arrays.toString(keys));
             long prev = (Long)keys[start];
             Utils.packLong(out,prev);
             for(int i=start+1;i<end;i++){
                 long curr = (Long)keys[i];
                 Utils.packLong(outcurr-prev);
                 prev = curr;
             }
         }
 
         @Override
         public Object[] deserialize(DataInput inint startint endint sizethrows IOException {
             Object[] ret = new Long[size];
             long prev = 0 ;
            for(int i = starti<endi++){
                ret[i] = prev = prev + Utils.unpackLong(in);
            }
            return ret;
        }
    };

    
Applies delta packing on java.lang.Integer. All keys must be non negative. Difference between consequential numbers is also packed itself, so for small diffs it takes only single byte per number.
    public static final  BTreeKeySerializer<IntegerZERO_OR_POSITIVE_INT = new BTreeKeySerializer<Integer>() {
        @Override
        public void serialize(DataOutput outint startint endObject[] keysthrows IOException {
            if(start>=endreturn;
//            System.out.println(start+" - "+end+" - "+Arrays.toString(keys));
            int prev = (Integer)keys[start];
            Utils.packLong(out,prev);
            for(int i=start+1;i<end;i++){
                int curr = (Integer)keys[i];
                Utils.packInt(outcurr-prev);
                prev = curr;
            }
        }
        @Override
        public Object[] deserialize(DataInput inint startint endint sizethrows IOException {
            Object[] ret = new Integer[size];
            int prev = 0 ;
            for(int i = starti<endi++){
                ret[i] = prev = prev + Utils.unpackInt(in);
            }
            return ret;
        }
    };


    
Applies delta packing on java.lang.String. This serializer splits consequent strings to two parts: shared prefix and different suffix. Only suffix is than stored.
    public static final  BTreeKeySerializer<StringSTRING = new BTreeKeySerializer<String>() {
        @Override
        public void serialize(DataOutput outint startint endObject[] keysthrows IOException {
            byte[] previous = null;
            for (int i = starti < endi++) {
                byte[] b = ((Stringkeys[i]).getBytes(.);
                leadingValuePackWrite(outbprevious, 0);
                previous = b;
            }
        }
        @Override
        public Object[] deserialize(DataInput inint startint endint sizethrows IOException {
            Object[] ret = new Object[size];
            byte[] previous = null;
            for (int i = starti < endi++) {
                byte[] b = leadingValuePackRead(inprevious, 0);
                if (b == nullcontinue;
                ret[i] = new String(b,.);
                previous = b;
            }
            return ret;
        }
    };

    
Read previously written data from leadingValuePackWrite() method.

Author(s):
Kevin Day
    public static byte[] leadingValuePackRead(DataInput inbyte[] previousint ignoreLeadingCountthrows IOException {
        int len = Utils.unpackInt(in) - 1;  // 0 indicates null
        if (len == -1)
            return null;
        int actualCommon = Utils.unpackInt(in);
        byte[] buf = new byte[len];
        if (previous == null) {
            actualCommon = 0;
        }
        if (actualCommon > 0) {
            in.readFully(buf, 0, ignoreLeadingCount);
            System.arraycopy(previousignoreLeadingCountbufignoreLeadingCountactualCommon - ignoreLeadingCount);
        }
        in.readFully(bufactualCommonlen - actualCommon);
        return buf;
    }

    
This method is used for delta compression for keys. Writes the contents of buf to the DataOutput out, with special encoding if there are common leading bytes in the previous group stored by this compressor.

Author(s):
Kevin Day
    public static void leadingValuePackWrite(DataOutput outbyte[] bufbyte[] previousint ignoreLeadingCountthrows IOException {
        if (buf == null) {
            Utils.packInt(out, 0);
            return;
        }
        int actualCommon = ignoreLeadingCount;
        if (previous != null) {
            int maxCommon = buf.length > previous.length ? previous.length : buf.length;
            if (maxCommon > .maxCommon = .;
            for (; actualCommon < maxCommonactualCommon++) {
                if (buf[actualCommon] != previous[actualCommon])
                    break;
            }
        }
        // there are enough common bytes to justify compression
        Utils.packInt(outbuf.length + 1);// store as +1, 0 indicates null
        Utils.packInt(outactualCommon);
        out.write(buf, 0, ignoreLeadingCount);
        out.write(bufactualCommonbuf.length - actualCommon);
    }

    
Tuple2 Serializer which uses Default Serializer from DB and expect values to implement Comparable interface.
    public static final Tuple2KeySerializer TUPLE2 = new Tuple2KeySerializer(nullnullnull);

    
Applies delta compression on array of tuple. First tuple value may be shared between consequentive tuples, so only first occurrence is serialized. An example:
     Value            Serialized as
     -------------------------
     Tuple(1, 1)       1, 1
     Tuple(1, 2)          2
     Tuple(1, 3)          3
     Tuple(1, 4)          4
 

Parameters:
<A> first tuple value
<B> second tuple value
    public final  static class Tuple2KeySerializer<A,B> extends  BTreeKeySerializer<Fun.Tuple2<A,B>> implements Serializable {
        protected final Comparator<A> aComparator;
        protected final Serializer<A> aSerializer;
        protected final Serializer<B> bSerializer;

        
Construct new Tuple2 Key Serializer. You may pass null for some value, In that case 'default' value will be used, Comparable comparator and Default Serializer from DB.

Parameters:
aComparator comparator used for first tuple value
aSerializer serializer used for first tuple value
bSerializer serializer used for second tuple value
        public Tuple2KeySerializer(Comparator<A> aComparator,Serializer aSerializerSerializer bSerializer){
            this. = aComparator;
            this. = aSerializer;
            this. = bSerializer;
        }
        @Override
        public void serialize(DataOutput outint startint endObject[] keysthrows IOException {
            int acount=0;
            for(int i=start;i<end;i++){
                Fun.Tuple2<A,B> t = (Fun.Tuple2<A, B>) keys[i];
                if(acount==0){
                    //write new A
                    .serialize(out,t.a);
                    //count how many A are following
                    acount=1;
                    while(i+acount<end && .compare(t.a, ((Fun.Tuple2<A, B>) keys[i+acount]).)==0){
                        acount++;
                    }
                    Utils.packInt(out,acount);
                }
                .serialize(out,t.b);
                acount--;
            }
        }
        @Override
        public Object[] deserialize(DataInput inint startint endint sizethrows IOException {
            Object[] ret = new Object[size];
            A a = null;
            int acount = 0;
            for(int i=start;i<end;i++){
                if(acount==0){
                    //read new A
                    a = .deserialize(in,-1);
                    acount = Utils.unpackInt(in);
                }
                B b = .deserialize(in,-1);
                ret[i]= Fun.t2(a,b);
                acount--;
            }
            if(acount!=0) throw new InternalError();
            return ret;
        }
        @Override
        public boolean equals(Object o) {
            if (this == oreturn true;
            if (o == null || getClass() != o.getClass()) return false;
            Tuple2KeySerializer that = (Tuple2KeySerializero;
            if ( != null ? !.equals(that.aComparator) : that.aComparator != nullreturn false;
            if ( != null ? !.equals(that.aSerializer) : that.aSerializer != nullreturn false;
            if ( != null ? !.equals(that.bSerializer) : that.bSerializer != nullreturn false;
            return true;
        }
        @Override
        public int hashCode() {
            int result =  != null ? .hashCode() : 0;
            result = 31 * result + ( != null ? .hashCode() : 0);
            result = 31 * result + ( != null ? .hashCode() : 0);
            return result;
        }
    }

    
Tuple3 Serializer which uses Default Serializer from DB and expect values to implement Comparable interface.
    public static final Tuple3KeySerializer TUPLE3 = new Tuple3KeySerializer(nullnullnullnullnull);

    
Applies delta compression on array of tuple. First and second tuple value may be shared between consequentive tuples, so only first occurrence is serialized. An example:
     Value            Serialized as
     ----------------------------
     Tuple(1, 2, 1)       1, 2, 1
     Tuple(1, 2, 2)             2
     Tuple(1, 3, 3)          3, 3
     Tuple(1, 3, 4)             4
 

Parameters:
<A> first tuple value
<B> second tuple value
<C> third tuple value
    public static class Tuple3KeySerializer<A,B,C> extends  BTreeKeySerializer<Fun.Tuple3<A,B,C>> implements Serializable {
        protected final Comparator<A> aComparator;
        protected final Comparator<B> bComparator;
        protected final Serializer<A> aSerializer;
        protected final Serializer<B> bSerializer;
        protected final Serializer<C> cSerializer;

        
Construct new Tuple3 Key Serializer. You may pass null for some value, In that case 'default' value will be used, Comparable comparator and Default Serializer from DB.

Parameters:
aComparator comparator used for first tuple value
bComparator comparator used for second tuple value
aSerializer serializer used for first tuple value
bSerializer serializer used for second tuple value
cSerializer serializer used for third tuple value
        public Tuple3KeySerializer(Comparator<A> aComparatorComparator<B> bComparator,  Serializer aSerializer,
                                   Serializer bSerializerSerializer cSerializer){
            this. = aComparator;
            this. = bComparator;
            this. = aSerializer;
            this. = bSerializer;
            this. = cSerializer;
        }
        @Override
        public void serialize(DataOutput outint startint endObject[] keysthrows IOException {
            int acount=0;
            int bcount=0;
            for(int i=start;i<end;i++){
                Fun.Tuple3<A,B,C> t = (Fun.Tuple3<A, B,C>) keys[i];
                if(acount==0){
                    //write new A
                    .serialize(out,t.a);
                    //count how many A are following
                    acount=1;
                    while(i+acount<end && .compare(t.a, ((Fun.Tuple3<A, B, C>) keys[i+acount]).)==0){
                        acount++;
                    }
                    Utils.packInt(out,acount);
                }
                if(bcount==0){
                    //write new B
                    .serialize(out,t.b);
                    //count how many B are following
                    bcount=1;
                    while(i+bcount<end && .compare(t.b, ((Fun.Tuple3<A, B,C>) keys[i+bcount]).)==0){
                        bcount++;
                    }
                    Utils.packInt(out,bcount);
                }
                .serialize(out,t.c);
                acount--;
                bcount--;
            }
        }
        @Override
        public Object[] deserialize(DataInput inint startint endint sizethrows IOException {
            Object[] ret = new Object[size];
            A a = null;
            int acount = 0;
            B b = null;
            int bcount = 0;
            for(int i=start;i<end;i++){
                if(acount==0){
                    //read new A
                    a = .deserialize(in,-1);
                    acount = Utils.unpackInt(in);
                }
                if(bcount==0){
                    //read new B
                    b = .deserialize(in,-1);
                    bcount = Utils.unpackInt(in);
                }
                C c = .deserialize(in,-1);
                ret[i]= Fun.t3(abc);
                acount--;
                bcount--;
            }
            if(acount!=0) throw new InternalError();
            if(bcount!=0) throw new InternalError();
            return ret;
        }
        @Override
        public boolean equals(Object o) {
            if (this == oreturn true;
            if (o == null || getClass() != o.getClass()) return false;
            Tuple3KeySerializer that = (Tuple3KeySerializero;
            if ( != null ? !.equals(that.aComparator) : that.aComparator != nullreturn false;
            if ( != null ? !.equals(that.aSerializer) : that.aSerializer != nullreturn false;
            if ( != null ? !.equals(that.bComparator) : that.bComparator != nullreturn false;
            if ( != null ? !.equals(that.bSerializer) : that.bSerializer != nullreturn false;
            if ( != null ? !.equals(that.cSerializer) : that.cSerializer != nullreturn false;
            return true;
        }
        @Override
        public int hashCode() {
            int result =  != null ? .hashCode() : 0;
            result = 31 * result + ( != null ? .hashCode() : 0);
            result = 31 * result + ( != null ? .hashCode() : 0);
            result = 31 * result + ( != null ? .hashCode() : 0);
            result = 31 * result + ( != null ? .hashCode() : 0);
            return result;
        }
    }

    
Tuple4 Serializer which uses Default Serializer from DB and expect values to implement Comparable interface.
    public static final Tuple4KeySerializer TUPLE4 = new Tuple4KeySerializer(nullnullnullnullnullnullnull);


    
Applies delta compression on array of tuple. First, second and third tuple value may be shared between consequential tuples, so only first occurrence is serialized. An example:
     Value                Serialized as
     ----------------------------------
     Tuple(1, 2, 1, 1)       1, 2, 1, 1
     Tuple(1, 2, 1, 2)                2
     Tuple(1, 3, 3, 3)          3, 3, 3
     Tuple(1, 3, 4, 4)             4, 4
 

Parameters:
<A> first tuple value
<B> second tuple value
<C> third tuple value
    public static class Tuple4KeySerializer<A,B,C,D> extends  BTreeKeySerializer<Fun.Tuple4<A,B,C,D>> implements Serializable {
        protected final Comparator<A> aComparator;
        protected final Comparator<B> bComparator;
        protected final Comparator<C> cComparator;
        protected final Serializer<A> aSerializer;
        protected final Serializer<B> bSerializer;
        protected final Serializer<C> cSerializer;
        protected final Serializer<D> dSerializer;

        
Construct new Tuple4 Key Serializer. You may pass null for some value, In that case 'default' value will be used, Comparable comparator and Default Serializer from DB.

Parameters:
aComparator comparator used for first tuple value
bComparator comparator used for second tuple value
cComparator comparator used for third tuple value*
aSerializer serializer used for first tuple value
bSerializer serializer used for second tuple value
cSerializer serializer used for third tuple value
dSerializer serializer used for fourth tuple value
        public Tuple4KeySerializer(Comparator<A> aComparatorComparator<B> bComparatorComparator<C> cComparator,
                                   Serializer aSerializerSerializer bSerializerSerializer cSerializerSerializer dSerializer){
            this. = aComparator;
            this. = bComparator;
            this. = cComparator;
            this. = aSerializer;
            this. = bSerializer;
            this. = cSerializer;
            this. = dSerializer;
        }
        @Override
        public void serialize(DataOutput outint startint endObject[] keysthrows IOException {
            int acount=0;
            int bcount=0;
            int ccount=0;
            for(int i=start;i<end;i++){
                Fun.Tuple4<A,B,C,D> t = (Fun.Tuple4<A, B,C,D>) keys[i];
                if(acount==0){
                    //write new A
                    .serialize(out,t.a);
                    //count how many A are following
                    acount=1;
                    while(i+acount<end && .compare(t.a, ((Fun.Tuple4<A, B, C,D>) keys[i+acount]).)==0){
                        acount++;
                    }
                    Utils.packInt(out,acount);
                }
                if(bcount==0){
                    //write new B
                    .serialize(out,t.b);
                    //count how many B are following
                    bcount=1;
                    while(i+bcount<end && .compare(t.b, ((Fun.Tuple4<A, B,C,D>) keys[i+bcount]).)==0){
                        bcount++;
                    }
                    Utils.packInt(out,bcount);
                }
                if(ccount==0){
                    //write new C
                    .serialize(out,t.c);
                    //count how many C are following
                    ccount=1;
                    while(i+ccount<end && .compare(t.c, ((Fun.Tuple4<A, B,C,D>) keys[i+ccount]).)==0){
                        ccount++;
                    }
                    Utils.packInt(out,ccount);
                }
                .serialize(out,t.d);
                acount--;
                bcount--;
                ccount--;
            }
        }
        @Override
        public Object[] deserialize(DataInput inint startint endint sizethrows IOException {
            Object[] ret = new Object[size];
            A a = null;
            int acount = 0;
            B b = null;
            int bcount = 0;
            C c = null;
            int ccount = 0;
            for(int i=start;i<end;i++){
                if(acount==0){
                    //read new A
                    a = .deserialize(in,-1);
                    acount = Utils.unpackInt(in);
                }
                if(bcount==0){
                    //read new B
                    b = .deserialize(in,-1);
                    bcount = Utils.unpackInt(in);
                }
                if(ccount==0){
                    //read new C
                    c = .deserialize(in,-1);
                    ccount = Utils.unpackInt(in);
                }
                D d = .deserialize(in,-1);
                ret[i]= Fun.t4(abcd);
                acount--;
                bcount--;
                ccount--;
            }
            if(acount!=0) throw new InternalError();
            if(bcount!=0) throw new InternalError();
            if(ccount!=0) throw new InternalError();
            return ret;
        }
        @Override
        public boolean equals(Object o) {
            if (this == oreturn true;
            if (o == null || getClass() != o.getClass()) return false;
            Tuple4KeySerializer that = (Tuple4KeySerializero;
            if ( != null ? !.equals(that.aComparator) : that.aComparator != nullreturn false;
            if ( != null ? !.equals(that.aSerializer) : that.aSerializer != nullreturn false;
            if ( != null ? !.equals(that.bComparator) : that.bComparator != nullreturn false;
            if ( != null ? !.equals(that.bSerializer) : that.bSerializer != nullreturn false;
            if ( != null ? !.equals(that.cComparator) : that.cComparator != nullreturn false;
            if ( != null ? !.equals(that.cSerializer) : that.cSerializer != nullreturn false;
            if ( != null ? !.equals(that.dSerializer) : that.dSerializer != nullreturn false;
            return true;
        }
        @Override
        public int hashCode() {
            int result =  != null ? .hashCode() : 0;
            result = 31 * result + ( != null ? .hashCode() : 0);
            result = 31 * result + ( != null ? .hashCode() : 0);
            result = 31 * result + ( != null ? .hashCode() : 0);
            result = 31 * result + ( != null ? .hashCode() : 0);
            result = 31 * result + ( != null ? .hashCode() : 0);
            result = 31 * result + ( != null ? .hashCode() : 0);
            return result;
        }
    }
New to GrepCode? Check out our FAQ X