Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   *  Copyright (c) 2012 Jan Kotek
   *
   *  Licensed under the Apache License, Version 2.0 (the "License");
   *  you may not use this file except in compliance with the License.
   *  You may obtain a copy of the License at
   *
   *    http://www.apache.org/licenses/LICENSE-2.0
   *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
 
 package org.mapdb;
 
 import java.io.*;
 import java.util.Map;
MapDB abstraction over raw storage (file, disk partition, memory etc...).

Implementations needs to be thread safe (especially 'ensureAvailable') operation. However updates do not have to be atomic, it is clients responsibility to ensure two threads are not writing/reading into the same location.

Author(s):
Jan Kotek
 
 public abstract class Volume {
 
 
     public static final int BUF_SHIFT = 30;
 
     public static final int BUF_SIZE = 1<<;
 
     public static final int BUF_SIZE_MOD_MASK = -1;

    
Check space allocated by Volume is bigger or equal to given offset. So it is safe to write into smaller offsets.

Parameters:
offset
Throws:
java.io.IOError if Volume can not be expanded beyond given offset
 
     public void ensureAvailable(final long offset){
         if(!tryAvailable(offset))
                 throw new IOError(new IOException("no free space to expand Volume"));
     }
 
 
     abstract public boolean tryAvailable(final long offset);
 
 
     abstract public void putLong(final long offsetfinal long value);
     abstract public void putInt(long offsetint value);
     abstract public void putByte(final long offsetfinal byte value);
 
     abstract public void putData(final long offsetfinal byte[] srcint srcPosint srcSize);
     abstract public void putData(final long offsetfinal ByteBuffer buf);
 
     abstract public long getLong(final long offset);
     abstract public int getInt(long offset);
     abstract public byte getByte(final long offset);
 
 
 
     abstract public DataInput2 getDataInput(final long offsetfinal int size);
 
     abstract public void close();
 
     abstract public void sync();
 
     public abstract boolean isEmpty();
 
     public abstract void deleteFile();
 
     public abstract boolean isSliced();
 
 
     public final void putUnsignedShort(final long offsetfinal int value){
         putByte(offset, (byte) (value>>8));
         putByte(offset+1, (byte) (value));
     }
 
     public final int getUnsignedShort(long offset) {
         return (( (getByte(offset) & 0xff) << 8) |
                 ( (getByte(offset+1) & 0xff)));
     }
    public int getUnsignedByte(long offset) {
        return getByte(offset) & 0xff;
    }
    public void putUnsignedByte(long offsetint b) {
        putByte(offset, (byte)(b & 0xff));
    }

    
Reads a long from the indicated position
    public long getSixLong(long pos) {
        return
                ((long) (getByte(pos + 0) & 0xff) << 40) |
                        ((long) (getByte(pos + 1) & 0xff) << 32) |
                        ((long) (getByte(pos + 2) & 0xff) << 24) |
                        ((long) (getByte(pos + 3) & 0xff) << 16) |
                        ((long) (getByte(pos + 4) & 0xff) << 8) |
                        ((long) (getByte(pos + 5) & 0xff) << 0);
    }

    
Writes a long to the indicated position
    public void putSixLong(long poslong value) {
        if(value<0) throw new IllegalArgumentException();
    	if(value >> (6*8)!=0)
    		throw new IllegalArgumentException("does not fit");
        //TODO read/write as integer+short, might be faster
        putByte(pos + 0, (byte) (0xff & (value >> 40)));
        putByte(pos + 1, (byte) (0xff & (value >> 32)));
        putByte(pos + 2, (byte) (0xff & (value >> 24)));
        putByte(pos + 3, (byte) (0xff & (value >> 16)));
        putByte(pos + 4, (byte) (0xff & (value >> 8)));
        putByte(pos + 5, (byte) (0xff & (value >> 0)));
    }

    
Writes packed long at given position and returns number of bytes used.
    public int putPackedLong(long poslong value) {
        if (value < 0) {
            throw new IllegalArgumentException("negative value: keys=" + value);
        }
        int ret = 0;
        while ((value & ~0x7FL) != 0) {
            putUnsignedByte(pos+(ret++), (((intvalue & 0x7F) | 0x80));
            value >>>= 7;
        }
        putUnsignedByte(pos + (ret++), (bytevalue);
        return ret;
    }



    
returns underlying file if it exists
    abstract public File getFile();
    public long getPackedLong(long pos){
        long result = 0;
        for (int offset = 0; offset < 64; offset += 7) {
            long b = getUnsignedByte(pos++);
            result |= (b & 0x7F) << offset;
            if ((b & 0x80) == 0) {
                return result;
            }
        }
        throw new Error("Malformed long.");
    }


    
Factory which creates two/three volumes used by each MapDB Storage Engine
    public static interface Factory {
        Volume createIndexVolume();
        Volume createPhysVolume();
        Volume createTransLogVolume();
    }
    public static Volume volumeForFile(File fboolean useRandomAccessFileboolean readOnlylong sizeLimit) {
        return useRandomAccessFile ?
                new FileChannelVol(freadOnly,sizeLimit):
                new MappedFileVol(freadOnly,sizeLimit);
    }
    public static Factory fileFactory(final boolean readOnlyfinal int rafModefinal File indexFilefinal long sizeLimit){
        return fileFactory(readOnlyrafModesizeLimitindexFile,
                new File(indexFile.getPath() + .),
                new File(indexFile.getPath() + .));
    }
    public static Factory fileFactory(final boolean readOnly,
                                      final int rafMode,
                                      final long sizeLimit,
                                      final File indexFile,
                                      final File physFile,
                                      final File transLogFile) {
        return new Factory() {
            @Override
            public Volume createIndexVolume() {
                return volumeForFile(indexFilerafMode>1, readOnlysizeLimit);
            }
            @Override
            public Volume createPhysVolume() {
                return volumeForFile(physFilerafMode>0, readOnlysizeLimit);
            }
            @Override
            public Volume createTransLogVolume() {
                return volumeForFile(transLogFilerafMode>0, readOnlysizeLimit);
            }
        };
    }
    public static Factory memoryFactory(final boolean useDirectBufferfinal long sizeLimit) {
        return new Factory() {
            @Override public synchronized  Volume createIndexVolume() {
                return new MemoryVol(useDirectBuffersizeLimit);
            }
            @Override public synchronized Volume createPhysVolume() {
                return new MemoryVol(useDirectBuffersizeLimit);
            }
            @Override public synchronized Volume createTransLogVolume() {
                return new MemoryVol(useDirectBuffersizeLimit);
            }
        };
    }


    
Abstract Volume over bunch of ByteBuffers It leaves ByteBufferVol details (allocation, disposal) on subclasses. Most methods are final for better performance (JIT compiler can inline those).
    abstract static public class ByteBufferVol extends Volume{
        protected final ReentrantLock growLock = new ReentrantLock();
        protected final long sizeLimit;
        protected final boolean hasLimit;
        protected volatile ByteBuffer[] buffers;
        protected final boolean readOnly;
        protected ByteBufferVol(boolean readOnlylong sizeLimit) {
            this. = readOnly;
            this. = sizeLimit;
            this. = sizeLimit>0;
        }
        @Override
        public final boolean tryAvailable(long offset) {
            if(&&offset>return false;
            int buffersPos = (int) (offset >>> );
            //check for most common case, this is already mapped
            if(buffersPos<. && [buffersPos]!=null &&
                    [buffersPos].capacity()>=(offset&.)){
                return true;
            }
            .lock();
            try{
                //check second time
                if(buffersPos<. && [buffersPos]!=null &&
                        [buffersPos].capacity()>=(offset&.))
                    return true;
                ByteBuffer[] buffers2 = ;
                //grow array if necessary
                if(buffersPos>=buffers2.length){
                    buffers2 = Arrays.copyOf(buffers2, Math.max(buffersPos+1, buffers2.length * 2));
                }
                //just remap file buffer
                ifbuffers2[buffersPos] == null){
                    //make sure previous buffer is fully expanded
                    if(buffersPos>0){
                        ByteBuffer oldPrev = buffers2[buffersPos-1];
                        if(oldPrev == null || oldPrev.capacity()!=){
                            buffers2[buffersPos-1]  = makeNewBuffer(1L*buffersPos*-1,buffers2);
                        }
                    }
                }
                ByteBuffer newBuf = makeNewBuffer(offsetbuffers2);
                if()
                    newBuf = newBuf.asReadOnlyBuffer();
                buffers2[buffersPos] = newBuf;
                 = buffers2;
            }finally{
                .unlock();
            }
            return true;
        }
        protected abstract ByteBuffer makeNewBuffer(long offsetByteBuffer[] buffers2);
        @Override public final void putLong(final long offsetfinal long value) {
            [(int)(offset >>>)].putLong((int) (offset&.), value);
        }
        @Override public final void putInt(final long offsetfinal int value) {
            [(int)(offset >>>)].putInt((int) (offset&.), value);
        }
        @Override public final void putByte(final long offsetfinal byte value) {
            [(int)(offset >>>)].put((int) (offset &.), value);
        }
        @Override public void putData(final long offsetfinal byte[] srcint srcPosint srcSize){
            final ByteBuffer b1 = [(int)(offset >>>)].duplicate();
            final int bufPos = (int) (offset&.);
            b1.position(bufPos);
            b1.put(srcsrcPossrcSize);
        }
        @Override public final void putData(final long offsetfinal ByteBuffer buf) {
            final ByteBuffer b1 = [(int)(offset >>>)].duplicate();
            final int bufPos = (int) (offset&.);
            //no overlap, so just write the value
            b1.position(bufPos);
            b1.put(buf);
        }
        @Override final public long getLong(long offset) {
             return [(int)(offset >>>)].getLong((int) (offset&.));
        }
        @Override final public int getInt(long offset) {
             return [(int)(offset >>>)].getInt((int) (offset&.));
        }
        @Override public final byte getByte(long offset) {
             return [(int)(offset >>>)].get((int) (offset&.));
        }
        @Override
        public final DataInput2 getDataInput(long offsetint size) {
            return new DataInput2([(int)(offset >>>)], (int) (offset&.));
        }
        @Override
        public boolean isEmpty() {
            return [0]==null || [0].capacity()==0;
        }
        @Override
        public boolean isSliced(){
            return true;
        }



        
Hack to unmap MappedByteBuffer. Unmap is necessary on Windows, otherwise file is locked until JVM exits or BB is GCed. There is no public JVM API to unmap buffer, so this tries to use SUN proprietary API for unmap. Any error is silently ignored (for example SUN API does not exist on Android).
        protected void unmap(MappedByteBuffer b){
            try{
                if(){
                    // need to dispose old direct buffer, see bug
                    // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038
                    Method cleanerMethod = b.getClass().getMethod("cleaner"new Class[0]);
                    if(cleanerMethod!=null){
                        cleanerMethod.setAccessible(true);
                        Object cleaner = cleanerMethod.invoke(bnew Object[0]);
                        if(cleaner!=null){
                            Method clearMethod = cleaner.getClass().getMethod("clean"new Class[0]);
                            if(cleanerMethod!=null)
                                clearMethod.invoke(cleanernew Object[0]);
                        }
                    }
                }
            }catch(Exception e){
                 = false;
                ..log(."ByteBufferVol Unmap failed"e);
            }
        }
        private static boolean unmapHackSupported = true;
        static{
            try{
                 =
                        Class.forName("sun.nio.ch.DirectBuffer")!=null;
            }catch(Exception e){
                 = false;
            }
        }
    }
    public static final class MappedFileVol extends ByteBufferVol {
        protected final File file;
        protected final FileChannel fileChannel;
        protected final FileChannel.MapMode mapMode;
        protected final java.io.RandomAccessFile raf;
        protected final Map<ByteBufferStringunreleasedBuffers =
                Utils.isWindows() ? new WeakHashMap<ByteBufferString>() : null;
        static final int BUF_SIZE_INC = 1024*1024;
        public MappedFileVol(File fileboolean readOnlylong sizeLimit) {
            super(readOnlysizeLimit);
            this. = file;
            this. = readOnly....;
            try {
                this. = new java.io.RandomAccessFile(filereadOnly?"r":"rw");
                this. = .getChannel();
                final long fileSize = .size();
                if(fileSize>0){
                    //map existing data
                     = new ByteBuffer[(int) (1+(fileSize>>>))];
                    for(int i=0;i<=fileSize>>>;i++){
                        final long offset = 1L**i;
                        [i] = .map(offset, Math.min(fileSize-offset));
                        if( == ..)
                            [i] = [i].asReadOnlyBuffer();
                        //TODO what if 'fileSize % 8 != 0'?
                    }
                }else{
                     = new ByteBuffer[1];
//                    buffers[0] = fileChannel.map(mapMode, 0, INITIAL_SIZE);
//                    if(mapMode == FileChannel.MapMode.READ_ONLY)
//                        buffers[0] = buffers[0].asReadOnlyBuffer();
                }
            } catch (IOException e) {
                throw new IOError(e);
            }
        }
        @Override
        public void close() {
            .lock();
            try{
                .close();
                .close();
                if(!)
                    sync();
                for(ByteBuffer b:){
                    if(b!=null && (b instanceof MappedByteBuffer)){
                        unmap((MappedByteBufferb);
                    }
                }
                 = null;
                if(!=null){
                    for(ByteBuffer b:.keySet().toArray(new MappedByteBuffer[0])){
                        if(b!=null && (b instanceof MappedByteBuffer)){
                            unmap((MappedByteBufferb);
                        }
                    }
                }
            } catch (IOException e) {
                throw new IOError(e);
            }finally{
                .unlock();
            }
        }
        @Override
        public void sync() {
            if(return;
            for(ByteBuffer b:){
                if(b!=null && (b instanceof MappedByteBuffer)){
                    ((MappedByteBuffer)b).force();
                }
            }
        }
        @Override
        public boolean isEmpty() {
            return [0]==null || [0].capacity()==0;
        }
        @Override
        public void deleteFile() {
            .delete();
        }
        @Override
        public File getFile() {
            return ;
        }
        @Override
        protected ByteBuffer makeNewBuffer(long offsetByteBuffer[] buffers2) {
            try {
                long newBufSize =  offset&.;
                newBufSize = newBufSize + (newBufSize&.); //round to BUF_SIZE_INC
                ByteBuffer buf =  .mapoffset - (offset&.), newBufSize );
                if(!=null.put(buf"");
                return buf;
            } catch (IOException e) {
                if(e.getCause()!=null && e.getCause() instanceof OutOfMemoryError){
                    throw new RuntimeException("File could not be mapped to memory, common problem on 32bit JVM. Use `DBMaker.newRandomAccessFileDB()` as workaround",e);
                }
                throw new IOError(e);
            }
        }
    }
    public static final class MemoryVol extends ByteBufferVol {
        protected final boolean useDirectBuffer;
        @Override
        public String toString() {
            return super.toString()+",direct="+;
        }
        public MemoryVol(boolean useDirectBufferlong sizeLimit) {
            super(false,sizeLimit);
            this. = useDirectBuffer;
//            ByteBuffer b0 = useDirectBuffer?
//                    ByteBuffer.allocateDirect(INITIAL_SIZE) :
//                    ByteBuffer.allocate(INITIAL_SIZE);
//            buffers = new ByteBuffer[]{b0};
            =new ByteBuffer[1];
        }
        @Override protected ByteBuffer makeNewBuffer(long offsetByteBuffer[] buffers2) {
            final int newBufSize = Utils.nextPowTwo((int) ((offset &.)));
            //double size of existing in-memory-buffer
            ByteBuffer newBuf = ?
                    ByteBuffer.allocateDirect(newBufSize):
                    ByteBuffer.allocate(newBufSize);
            final int buffersPos = (int) (offset >>> );
            ByteBuffer oldBuffer = buffers2[buffersPos];
            if(oldBuffer!=null){
                //copy old buffer if it exists
                oldBuffer = oldBuffer.duplicate();
                oldBuffer.rewind();
                newBuf.put(oldBuffer);
            }
            return newBuf;
        }
        @Override public void close() {
            .lock();
            try{
                for(ByteBuffer b:){
                    if(b!=null && (b instanceof MappedByteBuffer)){
                        unmap((MappedByteBuffer)b);
                    }
                }
                 = null;
            }finally{
                .unlock();
            }
        }
        @Override public void sync() {}
        @Override public void deleteFile() {}
        @Override
        public File getFile() {
            return null;
        }
    }


    
Volume which uses FileChannel. Uses global lock and does not use mapped memory.
    public static final class FileChannelVol extends Volume {
        protected final File file;
        protected FileChannel channel;
        protected final boolean readOnly;
        protected final long sizeLimit;
        protected final boolean hasLimit;
        protected volatile long size;
        protected Object growLock = new Object();
        public FileChannelVol(File fileboolean readOnly,long sizeLimit){
            this. = file;
            this. = readOnly;
            this. = sizeLimit;
            this. = sizeLimit>0;
            try {
                 = new RandomAccessFile(filereadOnly?"r":"rw").getChannel();
                 = .size();
            } catch (IOException e) {
                throw new IOError(e);
            }
        }
        @Override
        public boolean tryAvailable(long offset) {
            if( && offset>return false;
            if(offset>)synchronized (){
                try {
                    .truncate(offset);
                     = offset;
                } catch (IOException e) {
                    throw new IOError(e);
                }
            }
            return true;
        }
        protected void writeFully(long offsetByteBuffer bufthrows IOException {
            int remaining = buf.limit()-buf.position();
            while(remaining>0){
                int write = .write(bufoffset);
                if(write<0) throw new EOFException();
                remaining-=write;
            }
        }
        @Override
        public final void putSixLong(long offsetlong value) {
            if(value<0) throw new IllegalArgumentException();
            if(value >> (6*8)!=0)
                throw new IllegalArgumentException("does not fit");
            try{
                ByteBuffer buf = ByteBuffer.allocate(6);
                buf.put(0, (byte) (0xff & (value >> 40)));
                buf.put(1, (byte) (0xff & (value >> 32)));
                buf.put(2, (byte) (0xff & (value >> 24)));
                buf.put(3, (byte) (0xff & (value >> 16)));
                buf.put(4, (byte) (0xff & (value >> 8)));
                buf.put(5, (byte) (0xff & (value >> 0)));
                writeFully(offsetbuf);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public void putLong(long offsetlong value) {
            try{
                ByteBuffer buf = ByteBuffer.allocate(8);
                buf.putLong(0, value);
                writeFully(offsetbuf);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public void putInt(long offsetint value) {
            try{
                ByteBuffer buf = ByteBuffer.allocate(4);
                buf.putInt(0, value);
                writeFully(offsetbuf);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public void putByte(long offsetbyte value) {
            try{
                ByteBuffer buf = ByteBuffer.allocate(1);
                buf.put(0, value);
                writeFully(offsetbuf);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public void putData(long offsetbyte[] srcint srcPosint srcSize) {
            try{
                ByteBuffer buf = ByteBuffer.wrap(src,srcPossrcSize);
                writeFully(offsetbuf);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public void putData(long offsetByteBuffer buf) {
            try{
                writeFully(offset,buf);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        protected void readFully(long offsetByteBuffer bufthrows IOException {
            int remaining = buf.limit()-buf.position();
            while(remaining>0){
                int read = .read(bufoffset);
                if(read<0) throw new EOFException();
                remaining-=read;
            }
        }
        @Override
        public final long getSixLong(long offset) {
            try{
                ByteBuffer buf = ByteBuffer.allocate(6);
                readFully(offset,buf);
                return ((long) (buf.get(0) & 0xff) << 40) |
                        ((long) (buf.get(1) & 0xff) << 32) |
                        ((long) (buf.get(2) & 0xff) << 24) |
                        ((long) (buf.get(3) & 0xff) << 16) |
                        ((long) (buf.get(4) & 0xff) << 8) |
                        ((long) (buf.get(5) & 0xff) << 0);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public long getLong(long offset) {
            try{
                ByteBuffer buf = ByteBuffer.allocate(8);
                readFully(offset,buf);
                return buf.getLong(0);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public int getInt(long offset) {
            try{
                ByteBuffer buf = ByteBuffer.allocate(4);
                readFully(offset,buf);
                return buf.getInt(0);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public byte getByte(long offset) {
            try{
                ByteBuffer buf = ByteBuffer.allocate(1);
                readFully(offset,buf);
                return buf.get(0);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public DataInput2 getDataInput(long offsetint size) {
            try{
                ByteBuffer buf = ByteBuffer.allocate(size);
                readFully(offset,buf);
                return new DataInput2(buf,0);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public void close() {
            try{
                .close();
                 = null;
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public void sync() {
            try{
                .force(true);
            }catch(IOException e){
                throw new IOError(e);
            }
        }
        @Override
        public boolean isEmpty() {
            try {
                return ==null || .size()==0;
            } catch (IOException e) {
                throw new IOError(e);
            }
        }
        @Override
        public void deleteFile() {
            .delete();
        }
        @Override
        public boolean isSliced() {
            return false;
        }
        @Override
        public File getFile() {
            return ;
        }
    }
New to GrepCode? Check out our FAQ X