Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.facebook.presto.operator;
 
 
 import java.util.List;
 
 import static com.facebook.presto.operator.SyntheticAddress.decodePosition;
 import static com.facebook.presto.operator.SyntheticAddress.decodeSliceIndex;
 import static com.facebook.presto.operator.SyntheticAddress.encodeSyntheticAddress;
 import static com.facebook.presto.spi.type.BigintType.BIGINT;
 import static com.facebook.presto.sql.gen.JoinCompiler.LookupSourceFactory;
 import static com.facebook.presto.util.ImmutableCollectors.toImmutableList;
 import static com.google.common.base.MoreObjects.toStringHelper;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static io.airlift.slice.SizeOf.sizeOf;
 import static io.airlift.units.DataSize.Unit.BYTE;

PagesIndex a low-level data structure which contains the address of every value position of every channel. This data structure is not general purpose and is designed for a few specific uses:
 
 public class PagesIndex
         implements Swapper
 {
     private static final Logger log = Logger.get(PagesIndex.class);
 
     // todo this should be a services assigned in the constructor
     private static final OrderingCompiler orderingCompiler = new OrderingCompiler();
 
     // todo this should be a services assigned in the constructor
     private static final JoinCompiler joinCompiler = new JoinCompiler();
 
     private final List<Typetypes;
     private final LongArrayList valueAddresses;
     private final ObjectArrayList<Block>[] channels;
 
     private int nextBlockToCompact;
     private int positionCount;
     private long pagesMemorySize;
     private long estimatedSize;
 
     public PagesIndex(List<Typetypesint expectedPositions)
     {
         this. = ImmutableList.copyOf(checkNotNull(types"types is null"));
         this. = new LongArrayList(expectedPositions);
 
         //noinspection rawtypes
          = (ObjectArrayList<Block>[]) new ObjectArrayList[types.size()];
         for (int i = 0; i < .i++) {
             [i] = ObjectArrayList.wrap(new Block[1024], 0);
         }
     }
 
     public List<TypegetTypes()
     {
         return ;
     }
 
     public int getPositionCount()
     {
         return ;
     }
 
     public LongArrayList getValueAddresses()
     {
         return ;
    }
    public ObjectArrayList<BlockgetChannel(int channel)
    {
        return [channel];
    }
    public void clear()
    {
        for (ObjectArrayList<Blockchannel : ) {
            channel.clear();
        }
        .clear();
         = 0;
         = 0;
    }
    public void addPage(Page page)
    {
        // ignore empty pages
        if (page.getPositionCount() == 0) {
            return;
        }
         += page.getPositionCount();
        int pageIndex = (. > 0) ? [0].size() : 0;
        for (int i = 0; i < .i++) {
            Block block = page.getBlock(i);
            [i].add(block);
             += block.getRetainedSizeInBytes();
        }
        for (int position = 0; position < page.getPositionCount(); position++) {
            long sliceAddress = encodeSyntheticAddress(pageIndexposition);
            .add(sliceAddress);
        }
    }

    
Add positions in page with the specified partitionId. NOTE: this method does not track memory used in the page, since only part of the page will be used.
    public void addPage(Page pageint partitionIdBlock partitionIds)
    {
        // ignore empty pages
        if (page.getPositionCount() == 0) {
            return;
        }
         += page.getPositionCount();
        int pageIndex = (. > 0) ? [0].size() : 0;
        for (int i = 0; i < .i++) {
            Block block = page.getBlock(i);
            [i].add(block);
        }
        for (int position = 0; position < page.getPositionCount(); position++) {
            if (partitionId == .getLong(partitionIdsposition)) {
                long sliceAddress = encodeSyntheticAddress(pageIndexposition);
                .add(sliceAddress);
                ++;
            }
        }
    }
    public DataSize getEstimatedSize()
    {
        return new DataSize();
    }
    public void compact()
    {
        for (int channel = 0; channel < .size(); channel++) {
            Type type = .get(channel);
            ObjectArrayList<Blockblocks = [channel];
            for (int i = i < blocks.size(); i++) {
                Block block = blocks.get(i);
                if (block.getSizeInBytes() < block.getRetainedSizeInBytes()) {
                    // Copy the block to compact its size
                    Block compactedBlock = block.copyRegion(0, block.getPositionCount());
                    blocks.set(icompactedBlock);
                     -= block.getRetainedSizeInBytes();
                     += compactedBlock.getRetainedSizeInBytes();
                }
            }
        }
         = [0].size();
    }
    private long calculateEstimatedSize()
    {
        long elementsSize = (. > 0) ? sizeOf([0].elements()) : 0;
        long channelsArraySize = elementsSize * .;
        long addressesArraySize = sizeOf(.elements());
        return  + channelsArraySize + addressesArraySize;
    }
    public Type getType(int channel)
    {
        return .get(channel);
    }
    @Override
    public void swap(int aint b)
    {
        long[] elements = .elements();
        long temp = elements[a];
        elements[a] = elements[b];
        elements[b] = temp;
    }
    public int buildPage(int positionint[] outputChannelsPageBuilder pageBuilder)
    {
        while (!pageBuilder.isFull() && position < ) {
            long pageAddress = .getLong(position);
            int blockIndex = decodeSliceIndex(pageAddress);
            int blockPosition = decodePosition(pageAddress);
            // append the row
            pageBuilder.declarePosition();
            for (int i = 0; i < outputChannels.lengthi++) {
                int outputChannel = outputChannels[i];
                Type type = .get(outputChannel);
                Block block = this.[outputChannel].get(blockIndex);
                type.appendTo(blockblockPositionpageBuilder.getBlockBuilder(i));
            }
            position++;
        }
        return position;
    }
    public void appendTo(int channelint positionBlockBuilder output)
    {
        long pageAddress = .getLong(position);
        Type type = .get(channel);
        Block block = [channel].get(decodeSliceIndex(pageAddress));
        int blockPosition = decodePosition(pageAddress);
        type.appendTo(blockblockPositionoutput);
    }
    public boolean isNull(int channelint position)
    {
        long pageAddress = .getLong(position);
        Block block = [channel].get(decodeSliceIndex(pageAddress));
        int blockPosition = decodePosition(pageAddress);
        return block.isNull(blockPosition);
    }
    public boolean getBoolean(int channelint position)
    {
        long pageAddress = .getLong(position);
        Block block = [channel].get(decodeSliceIndex(pageAddress));
        int blockPosition = decodePosition(pageAddress);
        return .get(channel).getBoolean(blockblockPosition);
    }
    public long getLong(int channelint position)
    {
        long pageAddress = .getLong(position);
        Block block = [channel].get(decodeSliceIndex(pageAddress));
        int blockPosition = decodePosition(pageAddress);
        return .get(channel).getLong(blockblockPosition);
    }
    public double getDouble(int channelint position)
    {
        long pageAddress = .getLong(position);
        Block block = [channel].get(decodeSliceIndex(pageAddress));
        int blockPosition = decodePosition(pageAddress);
        return .get(channel).getDouble(blockblockPosition);
    }
    public Slice getSlice(int channelint position)
    {
        long pageAddress = .getLong(position);
        Block block = [channel].get(decodeSliceIndex(pageAddress));
        int blockPosition = decodePosition(pageAddress);
        return .get(channel).getSlice(blockblockPosition);
    }
    public void sort(List<IntegersortChannelsList<SortOrdersortOrders)
    {
        sort(sortChannelssortOrders, 0, getPositionCount());
    }
    public void sort(List<IntegersortChannelsList<SortOrdersortOrdersint startPositionint endPosition)
    {
        createPagesIndexComparator(sortChannelssortOrders).sort(thisstartPositionendPosition);
    }
    public boolean positionEqualsPosition(PagesHashStrategy partitionHashStrategyint leftPositionint rightPosition)
    {
        long leftAddress = .getLong(leftPosition);
        int leftPageIndex = decodeSliceIndex(leftAddress);
        int leftPagePosition = decodePosition(leftAddress);
        long rightAddress = .getLong(rightPosition);
        int rightPageIndex = decodeSliceIndex(rightAddress);
        int rightPagePosition = decodePosition(rightAddress);
        return partitionHashStrategy.positionEqualsPosition(leftPageIndexleftPagePositionrightPageIndexrightPagePosition);
    }
    public boolean positionEqualsRow(PagesHashStrategy pagesHashStrategyint indexPositionint rowPositionBlock... row)
    {
        long pageAddress = .getLong(indexPosition);
        int pageIndex = decodeSliceIndex(pageAddress);
        int pagePosition = decodePosition(pageAddress);
        return pagesHashStrategy.positionEqualsRow(pageIndexpagePositionrowPositionrow);
    }
    private PagesIndexOrdering createPagesIndexComparator(List<IntegersortChannelsList<SortOrdersortOrders)
    {
        List<TypesortTypes = sortChannels.stream()
                .map(::get)
                .collect(toImmutableList());
        return .compilePagesIndexOrdering(sortTypessortChannelssortOrders);
    }
    public LookupSource createLookupSource(List<IntegerjoinChannels)
    {
        return createLookupSource(joinChannels, Optional.empty());
    }
    public PagesHashStrategy createPagesHashStrategy(List<IntegerjoinChannelsOptional<IntegerhashChannel)
    {
        try {
            return .compilePagesHashStrategyFactory(joinChannels)
                    .createPagesHashStrategy(ImmutableList.copyOf(), hashChannel);
        }
        catch (Exception e) {
            .error(e"Lookup source compile failed for types=%s error=%s"e);
        }
        // if compilation fails, use interpreter
        return new SimplePagesHashStrategy(, ImmutableList.<List<Block>>copyOf(), joinChannelshashChannel);
    }
    public LookupSource createLookupSource(List<IntegerjoinChannelsOptional<IntegerhashChannel)
    {
        try {
            LookupSourceFactory lookupSourceFactory = .compileLookupSourceFactory(joinChannels);
            ImmutableList.Builder<TypejoinChannelTypes = ImmutableList.builder();
            for (Integer joinChannel : joinChannels) {
                joinChannelTypes.add(.get(joinChannel));
            }
            LookupSource lookupSource = lookupSourceFactory.createLookupSource(
                    ,
                    joinChannelTypes.build(),
                    ImmutableList.<List<Block>>copyOf(),
                    hashChannel);
            return lookupSource;
        }
        catch (Exception e) {
            .error(e"Lookup source compile failed for types=%s error=%s"e);
        }
        // if compilation fails
        PagesHashStrategy hashStrategy = new SimplePagesHashStrategy(
                ,
                ImmutableList.<List<Block>>copyOf(),
                joinChannels,
                hashChannel);
        ImmutableList.Builder<TypehashTypes = ImmutableList.builder();
        for (Integer channel : joinChannels) {
            hashTypes.add(.get(channel));
        }
        return new InMemoryJoinHash(hashTypes.build(), hashStrategy);
    }
    @Override
    public String toString()
    {
        return toStringHelper(this)
                .add("positionCount")
                .add("types")
                .add("estimatedSize")
                .toString();
    }
New to GrepCode? Check out our FAQ X