Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.facebook.presto.operator;
 
 
 import java.util.List;
 
 import static com.facebook.presto.spi.type.BigintType.BIGINT;
 import static it.unimi.dsi.fastutil.HashCommon.arraySize;
 
 @SuppressWarnings("MethodMayBeStatic")
 @Fork(2)
 @Warmup(iterations = 10, time = 500, timeUnit = .)
 @Measurement(iterations = 10, time = 500, timeUnit = .)
 public class BenchmarkGroupByHash
 {
     private static final int POSITIONS = 10_000_000;
     private static final String GROUP_COUNT_STRING = "3000000";
     private static final int GROUP_COUNT = Integer.parseInt();
     private static final int EXPECTED_SIZE = 10_000;
 
     @Benchmark
     public Object groupByHashPreCompute(BenchmarkData data)
     {
         GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), Optional.empty(), data.getHashChannel(), );
         data.getPages().forEach(groupByHash::getGroupIds);
 
         ImmutableList.Builder<Pagepages = ImmutableList.builder();
         PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
         for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) {
             pageBuilder.declarePosition();
             groupByHash.appendValuesTo(groupIdpageBuilder, 0);
             if (pageBuilder.isFull()) {
                 pages.add(pageBuilder.build());
                 pageBuilder.reset();
             }
         }
         pages.add(pageBuilder.build());
         return pageBuilder.build();
     }
 
     @Benchmark
     public Object addPagePreCompute(BenchmarkData data)
     {
         GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), Optional.empty(), data.getHashChannel(), );
         data.getPages().forEach(groupByHash::addPage);
 
         ImmutableList.Builder<Pagepages = ImmutableList.builder();
         PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
         for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) {
             pageBuilder.declarePosition();
             groupByHash.appendValuesTo(groupIdpageBuilder, 0);
            if (pageBuilder.isFull()) {
                pages.add(pageBuilder.build());
                pageBuilder.reset();
            }
        }
        pages.add(pageBuilder.build());
        return pageBuilder.build();
    }
    @Benchmark
    {
        GroupByHash groupByHash = new BigintGroupByHash(0, Optional.empty(), data.getHashEnabled(), );
        data.getPages().forEach(groupByHash::addPage);
        ImmutableList.Builder<Pagepages = ImmutableList.builder();
        PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
        for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) {
            pageBuilder.declarePosition();
            groupByHash.appendValuesTo(groupIdpageBuilder, 0);
            if (pageBuilder.isFull()) {
                pages.add(pageBuilder.build());
                pageBuilder.reset();
            }
        }
        pages.add(pageBuilder.build());
        return pageBuilder.build();
    }
    @Benchmark
    public long baseline(BaselinePagesData data)
    {
        int hashSize = arraySize(, 0.9f);
        int mask = hashSize - 1;
        long[] table = new long[hashSize];
        Arrays.fill(table, -1);
        long groupIds = 0;
        for (Page page : data.getPages()) {
            Block block = page.getBlocks()[0];
            int positionCount = block.getPositionCount();
            for (int position = 0; position < positionCountposition++) {
                long value = block.getLong(position, 0);
                int tablePosition = (int) (value & mask);
                while (table[tablePosition] != -1 && table[tablePosition] != value) {
                    tablePosition++;
                }
                if (table[tablePosition] == -1) {
                    table[tablePosition] = value;
                    groupIds++;
                }
            }
        }
        return groupIds;
    }
    @Benchmark
    public long baselineBigArray(BaselinePagesData data)
    {
        int hashSize = arraySize(, 0.9f);
        int mask = hashSize - 1;
        LongBigArray table = new LongBigArray(-1);
        table.ensureCapacity(hashSize);
        long groupIds = 0;
        for (Page page : data.getPages()) {
            Block block = page.getBlocks()[0];
            int positionCount = block.getPositionCount();
            for (int position = 0; position < positionCountposition++) {
                long value = .getLong(blockposition);
                int tablePosition = (int) XxHash64.hash(value) & mask;
                while (table.get(tablePosition) != -1 && table.get(tablePosition) != value) {
                    tablePosition++;
                }
                if (table.get(tablePosition) == -1) {
                    table.set(tablePositionvalue);
                    groupIds++;
                }
            }
        }
        return groupIds;
    }
    private static List<PagecreatePages(int positionCountint groupCountList<Typetypesboolean hashEnabled)
    {
        int channelCount = types.size();
        ImmutableList.Builder<Pagepages = ImmutableList.builder();
        if (hashEnabled) {
            types = ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of()));
        }
        PageBuilder pageBuilder = new PageBuilder(types);
        for (int position = 0; position < positionCountposition++) {
            int rand = ThreadLocalRandom.current().nextInt(groupCount);
            pageBuilder.declarePosition();
            for (int numChannel = 0; numChannel < channelCountnumChannel++) {
                .writeLong(pageBuilder.getBlockBuilder(numChannel), rand);
            }
            if (hashEnabled) {
                .writeLong(pageBuilder.getBlockBuilder(channelCount), BigintOperators.hashCode(rand));
            }
            if (pageBuilder.isFull()) {
                pages.add(pageBuilder.build());
                pageBuilder.reset();
            }
        }
        pages.add(pageBuilder.build());
        return pages.build();
    }
    @SuppressWarnings("FieldMayBeFinal")
    public static class BaselinePagesData
    {
        @Param("1")
        private int channelCount = 1;
        @Param("false")
        private boolean hashEnabled;
        @Param()
        private int groupCount;
        private List<Pagepages;
        @Setup
        public void setup()
        {
             = createPages(, ImmutableList.of(), );
        }
        public List<PagegetPages()
        {
            return ;
        }
    }
    @SuppressWarnings("FieldMayBeFinal")
    public static class SingleChannelBenchmarkData
    {
        @Param("1")
        private int channelCount = 1;
        @Param({"true""false"})
        private boolean hashEnabled = true;
        @Param()
        private int groupCount = ;
        private List<Pagepages;
        private List<Typetypes;
        private int[] channels;
        @Setup
        public void setup()
        {
             = createPages(, ImmutableList.of(), );
             = Collections.<Type>nCopies(1, );
             = new int[1];
            for (int i = 0; i < 1; i++) {
                [i] = i;
            }
        }
        public List<PagegetPages()
        {
            return ;
        }
        public List<TypegetTypes()
        {
            return ;
        }
        public boolean getHashEnabled()
        {
            return ;
        }
    }
    @SuppressWarnings("FieldMayBeFinal")
    public static class BenchmarkData
    {
        @Param({ "1""5""10""15""20" })
        private int channelCount = 1;
        // todo add more group counts when JMH support programmatic ability to set OperationsPerInvocation
        @Param()
        private int groupCount = ;
        @Param({"true""false"})
        private boolean hashEnabled;
        private List<Pagepages;
        private Optional<IntegerhashChannel;
        private List<Typetypes;
        private int[] channels;
        @Setup
        public void setup()
        {
             = createPages(, Collections.<Type>nCopies(), );
             =  ? Optional.of() : Optional.empty();
             = Collections.<Type>nCopies();
             = new int[];
            for (int i = 0; i < i++) {
                [i] = i;
            }
        }
        public List<PagegetPages()
        {
            return ;
        }
        public Optional<IntegergetHashChannel()
        {
            return ;
        }
        public List<TypegetTypes()
        {
            return ;
        }
        public int[] getChannels()
        {
            return ;
        }
    }
    public static void main(String[] args)
            throws RunnerException
    {
        // assure the benchmarks are valid before running
        BenchmarkData data = new BenchmarkData();
        data.setup();
        new BenchmarkGroupByHash().groupByHashPreCompute(data);
        new BenchmarkGroupByHash().addPagePreCompute(data);
        SingleChannelBenchmarkData singleChannelBenchmarkData = new SingleChannelBenchmarkData();
        singleChannelBenchmarkData.setup();
        new BenchmarkGroupByHash().bigintGroupByHash(singleChannelBenchmarkData);
        Options options = new OptionsBuilder()
                .verbosity(.)
                .include(".*" + BenchmarkGroupByHash.class.getSimpleName() + ".*")
                .build();
        new Runner(options).run();
    }
New to GrepCode? Check out our FAQ X