Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.facebook.presto.sql.planner.optimizations;
 
 
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes;
 import static com.facebook.presto.util.ImmutableCollectors.toImmutableList;
 import static com.facebook.presto.util.ImmutableCollectors.toImmutableSet;
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.util.stream.Collectors.toMap;
 
 {
     private PropertyDerivations() {}
 
     public static ActualProperties deriveProperties(PlanNode nodeActualProperties inputPropertiesMetadata metadataSession sessionMap<SymbolTypetypesSqlParser parser)
     {
         return deriveProperties(node, ImmutableList.of(inputProperties), metadatasessiontypesparser);
     }
 
     public static ActualProperties deriveProperties(PlanNode nodeList<ActualPropertiesinputPropertiesMetadata metadataSession sessionMap<SymbolTypetypesSqlParser parser)
     {
         return node.accept(new Visitor(metadatasessiontypesparser), inputProperties);
     }
 
     private static class Visitor
             extends PlanVisitor<List<ActualProperties>, ActualProperties>
     {
         private final Metadata metadata;
         private final Session session;
         private final Map<SymbolTypetypes;
         private final SqlParser parser;
 
         public Visitor(Metadata metadataSession sessionMap<SymbolTypetypesSqlParser parser)
         {
             this. = metadata;
            this. = session;
            this. = types;
            this. = parser;
        }
        @Override
        protected ActualProperties visitPlan(PlanNode nodeList<ActualPropertiesinputProperties)
        {
            throw new UnsupportedOperationException("not yet implemented: " + node.getClass().getName());
        }
        @Override
        public ActualProperties visitOutput(OutputNode nodeList<ActualPropertiesinputProperties)
        {
            return Iterables.getOnlyElement(inputProperties);
        }
        @Override
        public ActualProperties visitMarkDistinct(MarkDistinctNode nodeList<ActualPropertiesinputProperties)
        {
            return Iterables.getOnlyElement(inputProperties);
        }
        @Override
        public ActualProperties visitWindow(WindowNode nodeList<ActualPropertiesinputProperties)
        {
            // If the input is completely pre-partitioned and sorted, then the original input properties will be respected
            if (ImmutableSet.copyOf(node.getPartitionBy()).equals(node.getPrePartitionedInputs()) && node.getPreSortedOrderPrefix() == node.getOrderBy().size()) {
                return Iterables.getOnlyElement(inputProperties);
            }
            ImmutableList.Builder<LocalProperty<Symbol>> localProperties = ImmutableList.builder();
            if (!node.getPartitionBy().isEmpty()) {
                localProperties.add(new GroupingProperty<>(node.getPartitionBy()));
            }
            for (Symbol column : node.getOrderBy()) {
                localProperties.add(new SortingProperty<>(columnnode.getOrderings().get(column)));
            }
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            return ActualProperties.builder()
                    .partitioned(properties)
                    .coordinatorOnly(properties)
                    .local(localProperties.build())
                    .constants(properties)
                    .build();
        }
        @Override
        public ActualProperties visitAggregation(AggregationNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            if (!properties.isPartitioned()) {
                return ActualProperties.builder()
                        .unpartitioned()
                        .coordinatorOnly(properties)
                        .local(LocalProperties.grouped(node.getGroupBy()))
                        .constants(Maps.filterKeys(properties.getConstants(), ImmutableSet.of(node.getGroupBy())::contains))
                        .build();
            }
            return ActualProperties.builder()
                    .partitioned(properties)
                    .local(LocalProperties.grouped(node.getGroupBy()))
                    .constants(Maps.filterKeys(properties.getConstants(), ImmutableSet.of(node.getGroupBy())::contains))
                    .build();
        }
        @Override
        public ActualProperties visitRowNumber(RowNumberNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            return ActualProperties.builder()
                    .partitioned(properties)
                    .coordinatorOnly(properties)
                    .local(LocalProperties.grouped(node.getPartitionBy()))
                    .constants(properties)
                    .build();
        }
        @Override
        public ActualProperties visitTopNRowNumber(TopNRowNumberNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            ImmutableList.Builder<LocalProperty<Symbol>> localProperties = ImmutableList.builder();
            localProperties.add(new GroupingProperty<>(node.getPartitionBy()));
            for (Symbol column : node.getOrderBy()) {
                localProperties.add(new SortingProperty<>(columnnode.getOrderings().get(column)));
            }
            return ActualProperties.builder()
                    .partitioned(properties)
                    .coordinatorOnly(properties)
                    .local(localProperties.build())
                    .constants(properties)
                    .build();
        }
        @Override
        public ActualProperties visitTopN(TopNNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            List<SortingProperty<Symbol>> localProperties = node.getOrderBy().stream()
                    .map(column -> new SortingProperty<>(columnnode.getOrderings().get(column)))
                    .collect(toImmutableList());
            return ActualProperties.builder()
                    .partitioned(properties)
                    .coordinatorOnly(properties)
                    .local(localProperties)
                    .constants(properties)
                    .build();
        }
        @Override
        public ActualProperties visitSort(SortNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            List<SortingProperty<Symbol>> localProperties = node.getOrderBy().stream()
                    .map(column -> new SortingProperty<>(columnnode.getOrderings().get(column)))
                    .collect(toImmutableList());
            return ActualProperties.builder()
                    .partitioned(properties)
                    .coordinatorOnly(properties)
                    .local(localProperties)
                    .constants(properties)
                    .build();
        }
        @Override
        public ActualProperties visitLimit(LimitNode nodeList<ActualPropertiesinputProperties)
        {
            return Iterables.getOnlyElement(inputProperties);
        }
        @Override
        public ActualProperties visitDistinctLimit(DistinctLimitNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            return ActualProperties.builder()
                    .partitioned(properties)
                    .coordinatorOnly(properties)
                    .local(LocalProperties.grouped(node.getDistinctSymbols()))
                    .constants(properties)
                    .build();
        }
        @Override
        public ActualProperties visitTableCommit(TableCommitNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            return ActualProperties.builder()
                    .unpartitioned()
                    .coordinatorOnly(properties)
                    .build();
        }
        @Override
        public ActualProperties visitJoin(JoinNode nodeList<ActualPropertiesinputProperties)
        {
            // TODO: include all equivalent columns in partitioning properties
            // TODO: derive constants for right side
            return inputProperties.get(0);
        }
        @Override
        public ActualProperties visitSemiJoin(SemiJoinNode nodeList<ActualPropertiesinputProperties)
        {
            return inputProperties.get(0);
        }
        @Override
        public ActualProperties visitIndexJoin(IndexJoinNode nodeList<ActualPropertiesinputProperties)
        {
            return inputProperties.get(0);
        }
        @Override
        public ActualProperties visitExchange(ExchangeNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = inputProperties.get(0);
            switch (node.getType()) {
                case :
                    return ActualProperties.builder()
                            .unpartitioned()
                            .constants(properties)
                            .build();
                case :
                    return ActualProperties.builder()
                            .hashPartitioned(node.getPartitionKeys())
                            .constants(properties)
                            .build();
                case :
                    return ActualProperties.builder()
                            .partitioned(properties)
                            .constants(properties)
                            .build();
            }
            throw new UnsupportedOperationException("not yet implemented");
        }
        @Override
        public ActualProperties visitFilter(FilterNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            DomainTranslator.ExtractionResult decomposedPredicate = DomainTranslator.fromPredicate(
                    ,
                    ,
                    node.getPredicate(),
                    );
            Map<SymbolObjectconstants = new HashMap<>(properties.getConstants());
            constants.putAll(decomposedPredicate.getTupleDomain().extractFixedValues());
            return ActualProperties.builder()
                    .partitioned(properties)
                    .coordinatorOnly(properties)
                    .local(properties)
                    .constants(constants)
                    .build();
        }
        @Override
        public ActualProperties visitProject(ProjectNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            Map<SymbolSymbolidentities = computeIdentityTranslations(node.getAssignments());
            List<LocalProperty<Symbol>> localProperties = LocalProperties.translate(properties.getLocalProperties(), column -> Optional.ofNullable(identities.get(column)));
            Map<SymbolObjectconstants = new HashMap<>();
            for (Map.Entry<SymbolExpressionassignment : node.getAssignments().entrySet()) {
                Expression expression = assignment.getValue();
                IdentityHashMap<ExpressionTypeexpressionTypes = getExpressionTypes(expression);
                ExpressionInterpreter optimizer = ExpressionInterpreter.expressionOptimizer(expressionexpressionTypes);
                // TODO:
                // We want to use a symbol resolver that looks up in the constants from the input subplan
                // to take advantage of constant-folding for complex expressions
                // However, that currently causes errors when those expressions operate on arrays or row types
                // ("ROW comparison not supported for fields with null elements", etc)
                Object value = optimizer.optimize(.);
                if (value instanceof QualifiedNameReference) {
                    Symbol symbol = Symbol.fromQualifiedName(((QualifiedNameReferencevalue).getName());
                    value = constants.getOrDefault(symbolvalue);
                }
                // TODO: remove value null check when constants are supported
                if (value != null && !(value instanceof Expression)) {
                    constants.put(assignment.getKey(), value);
                }
            }
            properties.getConstants().entrySet().stream()
                    .filter(entry -> identities.containsKey(entry.getKey()))
                    .forEach(entry -> constants.put(identities.get(entry.getKey()), entry.getValue()));
            if (!properties.isPartitioned()) {
                return ActualProperties.builder()
                        .coordinatorOnly(properties)
                        .unpartitioned()
                        .local(localProperties)
                        .constants(constants)
                        .build();
            }
            if (properties.isHashPartitioned()) {
                Optional<List<Symbol>> translated = translate(properties.getHashPartitioningColumns().get(), identities);
                if (translated.isPresent()) {
                    return ActualProperties.builder()
                            .coordinatorOnly(properties)
                            .hashPartitioned(translated.get())
                            .local(localProperties)
                            .constants(constants)
                            .build();
                }
            }
            if (properties.hasKnownPartitioningScheme()) {
                Optional<List<Symbol>> translated = translate(properties.getPartitioningColumns().get(), identities);
                if (translated.isPresent()) {
                    return ActualProperties.builder()
                            .coordinatorOnly(properties)
                            .partitioned(ImmutableSet.copyOf(translated.get()))
                            .local(localProperties)
                            .constants(constants)
                            .build();
                }
            }
            return ActualProperties.builder()
                    .coordinatorOnly(properties)
                    .partitioned()
                    .local(localProperties)
                    .constants(constants)
                    .build();
        }
        @Override
        public ActualProperties visitTableWriter(TableWriterNode nodeList<ActualPropertiesinputProperties)
        {
            ActualProperties properties = Iterables.getOnlyElement(inputProperties);
            ActualProperties.Builder derived = ActualProperties.builder()
                    .coordinatorOnly(properties);
            if (properties.isPartitioned()) {
                derived.partitioned();
            }
            else {
                derived.unpartitioned();
            }
            return derived.build();
        }
        @Override
        public ActualProperties visitSample(SampleNode nodeList<ActualPropertiesinputProperties)
        {
            return Iterables.getOnlyElement(inputProperties);
        }
        @Override
        public ActualProperties visitUnnest(UnnestNode nodeList<ActualPropertiesinputProperties)
        {
            return Iterables.getOnlyElement(inputProperties);
        }
        @Override
        public ActualProperties visitTableScan(TableScanNode nodeList<ActualPropertiesinputProperties)
        {
            checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen");
            TableLayout layout = .getLayout(node.getLayout().get());
            Map<ColumnHandleSymbolassignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
            ActualProperties.Builder properties = ActualProperties.builder();
            // Constant assignments
            Map<ColumnHandleObjectconstants = new HashMap<>();
            LocalProperties.extractLeadingConstants(layout.getLocalProperties()).stream()
                    .forEach(column -> constants.put(columnnew Object())); // Use an arbitrary object value for property constants b/c we don't know its actual value
            // Do predicate constants after property constants so that we can override with known real predicate values (if they exist)
            node.getCurrentConstraint().extractFixedValues().entrySet().stream()
                    .forEach(entry -> constants.put(entry.getKey(), entry.getValue()));
            Map<SymbolObjectsymbolConstants = constants.entrySet().stream()
                    .filter(entry -> assignments.containsKey(entry.getKey()))
                    .collect(toMap(entry -> assignments.get(entry.getKey()), Map.Entry::getValue));
            properties.constants(symbolConstants);
            // Partitioning properties
            Optional<List<Symbol>> partitioningColumns = Optional.empty();
            if (layout.getPartitioningColumns().isPresent()) {
                // Strip off the constants from the partitioning columns (since those are not required for translation)
                Set<ColumnHandleconstantsStrippedPartitionColumns = layout.getPartitioningColumns().get().stream()
                        .filter(column -> !constants.containsKey(column))
                        .collect(toImmutableSet());
                partitioningColumns = translate(constantsStrippedPartitionColumnsassignments);
            }
            if (partitioningColumns.isPresent()) {
                properties.partitioned(ImmutableSet.copyOf(partitioningColumns.get()));
            }
            else {
                properties.partitioned();
            }
            // Append the constants onto the local properties to maximize their translation potential
            List<LocalProperty<ColumnHandle>> constantAppendedLocalProperties = ImmutableList.<LocalProperty<ColumnHandle>>builder()
                    .addAll(constants.keySet().stream().map(column -> new ConstantProperty<>(column)).iterator())
                    .addAll(layout.getLocalProperties())
                    .build();
            properties.local(LocalProperties.translate(constantAppendedLocalPropertiescolumn -> Optional.ofNullable(assignments.get(column))));
            return properties.build();
        }
        private static Map<SymbolSymbolcomputeIdentityTranslations(Map<SymbolExpressionassignments)
        {
            Map<SymbolSymbolinputToOutput = new HashMap<>();
            for (Map.Entry<SymbolExpressionassignment : assignments.entrySet()) {
                if (assignment.getValue() instanceof QualifiedNameReference) {
                    inputToOutput.put(Symbol.fromQualifiedName(((QualifiedNameReferenceassignment.getValue()).getName()), assignment.getKey());
                }
            }
            return inputToOutput;
        }

        

Returns:
Optional.empty() if not all columns could be translated
        private static <T> Optional<List<Symbol>> translate(Collection<T> columnsMap<T, Symbolmappings)
        {
            ImmutableList.Builder<Symbolbuilder = ImmutableList.builder();
            for (T column : columns) {
                Symbol translated = mappings.get(column);
                if (translated == null) {
                    return Optional.empty();
                }
                builder.add(translated);
            }
            return Optional.of(builder.build());
        }
    }
New to GrepCode? Check out our FAQ X