Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   * Licensed to the Apache Software Foundation (ASF) under one
   * or more contributor license agreements.  See the NOTICE file
   * distributed with this work for additional information
   * regarding copyright ownership.  The ASF licenses this file
   * to you under the Apache License, Version 2.0 (the
   * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.pig.newplan.logical.rules;
 
 import java.util.List;
 
This Rule moves Filter Above Foreach. It checks if uid on which filter works on is present in the predecessor of foreach. If so it transforms it.
 
 public class FilterAboveForeach extends Rule {
 
     public FilterAboveForeach(String n) {
         super(nfalse);
     }
 
     @Override
     protected OperatorPlan buildPattern() {
         // the pattern that this rule looks for
         // is foreach -> filter
         LogicalPlan plan = new LogicalPlan();
         LogicalRelationalOperator foreach = new LOForEach(plan);
         LogicalRelationalOperator filter = new LOFilter(plan);
 
         plan.add(foreach);
         plan.add(filter);
         plan.connect(foreachfilter);
 
         return plan;
     }
 
     @Override
     public Transformer getNewTransformer() {
         return new FilterAboveForEachTransformer();
     }
 
     public class FilterAboveForEachTransformer extends Transformer {
 
         LOFilter filter = null;
         LOForEach foreach = null;
         LogicalRelationalOperator forEachPred = null;
         OperatorSubPlan subPlan = null;
 
         @Override
         public boolean check(OperatorPlan matchedthrows FrontendException {
             Iterator<Operatoriter = matched.getOperators();
             whileiter.hasNext() ) {
                 Operator op = iter.next();
                 ifop instanceof LOForEach ) {
                      = (LOForEach)op;
                     break;
                 }
             }
 
             // This would be a strange case
             if == null ) return false;
 
             iter = matched.getOperators();
             whileiter.hasNext() ) {
                 Operator op = iter.next();
                 if( ( op instanceof LOFilter ) ) {
                      = (LOFilter)op;
                     break;
                 }
             }
            // This is for cheating, we look up more than one filter in the plan
            while != null ) {
                // Get uids of Filter
                Pair<List<Long>, List<Byte>> uidWithTypes = getFilterProjectionUids();
                // See if the previous operators have uids from project
                List<Operatorpreds = .getPredecessors();
                for(int j=0; jpreds.size(); j++) {
                    LogicalRelationalOperator logRelOp = (LogicalRelationalOperator)preds.get(j);
                    if (hasAll(logRelOpuidWithTypes)) {
                         = (LogicalRelationalOperatorpreds.get(j);
                        // If a filter is nondeterministic, we shouldn't push it up.
                        return !OptimizerUtils.planHasNonDeterministicUdf(.getFilterPlan());
                    }
                }
                // Chances are there are filters below this filter which can be
                // moved up. So searching for those filters
                List<Operatorsuccessors = .getSuccessors();
                ifsuccessors != null && successors.size() > 0 &&
                        successors.get(0) instanceof LOFilter ) {
                     = (LOFilter)successors.get(0);
                } else {
                     = null;
                }
            }
            return false;
        }

        
Get all uids from Projections of this FilterOperator

Parameters:
filter
Returns:
Set of uid
        private Pair<List<Long>, List<Byte>> getFilterProjectionUids(LOFilter filterthrows FrontendException {
            List<Longuids = new ArrayList<Long>();
            List<Bytetypes = new ArrayList<Byte>();
            iffilter != null ) {
                LogicalExpressionPlan filterPlan = filter.getFilterPlan();
                Iterator<Operatoriter = filterPlan.getOperators();
                Operator op = null;
                whileiter.hasNext() ) {
                    op = iter.next();
                    ifop instanceof ProjectExpression ) {
                        ProjectExpression proj = (ProjectExpression)op;
                        ifproj.isProjectStar() ) {
                            //project-range is always expanded when schema is
                            //available, so nothing to do here for it
                            LogicalRelationalOperator pred = (LogicalRelationalOperator)filter.getPlan().getPredecessors(filter).get(0);
                            LogicalSchema predSchema = pred.getSchema();
                            if (predSchema!=null) {
                                for (int i=0;i<predSchema.size();i++) {
                                    uids.add(predSchema.getField(i).);
                                    types.add(predSchema.getField(i).);
                                }
                            }
                        } else {
                            uids.add(proj.getFieldSchema().);
                            types.add(proj.getFieldSchema().);
                        }
                    }
                }
            }
            Pair<List<Long>, List<Byte>> result = new Pair<List<Long>, List<Byte>>(uidstypes);
            return result;
        }

        
checks if a relational operator contains all of the specified uids

Parameters:
op LogicalRelational operator that should contain the uid
uids Uids to check for
Returns:
true if given LogicalRelationalOperator has all the given uids
        private boolean hasAll(LogicalRelationalOperator opPair<List<Long>,
                List<Byte>> uidWithTypesthrows FrontendException {
            LogicalSchema schema = op.getSchema();
            if (schema==null)
                return false;
            List<Longuids = uidWithTypes.first;
            List<Bytetypes = uidWithTypes.second;
            for (int i=0;i<uids.size();i++) {
                boolean found = false;
                for (LogicalSchema.LogicalFieldSchema fs : schema.getFields()) {
                    if (fs.uid==uids.get(i) && fs.type==types.get(i))
                        found = true;
                }
                if (!found)
                    return false;
            }
            return true;
        }
        @Override
        public OperatorPlan reportChanges() {
            return ;
        }
        @Override
        public void transform(OperatorPlan matchedthrows FrontendException {
            List<OperatoropSet = .getPredecessors();
            if( ! ( opSet != null && opSet.size() > 0 ) ) {
                return;
            }
            Operator filterPred = opSet.get(0);
            opSet = .getSuccessors();
            if( ! ( opSet != null && opSet.size() > 0 ) ) {
                return;
            }
            Operator filterSuc = opSet.get(0);
             = new OperatorSubPlan();
            // Steps below do the following
            /*
             *          ForEachPred
             *               |
             *            ForEach
             *               |
             *             Filter*
             *      ( These are filters
             *      which cannot be moved )
             *               |
             *           FilterPred
             *         ( is a Filter )
             *               |
             *             Filter
             *        ( To be moved )
             *               |
             *            FilterSuc
             *
             *               |
             *               |
             *        Transforms into
             *               |
             *              \/
             *
             *            ForEachPred
             *               |
             *            Filter
             *     ( After being Moved )
             *               |
             *            ForEach
             *               |
             *             Filter*
             *       ( These are filters
             *      which cannot be moved )
             *               |
             *           FilterPred
             *         ( is a Filter )
             *               |
             *            FilterSuc
             *
             *  Above plan is assuming we are modifying the filter in middle.
             *  If we are modifying the first filter after ForEach then
             *  -- * (kleene star) becomes zero
             *  -- And ForEach is FilterPred
             */
            Pair<IntegerIntegerforEachPredPlaces = .disconnect();
            Pair<IntegerIntegerfilterPredPlaces = .disconnect(filterPred);
            Pair<IntegerIntegerfilterSucPlaces = .disconnect(filterSuc);
            .connect(forEachPredPlaces.firstfilterPredPlaces.second);
            .connect(filterSucPlaces.firstforEachPredPlaces.second);
            .connect(filterPredfilterPredPlaces.firstfilterSucfilterSucPlaces.second);
            .add();
            .add();
            .add(filterPred);
            .add();
            .add(filterSuc);
        }
    }
New to GrepCode? Check out our FAQ X