Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   * Licensed to the Apache Software Foundation (ASF) under one
   * or more contributor license agreements.  See the NOTICE file
   * distributed with this work for additional information
   * regarding copyright ownership.  The ASF licenses this file
   * to you under the Apache License, Version 2.0 (the
   * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators;
 
 
The load operator which is used in two ways: 1) As a local operator it can be used to load files 2) In the Map Reduce setting, it is used to create jobs from MapReduce operators which keep the loads and stores in the Map and Reduce Plans till the job is created
 
 public class POLoad extends PhysicalOperator {
     private static final Log log = LogFactory.getLog(POLoad.class);
    
 
     private static final long serialVersionUID = 1L;
     // The user defined load function or a default load function
     private transient LoadFunc loader = null;
     // The filespec on which the operator is based
     FileSpec lFile;
     // PigContext passed to us by the operator creator
     PigContext pc;
     //Indicates whether the loader setup is done or not
     boolean setUpDone = false;
     // Alias for the POLoad
     private String signature;
     // flag to distinguish user loads from MRCompiler loads.
     private boolean isTmpLoad;
     
     private long limit=-1;
     
     public POLoad(OperatorKey k) {
         this(k,-1, null);
     }
 
     public POLoad(OperatorKey kFileSpec lFile){
         this(k,-1,lFile);
     }
     
     public POLoad(OperatorKey kint rpFileSpec lFile) {
         super(krp);
         this. = lFile;
     }
     
     public POLoad(OperatorKey kLoadFunc lf){
         this(k);
         this. = lf;
     }
    
    
Set up the loader by 1) Instantiating the load func 2) Opening an input stream to the specified file and 3) Binding to the input stream at the specified offset.

 
     public void setUp() throws IOException{
          = new ReadToEndLoader((LoadFunc)
                 PigContext.instantiateFuncFromSpec(.getFuncSpec()), 
                 ConfigurationUtil.toConfiguration(.getProperties()), 
                 .getFileName(),0, );
     }
    
    
At the end of processing, the inputstream is closed using this method

    public void tearDown() throws IOException{
         = false;
    }
    
    
The main method used by this operator's successor to read tuples from the specified file using the specified load function.

Returns:
Whatever the loader returns A null from the loader is indicative of EOP and hence the tearDown of connection
    @Override
    public Result getNextTuple() throws ExecException {
        if(! && !=null){
            try {
                setUp();
            } catch (IOException ioe) {
                int errCode = 2081;
                String msg = "Unable to setup the load function.";
                throw new ExecException(msgerrCode.ioe);
            }
             = true;
        }
        Result res = new Result();
        try {
            res.result = .getNext();
            if(res.result==null){
                res.returnStatus = .;
                tearDown();
            }
            else
                res.returnStatus = .;
            if (res.returnStatus == .)
                res.result = illustratorMarkup(resres.result, 0);
        } catch (IOException e) {
            .error("Received error from loader function: " + e);
            return res;
        }
        return res;
    }
    @Override
    public String name() {
        return ( != null) ? getAliasString() + "Load" + "(" + .toString()
                + ")" + " - " + .toString() : getAliasString() + "Load" + "("
                + "DummyFil:DummyLdr" + ")" + " - " + .toString();
    }
    @Override
    public boolean supportsMultipleInputs() {
        return false;
    }
    @Override
    public boolean supportsMultipleOutputs() {
        return false;
    }
    @Override
    public void visit(PhyPlanVisitor vthrows VisitorException {
        v.visitLoad(this);
    }
    public FileSpec getLFile() {
        return ;
    }
    public void setLFile(FileSpec file) {
         = file;
    }
    public void setIsTmpLoad(boolean tmp) {
         = tmp;
    }
    public boolean isTmpLoad() {
        return ;
    }
    public PigContext getPc() {
        return ;
    }
    public void setPc(PigContext pc) {
        this. = pc;
    }
    public String getSignature() {
        return ;
    }
    
    public void setSignature(String signature) {
        this. = signature;
    }
    
    public LoadFunc getLoadFunc(){
        if (this.==null) {
            this. = (LoadFunc)PigContext.instantiateFuncFromSpec(.getFuncSpec());
            this..setUDFContextSignature();
        }
        return this.;
    }
    
    public Tuple illustratorMarkup(Object inObject outint eqClassIndex) {
        if( != null) {
          if (!.ceilingCheck()) {
              ((Resultin). = .;
              return null;
          }
          if (.getSchema() == null || .getSchema().size() <= ((Tupleout).size()) {
              boolean hasNull = false;
              for (int i = 0; i < ((Tupleout).size(); ++i)
                  try {
                      if (((Tupleout).get(i) == null)
                      {
                          hasNull = true;
                          break;
                      }
                  } catch (ExecException e) {
                      hasNull = true;
                      break;
                  }
              if (!hasNull) {
                  ExampleTuple tOut = new ExampleTuple((Tupleout);
                  .getLineage().insert(tOut);
                  .addData((TupletOut);
                  .getEquivalenceClasses().get(eqClassIndex).add(tOut);
                  return tOut;
              } else
                  return (Tupleout;
          } else
              return (Tupleout;
        } else
          return (Tupleout;
    }
    public long getLimit() {
        return ;
    }
    public void setLimit(long limit) {
        this. = limit;
    }
New to GrepCode? Check out our FAQ X