Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   * Copyright 2010-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
   *
   * Licensed under the Apache License, Version 2.0 (the "License").
   * You may not use this file except in compliance with the License.
   * A copy of the License is located at
   *
   *  http://aws.amazon.com/apache2.0
   *
  * or in the "license" file accompanying this file. This file is distributed
  * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
  * express or implied. See the License for the specific language governing
  * permissions and limitations under the License.
  */
 package com.amazonaws.services.s3.transfer.internal;
 
 import static com.amazonaws.event.SDKProgressPublisher.publishProgress;
 
 import java.util.List;
 import java.util.Map;
 
 
 
 public class UploadCallable implements Callable<UploadResult> {
     private final AmazonS3 s3;
     private final ExecutorService threadPool;
     private final PutObjectRequest origReq;
     private String multipartUploadId;
     private final UploadImpl upload;
 
     private static final Log log = LogFactory.getLog(UploadCallable.class);
     private final TransferManagerConfiguration configuration;
     private final List<Future<PartETag>> futures = new ArrayList<Future<PartETag>>();
     private final ProgressListenerChain listener;
     private final TransferProgress transferProgress;

    
ETags retrieved from Amazon S3 for a multi-part upload id. These parts will be skipped while resuming a paused upload.
 
     private final List<PartETageTagsToSkip = new ArrayList<PartETag>();
 
 
     public UploadCallable(TransferManager transferManager,
             ExecutorService threadPoolUploadImpl upload,
             PutObjectRequest origReq,
             ProgressListenerChain progressListenerChainString uploadId,
             TransferProgress transferProgress) {
         this. = transferManager.getAmazonS3Client();
         this. = transferManager.getConfiguration();
 
         this. = threadPool;
         this. = origReq;
         this. = progressListenerChain;
         this. = upload;
         this. = uploadId;
         this. = transferProgress;
     }
 
     List<Future<PartETag>> getFutures() {
         return ;
     }

    
Returns the ETags retrieved from Amazon S3 for a multi-part upload id. These parts will be skipped while resuming a paused upload.
    List<PartETaggetETags() {
        return ;
    }
        return ;
    }

    
Returns true if this UploadCallable is processing a multipart upload.

Returns:
True if this UploadCallable is processing a multipart upload.
    public boolean isMultipartUpload() {
        return TransferManagerUtils.shouldUseMultipartUpload();
    }
    public UploadResult call() throws Exception {
        if ( isMultipartUpload() ) {
            return uploadInParts();
        } else {
            return uploadInOneChunk();
        }
    }

    
Uploads the given request in a single chunk and returns the result.
    private UploadResult uploadInOneChunk() {
        PutObjectResult putObjectResult = .putObject();
        UploadResult uploadResult = new UploadResult();
        uploadResult.setBucketName(.getBucketName());
        uploadResult.setKey(.getKey());
        uploadResult.setETag(putObjectResult.getETag());
        uploadResult.setVersionId(putObjectResult.getVersionId());
        return uploadResult;
    }

    
Captures the state of the upload.
    private void captureUploadStateIfPossible() {
        if (.getSSECustomerKey() == null) {
             = new PersistableUpload(.getBucketName(),
                    .getKey(), .getFile()
                            .getAbsolutePath(), ,
                    .getMinimumUploadPartSize(),
                    .getMultipartUploadThreshold());
            notifyPersistableTransferAvailability();
        }
    }
        return ;
    }

    
Notifies to the callbacks that state is available
    private void notifyPersistableTransferAvailability() {
        S3ProgressPublisher.publishTransferPersistable(
                );
    }

    
Uploads the request in multiple chunks, submitting each upload chunk task to the thread pool and recording its corresponding Future object, as well as the multipart upload id.
    private UploadResult uploadInParts() throws Exception {
        boolean isUsingEncryption =  instanceof AmazonS3Encryption;
        long optimalPartSize = getOptimalPartSize(isUsingEncryption);
        if ( == null) {
                    isUsingEncryption);
        }
        try {
            UploadPartRequestFactory requestFactory = new UploadPartRequestFactory(optimalPartSize);
            if (TransferManagerUtils.isUploadParallelizable(isUsingEncryption)) {
                captureUploadStateIfPossible();
                uploadPartsInParallel(requestFactory);
                return null;
            } else {
                return uploadPartsInSeries(requestFactory);
            }
        } catch (Exception e) {
            performAbortMultipartUpload();
            throw e;
        } finally {
            if (.getInputStream() != null) {
                try {.getInputStream().close(); } catch (Exception e) {
                    .warn("Unable to cleanly close input stream: " + e.getMessage(), e);
                }
            }
        }
    }

    
        try {
            if ( != null)
                .abortMultipartUpload(new AbortMultipartUploadRequest(
                        .getBucketName(), 
                                .getKey(), ));
        } catch (Exception e2) {
            .info(
                    "Unable to abort multipart upload, you may need to manually remove uploaded parts: "
                            + e2.getMessage(), e2);
        }
    }

    
Computes and returns the optimal part size for the upload.
    private long getOptimalPartSize(boolean isUsingEncryption) {
        long optimalPartSize = TransferManagerUtils.calculateOptimalPartSize();
        if (isUsingEncryption && optimalPartSize % 32 > 0) {
            // When using encryption, parts must line up correctly along cipher block boundaries
            optimalPartSize = optimalPartSize - (optimalPartSize % 32) + 32 ;
        }
        .debug("Calculated optimal part size: " + optimalPartSize);
        return optimalPartSize;
    }

    
Uploads all parts in the request in serial in this thread, then completes the upload and returns the result.
    private UploadResult uploadPartsInSeries(UploadPartRequestFactory requestFactory) {
        final List<PartETagpartETags = new ArrayList<PartETag>();
        while (requestFactory.hasMoreRequests()) {
            if (.isShutdown()) throw new CancellationException("TransferManager has been shutdown");
            UploadPartRequest uploadPartRequest = requestFactory.getNextUploadPartRequest();
            // Mark the stream in case we need to reset it
            InputStream inputStream = uploadPartRequest.getInputStream();
            if (inputStream != null && inputStream.markSupported()) {
                if (uploadPartRequest.getPartSize() >= .) {
                    inputStream.mark(.);
                } else {
                    inputStream.mark((int)uploadPartRequest.getPartSize());
                }
            }
            partETags.add(.uploadPart(uploadPartRequest).getPartETag());
        }
        CompleteMultipartUploadRequest req =
            new CompleteMultipartUploadRequest(
                .getBucketName(), .getKey(), ,
                    partETags)
            ;
        UploadResult uploadResult = new UploadResult();
        uploadResult.setBucketName(res.getBucketName());
        uploadResult.setKey(res.getKey());
        uploadResult.setETag(res.getETag());
        uploadResult.setVersionId(res.getVersionId());
        return uploadResult;
    }

    
Submits a callable for each part to upload to our thread pool and records its corresponding Future.
    private void uploadPartsInParallel(UploadPartRequestFactory requestFactory,
            String uploadId) {
        Map<Integer,PartSummarypartNumbers = identifyExistingPartsForResume(uploadId);
        while (requestFactory.hasMoreRequests()) {
            if (.isShutdown()) throw new CancellationException("TransferManager has been shutdown");
            UploadPartRequest request = requestFactory.getNextUploadPartRequest();
            if (partNumbers.containsKey(request.getPartNumber())) {
                PartSummary summary = partNumbers.get(request.getPartNumber());
                .add(new PartETag(request.getPartNumber(), summary
                        .getETag()));
                .updateProgress(summary.getSize());
                continue;
            }
            .add(.submit(new UploadPartCallable(request)));
        }
    }
            String uploadId) {
        Map<IntegerPartSummarypartNumbers = new HashMap<IntegerPartSummary>();
        if (uploadId == null) {
            return partNumbers;
        }
        int partNumber = 0;
        while (true) {
            PartListing parts = .listParts(new ListPartsRequest(
                    .getBucketName(),
                    .getKey(), uploadId)
                    .withPartNumberMarker(partNumber));
            for (PartSummary partSummary : parts.getParts()) {
                partNumbers.put(partSummary.getPartNumber(), partSummary);
            }
            if (!parts.isTruncated()) {
                return partNumbers;
            }
            partNumber = parts.getNextPartNumberMarker();
        }
    }

    
Initiates a multipart upload and returns the upload id

Parameters:
isUsingEncryption
    private String initiateMultipartUpload(PutObjectRequest origReqboolean isUsingEncryption) {
        InitiateMultipartUploadRequest req = null;
        if (isUsingEncryption && origReq instanceof EncryptedPutObjectRequest) {
            req = new EncryptedInitiateMultipartUploadRequest(
                    origReq.getBucketName(), origReq.getKey()).withCannedACL(
                    origReq.getCannedAcl()).withObjectMetadata(origReq.getMetadata());
            ((EncryptedInitiateMultipartUploadRequestreq)
                    .setMaterialsDescription(((EncryptedPutObjectRequestorigReq).getMaterialsDescription());
        } else {
            req = new InitiateMultipartUploadRequest(origReq.getBucketName(), origReq.getKey())
                .withCannedACL(origReq.getCannedAcl())
                .withObjectMetadata(origReq.getMetadata());
        }
        TransferManager.appendMultipartUserAgent(req);
        req.withAccessControlList(origReq.getAccessControlList())
           .withStorageClass(origReq.getStorageClass())
           .withRedirectLocation(origReq.getRedirectLocation())
           .withSSECustomerKey(origReq.getSSECustomerKey())
           ;
        String uploadId = .initiateMultipartUpload(req).getUploadId();
        .debug("Initiated new multipart upload: " + uploadId);
        return uploadId;
    }
New to GrepCode? Check out our FAQ X