Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 
 package org.apache.marmotta.platform.core.services.http;
 
 
 import java.util.*;
 
 public class HttpClientServiceImpl implements HttpClientService {
 
     private static final String          TASK_GROUP_CLIENT       = "HttpClient";
 
     private static final String[]        KEYS                    = {
         "requests executed""payload sent""payload received""payload from cache",
         "ConnectionManager""max connections""available connections",
         "active connections""requests waiting"
     };
 
     private static final Charset         DEFAULT_CHARSET         = Charset.defaultCharset();
 
     @Inject
     private Logger                       log;
 
     @Inject
     private TaskManagerService           taskManagerService;
 
     @Inject
     private ConfigurationService         configurationService;
 
    @Inject
    private StatisticsService            statisticsService;
    @Inject
    @MarmottaCache("http-client-cache")
    private ConcurrentMap httpCache;
    private HttpClient                   httpClient;
    private BasicHttpParams              httpParams;
    private  AtomicLong             bytesSent               = new AtomicLong();
    private  AtomicLong             bytesReceived           = new AtomicLong();
    private  AtomicLong             bytesFromCache          = new AtomicLong();
    private  AtomicLong             requestsExecuted        = new AtomicLong();
    private  ReentrantReadWriteLock lock                    = new ReentrantReadWriteLock();

    
Execute a request and pass the response to the provided handler.

    @Override
    public <T> T execute(HttpRequestBase requestResponseHandler<? extends T> handlerthrows ClientProtocolExceptionIOException {
        if (handler == null)
            throw new IllegalArgumentException("Response handler must not be null.");
        final long start = System.nanoTime();
        String logMessageF = "Request '{}' failed after {}";
        try {
            final T result = .execute(requesthandler);
            logMessageF = "Request '{}' took {}";
            return result;
        } finally {
            .debug(logMessageFrequest.getRequestLine(), formatNanoDuration(System.nanoTime() - start));
        }
    }

    
Get a ready-to-use org.apache.http.client.HttpClient.
    @Override
    public HttpClient getHttpClient() {
        return new ReadLockHttpClient();
    }

    
    @Override
        final long start = System.nanoTime();
        String logMessageF = "Request '{}' failed after {}";
        try {
            final HttpResponse resp = .execute(request);
            logMessageF = "Request '{}' took {}";
            return resp;
        } finally {
            .debug(logMessageFrequest.getRequestLine(), formatNanoDuration(System.nanoTime() - start));
        }
    }
    @Override
    public void cleanupResponse(HttpResponse response) {
        if (response == nullthrow new IllegalArgumentException("Response must not be null");
        EntityUtils.consumeQuietly(response.getEntity());
    }
    private static String formatNanoDuration(long nano) {
        // convert to microseconds (1/1000s)
        final long micro = nano / 1000;
        if (micro > 1000 * 1000l) {
            // more than a second
            long millis = micro / 1000l;
            if (millis > 60000l)
                // more than a minute
                return String.format("%d min %.1f sec"millis / (1000 * 60), 0.001d * millis % 60);
            else
                // more than a second
                return String.format("%.1f sec", 0.001d * millis);
        } else
            // some millis
            return String.format("%f ms", 0.001d * micro);
    }
    @Override
    public String doGet(String requestUrlthrows IOException {
        return doGet(requestUrlnew StringBodyResponseHandler());
    }
    @Override
    public <T> T doGet(String requestUrlResponseHandler<? extends T> responseHandlerthrows IOException {
        final HttpGet get = new HttpGet(requestUrl);
        return execute(getresponseHandler);
    }
    @Override
    public <T> T doPost(String requestUrlHttpEntity bodyResponseHandler<? extends T> responseHandlerthrows IOException {
        HttpPost post = new HttpPost(requestUrl);
        post.setEntity(body);
        return execute(postresponseHandler);
    }
    @Override
    public String doPost(String urlString bodythrows IOException {
        return doPost(urlnew StringEntity(body, ContentType.create("text/plain")), new StringBodyResponseHandler());
    }
    @Override
    public <T> T doPut(String urlHttpEntity bodyResponseHandler<? extends T> responseHandlerthrows IOException {
        HttpPut put = new HttpPut(url);
        put.setEntity(body);
        return execute(putresponseHandler);
    }
    @Override
    public String doPut(String urlString bodythrows IOException {
        return doPut(urlnew StringEntity(body, ContentType.create("text/plain")), new StringBodyResponseHandler());
    }
    @Override
    public <T> T doDelete(String urlResponseHandler<? extends T> responseHandlerthrows IOException {
        HttpDelete delete = new HttpDelete(url);
        return execute(deleteresponseHandler);
    }
    @Override
    public int doDelete(String urlthrows IOException {
        return doDelete(urlnew StatusCodeResponseHandler());
    }
    @Override
    public <T> T doHead(String urlResponseHandler<? extends T> responseHandlerthrows IOException {
        HttpHead head = new HttpHead(url);
        return execute(headresponseHandler);
    }
    @Override
    public Date doHead(String urlthrows IOException {
        return doHead(urlnew LastModifiedResponseHandler());
    }
    protected void onConfigurationChange(@Observes ConfigurationChangedEvent event) {
        if (event.containsChangedKeyWithPrefix("core.http.")) {
            try {
                .writeLock().lock();
                shutdown();
                initialize();
            } finally {
                .writeLock().unlock();
            }
        }
    }
    protected void initialize() {
        try {
            .writeLock().lock();
             = new BasicHttpParams();
            String userAgentString =
                    "Apache Marmotta/" + .getStringConfiguration("kiwi.version") +
                    " (running at " + .getServerUri() + ")" +
                    " lmf-core/" + .getStringConfiguration("kiwi.version");
            userAgentString = .getStringConfiguration("core.http.user_agent"userAgentString);
            .setIntParameter(..getIntConfiguration("core.http.so_timeout", 60000));
                    .getIntConfiguration("core.http.connection_timeout", 10000));
            SchemeRegistry schemeRegistry = new SchemeRegistry();
            schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
            schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
            PoolingClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
            final DefaultHttpClient hc = new DefaultHttpClient(cm);
            hc.setRedirectStrategy(new LMFRedirectStrategy());
            hc.setHttpRequestRetryHandler(new LMFHttpRequestRetryHandler());
            hc.addRequestInterceptor(new LMFRequestUserAgent(userAgentString));
                CacheConfig cacheConfig = new CacheConfig();
                // FIXME: Hardcoded constants - is this useful?
                cacheConfig.setMaxCacheEntries(1000);
                cacheConfig.setMaxObjectSize(81920);
                final HttpCacheStorage cacheStore = new MapHttpCacheStorage();
                this. = new MonitoredHttpClient(new CachingHttpClient(hccacheStorecacheConfig));
            } else {
                this. = new MonitoredHttpClient(hc);
            }
            .set(0);
            .set(0);
            .set(0);
            .start();
            StatisticsProvider stats = new StatisticsProvider(cm);
            .registerModule(HttpClientService.class.getSimpleName(), stats);
        } finally {
            .writeLock().unlock();
        }
    }
    @PreDestroy
    protected void shutdown() {
        try {
            .writeLock().lock();
            .shutdown();
            .getConnectionManager().shutdown();
        } finally {
            .writeLock().unlock();
        }
    }
    private static class LMFRequestUserAgent implements HttpRequestInterceptor {
        private final String baseUserAgent;
        public LMFRequestUserAgent(String baseUserAgent) {
            this. = baseUserAgent;
        }
        private final String buildUserAgentString(String localPart) {
            if (localPart == null || localPart.length() == 0)
                return ;
            else if (localPart.endsWith()) return localPart;
            return localPart + " " + ;
        }
        @Override
        public void process(HttpRequest requestHttpContext contextthrows HttpExceptionIOException {
            if (request == nullthrow new IllegalArgumentException("HTTP request must not be null");
            if (!request.containsHeader(.)) {
                String useragent = HttpProtocolParams.getUserAgent(request.getParams());
                request.addHeader(.buildUserAgentString(useragent));
            } else {
                String useragent = request.getFirstHeader(.).getValue();
                request.setHeader(.buildUserAgentString(useragent));
            }
        }
    }
    private static class LMFRedirectStrategy extends DefaultRedirectStrategy {
        @Override
        public boolean isRedirected(HttpRequest requestHttpResponse responseHttpContext contextthrows ProtocolException {
            if (request == null)
                throw new IllegalArgumentException("HTTP request must not be null");
            if (response == null)
                throw new IllegalArgumentException("HTTP response must not be null");
            int statusCode = response.getStatusLine().getStatusCode();
            String method = request.getRequestLine().getMethod();
            Header locationHeader = response.getFirstHeader("location");
            switch (statusCode) {
            case .:
                return (method.equalsIgnoreCase(.) || method.equalsIgnoreCase(.))
                        && locationHeader != null;
            case .:
            case .:
                return method.equalsIgnoreCase(.)
                        || method.equalsIgnoreCase(.);
            case .:
                return true;
            case .:
                return true;
            default:
                return false;
            } // end of switch
        }
    }
    private static class LMFHttpRequestRetryHandler implements HttpRequestRetryHandler {
        
Determines if a method should be retried after an IOException occurs during execution.

Parameters:
exception the exception that occurred
executionCount the number of times this method has been unsuccessfully executed
context the context for the request execution
Returns:
true if the method should be retried, false otherwise
        @Override
        public boolean retryRequest(IOException exceptionint executionCountHttpContext context) {
            return false;
        }
    }
    private static class IdleConnectionMonitorThread extends Thread {
        private final ClientConnectionManager connMgr;
        private volatile boolean              shutdown;
        public IdleConnectionMonitorThread(ClientConnectionManager connMgr) {
            super("HttpClientService Idle Connection Manager");
            this. = connMgr;
            setDaemon(true);
        }
        @Override
        public void run() {
            try {
                while (!) {
                    synchronized (this) {
                        wait(5000);
                        // Close expired connections
                        .closeExpiredConnections();
                        // Optionally, close connections
                        // that have been idle longer than 30 sec
                        .closeIdleConnections(30, .);
                    }
                }
            } catch (InterruptedException ex) {
                // terminate
            }
        }
        public void shutdown() {
            synchronized (this) {
                 = true;
                notifyAll();
            }
        }
    }
    private class StatisticsProvider implements StatisticsModule {
        private boolean                        enabled;
            this. = cm;
             = true;
        }
        @Override
        public void enable() {
             = true;
        }
        @Override
        public void disable() {
             = false;
        }
        @Override
        public boolean isEnabled() {
            return ;
        }
        @Override
        public List<StringgetPropertyNames() {
            return Collections.unmodifiableList(Arrays.asList());
        }
        @Override
        public Map<StringStringgetStatistics() {
            int i = 0;
            final Map<StringStringdata = new LinkedHashMap<StringString>();
            data.put([i++], String.valueOf(.get()));
            data.put([i++], humanReadableBytes(.get(), false));
            data.put([i++], humanReadableBytes(.get(), false));
            data.put([i++], humanReadableBytes(.get(), false));
            final PoolStats cmStats = .getTotalStats();
            data.put([i++], .getClass().getSimpleName());
            data.put([i++], String.valueOf(cmStats.getMax()));
            data.put([i++], String.valueOf(cmStats.getAvailable()));
            data.put([i++], String.valueOf(cmStats.getLeased()));
            data.put([i++], String.valueOf(cmStats.getPending()));
            return data;
        }
        private String humanReadableBytes(long bytesboolean si) {
            final int unit = si ? 1000 : 1024;
            if (bytes < unitreturn bytes + " B";
            int exp = (int) (Math.log(bytes) / Math.log(unit));
            String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
            return String.format("%.2f %sB"bytes / Math.pow(unitexp), pre);
        }
        @Override
        public String getName() {
            return HttpClientService.class.getSimpleName();
        }
    }
    private class ReadLockHttpClient implements HttpClient {
        private HttpParams params;
        public ReadLockHttpClient() {
            this. = new DefaultedHttpParams(new BasicHttpParams(), );
        }
        @Override
        public HttpParams getParams() {
            return ;
        }
        @Override
        public ClientConnectionManager getConnectionManager() {
            return .getConnectionManager();
        }
        @Override
        public HttpResponse execute(HttpUriRequest requestthrows IOExceptionClientProtocolException {
            .readLock().lock();
            try {
                return .execute(request);
            } finally {
                .readLock().unlock();
            }
        }
        @Override
        public HttpResponse execute(HttpUriRequest requestHttpContext contextthrows IOExceptionClientProtocolException {
            .readLock().lock();
            try {
                return .execute(requestcontext);
            } finally {
                .readLock().unlock();
            }
        }
        @Override
        public HttpResponse execute(HttpHost targetHttpRequest requestthrows IOExceptionClientProtocolException {
            .readLock().lock();
            try {
                return .execute(targetrequest);
            } finally {
                .readLock().unlock();
            }
        }
        @Override
        public HttpResponse execute(HttpHost targetHttpRequest requestHttpContext contextthrows IOExceptionClientProtocolException {
            .readLock().lock();
            try {
                return .execute(targetrequestcontext);
            } finally {
                .readLock().unlock();
            }
        }
        @Override
        public <T> T execute(HttpUriRequest requestResponseHandler<? extends T> responseHandlerthrows IOExceptionClientProtocolException {
            .readLock().lock();
            try {
                return .execute(requestresponseHandler);
            } finally {
                .readLock().unlock();
            }
        }
        @Override
        public <T> T execute(HttpUriRequest requestResponseHandler<? extends T> responseHandlerHttpContext contextthrows IOException,
        ClientProtocolException {
            .readLock().lock();
            try {
                return .execute(requestresponseHandlercontext);
            } finally {
                .readLock().unlock();
            }
        }
        @Override
        public <T> T execute(HttpHost targetHttpRequest requestResponseHandler<? extends T> responseHandlerthrows IOException,
        ClientProtocolException {
            .readLock().lock();
            try {
                return .execute(targetrequestresponseHandler);
            } finally {
                .readLock().unlock();
            }
        }
        @Override
        public <T> T execute(HttpHost targetHttpRequest requestResponseHandler<? extends T> responseHandlerHttpContext context)
                throws IOExceptionClientProtocolException {
            .readLock().lock();
            try {
                return .execute(targetrequestresponseHandlercontext);
            } finally {
                .readLock().unlock();
            }
        }
    }
    protected class MonitoredHttpClient implements HttpClient {
        private final HttpClient delegate;
        public MonitoredHttpClient(HttpClient delegate) {
            this. = delegate;
        }
        @Override
        public HttpParams getParams() {
            return .getParams();
        }
        @Override
        public ClientConnectionManager getConnectionManager() {
            return .getConnectionManager();
        }
        @Override
        public HttpResponse execute(HttpUriRequest requestthrows IOExceptionClientProtocolException {
            final Task task = preProcess(request);
            final HttpResponse response;
            .readLock().lock();
            try {
                response = .execute(request);
            } catch (ClientProtocolException cpe) {
                task.endTask();
                throw cpe;
            } catch (IOException io) {
                task.endTask();
                throw io;
            } finally {
                .readLock().unlock();
            }
            return postProcess(responsenulltask);
        }
        @Override
        public HttpResponse execute(HttpUriRequest requestHttpContext contextthrows IOExceptionClientProtocolException {
            final Task task = preProcess(request);
            final HttpResponse response;
            .readLock().lock();
            try {
                response = .execute(requestcontext);
            } catch (ClientProtocolException cpe) {
                task.endTask();
                throw cpe;
            } catch (IOException io) {
                task.endTask();
                throw io;
            } finally {
                .readLock().unlock();
            }
            return postProcess(responsecontexttask);
        }
        private Task preProcess(HttpRequest request) {
            final RequestLine rl = request.getRequestLine();
            final String taskName = String.format("%S %s %S"rl.getMethod(), rl.getUri(), request.getProtocolVersion());
            final Task task = .createSubTask(taskName);
            task.updateMessage("preparing request");
            task.updateDetailMessage("method"rl.getMethod());
            task.updateDetailMessage("url"rl.getUri());
            // TODO: some more detail messages?
            if (request instanceof HttpEntityEnclosingRequest) {
                // To report upload progress, the entity is wrapped in a MonitoredHttpEntity.
                final HttpEntityEnclosingRequest entityRequest = (HttpEntityEnclosingRequestrequest;
                entityRequest.setEntity(new MonitoredHttpEntity(entityRequest.getEntity(), task));
            }
            task.updateMessage("sending request");
            return task;
        }
        private HttpResponse postProcess(final HttpResponse responseHttpContext contextfinal Task task) {
            .incrementAndGet();
            task.resetProgress();
            task.updateMessage("retrieving response");
            if (response.getEntity() != null) {
                boolean cachedResponse;
                if (context != null) {
                    CacheResponseStatus cacheRespStatus = (CacheResponseStatuscontext.getAttribute(.);
                    // To report download progress, the entity is wrapped in a MonitoredHttpEntity.
                    cachedResponse = cacheRespStatus != null && cacheRespStatus != .;
                } else {
                    cachedResponse = false;
                }
                response.setEntity(new MonitoredHttpEntity(response.getEntity(), taskcachedResponse ?  : ));
            } else {
                task.endTask();
            }
            return response;
        }
        @Override
        public HttpResponse execute(HttpHost targetHttpRequest requestthrows IOExceptionClientProtocolException {
            final Task task = preProcess(request);
            final HttpResponse response;
            .readLock().lock();
            try {
                response = .execute(targetrequest);
            } catch (ClientProtocolException cpe) {
                task.endTask();
                throw cpe;
            } catch (IOException io) {
                task.endTask();
                throw io;
            } finally {
                .readLock().unlock();
            }
            return postProcess(responsenulltask);
        }
        @Override
        public HttpResponse execute(HttpHost targetHttpRequest requestHttpContext contextthrows IOExceptionClientProtocolException {
            final Task task = preProcess(request);
            final HttpResponse response;
            .readLock().lock();
            try {
                response = .execute(targetrequestcontext);
            } catch (ClientProtocolException cpe) {
                task.endTask();
                throw cpe;
            } catch (IOException io) {
                task.endTask();
                throw io;
            } finally {
                .readLock().unlock();
            }
            return postProcess(responsecontexttask);
        }
        @Override
        public <T> T execute(HttpUriRequest requestResponseHandler<? extends T> responseHandlerthrows IOExceptionClientProtocolException {
            final HttpResponse response = execute(request);
            return processResponse(responseHandlerresponse);
        }
        private <T> T processResponse(ResponseHandler<? extends T> responseHandlerfinal HttpResponse responsethrows ClientProtocolException,
        IOException {
            try {
                return responseHandler.handleResponse(response);
            } finally {
                // Make sure everything is cleaned up properly
                EntityUtils.consume(response.getEntity());
            }
        }
        @Override
        public <T> T execute(HttpUriRequest requestResponseHandler<? extends T> responseHandlerHttpContext contextthrows IOException,
        ClientProtocolException {
            final HttpResponse response = execute(requestcontext);
            return processResponse(responseHandlerresponse);
        }
        @Override
        public <T> T execute(HttpHost targetHttpRequest requestResponseHandler<? extends T> responseHandlerthrows IOException,
        ClientProtocolException {
            final HttpResponse response = execute(targetrequest);
            return processResponse(responseHandlerresponse);
        }
        @Override
        public <T> T execute(HttpHost targetHttpRequest requestResponseHandler<? extends T> responseHandlerHttpContext context)
                throws IOExceptionClientProtocolException {
            final HttpResponse response = execute(targetrequestcontext);
            return processResponse(responseHandlerresponse);
        }
    }
    private static class MapHttpCacheStorage implements HttpCacheStorage {
        ConcurrentMap<Stringbyte[]> cache;
        private final HttpCacheEntrySerializer serializer;
        private MapHttpCacheStorage(ConcurrentMap<Stringbyte[]> cache) {
            this.      = cache;
            this. = new DefaultHttpCacheEntrySerializer();
        }

        
Store a given cache entry under the given key.

Parameters:
key where in the cache to store the entry
entry cached response to store
Throws:
java.io.IOException
        @Override
        public void putEntry(String keyHttpCacheEntry entrythrows IOException {
            final ByteArrayOutputStream bos = new ByteArrayOutputStream();
            .writeTo(entrybos);
            .put(key,bos.toByteArray());
        }

        
Retrieves the cache entry stored under the given key or null if no entry exists under that key.

Parameters:
key cache key
Returns:
an org.apache.http.client.cache.HttpCacheEntry or null if no entry exists
Throws:
java.io.IOException
        @Override
        public HttpCacheEntry getEntry(String keythrows IOException {
            byte[] data = .get(key);
            if(data == null) {
                return null;
            } else {
                return .readFrom(new ByteArrayInputStream(data));
            }
        }

        
Deletes/invalidates/removes any cache entries currently stored under the given key.

Parameters:
key
Throws:
java.io.IOException
        @Override
        public void removeEntry(String keythrows IOException {
            .remove(key);
        }

        
Atomically applies the given callback to update an existing cache entry under a given key.

Parameters:
key indicates which entry to modify
callback performs the update; see org.apache.http.client.cache.HttpCacheUpdateCallback for details, but roughly the callback expects to be handed the current entry and will return the new value for the entry.
Throws:
java.io.IOException
org.apache.http.client.cache.HttpCacheUpdateException
        @Override
        public void updateEntry(String keyHttpCacheUpdateCallback callbackthrows IOExceptionHttpCacheUpdateException {
            final byte[] oldData = .get(key);
            HttpCacheEntry existingEntry = null;
            if(oldData != null){
                existingEntry = .readFrom(new ByteArrayInputStream(oldData));
            }
            final HttpCacheEntry updatedEntry = callback.update(existingEntry);
            if (existingEntry == null) {
                putEntry(keyupdatedEntry);
                return;
            } else {
                // Attempt to do a CAS replace, if we fail then retry
                // While this operation should work fine within this instance, multiple instances
                //  could trample each others' data
                final ByteArrayOutputStream bos = new ByteArrayOutputStream();
                .writeTo(updatedEntrybos);
                .replace(keyoldDatabos.toByteArray());
            }
        }
    }