Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   * #%L
   * BroadleafCommerce Framework
   * %%
   * Copyright (C) 2009 - 2013 Broadleaf Commerce
   * %%
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
  * 
  *       http://www.apache.org/licenses/LICENSE-2.0
  * 
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  * #L%
  */
 package org.broadleafcommerce.core.search.service.solr;
 
 
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
Responsible for building and rebuilding the Solr index

Author(s):
Andre Azzolini (apazzolini)
Jeff Fischer
 
 @Service("blSolrIndexService")
 public class SolrIndexServiceImpl implements SolrIndexService {
 
     private static final Log LOG = LogFactory.getLog(SolrIndexServiceImpl.class);
 
     @Value("${solr.index.product.pageSize}")
     protected int pageSize;
 
     @Resource(name = "blProductDao")
     protected ProductDao productDao;
 
     @Resource(name = "blFieldDao")
     protected FieldDao fieldDao;
 
     @Resource(name = "blLocaleService")
     protected LocaleService localeService;
 
     @Resource(name = "blSolrHelperService")
     protected SolrHelperService shs;
 
    @Resource(name = "blSolrSearchServiceExtensionManager")
    @Resource(name = "blTransactionManager")
    @Resource(name = "blSolrIndexDao")
    protected SolrIndexDao solrIndexDao;
    public static String ATTR_MAP = "productAttributes";
    @Override
    public void performCachedOperation(SolrIndexCachedOperation.CacheOperation cacheOperationthrows ServiceException {
        try {
            CatalogStructure cache = new CatalogStructure();
            SolrIndexCachedOperation.setCache(cache);
            cacheOperation.execute();
        } finally {
            if (.isInfoEnabled()) {
                .info("Cleaning up Solr index cache from memory - size approx: " + getCacheSizeInMemoryApproximation(SolrIndexCachedOperation.getCache()) + " bytes");
            }
            SolrIndexCachedOperation.clearCache();
        }
    }
    protected int getCacheSizeInMemoryApproximation(CatalogStructure structure) {
        try {
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            ObjectOutputStream oos = new ObjectOutputStream(baos);
            oos.writeObject(structure);
            oos.close();
            return baos.size();
        } catch (IOException e) {
            throw ExceptionHelper.refineException(e);
        }
    }
    @Override
    public void rebuildIndex() throws ServiceExceptionIOException {
        .info("Rebuilding the solr index...");
        StopWatch s = new StopWatch();
        // If we are in single core mode, we have to delete the documents before reindexing
        if (SolrContext.isSingleCoreMode()) {
            SolrIndexServiceImpl.this.deleteAllDocuments();
        }
        Object[] pack = saveState();
        try {
            final Long numProducts = .readCountAllActiveProducts();
            if (.isDebugEnabled()) {
                .debug("There are " + numProducts + " total products");
            }
                @Override
                public void execute() throws ServiceException {
                    int page = 0;
                    while ((page * ) < numProducts) {
                        buildIncrementalIndex(page);
                        page++;
                    }
                }
            });
            optimizeIndex(SolrContext.getReindexServer());
        } finally {
            restoreState(pack);
        }
        // Swap the active and the reindex cores
        .swapActiveCores();
        // If we are not in single core mode, we delete the documents for the unused core after swapping
        if (!SolrContext.isSingleCoreMode()) {
            deleteAllDocuments();
        }
        .info(String.format("Finished building index in %s"s.toLapString()));
    }
    protected void deleteAllDocuments() throws ServiceException {
        try {
            String deleteQuery = .getNamespaceFieldName() + ":(\"" + .getCurrentNamespace() + "\")";
            .debug("Deleting by query: " + deleteQuery);
            SolrContext.getReindexServer().deleteByQuery(deleteQuery);
            SolrContext.getReindexServer().commit();
        } catch (Exception e) {
            throw new ServiceException("Could not delete documents"e);
        }
    }
    protected void buildIncrementalIndex(int pageint pageSizethrows ServiceException {
        buildIncrementalIndex(pagepageSizetrue);
    }
    @Override
    public void buildIncrementalIndex(int pageint pageSizeboolean useReindexServerthrows ServiceException {
        if (SolrIndexCachedOperation.getCache() == null) {
            .warn("Consider using SolrIndexService.performCachedOperation() in combination with " +
                    "SolrIndexService.buildIncrementalIndex() for better caching performance during solr indexing");
        }
        TransactionStatus status = TransactionUtils.createTransaction("readProducts",
                .true);
        if (.isDebugEnabled()) {
            .debug(String.format("Building index - page: [%s], pageSize: [%s]"pagepageSize));
        }
        StopWatch s = new StopWatch();
        boolean cacheOperationManaged = false;
        try {
            CatalogStructure cache = SolrIndexCachedOperation.getCache();
            if (cache != null) {
                cacheOperationManaged = true;
            } else {
                cache = new CatalogStructure();
                SolrIndexCachedOperation.setCache(cache);
            }
            List<Productproducts = readAllActiveProducts(pagepageSize);
            List<LongproductIds = BLCCollectionUtils.collectList(productsnew TypedTransformer<Long>() {
                @Override
                public Long transform(Object input) {
                    return ((Productinput).getId();
                }
            });
            .populateCatalogStructure(productIds, SolrIndexCachedOperation.getCache());
            List<Fieldfields = .readAllProductFields();
            List<Localelocales = getAllLocales();
            Collection<SolrInputDocumentdocuments = new ArrayList<SolrInputDocument>();
            for (Product product : products) {
                SolrInputDocument doc = buildDocument(productfieldslocales);
                //If someone overrides the buildDocument method and determines that they don't want a product 
                //indexed, then they can return null. If the document is null it does not get added to 
                //to the index.
                if (doc != null) {
                    documents.add(doc);
                }
            }
            logDocuments(documents);
            if (!CollectionUtils.isEmpty(documents)) {
                SolrServer server = useReindexServer ? SolrContext.getReindexServer() : SolrContext.getServer();
                server.add(documents);
                server.commit();
            }
            TransactionUtils.finalizeTransaction(statusfalse);
        } catch (SolrServerException e) {
            TransactionUtils.finalizeTransaction(statustrue);
            throw new ServiceException("Could not rebuild index"e);
        } catch (IOException e) {
            TransactionUtils.finalizeTransaction(statustrue);
            throw new ServiceException("Could not rebuild index"e);
        } catch (RuntimeException e) {
            TransactionUtils.finalizeTransaction(statustrue);
            throw e;
        } finally {
            if (!cacheOperationManaged) {
                SolrIndexCachedOperation.clearCache();
            }
        }
        if (.isDebugEnabled()) {
            .debug(String.format("Built index - page: [%s], pageSize: [%s] in [%s]"pagepageSizes.toLapString()));
        }
    }

    
This method to read all active products will be slow if you have a large catalog. In this case, you will want to read the products in a different manner. For example, if you know the fields that will be indexed, you can configure a DAO object to only load those fields. You could also use a JDBC based DAO for even faster access. This default implementation is only suitable for small catalogs.

Returns:
the list of all active products to be used by the index building task
    protected List<ProductreadAllActiveProducts() {
        return .readAllActiveProducts();
    }

    
This method to read active products utilizes paging to improve performance over readAllActiveProducts(). While not optimal, this will reduce the memory required to load large catalogs. It could still be improved for specific implementations by only loading fields that will be indexed or by accessing the database via direct JDBC (instead of Hibernate).

Returns:
the list of all active products to be used by the index building task
Since:
2.2.0
    protected List<ProductreadAllActiveProducts(int pageint pageSize) {
        return .readAllActiveProducts(pagepageSize);
    }
    @Override
    public List<LocalegetAllLocales() {
        return .findAllLocales();
    }
    @Override
    public SolrInputDocument buildDocument(final Product productList<FieldfieldsList<Localelocales) {
        final SolrInputDocument document = new SolrInputDocument();
        attachBasicDocumentFields(productdocument);
        // Keep track of searchable fields added to the index.   We need to also add the search facets if 
        // they weren't already added as a searchable field.
        List<StringaddedProperties = new ArrayList<String>();
        for (Field field : fields) {
            try {
                // Index the searchable fields
                if (field.getSearchable()) {
                    List<FieldTypesearchableFieldTypes = .getSearchableFieldTypes(field);
                    for (FieldType sft : searchableFieldTypes) {
                        Map<StringObjectpropertyValues = getPropertyValues(productfieldsftlocales);
                        // Build out the field for every prefix
                        for (Entry<StringObjectentry : propertyValues.entrySet()) {
                            String prefix = entry.getKey();
                            prefix = StringUtils.isBlank(prefix) ? prefix : prefix + "_";
                            String solrPropertyName = .getPropertyNameForFieldSearchable(fieldsftprefix);
                            Object value = entry.getValue();
                            document.addField(solrPropertyNamevalue);
                            addedProperties.add(solrPropertyName);
                        }
                    }
                }
                // Index the faceted field type as well
                FieldType facetType = field.getFacetFieldType();
                if (facetType != null) {
                    Map<StringObjectpropertyValues = getPropertyValues(productfieldfacetTypelocales);
                    // Build out the field for every prefix
                    for (Entry<StringObjectentry : propertyValues.entrySet()) {
                        String prefix = entry.getKey();
                        prefix = StringUtils.isBlank(prefix) ? prefix : prefix + "_";
                        String solrFacetPropertyName = .getPropertyNameForFieldFacet(fieldprefix);
                        Object value = entry.getValue();
                        if (!addedProperties.contains(solrFacetPropertyName)) {
                            document.addField(solrFacetPropertyNamevalue);
                        }
                    }
                }
            } catch (Exception e) {
                .trace("Could not get value for property[" + field.getQualifiedFieldName() + "] for product id["
                        + product.getId() + "]"e);
            }
        }
        return document;
    }

    
Adds the ID, category, and explicitCategory fields for the product to the document

Parameters:
product
document
    protected void attachBasicDocumentFields(Product productSolrInputDocument document) {
        boolean cacheOperationManaged = false;
        try {
            CatalogStructure cache = SolrIndexCachedOperation.getCache();
            if (cache != null) {
                cacheOperationManaged = true;
            } else {
                cache = new CatalogStructure();
                SolrIndexCachedOperation.setCache(cache);
                .populateCatalogStructure(Arrays.asList(product.getId()), SolrIndexCachedOperation.getCache());
            }
            // Add the namespace and ID fields for this product
            document.addField(.getNamespaceFieldName(), .getCurrentNamespace());
            document.addField(.getIdFieldName(), .getSolrDocumentId(documentproduct));
            document.addField(.getProductIdFieldName(), product.getId());
            .getProxy().attachAdditionalBasicFields(productdocument);
            // The explicit categories are the ones defined by the product itself
            if (cache.getParentCategoriesByProduct().containsKey(product.getId())) {
                for (Long categoryId : cache.getParentCategoriesByProduct().get(product.getId())) {
                    document.addField(.getExplicitCategoryFieldName(), .getCategoryId(categoryId));
                    String categorySortFieldName = .getCategorySortFieldName(.getCategoryId(categoryId));
                    String displayOrderKey = categoryId + "-" + .getProductId(product.getId());
                    BigDecimal displayOrder = cache.getDisplayOrdersByCategoryProduct().get(displayOrderKey);
                    if (displayOrder == null) {
                        displayOrderKey = categoryId + "-" + product.getId();
                        displayOrder = cache.getDisplayOrdersByCategoryProduct().get(displayOrderKey);
                    }
                    if (document.getField(categorySortFieldName) == null) {
                        document.addField(categorySortFieldNamedisplayOrder);
                    }
                    // This is the entire tree of every category defined on the product
                    buildFullCategoryHierarchy(documentcachecategoryId);
                }
            }
        } finally {
            if (!cacheOperationManaged) {
                SolrIndexCachedOperation.clearCache();
            }
        }
    }

    
Walk the category hierarchy upwards, adding a field for each level to the solr document

Parameters:
document the solr document for the product
cache the catalog structure cache
categoryId the current category id
    protected void buildFullCategoryHierarchy(SolrInputDocument documentCatalogStructure cacheLong categoryId) {
        Long catIdToAdd = .getCategoryId(categoryId); 
        Collection<ObjectexistingValues = document.getFieldValues(.getCategoryFieldName());
        if (existingValues == null || !existingValues.contains(catIdToAdd)) {
            document.addField(.getCategoryFieldName(), catIdToAdd);
        }
        Set<Longparents = cache.getParentCategoriesByCategory().get(categoryId);
        for (Long parent : parents) {
            buildFullCategoryHierarchy(documentcacheparent);
        }
    }

    
Returns a map of prefix to value for the requested attributes. For example, if the requested field corresponds to a Sku's description and the locales list has the en_US locale and the es_ES locale, the resulting map could be { "en_US" : "A description", "es_ES" : "Una descripcion" }

Parameters:
product
field
fieldType
locales
Returns:
the value of the property
Throws:
java.lang.IllegalAccessException
java.lang.reflect.InvocationTargetException
java.lang.NoSuchMethodException
    protected Map<StringObjectgetPropertyValues(Product productField fieldFieldType fieldType,
            List<Localelocalesthrows IllegalAccessExceptionInvocationTargetExceptionNoSuchMethodException {
        String propertyName = field.getPropertyName();
        Map<StringObjectvalues = new HashMap<StringObject>();
        if ( != null) {
            ExtensionResultStatusType result = .getProxy().addPropertyValues(productfieldfieldTypevaluespropertyNamelocales);
            if (..equals(result)) {
                Object propertyValue;
                if (propertyName.contains()) {
                    propertyValue = PropertyUtils.getMappedProperty(productpropertyName.substring(.length() + 1));
                    // It's possible that the value is an actual object, like ProductAttribute. We'll attempt to pull the 
                    // value field out of it if it exists.
                    if (propertyValue != null) {
                        try {
                            propertyValue = PropertyUtils.getProperty(propertyValue"value");
                        } catch (NoSuchMethodException e) {
                            // Do nothing, we'll keep the existing value
                        }
                    }
                } else {
                    propertyValue = PropertyUtils.getProperty(productpropertyName);
                }
                values.put(""propertyValue);
            }
        }
        return values;
    }

    
Converts a propertyName to one that is able to reference inside a map. For example, consider the property in Product that references a List<ProductAttribute>, "productAttributes". Also consider the utility method in Product called "mappedProductAttributes", which returns a map of the ProductAttributes keyed by the name property in the ProductAttribute. Given the parameters "productAttributes.heatRange", "productAttributes", "mappedProductAttributes" (which would represent a property called "productAttributes.heatRange" that references a specific ProductAttribute inside of a product whose "name" property is equal to "heatRange", this method will convert this property to mappedProductAttributes(heatRange).value, which is then usable by the standard beanutils PropertyUtils class to get the value.

Parameters:
propertyName
listPropertyName
mapPropertyName
Returns:
the converted property name
    protected String convertToMappedProperty(String propertyNameString listPropertyNameString mapPropertyName) {
        String[] splitName = StringUtils.split(propertyName".");
        StringBuilder convertedProperty = new StringBuilder();
        for (int i = 0; i < splitName.lengthi++) {
            if (convertedProperty.length() > 0) {
                convertedProperty.append(".");
            }
            if (splitName[i].equals(listPropertyName)) {
                convertedProperty.append(mapPropertyName).append("(");
                convertedProperty.append(splitName[i + 1]).append(").value");
                i++;
            } else {
                convertedProperty.append(splitName[i]);
            }
        }
        return convertedProperty.toString();
    }
    @Override
    public Object[] saveState() {
         return new Object[] {
             BroadleafRequestContext.getBroadleafRequestContext(),
             SkuPricingConsiderationContext.getSkuPricingConsiderationContext(),
             SkuPricingConsiderationContext.getSkuPricingService(),
             SkuActiveDateConsiderationContext.getSkuActiveDatesService()
         };
     }
         
    @Override
    @SuppressWarnings("rawtypes")
    public void restoreState(Object[] pack) {
         BroadleafRequestContext.setBroadleafRequestContext((BroadleafRequestContextpack[0]);
         SkuPricingConsiderationContext.setSkuPricingConsiderationContext((HashMappack[1]);
         SkuPricingConsiderationContext.setSkuPricingService((DynamicSkuPricingServicepack[2]);
         SkuActiveDateConsiderationContext.setSkuActiveDatesService((DynamicSkuActiveDatesServicepack[3]);
     }
     
    @Override
    public void optimizeIndex(SolrServer serverthrows ServiceExceptionIOException {
         try {
             if (.isDebugEnabled()) {
                 .debug("Optimizing the index...");
             }
             server.optimize();
         } catch (SolrServerException e) {
             throw new ServiceException("Could not optimize index"e);
         }
     }
    
    @Override
    public void logDocuments(Collection<SolrInputDocumentdocuments) {
        if (.isTraceEnabled()) {
            for (SolrInputDocument document : documents) {
                .trace(document);
            }
        }
    }
New to GrepCode? Check out our FAQ X