Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 
 
 package org.apache.hadoop.hbase.snapshot;
 
 import java.util.List;
 import java.util.Map;
 
Utility class to help read/write the Snapshot Manifest. The snapshot format is transparent for the users of this class, once the snapshot is written, it will never be modified. On open() the snapshot will be loaded to the current in-memory format.
 
 public class SnapshotManifest {
   private static final Log LOG = LogFactory.getLog(SnapshotManifest.class);
 
   private static final String DATA_MANIFEST_NAME = "data.manifest";
 
   private SnapshotDescription desc;
   private HTableDescriptor htd;
 
   private final ForeignExceptionSnare monitor;
   private final Configuration conf;
   private final Path workingDir;
   private final FileSystem fs;
 
   private SnapshotManifest(final Configuration conffinal FileSystem fs,
       final Path workingDirfinal SnapshotDescription desc,
       final ForeignExceptionSnare monitor) {
     this. = monitor;
     this. = desc;
     this. = workingDir;
     this. = conf;
     this. = fs;
   }

  
Return a SnapshotManifest instance, used for writing a snapshot. There are two usage pattern: - The Master will create a manifest, add the descriptor, offline regions and consolidate the snapshot by writing all the pending stuff on-disk. manifest = SnapshotManifest.create(...) manifest.addRegion(tableDir, hri) manifest.consolidate() - The RegionServer will create a single region manifest manifest = SnapshotManifest.create(...) manifest.addRegion(region)
  public static SnapshotManifest create(final Configuration conffinal FileSystem fs,
      final Path workingDirfinal SnapshotDescription desc,
      final ForeignExceptionSnare monitor) {
    return new SnapshotManifest(conffsworkingDirdescmonitor);
  }

  
Return a SnapshotManifest instance with the information already loaded in-memory. SnapshotManifest manifest = SnapshotManifest.open(...) HTableDescriptor htd = manifest.getTableDescriptor() for (SnapshotRegionManifest regionManifest: manifest.getRegionManifests()) hri = regionManifest.getRegionInfo() for (regionManifest.getFamilyFiles()) ...
  public static SnapshotManifest open(final Configuration conffinal FileSystem fs,
      final Path workingDirfinal SnapshotDescription descthrows IOException {
    SnapshotManifest manifest = new SnapshotManifest(conffsworkingDirdescnull);
    manifest.load();
    return manifest;
  }


  
Add the table descriptor to the snapshot manifest
  public void addTableDescriptor(final HTableDescriptor htdthrows IOException {
    this. = htd;
  }
  interface RegionVisitor<TRegion, TFamily> {
    TRegion regionOpen(final HRegionInfo regionInfothrows IOException;
    void regionClose(final TRegion regionthrows IOException;
    TFamily familyOpen(final TRegion regionfinal byte[] familyNamethrows IOException;
    void familyClose(final TRegion regionfinal TFamily familythrows IOException;
    void storeFile(final TRegion regionfinal TFamily familyfinal StoreFileInfo storeFile)
      throws IOException;
  }
    switch (getSnapshotFormat(desc)) {
        return new SnapshotManifestV1.ManifestBuilder();
        return new SnapshotManifestV2.ManifestBuilder();
      default:
        throw new CorruptedSnapshotException("Invalid Snapshot version: "desc.getVersion(), desc);
    }
  }

  
Creates a 'manifest' for the specified region, by reading directly from the HRegion object. This is used by the "online snapshot" when the table is enabled.
  public void addRegion(final HRegion regionthrows IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    // 1. dump region meta info into the snapshot directory
    .debug("Storing '" + region + "' region-info for snapshot.");
    Object regionData = visitor.regionOpen(region.getRegionInfo());
    // 2. iterate through all the stores in the region
    .debug("Creating references for hfiles");
    for (Store store : region.getStores().values()) {
      // 2.1. build the snapshot reference for the store
      Object familyData = visitor.familyOpen(regionDatastore.getFamily().getName());
      List<StoreFilestoreFiles = new ArrayList<StoreFile>(store.getStorefiles());
      if (.isDebugEnabled()) {
        .debug("Adding snapshot references for " + storeFiles  + " hfiles");
      }
      // 2.2. iterate through all the store's files and create "references".
      for (int i = 0, sz = storeFiles.size(); i < szi++) {
        StoreFile storeFile = storeFiles.get(i);
        .rethrowException();
        // create "reference" to this store file.
        .debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath());
        visitor.storeFile(regionDatafamilyDatastoreFile.getFileInfo());
      }
      visitor.familyClose(regionDatafamilyData);
    }
    visitor.regionClose(regionData);
  }

  
Creates a 'manifest' for the specified region, by reading directly from the disk. This is used by the "offline snapshot" when the table is disabled.
  public void addRegion(final Path tableDirfinal HRegionInfo regionInfothrows IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    // Open the RegionFS
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(,
          tableDirregionInfotrue);
    // 1. dump region meta info into the snapshot directory
    .debug("Storing region-info for snapshot.");
    Object regionData = visitor.regionOpen(regionInfo);
    // 2. iterate through all the stores in the region
    .debug("Creating references for hfiles");
    // This ensures that we have an atomic view of the directory as long as we have < ls limit
    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
    // batches and may miss files being added/deleted. This could be more robust (iteratively
    // checking to see if we have all the files until we are sure), but the limit is currently 1000
    // files/batch, far more than the number of store files under a single column family.
    Collection<StringfamilyNames = regionFs.getFamilies();
    if (familyNames != null) {
      for (String familyNamefamilyNames) {
        Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
        .rethrowException();
        Collection<StoreFileInfostoreFiles = regionFs.getStoreFiles(familyName);
        if (storeFiles == null) {
          .debug("No files under family: " + familyName);
          continue;
        }
        // 2.1. build the snapshot reference for the store
        if (.isDebugEnabled()) {
          .debug("Adding snapshot references for " + storeFiles  + " hfiles");
        }
        // 2.2. iterate through all the store's files and create "references".
        int i = 0;
        int sz = storeFiles.size();
        for (StoreFileInfo storeFilestoreFiles) {
          .rethrowException();
          // create "reference" to this store file.
          .debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
          visitor.storeFile(regionDatafamilyDatastoreFile);
        }
        visitor.familyClose(regionDatafamilyData);
      }
    }
    visitor.regionClose(regionData);
  }

  
Load the information in the SnapshotManifest. Called by SnapshotManifest.open() If the format is v2 and there is no data-manifest, means that we are loading an in-progress snapshot. Since we support rolling-upgrades, we loook for v1 and v2 regions format.
  private void load() throws IOException {
    switch (getSnapshotFormat()) {
        this. = FSTableDescriptors.getTableDescriptorFromFs();
        ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
        try {
          this. =
            SnapshotManifestV1.loadRegionManifests(tpool);
        } finally {
          tpool.shutdown();
        }
        break;
      }
        SnapshotDataManifest dataManifest = readDataManifest();
        if (dataManifest != null) {
           = HTableDescriptor.convert(dataManifest.getTableSchema());
           = dataManifest.getRegionManifestsList();
        } else {
          // Compatibility, load the v1 regions
          // This happens only when the snapshot is in-progress and the cache wants to refresh.
          List<SnapshotRegionManifestv1Regionsv2Regions;
          ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
          try {
            v1Regions = SnapshotManifestV1.loadRegionManifests(tpool);
            v2Regions = SnapshotManifestV2.loadRegionManifests(tpool);
          } finally {
            tpool.shutdown();
          }
          if (v1Regions != null && v2Regions != null) {
             =
              new ArrayList<SnapshotRegionManifest>(v1Regions.size() + v2Regions.size());
            .addAll(v1Regions);
            .addAll(v2Regions);
          } else if (v1Regions != null) {
             = v1Regions;
          } else /* if (v2Regions != null) */ {
             = v2Regions;
          }
        }
        break;
      }
      default:
        throw new CorruptedSnapshotException("Invalid Snapshot version: ".getVersion(), );
    }
  }

  
Get the current snapshot working dir
  public Path getSnapshotDir() {
    return this.;
  }

  
Get the SnapshotDescription
    return this.;
  }

  
Get the table descriptor from the Snapshot
    return this.;
  }

  
Get all the Region Manifest from the snapshot
    return this.;
  }

  
Get all the Region Manifest from the snapshot. This is an helper to get a map with the region encoded name
    if ( == null || .size() == 0) return null;
    HashMap<StringSnapshotRegionManifestregionsMap =
    for (SnapshotRegionManifest manifest) {
      String regionName = getRegionNameFromManifest(manifest);
      regionsMap.put(regionNamemanifest);
    }
    return regionsMap;
  }
  public void consolidate() throws IOException {
      Path rootDir = FSUtils.getRootDir();
      .info("Using old Snapshot Format");
      // write a copy of descriptor to the snapshot directory
      new FSTableDescriptors(rootDir)
    } else {
      .debug("Convert to Single Snapshot Manifest");
    }
  }
  /*
   * In case of rolling-upgrade, we try to read all the formats and build
   * the snapshot with the latest format.
   */
  private void convertToV2SingleManifest() throws IOException {
    // Try to load v1 and v2 regions
    List<SnapshotRegionManifestv1Regionsv2Regions;
    ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
    try {
      v1Regions = SnapshotManifestV1.loadRegionManifests(tpool);
      v2Regions = SnapshotManifestV2.loadRegionManifests(tpool);
    } finally {
      tpool.shutdown();
    }
    SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder();
    dataManifestBuilder.setTableSchema(.convert());
    if (v1Regions != null && v1Regions.size() > 0) {
      dataManifestBuilder.addAllRegionManifests(v1Regions);
    }
    if (v2Regions != null && v2Regions.size() > 0) {
      dataManifestBuilder.addAllRegionManifests(v2Regions);
    }
    // Write the v2 Data Manifest.
    // Once the data-manifest is written, the snapshot can be considered complete.
    // Currently snapshots are written in a "temporary" directory and later
    // moved to the "complated" snapshot directory.
    SnapshotDataManifest dataManifest = dataManifestBuilder.build();
    writeDataManifest(dataManifest);
    this. = dataManifest.getRegionManifestsList();
    // Remove the region manifests. Everything is now in the data-manifest.
    // The delete operation is "relaxed", unless we get an exception we keep going.
    // The extra files in the snapshot directory will not give any problem,
    // since they have the same content as the data manifest, and even by re-reading
    // them we will get the same information.
    if (v1Regions != null && v1Regions.size() > 0) {
      for (SnapshotRegionManifest regionManifestv1Regions) {
        SnapshotManifestV1.deleteRegionManifest(regionManifest);
      }
    }
    if (v2Regions != null && v2Regions.size() > 0) {
      for (SnapshotRegionManifest regionManifestv2Regions) {
        SnapshotManifestV2.deleteRegionManifest(regionManifest);
      }
    }
  }
  /*
   * Write the SnapshotDataManifest file
   */
  private void writeDataManifest(final SnapshotDataManifest manifest)
      throws IOException {
    try {
      manifest.writeTo(stream);
    } finally {
      stream.close();
    }
  }
  /*
   * Read the SnapshotDataManifest file
   */
    FSDataInputStream in = null;
    try {
      in = .open(new Path());
      return SnapshotDataManifest.parseFrom(in);
    } catch (FileNotFoundException e) {
      return null;
    } finally {
      if (in != nullin.close();
    }
  }
  private ThreadPoolExecutor createExecutor(final String name) {
    return createExecutor(name);
  }
  public static ThreadPoolExecutor createExecutor(final Configuration conffinal String name) {
    int maxThreads = conf.getInt("hbase.snapshot.thread.pool.max", 8);
    return Threads.getBoundedCachedThreadPool(maxThreads, 30L, .,
              Threads.getNamedThreadFactory(name));
  }

  
Extract the region encoded name from the region manifest
    byte[] regionName = HRegionInfo.createRegionName(
            ProtobufUtil.toTableName(manifest.getRegionInfo().getTableName()),
            manifest.getRegionInfo().getStartKey().toByteArray(),
            manifest.getRegionInfo().getRegionId(), true);
    return HRegionInfo.encodeRegionName(regionName);
  }
  /*
   * Return the snapshot format
   */
  private static int getSnapshotFormat(final SnapshotDescription desc) {
    return desc.hasVersion() ? desc.getVersion() : .;
  }
New to GrepCode? Check out our FAQ X