Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 
 
 package org.apache.hadoop.hbase.snapshot;
 
 import java.net.URI;
 import java.util.Date;
 import java.util.List;
 
 
 
Tool for dumping snapshot information.
  1. Table Descriptor
  2. Snapshot creation time, type, format version, ...
  3. List of hfiles and hlogs
  4. Stats about hfiles and logs sizes, percentage of shared with the source table, ...
 
 public final class SnapshotInfo extends Configured implements Tool {
   private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);

  
Statistics about the snapshot
  1. How many store files and logs are in the archive
  2. How many store files and logs are shared with the table
  3. Total store files and logs size and shared amount
 
   public static class SnapshotStats {
    
Information about the file referenced by the snapshot
 
     static class FileInfo {
       private final boolean corrupted;
       private final boolean inArchive;
       private final long size;
 
       FileInfo(final boolean inArchivefinal long sizefinal boolean corrupted) {
         this. = corrupted;
         this. = inArchive;
         this. = size;
       }

      

Returns:
true if the file is in the archive
 
       public boolean inArchive() {
         return this.;
       }

      

Returns:
true if the file is corrupted
 
       public boolean isCorrupted() {
         return this.;
       }

      

Returns:
true if the file is missing
      public boolean isMissing() {
        return this. < 0;
      }

      

Returns:
the file size
      public long getSize() {
        return this.;
      }
      String getStateToString() {
        if (isCorrupted()) return "CORRUPTED";
        if (isMissing()) return "NOT FOUND";
        if (inArchive()) return "archive";
        return null;
      }
    }
    private AtomicInteger hfileArchiveCount = new AtomicInteger();
    private AtomicInteger hfilesCorrupted = new AtomicInteger();
    private AtomicInteger hfilesMissing = new AtomicInteger();
    private AtomicInteger hfilesCount = new AtomicInteger();
    private AtomicInteger logsMissing = new AtomicInteger();
    private AtomicInteger logsCount = new AtomicInteger();
    private AtomicLong hfileArchiveSize = new AtomicLong();
    private AtomicLong hfileSize = new AtomicLong();
    private AtomicLong logSize = new AtomicLong();
    private final SnapshotDescription snapshot;
    private final TableName snapshotTable;
    private final Configuration conf;
    private final FileSystem fs;
    SnapshotStats(final Configuration conffinal FileSystem fsfinal SnapshotDescription snapshot)
    {
      this. = snapshot;
      this. = TableName.valueOf(snapshot.getTable());
      this. = conf;
      this. = fs;
    }

    

Returns:
the snapshot descriptor
      return this.;
    }

    

Returns:
true if the snapshot is corrupted
    public boolean isSnapshotCorrupted() {
      return .get() > 0 ||
             .get() > 0 ||
             .get() > 0;
    }

    

Returns:
the number of available store files
    public int getStoreFilesCount() {
      return .get() + .get();
    }

    

Returns:
the number of available store files in the archive
    public int getArchivedStoreFilesCount() {
      return .get();
    }

    

Returns:
the number of available log files
    public int getLogsCount() {
      return .get();
    }

    

Returns:
the number of missing store files
    public int getMissingStoreFilesCount() {
      return .get();
    }

    

Returns:
the number of corrupted store files
    public int getCorruptedStoreFilesCount() {
      return .get();
    }

    

Returns:
the number of missing log files
    public int getMissingLogsCount() {
      return .get();
    }

    

Returns:
the total size of the store files referenced by the snapshot
    public long getStoreFilesSize() {
      return .get() + .get();
    }

    

Returns:
the total size of the store files shared
    public long getSharedStoreFilesSize() {
      return .get();
    }

    

Returns:
the total size of the store files in the archive
    public long getArchivedStoreFileSize() {
      return .get();
    }

    

Returns:
the percentage of the shared store files
    public float getSharedStoreFilePercentage() {
      return ((float).get() / (.get() + .get())) * 100;
    }

    

Returns:
the total log size
    public long getLogsSize() {
      return .get();
    }

    
Add the specified store file to the stats

Parameters:
region region encoded Name
family family name
hfile store file name
Returns:
the store file information
    FileInfo addStoreFile(final HRegionInfo regionfinal String family,
        final SnapshotRegionManifest.StoreFile storeFilethrows IOException {
      HFileLink link = HFileLink.create(region.getEncodedName(),
                                        familystoreFile.getName());
      boolean isCorrupted = false;
      boolean inArchive = false;
      long size = -1;
      try {
        if ((inArchive = .exists(link.getArchivePath()))) {
          size = .getFileStatus(link.getArchivePath()).getLen();
          .addAndGet(size);
          .incrementAndGet();
        } else {
          size = link.getFileStatus().getLen();
          .addAndGet(size);
          .incrementAndGet();
        }
        isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size);
        if (isCorrupted.incrementAndGet();
      } catch (FileNotFoundException e) {
      }
      return new FileInfo(inArchivesizeisCorrupted);
    }

    
Add the specified log file to the stats

Parameters:
server server name
logfile log file name
Returns:
the log information
    FileInfo addLogFile(final String serverfinal String logfilethrows IOException {
      HLogLink logLink = new HLogLink(serverlogfile);
      long size = -1;
      try {
        size = logLink.getFileStatus().getLen();
        .addAndGet(size);
        .incrementAndGet();
      } catch (FileNotFoundException e) {
        .incrementAndGet();
      }
      return new FileInfo(falsesizefalse);
    }
  }
  private boolean printSizeInBytes = false;
  private FileSystem fs;
  private Path rootDir;
  public int run(String[] argsthrows IOExceptionInterruptedException {
    final Configuration conf = getConf();
    boolean listSnapshots = false;
    String snapshotName = null;
    boolean showSchema = false;
    boolean showFiles = false;
    boolean showStats = false;
    // Process command line args
    for (int i = 0; i < args.lengthi++) {
      String cmd = args[i];
      try {
        if (cmd.equals("-snapshot")) {
          snapshotName = args[++i];
        } else if (cmd.equals("-files")) {
          showFiles = true;
          showStats = true;
        } else if (cmd.equals("-stats")) {
          showStats = true;
        } else if (cmd.equals("-schema")) {
          showSchema = true;
        } else if (cmd.equals("-remote-dir")) {
          Path sourceDir = new Path(args[++i]);
          URI defaultFs = sourceDir.getFileSystem(conf).getUri();
          FSUtils.setFsDefault(confnew Path(defaultFs));
          FSUtils.setRootDir(confsourceDir);
        } else if (cmd.equals("-list-snapshots")) {
          listSnapshots = true;
        } else if (cmd.equals("-size-in-bytes")) {
           = true;
        } else if (cmd.equals("-h") || cmd.equals("--help")) {
          printUsageAndExit();
        } else {
          ..println("UNEXPECTED: " + cmd);
          printUsageAndExit();
        }
      } catch (Exception e) {
        printUsageAndExit();
      }
    }
    // List Available Snapshots
    if (listSnapshots) {
      SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
      ..printf("%-20s | %-20s | %s%n""SNAPSHOT""CREATION TIME""TABLE NAME");
      for (SnapshotDescription descgetSnapshotList(conf)) {
        ..printf("%-20s | %20s | %s%n",
                          desc.getName(),
                          df.format(new Date(desc.getCreationTime())),
                          desc.getTable());
      }
      return 0;
    }
    if (snapshotName == null) {
      ..println("Missing snapshot name!");
      printUsageAndExit();
      return 1;
    }
     = FSUtils.getRootDir(conf);
     = FileSystem.get(.toUri(), conf);
    .debug("fs=" + .getUri().toString() + " root=" + );
    // Load snapshot information
    if (!loadSnapshotInfo(snapshotName)) {
      ..println("Snapshot '" + snapshotName + "' not found!");
      return 1;
    }
    printInfo();
    if (showSchemaprintSchema();
    printFiles(showFilesshowStats);
    return 0;
  }

  
Load snapshot info and table descriptor for the specified snapshot

Parameters:
snapshotName name of the snapshot to load
Returns:
false if snapshot is not found
  private boolean loadSnapshotInfo(final String snapshotNamethrows IOException {
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName);
    if (!.exists(snapshotDir)) {
      .warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
      return false;
    }
    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(snapshotDir);
     = SnapshotManifest.open(getConf(), snapshotDirsnapshotDesc);
    return true;
  }

  
  private void printInfo() {
    SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
    ..println("Snapshot Info");
    ..println("----------------------------------------");
    ..println("   Name: " + snapshotDesc.getName());
    ..println("   Type: " + snapshotDesc.getType());
    ..println("  Table: " + snapshotDesc.getTable());
    ..println(" Format: " + snapshotDesc.getVersion());
    ..println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
    ..println();
  }

  
  private void printSchema() {
    ..println("Table Descriptor");
    ..println("----------------------------------------");
    ..println();
  }

  
Collect the hfiles and logs statistics of the snapshot and dump the file list if requested and the collected information.
  private void printFiles(final boolean showFilesfinal boolean showStatsthrows IOException {
    if (showFiles) {
      ..println("Snapshot Files");
      ..println("----------------------------------------");
    }
    // Collect information about hfiles and logs in the snapshot
    final String table = snapshotDesc.getTable();
    final SnapshotStats stats = new SnapshotStats(this.getConf(), this.snapshotDesc);
    SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), ,
        @Override
        public void storeFile(final HRegionInfo regionInfofinal String family,
            final SnapshotRegionManifest.StoreFile storeFilethrows IOException {
          if (storeFile.hasReference()) return;
          SnapshotStats.FileInfo info = stats.addStoreFile(regionInfofamilystoreFile);
          if (showFiles) {
            String state = info.getStateToString();
            ..printf("%8s %s/%s/%s/%s %s%n",
              (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
              tableregionInfo.getEncodedName(), familystoreFile.getName(),
              state == null ? "" : "(" + state + ")");
          }
        }
        @Override
        public void logFile (final String serverfinal String logfile)
            throws IOException {
          SnapshotStats.FileInfo info = stats.addLogFile(serverlogfile);
          if (showFiles) {
            String state = info.getStateToString();
            ..printf("%8s log %s on server %s (%s)%n",
              (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
              logfileserver,
              state == null ? "" : "(" + state + ")");
          }
        }
    });
    // Dump the stats
    ..println();
    if (stats.isSnapshotCorrupted()) {
      ..println("**************************************************************");
      ..printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
        stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
      ..printf("              %d hfile(s) corrupted.%n",
        stats.getCorruptedStoreFilesCount());
      ..println("**************************************************************");
    }
    if (showStats) {
      ..printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
        stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
        fileSizeToString(stats.getStoreFilesSize()),
        stats.getSharedStoreFilePercentage(),
      );
      ..printf("%d Logs, total size %s%n",
        stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
      ..println();
    }
  }
  private String fileSizeToString(long size) {
    return  ? Long.toString(size) : StringUtils.humanReadableInt(size);
  }
  private void printUsageAndExit() {
    ..printf("Usage: bin/hbase %s [options]%n"getClass().getName());
    ..println(" where [options] are:");
    ..println("  -h|-help                Show this help and exit.");
    ..println("  -remote-dir             Root directory that contains the snapshots.");
    ..println("  -list-snapshots         List all the available snapshots and exit.");
    ..println("  -size-in-bytes          Print the size of the files in bytes.");
    ..println("  -snapshot NAME          Snapshot to examine.");
    ..println("  -files                  Files and logs list.");
    ..println("  -stats                  Files and logs stats.");
    ..println("  -schema                 Describe the snapshotted table.");
    ..println();
    ..println("Examples:");
    ..println("  hbase " + getClass() + " \\");
    ..println("    -snapshot MySnapshot -files");
    System.exit(1);
  }

  
Returns the snapshot stats

Parameters:
conf the org.apache.hadoop.conf.Configuration to use
snapshot org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription to get stats from
Returns:
the snapshot stats
  public static SnapshotStats getSnapshotStats(final Configuration conf,
      final SnapshotDescription snapshotthrows IOException {
    Path rootDir = FSUtils.getRootDir(conf);
    FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotrootDir);
    SnapshotManifest manifest = SnapshotManifest.open(conffssnapshotDirsnapshot);
    final SnapshotStats stats = new SnapshotStats(conffssnapshot);
    SnapshotReferenceUtil.concurrentVisitReferencedFiles(conffsmanifest,
        @Override
        public void storeFile(final HRegionInfo regionInfofinal String family,
            final SnapshotRegionManifest.StoreFile storeFilethrows IOException {
          if (!storeFile.hasReference()) {
            stats.addStoreFile(regionInfofamilystoreFile);
          }
        }
        @Override
        public void logFile (final String serverfinal String logfilethrows IOException {
          stats.addLogFile(serverlogfile);
        }
    });
    return stats;
  }

  
Returns the list of available snapshots in the specified location

Parameters:
conf the org.apache.hadoop.conf.Configuration to use
Returns:
the list of snapshots
  public static List<SnapshotDescriptiongetSnapshotList(final Configuration conf)
      throws IOException {
    Path rootDir = FSUtils.getRootDir(conf);
    FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
    FileStatus[] snapshots = fs.listStatus(snapshotDir,
    List<SnapshotDescriptionsnapshotLists =
      new ArrayList<SnapshotDescription>(snapshots.length);
    for (FileStatus snapshotDirStatsnapshots) {
      snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fssnapshotDirStat.getPath()));
    }
    return snapshotLists;
  }

  
The guts of the main(java.lang.String[]) method. Call this method to avoid the main(java.lang.String[]) System.exit.

Parameters:
args
Returns:
errCode
Throws:
java.lang.Exception
  static int innerMain(final String [] argsthrows Exception {
    return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
  }
  public static void main(String[] argsthrows Exception {
     System.exit(innerMain(args));
  }
New to GrepCode? Check out our FAQ X