Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*-
   * See the file LICENSE for redistribution information.
   *
   * Copyright (c) 2002, 2013 Oracle and/or its affiliates.  All rights reserved.
   *
   */
  
  package com.sleepycat.je.rep.impl.networkRestore;
  
 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.BACKUP_FILE_COUNT;
 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.DISPOSED_COUNT;
 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.FETCH_COUNT;
 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.SKIP_COUNT;
 
 import java.io.File;
 import java.util.Date;
 import java.util.Set;
 
This class implements a hot network backup that permits it to obtain a consistent set of log files from any running environment that provides a LogFileFeeder service. This class thus plays the role of a client, and the running environment that of a server.

The log files that are retrieved over the network are placed in a directory that can serve as an environment directory for a JE stand alone or HA environment. If log files are already present in the target directory, it will try reuse them, if they are really consistent with those on the server. Extant log files that are no longer part of the current backup file set are deleted or are renamed, depending on how the backup operation was configured.

Renamed backup files have the following syntax: NNNNNNNN.bup.<backup number> where the backup number is the number associated with the backup attempt, rather than with an individual file. That is, the backup number is increased by one each time a backup is repeated in the same directory and log files actually needed to be renamed.

The implementation tries to be resilient in the face of network failures and minimizes the amount of work that might need to be done if the client or server were to fail and had to be restarted. Users of this API must be careful to ensure that the execute() completes successfully before accessing the environment. [Sam: should we create a je.lck file to ensure that this is the case?]

 
 public class NetworkBackup {
     /* The server that was chosen to supply the log files. */
     private final InetSocketAddress serverAddress;
 
     /* The environment directory into which the log files will be backed up */
     private final File envDir;
 
     /* The id used during logging to identify a node. */
     private final NameIdPair clientNameId;
 
     /*
      * Determines whether any existing log files in the envDir should be
      * retained under a different name (with a BUP_SUFFIX), or whether it
      * should be deleted.
      */
     private final boolean retainLogfiles;
 
     /*
     * The minimal VLSN that the backup must cover. Used to ensure that the
     * backup is sufficient to permit replay of a replication stream from a
     * feeder. It's NULL_VLSN if the VLSN does not matter, that is, it's a
     * backup for a standalone environment.
     */
    private final VLSN minVLSN;
    /*
     * The client abandons a backup attempt if the server is loaded beyond this
     * threshold
     */
    final int serverLoadThreshold;
    /* The RepImpl instance used in Protocol. */
    private final RepImpl repImpl;
    private final FileManager fileManager;
    /* The protocol used to communicate with the server. */
    private Protocol protocol;
    /* The channel connecting this client to the server. */
    private SocketChannel channel;
    /*
     * The message digest used to compute the digest as each log file is pulled
     * over the network.
     */
    /* Statistics on number of files actually fetched and skipped */
    private final StatGroup statistics;
    private final IntStat backupFileCount;
    private final IntStat disposedCount;
    private final IntStat fetchCount;
    private final IntStat skipCount;
    private final Logger logger;
    private CyclicBarrier testBarrier = null;

    
The receive buffer size associated with the socket used for the log file transfers
    private final int receiveBufferSize;

    
Time to wait for a request from the client.
    private static final int SOCKET_TIMEOUT_MS = 10000;

    
The number of times to retry on a digest exception. That is, when the SHA1 hash as computed by the server for the file does not match the hash as computed by the client for the same file.
    private static final int DIGEST_RETRIES = 5;

    
Creates a configured backup instance which when executed will backup the files to the environment directory.

Parameters:
serverSocket the socket on which to contact the server
receiveBufferSize the receive buffer size to be associated with the socket used for the log file transfers.
envDir the directory in which to place the log files
clientNameId the id used to identify this client
retainLogfiles determines whether obsolete log files should be retained by renaming them, instead of deleting them.
serverLoadThreshold only backup from this server if it has fewer than this number of feeders active.
minVLSN the VLSN that should be covered by the server. It ensures that the log files are sufficiently current for this client's needs.
Throws:
java.lang.IllegalArgumentException if the environment directory is not valid. When used internally, this should be caught appropriately.
    public NetworkBackup(InetSocketAddress serverSocket,
                         int receiveBufferSize,
                         File envDir,
                         NameIdPair clientNameId,
                         boolean retainLogfiles,
                         int serverLoadThreshold,
                         VLSN minVLSN,
                         RepImpl repImpl,
                         FileManager fileManager)
        throws IllegalArgumentException {
        super();
        this. = serverSocket;
        this. = receiveBufferSize;
        if (!envDir.exists()) {
            throw new IllegalArgumentException("Environment directory: " +
                                               envDir + " not found");
        }
        this. = envDir;
        this. = clientNameId;
        this. = retainLogfiles;
        this. = serverLoadThreshold;
        this. = minVLSN;
        this. = repImpl;
        this. = fileManager;
        try {
             = MessageDigest.getInstance("SHA1");
        } catch (NoSuchAlgorithmException e) {
            // Should not happen -- if it does it's a JDK config issue
            throw EnvironmentFailureException.unexpectedException(e);
        }
         = LoggerUtils.getLoggerFixedPrefix(getClass(),
                                                  clientNameId.toString(),
                                                  repImpl);
                                   .);
         = new IntStat();
         = new IntStat();
         = new IntStat();
    }

    
    public NetworkBackup(InetSocketAddress serverSocket,
                         File envDir,
                         NameIdPair clientNameId,
                         boolean retainLogfiles,
                         FileManager fileManager)
        throws DatabaseException {
        this(serverSocket,
             0,
             envDir,
             clientNameId,
             retainLogfiles,
             .,
             .,
             null,
             fileManager);
    }

    
Returns statistics associated with the NetworkBackup execution.
    public StatGroup getStats() {
        StatGroup ret = .cloneGroup(false);
        return ret;
    }

    
    public String[] execute()
        throws IOException,
               DatabaseException,
               ServiceConnectFailedException,
               LoadThresholdExceededException,
               InsufficientVLSNRangeException {
        try {
             = RepUtils.openBlockingChannel(true,
                                                   ,
                                                   );
            ServiceDispatcher.doServiceHandshake
                (.);
             = checkProtocol(new Protocol(,
                                                  .,
                                                  ));
            checkServer();
            final String[] fileNames = getFileList();
            .info("Restoring from:" +  +
                        " Allocated network receive buffer size:" +
                        .socket().getReceiveBufferSize() +
                        "(" +  + ")" +
                        " candidate log file count:" + fileNames.length);
            getFiles(fileNames);
            cleanup(fileNames);
            assert .listJDBFiles().length == fileNames.length :
                "envDir=" +  + " list=" +
                Arrays.asList(.listJDBFiles()) +
                " fileNames=" + Arrays.asList(fileNames);
            /*
             * The fileNames array is sorted in getFileList method, so we can
             * use the first and last array elements to get the range of the
             * files to be restored.
             */
            final long fileBegin = .getNumFromName(fileNames[0]);
            final long fileEnd =
                .getNumFromName(fileNames[fileNames.length - 1]);
            /* Return file names with sub directories' names if exists. */
            return .listFileNames(fileBeginfileEnd);
        } finally {
            if ( != null) {
                .socket().close();
                .close();
            }
            .info("Backup file total: " +
                        .get() +
                        ".  Files actually fetched: " +
                        .get() +
                        ".  Files skipped(available locally): " +
                        .get() +
                        ".  Local files renamed/deleted: " +
                        .get());
        }
    }

    
Ensures that the log file feeder is a suitable choice for this backup: The feeder must cover the VLSN if it's not null and must not be too busy.
    private void checkServer()
        throws IOException,
               ProtocolException,
               LoadThresholdExceededException,
               InsufficientVLSNRangeException {
        .write(.new FeederInfoReq(), );
        FeederInfoResp resp = .read(FeederInfoResp.class);
        if ((!..equals()) &&
            ((resp.getRangeFirst().compareTo() > 0) ||
             (resp.getRangeLast().compareTo()) < 0)) {
            throw new InsufficientVLSNRangeException(,
                                                     resp.getRangeFirst(),
                                                     resp.getRangeLast());
        }
        if (resp.getActiveFeeders() > ) {
            LoadThresholdExceededException exception
                = new LoadThresholdExceededException(,
                                                     resp.getActiveFeeders());
            throw exception;
        }
    }

    
Delete or rename residual jdb files that are not part of the log file set. This method is only invoked after all required files have been copied over from the server.

    private void cleanup(String[] fileNames)
        throws IOException {
        .fine("Cleaning up");
        Set<StringlogFileSet = new HashSet<String>(Arrays.asList(fileNames));
        for (File file : .listJDBFiles()) {
            if (!logFileSet.contains(file.getName())) {
                disposeFile(file);
            }
        }
        StringBuilder logFiles = new StringBuilder();
        for (String string : logFileSet) {
            /*
             * Use the full path of this file in case the environment uses
             * multiple data directories.
             */
            File file = new File(.getFullFileName(string));
            if (!file.exists()) {
                throw EnvironmentFailureException.unexpectedState
                    ("Missing file: " + file);
            }
            logFiles.append(file.getCanonicalPath()).append(", ");
        }
        String names = logFiles.toString();
        if (names.length() > 0) {
            names = names.substring(0, names.length()-2);
        }
        .fine("Log file set: " + names);
    }

    
Retrieves all the files in the list, that are not already in the envDir.

    private void getFiles(String[] fileNames)
        throws IOExceptionDatabaseException {
        .info(fileNames.length + " files in backup set");
        for (String fileName : fileNames) {
            if ( != null) {
                try {
                    .await();
                } catch (InterruptedException e) {
                    // Ignore just a test mechanism
                } catch (BrokenBarrierException e) {
                    throw EnvironmentFailureException.unexpectedException(e);
                }
            }
            /*
             * Use the full path of this file in case the environment uses
             * multiple data directories.
             */
            File file = new File(.getFullFileName(fileName));
            if (haveFile(file)) {
                .info("File: " + file.getCanonicalPath() +
                            " length: " + file.length() +
                            " available with matching SHA1, copy skipped");
                .increment();
                continue;
            }
            for (int i = 0; i < i++) {
                try {
                    getFile(file);
                    .increment();
                    break;
                } catch (DigestException e) {
                    if ((i + 1) == ) {
                        throw new IOException("Digest mismatch despite "
                                +  + " attempts");
                    }
                    continue;
                }
            }
        }
        /* All done, shutdown conversation with the server. */
        .write(.new Done(), );
    }

    
Returns true if the file in this directory is the same as the file on the server. The method tries to avoid requesting the SHA1 if the file lengths are not equal, since computing the SHA1 if it's not already cached requires a pass over the the log file. Note that the server will always send back the SHA1 value if it has it cached.
    private boolean haveFile(File fileboolean getSHA1)
        throws IOExceptionDatabaseException {
        if (!file.exists()) {
            return false;
        }
        .write(.new
                       FileInfoReq(file.getName(), getSHA1), );
        FileInfoResp statResp =
            .read(Protocol.FileInfoResp.class);
        long fileLength = file.length();
        if (statResp.getFileLength() != fileLength) {
            return false;
        }
        if (statResp.getDigestSHA1().length == 0) {
            assert(!getSHA1);
            return haveFile(filetrue); // request SHA1 this time around
        }
        byte digest[] =
            LogFileFeeder.getSHA1Digest(filefile.length()).digest();
        return Arrays.equals(digeststatResp.getDigestSHA1());
    }

    
Convenience overloading for the above method.
    private boolean haveFile(File file)
        throws IOExceptionDatabaseException {
        return haveFile(filefalse);
    }

    
Requests and obtains the specific log file from the server. The file is first created under a name with the .tmp suffix and is renamed to its true name only after its digest has been verified. This method is protected to facilitate error testing.
    protected void getFile(File file)
        throws IOExceptionProtocolExceptionDigestException {
        .fine("Requesting file: " + file);
        .write(.new FileReq(file.getName()), );
        FileStart fileResp = .read(Protocol.FileStart.class);
        /*
         * Delete the tmp file if it already exists.
         *
         * Use the full path of this file in case the environment uses multiple
         * data directories.
         */
        File tmpFile = new File(.getFullFileName(file.getName()) +
                                .);
        if (tmpFile.exists()) {
            boolean deleted = tmpFile.delete();
            if (!deleted) {
                throw EnvironmentFailureException.unexpectedState
                    ("Could not delete file: " + tmpFile);
            }
        }
        /*
         * Use a direct buffer to avoid an unnecessary copies into and out of
         * native buffers.
         */
        final ByteBuffer buffer =
                ByteBuffer.allocateDirect(.);
        .reset();
        /* Write the tmp file. */
        final FileOutputStream fileStream = new FileOutputStream(tmpFile);
        final FileChannel fileChannel = fileStream.getChannel();
        try {
            /* Copy over the file contents. */
            for (long bytes = fileResp.getFileLength(); bytes > 0;) {
                int readSize =
                    (int) Math.min(.bytes);
                buffer.clear();
                buffer.limit(readSize);
                int actualBytes = .read(buffer);
                if (actualBytes == -1) {
                    throw new IOException("Premature EOF. Was expecting:"
                                          + readSize);
                }
                bytes -= actualBytes;
                buffer.flip();
                fileChannel.write(buffer);
                buffer.rewind();
                .update(buffer);
            }
            .info(String.format("Fetched log file: %s, size: %,d bytes",
                                      file.getName(),
                                      fileResp.getFileLength()));
        } finally {
            fileStream.close();
        }
        final FileEnd fileEnd = .read(Protocol.FileEnd.class);
        /* Check that the read is successful. */
        if (!Arrays.equals(.digest(), fileEnd.getDigestSHA1())) {
            .warning("digest mismatch on file: " + file);
            throw new DigestException();
        }
        /* Now that we know it's good, move the file into place. */
        if (file.exists()) {
            /* delete or back up this and all subsequent obsolete files */
            disposeObsoleteFiles(file);
        }
        /* Rename the tmp file. */
        .fine("Renamed " + tmpFile + " to " + file);
        boolean renamed = tmpFile.renameTo(file);
        if (!renamed) {
            throw EnvironmentFailureException.unexpectedState
                ("Rename from: " + tmpFile + " to " + file + " failed");
        }
        /* Retain last modified time, to leave an audit trail. */
        if (!file.setLastModified(fileResp.getLastModifiedTime())) {
            throw EnvironmentFailureException.unexpectedState
                ("File.setlastModifiedTime() for:" + file + " and time " +
                 new Date(fileResp.getLastModifiedTime()) + " failed.");
        }
    }

    
Renames (or deletes) this log file, and all other files following it in the log sequence. The operation is done from the highest file down to this one, to ensure the integrity of the log files in the directory is always preserved.

Parameters:
file the lowest numbered log file that must be renamed or deleted
    private void disposeObsoleteFiles(File startFile) {
        File[] dirFiles = .listJDBFiles();
        Arrays.sort(dirFiles); // sorts in ascending order
        /* Start with highest numbered file to be robust in case of failure. */
        for (int i = dirFiles.length - 1; i >= 0; i--) {
            File file = dirFiles[i];
            disposeFile(file);
            if (startFile.equals(file)) {
                break;
            }
        }
    }

    
Remove the file from the current set of log files in the directory.

Parameters:
file
    private void disposeFile(File file) {
        .increment();
        if () {
            long fileNumber = .getNumFromName(file.getName());
            boolean renamed = false;
            try {
                renamed =
                    .renameFile(fileNumber.);
            } catch (IOException e) {
                throw EnvironmentFailureException.unexpectedState
                    ("Could not rename log file " + file.getPath() +
                     " because of exception: " + e.getMessage());
            }
            if (!renamed) {
                throw EnvironmentFailureException.unexpectedState
                    ("Could not rename log file " +  file.getPath());
            }
            .fine("Renamed log file: " + file.getPath());
        } else {
            boolean deleted = file.delete();
            if (!deleted) {
                throw EnvironmentFailureException.unexpectedState
                    ("Could not delete log file " +  file.getPath());
            }
            .fine("deleted log file: " + file.getPath());
        }
    }

    
Carries out the message exchange to obtain the list of backup files.

    private String[] getFileList()
        throws IOExceptionProtocolException {
        .write(.new FileListReq(), );
        FileListResp fileListResp = .read(,
                                                  Protocol.FileListResp.class);
        String[] fileList = fileListResp.getFileNames();
        Arrays.sort(fileList); //sort the file names in ascending order
        .set(fileList.length);
        return fileList;
    }

    
Verify that the protocols are compatible, switch to a different protocol version, if we need to.

    private Protocol checkProtocol(Protocol candidateProtocol)
        throws IOExceptionProtocolException {
        candidateProtocol.write
            (candidateProtocol.new ClientVersion(), );
        ServerVersion serverVersion =
            candidateProtocol.read(Protocol.ServerVersion.class);
        if (serverVersion.getVersion() != candidateProtocol.getVersion()) {
            String message = "Server requested protocol version:"
                    + serverVersion.getVersion()
                    + " but the client version is " +
                    candidateProtocol.getVersion();
            .info(message);
            throw new ProtocolException(message);
        }
        /*
         * In future we may switch protocol versions to accommodate the server.
         * For now, simply return the one and only version.
         */
        return candidateProtocol;
    }
    /*
     * @hidden
     *
     * A test entry point used to simulate a slow network restore.
     */
    public void setTestBarrier(CyclicBarrier  testBarrier) {
        this. = testBarrier;
    }

    
Exception indicating that the digest sent by the server did not match the digest computed by the client, that is, the log file was corrupted during transit.
    @SuppressWarnings("serial")
    protected static class DigestException extends Exception {
    }

    
Exception indicating that the server vlsn range did not cover the VLSN of interest.
    @SuppressWarnings("serial")
    public static class InsufficientVLSNRangeException extends Exception {
        /* The VLSN that must be covered by the server. */
        private final VLSN minVLSN;
        /* The actual range covered by the server. */
        private final VLSN rangeFirst;
        private final VLSN rangeLast;
        public InsufficientVLSNRangeException(VLSN minVLSN,
                                              VLSN rangeFirst,
                                              VLSN rangeLast) {
            this. = minVLSN;
            this. = rangeFirst;
            this. = rangeLast;
        }
        public VLSN getMinVLSN() {
            return ;
        }
        public VLSN getRangeFirst() {
            return ;
        }
        @Override
        public String getMessage() {
            return "Insufficient VLSN range. Needed VLSN: " +  +
                   " Available range: " +
                   "[" +  + ", " +  + "]";
        }
    }
    @SuppressWarnings("serial")
    public static class LoadThresholdExceededException extends Exception {
        final int threshold;
        final int activeServers;
        LoadThresholdExceededException(int threshold,
                                       int activeServers) {
            assert(activeServers > threshold);
            this. = threshold;
            this. = activeServers;
        }
        public int getActiveServers() {
            return ;
        }
        public int getThreshold() {
            return ;
        }
        @Override
        public String getMessage() {
            return "Active server threshold: " +  + " exceeded. " +
            "Active servers: " + ;
        }
    }
New to GrepCode? Check out our FAQ X