Verifying Full Backup Files

| backup_database
  \  
  | BACKUP_COMPLETE
  | db.backup
  | db_0.backup
  | db_0.backup_columns0.tar
  | db_0.backup_columns1.tar
  | db_1.backup
  | db_2.backup
  | db_2.backup_columns0.tar
  | db_2.backup_columns1.tar
  | db_2.backup_columns2.tar
  | db_3.backup
  | db_3.backup_columns0.tar

# verify_backup.py
#
# Given a directory, this script will verify that the backup crc and size both
# are unchanged.
#
# REQUIRES: crcmod to be installed: https://pypi.org/project/crcmod/
#
# USAGE: python verify_backup.py /absolute/path/to/backup
#
# NOTE: This script needs read privileges on all files being verified.
#
import crcmod
import glob
import json
import sys
import errno

# VerifyBackup:
#   Verifies the CRC located in the backup sentinel file (BACKUP_COMPLETE)
#   matches the calculated CRC of files in backupDirectory.
#
# Param backupDirectory: absolute path to directory where backup exists.
# Return: 0 on success, -1 on failure.
#
def verifyBackup(backupDirectory):
    # Strip off trailing '/' if exists.
    #
    if backupDirectory[-1] == '/':
        backupDirectory = backupDirectory[:(len(backupDirectory)-1)]

    with open("%s/BACKUP_COMPLETE" % backupDirectory, "r") as f:
        buf = f.read()
        backupDictionary = json.loads(buf)
        try:
            finalCrc = backupDictionary["Checksum"]
            dbName = backupDictionary["Database_Name"]
            numPartitions = int(backupDictionary["Num_Partitions"])
        except KeyError as e:
            print e
            print "Sentinel File 'BACKUP_COMPLETE' is from unsupported version of backup."
            return -1

    # This is in the crc32c specification, crcmod also has crc32c hardcoded,
    # so either can be used.
    #
    crc = crcmod.Crc(0x11EDC6F41, rev=True, initCrc=0x00000000, xorOut =0xFFFFFFFF)
    crclist = ""

    # Process the reference snapshot.
    #
    with open("%s/%s.backup" % (backupDirectory, dbName), "r") as f:
            buf = f.read()
            crc.update(buf)
    crclist += crc.hexdigest()

    # Process each partition.
    #
    for i in range (numPartitions):
        crc = crc.new()

        # Process Partition snapshot.
        # Each Partition MUST have a snapshot.
        #
        with open("%s/%s_%d.backup" % (backupDirectory, dbName, i), "r") as f:
            buf = f.read()
            crc.update(buf)
        crclist += crc.hexdigest()

        # Snapshots and Columns are checksummed with separate CRC's.
        #
        crc = crc.new()

        # To emulate a do while loop in Python.
        tarFound = True

        # If the columnar blobs is non empty, append the crc to the list.
        columnCrc = False
        j = 0

        # Process all tarballed columnstore files.
        #
        # NOTE: Even if a database has a columnstore, this does not imply
        # each partition has columnar blobs. The data might exist in the
        # rowstore snapshot or might be skewed such that all rows exist
        # in other partitions.
        #
        while tarFound:
            try:
                with open("%s/%s_%d.backup_columns%d.tar" % (backupDirectory, dbName, i,j) , "r") as f:
                    buf = f.read()
                    crc.update(buf)
                    j += 1
                    columnCrc = True
            except IOError as e:
                if e.errno == errno.ENOENT:
                    tarFound = False
                else:
                    assert e
        if columnCrc:
            crclist += crc.hexdigest()

    # CRC's will be of different case, make both uppercase.
    #
    if crclist != finalCrc.upper():
        print "Crc calculated from directory:" + crclist
        print "Crc in backup file :" + finalCrc
        print "Crcs do not match!"
        return -1
    return 0

if __name__ == '__main__':
    backupDirectory = sys.argv[1]
    if len(sys.argv) != 2:
        print "Incorrect usage: please include just the directory where the backup is located."
    sys.exit(verifyBackup(backupDirectory))