#!/usr/bin/python3 -BEsStt
"""This tool performs operations on a local data storage, at
the moment only checking that all file names are sane, warning
about unexpected files, e.g. from failed transfers."""

import datetime
import json
import os
import re
import sys

# Adjust the Python sites path to include only the guerillabackup
# library addons, thus avoiding a large set of python site packages
# to be included in code run with root privileges. Also remove
# the local directory from the site path.
sys.path = sys.path[1:]+['/usr/lib/guerillabackup/lib', '/etc/guerillabackup/lib-enabled']

import guerillabackup.Utils
from guerillabackup.storagetool.PolicyTypeInterval import PolicyTypeInterval
from guerillabackup.storagetool.PolicyTypeLevelRetention import PolicyTypeLevelRetention
from guerillabackup.storagetool.PolicyTypeSize import PolicyTypeSize


class PolicyGroup():
  """A policy group is a list of policies to be applied to sources
  matching a given regular expression."""

  def __init__(self, groupConfig):
    """Create a new policy configuration group."""
    if not isinstance(groupConfig, dict):
      raise Exception('Policies entry not a dictionary')
    self.inheritFlag = True
    self.resourceRegex = re.compile(groupConfig['Sources'])
    if 'Inherit' in groupConfig:
      self.inheritFlag = groupConfig['Inherit']
      if not isinstance(self.inheritFlag, bool):
        raise Exception(
            'Inherit policy configuration flag has to be ' \
            'true or false (defaults to true)')

    self.policyList = []
    for policyConfig in groupConfig['List']:
      policy = {
          PolicyTypeInterval.POLICY_NAME: PolicyTypeInterval,
          PolicyTypeLevelRetention.POLICY_NAME: PolicyTypeLevelRetention,
          PolicyTypeSize.POLICY_NAME: PolicyTypeSize
      }.get(policyConfig['Name'], None)
      if policy is None:
        raise Exception(
            'Unknown policy type "%s"' % policyConfig['Name'])
      self.policyList.append(policy(policyConfig))

  def isApplicableToSource(self, sourceName):
    """Check if this policy is applicable to a given source."""
    return self.resourceRegex.match(sourceName) is not None

  def isPolicyInheritanceEnabled(self):
    """Check if policy inheritance is enabled for this group."""
    return self.inheritFlag

  def getPolicies(self):
    """Get the list of policies in this group."""
    return self.policyList


class StorageConfig():
  """This class implements the configuration of a storage location
  without the status information."""

  def __init__(self, fileName, parentConfig):
    """Create a storage configuration from a given file name.
    @param fileName the filename where to load the storage configuration.
    @param parentConfig this is the parent configuration that
    included this configuration and therefor may also define
    policies."""
    self.configFileName = fileName
    self.parentConfig = parentConfig
    self.dataDir = None
# This is the list of files to ignore within the data directory.
    self.ignoreFileList = []
# Keep all policies in a list as we need to loop over all of
# them anyway.
    self.policyList = None
    self.storageStatusFileName = None
    self.storageStatus = None
    self.includedConfigList = []

    if not os.path.exists(self.configFileName):
      raise Exception(
          'Configuration file "%s" does not exist' % self.configFileName)

    config = guerillabackup.Utils.jsonLoadWithComments(self.configFileName)
    includeFileList = []
    if not isinstance(config, dict):
      raise Exception()
    for configName, configData in config.items():
      if configName == 'DataDir':
        if not isinstance(configData, str):
          raise Exception()
        self.dataDir = self.canonicalizePathname(configData)
        if not os.path.isdir(self.dataDir):
          raise Exception(
              'Data directory %s in configuration %s does ' \
              'not exist or is not a directory' % (
                  self.dataDir, self.configFileName))
        continue
      if configName == 'Ignore':
        self.ignoreFileList = configData
        continue
      if configName == 'Include':
        includeFileList = configData
        continue
      if configName == 'Policies':
        self.initPolicies(configData)
        continue
      if configName == 'Status':
        if not isinstance(configData, str):
          raise Exception()
        self.storageStatusFileName = self.canonicalizePathname(configData)
        if os.path.exists(self.storageStatusFileName):
          if not os.path.isfile(self.storageStatusFileName):
            raise Exception('Status file "%s" has to be file' % configData)
          self.storageStatus = StorageStatus(
              self,
              guerillabackup.Utils.jsonLoadWithComments(
                  self.storageStatusFileName))
        continue
      raise Exception('Invalid configuration section "%s"' % configName)

# Always have a status object even when not loaded from file.
    if self.storageStatus is None:
      self.storageStatus = StorageStatus(self, None)

# Initialize the included configuration only after initialization
# of this object was completed because we pass this object as
# parent so it might already get used.
    for includeFileName in includeFileList:
      includeFileName = self.canonicalizePathname(includeFileName)
      if not os.path.isfile(includeFileName):
        raise Exception(
            'Included configuration %s in configuration %s ' \
            'does not exist or is not a file' % (
                includeFileName, self.configFileName))
      try:
        includedConfig = StorageConfig(includeFileName, self)
        self.includedConfigList.append(includedConfig)
      except:
        print(
            'Failed to load configuration "%s" included from "%s".' % (
                includeFileName, self.configFileName),
            file=sys.stderr)
        raise

  def getConfigFileName(self):
    """Get the name of the file defining this configuration."""
    return self.configFileName

  def canonicalizePathname(self, pathname):
    """Canonicalize a pathname which might be a name relative
    to the configuration file directory or a noncanonical absolute
    path."""
    if not os.path.isabs(pathname):
      pathname = os.path.join(os.path.dirname(self.configFileName), pathname)
    return os.path.realpath(pathname)

  def getStatusFileName(self):
    """Get the name of the status file to use to keep validation
    status information between validation runs."""
    return self.storageStatusFileName

  def getStatus(self):
    """Get the status object associated with this configuration."""
    return self.storageStatus

  def initPolicies(self, policyConfig):
    """Initialize all policies from the given JSON policy configuration.
    The configuration may contain multiple policy definitions
    per resource name regular expression, but that is just for
    convenience and not reflected in the policy data structures."""
    if not isinstance(policyConfig, list):
      raise Exception('Policies not a list')

    policyList = []
    for policyGroupConfig in policyConfig:
      policyList.append(PolicyGroup(policyGroupConfig))
    self.policyList = policyList

  def getDataDirectoryRelativePath(self, pathname):
    """Get the pathname of a file relative to the data directory.
    @param pathname the absolute path name to resolve."""
    return os.path.relpath(pathname, self.dataDir)

  def initializeStorage(self, storageFileDict):
    """Initialize the storage by locating all files in this data
    storage directory and also apply already known status information."""

# First load included configuration data to assign the most specific
# configuration and status to those resources.
    for includeConfig in self.includedConfigList:
      includeConfig.initializeStorage(storageFileDict)

# Walk the data directory to include any files not included yet.
    for dirName, subDirs, subFiles in os.walk(self.dataDir):
      for fileName in subFiles:
        fileName = os.path.join(dirName, fileName)
        fileInfo = storageFileDict.get(fileName, None)
        if fileInfo is not None:
          refConfig = fileInfo.getConfig()
          if refConfig == self:
            raise Exception('Logic error')
          if self.dataDir == refConfig.dataDir:
            raise Exception(
                'Data directory "%s" part of at least two ' \
                '(included) configurations' % self.dataDir)
          if (not refConfig.dataDir.startswith(self.dataDir)) or \
              (refConfig.dataDir[len(self.dataDir)] != '/'):
            raise Exception(
                'Directory tree inconsistency due to logic ' \
                'error or concurrent (malicious) modifictions')
        else:
# Add the file and reference to this configuration or override
# a less specific previous one.
          storageFileDict[fileName] = StorageFileInfo(self.storageStatus)

# Now detect all the valid resources and group them.
    self.storageStatus.updateResources(storageFileDict)
    self.storageStatus.validate(storageFileDict)

  def getSourcePolicies(self, sourceName):
    """Get the list of policies to be applied to a source."""
    result = []
    if self.parentConfig is not None:
# No need to clone the list as the topmost configuration will
# have started with a newly created list anyway.
      result = self.parentConfig.getSourcePolicies(sourceName)

    for policyGroup in self.policyList:
# Ignore policies not matching the source.
      if not policyGroup.isApplicableToSource(sourceName):
        continue

      if not policyGroup.isPolicyInheritanceEnabled():
        result = []

      for policy in policyGroup.getPolicies():
        for refPos, refPolicy in enumerate(result):
          refPolicy = result[refPos]
          if refPolicy.getPolicyName() == policy.getPolicyName():
            if refPolicy.getPriority() < policy.getPriority():
              result[refPos] = policy
            policy = None
            break
        if policy is not None:
          result.append(policy)
    return result


class BackupDataElement():
  """This class stores information about a single backup data
  element that is the reference to the backup data itself but
  also the meta information. The class is designed in a way to
  support current file based backup data element storage but
  also future storages where the meta information may end up
  in a relational database and the backup data is kept in a storage
  system."""

# This name is used to store deletion status information in the
# storage status information in the same way as policy data. Unlike
# policy data, the delete status data cannot be serialized.
  DELETE_STATUS_POLICY = 'Delete'

  def __init__(self, sourceStatus, dateTimeId, elementType):
    """Create a element as part of the status of one backup source.
    @param sourceStatus this is the reference to the complete
    status of the backup source this element is belonging to.
    @param dateTimeId the datetime string when this element was
    created and optional ID number part. It has to contain at
    least a 14 digit timestamp but may include more digits
    which then sorted according to their integer value."""
    self.sourceStatus = sourceStatus
    if (len(dateTimeId) < 14) or (not dateTimeId.isnumeric()):
      raise Exception('Datetime to short or not numeric')
    self.dateTimeId = dateTimeId
    if elementType not in ('full', 'inc'):
      raise Exception()
    self.type = elementType
    self.dataFileName = None
    self.dataLength = None
    self.infoFileName = None
# This dictionary contains policy data associated with this element.
# The key is the name of the policy holding the data.
    self.policyData = {}

  def getSourceStatus(self):
    """Get the complete source status information this backup
    data element is belonging to.
    @return the source status object."""
    return self.sourceStatus

  def getDateTimeId(self):
    """Get the date, time and ID string of this element."""
    return self.dateTimeId

  def setFile(self, fileName, fileType):
    """Set a file of given type to define this backup data element."""
    if fileType == 'data':
      if self.dataFileName is not None:
        raise Exception(
            'Logic error redefining data file for "%s"' % (
                self.sourceStatus.getSourceName()))
      if not fileName.endswith('.data'):
        raise Exception()
      self.dataFileName = fileName
    elif fileType == 'info':
      if self.infoFileName is not None:
        raise Exception()
      self.infoFileName = fileName
    else:
      raise Exception('Invalid file type %s' % fileType)

  def getElementName(self):
    """Get the name of the element in the storage."""
    sourceName = self.sourceStatus.getSourceName()
    elementStart = sourceName.rfind('/') +1
    partStr = '%s-%s-%s' % (
        self.dateTimeId, sourceName[elementStart:], self.type)
    if elementStart:
      return '%s%s' % (sourceName[:elementStart], partStr)
    return partStr

  def getDatetimeSeconds(self):
    """Get the datetime part of this element as seconds since
    epoche."""
    dateTime = datetime.datetime.strptime(self.dateTimeId[:14], '%Y%m%d%H%M%S')
# FIXME: no UTC conversion yet.
    return int(dateTime.timestamp())

  def getDatetimeTuple(self):
    """Get the datetime of this element as a tuple.
    @return a tuple with the datetime part as string and the
    serial part as integer or -1 when there is no serial part."""
    dateTime = self.dateTimeId[:14]
    serial = -1
    if len(self.dateTimeId) > 14:
      serialStr = self.dateTimeId[14:]
      if (serialStr[0] == '0') and (len(serialStr) > 1):
        raise Exception()
      serial = int(serialStr)
    return (dateTime, serial)

  def getType(self):
    """Get the type of this backup data element."""
    return self.type

  def getDataLength(self):
    """Get the length of the binary backup data of this element."""
    if self.dataLength is None:
      self.dataLength = os.stat(self.dataFileName).st_size
    return self.dataLength

  def getStatusData(self):
    """Get the complete status data associated with this element.
    Currently this data is identical to the complete policy data.
    @return the status data or None when there is no status data
    associated with this element."""
    return self.policyData

  def getPolicyData(self, policyName):
    """Get the policy data for a given policy name.
    @return the data or None when there was no data defined yet."""
    return self.policyData.get(policyName, None)

  def setPolicyData(self, policyName, data):
    """Set the policy data for a given policy name."""
    self.policyData[policyName] = data

  def updatePolicyData(self, policyName, data):
    """Update the policy data for a given policy name by adding
    or overriding the data.
    @return the updated policy data"""
    if not isinstance(data, dict):
      raise Exception()
    policyData = self.policyData.get(policyName, None)
    if policyData is None:
      policyData = dict(data)
      self.policyData[policyName] = policyData
    else:
      policyData.update(data)
    return policyData

  def removePolicyData(self, policyName):
    """Remove the policy data for a given policy name if it exists."""
    if policyName in self.policyData:
      del self.policyData[policyName]

  def initPolicyData(self, data):
    """Initialize the complete policy data of this backup data
    element. This function may only be called while there is
    no policy data defined yet."""
    if self.policyData:
      raise Exception(
          'Attempted to initialize policy data twice for %s' % (
              repr(self.dataFileName)))
    self.policyData = data

  def findUnsuedPolicyData(self, policyNames):
    """Check if this element contains policy data not belonging
    to any policy.
    @return the name of the first unused policy found or None."""
    for key in self.policyData.keys():
      if key not in policyNames:
        return key
    return None

  def markForDeletion(self, deleteFlag):
    """Mark this element for deletion if it was not marked at
    all or marked in the same way.
    @param deleteFlag the deletion mark to set or None to remove
    the current mark.
    @raise Exception if there is already a conflicting mark."""

    if (deleteFlag is not None) and (not isinstance(deleteFlag, bool)):
      raise Exception()

    policyData = self.getPolicyData(BackupDataElement.DELETE_STATUS_POLICY)
    if (policyData is not None) and (not isinstance(policyData, bool)):
      raise Exception()

    if deleteFlag is None:
      if policyData:
        self.removePolicyData(BackupDataElement.DELETE_STATUS_POLICY)
    else:
      if (policyData is not None) and (policyData != deleteFlag):
        raise Exception()
      self.setPolicyData(BackupDataElement.DELETE_STATUS_POLICY, deleteFlag)

  def isMarkedForDeletion(self):
    """Check if this element is marked for deletion."""
    policyData = self.getPolicyData(BackupDataElement.DELETE_STATUS_POLICY)
    if (policyData is not None) and (not isinstance(policyData, bool)):
      raise Exception()
    return bool(policyData)

  def delete(self):
    """Delete all resources associated with this backup data
    element."""
    os.unlink(self.dataFileName)
    os.unlink(self.infoFileName)


class BackupSourceStatus():
  """This class stores status information about a single backup
  source, e.g. all BackupDataElements belonging to this source,
  current policy status information ..."""

  def __init__(self, storageStatus, sourceName):
    self.storageStatus = storageStatus
# This is the unique name of this source.
    self.sourceName = sourceName
# This dictionary contains all backup data elements belonging
# to this source with datetime and type as key.
    self.dataElements = {}

  def getStorageStatus(self):
    """Get the complete storage status containing the status
    of this backup source.
    @return the storage status object."""
    return self.storageStatus

  def getSourceName(self):
    """Get the name of the source that created the backup data
    elements.
    @return the name of the source."""
    return self.sourceName

  def addFile(self, fileName, dateTimeId, elementType, fileType):
    """Add a file to this storage status. With the current file
    storage model, two files will define a backup data element.
    @param fileName the file to add.
    @param dateTimeId the datetime and additional serial number
    information.
    @param elementType the type of element to create, i.e. full
    or incremental.
    @param fileType the type of the file, i.e. data or info."""
    key = (dateTimeId, elementType)
    element = self.dataElements.get(key, None)
    if element is None:
      element = BackupDataElement(self, dateTimeId, elementType)
      self.dataElements[key] = element
    element.setFile(fileName, fileType)

  def getDataElementList(self):
    """Get the sorted list of all data elements."""
    result = list(self.dataElements.values())
    result.sort(key=lambda x: x.getDatetimeTuple())
    return result

  def findElementByKey(self, dateTimeId, elementType):
    """Find an element by the identification key values.
    @return the element or None when not found."""
    return self.dataElements.get((dateTimeId, elementType), None)

  def getElementIdString(self, element):
    """Get the identification string of a given element of this
    source."""
    idStr = ''
    pathEndPos = self.sourceName.rfind('/')
    if pathEndPos >= 0:
      idStr = '%s/%s-%s-%s' % (
          self.sourceName[:pathEndPos],
          element.getDateTimeId(),
          self.sourceName[pathEndPos+1:],
          element.getType())
    else:
      idStr = '%s-%s-%s' % (
          element.getDateTimeId(), self.sourceName,
          element.getType())
    return idStr

  def removeDeleted(self):
    """Remove all elements that are marked deleted. The method
    should only be invoked after applying all deletion policies."""
    for key in list(self.dataElements.keys()):
      element = self.dataElements[key]
      if element.isMarkedForDeletion():
        del self.dataElements[key]

  def serializeStatus(self):
    """Serialize the status of all files belonging to this source.
    @return a dictionary with the status information."""
    status = {}
    for element in self.dataElements.values():
      statusData = element.getStatusData()
# Do not serialize the deletion policy data.
      if BackupDataElement.DELETE_STATUS_POLICY in statusData:
        statusData = dict(statusData)
        del statusData[BackupDataElement.DELETE_STATUS_POLICY]
        if not statusData:
          statusData = None
      if statusData:
        status[self.getElementIdString(element)] = statusData
    return status


class StorageStatus():
  """This class keeps track about the status of one storage location,
  that are all tracked resources but also unrelated files within
  the storage location."""

  RESOURCE_NAME_REGEX = re.compile(
      '^(?P<datetime>[0-9]{14,})-(?P<name>[0-9A-Za-z.-]+)-' \
      '(?P<type>full|inc)\\.(?P<element>data|info)$')

  def __init__(self, config, statusData):
    self.config = config
    self.statusData = statusData
    if self.statusData is None:
      self.statusData = {}
# This dictionary contains the name of each source (not file)
# found in this storage as key and the BackupSourceStatus element
# bundling all relevant information about the source.
    self.trackedSources = {}

  def getConfig(self):
    """Get the configuration managing this status."""
    return self.config

  def getStatusFileName(self):
    """Get the file name holding this backup status data."""
    return self.config.getStatusFileName()

  def findElementByName(self, name):
    """Find a backup element tracked by this status object by
    name.
    @return the element when found or None"""
    relFileName = name + '.data'
    nameStartPos = relFileName.rfind('/') + 1
    match = StorageStatus.RESOURCE_NAME_REGEX.match(
        relFileName[nameStartPos:])
    if match is None:
      raise Exception(
          'Invalid element name "%s"' % str(name))
    sourceName = match.group('name')
    if nameStartPos != 0:
      sourceName = relFileName[:nameStartPos] + sourceName
    if sourceName not in self.trackedSources:
      return None
    sourceStatus = self.trackedSources[sourceName]
    return sourceStatus.findElementByKey(
        match.group('datetime'), match.group('type'))

  def validate(self, storageFileDict):
    """Validate that all files tracked in the status can be still
    found in the list of all storage files."""
    for elemName in self.statusData.keys():
      targetFileName = os.path.join(self.config.dataDir, elemName + '.data')
      if targetFileName not in storageFileDict:
        raise Exception(
            'Invalid status of nonexisting file "%s.data" ' \
            'in data directory "%s"' % (
                elemName, self.config.dataDir))

  def updateResources(self, storageFileDict):
    """Update the status of valid sources.
    @param storageFileDict the dictionary keeping track of known
    files from all configurations."""

    ignoreSet = set()
    ignoreSet.update(self.config.ignoreFileList)
    for fileName, fileInfo in storageFileDict.items():
      if fileInfo.getStatus() != self:
        continue
      relFileName = self.config.getDataDirectoryRelativePath(fileName)
      if relFileName in ignoreSet:
        ignoreSet.remove(relFileName)
        continue

      nameStartPos = relFileName.rfind('/') + 1
      match = StorageStatus.RESOURCE_NAME_REGEX.match(
          relFileName[nameStartPos:])
      if match is None:
        print(
            'File "%s" (absolute "%s") should be ignored by ' \
            'config "%s".' % (
                fileInfo.getConfig().getDataDirectoryRelativePath(fileName),
                fileName, fileInfo.getConfig().getConfigFileName()),
            file=sys.stderr)
        continue
      sourceName = match.group('name')
      if nameStartPos != 0:
        sourceName = relFileName[:nameStartPos] + sourceName

      sourceStatus = self.trackedSources.get(sourceName, None)
      if sourceStatus is None:
        sourceStatus = BackupSourceStatus(self, sourceName)
        self.trackedSources[sourceName] = sourceStatus
      sourceStatus.addFile(
          fileName, match.group('datetime'), match.group('type'),
          match.group('element'))
      fileInfo.setBackupSource(sourceStatus)

    for unusedIgnoreFile in ignoreSet:
      print(
          'WARNING: Nonexisting file "%s" ignored in configuration "%s".' % (
              unusedIgnoreFile, self.config.getConfigFileName()),
          file=sys.stderr)

    for elemName, statusData in self.statusData.items():
      element = self.findElementByName(elemName)
      if element is None:
        raise Exception(
            'Invalid status of nonexisting element "%s" ' \
            'in data directory "%s"' % (
                elemName, self.config.dataDir))
      element.initPolicyData(statusData)

  def applyPolicies(self):
    """Apply the policy to this storage and all storages defined
    in subconfigurations. For each storage this will check all
    policy templates if one or more of them should be applied
    to known sources managed by this configuration."""
    for includeConfig in self.config.includedConfigList:
      includeConfig.getStatus().applyPolicies()

    for sourceName, sourceStatus in self.trackedSources.items():
      policyList = self.config.getSourcePolicies(sourceName)
      if len(policyList) == 0:
        print('WARNING: no policies for "%s" in "%s"' % (
            sourceName, self.config.getConfigFileName()), file=sys.stderr)
      policyNames = set()
      policyNames.add(BackupDataElement.DELETE_STATUS_POLICY)
      for policy in policyList:
        policyNames.add(policy.getPolicyName())
        policy.apply(sourceStatus)

# Now check if any element is marked for deletion. In the same
# round detect policy status not belonging to any policy.
      deleteList = []
      for element in sourceStatus.getDataElementList():
        if element.findUnsuedPolicyData(policyNames):
          print(
              'WARNING: Unused policy data for "%s" in ' \
              'element "%s".' % (
                  element.findUnsuedPolicyData(policyNames),
                  element.getElementName()), file=sys.stderr)

        if element.isMarkedForDeletion():
          deleteList.append(element)
      if not deleteList:
        continue

# So there are deletions, show them and ask for confirmation.
      print('Backup data to be kept/deleted for "%s" in storage "%s":' %
          (sourceName, self.config.getConfigFileName()))
      for element in sourceStatus.getDataElementList():
        marker = '*'
        if element.isMarkedForDeletion():
          marker = ' '
        print(
            '%s %s %s' % (marker, element.getDateTimeId(), element.getType()))
      inputText = None
      if StorageTool.INTERACTIVE_MODE == 'keyboard':
        inputText = input('Delete elements in "%s" (y/n)? ' % sourceName)
      elif StorageTool.INTERACTIVE_MODE == 'force-no':
        pass
      elif StorageTool.INTERACTIVE_MODE == 'force-yes':
        inputText = 'y'
      if inputText != 'y':
# Deletion was not confirmed.
        continue

# So there are deletions. Deleting data may corrupt the status
# data if there is any software or system error while processing
# the deletions. Therefore save the current status in a temporary
# file before modifying the data by applying deletion policies.
# Verify that there was no logic flaw assinging sources to the
# wrong storage.
      if sourceStatus.getStorageStatus() != self:
        raise Exception()
# Now save the current storage status to a temporary file as
# applying policies might have modified it already.
      statusFileNamePreDelete = self.save(suffix='.pre-delete')

      for policy in policyList:
        policy.delete(sourceStatus)
# All policies were invoked, so remove the deleted elements from
# the status.
      sourceStatus.removeDeleted()

# So at least updating the status data has worked, so save the
# status data.
      statusFileNamePostDelete = self.save(suffix='.post-delete')

# Now just delete the files. File system errors or system crashes
# will still cause an inconsistent state, but that is easy to
# detect.
      for element in deleteList:
        element.delete()

      os.unlink(statusFileNamePreDelete)
      os.rename(statusFileNamePostDelete, self.config.getStatusFileName())

  def save(self, suffix=None):
    """Save the current storage status to the status file or
    a new file derived from the status file name by adding a
    suffix. When saving to the status file, the old status file
    may exist and is replaced. Saving to files with suffix is
    only intended to create temporary files to avoid status data
    corruption during critical operations. These files shall
    be removed or renamed by the caller as soon as not needed
    any more.
    @return the file name the status data was written to."""
# First serialize all status data.
    statusData = {}
    for sourceStatus in self.trackedSources.values():
      statusData.update(sourceStatus.serializeStatus())
    targetFileName = self.config.getStatusFileName()
    if suffix is not None:
      targetFileName += suffix
      if os.path.exists(targetFileName):
        raise Exception()
    targetFile = open(targetFileName, 'wb')
    targetFile.write(bytes(json.dumps(
        statusData, indent=2, sort_keys=True), 'ascii'))
    targetFile.close()
    return targetFileName


class StorageFileInfo():
  """This class stores information about one file found in the
  file data storage directory."""

  def __init__(self, status):
# This is the storage configuration authoritative for defining
# the status and policy of this file.
    self.status = status
# This is the backup source that created the file data.
    self.backupSource = None

  def getStatus(self):
    """Get the authoritative status for this file."""
    return self.status

  def setBackupSource(self, backupSource):
    """Set the backup source this file belongs to."""
    if self.backupSource is not None:
      raise Exception('Logic error')
    self.backupSource = backupSource

  def getConfig(self):
    """Get the configuration associated with this storage file."""
    return self.status.config


class StorageTool():
  """This class implements the storage tool main functions."""

# Use a singleton variable to define interactive behaviour.
  INTERACTIVE_MODE = 'keyboard'

  def __init__(self):
    """Create a StorageTool object with default configuration.
    The object has to be properly initialized by loading configuration
    data from files."""
    self.configFileName = '/etc/guerillabackup/storage-tool-config.json'
    self.config = None
# This is the dictionary of all known storage files found in
# the data directory of the main configuration or any subconfiguration.
    self.storageFileDict = {}

  def parseCommandLine(self):
    """This function will parse command line arguments and update
    settings before loading of configuration."""
    if self.config is not None:
      raise Exception('Cannot reload configuration')
    argPos = 1
    while argPos < len(sys.argv):
      argName = sys.argv[argPos]
      argPos += 1
      if not argName.startswith('--'):
        raise Exception('Invalid argument "%s"' % argName)
      if argName == '--Config':
        self.configFileName = sys.argv[argPos]
        argPos += 1
        continue
      if argName == '--Help':
        print(
            'Usage: %s [options]\n' \
            '* --Config [file]: Use this configuration file not the default\n' \
            '  file at "/etc/guerillabackup/storage-tool-config.json".\n' \
            '* --DryRun: Just report check results but do not ' \
            'modify storage.\n' \
            '* --Help: This output' % sys.argv[0],
            file=sys.stderr)
        sys.exit(0)
      if argName == '--DryRun':
        StorageTool.INTERACTIVE_MODE = 'force-no'
        continue
      print(
          'Unknown parameter "%s", use "--Help" or see man page.' % argName,
              file=sys.stderr)
      sys.exit(1)


  def loadConfiguration(self):
    """Load the configuration from the specified configuration file."""
    if self.config is not None:
      raise Exception('Cannot reload configuration')
    self.config = StorageConfig(self.configFileName, None)


  def initializeStorage(self):
    """Initialize the storage by locating all files in data storage
    directories and also apply already known status information."""
    self.config.initializeStorage(self.storageFileDict)

  def applyPolicies(self):
    """Check all policy templates if one or more should be applied
    to any known resource."""
    self.config.getStatus().applyPolicies()


def main():
  """This is the program main function."""
  tool = StorageTool()
  tool.parseCommandLine()
  tool.loadConfiguration()
# Now all recursive configurations are loaded. First initialize
# the storage.
  tool.initializeStorage()

# Next check for files not already covered by policies according
# to status data. Suggest status changes for those.
  tool.applyPolicies()
  print('All policies applied.', file=sys.stderr)


if __name__ == '__main__':
  main()
