# Back In Time
# Copyright (C) 2008-2022 Oprea Dan, Bart de Koning, Richard Bailey, Germar Reitze, Taylor Raack
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import json
import os
from pathlib import Path
import stat
import datetime
import gettext
import bz2
import pwd
import grp
import subprocess
import shutil
import time
import re
import fcntl
from tempfile import TemporaryDirectory
import config
import configfile
import logger
import tools
import encfstools
import mount
import progress
import snapshotlog
from applicationinstance import ApplicationInstance
from exceptions import MountException, LastSnapshotSymlink
[docs]
class Snapshots:
"""
Collection of take-snapshot and restore commands.
BUHTZ 2022-10-09: In my understanding this is the representation of a
snapshot in the "application layer". This seems to be the difference to
the class `SID` which represents a snapshot in the "data layer".
BUHTZ 2024-02-23: Not sure but it seems to be one concret snapshot and
not a collection of snapshots. In this case the class name is missleading
because it is in plural form.
Args:
cfg (config.Config): current config
"""
SNAPSHOT_VERSION = 3
GLOBAL_FLOCK = '/tmp/backintime.lock'
def __init__(self, cfg = None):
self.config = cfg
if self.config is None:
self.config = config.Config()
self.snapshotLog = snapshotlog.SnapshotLog(self.config)
self.clearIdCache()
self.clearNameCache()
#rsync --info=progress2 output
#search for: 517.38K 26% 14.46MB/s 0:02:36
#or: 497.84M 4% -449.39kB/s ??:??:??
#but filter out: 517.38K 26% 14.46MB/s 0:00:53 (xfr#53, to-chk=169/452)
# because this shows current run time
self.reRsyncProgress = re.compile(r'.*?' #trash at start
r'(\d*[,\.]?\d+[KkMGT]?)\s+' #bytes sent
r'(\d*)%\s+' #percent done
r'(-?\d*[,\.]?\d*[KkMGT]?B/s)\s+' #speed
r'([\d\?]+:[\d\?]{2}:[\d\?]{2})' #estimated time of arrival
r'(.*$)') #trash at the end
self.lastBusyCheck = datetime.datetime(1, 1, 1)
self.flock = None
self.restorePermissionFailed = False
# TODO: make own class for takeSnapshotMessage
[docs]
def clearTakeSnapshotMessage(self):
"""Delete message and progress file"""
Path(self.config.takeSnapshotMessageFile()).unlink(missing_ok=True)
Path(self.config.takeSnapshotProgressFile()).unlink(missing_ok=True)
# TODO: make own class for takeSnapshotMessage
[docs]
def takeSnapshotMessage(self):
"""Get the current message from the message file.
Returns:
(tuple(int, str)): The message type and its string or `None`. See
`setTakeSnapshotMessage()` for further details.
Dev note (buhtz):
Too many try..excepts in here.
"""
# Dev note (buhtz): Not sure what happens here or why this is usefull.
wait = datetime.datetime.now() - datetime.timedelta(seconds=5)
if self.lastBusyCheck < wait:
self.lastBusyCheck = datetime.datetime.now()
if not self.busy():
self.clearTakeSnapshotMessage()
return None
# Filename of the message file
message_fn = self.config.takeSnapshotMessageFile()
if not os.path.exists(message_fn):
return None
try:
with open(message_fn, 'rt') as handle:
items = handle.read().split('\n')
# TODO (buhtz): Too broad exception
except Exception as exc:
logger.debug('Failed to get takeSnapshot message from '
f'{message_fn}: {str(exc)}', self)
return None
if len(items) < 2:
return None
# "Message id": Type of the message.
mid = 0
try:
mid = int(items[0])
# TODO (buhtz): Too broad exception
except Exception as exc:
logger.debug('Failed to extract message ID from '
f'{items}: {str(exc)}', self)
return (mid, '\n'.join(items[1:]))
# TODO: make own class for takeSnapshotMessage
[docs]
def setTakeSnapshotMessage(self, type_id, message, timeout=-1):
"""Update the status message of the active snapshot creation job
Write the status message into a message file to allow async
processing for the GUI, plug-ins (like user-callback)
and desktop notifications.
Args:
type_id: Simplified severity level of the status message:
0: INFO
1: ERROR
other values: defaults to INFO (may change in the future)
message: status message string
timeout: Requested maximum processing duration in plug-ins.
Default: -1 (no limit)
Called plug-ins must try to keep the timeout itself
(it is not enforced by the ``PluginManager``.
In fact currently all known plug-ins do
ignore the timeout value!
"""
message_fn = self.config.takeSnapshotMessageFile()
try:
# Write message to file (and overwrites the previos one)
with open(message_fn, 'wt') as f:
f.write(str(type_id) + '\n' + message)
except Exception as exc:
logger.debug('Failed to set takeSnapshot message '
f'to {message_fn}: {str(exc)}', self)
# Error message?
if type_id == 1:
self.snapshotLog.append('[E] ' + message, 1)
else:
self.snapshotLog.append('[I] ' + message, 3)
try:
profile_id = self.config.currentProfile()
profile_name = self.config.profileName(profile_id)
self.config.PLUGIN_MANAGER.message(
profile_id, profile_name, type_id, message, timeout)
except Exception as exc:
logger.debug(f'Failed to send message to plugins: {str(exc)}', self)
[docs]
def busy(self):
instance = ApplicationInstance(self.config.takeSnapshotInstanceFile(), False)
return instance.busy()
[docs]
def pid(self):
instance = ApplicationInstance(self.config.takeSnapshotInstanceFile(), False)
return instance.readPidFile()[0]
[docs]
def clearNameCache(self):
"""
Reset the cache for user and group names.
"""
self.userCache = {}
self.groupCache = {}
[docs]
def clearIdCache(self):
"""
Reset the cache for UIDs and GIDs.
"""
self.uidCache = {}
self.gidCache = {}
[docs]
def uid(self, name, callback = None, backup = None):
"""
Get the User identifier (UID) for the user in ``name``.
name->uid will be cached to speed up subsequent requests.
Args:
name (:py:class:`str`, :py:class:`bytes`):
username to search for
callback (method): callable which will handle a given message
backup (int): UID which will be used if the username is unknown
on this machine
Returns:
int: UID of the user in name or -1 if not found
"""
if isinstance(name, bytes):
name = name.decode()
if name in self.uidCache:
return self.uidCache[name]
else:
uid = -1
try:
uid = pwd.getpwnam(name).pw_uid
except Exception as e:
if backup:
uid = backup
msg = "UID for '%s' is not available on this system. Using UID %s from snapshot." %(name, backup)
logger.info(msg, self)
if callback is not None:
callback(msg)
else:
self.restorePermissionFailed = True
msg = 'Failed to get UID for %s: %s' %(name, str(e))
logger.error(msg, self)
if callback:
callback(msg)
self.uidCache[name] = uid
return uid
[docs]
def gid(self, name, callback = None, backup = None):
"""
Get the Group identifier (GID) for the group in ``name``.
name->gid will be cached to speed up subsequent requests.
Args:
name (:py:class:`str`, :py:class:`bytes`):
groupname to search for
callback (method): callable which will handle a given message
backup (int): GID which will be used if the groupname is unknown
on this machine
Returns:
int: GID of the group in name or -1 if not found
"""
if isinstance(name, bytes):
name = name.decode()
if name in self.gidCache:
return self.gidCache[name]
else:
gid = -1
try:
gid = grp.getgrnam(name).gr_gid
except Exception as e:
if backup is not None:
gid = backup
msg = "GID for '%s' is not available on this system. Using GID %s from snapshot." %(name, backup)
logger.info(msg, self)
if callback:
callback(msg)
else:
self.restorePermissionFailed = True
msg = 'Failed to get GID for %s: %s' %(name, str(e))
logger.error(msg, self)
if callback:
callback(msg)
self.gidCache[name] = gid
return gid
[docs]
def userName(self, uid):
"""
Get the username for the given uid.
uid->name will be cached to speed up subsequent requests.
Args:
uid (int): User identifier (UID) to search for
Returns:
str: name of the user with UID uid or '-' if not found
"""
if uid in self.userCache:
return self.userCache[uid]
else:
name = '-'
try:
name = pwd.getpwuid(uid).pw_name
except Exception as e:
logger.debug('Failed to get user name for UID %s: %s'
%(uid, str(e)),
self)
self.userCache[uid] = name
return name
[docs]
def groupName(self, gid):
"""
Get the groupname for the given gid.
gid->name will be cached to speed up subsequent requests.
Args:
gid (int): Group identifier (GID) to search for
Returns:
str: name of the Group with GID gid or '.' if not found
"""
if gid in self.groupCache:
return self.groupCache[gid]
else:
name = '-'
try:
name = grp.getgrgid(gid).gr_name
except Exception as e:
logger.debug('Failed to get group name for GID %s: %s'
%(gid, str(e)),
self)
self.groupCache[gid] = name
return name
[docs]
def restoreCallback(self, callback, ok, msg):
"""
Format messages thrown by restore depending on whether they where
successful or failed.
Args:
callback (method): callable instance which will handle the message
ok (bool): ``True`` if the logged action was successful
or ``False`` if it failed
msg (str): message that should be send to callback
"""
if not callback is None:
if not ok:
msg = msg + ' : ' + _('FAILED')
self.restorePermissionFailed = True
callback(msg)
[docs]
def restorePermission(self, key_path, path, fileInfoDict, callback = None):
"""
Restore permissions (owner, group and mode). If permissions are
already identical with the new ones just skip. Otherwise try to
'chown' to new owner and new group. If that fails (most probably because
we are not running as root and normal user has no rights to change
ownership of files) try to at least 'chgrp' to the new group. Finally
'chmod' the new mode.
Args:
key_path (bytes): original path during backup.
Same as in fileInfoDict.
path (bytes): current path of file that should be changed.
fileInfoDict (FileInfoDict): FileInfoDict
"""
assert isinstance(key_path, bytes), 'key_path is not bytes type: %s' % key_path
assert isinstance(path, bytes), 'path is not bytes type: %s' % path
assert isinstance(fileInfoDict, FileInfoDict), 'fileInfoDict is not FileInfoDict type: %s' % fileInfoDict
if key_path not in fileInfoDict or not os.path.exists(path):
return
info = fileInfoDict[key_path]
#restore uid/gid
uid = self.uid(info[1], callback)
gid = self.gid(info[2], callback)
#current file stats
st = os.stat(path)
# logger.debug('%(path)s: uid %(target_uid)s/%(cur_uid)s, gid %(target_gid)s/%(cur_gid)s, mod %(target_mod)s/%(cur_mod)s'
# %{'path': path.decode(),
# 'target_uid': uid,
# 'cur_uid': st.st_uid,
# 'target_gid': gid,
# 'cur_gid': st.st_gid,
# 'target_mod': info[0],
# 'cur_mod': st.st_mode
# })
if uid != -1 or gid != -1:
ok = False
if uid != st.st_uid:
try:
os.chown(path, uid, gid)
ok = True
except:
pass
self.restoreCallback(callback, ok, "chown %s %s : %s" % (path.decode(errors = 'ignore'), uid, gid))
st = os.stat(path)
#if restore uid/gid failed try to restore at least gid
if not ok and gid != st.st_gid:
try:
os.chown(path, -1, gid)
ok = True
except:
pass
self.restoreCallback(callback, ok, "chgrp %s %s" % (path.decode(errors = 'ignore'), gid))
st = os.stat(path)
#restore perms
ok = False
if info[0] != st.st_mode:
try:
os.chmod(path, info[0])
ok = True
except:
pass
self.restoreCallback(callback, ok, "chmod %s %04o" % (path.decode(errors = 'ignore'), info[0]))
[docs]
def restore(self,
sid,
paths,
callback = None,
restore_to = '',
delete = False,
backup = True,
only_new = False):
"""
Restore one or more files from snapshot ``sid`` to either original
or a different destination. Restore is done with rsync. If available
permissions will be restored from ``fileinfo.bz2``.
Args:
sid (SID): snapshot from whom to restore
paths (:py:class:`list`, :py:class:`tuple` or :py:class:`str`):
single path (str) or multiple
paths (list, tuple) that should be
restored. For every path this will run
a new rsync process. Permissions will be
restored for all paths in one run
callback (method): callable instance which will handle
messages
restore_to (str): full path to restore to. If empty
restore to original destination
delete (bool): delete newer files which are not in the
snapshot
backup (bool): create backup files (``*.backup.YYYYMMDD``)
before changing or deleting local files.
only_new (bool): Only restore files which do not exist
or are newer than those in destination.
Using ``rsync --update`` option.
"""
instance = ApplicationInstance(
pidFile=self.config.restoreInstanceFile(),
autoExit=False,
flock=True)
if instance.check():
instance.startApplication()
else:
logger.warning('Restore is already running', self)
return
if restore_to.endswith('/'):
restore_to = restore_to[: -1]
if not isinstance(paths, (list, tuple)):
paths = (paths,)
logger.info("Restore: %s to: %s"
%(', '.join(paths), restore_to),
self)
info = sid.info
cmd_prefix = tools.rsyncPrefix(self.config, no_perms=False, use_mode=['ssh'])
cmd_prefix.extend(('-R', '-v'))
if backup:
cmd_prefix.extend(('--backup', '--suffix=%s' % self.backupSuffix()))
if delete:
cmd_prefix.append('--delete')
cmd_prefix.append('--filter=protect %s' % self.config.snapshotsPath())
cmd_prefix.append('--filter=protect %s' % self.config._LOCAL_DATA_FOLDER)
cmd_prefix.append('--filter=protect %s' % self.config._MOUNT_ROOT)
if only_new:
cmd_prefix.append('--update')
restored_paths = []
for path in paths:
tools.makeDirs(os.path.dirname(path))
src_path = path
src_delta = 0
src_base = sid.pathBackup(use_mode = ['ssh'])
if not src_base.endswith(os.sep):
src_base += os.sep
cmd = cmd_prefix[:]
if restore_to:
items = os.path.split(src_path)
aux = items[0].lstrip(os.sep)
# bugfix: restore system root ended in <src_base>//.<src_path>
if aux:
src_base = os.path.join(src_base, aux) + '/'
src_path = '/' + items[1]
if items[0] == '/':
src_delta = 0
else:
src_delta = len(items[0])
cmd.append(self.rsyncRemotePath('%s.%s' % (src_base, src_path), use_mode=['ssh'], quote=''))
cmd.append('%s/' % restore_to)
proc = tools.Execute(cmd,
callback=callback,
filters=(self.filterRsyncProgress,),
parent=self)
self.restoreCallback(callback, True, proc.printable_cmd)
proc.run()
self.restoreCallback(callback, True, ' ')
restored_paths.append((path, src_delta))
try:
os.remove(self.config.takeSnapshotProgressFile())
except Exception as e:
logger.debug('Failed to remove snapshot progress file %s: %s'
%(self.config.takeSnapshotProgressFile(), str(e)),
self)
#restore permissions
logger.info('Restore permissions', self)
self.restoreCallback(callback, True, ' ')
self.restoreCallback(
callback, True, '{}:'.format(_('Restore permissions')))
self.restorePermissionFailed = False
fileInfoDict = sid.fileInfo
#cache uids/gids
for uid, name in info.listValue('user', ('int:uid', 'str:name')):
self.uid(name.encode(), callback = callback, backup = uid)
for gid, name in info.listValue('group', ('int:gid', 'str:name')):
self.gid(name.encode(), callback = callback, backup = gid)
if fileInfoDict:
all_dirs = [] #restore dir permissions after all files are done
for path, src_delta in restored_paths:
#explore items
snapshot_path_to = sid.pathBackup(path).rstrip('/')
root_snapshot_path_to = sid.pathBackup().rstrip('/')
#use bytes instead of string from here
if isinstance(path, str):
path = path.encode()
if isinstance(restore_to, str):
restore_to = restore_to.encode()
if not restore_to:
path_items = path.strip(b'/').split(b'/')
curr_path = b'/'
for path_item in path_items:
curr_path = os.path.join(curr_path, path_item)
if curr_path not in all_dirs:
all_dirs.append(curr_path)
else:
if path not in all_dirs:
all_dirs.append(path)
if os.path.isdir(snapshot_path_to) and not os.path.islink(snapshot_path_to):
head = len(root_snapshot_path_to.encode())
for explore_path, dirs, files in os.walk(snapshot_path_to.encode()):
for item in dirs:
item_path = os.path.join(explore_path, item)[head:]
if item_path not in all_dirs:
all_dirs.append(item_path)
for item in files:
item_path = os.path.join(explore_path, item)[head:]
real_path = restore_to + item_path[src_delta:]
self.restorePermission(item_path, real_path, fileInfoDict, callback)
all_dirs.reverse()
for item_path in all_dirs:
real_path = restore_to + item_path[src_delta:]
self.restorePermission(item_path, real_path, fileInfoDict, callback)
self.restoreCallback(callback, True, '')
if self.restorePermissionFailed:
status = _('FAILED')
else:
status = _('Done')
self.restoreCallback(
callback,
True,
'{}: {}'.format(_('Restore permissions'), status)
)
instance.exitApplication()
[docs]
def backupSuffix(self):
"""
Get suffix for backup files.
Returns:
str: backup suffix in form of '.backup.YYYYMMDD'
"""
return '.backup.' + datetime.date.today().strftime('%Y%m%d')
[docs]
def remove(self, sid):
"""
Remove snapshot ``sid``.
BUHTZ 2022-10-11: From my understanding rsync is used here to sync the
directory of a concrete snapshot (``sid``) against an empty temporary
directory. In the consequence the sid directory is empty but not
deleted.
To delete that directory simple `rm` call (via `shutil` package) is
used to delete the directory. No need to do this via SSH because the
directory is temporary mounted.
It is not clear for me why it is done that way. Why not simply "rm"
the directory when it is mounted instead of using rsync in a previous
step?! But I won't change it yet.
Args:
sid (SID): snapshot to remove
Returns:
(bool): ``True`` if succeeded otherwise ``False``.
"""
if isinstance(sid, RootSnapshot):
return
# build the rsync command and it's arguments
rsync = tools.rsyncRemove(self.config)
# an empty temporary directory
# e.g. /tmp/tmp8g59onuz
with TemporaryDirectory() as d:
# the temp dir
rsync.append(d + os.sep)
# the real remote path of a concrete snapshot (a "sid")
# e.g. user@myserver:"/MyBackup/.backintime/backintime/HOST/user/ \
# MyProfile/20221005-000003-880"
rsync.append(
self.rsyncRemotePath(
sid.path(use_mode=['ssh', 'ssh_encfs']),
# No quoting because of new argument protection of rsync.
quote=''
)
)
# Syncing the empty tmp directory against the sid directory
# will clear the sid directory.
rc = tools.Execute(rsync).run()
#
if rc != 0:
logger.error(
f'Last rsync command failed with return code "{rc}". '
'See previous WARNING message in the logs for details.')
return False
# Delete the sid dir. BUT here isn't the remote path used but the
# temporary mounted variant of it.
# e.g. /home/user/.local/share/backintime/mnt/4_8030/backintime/ \
# HOST/user/MyProfile/20221005-000003-880
shutil.rmtree(sid.path())
return True
# TODO Refactor: This functions is extremely difficult to understand:
# - Nested "if"s
# - Fuzzy names of classes, attributes and methods
# - unclear variable names (at least for the return values)
[docs]
def backup(self, force = False):
"""Wrapper for :py:func:`takeSnapshot` which will prepare and clean up
things for the main :py:func:`takeSnapshot` method.
This will check that no other snapshots are running at the same time,
there is nothing prohibiting a new snapshot (e.g. on battery) and the
profile is configured correctly. This will also mount and unmount
remote destinations.
Args:
force (bool): Force taking a new snapshot even if the profile is
not scheduled or the machine is running on battery.
Returns:
bool: ``True`` if there was an error.
"""
ret_val, ret_error = False, True
sleep = True
self.config.PLUGIN_MANAGER.load(self)
if not self.config.isConfigured():
logger.warning('Not configured', self)
# not configured
self.config.PLUGIN_MANAGER.error(1)
elif (not force
and self.config.noSnapshotOnBattery()
and tools.onBattery()):
self.setTakeSnapshotMessage(
0, _('Deferring backup while on battery'))
logger.info('Deferring backup while on battery', self)
logger.warning('Backup not performed', self)
ret_error = False
elif not force and not self.config.backupScheduled():
logger.info(f'Profile "{self.config.profileName()}" is not '
'scheduled to run now.', self)
ret_error = False
else:
instance = ApplicationInstance(
self.config.takeSnapshotInstanceFile(),
False,
flock=True
)
restore_instance = ApplicationInstance(
self.config.restoreInstanceFile(),
False
)
if not instance.check():
logger.warning(
'A backup is already running. The pid of the already '
f'running backup is in file {instance.pidFile}. Maybe '
'delete it.', self )
# a backup is already running
self.config.PLUGIN_MANAGER.error(2)
elif not restore_instance.check():
logger.warning(
'Restore is still running. Stop backup until restore is '
'done. The pid of the already running restore is in '
f'{restore_instance.pidFile}. Maybe delete it.', self)
else:
if (self.config.noSnapshotOnBattery ()
and not tools.powerStatusAvailable()):
logger.warning('Backups disabled on battery but power '
'status is not available', self)
instance.startApplication()
# global flock to block backups from other profiles or users
# (and run them serialized)
self.flockExclusive()
logger.info('Lock', self)
now = datetime.datetime.today()
# inhibit suspend/hibernate during snapshot is running
self.config.inhibitCookie \
= tools.inhibitSuspend(toplevel_xid=self.config.xWindowId)
# mount
try:
hash_id = mount.Mount(cfg = self.config).mount()
except MountException as ex:
logger.error(str(ex), self)
instance.exitApplication()
logger.info('Unlock', self)
time.sleep(2)
return True
else:
self.config.setCurrentHashId(hash_id)
include_folders = self.config.include()
if not include_folders:
logger.info('Nothing to do', self)
elif not self.config.PLUGIN_MANAGER.processBegin():
logger.info('A plugin prevented the backup', self)
else:
# take snapshot process begin
self.setTakeSnapshotMessage(0, '...')
self.snapshotLog.new(now)
profile_id = self.config.currentProfile()
profile_name = self.config.profileName()
logger.info(f"Take a new snapshot. Profile: {profile_id} "
f"{profile_name}", self)
if not self.config.canBackup(profile_id):
if (self.config.PLUGIN_MANAGER.hasGuiPlugins
and self.config.notify()):
message = (
_('Can\'t find snapshots folder.\nIf it is '
'on a removable drive please plug it in.')
+ '\n'
+ gettext.ngettext('Waiting %s second.',
'Waiting %s seconds.',
30) % 30
)
self.setTakeSnapshotMessage(
type_id=1,
message=message,
timeout=30)
counter = 0
for counter in range(0, 30):
logger.debug(
"Cannot start snapshot yet: target directory "
"not accessible. Waiting 1s.")
time.sleep(1)
if self.config.canBackup():
break
if counter != 0:
logger.info(
f"Waited {counter} seconds for target "
"directory to be available", self)
if not self.config.canBackup(profile_id):
logger.warning("Can't find snapshots folder!", self)
# Can't find snapshots directory (is it on a removable
# drive ?)
self.config.PLUGIN_MANAGER.error(3)
else:
ret_error = False
sid = SID(now, self.config)
if sid.exists():
logger.warning(f'Snapshot path "{sid.path()}" '
'already exists', self)
# This snapshot already exists
self.config.PLUGIN_MANAGER.error(4, sid)
else:
try:
# TODO
# rename ret_val to new_snapshot_created and
# ret_error to has_error for clearer code
ret_val, ret_error = self.takeSnapshot(
sid, now, include_folders)
except:
new = NewSnapshot(self.config)
if new.exists():
new.saveToContinue = False
new.failed = True
raise
if not ret_val:
self.remove(sid)
if ret_error:
logger.error('Failed to take snapshot.', self)
msg = _('Failed to take snapshot '
'{snapshot_id}.').format(
snapshot_id=sid.displayID)
self.setTakeSnapshotMessage(1, msg)
# Fixes #1491
self.config.PLUGIN_MANAGER.error(5, msg)
time.sleep(2)
else:
logger.warning("No new snapshot", self)
else: # new snapshot taken...
if ret_error:
logger.error(
'New snapshot taken but errors detected',
self)
# Fixes #1491
self.config.PLUGIN_MANAGER.error(
6, sid.displayID)
# Why ignore errors now?
ret_error = False
# Probably because a new snapshot has been created
# (= changes transferred) and "continue on errors"
# is enabled
if not ret_error:
self.freeSpace(now)
self.setTakeSnapshotMessage(0, _('Finalizing'))
time.sleep(2)
sleep = False
if ret_val:
# new snapshot
self.config.PLUGIN_MANAGER.newSnapshot(
sid, sid.path())
# Take snapshot process end
self.config.PLUGIN_MANAGER.processEnd()
if sleep:
time.sleep(2)
sleep = False
if not ret_error:
self.clearTakeSnapshotMessage()
# unmount
try:
mount.Mount(cfg = self.config).umount(self.config.current_hash_id)
except MountException as ex:
logger.error(str(ex), self)
instance.exitApplication()
self.flockRelease()
logger.info('Unlock', self)
if sleep:
# max 1 backup / second
time.sleep(2)
# release inhibit suspend
if self.config.inhibitCookie:
self.config.inhibitCookie = tools.unInhibitSuspend(
*self.config.inhibitCookie)
return ret_error
[docs]
def filterRsyncProgress(self, line):
"""
Filter rsync's stdout for progress information and store them in
'~/.local/share/backintime/worker<N>.progress' file.
Args:
line (str): stdout line from rsync
Returns:
str: ``line`` if it had no progress infos. ``None`` if
``line`` was a progress
"""
ret = []
for l in line.split('\n'):
m = self.reRsyncProgress.match(l)
if m:
# if m.group(5).strip():
# return
pg = progress.ProgressFile(self.config)
pg.setIntValue('status', pg.RSYNC)
pg.setStrValue('sent', m.group(1))
pg.setIntValue('percent', int(m.group(2)))
pg.setStrValue('speed', m.group(3))
#pg.setStrValue('eta', m.group(4))
pg.save()
del(pg)
else:
ret.append(l)
return '\n'.join(ret)
[docs]
def rsyncCallback(self, line, params):
"""
Parse rsync's stdout, send it to takeSnapshotMessage and
takeSnapshotLog. Also check if there has been changes or errors in
current rsync.
Args:
line (str): stdout line from rsync
params (list): list of two bool '[error, changes]'.
Uses a side effect by changing list items here
to change the original list of the caller, too
(lists are passed as reference in Python).
If rsync reported an error ``params[0]``
will be set to ``True``. If rsync reported a changed
file ``params[1]`` will be set to ``True``
"""
if not line:
return
# Warning (2023-11): Do not modify the source string.
# See #1559 for details.
self.setTakeSnapshotMessage(
0, _('Take snapshot') + " (rsync: %s)" % line)
# Did rsync report an error?
if line.endswith(')'):
if line.startswith('rsync:'):
if not line.startswith('rsync: chgrp ') and not line.startswith('rsync: chown '):
# matches rsync error lines like:
# rsync: [generator] link [...] failed: Invalid cross-device link (18)
params[0] = True
self.setTakeSnapshotMessage(1, 'Error: ' + line)
if len(line) >= 13:
# The prefix is created by rsync via the argument "--out-format=BACKINTIME: %i %n%L"
if line.startswith('BACKINTIME: '):
if line[12] != '.' and line[12:14] != 'cd':
params[1] = True
self.snapshotLog.append('[C] ' + line[12:], 2)
[docs]
def makeDirs(self, path):
"""
Wrapper for :py:func:`tools.makeDirs()`. Create directories ``path``
recursive and return success. If not successful send error-message to
log.
Args:
path (str): fullpath to directories that should be created
Returns:
bool: ``True`` if successful
"""
if not tools.makeDirs(path):
logger.error(f"Can't create folder: {path}", self)
self.setTakeSnapshotMessage(
1,
'{}: {}'.format(
_("Can't create folder"),
path)
)
time.sleep(2) # max 1 backup / second
return False
return True
[docs]
def backupConfig(self, sid):
"""
Backup the config file to the snapshot and to the backup root if backup
is encrypted.
Args:
sid (SID): snapshot in which the config should be stored
"""
logger.info('Save config file', self)
self.setTakeSnapshotMessage(0, _('Saving config file…'))
with open(self.config._LOCAL_CONFIG_PATH, 'rb') as src:
with open(sid.path('config'), 'wb') as dst1:
dst1.write(src.read())
if self.config.snapshotsMode() == 'local_encfs':
src.seek(0)
dst2_path = os.path.join(
self.config.localEncfsPath(),
'config'
)
with open(dst2_path, 'wb') as dst2:
dst2.write(src.read())
elif self.config.snapshotsMode() == 'ssh_encfs':
cmd = tools.rsyncPrefix(self.config, no_perms=False)
cmd.append(self.config._LOCAL_CONFIG_PATH)
remote_path = self.rsyncRemotePath(
self.config.sshSnapshotsPath(),
# no quoting because of rsync modern argument
# protection (argument -s)
quote=''
)
cmd.append(remote_path)
proc = tools.Execute(cmd, parent=self)
rc = proc.run()
# WORKAROUND
# tools.Execute only create warnings if 'cmd' fails.
# But we need a real ERROR here.
if rc != 0:
logger.error(
f'Backing up the config in "{self.config.snapshotsMode()}"'
f' mode failed! The return code was {rc} and the'
f' command was {cmd}. Also see the previous '
'WARNING message for a more details.', parent=self)
[docs]
def backupInfo(self, sid):
"""
Save infos about the snapshot into the 'info' file.
Args:
sid (SID): snapshot that should get an info file
"""
logger.info("Create info file", self)
machine = self.config.host()
user = self.config.user()
profile_id = self.config.currentProfile()
i = configfile.ConfigFile()
i.setIntValue('snapshot_version', self.SNAPSHOT_VERSION)
i.setStrValue('snapshot_date', sid.withoutTag)
i.setStrValue('snapshot_machine', machine)
i.setStrValue('snapshot_user', user)
i.setIntValue('snapshot_profile_id', profile_id)
i.setIntValue('snapshot_tag', sid.tag)
i.setListValue('user', ('int:uid', 'str:name'), list(self.userCache.items()))
i.setListValue('group', ('int:gid', 'str:name'), list(self.groupCache.items()))
i.setStrValue('filesystem_mounts', json.dumps(tools.filesystemMountInfo()))
sid.info = i
[docs]
def backupPermissions(self, sid):
"""
Save permissions (owner, group, read-, write- and executable)
for all files in Snapshot ``sid`` into snapshots fileInfoDict.
Args:
sid (SID): snapshot that should be scanned
Returns:
int: Return code of rsync.
"""
logger.info('Save permissions', self)
self.setTakeSnapshotMessage(0, _('Saving permissions…'))
fileInfoDict = FileInfoDict()
if self.config.snapshotsMode() == 'ssh_encfs':
decode = encfstools.Decode(self.config, False)
else:
decode = encfstools.Bounce()
# backup permissions of /
# bugfix for https://github.com/bit-team/backintime/issues/708
self.backupPermissionsCallback(b'/', (fileInfoDict, decode))
rsync = ['rsync', '--dry-run', '-s', '-r', '--out-format=%n']
rsync.extend(tools.rsyncSshArgs(self.config))
rsync.append(
self.rsyncRemotePath(
path=sid.pathBackup(
use_mode=['ssh', 'ssh_encfs']
),
quote=''
) + os.sep
)
with TemporaryDirectory() as d:
rsync.append(d + os.sep)
proc = tools.Execute(rsync,
callback=self.backupPermissionsCallback,
user_data=(fileInfoDict, decode),
parent=self,
conv_str=False,
join_stderr=False)
rc = proc.run()
sid.fileInfo = fileInfoDict
return rc
[docs]
def backupPermissionsCallback(self, line, user_data):
"""
Rsync callback for :py:func:`Snapshots.backupPermissions`.
Args:
line(bytes): output from rsync command
user_data (tuple): two item tuple of (:py:class:`FileInfoDict`,
:py:class:`encfstools.Decode`)
"""
fileInfoDict, decode = user_data
self.collectPermission(fileInfoDict, b'/' + decode.path(line).rstrip(b'/'))
[docs]
def collectPermission(self, fileinfo, path):
"""
Collect permission infos about ``path`` and store them into
``fileinfo``.
Args:
fileinfo (FileInfoDict):
dict of: {path: (permission, user, group)}
Using sideefect on changing dict item will change
original dict, too.
path (bytes): full path to file or folder
"""
assert isinstance(path, bytes), 'path is not bytes type: %s' % path
if path and os.path.exists(path):
info = os.stat(path)
mode = info.st_mode
user = self.userName(info.st_uid).encode('utf-8', 'replace')
group = self.groupName(info.st_gid).encode('utf-8', 'replace')
fileinfo[path] = (mode, user, group)
[docs]
def takeSnapshot(self, sid, now, include_folders):
"""This is the main backup routine.
It will take a new snapshot and store permissions of included files
and folders into ``fileinfo.bz2``.
Args:
sid (SID): snapshot ID which the new snapshot should get
now (datetime.datetime): date and time when this snapshot was
started
include_folders (list): folders to include. list of tuples
(item, int) where ``int`` is 0 if ``item`` is a folder or 1
if ``item`` is a file
Returns:
list: list of two bool (``ret_val``, ``ret_error``) where
``ret_val`` is ``True`` if a new snapshot has been created and
``ret_error`` is ``True`` if there was an error during taking
the snapshot
"""
self.setTakeSnapshotMessage(0, '...')
new_snapshot = NewSnapshot(self.config)
encode = self.config.ENCODE
# "return" values set during async rsync execution (as user data "by ref")
params = [False, False] # [error, changes]
# TODO
# docstring of return value for this function swaps the meaning of the
# elements, this is confusing (``ret_val``, ``ret_error``) and
# error-prone. Use a mutable data structure with named elements
# instead, e.g. a DataClass
if new_snapshot.exists() and new_snapshot.saveToContinue:
logger.info(f"Found leftover '{new_snapshot.displayID}' which "
"can be continued.", self)
self.setTakeSnapshotMessage(
0,
_('Found leftover {snapshot_id} which can be continued.')
.format(snapshot_id=new_snapshot.displayID)
)
# fix permissions
for file in os.listdir(new_snapshot.path()):
file = os.path.join(new_snapshot.path(), file)
mode = os.stat(file).st_mode
try:
os.chmod(file, mode | stat.S_IWUSR)
except PermissionError:
pass
# search previous log for changes and set params
params[1] = new_snapshot.hasChanges
elif new_snapshot.exists() and not new_snapshot.saveToContinue:
logger.info(f'Remove leftover {new_snapshot.displayID} folder '
'from last run')
self.setTakeSnapshotMessage(
0,
_('Removing leftover {snapshot_id} folder from last run')
.format(snapshot_id=new_snapshot.displayID)
)
self.remove(new_snapshot)
if os.path.exists(new_snapshot.path()):
logger.error(
f"Can't remove folder: {new_snapshot.path()}", self)
self.setTakeSnapshotMessage(
1,
'{}: {}'.format(_("Can't remove folder"),
new_snapshot.path())
)
time.sleep(2) # max 1 backup / second
return [False, True]
if not new_snapshot.saveToContinue and not new_snapshot.makeDirs():
return [False, True]
prev_sid = None
snapshots = listSnapshots(self.config)
if snapshots:
prev_sid = snapshots[0]
# rsync prefix & suffix
rsync_prefix = tools.rsyncPrefix(self.config, no_perms=False)
if self.config.excludeBySizeEnabled():
rsync_prefix.append('--max-size=%sM' % self.config.excludeBySize())
rsync_suffix = self.rsyncSuffix(include_folders)
# When there is no snapshots it takes the last snapshot from the other folders
# It should delete the excluded folders then
rsync_prefix.extend(('--delete', '--delete-excluded'))
rsync_prefix.append('-v')
# Use a fixed logging format for the rsync "changed files" list to
# make it parsable e.g. in rsyncCallback()
# %i = itemized list (11 characters) of what is being updated
# (see "--itemize-changes" in "man rsync")
# %n = the filename (short form; trailing "/" on dir)
# %L = the string " -> SYMLINK", " => HARDLINK", or ""
# (where SYMLINK or HARDLINK is a filename)
# (see log format section in "man rsyncd.conf")
rsync_prefix.extend(('-i', '--out-format=BACKINTIME: %i %n%L'))
if prev_sid:
link_dest = encode.path(os.path.join(prev_sid.sid, 'backup'))
link_dest = os.path.join(os.pardir, os.pardir, link_dest)
rsync_prefix.append('--link-dest=%s' % link_dest)
# sync changed folders
logger.info("Call rsync to take the snapshot", self)
new_snapshot.saveToContinue = True
cmd = rsync_prefix + rsync_suffix
# No quoting (quote='') because of new argument protection of rsync.
cmd.append(self.rsyncRemotePath(
new_snapshot.pathBackup(use_mode=['ssh', 'ssh_encfs']),
quote=''))
self.setTakeSnapshotMessage(0, _('Taking snapshot'))
# run rsync
proc = tools.Execute(cmd,
# TODO
# interprets the user_data in params as: list of
# two bool [error, changes] but params is reused
# as return value of this function with [changes,
# error]. Use a separate variable to avoid
# confusion!
callback=self.rsyncCallback,
user_data=params,
filters=(self.filterRsyncProgress,),
parent=self)
# TODO
# introduce centralized log msg builder to avoid spread severity level
# indicators like "[I]" here?
self.snapshotLog.append('[I] ' + proc.printable_cmd, 3)
# TODO
# Process return value with rsync exit code to recognize errors that
# cannot be recognized by parsing the rsync output currently
rsync_exit_code = proc.run()
# Fix for #1491 and #489
# Note that the return value (containing the exit code) of the
# rsync child process is not the only way to detect errors (and
# sometimes not reliably delivers <> 0 in case of an error):
# Errors are also indicated via the pass-by-ref argument
# user_data="params" list (updated by the callback function that
# parses the rsync output for error message patterns).
# cleanup
try:
os.remove(self.config.takeSnapshotProgressFile())
except Exception as e:
logger.debug('Failed to remove snapshot progress file %s: %s'
% (self.config.takeSnapshotProgressFile(), str(e)),
self)
# handle errors
# TODO
# Fix inconsistent usage: Collects return value, but errors are also
# checked via params[0]
has_errors = False
# dict of exit codes (as keys) that are treated as INFO only by BiT
# (not as ERROR). The values are message strings for the snapshot log.
rsync_non_error_exit_codes = {
0: _("Success"),
# ignored as fix for #1587 (until we introduce a new snapshot
# result category "(with warnings)")
23: _("Partial transfer due to error"),
24: _("Partial transfer due to vanished source files "
"(see 'man rsync')")
}
rsync_exit_code_msg = _("'rsync' ended with exit code {exit_code}") \
.format(exit_code=rsync_exit_code)
if rsync_exit_code in rsync_non_error_exit_codes:
self.setTakeSnapshotMessage(
0, rsync_exit_code_msg + ": "
+ rsync_non_error_exit_codes[rsync_exit_code])
elif rsync_exit_code > 0: # an rsync error
# HACK to fix #489 (params[0] and has_errors should be merged)
params[0] = True
self.setTakeSnapshotMessage(
1, rsync_exit_code_msg + ": "
+ _("See 'man rsync' for more details"))
elif rsync_exit_code < 0: # an rsync error caused by a signal
# HACK to fix #489 (params[0] and has_errors should be merged)
params[0] = True
self.setTakeSnapshotMessage(
1, rsync_exit_code_msg + ": "
+ _("Negative rsync exit codes are signal numbers, see "
"'kill -l' and 'man kill'"))
# params[0] -> error?
if params[0]:
if not self.config.continueOnErrors():
self.remove(new_snapshot)
return [False, True]
has_errors = True
new_snapshot.failed = True
# params[1] -> changes?
if not params[1] and not self.config.takeSnapshotRegardlessOfChanges():
self.remove(new_snapshot)
logger.info("Nothing changed, no new snapshot necessary", self)
self.snapshotLog.append(
'[I] ' + _('Nothing changed, no new snapshot necessary'), 3)
if prev_sid:
prev_sid.setLastChecked()
if not has_errors and not list(self.config.anacrontabFiles()):
tools.writeTimeStamp(self.config.anacronSpoolFile())
# Part of fix for #1491:
# Returns "has_errors" instead of False now to signal rsync errors
# (which may have prevented processing any changes)
return [False, has_errors]
self.backupConfig(new_snapshot)
self.backupPermissions(new_snapshot)
# copy snapshot log
try:
self.snapshotLog.flush()
with open(self.snapshotLog.logFileName, 'rb') as logfile:
new_snapshot.setLog(logfile.read())
except Exception as e:
logger.debug('Failed to write takeSnapshot log %s into '
'compressed file %s: %s' % (
self.config.takeSnapshotLogFile(),
new_snapshot.path(SID.LOG),
str(e)),
self)
# TODO How is this error handled? Currently it looks like it is
# ignored (just logged)!
new_snapshot.saveToContinue = False
# rename snapshot
os.rename(new_snapshot.path(), sid.path())
if not sid.exists():
logger.error(
f"Can't rename {new_snapshot.path()} to {sid.path()}", self)
self.setTakeSnapshotMessage(
1,
_("Can't rename {new_path} to {path}")
.format(new_path=new_snapshot.path(), path=sid.path())
)
time.sleep(2) # max 1 backup / second
return [False, True]
self.backupInfo(sid)
if not has_errors and not list(self.config.anacrontabFiles()):
tools.writeTimeStamp(self.config.anacronSpoolFile())
# create last_snapshot symlink
self.createLastSnapshotSymlink(sid)
return [True, has_errors]
[docs]
def smartRemoveKeepAll(self,
snapshots,
min_date,
max_date):
"""
Return all snapshots between ``min_date`` and ``max_date``.
Args:
snapshots (list): full list of :py:class:`SID` objects
min_date (datetime.date): minimum date for snapshots to keep
max_date (datetime.date): maximum date for snapshots to keep
Returns:
set: set of snapshots that should be kept
"""
min_id = SID(min_date, self.config)
max_id = SID(max_date, self.config)
logger.debug("Keep all >= %s and < %s" %(min_id, max_id), self)
return set([sid for sid in snapshots if sid >= min_id and sid < max_id])
[docs]
def smartRemoveKeepFirst(self,
snapshots,
min_date,
max_date,
keep_healthy = False):
"""
Return only the first snapshot between ``min_date`` and ``max_date``.
Args:
snapshots (list): full list of :py:class:`SID` objects
min_date (datetime.date): minimum date for snapshots to keep
max_date (datetime.date): maximum date for snapshots to keep
keep_healthy (bool): return the first healthy snapshot (not
marked as failed) instead of the first
at all. If all snapshots failed this
will again return the very first
snapshot
Returns:
set: set of snapshots that should be kept
"""
min_id = SID(min_date, self.config)
max_id = SID(max_date, self.config)
logger.debug("Keep first >= %s and < %s" %(min_id, max_id), self)
for sid in snapshots:
# try to keep the first healty snapshot
if keep_healthy and sid.failed:
logger.debug("Do not keep failed snapshot %s" %sid, self)
continue
if sid >= min_id and sid < max_id:
return set([sid])
# if all snapshots failed return the first snapshot
# no matter if it has errors
if keep_healthy:
return self.smartRemoveKeepFirst(snapshots,
min_date,
max_date,
keep_healthy = False)
return set()
[docs]
def incMonth(self, date):
"""
First day of next month of ``date`` with respect on new years. So if
``date`` is December this will return 1st of January next year.
Args:
date (datetime.date): old date that should be increased
Returns:
datetime.date: 1st day of next month
"""
y = date.year
m = date.month + 1
if m > 12:
m = 1
y = y + 1
return datetime.date(y, m, 1)
[docs]
def decMonth(self, date):
"""
First day of previous month of ``date`` with respect on previous years.
So if ``date`` is January this will return 1st of December previous
year.
Args:
date (datetime.date): old date that should be decreased
Returns:
datetime.date: 1st day of previous month
"""
y = date.year
m = date.month - 1
if m < 1:
m = 12
y = y - 1
return datetime.date(y, m, 1)
[docs]
def smartRemoveList(self,
now_full,
keep_all,
keep_one_per_day,
keep_one_per_week,
keep_one_per_month):
"""
Get a list of old snapshots that should be removed based on configurable
intervals.
Args:
now_full (datetime.datetime): date and time when takeSnapshot was
started
keep_all (int): keep all snapshots for the
last ``keep_all`` days
keep_one_per_day (int): keep one snapshot per day for the
last ``keep_one_per_day`` days
keep_one_per_week (int): keep one snapshot per week for the
last ``keep_one_per_week`` weeks
keep_one_per_month (int): keep one snapshot per month for the
last ``keep_one_per_month`` months
Returns:
list: snapshots that should be removed
"""
snapshots = listSnapshots(self.config)
logger.debug(f'Considered: {snapshots}', self)
if len(snapshots) <= 1:
logger.debug('There is only one snapshot, so keep it', self)
return []
if now_full is None:
now_full = datetime.datetime.today()
now = now_full.date()
# keep the last snapshot
keep = set([snapshots[0]])
# keep all for the last keep_all days
if keep_all > 0:
keep |= self.smartRemoveKeepAll(
snapshots,
now - datetime.timedelta(days=keep_all-1),
now + datetime.timedelta(days=1))
# keep one per day for the last keep_one_per_day days
if keep_one_per_day > 0:
d = now
for i in range(0, keep_one_per_day):
keep |= self.smartRemoveKeepFirst(
snapshots,
d,
d + datetime.timedelta(days=1),
keep_healthy=True)
d -= datetime.timedelta(days=1)
# keep one per week for the last keep_one_per_week weeks
if keep_one_per_week > 0:
d = now - datetime.timedelta(days=now.weekday() + 1)
for i in range(0, keep_one_per_week):
keep |= self.smartRemoveKeepFirst(
snapshots,
d,
d + datetime.timedelta(days=8),
keep_healthy=True)
d -= datetime.timedelta(days=7)
# keep one per month for the last keep_one_per_month months
if keep_one_per_month > 0:
d1 = datetime.date(now.year, now.month, 1)
d2 = self.incMonth(d1)
for i in range(0, keep_one_per_month):
keep |= self.smartRemoveKeepFirst(
snapshots, d1, d2, keep_healthy=True)
d2 = d1
d1 = self.decMonth(d1)
# keep one per year for all years
first_year = int(snapshots[-1].sid[:4])
for i in range(first_year, now.year+1):
keep |= self.smartRemoveKeepFirst(snapshots,
datetime.date(i, 1, 1),
datetime.date(i+1, 1, 1),
keep_healthy=True)
logger.debug(f'Keep snapshots: {keep}', self)
del_snapshots = []
for sid in snapshots:
if sid in keep:
continue
if self.config.dontRemoveNamedSnapshots():
if sid.name:
logger.debug(
f'Keep snapshot: {sid}, because it has a name', self)
continue
del_snapshots.append(sid)
return del_snapshots
[docs]
def smartRemove(self, del_snapshots, log = None):
"""
Remove multiple snapshots either with
:py:func:`Snapshots.remove` or in background on the remote host
if mode is `ssh` or `ssh_encfs` and smart-remove in background is
activated.
Args:
del_snapshots (list): list of :py:class:`SID` that should be removed
log (method): callable method that will handle progress log
"""
if not del_snapshots:
return
if not log:
log = lambda x: self.setTakeSnapshotMessage(0, x)
if self.config.snapshotsMode() in ['ssh', 'ssh_encfs'] and self.config.smartRemoveRunRemoteInBackground():
logger.info('[smart remove] remove snapshots in background: %s'
% del_snapshots, self)
lckFile = os.path.normpath(
os.path.join(
del_snapshots[0].path(use_mode=['ssh', 'ssh_encfs']),
os.pardir,
'smartremove.lck'
)
)
maxLength = self.config.sshMaxArgLength()
if not maxLength:
import sshMaxArg
user_host = '%s@%s' % (self.config.sshUser(),
self.config.sshHost())
maxLength = sshMaxArg.probe_max_ssh_command_size(self.config)
self.config.setSshMaxArgLength(maxLength)
self.config.save()
sshMaxArg.report_result(user_host, maxLength)
additionalChars = len(self.config.sshPrefixCmd(cmd_type = str))
head = 'screen -d -m bash -c "('
# create temp dir used for delete files with rsync
head += 'TMP=\\$(mktemp -d); '
# make sure $TMP dir was created
head += 'test -z \\\"\\$TMP\\\" && exit 1; '
# make sure $TMP is empty
head += 'test -n \\\"\\$(ls \\$TMP)\\\" && exit 1; '
if logger.DEBUG:
head += 'logger -t \\\"backintime smart-remove [$BASHPID]\\\" \\\"start\\\"; '
head += 'flock -x 9; '
if logger.DEBUG:
head += 'logger -t \\\"backintime smart-remove [$BASHPID]\\\" \\\"got exclusive flock\\\"; '
tail = 'rmdir \\$TMP) 9>\\\"%s\\\""' %lckFile
cmds = []
for sid in del_snapshots:
remote = self.rsyncRemotePath(sid.path(use_mode = ['ssh', 'ssh_encfs']), use_mode = [], quote = '\\\"')
rsync = ' '.join(tools.rsyncRemove(self.config, run_local = False))
rsync += ' \\\"\\$TMP/\\\" {}; '.format(remote)
s = 'test -e \\\"%s\\\" && (' %sid.path(use_mode = ['ssh', 'ssh_encfs'])
if logger.DEBUG:
s += 'logger -t \\\"backintime smart-remove [$BASHPID]\\\" '
s += '\\\"snapshot %s still exist\\\"; ' %sid
s += 'sleep 1; ' #add one second delay because otherwise you might not see serialized process with small snapshots
s += rsync
s += 'rmdir \\\"%s\\\"; ' %sid.path(use_mode = ['ssh', 'ssh_encfs'])
if logger.DEBUG:
s += 'logger -t \\\"backintime smart-remove [$BASHPID]\\\" '
s += '\\\"snapshot %s remove done\\\"' %sid
s += '); '
cmds.append(s)
for cmd in tools.splitCommands(cmds,
head = head,
tail = tail,
maxLength = maxLength - additionalChars):
tools.Execute(self.config.sshCommand([cmd,],
quote = False,
nice = False,
ionice = False)).run()
else:
logger.info("[smart remove] remove snapshots: %s"
%del_snapshots, self)
for i, sid in enumerate(del_snapshots, 1):
log(_('Smart remove') + ' %s/%s' %(i, len(del_snapshots)))
self.remove(sid)
[docs]
def freeSpace(self, now):
"""
Remove old snapshots on based on different rules (only if enabled).
First rule is to remove snapshots older than X years. Next will call
:py:func:`smartRemove` to remove snapshots based on
configurable intervals. Third rule is to remove the oldest snapshot
until there is enough free space. Last rule will remove the oldest
snapshot until there are enough free inodes.
'last_snapshot' symlink will be fixed when done.
Args:
now (datetime.datetime): date and time when takeSnapshot was
started
"""
snapshots = listSnapshots(self.config, reverse = False)
if not snapshots:
logger.debug('No snapshots. Skip freeSpace', self)
return
last_snapshot = snapshots[-1]
#remove old backups
if self.config.removeOldSnapshotsEnabled():
self.setTakeSnapshotMessage(0, _('Removing old snapshots'))
oldBackupId = SID(self.config.removeOldSnapshotsDate(), self.config)
logger.debug("Remove snapshots older than: {}".format(oldBackupId.withoutTag), self)
while True:
if len(snapshots) <= 1:
break
if snapshots[0] >= oldBackupId:
break
if self.config.dontRemoveNamedSnapshots():
if snapshots[0].name:
del snapshots[0]
continue
msg = 'Remove snapshot {} because it is older than {}'
logger.debug(msg.format(snapshots[0].withoutTag, oldBackupId.withoutTag), self)
self.remove(snapshots[0])
del snapshots[0]
# smart remove
enabled, keep_all, keep_one_per_day, keep_one_per_week, keep_one_per_month = self.config.smartRemove()
if enabled:
self.setTakeSnapshotMessage(0, _('Smart remove'))
del_snapshots = self.smartRemoveList(now,
keep_all,
keep_one_per_day,
keep_one_per_week,
keep_one_per_month)
self.smartRemove(del_snapshots)
# try to keep min free space
if self.config.minFreeSpaceEnabled():
self.setTakeSnapshotMessage(0, _('Trying to keep min free space'))
minFreeSpace = self.config.minFreeSpaceMib()
logger.debug("Keep min free disk space: {} MiB".format(minFreeSpace), self)
snapshots = listSnapshots(self.config, reverse = False)
while True:
if len(snapshots) <= 1:
break
free_space = self.statFreeSpaceLocal(self.config.snapshotsFullPath())
if free_space is None:
free_space = self.statFreeSpaceSsh()
if free_space is None:
logger.warning('Failed to get free space. Skipping', self)
break
if free_space >= minFreeSpace:
break
if self.config.dontRemoveNamedSnapshots():
if snapshots[0].name:
del snapshots[0]
continue
msg = "free disk space: {} MiB. Remove snapshot {}"
logger.debug(msg.format(free_space, snapshots[0].withoutTag), self)
self.remove(snapshots[0])
del snapshots[0]
#try to keep free inodes
if self.config.minFreeInodesEnabled():
minFreeInodes = self.config.minFreeInodes()
self.setTakeSnapshotMessage(
0,
_('Trying to keep min {perc} free inodes')
.format(perc=f'{minFreeInodes}%')
)
logger.debug(
"Keep min {perc}% free inodes".format(perc=minFreeInodes),
self)
snapshots = listSnapshots(self.config, reverse = False)
while True:
if len(snapshots) <= 1:
break
try:
info = os.statvfs(self.config.snapshotsPath())
free_inodes = info.f_favail
max_inodes = info.f_files
except Exception as e:
logger.debug('Failed to get free inodes for snapshot path %s: %s'
% (self.config.snapshotsPath(), str(e)),
self)
break
if free_inodes >= max_inodes * (minFreeInodes / 100.0):
break
if self.config.dontRemoveNamedSnapshots():
if snapshots[0].name:
del snapshots[0]
continue
logger.debug("free inodes: %.2f%%. Remove snapshot %s"
%((100.0 / max_inodes * free_inodes), snapshots[0].withoutTag),
self)
self.remove(snapshots[0])
del snapshots[0]
#set correct last snapshot again
if last_snapshot is not snapshots[-1]:
self.createLastSnapshotSymlink(snapshots[-1])
[docs]
def statFreeSpaceLocal(self, path):
"""
Get free space on filesystem containing ``path`` in MiB using
:py:func:`os.statvfs()`. Depending on remote SFTP server this might fail
on sshfs mounted shares.
Args:
path (str): full path
Returns:
int free space in MiB
"""
try:
info = os.statvfs(path)
if info.f_blocks != info.f_bavail:
return info.f_frsize * info.f_bavail // (1024 * 1024)
except Exception as e:
logger.debug('Failed to get free space for %s: %s'
%(path, str(e)),
self)
logger.warning('Failed to stat snapshot path', self)
[docs]
def statFreeSpaceSsh(self):
"""
Get free space on remote filesystem in MiB. This will call ``df`` on
remote host and parse its output.
Returns:
int free space in MiB
"""
if self.config.snapshotsMode() not in ('ssh', 'ssh_encfs'):
return None
snapshots_path_ssh = self.config.sshSnapshotsFullPath()
if not len(snapshots_path_ssh):
snapshots_path_ssh = './'
cmd = self.config.sshCommand(['df', snapshots_path_ssh],
nice=False,
ionice=False)
df = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = df.communicate()[0]
# Filesystem 1K-blocks Used Available Use% Mounted on
# /tmp 127266564 115596412 5182296 96% /
# ^^^^^^^
for line in output.split(b'\n'):
m = re.match(r'^.*?\s+\d+\s+\d+\s+(\d+)\s+\d+%', line.decode(), re.M)
if m:
return int(int(m.group(1)) / 1024)
logger.warning('Failed to get free space on remote', self)
[docs]
def filter(self,
base_sid,
base_path,
snapshotsList,
list_diff_only = False,
flag_deep_check = False,
list_equal_to = ''):
"""
Filter snapshots from ``snapshotsList`` based on whether ``base_path``
file is included and optional if the snapshot is unique or equal to
``list_equal_to``.
Args:
base_sid (SID): snapshot ID that contained the original
file ``base_path``
base_path (str): path to file on root filesystem.
snapshotsList (list): List of :py:class:`SID` objects that should
be filtered
list_diff_only (bool): if ``True`` only return unique snapshots.
Which means if a file is exactly the same in
different snapshots only the first snapshot
will be listed
flag_deep_check (bool): use md5sum to check uniqueness of files.
More accurate but slow
list_equal_to (str): full path to file. If not empty only return
snapshots which have exactly the same file
as this file
Returns:
list: filtered list of :py:class:`SID` objects
"""
snapshotsFiltered = []
base_full_path = base_sid.pathBackup(base_path)
if not os.path.lexists(base_full_path):
return []
allSnapshotsList = [RootSnapshot(self.config)]
allSnapshotsList.extend(snapshotsList)
#links
if os.path.islink(base_full_path):
targets = []
for sid in allSnapshotsList:
path = sid.pathBackup(base_path)
if os.path.lexists(path) and os.path.islink(path):
if list_diff_only:
target = os.readlink(path)
if target in targets:
continue
targets.append(target)
snapshotsFiltered.append(sid)
return snapshotsFiltered
#directories
if os.path.isdir(base_full_path):
for sid in allSnapshotsList:
path = sid.pathBackup(base_path)
if os.path.exists(path) and not os.path.islink(path) and os.path.isdir(path):
snapshotsFiltered.append(sid)
return snapshotsFiltered
#files
if not list_diff_only and not list_equal_to:
for sid in allSnapshotsList:
path = sid.pathBackup(base_path)
if os.path.exists(path) and not os.path.islink(path) and os.path.isfile(path):
snapshotsFiltered.append(sid)
return snapshotsFiltered
# check for duplicates
uniqueness = tools.UniquenessSet(flag_deep_check, follow_symlink = False, list_equal_to = list_equal_to)
for sid in allSnapshotsList:
path = sid.pathBackup(base_path)
if os.path.exists(path) and not os.path.islink(path) and os.path.isfile(path) and uniqueness.check(path):
snapshotsFiltered.append(sid)
return snapshotsFiltered
#TODO: move this to config.Config -> Don't!
[docs]
def rsyncRemotePath(self, path, use_mode = ['ssh', 'ssh_encfs'], quote = '"'):
"""
Format the destination string for rsync depending on which profile is
used.
Args:
path (str): destination path
use_mode (list): list of modes in which the result should
change to ``user@host:path`` instead of
just ``path``
quote (str): use this to quote the path
Returns:
str: quoted ``path`` like '"/foo"'
or if the current mode is using ssh and
current mode is in ``use_mode`` a combination
of user, host and ``path``
like ''user@host:"/foo"''
"""
mode = self.config.snapshotsMode()
if mode in ['ssh', 'ssh_encfs'] and mode in use_mode:
user = self.config.sshUser()
host = tools.escapeIPv6Address(self.config.sshHost())
return '%(u)s@%(h)s:%(q)s%(p)s%(q)s' % {'u': user,
'h': host,
'q': quote,
'p': path}
else:
return path
[docs]
def deletePath(self, sid, path):
"""
Delete ``path`` and all files and folder inside in snapshot ``sid``.
Args:
sid (SID): snapshot ID in which ``path`` should be deleted
path (str): path to delete
"""
def errorHandler(fn, path, excinfo):
"""
Error handler for :py:func:`deletePath`. This will fix permissions
and try again to remove the file.
Args:
fn (method): callable which failed before
path (str): file to delete
excinfo: NotImplemented
"""
dirname = os.path.dirname(path)
st = os.stat(dirname)
os.chmod(dirname, st.st_mode | stat.S_IWUSR)
st = os.stat(path)
os.chmod(path, st.st_mode | stat.S_IWUSR)
fn(path)
full_path = sid.pathBackup(path)
dirname = os.path.dirname(full_path)
dir_st = os.stat(dirname)
os.chmod(dirname, dir_st.st_mode | stat.S_IWUSR)
if os.path.isdir(full_path) and not os.path.islink(full_path):
shutil.rmtree(full_path, onerror = errorHandler)
else:
st = os.stat(full_path)
os.chmod(full_path, st.st_mode | stat.S_IWUSR)
os.remove(full_path)
os.chmod(dirname, dir_st.st_mode)
[docs]
def createLastSnapshotSymlink(self, sid):
"""
Create symlink 'last_snapshot' to snapshot ``sid``
Args:
sid (SID): snapshot that should be linked.
Returns:
bool: ``True`` if successful
"""
if sid is None:
return
symlink = self.config.lastSnapshotSymlink()
try:
if os.path.islink(symlink):
if os.path.basename(os.path.realpath(symlink)) == sid.sid:
return True
os.remove(symlink)
if os.path.exists(symlink):
logger.error('Could not remove symlink %s' %symlink, self)
return False
logger.debug('Create symlink %s => %s' %(symlink, sid), self)
os.symlink(sid.sid, symlink)
return True
except Exception as e:
logger.error('Failed to create symlink %s: %s' %(symlink, str(e)), self)
return False
# TODO Rename to make the purpose obvious, eg: Serialize_[backup|execution]_via_exclusive_global_flock()
[docs]
def flockExclusive(self):
"""
Block :py:func:`backup` from other profiles or users
and run them serialized
"""
if self.config.globalFlock():
logger.debug('Set flock %s' %self.GLOBAL_FLOCK, self)
self.flock = open(self.GLOBAL_FLOCK, 'w')
fcntl.flock(self.flock, fcntl.LOCK_EX) # blocks (waits) until an existing flock is released
# make it rw by all if that's not already done.
perms = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | \
stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH
s = os.fstat(self.flock.fileno())
if not s.st_mode & perms == perms:
logger.debug('Set flock permissions %s' %self.GLOBAL_FLOCK, self)
os.fchmod(self.flock.fileno(), perms)
[docs]
def flockRelease(self):
"""
Release lock so other snapshots can continue
"""
if self.flock:
logger.debug('Release flock %s' %self.GLOBAL_FLOCK, self)
fcntl.fcntl(self.flock, fcntl.LOCK_UN)
self.flock.close()
self.flock = None
[docs]
def rsyncSuffix(self, includeFolders = None, excludeFolders = None):
"""
Create suffixes for rsync.
Args:
includeFolders (list): folders to include. list of tuples (item, int)
Where ``int`` is ``0`` if ``item`` is a
folder or ``1`` if ``item`` is a file
excludeFolders (list): list of folders to exclude
Returns:
list: rsync include and exclude options
"""
#create exclude patterns string
rsync_exclude = self.rsyncExclude(excludeFolders)
#create include patterns list
rsync_include, rsync_include2 = self.rsyncInclude(includeFolders)
encode = self.config.ENCODE
ret = ['--chmod=Du+wx']
ret.extend(['--exclude=' + i for i in (encode.exclude(self.config.snapshotsPath()),
encode.exclude(self.config._LOCAL_DATA_FOLDER),
encode.exclude(self.config._MOUNT_ROOT)
)])
# TODO: fix bug #561:
# after rsync_exclude we need to explicitly include files inside excluded
# folders, recursive exclude folder-content again and finally add the
# rest from rsync_include2
ret.extend(rsync_include)
ret.extend(rsync_exclude)
ret.extend(rsync_include2)
ret.append('--exclude=*')
ret.append(encode.chroot)
return ret
[docs]
def rsyncExclude(self, excludeFolders = None):
"""
Format exclude list for rsync
Args:
excludeFolders (list): list of folders to exclude
Returns:
OrderedSet: rsync exclude options
"""
items = tools.OrderedSet()
encode = self.config.ENCODE
if excludeFolders is None:
excludeFolders = self.config.exclude()
for exclude in excludeFolders:
exclude = encode.exclude(exclude)
if exclude is None:
continue
items.add('--exclude=' + exclude)
return items
[docs]
def rsyncInclude(self, includeFolders = None):
"""
Format include list for rsync. Returns a tuple of two include strings.
First string need to come before exclude, second after exclude.
Args:
includeFolders (list): folders to include. list of
tuples (item, int) where ``int`` is ``0``
if ``item`` is a folder or ``1`` if ``item``
is a file
Returns:
tuple: two item tuple of
``(OrderedSet('include1 opions'),
OrderedSet('include2 options'))``
"""
items1 = tools.OrderedSet()
items2 = tools.OrderedSet()
encode = self.config.ENCODE
if includeFolders is None:
includeFolders = self.config.include()
for include_folder in includeFolders:
folder = include_folder[0]
if folder == "/": # If / is selected as included folder it should be changed to ""
#folder = "" # because an extra / is added below. Patch thanks to Martin Hoefling
items2.add('--include=/')
items2.add('--include=/**')
continue
folder = encode.include(folder)
if include_folder[1] == 0:
items2.add('--include={}/**'.format(folder))
else:
items2.add('--include={}'.format(folder))
folder = os.path.split(folder)[0]
while True:
if len(folder) <= 1:
break
items1.add('--include={}/'.format(folder))
folder = os.path.split(folder)[0]
return (items1, items2)
[docs]
class FileInfoDict(dict):
"""
A :py:class:`dict` that maps a path (as :py:class:`bytes`) to a
tuple (:py:class:`int`, :py:class:`bytes`, :py:class:`bytes`).
"""
def __init__(self):
# default permissions for /
# only used if fileinfo.bz2 does not contain a value for /
# when it was created with version <= 1.1.12
# bugfix for https://github.com/bit-team/backintime/issues/708
self[b'/'] = (16877, b'root', b'root')
def __setitem__(self, key, value):
assert isinstance(key, bytes), "key '{}' is not bytes instance".format(key)
assert isinstance(value, tuple), "value '{}' is not tuple instance".format(value)
assert len(value) == 3, "value '{}' does not have 3 items".format(value)
assert isinstance(value[0], int), "first value '{}' is not int instance".format(value[0])
assert isinstance(value[1], bytes), "second value '{}' is not bytes instance".format(value[1])
assert isinstance(value[2], bytes), "third value '{}' is not bytes instance".format(value[2])
super(FileInfoDict, self).__setitem__(key, value)
[docs]
class SID(object):
"""
Snapshot ID object used to gather all information for a snapshot
See :py:class:`Snapshots` to understand the difference.
Args:
date (:py:class:`str`, :py:class:`datetime.date` or :py:class:`datetime.datetime`):
used for creating this snapshot. str must be in
snapshot ID format (e.g 20151218-173512-123)
cfg (config.Config): current config
Raises:
ValueError: if ``date`` is :py:class:`str` instance and
doesn't match the snapshot ID format
(20151218-173512-123 or 20151218-173512)
TypeError: if ``date`` is not :py:class:`str`,
:py:class:`datetime.date` or
:py:class:`datetime.datetime` type
"""
__cValidSID = re.compile(r'^\d{8}-\d{6}(?:-\d{3})?$')
INFO = 'info'
NAME = 'name'
FAILED = 'failed'
FILEINFO = 'fileinfo.bz2'
LOG = 'takesnapshot.log.bz2'
def __init__(self, date, cfg):
self.config = cfg
self.profileID = cfg.currentProfile()
self.isRoot = False
if isinstance(date, datetime.datetime):
self.sid = '-'.join((date.strftime('%Y%m%d-%H%M%S'), self.config.tag(self.profileID)))
self.date = date
elif isinstance(date, datetime.date):
self.sid = '-'.join((date.strftime('%Y%m%d-000000'), self.config.tag(self.profileID)))
self.date = datetime.datetime.combine(date, datetime.datetime.min.time())
elif isinstance(date, str):
if self.__cValidSID.match(date):
self.sid = date
self.date = datetime.datetime(*self.split())
elif date == 'last_snapshot':
raise LastSnapshotSymlink()
else:
raise ValueError("'date' must be in snapshot ID format (e.g 20151218-173512-123)")
else:
raise TypeError("'date' must be an instance of str, datetime.date or datetime.datetime")
def __repr__(self):
return self.sid
def __eq__(self, other):
"""
Compare snapshots based on self.sid
Args:
other (:py:class:`SID`, :py:class:`str`):
an other :py:class:`SID` or str instance
Returns:
bool: ``True`` if other is equal
"""
if isinstance(other, SID):
return self.sid == other.sid and self.profileID == other.profileID
elif isinstance(other, str):
return self.sid == other
else:
return NotImplemented
def __ne__(self, other):
return not self.__eq__(other)
def __lt__(self, other):
"""
Sort snapshots (alphabetical order) based on self.sid
Args:
other (:py:class:`SID`, :py:class:`str`):
an other :py:class:`SID` or str instance
Returns:
bool: ``True`` if other is lower
"""
if isinstance(other, SID):
return self.sid < other.sid
elif isinstance(other, str) and self.__cValidSID.match(other):
return self.sid < other
else:
return NotImplemented
def __le__(self, other):
if isinstance(other, SID):
return self.sid <= other.sid
elif isinstance(other, str) and self.__cValidSID.match(other):
return self.sid <= other
else:
return NotImplemented
def __gt__(self, other):
if isinstance(other, SID):
return self.sid > other.sid
elif isinstance(other, str) and self.__cValidSID.match(other):
return self.sid > other
else:
return NotImplemented
def __ge__(self, other):
if isinstance(other, SID):
return self.sid >= other.sid
elif isinstance(other, str) and self.__cValidSID.match(other):
return self.sid >= other
else:
return NotImplemented
def __hash__(self):
return hash(self.sid + self.profileID)
[docs]
def split(self):
"""
Split self.sid into a tuple of int's
with Year, Month, Day, Hour, Minute, Second
Returns:
tuple: tuple of 6 int
"""
def split(s, e):
return int(self.sid[s:e])
return (split(0, 4), split(4, 6), split(6, 8), split(9, 11), split(11, 13), split(13, 15))
@property
def displayID(self):
"""
Snapshot ID in a user-readable format:
YYYY-MM-DD HH:MM:SS
Returns:
str: formatted sID
"""
return "{:04}-{:02}-{:02} {:02}:{:02}:{:02}".format(*self.split())
@property
def displayName(self):
"""
Combination of displayID, name and error indicator (if any)
Returns:
str: name
"""
ret = self.displayID
name = self.name
if name:
ret += ' - {}'.format(name)
if self.failed:
ret += ' ({})'.format('WITH ERRORS !')
return ret
@property
def tag(self):
"""
Snapshot ID's tag
Returns:
str: tag (last three digits)
"""
return self.sid[16:]
@property
def withoutTag(self):
"""
Snapshot ID without tag
Returns:
str: YYYYMMDD-HHMMSS
"""
return self.sid[0:15]
[docs]
def path(self, *path, use_mode = []):
"""
Current path of this snapshot automatically altered for
remote/encrypted version of this path
Args:
*path (str): one or more folder/files to join at the end of
the path.
use_mode (list): list of modes that should alter this path.
If the current mode is in this list, the path
will automatically be altered for the
remote/encrypted version of this path.
Returns:
str: full snapshot path
"""
path = [i.strip(os.sep) for i in path]
current_mode = self.config.snapshotsMode(self.profileID)
if 'ssh' in use_mode and current_mode == 'ssh':
return os.path.join(self.config.sshSnapshotsFullPath(self.profileID),
self.sid, *path)
if 'ssh_encfs' in use_mode and current_mode == 'ssh_encfs':
ret = os.path.join(self.config.sshSnapshotsFullPath(self.profileID),
self.sid, *path)
return self.config.ENCODE.remote(ret)
return os.path.join(self.config.snapshotsFullPath(self.profileID),
self.sid, *path)
[docs]
def pathBackup(self, *path, **kwargs):
"""
'backup' folder inside snapshots path
Args:
*path (str): one or more folder/files to join at the end of
the path.
use_mode (list): list of modes that should alter this path.
If the current mode is in this list, the path
will automatically be altered for the
remote/encrypted version of this path.
Returns:
str: full snapshot path
"""
return self.path('backup', *path, **kwargs)
[docs]
def makeDirs(self, *path):
"""
Create snapshot directory
Args:
*path (str): one or more folder/files to join at the end
of the path
Returns:
bool: ``True`` if successful
"""
if not os.path.isdir(self.config.snapshotsFullPath(self.profileID)):
logger.error('Snapshots path {} doesn\'t exist. Unable to make dirs for snapshot ID {}'.format(
self.config.snapshotsFullPath(self.profileID), self.sid),
self)
return False
return tools.makeDirs(self.pathBackup(*path))
[docs]
def exists(self):
"""
``True`` if the snapshot folder and the "backup" folder inside exist
Returns:
bool: ``True`` if exists
"""
return os.path.isdir(self.path()) and os.path.isdir(self.pathBackup())
[docs]
def isExistingPathInsideSnapshotFolder(self, path):
"""
``True`` if path is a file inside this snapshot
Args:
path (str): path from local filesystem (no snapshot path)
Returns:
bool: ``True`` if file exists
"""
fullPath = self.pathBackup(path)
if not os.path.exists(fullPath):
return False
if not os.path.islink(fullPath):
return True
basePath = self.pathBackup()
target = os.readlink(fullPath)
target = os.path.join(os.path.abspath(os.path.dirname(fullPath)), target)
return target.startswith(basePath)
@property
def name(self):
"""
Name of this snapshot
Args:
name (str): new name of the snapshot
Returns:
str: name of this snapshot
"""
nameFile = self.path(self.NAME)
if not os.path.isfile(nameFile):
return ''
try:
with open(nameFile, 'rt') as f:
return f.read()
except Exception as e:
logger.debug('Failed to get snapshot {} name: {}'.format(
self.sid, str(e)),
self)
@name.setter
def name(self, name):
nameFile = self.path(self.NAME)
self.makeWritable()
try:
with open(nameFile, 'wt') as f:
f.write(name)
except Exception as e:
logger.debug('Failed to set snapshot {} name: {}'.format(
self.sid, str(e)),
self)
@property
def lastChecked(self):
"""
Date when snapshot has finished last time.
This can be the end of creation of this snapshot or the last time when
this snapshot was checked against source without changes.
Returns:
str: date and time of last check (YYYY-MM-DD HH:MM:SS)
"""
info = self.path(self.INFO)
if os.path.exists(info):
return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getatime(info)))
return self.displayID
#using @property.setter would be confusing here as there is no value to give
[docs]
def setLastChecked(self):
"""
Set info files atime to current time to indicate this snapshot was
checked against source without changes right now.
"""
info = self.path(self.INFO)
if os.path.exists(info):
os.utime(info, None)
@property
def failed(self):
"""
This snapshot has failed
Args:
enable (bool): set or remove flag
Returns:
bool: ``True`` if flag is set
"""
failedFile = self.path(self.FAILED)
return os.path.isfile(failedFile)
@failed.setter
def failed(self, enable):
failedFile = self.path(self.FAILED)
if enable:
self.makeWritable()
try:
with open(failedFile, 'wt') as f:
f.write('')
except Exception as e:
logger.debug('Failed to mark snapshot {} failed: {}'.format(
self.sid, str(e)),
self)
elif os.path.exists(failedFile):
os.remove(failedFile)
@property
def info(self):
"""
Load/save "info" file which contains additional information
about this snapshot (using configfile.ConfigFile)
Args:
i (configfile.ConfigFile): info that should be saved.
Returns:
configfile.ConfigFile: snapshots information
"""
i = configfile.ConfigFile()
i.load(self.path(self.INFO))
return i
@info.setter
def info(self, i):
assert isinstance(i, configfile.ConfigFile), 'i is not configfile.ConfigFile type: {}'.format(i)
i.save(self.path(self.INFO))
@property
def fileInfo(self):
"""
Load/save "fileinfo.bz2"
Args:
d (FileInfoDict): dict of: {path: (permission, user, group)}
Returns:
FileInfoDict: dict of: {path: (permission, user, group)}
"""
d = FileInfoDict()
infoFile = self.path(self.FILEINFO)
if not os.path.isfile(infoFile):
return d
try:
with bz2.BZ2File(infoFile, 'rb') as fileinfo:
for line in fileinfo:
line = line.strip(b'\n')
if not line:
continue
index = line.find(b'/')
if index < 0:
continue
f = line[index:]
if not f:
continue
info = line[:index].strip().split(b' ')
if len(info) == 3:
d[f] = (int(info[0]), info[1], info[2]) #perms, user, group
except (FileNotFoundError, PermissionError) as e:
logger.error('Failed to load {} from snapshot {}: {}'.format(
self.FILEINFO, self.sid, str(e)),
self)
return d
@fileInfo.setter
def fileInfo(self, d):
assert isinstance(d, FileInfoDict), 'd is not FileInfoDict type: {}'.format(d)
try:
with bz2.BZ2File(self.path(self.FILEINFO), 'wb') as f:
for path, info in d.items():
f.write(b' '.join((str(info[0]).encode('utf-8', 'replace'),
info[1],
info[2],
path))
+ b'\n')
except PermissionError as e:
logger.error('Failed to write {}: {}'.format(self.FILEINFO, str(e)))
# TODO use @property decorator? IMHO not because it is not a "getter" but processes data
# TODO Should have an action name like "loadLogFile"
[docs]
def log(self, mode = None, decode = None):
"""
Load log from "takesnapshot.log.bz2"
Args:
mode (int): Mode used for filtering. Take a look at
:py:class:`snapshotlog.LogFilter`
decode (encfstools.Decode): instance used for decoding lines or ``None``
Yields:
str: filtered and decoded log lines
"""
logFile = self.path(self.LOG)
logFilter = snapshotlog.LogFilter(mode, decode)
try:
with bz2.BZ2File(logFile, 'rb') as f:
if logFilter.header:
yield logFilter.header
for line in f.readlines():
line = logFilter.filter(line.decode('utf-8').rstrip('\n'))
if not line is None:
yield line
except Exception as e:
msg = ('Failed to get snapshot log from {}:'.format(logFile), str(e))
logger.debug(' '.join(msg), self)
for line in msg:
yield line
[docs]
def setLog(self, log):
"""
Write log to "takesnapshot.log.bz2"
Args:
log: full snapshot log
"""
if isinstance(log, str):
log = log.encode('utf-8', 'replace')
logFile = self.path(self.LOG)
try:
with bz2.BZ2File(logFile, 'wb') as f:
f.write(log)
except Exception as e:
logger.error('Failed to write log into compressed file {}: {}'.format(
logFile, str(e)),
self)
[docs]
def makeWritable(self):
"""
Make the snapshot path writable so we can change files inside
Returns:
bool: ``True`` if successful
"""
path = self.path()
rw = os.stat(path).st_mode | stat.S_IWUSR
return os.chmod(path, rw)
[docs]
class GenericNonSnapshot(SID):
@property
def displayID(self):
return self.name
@property
def displayName(self):
return self.name
@property
def tag(self):
return self.name
@property
def withoutTag(self):
return self.name
[docs]
class NewSnapshot(GenericNonSnapshot):
"""
Snapshot ID object for 'new_snapshot' folder
Args:
cfg (config.Config): current config
"""
NEWSNAPSHOT = 'new_snapshot'
SAVETOCONTINUE = 'save_to_continue'
def __init__(self, cfg):
self.config = cfg
self.profileID = cfg.currentProfile()
self.isRoot = False
self.sid = self.NEWSNAPSHOT
self.date = datetime.datetime(1, 1, 1)
self.__le__ = self.__lt__
self.__ge__ = self.__gt__
def __lt__(self, other):
return False
def __gt__(self, other):
return True
@property
def name(self):
"""
Name of this snapshot
Returns:
str: name of this snapshot
"""
return self.sid
@property
def saveToContinue(self):
"""
Check if 'save_to_continue' flag is set
Args:
enable (bool): set or remove flag
Returns:
bool: ``True`` if flag is set
"""
return os.path.exists(self.path(self.SAVETOCONTINUE))
@saveToContinue.setter
def saveToContinue(self, enable):
flag = self.path(self.SAVETOCONTINUE)
if enable:
try:
with open(flag, 'wt') as f:
pass
except Exception as e:
logger.error("Failed to set 'save_to_continue' flag: %s" %str(e)) # should be "safe", throughout
elif os.path.exists(flag):
try:
os.remove(flag)
except Exception as e:
logger.error("Failed to remove 'save_to_continue' flag: %s" %str(e)) # should be "safe", throughout
@property
def hasChanges(self):
"""
Check if there where changes in previous sessions.
Returns:
bool: ``True`` if there where changes
"""
log = snapshotlog.SnapshotLog(self.config, self.profileID)
c = re.compile(r'^\[C\] ')
for line in log.get(mode = snapshotlog.LogFilter.CHANGES):
if c.match(line):
return True
return False
[docs]
class RootSnapshot(GenericNonSnapshot):
"""
Snapshot ID for the filesystem root folder ('/')
Args:
cfg (config.Config): current config
"""
def __init__(self, cfg):
self.config = cfg
self.profileID = cfg.currentProfile()
self.isRoot = True
self.sid = '/'
self.date = datetime.datetime(datetime.MAXYEAR, 12, 31)
self.__le__ = self.__lt__
self.__ge__ = self.__gt__
def __lt__(self, other):
return False
def __gt__(self, other):
return True
@property
def name(self):
"""
Name of this snapshot
Returns:
str: name of this snapshot
"""
return _('Now')
[docs]
def path(self, *path, use_mode = []):
"""
Current path of this snapshot automatically altered for
remote/encrypted version of this path
Args:
*path (str): one or more folder/files to join at the end of
the path.
use_mode (list): list of modes that should alter this path.
If the current mode is in this list, the path
will automatically altered for the
remote/encrypted version of this path.
Returns:
str: full snapshot path
"""
current_mode = self.config.snapshotsMode(self.profileID)
if 'ssh_encfs' in use_mode and current_mode == 'ssh_encfs':
if path:
path = self.config.ENCODE.remote(os.path.join(*path))
return os.path.join(self.config.ENCODE.chroot, path)
else:
return os.path.join(os.sep, *path)
[docs]
def iterSnapshots(cfg, includeNewSnapshot = False):
"""
Iterate over snapshots in current snapshot path. Use this in a 'for' loop
for faster processing than list object
Args:
cfg (config.Config): current config
includeNewSnapshot (bool): include a NewSnapshot instance if
'new_snapshot' folder is available.
Yields:
SID: snapshot IDs
"""
path = cfg.snapshotsFullPath()
if not os.path.exists(path):
return None
for item in os.listdir(path):
if item == NewSnapshot.NEWSNAPSHOT:
newSid = NewSnapshot(cfg)
if newSid.exists() and includeNewSnapshot:
yield newSid
continue
try:
sid = SID(item, cfg)
if sid.exists():
yield sid
except Exception as e:
if not isinstance(e, LastSnapshotSymlink):
logger.debug("'{}' is not a snapshot ID: {}".format(item, str(e)))
[docs]
def listSnapshots(cfg, includeNewSnapshot = False, reverse = True):
"""
List of snapshots in current snapshot path.
Args:
cfg (config.Config): current config (config.Config instance)
includeNewSnapshot (bool): include a NewSnapshot instance if
'new_snapshot' folder is available
reverse (bool): sort reverse
Returns:
list: list of :py:class:`SID` objects
"""
ret = list(iterSnapshots(cfg, includeNewSnapshot))
ret.sort(reverse = reverse)
return ret
[docs]
def lastSnapshot(cfg):
"""
Most recent snapshot.
Args:
cfg (config.Config): current config (config.Config instance)
Returns:
SID: most recent snapshot ID
"""
sids = listSnapshots(cfg)
if sids:
return sids[0]
if __name__ == '__main__':
config = config.Config()
snapshots = Snapshots(config)
snapshots.backup()