Rockstor Share management scripts

- `sync-shares`: backup files to, or restore from, an external drive, for all specified file shares.
- `defrag-btrfs`: defrag all BTRFS subvolumes in the specified directory.
  - Should be run as a cronjob as root

linux/defrag-btrfs

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -e
+#set -x
+
+DEFRAG_ROOT="$1"
+SHARE_ROOT="/mnt2"
+
+LOG=~jason/logs/btrfs-defrag-$(date '+%Y-%m-%d_%H-%M').log
+
+function log() {
+	date_str=`date '+%Y-%m-%d %H:%M:%S'`
+	echo "[$date_str]: $1" 2>&1 | tee -a $LOG
+}
+
+CURR_USER=`whoami`
+if [ "$CURR_USER" != "root" ]; then
+	echo "Must be run as root"
+	exit -1
+fi
+
+function logCmd() {
+	cmd="$@"
+	log "Running '${cmd}'"
+	time $cmd 2>&1 | tee -a $LOG
+}
+
+function doDefrag() {
+	log "Starting defrag of: $DEFRAG_ROOT"
+	logCmd /usr/sbin/btrfs filesystem defrag -rvf "$DEFRAG_ROOT"
+}
+
+function defragAll() {
+	for d in $SHARE_ROOT/*; do
+		DEFRAG_ROOT=$d
+		time doDefrag
+	done
+}
+
+if [ "$DEFRAG_ROOT" = "" ]; then
+	echo "No share specified. Going to defrag all shares under $SHARE_ROOT"
+	#read -p "Press any key to continue, or ctrl+C to cancel."
+	time defragAll
+else
+	time doDefrag
+fi
+
+log "Done"
+

linux/sync-shares

@@ -0,0 +1,270 @@
+#!/bin/bash
+
+################################################################################
+# Sync shares from one location to another.
+#
+# For backing up shares (usual action), use `sync-shares backup {OPTIONS}`
+# To restore from a backup, use `sync-shares restore {OPTIONS}`
+# To see options, use `sync-shares --help`
+################################################################################
+
+set -e
+
+# Uncomment to show all commands as they run (used to debug the script).
+# set -x
+
+# The root directory where everything is stored in.
+# The specific file is determined after a mode is specified.
+LOG_ROOT="~jason/logs"
+
+# The external drive
+EXTERN_ROOT='/mnt/external'
+
+# The root of BTRFS shares to sync to/from
+SHARES_ROOT='/mnt2'
+
+# The external drive to mount.
+EXTERN_DEV=`ls /dev/sd* | sort -r | head -n1`
+
+CHECKSUM=""
+DRY_RUN=""
+DELETE=""
+VERBOSE=""
+UNMOUNT=""
+LARGE=""
+
+FULL_ARGS="$@"
+
+USAGE="sudo $0 (backup|restore) [--help] [<options>]"
+DESC=`cat << EOT
+
+Modes:
+    backup:  Backup shares from $SHARES_ROOT to the external drive at $EXTERN_ROOT
+    restore: Restore files in directories in $EXTERN_ROOT to $SHARES_ROOT
+
+Options:
+    --help:       Show this message.
+    --checksum:   Compare duplicate files using checksums, rather than date/size.
+    --dry-run:    Show what would be transferred, without actually copying anything.
+    --delete:     Remove files from the destination that no longer exist in the source.
+                  eg: When running 'backup' any files deleted from a share since the
+                  last run will be deleted from the backup.
+    --mount:      Mount ${EXTERN_DEV?} to ${EXTERN_ROOT?}.
+    --unmount:    Unmount $EXTERN_ROOT when the sync has completed.
+    --large:      Include large shares that would otherwise be excluded.
+    --verbose:    Run rsync in verbose mode.
+EOT`
+
+function showUsage() {	
+	echo "Usage:" >&2
+	echo "$ $USAGE" >&2
+	echo "$DESC" >&2
+}
+
+function checkModeSet() {
+	if [ "$mode" ]; then
+		echo "Only one mode can be set." >&2
+		exit -1
+	fi
+}
+
+while :; do
+	case $1 in
+		-h|--help)
+			showUsage
+			exit -2
+			;;
+		backup)
+			checkModeSet
+			mode=backup
+			SRC_ROOT=$SHARES_ROOT
+			DEST_ROOT=$EXTERN_ROOT
+			;;
+		restore)
+			checkModeSet
+			mode=restore
+			SRC_ROOT=$EXTERN_ROOT
+			DEST_ROOT=$SHARES_ROOT
+			;;
+		--checksum)
+			echo "Using checksum comparison."
+			CHECKSUM="--checksum"
+			;;
+		--dry-run)
+			echo "Running in dry-run mode."
+			DRY_RUN="--dry-run"
+			VERBOSE="--verbose"
+			;;
+		--delete)
+			echo "Deleting extraneous files."
+			DELETE="--delete-after"
+			;;
+		--mount)
+			echo "Will mount ${EXTERN_DEV?}, and unmount when done."
+			MOUNT="mount"
+			UNMOUNT="unmount"
+			;;
+		--unmount)
+			echo "Will unmount when done."
+			UNMOUNT="unmount"
+			;;
+		--large)
+			echo "Will include large shares."
+			LARGE="large"
+			;;
+		-v|--verbose)
+			VERBOSE="--verbose"
+			;;
+		"") break ;;
+		*)
+			printf "ERROR: Unknown option: %s\n" "$1" >&2
+			showUsage
+			exit -1
+			;;
+	esac
+	shift
+done
+
+echo
+
+# Check that the mode has been set
+if [ "$mode" = "" ]; then
+	showUsage
+	exit -2
+fi
+
+echo "Checking that script is being run as root..."
+CURR_USER=`whoami`
+if [ "$CURR_USER" != "root" ]; then
+	echo
+	>&2 echo "ERROR: Must be run as root"
+	exit -3
+fi
+
+# Check if the drive is mounted.
+# Grep returns 1 when the string isn't found, which is an error, and `set -e`
+# stops the script. Extra bit at the end makes it return an empty string
+# when it's not found.
+MOUNTED=`mount | grep $EXTERN_ROOT || [[ $? == 1 ]]`
+if [ "$MOUNTED" != "" ]; then
+	echo "${EXTERN_ROOT?} is already mounted."
+elif [ "$MOUNT" = "mount" ]; then
+	echo "Mounting ${EXTERN_DEV?} to ${EXTERN_ROOT?}"
+	mount ${EXTERN_DEV?} ${EXTERN_ROOT?}
+fi
+
+# Do a second check regardless of --mount to ensure everything is OK.
+echo "Checking that drive is mounted at ${EXTERN_ROOT?} ..."
+MOUNTED=`mount | grep $EXTERN_ROOT || [[ $? == 1 ]]`
+if [ "$MOUNTED" = "" ]; then
+	>&2 echo "$EXTERN_ROOT is not mounted!"
+	exit -4
+fi
+
+
+LOG="${LOG_ROOT?}/$mode-shares-$(date '+%Y-%m-%d_%H-%M').log"
+echo "Everything looks good. Will log to ${LOG?}"
+
+function log() {
+	date_str=`date '+%Y-%m-%d %H:%M:%S'`
+	echo "[$date_str]: $1" 2>&1 | tee -a $LOG
+}
+
+function logCmd() {
+	cmd="$@"
+	log "Running '${cmd}'"
+	(time $cmd) 2>&1 | tee -a $LOG
+}
+
+function pause() {
+	read -s -n 1 -p "Press any key to continue, or ctrl+c to cancel."
+}
+
+function syncShare() {
+	localShare="$1"
+	if [ "$2" = "" ]; then
+		remoteShare="$1"
+	else
+		remoteShare="$2"
+	fi
+	
+	if [ "$mode" = "restore" ]; then
+		srcShare="$remoteShare"
+		destShare="$localShare"
+	else
+		srcShare="$localShare"
+		destShare="$remoteShare"
+	fi
+	# use a trailing slash to sync the contents of the directory
+	src="$SRC_ROOT/$srcShare/"
+	dest="$DEST_ROOT/$destShare/"
+
+	echo "" | tee -a $LOG
+	log '============================'
+	log "Syncing $src to $dest"
+	log "Checking if $dest exists"
+	if [[ ! -d "$dest" ]]; then
+		if [ "$mode" = "backup" ]; then
+			echo "$dest does not exist. Creating it."
+			mkdir -p $dest
+		else
+			echo "$dest does not exist. Exiting."
+			exit 1
+		fi
+	fi
+
+	if [ "$1" = "bittorrent" ]; then
+		DELETE="--delete-before"
+	fi
+
+	# Set noglob to prevent * from being expanded before logCmd is called
+	set -o noglob
+	logCmd rsync --archive --hard-links $DRY_RUN $VERBOSE \
+			--progress --partial \
+			--human-readable \
+			--exclude ".[!.]*-daily_20*/" \
+			--exclude ".[!.]*-weekly_20*/" \
+			--exclude ".daily_20*/" \
+			--exclude ".weekly_20*/" \
+			--exclude "transcoding-temp/" \
+			--exclude ".mount" \
+			$CHECKSUM \
+			$DELETE \
+			$src \
+			$dest
+	set +o noglob
+}
+
+function syncShares() {
+	pause
+	log "Starting sync"
+	syncShare bitwarden
+	syncShare jason Jason
+	syncShare jenn Jenn
+	syncShare nginx-config
+	syncShare git-config
+	syncShare git-storage
+	syncShare party-rescue
+	syncShare shared
+	#syncShare video-recordings
+	syncShare music
+	#syncShare nginx-data
+	syncShare emby-config
+	#syncShare odoo-data
+	if [ "$LARGE" ]; then
+		syncShare bittorrent
+	fi
+}
+
+function unmount() {
+	if [ "$UNMOUNT" ]; then
+		log "Unmounting drive"
+		logCmd sudo umount $EXTERN_ROOT
+	fi
+}
+
+log "Arguments: $FULL_ARGS"
+logCmd syncShares
+unmount
+log "Done"
+