Our backup approach leverages a custom wrapper for syncoid, packaged in sanoid. Sanoid handles the snapshots, and then bacoid sends those snapshots incrementally to a server that can receive via SSH. Bacoid uses custom ZFS config variables to know which snapshots to send and where. A dataset that doesn't have snapshots does not get backed-up. If the dataset is encrypted, the backup will also be encrypted. The backups on the server-side then gets purged via that server's sanoid configuration.

The bacoid script:

#!/bin/bash

for copy in _a _b; do
	subvolArray=($(zfs get syncoid:pool$copy -H -o name,value -r $1  | grep -v '@' | awk '{if($2 != "-" && $2 != "") print $1}'))

	COUNT=1
	for subvol in ${subvolArray[@]}; do
		targetArray[$COUNT]="$(zfs get syncoid:target$copy ${subvol} | grep -v 'NAME' | awk '{print $3}')"
		COUNT=$(( ${COUNT} + 1 ))
	done
	targetArray=($(printf '%s\n' ${targetArray[@]} | awk '!a[$0]++'))


	for target in ${targetArray[@]}; do
		[[ ${target} != *@* ]] && continue
		echo "Testing SSH connection to ${target}"
		USERHOST="$(sed 's|:.*||' <<< ${target})"
		HOST="$(sed 's|.*@||' <<< ${USERHOST})"
		USER="$(sed 's|@.*||' <<< ${USERHOST})"
		PORT="$(sed 's|.*:||' <<< ${target})"
		if [[ ! -f "${HOME}/.ssh/bkp_rsa" ]]; then
			echo "Generating bkp_rsa key"
			ssh-keygen -N "" -f "${HOME}/.ssh/bkp_rsa"
		fi

		ssh -i "${HOME}/.ssh/bkp_rsa" \
       			-p ${PORT} \
       			-o ControlPath=none \
       			-o LogLevel=INFO \
       			-o PreferredAuthentications=publickey \
       			-o IdentitiesOnly=yes ${USERHOST} exit
		[[ $? -ne 0 ]] && ssh-copy-id -p ${PORT} -i "${HOME}/.ssh/bkp_rsa" ${USERHOST}
	done


	for subvol in ${subvolArray[@]}; do
		TARGET="$(zfs get syncoid:target$copy ${subvol} | grep -v NAME | awk '{print $3}')"
		if [[ ${target} != "-" ]]; then
			USERHOST="$(sed 's|:.*||' <<< ${TARGET})"
			HOST="$(sed 's|.*@||' <<< ${USERHOST})"
			USER="$(sed 's|@.*||' <<< ${USERHOST})"
			PORT="$(sed 's|.*:||' <<< ${TARGET})"
			MACHINE="$(zfs get syncoid:machine ${subvol} | grep -v NAME | awk '{print $3}')"
			[[ ${MACHINE} == '-' ]] && { echo "syncoid:machine not set for ${subvol}"; continue; }
			POOL="$(zfs get syncoid:pool$copy ${subvol} | grep -v NAME | awk '{print $3}')"
			[[ ${POOL} == '-' ]] && { echo "syncoid:pool$copy not set for ${subvol}"; continue; }
			echo "Sending ${subvol} to ${USERHOST}:${POOL}/${USER/-*}/$MACHINE/$subvol"
			syncoid --sendoptions="w" --recvoptions="u" --force-delete  --sshport=${PORT} --sshkey="${HOME}/.ssh/bkp_rsa" --no-sync-snap --no-privilege-elevation  ${subvol} ${USERHOST}:${POOL}/${USER/-*}/${MACHINE}/${subvol}
		else
			MACHINE="$(zfs get syncoid:machine ${subvol} | grep -v NAME | awk '{print $3}')"
			[[ ${MACHINE} == '-' ]] && { echo "syncoid:machine not set for ${subvol}"; continue; }
			POOL="$(zfs get syncoid:pool$copy ${subvol} | grep -v NAME | awk '{print $3}')"
			[[ ${POOL} == '-' ]] && { echo "syncoid:pool$copy not set for ${subvol}"; continue; }
			echo "Sending ${subvol} to ${POOL}/$MACHINE/$subvol"
			syncoid --sendoptions="w" --recvoptions="u" --no-sync-snap ${subvol} ${POOL}/${MACHINE}/${subvol}
		fi
	done
done