#!/usr/bin/env bash
#  flashstore-pve-setup.sh - put croit FlashStore OSDs on a Proxmox VE node.
#
#  Written for https://blogs.damiendye.uk/en/proxmox/flashstore-osd-on-proxmox/
#  and tested on the node that post was written on. Read the post first: every
#  step below is one of its sections, and the reasons are there.
#
#  What it does, in order:
#    1. checks this is Proxmox VE 9 with Ceph Umbrella 21.2.0 already set up
#       from the GUI (monitor and manager running), and stops if not
#    2. checks every drive you name: NVMe, no volatile write cache, not
#       mounted, partitioned or held, and 4 KiB blocks (or formats it to
#       4 KiB if you pass --format-4k)
#    3. downloads the FlashStore .deb for Debian 13, checks its SHA-256 and
#       installs it
#    4. writes the bootstrap-osd keyring, the systemd drop-in that lets the
#       OSD read /etc/pve/ceph.conf, and a udev rule giving user ceph the
#       namespace nodes of the drives you named (matched by serial number)
#    5. tests whether this kernel lets user ceph do NVMe passthrough I/O and
#       picks kernel-cmd if it does, kernel (direct I/O) if it does not
#    6. adds an [osd.N] section per drive, one physical core each, reserves
#       the hugepages the OSDs use, prepares and starts the OSDs
#
#  It only ever touches the drives you name, by PCI address. Nothing is
#  destroyed unless you ask: --format-4k reformats, --wipe overwrites a drive
#  that still holds data. Without --yes it shows the plan and asks first.
#
#  Usage:
#    flashstore-pve-setup.sh [options] PCI-ADDRESS...
#    flashstore-pve-setup.sh --dry-run 0000:05:00.0 0000:06:00.0
#
#  Find the addresses with:  lspci -D -d ::0108
#
#  Options:
#    --dry-run          show what would be done, change nothing
#    --yes              do not ask for confirmation
#    --format-4k        reformat drives that are not at 4 KiB (destroys data)
#    --wipe             let prepare overwrite a drive holding data (destroys it)
#    --backend B        auto (default), kernel or kernel-cmd
#    --single-node      add CRUSH rule replicated_osd (failure domain osd) and
#                       move the .mgr pool onto it; only for a one-node cluster
#    --release TAG      ceph-rust release to install
#                       (default snapshot-2026-11-10-0b6783237291)
#
#  FlashStore is in public preview and croit say it is not yet for production
#  data. Neither is this script. Public domain; do what you like with it.

set -euo pipefail

RELEASE=snapshot-2026-11-10-0b6783237291
REPO=https://github.com/croit/ceph-rust/releases/download
CEPH_WANT=21.2.0
DRY=0 YES=0 FORMAT=0 WIPE=0 SINGLE=0 BACKEND=auto
CONF=/etc/pve/ceph.conf
DROPIN=/etc/systemd/system/croit-flashstore-osd@.service.d/pve-ceph-conf.conf
UDEV=/etc/udev/rules.d/70-croit-flashstore.rules
SYSCTL=/etc/sysctl.d/90-croit-flashstore-hugepages.conf
KEYRING=/var/lib/ceph/bootstrap-osd/ceph.keyring
HUGE_MB=2048   # per OSD: the daemon's default --mem, which prep needs free at each start

say()  { printf '%s\n' "$*"; }
step() { printf '\n== %s\n' "$*"; }
die()  { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
run()  { if [ "$DRY" = 1 ]; then printf '  would run: %s\n' "$*"; else "$@"; fi; }
# write FILE (content on stdin), or show it in a dry run
put()  {
	if [ "$DRY" = 1 ]; then printf '  would write %s:\n' "$1"; sed 's/^/    | /'; return; fi
	mkdir -p "$(dirname "$1")"; cat > "$1"
}

usage() { sed -n '/^#  Usage:/,/^#  FlashStore is/p' "$0" | sed 's/^#  \{0,1\}//' | head -n -1; exit "${1:-0}"; }

BDFS=()
while [ $# -gt 0 ]; do
	case $1 in
		--dry-run) DRY=1 ;;
		--yes) YES=1 ;;
		--format-4k) FORMAT=1 ;;
		--wipe) WIPE=1 ;;
		--single-node) SINGLE=1 ;;
		--backend) BACKEND=${2:?}; shift ;;
		--release) RELEASE=${2:?}; shift ;;
		-h|--help) usage 0 ;;
		-*) die "unknown option $1 (try --help)" ;;
		*) BDFS+=("$1") ;;
	esac
	shift
done
[ ${#BDFS[@]} -gt 0 ] || usage 1
case $BACKEND in auto|kernel|kernel-cmd) ;; *) die "--backend must be auto, kernel or kernel-cmd" ;; esac
[ "$(id -u)" = 0 ] || die "run as root on the Proxmox node"

# ------------------------------------------------------------------ 1. the node
step "Proxmox VE and Ceph"
command -v pveversion >/dev/null || die "pveversion not found: this is not a Proxmox VE node"
pve=$(pveversion | sed -n 's|^pve-manager/\([0-9]*\)\..*|\1|p')
[ "$pve" = 9 ] || die "needs Proxmox VE 9 (found pve-manager major ${pve:-unknown})"
for t in nvme lspci lsblk wipefs curl sha256sum systemd-run udevadm; do
	command -v $t >/dev/null || die "$t not found"
done
[ -f "$CONF" ] || die "no $CONF: install Ceph first, in the GUI: Datacenter > Ceph > Install,
       version umbrella (21.2), then create the first monitor and manager"
vers=$(timeout 20 ceph versions -f json 2>/dev/null) || die "ceph versions failed: is the monitor up?"
grep -q "ceph version $CEPH_WANT .*umbrella" <<<"$vers" \
	|| die "the cluster is not Ceph $CEPH_WANT Umbrella, which this FlashStore build speaks"
say "pve-manager $pve, Ceph $CEPH_WANT Umbrella: ok"

# ---------------------------------------------------------------- 2. the drives
step "Drives"
declare -A NS NG SERIAL NEEDFMT
for bdf in "${BDFS[@]}"; do
	[[ $bdf =~ ^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$ ]] || die "$bdf: not a PCI address like 0000:05:00.0"
	d=/sys/bus/pci/devices/$bdf
	[ -d "$d" ] || die "$bdf: no such PCI device"
	[ "$(cat "$d/class")" = 0x010802 ] || die "$bdf: not an NVMe controller"
	[ "$(basename "$(readlink -f "$d/driver")")" = nvme ] || die "$bdf: not on the kernel nvme driver (passed through to a VM?)"
	ctrl=$(basename "$(ls -d "$d"/nvme/nvme* | head -1)")
	serial=$(sed 's/[[:space:]]*$//' "$d/nvme/$ctrl/serial")
	ng=$(basename "$(ls -d "$d/nvme/$ctrl"/ng*n1 2>/dev/null | head -1)")
	ns=$(lsblk -dn -o NAME,SERIAL | awk -v s="$serial" '$2==s && $1 ~ /^nvme/ {print $1; exit}')
	[ -n "$ns" ] && [ -n "$ng" ] || die "$bdf: cannot find its namespace nodes"
	[ "$(ls -d "$d/nvme/$ctrl"/ng*n* | wc -l)" = 1 ] || die "$bdf: more than one namespace; this script handles one per controller"
	NS[$bdf]=$ns NG[$bdf]=$ng SERIAL[$bdf]=$serial
	model=$(sed 's/[[:space:]]*$//' "$d/nvme/$ctrl/model")

	# an OSD already set up on this drive is left alone
	if grep -q "flashstore_dev = spdk:$bdf\$" "$CONF"; then
		say "$bdf $ns $model: already in $CONF, skipped"; continue
	fi
	# power-loss protection: FlashStore's mkfs refuses a volatile write cache
	vwc=$(nvme id-ctrl "/dev/$ctrl" | awk '/^vwc /{print $3}')
	[ "$vwc" = 0 ] || die "$bdf $model: volatile write cache present (vwc $vwc), no power-loss protection"
	# never the boot drive, never a drive in use
	[ -z "$(ls "/sys/block/$ns/holders")" ] || die "$bdf /dev/$ns: held by $(ls "/sys/block/$ns/holders" | tr '\n' ' ')"
	[ "$(lsblk -n -o NAME "/dev/$ns" | wc -l)" = 1 ] || die "$bdf /dev/$ns: has partitions"
	! grep -q "^/dev/$ns" /proc/mounts || die "$bdf /dev/$ns: mounted"
	# wipefs misses plenty (an old FlashStore store, LVM text metadata whose
	# label is gone), so also read the windows FlashStore's own probe reads:
	# the first 8 KiB and 8 KiB at 64 KiB and 256 KiB. Anything non-zero is data.
	data=0
	for off in 0 8 32; do
		[ -n "$(dd if="/dev/$ns" bs=8192 skip=$off count=1 iflag=direct 2>/dev/null | tr -d '\0' | head -c1)" ] && data=1
	done
	[ -n "$(wipefs -n "/dev/$ns" 2>/dev/null)" ] && data=1
	if [ "$data" = 1 ] && [ "$WIPE" = 0 ]; then
		die "$bdf /dev/$ns: holds data; pass --wipe to overwrite it (destroys it)"
	fi
	if [ "$(cat "/sys/block/$ns/queue/logical_block_size")" != 4096 ]; then
		[ "$FORMAT" = 1 ] || die "$bdf /dev/$ns: not 4 KiB blocks; pass --format-4k to reformat it (destroys data)"
		NEEDFMT[$bdf]=1
	fi
	say "$bdf /dev/$ns $model sn $serial: ok${NEEDFMT[$bdf]:+, will be formatted to 4 KiB}"
done

TODO=()
for bdf in "${BDFS[@]}"; do grep -q "flashstore_dev = spdk:$bdf\$" "$CONF" || TODO+=("$bdf"); done
[ ${#TODO[@]} -gt 0 ] || { say; say "Every drive named is already a FlashStore OSD. Nothing to do."; exit 0; }

if [ "$DRY" = 0 ] && [ "$YES" = 0 ]; then
	say; say "About to set up ${#TODO[@]} OSD(s) on: ${TODO[*]}"
	[ ${#NEEDFMT[@]} -gt 0 ] && say "Reformatting to 4 KiB: ${!NEEDFMT[*]} (destroys their data)"
	[ "$WIPE" = 1 ] && say "--wipe given: prepare will overwrite anything on those drives"
	read -r -p "Carry on? [y/N] " a; [[ $a == [yY]* ]] || exit 1
fi

# ------------------------------------------------------------- 2b. 4 KiB format
# A P4800X takes about six minutes per drive and answers nothing else while it
# formats. Any admin command that arrives meanwhile (smartctl, nvme list, the
# GUI's disk list) times out after 60 s, the kernel resets the controller and
# the reset aborts the format. So: one drive at a time, in its own unit, with
# a long timeout, and nothing else touches the drives until it finishes.
for bdf in "${!NEEDFMT[@]}"; do
	ns=${NS[$bdf]}
	lbaf=$(nvme id-ns "/dev/$ns" | awk '/^lbaf/ && /ms:0 / && /lbads:12 /{print $2; exit}')
	[ -n "$lbaf" ] || die "$bdf: the drive offers no 4096+0 LBA format"
	step "Formatting /dev/$ns to LBA format $lbaf (4 KiB). Leave the drives alone until it finishes."
	run systemd-run --unit="flashstore-fmt-${ns}" --wait --collect \
		nvme format "/dev/$ns" --lbaf="$lbaf" --force --timeout=900000
	[ "$DRY" = 1 ] || [ "$(cat "/sys/block/$ns/queue/logical_block_size")" = 4096 ] \
		|| die "$bdf: still not 4 KiB after the format; see journalctl -u flashstore-fmt-${ns}"
	run wipefs -a "/dev/$ns"
done

# ------------------------------------------------------------------- 3. package
step "FlashStore package ($RELEASE)"
tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
curl -fsSL -o "$tmp/sha256sums.txt" "$REPO/$RELEASE/sha256sums.txt"
# the checksum list names it debian-trixie/flashstore_V_amd64.deb, the release
# asset is flashstore_V_debian-trixie_amd64.deb: same file, different names
line=$(grep -E ' debian-trixie/flashstore_.*_amd64\.deb$' "$tmp/sha256sums.txt") || die "no Debian 13 package in $RELEASE"
want=${line%% *}
ver=$(sed -E 's|.*/flashstore_(.*)_amd64\.deb$|\1|' <<<"$line")
have=$(dpkg-query -W -f='${Version}' croit-flashstore-osd 2>/dev/null || true)
if [ "$have" = "$ver" ]; then
	say "croit-flashstore-osd $ver already installed"
else
	deb="flashstore_${ver}_debian-trixie_amd64.deb"
	curl -fsSL -o "$tmp/$deb" "$REPO/$RELEASE/$deb"
	got=$(sha256sum "$tmp/$deb" | cut -d' ' -f1)
	[ "$got" = "$want" ] || die "$deb: SHA-256 $got does not match the release's $want"
	say "$deb: SHA-256 matches"
	run env DEBIAN_FRONTEND=noninteractive apt-get install -y "$tmp/$deb"
fi
[ "$DRY" = 1 ] || croit-flashstore-osd --version | head -1

# ------------------------------------------------- 4. keyring, drop-in and udev
step "Proxmox fixes"
# pveceph only writes this the first time it creates a classic OSD
if [ ! -s "$KEYRING" ]; then
	run mkdir -p "$(dirname "$KEYRING")"
	run ceph auth get client.bootstrap-osd -o "$KEYRING"
fi
# The shipped unit runs as ceph from the start and cannot read pmxcfs
# (root:www-data 0640). Start as root:ceph, the way pveceph starts its own
# OSDs, and let the daemon drop to ceph before it opens anything.
put "$DROPIN" <<'EOF'
# Proxmox VE: /etc/ceph/ceph.conf -> /etc/pve/ceph.conf (pmxcfs, root:www-data 0640).
# Root reads it as owner, group ceph opens the NVMe nodes, then the daemon drops to ceph.
[Service]
User=root
Group=ceph
ExecStart=
ExecStart=/usr/bin/croit-flashstore-osd run --id %i --cluster ${CLUSTER} -f --setuser ceph --setgroup ceph
EOF
run systemctl daemon-reload
# The kernel tier needs ceph to own the namespace nodes. Matched by serial, so
# the rule cannot catch another drive whatever the nvme numbering does.
{
	[ -f "$UDEV" ] && cat "$UDEV" || echo "# croit FlashStore: user ceph owns the namespace nodes of these drives only"
	for bdf in "${TODO[@]}"; do
		s=${SERIAL[$bdf]}
		grep -qF "\"$s\"" "$UDEV" 2>/dev/null && continue
		echo "# $bdf"
		echo "SUBSYSTEM==\"nvme-generic\", KERNEL==\"ng*n*\", ATTRS{serial}==\"$s\", OWNER=\"ceph\", GROUP=\"ceph\", MODE=\"0660\""
		echo "SUBSYSTEM==\"block\", KERNEL==\"nvme*n*\", ENV{DEVTYPE}==\"disk\", ATTRS{serial}==\"$s\", OWNER=\"ceph\", GROUP=\"ceph\", MODE=\"0660\""
	done
} > "$tmp/udev.rules"
put "$UDEV" < "$tmp/udev.rules"
run udevadm control --reload
run udevadm trigger --action=change --subsystem-match=nvme-generic --subsystem-match=block
run udevadm settle
if [ "$DRY" = 0 ]; then
	for bdf in "${TODO[@]}"; do
		for n in "${NS[$bdf]}" "${NG[$bdf]}"; do
			[ "$(stat -c %U "/dev/$n")" = ceph ] || die "/dev/$n is not owned by ceph after the udev rule"
		done
	done
	say "namespace nodes owned by ceph"
fi

# ------------------------------------------------------------------ 5. backend
# kernel-cmd (NVMe passthrough on /dev/ngXnY) is croit's faster mode, but some
# kernels refuse passthrough I/O to an unprivileged user, Proxmox's 7.0.14 among
# them, and FlashStore does not fall back by itself. Ask the kernel, as ceph.
if [ "$BACKEND" = auto ]; then
	if [ "$DRY" = 1 ]; then
		BACKEND="kernel (or kernel-cmd: decided by a passthrough read as ceph)"
	else
		ng=${NG[${TODO[0]}]}
		if runuser -u ceph -- nvme io-passthru "/dev/$ng" --opcode=0x02 --namespace-id=1 \
			--data-len=4096 --read -b >/dev/null 2>&1; then BACKEND=kernel-cmd; else BACKEND=kernel; fi
	fi
fi
say "store backend: $BACKEND"

# -------------------------------------------------------- 6. config and cores
step "OSD sections in $CONF"
# one physical core per OSD (the first thread of each core), skipping cores
# already given to a FlashStore OSD; SMT siblings are left to the host
mapfile -t CORES < <(for c in /sys/devices/system/cpu/cpu[0-9]*/topology/thread_siblings_list; do
	cut -d, -f1 "$c" | cut -d- -f1; done | sort -nu)
mapfile -t USED < <(sed -n 's/^[[:space:]]*flashstore_lcores = //p' "$CONF")
mapfile -t IDS < <({ timeout 20 ceph osd ls; sed -n 's/^\[osd\.\([0-9]*\)\]$/\1/p' "$CONF"; } | sort -nu)
next=$(( ${#IDS[@]} ? ${IDS[-1]} + 1 : 0 ))
declare -A OSDID
for bdf in "${TODO[@]}"; do
	core=
	for c in "${CORES[@]}"; do
		printf '%s\n' "${USED[@]}" | grep -qx "$c" && continue
		core=$c; USED+=("$c"); break
	done
	OSDID[$bdf]=$next
	{
		printf '\n[osd.%s]\n\tflashstore_dev = spdk:%s\n\tflashstore_store_backend = %s\n' "$next" "$bdf" "$BACKEND"
		[ -n "$core" ] && printf '\tflashstore_lcores = %s\n' "$core"
	} > "$tmp/osd.$next"
	[ -n "$core" ] || say "WARNING: no free physical core for osd.$next; it will share"
	if [ "$DRY" = 1 ]; then put "$CONF (append)" < "$tmp/osd.$next"; else cat "$tmp/osd.$next" >> "$CONF"; fi
	say "osd.$next on $bdf${core:+, core $core}"
	next=$((next + 1))
done

# hugepages: the kernel backend uses a quarter of --mem (512 MiB, 256 pages),
# but croit-flashstore-prep wants a whole --mem (1024 pages) free before each
# OSD starts, and tops the pool up itself if it is short. So reserve what the
# OSDs use plus one start's margin, not the 2048 MiB per OSD the check asks for
n=$(grep -c 'flashstore_dev = ' "$CONF" || true)
[ "$DRY" = 1 ] && n=$(( n + ${#TODO[@]} ))
use=$(( HUGE_MB / 4 / 2 )); need=$(( HUGE_MB / 2 ))
pages=$(( n * use + need - use ))
printf '# croit FlashStore: %s OSDs x %s MiB used, plus %s MiB free for an OSD start\nvm.nr_hugepages = %s\n' "$n" "$(( use * 2 ))" "$(( (need - use) * 2 ))" "$pages" | put "$SYSCTL"
run sysctl -q -p "$SYSCTL"

# the host check: print what is wrong, but a boot SSD with a write cache fails
# it on every host, so only stop on the things that would stop the OSDs. Its
# hugepage finding is left out on purpose: it sizes 2048 MiB per OSD, and prep
# tops the pool up at each start and refuses to start an OSD it cannot feed
if [ "$DRY" = 0 ]; then
	step "Host check"
	croit-flashstore-osd --check > "$tmp/check.txt" 2>&1 || true
	grep -E '^(FAIL|WARN) ' "$tmp/check.txt" || true
	if grep -E '^FAIL ' "$tmp/check.txt" | grep -qE "io_uring|$(IFS='|'; echo "${TODO[*]}")"; then
		die "the host check failed on something the OSDs need; see above"
	fi
fi

# --------------------------------------------------------- 7. prepare, start
step "Prepare and start"
WIPEFLAG=(); [ "$WIPE" = 1 ] && WIPEFLAG=(--wipe)
for bdf in "${TODO[@]}"; do
	id=${OSDID[$bdf]}
	run croit-flashstore-tool prepare --dev "spdk:$bdf" --id "$id" "${WIPEFLAG[@]}"
	run systemctl enable --now "croit-flashstore-osd@$id"
done
if [ "$DRY" = 0 ]; then
	ids=$(for bdf in "${TODO[@]}"; do echo "${OSDID[$bdf]}"; done | paste -sd,)
	for _ in $(seq 1 30); do
		down=$(timeout 10 ceph osd dump -f json | python3 -c '
import json, sys
want = {int(x) for x in sys.argv[1].split(",")}
up = {o["osd"] for o in json.load(sys.stdin)["osds"] if o["up"]}
print(len(want - up))' "$ids")
		[ "$down" = 0 ] && break; sleep 2
	done
	[ "$down" = 0 ] || die "not every new OSD came up; see journalctl -u croit-flashstore-osd@N"
fi

if [ "$SINGLE" = 1 ]; then
	step "Single-node CRUSH rule"
	timeout 20 ceph osd crush rule ls | grep -qx replicated_osd \
		|| run ceph osd crush rule create-replicated replicated_osd default osd nvme
	run ceph osd pool set .mgr crush_rule replicated_osd
fi

step "Done"
[ "$DRY" = 1 ] && { say "Dry run: nothing was changed."; exit 0; }
timeout 20 ceph osd tree
say
say "Create a pool in the GUI: Ceph > Pools > Create, Add as Storage ticked."
[ "$SINGLE" = 1 ] && say "On this single node, pick the CRUSH rule replicated_osd."
say "The Proxmox OSD panel will call these OSDs filestore, and its Start/Stop"
say "buttons drive ceph-osd@N. Use systemctl ... croit-flashstore-osd@N instead."