feat: federated learning demo on the actor framework
Add a FedAvg fraud-detection application in two topologies: provider mode (coordinator/trainer star) and P2P mode (full mesh with CRDT-gated rounds), backed by an MLP trained on PaySim data with a synthetic fallback, plus demo scripts for both modes. Also harden the actor runtime: - supervisor: ignore stale death messages (OneForAll restart storm), drop normally-exited children instead of respawning, add a restart-rate limit - register actors before their thread starts so UUID dispatch and getSelf cannot race the spawn - killActor now kills the thread so links see Killed - fulfill connection promises with the pool winner on simultaneous connects - clamp the PaySim test split on small datasets and split shared transaction types across trainers in non-IID partitioning Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Demo: federated learning in P2P mode (full mesh, no central coordinator).
|
||||
# Starts N peer nodes as separate processes; each trains locally, gossips
|
||||
# updates via PeerSync (CRDT-gated rounds), and aggregates independently.
|
||||
# At the end every peer must hold the SAME model — the script verifies the
|
||||
# saved model files are byte-identical.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/demo-p2p.sh [ROUNDS] [PEERS] [EPOCHS] [/path/to/paysim.csv]
|
||||
#
|
||||
# Without a CSV path, nodes fall back to synthetic data (fast; the metrics are
|
||||
# meaningless, but the gossip protocol and aggregation are fully exercised).
|
||||
set -euo pipefail
|
||||
|
||||
ROUNDS="${1:-3}"
|
||||
PEERS="${2:-3}"
|
||||
EPOCHS="${3:-2}"
|
||||
DATA="${4:-}"
|
||||
|
||||
BASE_PORT=9600
|
||||
OUT="demo-out/p2p"
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
echo "==> Building..."
|
||||
cabal build exe:fl-actors >/dev/null
|
||||
BIN="$(cabal list-bin fl-actors)"
|
||||
|
||||
mkdir -p "$OUT"
|
||||
rm -f "$OUT"/*.log fl_model_peer-*.bin
|
||||
|
||||
DATA_ARGS=()
|
||||
[ -n "$DATA" ] && DATA_ARGS=(--data "$DATA")
|
||||
|
||||
PIDS=()
|
||||
cleanup() { kill "${PIDS[@]}" 2>/dev/null || true; wait 2>/dev/null || true; }
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
for i in $(seq 0 $((PEERS - 1))); do
|
||||
PORT=$((BASE_PORT + i))
|
||||
ARGS=(--mode p2p --role peer --id "$i" --port "$PORT"
|
||||
--rounds "$ROUNDS" --trainers "$PEERS" --epochs "$EPOCHS")
|
||||
for j in $(seq 0 $((PEERS - 1))); do
|
||||
[ "$j" = "$i" ] && continue
|
||||
ARGS+=(--peer "localhost:$((BASE_PORT + j))")
|
||||
done
|
||||
echo "==> Starting peer-$i on port $PORT"
|
||||
"$BIN" "${ARGS[@]}" "${DATA_ARGS[@]}" > "$OUT/peer$i.log" 2>&1 &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
echo "==> Federation running; waiting for all $PEERS peers to finish..."
|
||||
echo " (live log: tail -f $OUT/peer0.log)"
|
||||
FAIL=0
|
||||
for pid in "${PIDS[@]}"; do
|
||||
wait "$pid" || FAIL=1
|
||||
done
|
||||
PIDS=()
|
||||
if [ "$FAIL" -ne 0 ]; then
|
||||
echo "!! A peer exited abnormally; check $OUT/ logs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "==> Evaluation results (peer-0's view):"
|
||||
grep "Eval round=" "$OUT/peer0.log" || true
|
||||
echo
|
||||
|
||||
MODELS=(fl_model_peer-*.bin)
|
||||
if [ "${#MODELS[@]}" -ne "$PEERS" ]; then
|
||||
echo "!! Expected $PEERS model files, found ${#MODELS[@]}; check $OUT/ logs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FIRST="${MODELS[0]}"
|
||||
for m in "${MODELS[@]:1}"; do
|
||||
if ! cmp -s "$FIRST" "$m"; then
|
||||
echo "!! Divergence: $m differs from $FIRST"
|
||||
md5sum "${MODELS[@]}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "==> Success: all $PEERS peers converged to a byte-identical model:"
|
||||
ls -la "${MODELS[@]}"
|
||||
echo " Full logs in $OUT/"
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Demo: federated learning in PROVIDER mode (star topology).
|
||||
# Starts one coordinator and N trainer nodes as separate processes, runs the
|
||||
# federation, then prints the evaluation log and the saved model file.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/demo-provider.sh [ROUNDS] [TRAINERS] [EPOCHS] [/path/to/paysim.csv]
|
||||
#
|
||||
# Without a CSV path, nodes fall back to synthetic data (fast; the metrics are
|
||||
# meaningless, but the message flow and aggregation are fully exercised).
|
||||
set -euo pipefail
|
||||
|
||||
ROUNDS="${1:-3}"
|
||||
TRAINERS="${2:-3}"
|
||||
EPOCHS="${3:-2}"
|
||||
DATA="${4:-}"
|
||||
|
||||
BASE_PORT=9500
|
||||
OUT="demo-out/provider"
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
echo "==> Building..."
|
||||
cabal build exe:fl-actors >/dev/null
|
||||
BIN="$(cabal list-bin fl-actors)"
|
||||
|
||||
mkdir -p "$OUT"
|
||||
rm -f "$OUT"/*.log fl_model_coordinator.bin
|
||||
|
||||
DATA_ARGS=()
|
||||
[ -n "$DATA" ] && DATA_ARGS=(--data "$DATA")
|
||||
|
||||
PIDS=()
|
||||
cleanup() { kill "${PIDS[@]}" 2>/dev/null || true; wait 2>/dev/null || true; }
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
echo "==> Starting coordinator on port $BASE_PORT (rounds=$ROUNDS, trainers=$TRAINERS, epochs=$EPOCHS)"
|
||||
"$BIN" --mode provider --role coordinator \
|
||||
--port "$BASE_PORT" --rounds "$ROUNDS" --trainers "$TRAINERS" --epochs "$EPOCHS" \
|
||||
"${DATA_ARGS[@]}" > "$OUT/coordinator.log" 2>&1 &
|
||||
COORD=$!
|
||||
PIDS+=("$COORD")
|
||||
sleep 1
|
||||
|
||||
for i in $(seq 0 $((TRAINERS - 1))); do
|
||||
echo "==> Starting trainer-$i on port $((BASE_PORT + i + 1))"
|
||||
"$BIN" --mode provider --role trainer --id "$i" \
|
||||
--base-port "$BASE_PORT" --coord-host localhost --coord-port "$BASE_PORT" \
|
||||
--rounds "$ROUNDS" --trainers "$TRAINERS" --epochs "$EPOCHS" \
|
||||
"${DATA_ARGS[@]}" > "$OUT/trainer$i.log" 2>&1 &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
echo "==> Federation running; waiting for the coordinator to finish..."
|
||||
echo " (live log: tail -f $OUT/coordinator.log)"
|
||||
if ! wait "$COORD"; then
|
||||
echo "!! Coordinator exited abnormally; last log lines:"
|
||||
tail -20 "$OUT/coordinator.log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "==> Evaluation results:"
|
||||
grep "Eval round=" "$OUT/coordinator.log" || true
|
||||
echo
|
||||
if [ -f fl_model_coordinator.bin ]; then
|
||||
echo "==> Success: model saved to fl_model_coordinator.bin ($(stat -c %s fl_model_coordinator.bin) bytes)"
|
||||
echo " Full logs in $OUT/"
|
||||
else
|
||||
echo "!! No model file was produced; check $OUT/ logs"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user