A job runner for working with an AI

#!/usr/bin/env bash
# jobd.sh - runs job files an assistant drops into ~/jobs/inbox through a filesystem connector, so
# nobody has to run script after script and paste the results back. One job at a time, in file-name
# order; each runs with bash in ~/scripts; its output goes to ~/jobs/done/<name>.log and a one-line
# status to ~/jobs/done/<name>.status ("running <time>" then "exit <code> <time>"); the job file
# itself moves to ~/jobs/done/<name>.sh.
# When the queue has emptied it sends a Telegram (~/scripts/tg.sh; quiet until the bot is set up),
# with each job's result and its "NOTIFY: ..." line if it printed one. It reloads itself when this
# file changes, so an update needs no restart.
#   as a service (survives reboots):  bash ~/scripts/install-jobd-service.sh   -> jobd.service
#       is it up?   systemctl --user status jobd      log: ~/logs/jobd.log
#       (touch ~/jobs/STOP only makes the service restart itself; a stop that sticks is systemctl)
#   by hand (no service):   setsid nohup bash ~/scripts/jobd.sh > ~/logs/jobd.log 2>&1 < /dev/null &
# A job may itself start long work detached (setsid nohup) and finish at once; that work can end
# with tg.sh itself. Jobs that change real data are only dropped in after you have said go in chat.
#   lanes: JOBS_DIR picks the tree - ~/jobs is the GPU lane (jobd.service), ~/jobs-cpu the CPU lane
#   (jobd-cpu.service, from `install-jobd-service.sh cpu`); the two run in parallel and the
#   Telegram names the lane. GPU-heavy work goes to ~/jobs/inbox; everything else to ~/jobs-cpu/inbox.
set -u
J=${JOBS_DIR:-$HOME/jobs}
LANE=$(basename "$J"); LANE=${LANE#jobs}; LANE=${LANE#-}        # "" for ~/jobs, "cpu" for ~/jobs-cpu
SELF=$(readlink -f "$0"); STAMP=$(stat -c %Y "$SELF")
mkdir -p "$J/inbox" "$J/running" "$J/done" "$HOME/logs"
# a job in flight is a known state: the lane publishes jobd_busy for Prometheus through
# node_exporter's textfile directory when that directory exists, so a CPU-saturation alert can tell
# a measurement run from a fault. Atomic write; silent where there is no exporter.
METRICS_DIR=/var/lib/node_exporter/textfile
busy() {  # 1|0 [job name]
  [ -d "$METRICS_DIR" ] && [ -w "$METRICS_DIR" ] || return 0
  { echo "# HELP jobd_busy 1 while a job runner lane has a job in flight"; echo "# TYPE jobd_busy gauge";
    echo "jobd_busy{lane=\"${LANE:-gpu}\"} $1"; } > "$METRICS_DIR/jobd-${LANE:-gpu}.prom.tmp" && mv -f "$METRICS_DIR/jobd-${LANE:-gpu}.prom.tmp" "$METRICS_DIR/jobd-${LANE:-gpu}.prom"
}
busy 0
echo "$(date +%T) jobd up (pid $$, lane ${LANE:-gpu}), watching $J/inbox"
ran=""
while true; do
  if [ -e "$J/STOP" ]; then rm -f "$J/STOP"; echo "$(date +%T) stopped"; exit 0; fi
  if [ "$(stat -c %Y "$SELF" 2>/dev/null)" != "$STAMP" ]; then echo "$(date +%T) jobd.sh changed: reloading"; exec bash "$SELF"; fi
  f=$(ls "$J"/inbox/*.sh 2>/dev/null | head -1)
  if [ -z "$f" ]; then
    if [ -n "$ran" ]; then                            # the queue has emptied: say so, once
      bash "$HOME/scripts/tg.sh" "$(hostname)${LANE:+ [$LANE]} jobs finished:
$ran
Say 'check' in the chat."
      ran=""
    fi
    sleep 5; continue
  fi
  n=$(basename "$f" .sh)
  mv "$f" "$J/running/$n.sh"
  echo "running $(date -Is)" > "$J/done/$n.status"
  echo "$(date +%T) start $n"
  busy 1 "$n"
  ( cd "$HOME/scripts" && bash "$J/running/$n.sh" ) > "$J/done/$n.log" 2>&1
  rc=$?
  busy 0
  echo "exit $rc $(date -Is)" > "$J/done/$n.status"
  mv "$J/running/$n.sh" "$J/done/$n.sh"
  echo "$(date +%T) done $n rc=$rc"
  note=$(grep -m1 '^NOTIFY:' "$J/done/$n.log" | cut -c9-)
  if [ "$rc" -eq 0 ]; then r="done"; else r="FAILED (exit $rc)"; fi
  ran="${ran:+$ran
}- $r: ${n#*-*-}${note:+ - $note}"
done

#!/usr/bin/env bash
# tg.sh "text" - a Telegram message through your bot. Reads TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID
# from ~/.config/secrets.env; quietly does nothing if either is missing. Forced IPv4, because
# Telegram over some dual-stack hosts times out on v6.
# Detached work can call it when it ends:
#   setsid nohup sh -c 'long-thing; bash ~/scripts/tg.sh "long-thing done"' &
[ -f "$HOME/.config/secrets.env" ] && source "$HOME/.config/secrets.env"
[ -n "${TELEGRAM_BOT_TOKEN:-}" ] && [ -n "${TELEGRAM_CHAT_ID:-}" ] || exit 0
curl -s -4 --max-time 20 -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
  --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
  --data-urlencode "disable_web_page_preview=true" \
  --data-urlencode "text=$*" > /dev/null

#!/usr/bin/env bash
# install-jobd-service.sh - run the job runner (~/scripts/jobd.sh) as a systemd USER service, so it
# comes back after a reboot instead of needing the setsid/nohup line by hand. Needs jobd.sh,
# job-now.sh and tg.sh in ~/scripts (tg.sh needs its two values in ~/.config/secrets.env to speak).
#   bash ~/scripts/install-jobd-service.sh          # GPU lane: ~/jobs, jobd.service
#   bash ~/scripts/install-jobd-service.sh cpu      # CPU lane: ~/jobs-cpu, jobd-cpu.service
# It refuses to run while a job is in flight (the old runner would lose the job's result), stops a
# hand-started runner cleanly, writes the unit, turns on linger, enables and starts it.
# Afterwards:  systemctl --user status jobd   -  log ~/logs/jobd.log  -  stop: systemctl --user stop jobd
set -euo pipefail
LANE=${1:-}
case "$LANE" in ""|gpu) LANE="";; cpu) ;; *) echo "lane must be empty/gpu or cpu"; exit 1;; esac
J=$HOME/jobs${LANE:+-$LANE}
UNIT_NAME=jobd${LANE:+-$LANE}
UNIT_DIR=$HOME/.config/systemd/user
UNIT=$UNIT_DIR/$UNIT_NAME.service
mkdir -p "$J/inbox" "$J/running" "$J/done" "$J/staged" "$HOME/logs" "$UNIT_DIR"
[ -f "$HOME/scripts/jobd.sh" ] || { echo "no ~/scripts/jobd.sh here - copy jobd.sh, job-now.sh and tg.sh first"; exit 1; }

echo "== 1. a job in flight?"
if ls "$J"/running/*.sh >/dev/null 2>&1; then
  echo "   $(ls "$J"/running/*.sh | xargs -n1 basename) is running under the current runner."
  echo "   Run this again when it has finished (its Telegram), or it would end unrecorded."; exit 1
fi
echo "   none"

echo "== 2. a hand-started runner (this lane only)?"
if [ -z "$LANE" ] && pgrep -f 'bash .*scripts/jobd\.sh' >/dev/null && ! systemctl --user is-active -q jobd.service; then
  touch "$J/STOP"
  for i in $(seq 1 30); do pgrep -f 'bash .*scripts/jobd\.sh' >/dev/null || break; sleep 1; done
  if pgrep -f 'bash .*scripts/jobd\.sh' >/dev/null; then pkill -f 'bash .*scripts/jobd\.sh' || true; sleep 1; fi
  rm -f "$J/STOP"
  echo "   stopped"
else
  echo "   none"
fi

echo "== 3. the unit: $UNIT"
cat > "$UNIT" <<EOF
[Unit]
Description=job runner${LANE:+ ($LANE lane)}: runs $J/inbox/*.sh one at a time, logs to $J/done, Telegram when the queue empties
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
ExecStart=/usr/bin/bash %h/scripts/jobd.sh
WorkingDirectory=%h/scripts
Environment=JOBS_DIR=$J
Environment=PATH=/usr/local/bin:/usr/bin:/bin:%h/.local/bin:%h/.npm-global/bin
# the runner is a 5-second polling loop that reloads itself when jobd.sh changes; touching $J/STOP
# makes it exit 0, which under this unit just restarts it - a stop that sticks is:
#   systemctl --user stop $UNIT_NAME
Restart=always
RestartSec=5
# a job is a child bash; on stop/restart only the runner is signalled, a running job carries on to
# its end (its exit will not be recorded - restart the runner between jobs when you can)
KillMode=process
TimeoutStopSec=20
StandardOutput=append:%h/logs/$UNIT_NAME.log
StandardError=append:%h/logs/$UNIT_NAME.log

[Install]
WantedBy=default.target
EOF
echo "   written"

echo "== 4. linger (user units start at boot, no login needed)"
if [ "$(loginctl show-user "$USER" -p Linger --value 2>/dev/null)" = "yes" ]; then
  echo "   already on"
else
  if loginctl enable-linger "$USER" 2>/dev/null; then echo "   enabled"; else
    echo "   needs root: sudo loginctl enable-linger $USER"; sudo loginctl enable-linger "$USER" && echo "   enabled"; fi
fi

echo "== 5. enable + start"
systemctl --user daemon-reload
systemctl --user enable --now "$UNIT_NAME.service"
sleep 2
echo "   state: $(systemctl --user is-active "$UNIT_NAME.service")   enabled: $(systemctl --user is-enabled "$UNIT_NAME.service")"
tail -n 2 "$HOME/logs/$UNIT_NAME.log" | sed 's/^/   log: /'
echo
echo "done. The runner now starts at boot. Check:  systemctl --user status $UNIT_NAME --no-pager"

#!/usr/bin/env bash
# job-now.sh - run one queued job NOW, outside jobd. Takes the job file out of the inbox so jobd
# cannot run it a second time, runs it detached (setsid nohup) with jobd's conventions - log and
# status in done/ under the job's name, the job file moved to done/ at the end - and sends a
# Telegram with the job's NOTIFY line.
#   bash ~/scripts/job-now.sh 20261001-03-release-pack      (the file name without .sh)
set -u
n=${1:?job name (the inbox file name without .sh)}
n=${n%.sh}
# the job may sit in either lane's tree: ~/jobs (GPU) or ~/jobs-cpu (CPU); JOBS_DIR forces one
J=""
for cand in ${JOBS_DIR:-} "$HOME/jobs" "$HOME/jobs-cpu"; do
  [ -n "$cand" ] || continue
  if [ -f "$cand/inbox/$n.sh" ] || [ -f "$cand/staged/$n.sh" ]; then J=$cand; break; fi
done
[ -n "$J" ] || { echo "no $n.sh in ~/jobs or ~/jobs-cpu (inbox or staged)"; exit 1; }
mkdir -p "$J/staged" "$J/done"
if [ -f "$J/inbox/$n.sh" ]; then mv "$J/inbox/$n.sh" "$J/staged/$n.sh"; fi
cat > "$J/staged/$n.runner.sh" <<EOF
J=$J; n=$n
echo "running \$(date -Is)" > "\$J/done/\$n.status"
( cd "\$HOME/scripts" && bash "\$J/staged/\$n.sh" ) > "\$J/done/\$n.log" 2>&1
rc=\$?
echo "exit \$rc \$(date -Is)" > "\$J/done/\$n.status"
mv "\$J/staged/\$n.sh" "\$J/done/\$n.sh"
note=\$(grep -m1 '^NOTIFY:' "\$J/done/\$n.log" | cut -c9-)
if [ "\$rc" -eq 0 ]; then r="done"; else r="FAILED (exit \$rc)"; fi
bash "\$HOME/scripts/tg.sh" "\$(hostname) job (run now) \$r: \${n#*-*-}\${note:+ - \$note}
Say 'check' in the chat."
rm -f "\$J/staged/\$n.runner.sh"
EOF
setsid nohup bash "$J/staged/$n.runner.sh" > /dev/null 2>&1 < /dev/null &
echo "started $n, detached - log: $J/done/$n.log - Telegram when it finishes"