النشر في الإنتاج

# على الـ instance
sudo apt update && sudo apt upgrade -y
sudo apt install -y build-essential

# اسم مضيف واضح + linger (مهم جداً — شرحه تحت)
sudo hostnamectl set-hostname openclaw
sudo passwd ubuntu
sudo loginctl enable-linger ubuntu

# Hermes
curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash
hermes --version
hermes setup          # model + provider + API keys
hermes doctor         # لازم يخرج 0

# OpenClaw (اختياري — على نفس الجهاز بـ profile منفصل)
curl -fsSL https://openclaw.ai/install.sh | bash
source ~/.bashrc
openclaw --version
openclaw doctor

# Hermes — الـ subcommands الموثّقة
hermes gateway setup     # قنوات + allowlists
hermes gateway install   # systemd (Linux) / launchd (macOS)
hermes gateway start
hermes gateway status
hermes gateway list      # كل الـ profiles وحالة كل gateway (+ PID)

# OpenClaw — من الـ runbook الرسمي
openclaw gateway install
systemctl --user enable --now openclaw-gateway.service
openclaw gateway status

# الاستمرارية بعد logout
sudo loginctl enable-linger $(whoami)

[Unit]
Description=OpenClaw Gateway
After=network-online.target
Wants=network-online.target
StartLimitBurst=10
StartLimitIntervalSec=300

[Service]
ExecStart=/usr/local/bin/openclaw gateway --port 18789
Restart=always
RestartSec=5
RestartPreventExitStatus=78
TimeoutStopSec=330
TimeoutStartSec=30
SuccessExitStatus=0 143
OOMPolicy=continue
KillMode=mixed

[Install]
WantedBy=default.target

# تعديل آمن على الـ unit المُدارة
systemctl --user edit openclaw-gateway.service

[Service]
Environment=OPENCLAW_NO_RESPAWN=1
Environment=NODE_COMPILE_CACHE=/var/tmp/openclaw-compile-cache
TimeoutStartSec=90

# Hermes
systemctl --user is-active hermes-gateway.service   # exit 0 = active
systemctl --user show hermes-gateway.service -p NRestarts   # عدد مرات إعادة التشغيل
systemctl --user reset-failed hermes-gateway   # لو معلّقة في failed state
hermes gateway status
hermes gateway list
hermes doctor                # exit 0 = مفيش مشاكل، exit 1 = في مشكلة

# OpenClaw — فحص أعمق
openclaw gateway status --deep          # system-level service scan
openclaw gateway status --require-rpc       # إثبات read-scope RPC، مش مجرد reachability
openclaw channels status --probe

#!/usr/bin/env bash
# /usr/local/bin/agent-healthcheck
set -euo pipefail
rc=0

# 1) systemd active
systemctl --user is-active --quiet hermes-gateway.service \
  || { echo "FAIL: unit not active"; rc=1; }

# 2) no restart storm
n=$(systemctl --user show hermes-gateway.service -p NRestarts --value)
if [ "$n" -gt 3 ]; then echo "WARN: $n restarts"; fi

# 3) API server end-to-end (اختياري لو مفعّل)
if [ -n "${API_SERVER_KEY:-}" ]; then
  curl -sf --max-time 20 http://127.0.0.1:8642/v1/chat/completions \
    -H "Authorization: Bearer ***" \
    -H "Content-Type: application/json" \
    -d '{"model":"hermes-agent","messages":[{"role":"user","content":"ping"}]}' \
    >/dev/null || { echo "FAIL: API server unhealthy"; rc=1; }
fi

# 4) disk space
used=$(df --output=pcent / | tail -1 | tr -dc '0-9')
[ "$used" -gt 90 ] && { echo "FAIL: disk ${used}%"; rc=1; }

[ $rc -eq 0 ] && echo "OK"
exit $rc

# كل 5 دقايق
*/5 * * * * /usr/local/bin/agent-healthcheck 2>&1 | tee -a ~/healthcheck.log

# /etc/caddy/Caddyfile
agent.example.com {
    encode zstd gzip

    # WebSocket / streaming يحتاج headers صريحة
    reverse_proxy 127.0.0.1:8642 {
        flush_interval -1
        header_up Connection {>Connection}
        header_up Upgrade {>Upgrade}
    }

    log {
        output file /var/log/caddy/agent-access.log
        format json
    }
}

# تثبيت Caddy على Ubuntu 24.04 ARM
sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https curl
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' \
  | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' \
  | sudo tee /etc/apt/sources.list.d/caddy-stable.list
sudo apt update && sudo apt install -y caddy

# تحقّق ثم أعد التحميل
sudo caddy validate --config /etc/caddy/Caddyfile
sudo systemctl reload caddy
curl -I https://agent.example.com

# /etc/nginx/sites-available/agent
server {
    listen 443 ssl http2;
    server_name agent.example.com;

    ssl_certificate     /etc/letsencrypt/live/agent.example.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/agent.example.com/privkey.pem;

    # مهم: الوقت كبير — نداءات الوكيل ممكن تاخد دقايق
    proxy_read_timeout  600s;
    proxy_buffering     off;          # للـ SSE streaming
    client_max_body_size 100M;

    location / {
        proxy_pass         http://127.0.0.1:8642;
        proxy_http_version 1.1;
        proxy_set_header   Upgrade           $http_upgrade;
        proxy_set_header   Connection        "upgrade";
    }
}

sudo apt install -y nginx certbot python3-certbot-nginx
sudo ln -s /etc/nginx/sites-available/agent /etc/nginx/sites-enabled/
sudo nginx -t
sudo certbot --nginx -d agent.example.com
sudo systemctl reload nginx

# تدوير اللوجات — journald لازم persistent على الإنتاج
# /etc/systemd/journald.conf.d/agent.conf
[Journal]
Storage=persistent
SystemMaxUse=2G
MaxRetentionSec=30day
Compress=yes

# hermes logs
hermes logs                       # آخر 50 سطر من agent.log
hermes logs -f                    # real-time
hermes logs gateway -n 100        # آخر 100 سطر من gateway.log
hermes logs --level WARNING --since 1h
hermes logs errors --since 30m -f
hermes logs list                  # كل ملفات اللوج + أحجامها

# journalctl على إنتاج
sudo systemctl restart systemd-journald
journalctl --disk-usage
journalctl -u hermes-gateway.service --since today
journalctl -u 'hermes-gateway*' --since 1h -p err
journalctl -u hermes-gateway.service --since 1h -o json > /tmp/j.json

# كشف تسريب محتمل لأسرار في اللوجات
journalctl -u hermes-gateway.service -p warning --since 1d \
  | grep -iE 'api[_-]?key|token|password' && echo "🔴 possible secret leak"

# تنبيه متعدد القنوات — دايم بلّغ المستخدم على القناة
hermes send --to telegram "[ALERT] gateway down on $(hostname)"
hermes send --to discord:#ops --subject "[ALERT] agent degraded"
curl -d "gateway down on $(hostname)" https://ntfy.sh/your-topic-alerts

#!/usr/bin/env bash
# /usr/local/bin/agent-autofix
set -euo pipefail

if ! systemctl --user is-active --quiet hermes-gateway.service; then
  n=$(systemctl --user show hermes-gateway.service -p NRestarts --value)
  if [ "$n" -ge 5 ]; then
    # exit storm — متحاولش تعمل loop لا نهائي
    hermes send --to telegram "🔴 gateway dead ($n restarts) — manual intervention needed"
    exit 1
  fi
  hermes send --to telegram "🟡 gateway down, restarting…"
  systemctl --user restart hermes-gateway.service
  sleep 15
  if systemctl --user is-active --quiet hermes-gateway.service; then
    hermes send --to telegram "🟢 gateway recovered"
  else
    hermes send --to telegram "🔴 restart failed — see journalctl -u hermes-gateway"
  fi
fi

hermes pause     # مفيش cron جديد + مفيش kanban dispatch + مفيش gateway turns
hermes resume    # يرجّع

# Hermes — أداة backup مدمجة
hermes backup                                  # zip كامل → ~/hermes-backup-<ts>.zip
hermes backup -o /var/backups/agent.zip
hermes backup --quick                          # config, state.db, .env, auth, cron
hermes backup --quick --label pre-upgrade
hermes backup --keep 5                         # احتفظ بآخر 5 (default 3، 0 = الكل)
hermes import                                  # استرجع من zip

# OpenClaw — نسخ ما قبل الترحيل بتتعمل تلقائي
openclaw gateway status --deep
openclaw doctor
openclaw secrets reload

# النسخة اللي على نفس الجهاز مش نسخة — استخدم الجهاز التاني كـ replica
rsync -avz --delete \
  /var/backups/agent/ \
  ubuntu@hermes-2:/var/backups/agent-replica/

# تشفير قبل ما النسخة تغادر الجهاز
tar czf - /var/backups/agent | gpg --encrypt --recipient ops@example.com > /tmp/agent-enc.tar.gz.gpg

# 3-2-1: 3 نسخ، 2 وسائط مختلفة، 1 نسخة برّه الموقع
find "$DEST" -type f -mtime +30 -delete

# Hermes
hermes update              # آخر كود من main + dependencies + config migration
hermes update --check      # preview بدون install
hermes update --backup     # full zip قبل الـ pull
hermes config check        # أي options جديدة ناقصة
hermes config migrate      # أضفها تفاعلياً
hermes update --status     # read-only

# OpenClaw
openclaw update                    # يكتشف النوع + يجيب آخر نسخة + doctor + restart
openclaw update --dry-run
openclaw update --channel extended-stable   # exact pins، fails closed

# بعد الترقية على السيرفر
sudo loginctl enable-linger $USER
hermes gateway restart --all     # كل الـ profiles
hermes gateway list
hermes doctor                    # لازم exit 0

# Rollback
git -C ~/.hermes/hermes-agent log --oneline -5
git -C ~/.hermes/hermes-agent reset --hard <good-sha>
hermes import /var/backups/agent/hermes-full-<ts>.zip

# العزل عبر الأجهزة — Tailscale
curl -fsSL https://tailscale.com/install.sh | sh
sudo tailscale up --ssh --hostname=hermes-1
sudo tailscale up --ssh --hostname=hermes-2

# OpenClaw على Tailscale Serve
openclaw config set gateway.bind loopback
openclaw config set gateway.auth.mode token
openclaw doctor --generate-gateway-token
openclaw config set gateway.tailscale.mode serve
openclaw config set gateway.trustedProxies '["127.0.0.1"]'

# Rescue bot بـ profile ومنفذ منفصلين
openclaw --profile rescue gateway install --port 19789
hermes profile create rescue