diff --git a/gui-rl/gui_qwen3vl_8b_rl.sh b/gui-rl/gui_qwen3vl_8b_rl.sh index e237090..591d963 100644 --- a/gui-rl/gui_qwen3vl_8b_rl.sh +++ b/gui-rl/gui_qwen3vl_8b_rl.sh @@ -1,6 +1,5 @@ #!/bin/bash -# for rerun the task (same style as retool scripts) pkill -9 sglang || true sleep 3 ray stop --force || true @@ -14,20 +13,19 @@ set -ex SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" SLIME_DIR="$(cd -- "${SCRIPT_DIR}/../slime" &>/dev/null && pwd)" -MODEL_ARGS_ROTARY_BASE=5000000 source "${SLIME_DIR}/scripts/models/qwen3-8B.sh" MEGATRON_LM_PATH=${MEGATRON_LM_PATH:-"${SCRIPT_DIR}/../Megatron-LM"} -# keep stdout/stderr unbuffered in ray jobs +HF_CKPT=${HF_CKPT:-/data_storage/wyj/systems/huggingface/hub/models--Qwen--Qwen3-VL-8B-Thinking/snapshots/41ea130ce6eaaf7829c72dfc0e4597d49741ed18} +MODEL_ARGS_ROTARY_BASE=5000000 source "${SLIME_DIR}/scripts/models/qwen3-8B.sh" + export PYTHONUNBUFFERED=1 export PYTHONFAULTHANDLER=1 -# reduce false ray node death under heavy initialization export RAY_health_check_failure_threshold=${RAY_health_check_failure_threshold:-20} export RAY_health_check_period_ms=${RAY_health_check_period_ms:-5000} export RAY_health_check_timeout_ms=${RAY_health_check_timeout_ms:-30000} export RAY_num_heartbeats_timeout=${RAY_num_heartbeats_timeout:-60} -# default to 8 GPUs if not set by scheduler NUM_GPUS=${NUM_GPUS:-8} ACTOR_GPUS=${ACTOR_GPUS:-4} ROLLOUT_GPUS=${ROLLOUT_GPUS:-4} @@ -38,14 +36,11 @@ if (( ACTOR_GPUS + ROLLOUT_GPUS > NUM_GPUS )); then exit 1 fi -# --------------------------- -# GUI env pool server configs -# --------------------------- -export GUI_ENV_SERVER_HOST=${GUI_ENV_SERVER_HOST:-"127.0.0.1"} +export GUI_ENV_SERVER_HOST=${GUI_ENV_SERVER_HOST:-127.0.0.1} export GUI_ENV_SERVER_PORT=${GUI_ENV_SERVER_PORT:-18080} export GUI_ENV_SERVER_URL=${GUI_ENV_SERVER_URL:-"http://${GUI_ENV_SERVER_HOST}:${GUI_ENV_SERVER_PORT}"} -export GUI_ENV_SERVER_MAX_ENVS=${GUI_ENV_SERVER_MAX_ENVS:-32} -export GUI_PREWARM_CONCURRENCY=${GUI_PREWARM_CONCURRENCY:-32} +export GUI_ENV_SERVER_MAX_ENVS=${GUI_ENV_SERVER_MAX_ENVS:-64} +export GUI_PREWARM_CONCURRENCY=${GUI_PREWARM_CONCURRENCY:-64} export GUI_POOL_MAX_ENVS=${GUI_POOL_MAX_ENVS:-${GUI_ENV_SERVER_MAX_ENVS}} export GUI_PREWARM_ENVS=${GUI_PREWARM_ENVS:-${GUI_POOL_MAX_ENVS}} export GUI_FORCE_PREWARM_ALL=${GUI_FORCE_PREWARM_ALL:-1} @@ -54,20 +49,19 @@ if [[ "${GUI_FORCE_PREWARM_ALL}" == "1" ]]; then fi export GUI_TRAJECTORY_CONCURRENCY=${GUI_TRAJECTORY_CONCURRENCY:-${GUI_POOL_MAX_ENVS}} export GUI_POOL_IDLE_TTL_SECONDS=${GUI_POOL_IDLE_TTL_SECONDS:-600} -export GUI_PROVIDER_NAME=${GUI_PROVIDER_NAME:-"volcengine"} -export GUI_REGION=${GUI_REGION:-"cn-beijing"} +export GUI_PROVIDER_NAME=${GUI_PROVIDER_NAME:-volcengine} +export GUI_REGION=${GUI_REGION:-cn-beijing} export GUI_PATH_TO_VM=${GUI_PATH_TO_VM:-""} -export GUI_ACTION_SPACE=${GUI_ACTION_SPACE:-"pyautogui"} -export GUI_OBSERVATION_TYPE=${GUI_OBSERVATION_TYPE:-"screenshot"} -export GUI_COORDINATE_TYPE=${GUI_COORDINATE_TYPE:-"relative"} +export GUI_ACTION_SPACE=${GUI_ACTION_SPACE:-pyautogui} +export GUI_OBSERVATION_TYPE=${GUI_OBSERVATION_TYPE:-screenshot} +export GUI_COORDINATE_TYPE=${GUI_COORDINATE_TYPE:-relative} export GUI_AGENT_CLASS_PATH=${GUI_AGENT_CLASS_PATH:-"agents.qwen3vl_agent.Qwen3VLAgentLocal"} -MULTIMODAL_KEYS=${MULTIMODAL_KEYS:-'{"image":"images"}'} export GUI_REUSE_VM_ON_RESET=${GUI_REUSE_VM_ON_RESET:-0} export GUI_RESET_ON_CLOSE=${GUI_RESET_ON_CLOSE:-1} -export GUI_CLIENT_PASSWORD=${GUI_CLIENT_PASSWORD:-"WWbbb8b7b6314"} +export GUI_CLIENT_PASSWORD=${GUI_CLIENT_PASSWORD:-WWbbb8b7b6314} export GUI_SCREEN_WIDTH=${GUI_SCREEN_WIDTH:-1920} export GUI_SCREEN_HEIGHT=${GUI_SCREEN_HEIGHT:-1080} -# Keep wandb project independent from OSWORLD/result project naming. + WANDB_PROJECT=${WANDB_PROJECT:-slime_gui} GUI_PROJECT_NAME=${GUI_PROJECT_NAME:-slime_gui-8b-rl} export OSWORLD_PROJECT="${GUI_PROJECT_NAME}" @@ -75,61 +69,58 @@ export GUI_RESULT_DIR=${GUI_RESULT_DIR:-"${SCRIPT_DIR}/results"} export GUI_RESULT_DIR="${GUI_RESULT_DIR}/${GUI_PROJECT_NAME}" export GUI_TEST_CONFIG_BASE_DIR=${GUI_TEST_CONFIG_BASE_DIR:-"${SCRIPT_DIR}/evaluation_examples"} export GUI_TRAIN_META_PATH=${GUI_TRAIN_META_PATH:-"${GUI_TEST_CONFIG_BASE_DIR}/train_nochrome.json"} -export GUI_EVAL_META_PATH=${GUI_EVAL_META_PATH:-"${GUI_TEST_CONFIG_BASE_DIR}/test_multinode.json"} +export GUI_EVAL_META_PATH=${GUI_EVAL_META_PATH:-"${GUI_TEST_CONFIG_BASE_DIR}/test_nochrome.json"} +MULTIMODAL_KEYS=${MULTIMODAL_KEYS:-'{"image":"images"}'} - - -# Start each run with a clean project result directory. if [[ -n "${GUI_RESULT_DIR}" && "${GUI_RESULT_DIR}" != "/" ]]; then rm -rf "${GUI_RESULT_DIR}" fi mkdir -p "${GUI_RESULT_DIR}" - - -# --------------------------- -# Volcengine non-secret configs -# Secrets (AK/SK) should be exported in shell before running this script. -# --------------------------- -export VOLCENGINE_REGION=${VOLCENGINE_REGION:-"cn-beijing"} -export VOLCENGINE_IMAGE_ID=${VOLCENGINE_IMAGE_ID:-"image-id"} -export VOLCENGINE_SUBNET_ID=${VOLCENGINE_SUBNET_ID:-"subnet-id"} -export VOLCENGINE_SECURITY_GROUP_ID=${VOLCENGINE_SECURITY_GROUP_ID:-"sg-id"} -export VOLCENGINE_ZONE_ID=${VOLCENGINE_ZONE_ID:-"cn-beijing-a"} -export VOLCENGINE_DEFAULT_PASSWORD=${VOLCENGINE_DEFAULT_PASSWORD:-"WWbbb180314"} -# Lower interval speeds up create/delete under moderate throttling. +export VOLCENGINE_REGION=${VOLCENGINE_REGION:-cn-beijing} +export VOLCENGINE_IMAGE_ID=${VOLCENGINE_IMAGE_ID:-image-xxx} +export VOLCENGINE_SUBNET_ID=${VOLCENGINE_SUBNET_ID:-subnet-xxx} +export VOLCENGINE_SECURITY_GROUP_ID=${VOLCENGINE_SECURITY_GROUP_ID:-sg-xxx} +export VOLCENGINE_ZONE_ID=${VOLCENGINE_ZONE_ID:-cn-beijing-a} +export VOLCENGINE_DEFAULT_PASSWORD=${VOLCENGINE_DEFAULT_PASSWORD:-WWbbb180314} export VOLCENGINE_RUNINST_MIN_INTERVAL=${VOLCENGINE_RUNINST_MIN_INTERVAL:-0.1} export VOLCENGINE_DELINST_MIN_INTERVAL=${VOLCENGINE_DELINST_MIN_INTERVAL:-0.1} - -# Instance type fallback list (comma-separated). -# desktop_env/providers/volcengine/manager.py already supports this format. export VOLCENGINE_INSTANCE_TYPE=${VOLCENGINE_INSTANCE_TYPE:-"ecs.e-c1m2.large,ecs.e-c1m4.large,ecs.e-c1m8.large,ecs.e-c1m1.large,ecs.c3al.large,ecs.c3a.large,ecs.c3il.large,ecs.g3il.large,ecs.r3il.large,ecs.c3a.large,ecs.g3a.large,ecs.r3a.large,ecs.c3i.large,ecs.g3i.large,ecs.r3i.large,ecs.g3al.large,ecs.r3al.large,ecs.r1ie.large,ecs.g1ie.large,ecs.c1ie.large,ecs.g3ine.large"} - -# Optional download proxy used by setup/evaluator code. -# Do not put a default value here; export it in terminal when needed (same as keys). export download_proxy=${download_proxy:-} -HF_CKPT=${HF_CKPT:-/absolute/path/to/models--Qwen--Qwen3-VL-8B-Thinking/snapshots/41ea130ce6eaaf7829c72dfc0e4597d49741ed18} -# For bridge+multimodal path, prefer HF checkpoint path for ref model loading. +if [[ -z "${VOLCENGINE_ACCESS_KEY_ID:-}" ]]; then + echo "VOLCENGINE_ACCESS_KEY_ID must be set before starting GUI RL." + exit 1 +fi +if [[ -z "${VOLCENGINE_SECRET_ACCESS_KEY:-}" ]]; then + echo "VOLCENGINE_SECRET_ACCESS_KEY must be set before starting GUI RL." + exit 1 +fi + +if [[ -z "${HF_CKPT}" ]]; then + echo "Set HF_CKPT to your Qwen3-VL-8B checkpoint path" + exit 1 +fi +if [[ ! -e "${HF_CKPT}" ]]; then + echo "HF_CKPT does not exist: ${HF_CKPT}" + exit 1 +fi REF_LOAD=${REF_LOAD:-${HF_CKPT}} CKPT_ARGS=( - --hf-checkpoint ${HF_CKPT} - --ref-load ${REF_LOAD} + --hf-checkpoint "${HF_CKPT}" + --ref-load "${REF_LOAD}" --save "${SAVE_CKPT:-${SCRIPT_DIR}/../ckpt/gui-qwen3vl-8b-rl}" --save-interval 20 ) ENABLE_RESUME_LOAD=${ENABLE_RESUME_LOAD:-0} -RESUME_LOAD=${RESUME_LOAD:-/absolute/path/to/OpenClaw-RL/ckpt/gui-qwen3vl-8b-rl} +RESUME_LOAD=${RESUME_LOAD:-"${SCRIPT_DIR}/../ckpt/gui-qwen3vl-8b-rl"} if [[ "${ENABLE_RESUME_LOAD}" == "1" ]]; then CKPT_ARGS+=(--load "${RESUME_LOAD}") - echo "Resume load enabled: ${RESUME_LOAD}" -else - echo "Resume load disabled (ENABLE_RESUME_LOAD=${ENABLE_RESUME_LOAD})" fi -ROLLOUT_BATCH_SIZE=${ROLLOUT_BATCH_SIZE:-4} +ROLLOUT_BATCH_SIZE=${ROLLOUT_BATCH_SIZE:-8} N_SAMPLES_PER_PROMPT=${N_SAMPLES_PER_PROMPT:-8} ROLLOUT_ARGS=( @@ -138,14 +129,14 @@ ROLLOUT_ARGS=( --num-rollout 1000 --rollout-batch-size ${ROLLOUT_BATCH_SIZE} --n-samples-per-prompt ${N_SAMPLES_PER_PROMPT} - --rollout-max-response-len 512 + --rollout-max-response-len 1024 --rollout-temperature 1.0 --gui-max-steps 30 --gui-wait-after-reset 60 - --gui-sleep-after-execution 0.0 + --gui-sleep-after-execution 0.5 --gui-max-image-history-length 3 - --gui-max-reward-image-history-length 2 - --num-steps-per-rollout 2 + --gui-max-reward-image-history-length 1 + --num-steps-per-rollout 1 ) ROLLOUT_NUM_GPUS_PER_ENGINE=${ROLLOUT_NUM_GPUS_PER_ENGINE:-1} @@ -157,9 +148,12 @@ echo "GUI_TRAJECTORY_CONCURRENCY=${GUI_TRAJECTORY_CONCURRENCY} (independent from EVAL_ARGS=( --eval-temperature 0.0 --gui-eval-max-steps 30 - --gui-eval-sleep-after-execution 0.0 + --gui-eval-sleep-after-execution 5.0 --gui-eval-wait-after-reset 60 --n-samples-per-eval-prompt 1 + --eval-interval 20 + --eval-reward-key acc + --eval-function-path generate_with_gui.gui_generate_rollout ) if [ -n "${GUI_EVAL_INTERVAL}" ]; then EVAL_ARGS+=(--eval-interval "${GUI_EVAL_INTERVAL}") @@ -181,18 +175,15 @@ PERF_ARGS=( --tensor-model-parallel-size 4 --sequence-parallel --pipeline-model-parallel-size 1 - # VLM train forward in current Megatron bridge expects unsliced vision inputs. - # CP>1 slices text tokens per rank but does not slice multimodal tensors here, - # which can trigger "assert video_embeds is None" with image-only data. --context-parallel-size 1 --expert-model-parallel-size 1 --expert-tensor-parallel-size 1 --recompute-granularity full --recompute-method uniform - --recompute-num-layers 36 + --recompute-num-layers 1 --megatron-to-hf-mode bridge --use-dynamic-batch-size - --max-tokens-per-gpu 512 + --max-tokens-per-gpu 1024 ) GRPO_ARGS=( @@ -203,12 +194,6 @@ GRPO_ARGS=( --kl-loss-coef 0.01 ) -#ROUTER_ARGS=( - # Use SlimeRouter's least-active-worker routing to avoid over-pinning - # GUI requests with similar prefixes to a single SGLang worker. - #--use-slime-router -#) - SGLANG_ARGS=( --rollout-num-gpus-per-engine ${ROLLOUT_NUM_GPUS_PER_ENGINE} --sglang-mem-fraction-static 0.85 @@ -231,9 +216,8 @@ else WANDB_ARGS=() fi -# Start GUI env pool server mkdir -p logs -ENV_SERVER_LOG=${ENV_SERVER_LOG:-"./logs/gui_env_pool_server.log"} +ENV_SERVER_LOG=${ENV_SERVER_LOG:-"./logs/gui_env_pool_server_qwen3vl_8b.log"} PYTHONPATH="${SLIME_DIR}:${SCRIPT_DIR}:${PYTHONPATH}" \ python3 -m env_pool_server \ --host "${GUI_ENV_SERVER_HOST}" \ @@ -252,7 +236,6 @@ PYTHONPATH="${SLIME_DIR}:${SCRIPT_DIR}:${PYTHONPATH}" \ --screen-height "${GUI_SCREEN_HEIGHT}" \ > "${ENV_SERVER_LOG}" 2>&1 & GUI_ENV_SERVER_PID=$! -echo "GUI env pool server PID=${GUI_ENV_SERVER_PID}, log=${ENV_SERVER_LOG}" cleanup() { set +e @@ -262,23 +245,19 @@ cleanup() { } trap cleanup EXIT INT TERM -# Wait until env server is healthy for i in {1..60}; do if curl -fsS "${GUI_ENV_SERVER_URL}/healthz" >/dev/null 2>&1; then - echo "GUI env pool server is ready: ${GUI_ENV_SERVER_URL}" break fi sleep 2 done -# Block until all prewarm envs are visible in the pool. if (( GUI_PREWARM_ENVS > 0 )); then for i in {1..600}; do if python3 - "${GUI_ENV_SERVER_URL}" "${GUI_PREWARM_ENVS}" <<'PY' import json import sys import urllib.request - status_url = sys.argv[1].rstrip("/") + "/status" target = int(sys.argv[2]) with urllib.request.urlopen(status_url, timeout=5) as resp: @@ -286,32 +265,24 @@ with urllib.request.urlopen(status_url, timeout=5) as resp: pool = data.get("pool", {}) total_envs = int(pool.get("total_envs", 0)) ok = bool(data.get("ok", False)) -print(f"pool total_envs={total_envs}, target={target}, ok={ok}") raise SystemExit(0 if ok and total_envs >= target else 1) PY then - echo "GUI prewarm complete: ${GUI_PREWARM_ENVS}/${GUI_POOL_MAX_ENVS}" break fi sleep 2 - if (( i == 600 )); then - echo "Timed out waiting prewarm completion: target=${GUI_PREWARM_ENVS}" - exit 1 - fi done fi NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) -if [ "$NVLINK_COUNT" -gt 0 ]; then +HAS_NVLINK=0 +if [[ "${NVLINK_COUNT}" -gt 0 ]]; then HAS_NVLINK=1 -else - HAS_NVLINK=0 fi -echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)" export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:2048 -ray start --head --node-ip-address 127.0.0.1 --num-gpus ${NUM_GPUS} --disable-usage-stats --dashboard-host=0.0.0.0 --dashboard-port=8265 +ray start --head --node-ip-address 127.0.0.1 --num-gpus "${NUM_GPUS}" --disable-usage-stats --dashboard-host=0.0.0.0 --dashboard-port=8265 RUNTIME_ENV_JSON="{ \"env_vars\": { @@ -333,17 +304,21 @@ RUNTIME_ENV_JSON="{ \"GUI_TRAIN_META_PATH\": \"${GUI_TRAIN_META_PATH}\", \"GUI_EVAL_META_PATH\": \"${GUI_EVAL_META_PATH}\", \"OSWORLD_PROJECT\": \"${OSWORLD_PROJECT}\", - \"download_proxy\": \"${download_proxy}\" + \"download_proxy\": \"${download_proxy}\", + \"MEGATRON_LM_PATH\": \"${MEGATRON_LM_PATH}\", + \"HF_CKPT\": \"${HF_CKPT}\", + \"GUI_AGENT_CLASS_PATH\": \"${GUI_AGENT_CLASS_PATH}\" } }" -RAY_JOB_SUBMISSION_ID=${RAY_JOB_SUBMISSION_ID:-"gui_qwen3vl_$(date +%Y%m%d_%H%M%S)"} +TRAIN_ENTRY=${TRAIN_ENTRY:-train_async.py} +RAY_JOB_SUBMISSION_ID=${RAY_JOB_SUBMISSION_ID:-"gui_qwen3vl_8b_rl_$(date +%Y%m%d_%H%M%S)"} ray job submit --address="http://127.0.0.1:8265" \ --submission-id "${RAY_JOB_SUBMISSION_ID}" \ --no-wait \ --runtime-env-json="${RUNTIME_ENV_JSON}" \ - -- python3 -u train.py \ + -- python3 -u "${TRAIN_ENTRY}" \ --actor-num-nodes 1 \ --actor-num-gpus-per-node ${ACTOR_GPUS} \ --rollout-num-gpus ${ROLLOUT_GPUS} \ @@ -374,4 +349,3 @@ fi echo "Ray job failed (submission id: ${RAY_JOB_SUBMISSION_ID}, logs exit: ${RAY_LOG_EXIT})" exit 1 -