Mulit node train demo for bert

pytorch/nlp/bert/README.md

@@ -1,42 +1,56 @@
-0. Start docker
+0. Start docker  
 启动命令可参考: [README.md](../../README.md)
 
 1. Prepare model
-```
-apt-get update
-apt-get install git-lfs
+```shell
+cd tutorial_on_musa/pytorch/nlp/bert
+mkdir -p data
+cd data
+apt-get update && apt-get install -y git-lfs
 git lfs install
 git clone https://huggingface.co/google-bert/bert-base-chinese
-
 ```
 
 2. Prepare dataset
-```
-wget http://s3.bmio.net/kashgari/china-people-daily-ner-corpus.tar.gz && tar -zxvf china-people-daily-ner-corpus.tar.gz 
-
+```shell
+cd tutorial_on_musa/pytorch/nlp/bert/data
+wget http://s3.bmio.net/kashgari/china-people-daily-ner-corpus.tar.gz
+tar -zxvf china-people-daily-ner-corpus.tar.gz
+# 整理为：./china-people-daily-ner-corpus/{example.train,example.dev,example.test}
 ```
 
-3. Prepare scripts
-```
+3. Prepare bert4torch
+```shell
+cd tutorial_on_musa/pytorch/nlp/bert
 git clone https://github.com/Tongjilibo/bert4torch.git
 cd bert4torch
 git reset --hard ebd53d61c28295
+# 每台训练节点/容器都要执行
 python setup.py install
-
-cp -r task_sequence_labeling_ner_crf.py bert4torch/examples/sequence_labeling
+pip install torch4keras==0.2.7
+cd ..
+# 仅单机训练需要；多机用本目录 task_sequence_labeling_ner_crf_2node.py，无需此 cp
+cp task_sequence_labeling_ner_crf.py bert4torch/examples/sequence_labeling/
 ```
 
 4. Train
 ```shell
-# single GPU
-bash run_train.sh
-# Multi-GPU DDP
-bash run_dist_train.sh
+# 单机：在 bert4torch 下跑（需要 ./data → 本目录 data）
+cd tutorial_on_musa/pytorch/nlp/bert/bert4torch
+ln -sfn ../data data
+bash ../run_train.sh          # 单卡
+bash ../run_dist_train.sh     # 单机 8 卡
+
+# 多机 2×8：本目录须在两机共享盘（含 data/ 与脚本）；两机免密 SSH（默认端口 62216，可用 SSH_PORT 覆盖）
+cd tutorial_on_musa/pytorch/nlp/bert
+bash run_dist_train_2node.sh <NODE0_IP> <NODE1_IP> 50
 ```
 
 5. Inference
 ```shell
-# ⚠️  Make sure the model_path in test_bert.py is correctly set before running
-cp -r test_bert.py bert4torch/test/models/
-python bert4torch/test/models/test_bert.py
+cd tutorial_on_musa/pytorch/nlp/bert
+cp test_bert.py bert4torch/test/models/
+cd bert4torch
+# 先确认 test_bert.py 里 model_path 指向正确权重
+python test/models/test_bert.py
 ```

pytorch/nlp/bert/run_dist_train_2node.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+# Multi-node bert+CRF NER DDP (2 nodes x 8 GPUs = 16 ranks)
+# Uses REAL data/model from README: bert-base-chinese + china-people-daily-ner-corpus
+#
+# Usage (from a node that can SSH to both peers on port 62216):
+#   bash run_dist_train_2node.sh <NODE0_IP> <NODE1_IP> [MAX_STEPS]
+#
+# Prerequisites (shared PVC, see README "多机（2×8）"):
+#   WORK_DIR/data/bert-base-chinese/
+#   WORK_DIR/data/china-people-daily-ner-corpus/
+#   bert4torch installed; task_sequence_labeling_ner_crf_2node.py in WORK_DIR
+#
+# Env overrides:
+#   SSH_PORT=62216 WORK_DIR=... MASTER_PORT=29501 NPROC_PER_NODE=8
+
+set -euo pipefail
+
+NODE0="${1:?usage: $0 <node0_ip> <node1_ip> [max_steps]}"
+NODE1="${2:?usage: $0 <node0_ip> <node1_ip> [max_steps]}"
+MAX_STEPS="${3:-50}"
+
+SSH_PORT="${SSH_PORT:-62216}"
+MASTER_PORT="${MASTER_PORT:-29511}"
+NNODES=2
+NPROC_PER_NODE="${NPROC_PER_NODE:-8}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WORK_DIR="${WORK_DIR:-${SCRIPT_DIR}}"
+TRAIN_PY="${WORK_DIR}/task_sequence_labeling_ner_crf_2node.py"
+LOG_DIR="${WORK_DIR}/log_2node_$(date +%Y%m%d_%H%M%S)"
+LOCAL_CACHE="${LOCAL_CACHE:-/tmp/bert_ddp_2node_cache}"
+BATCH_SIZE="${BATCH_SIZE:-16}"
+
+ssh_cmd() {
+  local ip="$1"; shift
+  ssh -p "${SSH_PORT}" -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=15 "$ip" "$@"
+}
+
+echo "[info] NODE0(master)=${NODE0} NODE1=${NODE1}"
+echo "[info] WORK_DIR=${WORK_DIR} MAX_STEPS=${MAX_STEPS} LOG_DIR=${LOG_DIR}"
+echo "[info] MASTER_PORT=${MASTER_PORT} BATCH_SIZE=${BATCH_SIZE} LOCAL_CACHE=${LOCAL_CACHE}"
+
+# Validate shared assets (PVC)
+for req in \
+  "${TRAIN_PY}" \
+  "${WORK_DIR}/data/bert-base-chinese/pytorch_model.bin" \
+  "${WORK_DIR}/data/bert-base-chinese/config.json" \
+  "${WORK_DIR}/data/bert-base-chinese/vocab.txt" \
+  "${WORK_DIR}/data/china-people-daily-ner-corpus/example.train"
+do
+  if [ ! -e "${req}" ]; then
+    echo "[fail] missing: ${req}"
+    echo "       Follow README multi-node prep (model + dataset + this script)."
+    exit 1
+  fi
+done
+
+for ip in "${NODE0}" "${NODE1}"; do
+  ssh_cmd "$ip" "mkdir -p '${LOG_DIR}' '${LOCAL_CACHE}' && test -f '${TRAIN_PY}'"
+done
+
+# Prefetch bert weights to per-node local disk (one copy per node) before torchrun
+for ip in "${NODE0}" "${NODE1}"; do
+  echo "[info] prefetch bert weights on ${ip} -> ${LOCAL_CACHE}"
+  ssh_cmd "$ip" "bash -lc '
+    set -euo pipefail
+    mkdir -p \"${LOCAL_CACHE}\"
+    if [ ! -f \"${LOCAL_CACHE}/bert-base-chinese/pytorch_model.bin\" ]; then
+      rm -rf \"${LOCAL_CACHE}/bert-base-chinese\"
+      cp -a \"${WORK_DIR}/data/bert-base-chinese\" \"${LOCAL_CACHE}/bert-base-chinese\"
+    fi
+    ls -lh \"${LOCAL_CACHE}/bert-base-chinese/pytorch_model.bin\"
+  '"
+done
+
+# best-effort cleanup (bracket pattern avoids killing this shell)
+for ip in "${NODE0}" "${NODE1}"; do
+  ssh_cmd "$ip" "pkill -f '[t]ask_sequence_labeling_ner_crf_2node' || true; pkill -f 'torchrun.*[t]ask_sequence_labeling_ner_crf_2node' || true" || true
+done
+sleep 2
+
+launch_one() {
+  local ip="$1"
+  local node_rank="$2"
+  local logfile="${LOG_DIR}/rank${node_rank}.${ip}.log"
+  ssh_cmd "$ip" "bash -lc '
+    set -euo pipefail
+    cd \"${WORK_DIR}\"
+    export MASTER_ADDR=${NODE0}
+    export MASTER_PORT=${MASTER_PORT}
+    export MUSA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+    export PYTHONUNBUFFERED=1
+    export PYTHONPATH=\"${WORK_DIR}:${WORK_DIR}/bert4torch:${WORK_DIR}/pydeps:\${PYTHONPATH:-}\"
+    nohup torchrun \\
+      --nnodes=${NNODES} \\
+      --nproc_per_node=${NPROC_PER_NODE} \\
+      --node_rank=${node_rank} \\
+      --master_addr=${NODE0} \\
+      --master_port=${MASTER_PORT} \\
+      \"${TRAIN_PY}\" --max-steps ${MAX_STEPS} --log-interval 5 --batch-size ${BATCH_SIZE} \\
+      --data-dir ./data --local-cache ${LOCAL_CACHE} \\
+      > \"${logfile}\" 2>&1 &
+    echo STARTED_PID=\$! LOG=${logfile}
+    sleep 1
+    head -5 \"${logfile}\" || true
+  '"
+}
+
+launch_one "${NODE0}" 0
+launch_one "${NODE1}" 1
+
+echo "[info] waiting for completion (poll rank0 log)..."
+LOG0="${LOG_DIR}/rank0.${NODE0}.log"
+# bert+CRF load can take minutes; allow up to ~20 min
+for i in $(seq 1 240); do
+  if ssh_cmd "${NODE0}" "grep -q '\\[bert-crf-2node\\] DONE' '${LOG0}' 2>/dev/null"; then
+    echo "[ok] finished"
+    ssh_cmd "${NODE0}" "tail -40 '${LOG0}'"
+    echo "LOG_DIR=${LOG_DIR}"
+    exit 0
+  fi
+  if ssh_cmd "${NODE0}" "grep -Eq 'ChildFailedError|Traceback \\(most recent call last\\)|FileNotFoundError' '${LOG0}' 2>/dev/null"; then
+    echo "[fail] error detected in rank0 log"
+    ssh_cmd "${NODE0}" "tail -100 '${LOG0}'" || true
+    exit 1
+  fi
+  sleep 5
+done
+
+echo "[fail] timeout waiting for DONE"
+ssh_cmd "${NODE0}" "tail -100 '${LOG0}'" || true
+exit 1