File size: 4,692 Bytes
78a0ca9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | #!/usr/bin/env python3
import argparse
import subprocess
import os
import sys
def main():
parser = argparse.ArgumentParser(description="Qwen Trainer CLI - Unified interface for data gen and fine-tuning.")
subparsers = parser.add_subparsers(dest="command", help="Command to run")
# Data Gen Subcommand
data_parser = subparsers.add_parser("data", help="Generate synthetic agentic data")
data_parser.add_argument("--task", type=str, required=True, help="Task description")
data_parser.add_argument("--num", type=int, default=10, help="Number of records")
data_parser.add_argument("--output", type=str, default="synthetic_data.jsonl", help="Output path")
data_parser.add_argument("--reasoning", action="store_true", help="Generate reasoning format")
data_parser.add_argument("--dpo", action="store_true", help="Generate DPO pairs")
data_parser.add_argument("--max-tokens", type=int, default=4096, help="Max tokens for generation")
# Train Subcommand
train_parser = subparsers.add_parser("train", help="Run fine-tuning")
train_parser.add_argument("--model", type=str, default="Qwen/Qwen3.5-2B", help="Base model")
train_parser.add_argument("--dataset", type=str, help="Dataset path/name")
train_parser.add_argument("--method", choices=["sft", "dpo", "grpo"], default="sft", help="Method")
train_parser.add_argument("--task", type=str, help="Auto-generate data for this task")
train_parser.add_argument("--num_synthetic", type=int, default=50, help="Number of synthetic records if --task is set")
train_parser.add_argument("--push", action="store_true", help="Push to Hub")
train_parser.add_argument("--hub_id", type=str, help="HF Hub ID")
# Submit Subcommand
submit_parser = subparsers.add_parser("submit", help="Submit a job to HF or Kaggle")
submit_parser.add_argument("--platform", choices=["hf", "kaggle"], required=True)
submit_parser.add_argument("--flavor", type=str, default="a10g-small", help="HF Job flavor")
submit_parser.add_argument("--image", type=str, default="pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel")
submit_parser.add_argument("--cmd", type=str, help="Full command to run in the job")
# Benchmark Subcommand
benchmark_parser = subparsers.add_parser("benchmark", help="Benchmark a model on a dataset")
benchmark_parser.add_argument("--model", type=str, default="Qwen/Qwen3.5-7B", help="Model ID")
benchmark_parser.add_argument("--dataset", type=str, default="reasoning_assistant_v2_10.jsonl", help="Dataset path")
benchmark_parser.add_argument("--num", type=int, default=10, help="Number of samples")
args = parser.parse_args()
if args.command == "data":
cmd = [
f"{os.path.expanduser('~/datadesigner-env-py312/bin/python3')}",
"skills/qwen-trainer/scripts/agentic_data_gen.py",
"--task", args.task,
"--num", str(args.num),
"--output", args.output,
"--max-tokens", str(args.max_tokens)
]
if args.reasoning: cmd.append("--reasoning")
if args.dpo: cmd.append("--dpo")
print(f"Running Data Generation: {' '.join(cmd)}")
subprocess.run(cmd, check=True)
elif args.command == "train":
cmd = [
"python3",
"skills/qwen-trainer/scripts/train.py",
"--model", args.model,
"--method", args.method
]
if args.dataset:
cmd.extend(["--dataset", args.dataset])
if args.task:
cmd.extend(["--use_agentic", "--task", args.task, "--num_synthetic", str(args.num_synthetic)])
if args.push and args.hub_id:
cmd.extend(["--push", "--hub_id", args.hub_id])
print(f"Running Training: {' '.join(cmd)}")
subprocess.run(cmd, check=True)
elif args.command == "submit":
cmd = [
"python3",
"skills/qwen-trainer/scripts/submit.py",
"--platform", args.platform,
"--flavor", args.flavor,
"--image", args.image
]
if args.cmd:
cmd.extend(["--command", args.cmd])
print(f"Submitting Job: {' '.join(cmd)}")
subprocess.run(cmd, check=True)
elif args.command == "benchmark":
cmd = [
"python3",
"skills/qwen-trainer/scripts/benchmark.py",
"--model", args.model,
"--dataset", args.dataset,
"--num", str(args.num)
]
print(f"Running Benchmark: {' '.join(cmd)}")
subprocess.run(cmd, check=True)
else:
parser.print_help()
if __name__ == "__main__":
main()
|