#!/usr/bin/env python3 import argparse import subprocess import os import sys def main(): parser = argparse.ArgumentParser(description="Qwen Trainer CLI - Unified interface for data gen and fine-tuning.") subparsers = parser.add_subparsers(dest="command", help="Command to run") # Data Gen Subcommand data_parser = subparsers.add_parser("data", help="Generate synthetic agentic data") data_parser.add_argument("--task", type=str, required=True, help="Task description") data_parser.add_argument("--num", type=int, default=10, help="Number of records") data_parser.add_argument("--output", type=str, default="synthetic_data.jsonl", help="Output path") data_parser.add_argument("--reasoning", action="store_true", help="Generate reasoning format") data_parser.add_argument("--dpo", action="store_true", help="Generate DPO pairs") data_parser.add_argument("--max-tokens", type=int, default=4096, help="Max tokens for generation") # Train Subcommand train_parser = subparsers.add_parser("train", help="Run fine-tuning") train_parser.add_argument("--model", type=str, default="Qwen/Qwen3.5-2B", help="Base model") train_parser.add_argument("--dataset", type=str, help="Dataset path/name") train_parser.add_argument("--method", choices=["sft", "dpo", "grpo"], default="sft", help="Method") train_parser.add_argument("--task", type=str, help="Auto-generate data for this task") train_parser.add_argument("--num_synthetic", type=int, default=50, help="Number of synthetic records if --task is set") train_parser.add_argument("--push", action="store_true", help="Push to Hub") train_parser.add_argument("--hub_id", type=str, help="HF Hub ID") # Submit Subcommand submit_parser = subparsers.add_parser("submit", help="Submit a job to HF or Kaggle") submit_parser.add_argument("--platform", choices=["hf", "kaggle"], required=True) submit_parser.add_argument("--flavor", type=str, default="a10g-small", help="HF Job flavor") submit_parser.add_argument("--image", type=str, default="pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel") submit_parser.add_argument("--cmd", type=str, help="Full command to run in the job") # Benchmark Subcommand benchmark_parser = subparsers.add_parser("benchmark", help="Benchmark a model on a dataset") benchmark_parser.add_argument("--model", type=str, default="Qwen/Qwen3.5-7B", help="Model ID") benchmark_parser.add_argument("--dataset", type=str, default="reasoning_assistant_v2_10.jsonl", help="Dataset path") benchmark_parser.add_argument("--num", type=int, default=10, help="Number of samples") args = parser.parse_args() if args.command == "data": cmd = [ f"{os.path.expanduser('~/datadesigner-env-py312/bin/python3')}", "skills/qwen-trainer/scripts/agentic_data_gen.py", "--task", args.task, "--num", str(args.num), "--output", args.output, "--max-tokens", str(args.max_tokens) ] if args.reasoning: cmd.append("--reasoning") if args.dpo: cmd.append("--dpo") print(f"Running Data Generation: {' '.join(cmd)}") subprocess.run(cmd, check=True) elif args.command == "train": cmd = [ "python3", "skills/qwen-trainer/scripts/train.py", "--model", args.model, "--method", args.method ] if args.dataset: cmd.extend(["--dataset", args.dataset]) if args.task: cmd.extend(["--use_agentic", "--task", args.task, "--num_synthetic", str(args.num_synthetic)]) if args.push and args.hub_id: cmd.extend(["--push", "--hub_id", args.hub_id]) print(f"Running Training: {' '.join(cmd)}") subprocess.run(cmd, check=True) elif args.command == "submit": cmd = [ "python3", "skills/qwen-trainer/scripts/submit.py", "--platform", args.platform, "--flavor", args.flavor, "--image", args.image ] if args.cmd: cmd.extend(["--command", args.cmd]) print(f"Submitting Job: {' '.join(cmd)}") subprocess.run(cmd, check=True) elif args.command == "benchmark": cmd = [ "python3", "skills/qwen-trainer/scripts/benchmark.py", "--model", args.model, "--dataset", args.dataset, "--num", str(args.num) ] print(f"Running Benchmark: {' '.join(cmd)}") subprocess.run(cmd, check=True) else: parser.print_help() if __name__ == "__main__": main()