CUDA_VISIBLE_DEVICES=0,1 torchrun --nproc_per_node 2 pretraining.py \

--model_type bloom \

--model_name_or_path bigscience/bloomz-560m \

--train_file_dir ./data/pretrain \

--validation_file_dir ./data/pretrain \

--per_device_train_batch_size 4 \

--per_device_eval_batch_size 4 \

--do_train \

--do_eval \

--use_peft True \

--seed 42 \

--max_train_samples 10000 \

--max_eval_samples 10 \

--num_train_epochs 0.5 \

--learning_rate 2e-4 \

--warmup_ratio 0.05 \

--weight_decay 0.01 \

--logging_strategy steps \

--logging_steps 10 \

--eval_steps 50 \

--evaluation_strategy steps \

--save_steps 500 \

--save_strategy steps \

--save_total_limit 13 \

--gradient_accumulation_steps 1 \

--preprocessing_num_workers 10 \

--block_size 512 \

--output_dir outputs-pt-bloom-v1 \

--overwrite_output_dir \

--ddp_timeout 30000 \

--logging_first_step True \

--target_modules all \

--lora_rank 8 \

--lora_alpha 16 \

--lora_dropout 0.05 \

--torch_dtype bfloat16 \

--bf16 \

--device_map auto \

--report_to tensorboard \

--ddp_find_unused_parameters False \

--gradient_checkpointing True \

--cache_dir ./cache