removed taia, added adamuon, rmsspectral (sania), adam sania

requirements.txt

@@ -7,7 +7,7 @@ peft==0.16.0
 datasets==2.19.0
 
 # Data science
-numpy==1.24.4
+numpy==1.26.4
 pandas==2.2.3
 matplotlib==3.9.3
 seaborn==0.13.2

scripts/glue/madam/deberta_full.sh → scripts/glue/precond/deberta_full.sh

@@ -20,6 +20,5 @@ do
         --eval_strategy epoch \
         --save_strategy no \
         --ft_strategy Full \
-        --dtype bfloat16 \
         --wandb
 done

scripts/glue/precond/deberta_full_adam_sania.sh

@@ -0,0 +1,24 @@
+clear
+
+# datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+datasets=(cola rte)
+lrs=(3e-4 1e-3 2e-5)
+
+for dataset in "${datasets[@]}"; do
+  for lr in "${lrs[@]}"; do
+    CUDA_VISIBLE_DEVICES=0 python ./src/run_experiment.py \
+      --dataset $dataset \
+      --model distilbert/distilbert-base-uncased \
+      --optimizer adam_sania \
+      --batch_size 16 \
+      --gradient_accumulation_steps 2 \
+      --lr $lr \
+      --lr_scheduler_type linear \
+      --warmup_ratio 0.1 \
+      --max_train_steps 10000 \
+      --eval_strategy epoch \
+      --save_strategy no \
+      --ft_strategy Full \
+      --wandb
+  done
+  done

scripts/glue/precond/deberta_full_adamuon_ns.sh

@@ -0,0 +1,25 @@
+clear
+
+datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+lrs=(3e-4 1e-3 2e-5)
+
+for dataset in "${datasets[@]}"; do
+    for lr in "${lrs[@]}"; do
+        CUDA_VISIBLE_DEVICES=0 python ./src/run_experiment.py \
+            --dataset $dataset \
+            --model distilbert/distilbert-base-uncased \
+            --optimizer adamuon \
+            --ns_steps 6 \
+            --batch_size 32 \
+            --gradient_accumulation_steps 2 \
+            --lr $lr \
+            --weight_decay 0.1 \
+            --lr_scheduler_type linear \
+            --warmup_ratio 0.1 \
+            --max_train_steps 10000 \
+            --eval_strategy epoch \
+            --save_strategy no \
+            --ft_strategy Full \
+            --wandb
+    done
+done

scripts/glue/precond/deberta_full_adamuon_pe.sh

@@ -0,0 +1,25 @@
+clear
+
+datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+lrs=(3e-4 1e-3 2e-5)
+
+for dataset in "${datasets[@]}"; do
+    for lr in "${lrs[@]}"; do
+        CUDA_VISIBLE_DEVICES=1 python ./src/run_experiment.py \
+            --dataset $dataset \
+            --model distilbert/distilbert-base-uncased \
+            --optimizer adamuon \
+            --ns_steps 6 \
+            --batch_size 32 \
+            --gradient_accumulation_steps 2 \
+            --lr $lr \
+            --weight_decay 0.1 \
+            --lr_scheduler_type linear \
+            --warmup_ratio 0.1 \
+            --max_train_steps 10000 \
+            --eval_strategy epoch \
+            --save_strategy no \
+            --ft_strategy Full \
+            --wandb
+    done
+done

scripts/glue/madam/deberta_full_adamw.sh → scripts/glue/precond/deberta_full_adamw.sh

@@ -19,7 +19,6 @@ for dataset in "${datasets[@]}"; do
       --eval_strategy epoch \
       --save_strategy no \
       --ft_strategy Full \
-      --dtype bfloat16 \
       --wandb
   done
   done

scripts/glue/madam/deberta_full_madam.sh → scripts/glue/precond/deberta_full_madam.sh

@@ -22,7 +22,6 @@ for dataset in "${datasets[@]}"; do
             --eval_strategy epoch \
             --save_strategy no \
             --ft_strategy Full \
-            --dtype bfloat16 \
             --wandb
     done
 done

scripts/glue/precond/deberta_full_madam_pe.sh

@@ -0,0 +1,26 @@
+clear
+
+datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+lrs=(3e-4 1e-3 2e-5)
+
+for dataset in "${datasets[@]}"; do
+    for lr in "${lrs[@]}"; do
+        CUDA_VISIBLE_DEVICES=1 python ./src/run_experiment.py \
+            --dataset $dataset \
+            --model distilbert/distilbert-base-uncased \
+            --optimizer rmsspectral \
+            --rms_power 0.25 \
+            --ns_steps 6 \
+            --batch_size 32 \
+            --gradient_accumulation_steps 2 \
+            --lr $lr \
+            --weight_decay 0.1 \
+            --lr_scheduler_type linear \
+            --warmup_ratio 0.1 \
+            --max_train_steps 10000 \
+            --eval_strategy epoch \
+            --save_strategy no \
+            --ft_strategy Full \
+            --wandb
+    done
+done

scripts/glue/madam/deberta_full_muon.sh → scripts/glue/precond/deberta_full_muon.sh

@@ -19,7 +19,6 @@ for dataset in "${datasets[@]}"; do
       --eval_strategy epoch \
       --save_strategy no \
       --ft_strategy Full \
-      --dtype bfloat16 \
       --wandb
   done
   done

scripts/glue/precond/deberta_full_muon_pe.sh

@@ -0,0 +1,25 @@
+clear
+
+datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+# datasets=(cola rte)
+lrs=(3e-4 1e-3 2e-5)
+
+for dataset in "${datasets[@]}"; do
+  for lr in "${lrs[@]}"; do
+    CUDA_VISIBLE_DEVICES=2 python ./src/run_experiment.py \
+      --dataset $dataset \
+      --model distilbert/distilbert-base-uncased \
+      --optimizer muon \
+      --batch_size 32 \
+      --gradient_accumulation_steps 2 \
+      --lr $lr \
+      --lr_scheduler_type linear \
+      --warmup_ratio 0.1 \
+      --max_train_steps 10000 \
+      --eval_strategy epoch \
+      --save_strategy no \
+      --ft_strategy Full \
+      --orth_algo ns \
+      --wandb
+  done
+  done

scripts/glue/precond/deberta_full_rmsspectral_ns.sh

@@ -0,0 +1,26 @@
+clear
+
+datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+lrs=(3e-4 1e-3 2e-5)
+
+for dataset in "${datasets[@]}"; do
+  for lr in "${lrs[@]}"; do
+    CUDA_VISIBLE_DEVICES=0 python ./src/run_experiment.py \
+      --dataset $dataset \
+      --model distilbert/distilbert-base-uncased \
+      --optimizer rmsspectral \
+      --rms_power 0.25 \
+      --ns_steps 6 \
+      --batch_size 32 \
+      --gradient_accumulation_steps 2 \
+      --lr $lr \
+      --weight_decay 0.1 \
+      --lr_scheduler_type linear \
+      --warmup_ratio 0.1 \
+      --max_train_steps 10000 \
+      --eval_strategy epoch \
+      --save_strategy no \
+      --ft_strategy Full \
+      --wandb
+  done
+done

scripts/glue/precond/deberta_full_rmsspectral_sania_ns.sh

@@ -0,0 +1,26 @@
+clear
+
+datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+lrs=(3e-4 1e-3 2e-5)
+
+for dataset in "${datasets[@]}"; do
+  for lr in "${lrs[@]}"; do
+    CUDA_VISIBLE_DEVICES=0 python ./src/run_experiment.py \
+      --dataset $dataset \
+      --model distilbert/distilbert-base-uncased \
+      --optimizer rmsspectral_sania \
+      --rms_power 0.5 \
+      --ns_steps 6 \
+      --batch_size 32 \
+      --gradient_accumulation_steps 2 \
+      --lr $lr \
+      --weight_decay 0.1 \
+      --lr_scheduler_type linear \
+      --warmup_ratio 0.1 \
+      --max_train_steps 10000 \
+      --eval_strategy epoch \
+      --save_strategy no \
+      --ft_strategy Full \
+      --wandb
+  done
+done

scripts/glue/precond/deberta_full_rmsspectral_sania_pe.sh

@@ -0,0 +1,26 @@
+clear
+
+datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+lrs=(3e-4 1e-3 2e-5)
+
+for dataset in "${datasets[@]}"; do
+    for lr in "${lrs[@]}"; do
+        CUDA_VISIBLE_DEVICES=1 python ./src/run_experiment.py \
+            --dataset $dataset \
+            --model distilbert/distilbert-base-uncased \
+            --optimizer rmsspectral_sania \
+            --rms_power 0.5 \
+            --ns_steps 6 \
+            --batch_size 32 \
+            --gradient_accumulation_steps 2 \
+            --lr $lr \
+            --weight_decay 0.1 \
+            --lr_scheduler_type linear \
+            --warmup_ratio 0.1 \
+            --max_train_steps 10000 \
+            --eval_strategy epoch \
+            --save_strategy no \
+            --ft_strategy Full \
+            --wandb
+    done
+done

scripts/glue/precond/deberta_lora_adam_sania.sh

@@ -0,0 +1,27 @@
+clear
+
+# datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+datasets=(cola rte)
+lrs=(3e-4 1e-3 2e-5)
+
+for dataset in "${datasets[@]}"; do
+  for lr in "${lrs[@]}"; do
+    CUDA_VISIBLE_DEVICES=0 python ./src/run_experiment.py \
+      --dataset $dataset \
+      --model distilbert/distilbert-base-uncased \
+      --optimizer adam_sania \
+      --batch_size 16 \
+      --gradient_accumulation_steps 2 \
+      --lr $lr \
+      --lr_scheduler_type linear \
+      --warmup_ratio 0.1 \
+      --max_train_steps 10000 \
+      --eval_strategy epoch \
+      --save_strategy no \
+      --ft_strategy LoRA \
+      --lora_r 4 \
+      --lora_alpha 32 \
+      --lora_dropout 0.05 \
+      --wandb
+  done
+  done

scripts/glue/precond/deberta_lora_adamuon_ns.sh

@@ -0,0 +1,28 @@
+clear
+
+datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+lrs=(3e-4 1e-3 2e-5 1e-4)
+
+for dataset in "${datasets[@]}"; do
+  for lr in "${lrs[@]}"; do
+    CUDA_VISIBLE_DEVICES=0 python ./src/run_experiment.py \
+      --dataset $dataset \
+      --model distilbert/distilbert-base-uncased \
+      --optimizer adamuon \
+      --ns_steps 6 \
+      --batch_size 16 \
+      --gradient_accumulation_steps 2 \
+      --lr $lr \
+      --weight_decay 0.1 \
+      --lr_scheduler_type linear \
+      --warmup_ratio 0.1 \
+      --max_train_steps 10000 \
+      --eval_strategy epoch \
+      --save_strategy no \
+      --ft_strategy LoRA \
+      --lora_r 4 \
+      --lora_alpha 32 \
+      --lora_dropout 0.1 \
+      --wandb
+  done
+done

scripts/glue/madam/deberta_lora_adamw.sh → scripts/glue/precond/deberta_lora_adamw.sh

@@ -19,7 +19,6 @@ for dataset in "${datasets[@]}"; do
       --eval_strategy epoch \
       --save_strategy no \
       --ft_strategy LoRA \
-      --dtype bfloat16 \
       --lora_r 4 \
       --lora_alpha 32 \
       --lora_dropout 0.05 \

scripts/glue/madam/deberta_lora_madam.sh → scripts/glue/precond/deberta_lora_madam.sh

@@ -22,7 +22,6 @@ for dataset in "${datasets[@]}"; do
             --eval_strategy epoch \
             --save_strategy no \
             --ft_strategy LoRA \
-            --dtype bfloat16 \
             --lora_r 4 \
             --lora_alpha 32 \
             --lora_dropout 0.05 \

scripts/glue/precond/deberta_lora_madam_pe.sh

@@ -0,0 +1,31 @@
+clear
+
+datasets=(cola mnli mrpc qnli qqp rte sst2 stsb)
+# datasets=(cola rte)
+lrs=(3e-4 1e-3 2e-5 1e-4)
+
+for dataset in "${datasets[@]}"; do
+    for lr in "${lrs[@]}"; do
+        CUDA_VISIBLE_DEVICES=0 python ./src/run_experiment.py \
+            --dataset $dataset \
+            --model distilbert/distilbert-base-uncased \
+            --optimizer taia \
+            --lmo spectral \
+            --precondition_type adam \
+            --init eps \
+            --batch_size 32 \
+            --gradient_accumulation_steps 2 \
+            --lr $lr \
+            --lr_scheduler_type linear \
+            --warmup_ratio 0.1 \
+            --max_train_steps 10000 \
+            --eval_strategy epoch \
+            --save_strategy no \
+            --ft_strategy LoRA \
+            --lora_r 4 \
+            --lora_alpha 32 \
+            --lora_dropout 0.1 \
+            --orth_algo polar \
+            --wandb
+    done
+done

scripts/glue/madam/deberta_lora_muon.sh → scripts/glue/precond/deberta_lora_muon.sh

@@ -19,7 +19,6 @@ for dataset in "${datasets[@]}"; do
       --eval_strategy epoch \
       --save_strategy no \
       --ft_strategy LoRA \
-      --dtype bfloat16 \
       --lora_r 4 \
       --lora_alpha 32 \
       --lora_dropout 0.05 \