git/deap-based-automl-experiment-framework
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated code for other, non-NSGA experiments

Browse Source
This commit is contained in:
Varyngoth
2025-12-30 16:41:12 -04:00
parent 3eef3c7ed5
commit 7737997e7a
2 changed files with 40 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
run_deap.py
View File

@@ -15,6 +15,7 @@ from src.preprocessing import build_preprocessor
from src.models import make_model from src.models import make_model
from src.stability import compute_shap_matrix from src.stability import compute_shap_matrix
def save_checkpoint(path, gen, pop, seed): def save_checkpoint(path, gen, pop, seed):
state = { state = {
"gen": gen, "gen": gen,

47
runs.txt
View File

@@ -5,28 +5,29 @@ python3 -m src.protocols_methodology.exp_cv --dataset adult --algo rf --n-folds
Bootstrap baseline: Bootstrap baseline:
python3 -m src.protocols_methodology.exp_bootstrap --dataset adult --algo rf --n-bootstrap 30 --seeds 0 1 2 3 4 python3 -m src.protocols_methodology.exp_bootstrap --dataset adult --algo rf --n-bootstrap 30 --seeds 0 1 2 3 4
Noise baseline:
python3 -m src.protocols_methodology.exp_noise --dataset adult --algo rf --n-replicates 30 --noise-std 0.01 --seeds 0 1 2 3 4
Small run: compute 0 Small run: compute 0
tmux new -s small tmux new -s small
python3 -m src.protocols_methodology.run_nsga_protocols --dataset adult --generations 3 --pop-size 8 --seed 0 python3 -m src.protocols_methodology.run_nsga_protocols --dataset adult --generations 3 --pop-size 8 --seed 0 --experiment small_deap_nsga_protocol_study
tmux attach-session -t small tmux attach-session -t small
Medium run: compute 1 Medium run: compute 1
tmux new -s medium tmux new -s medium
python3 -m src.protocols_methodology.run_nsga_protocols --dataset adult --generations 10 --pop-size 24 --seed 0 python3 -m src.protocols_methodology.run_nsga_protocols --dataset adult --generations 10 --pop-size 24 --seed 0 --experiment medium_deap_nsga_protocol_study
tmux attach-session -t medium tmux attach-session -t medium
Larger run: compute 3 Larger run: compute 3
tmux new -s larger tmux new -s larger
python3 -m src.protocols_methodology.run_nsga_protocols --dataset adult --generations 20 --pop-size 40 --seed 0 python3 -m src.protocols_methodology.run_nsga_protocols --dataset adult --generations 20 --pop-size 40 --seed 0 --experiment larger_deap_nsga_protocol_study
tmux attach-session -t larger tmux attach-session -t larger
rsync -avz --progress /home/mlly/projects/deap-based-automl-experiment-framework/. root@192.168.2.69:/mnt/user/exp_results/small_deap_nsga_protocol_study
rsync -avz --progress /home/mlly/projects/deap-based-automl-experiment-framework/. root@192.168.2.69:/mnt/user/exp_results/medium_deap_nsga_protocol_study
rsync -avz --progress /home/mlly/projects/deap-based-automl-experiment-framework/. root@192.168.2.69:/mnt/user/exp_results/larger_deap_nsga_protocol_study
Send back: Send back:
The three baseline summary csv files: The three baseline summary csv files:
runs/protocol_cv/__cv_summary.csv runs/protocol_cv/__cv_summary.csv
@@ -43,3 +44,31 @@ runs/_protocol_study/cv/pareto_front.csv
runs/_protocol_study/bootstrap/pareto_front.csv runs/_protocol_study/bootstrap/pareto_front.csv
runs/_protocol_study/noise/pareto_front.csv runs/_protocol_study/noise/pareto_front.csv
Baseline Error:
(.venv) [root@compute2 deap-based-automl-experiment-framework]# python3 -m src.protocols_methodology.exp_cv --dataset adult --algo rf --n-folds 5 --seeds 0 1 2 3 4
99%|===================| 2018/2048 [00:20<00:00] Traceback (most recent call last):
File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/mlly/projects/deap-based-automl-experiment-framework/src/protocols_methodology/exp_cv.py", line 195, in <module>
main()
File "/home/mlly/projects/deap-based-automl-experiment-framework/src/protocols_methodology/exp_cv.py", line 153, in main
summary, rep_rows = run_cv_protocol(
File "/home/mlly/projects/deap-based-automl-experiment-framework/src/protocols_methodology/exp_cv.py", line 81, in run_cv_protocol
agg_std_rep, stability_rep, _, _ = shap_stability_from_matrices(shap_mats_with_names)
File "/home/mlly/projects/deap-based-automl-experiment-framework/src/stability.py", line 251, in shap_stability_from_matrices
raise ValueError(
ValueError: Need at least 2 models to estimate stability, got 1
Noise baseline:
python3 -m src.protocols_methodology.exp_noise --dataset adult --algo rf --n-replicates 30 --noise-std 0.01 --seeds 0 1 2 3 4