-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathtrain.py
More file actions
60 lines (47 loc) · 2.57 KB
/
Copy pathtrain.py
File metadata and controls
60 lines (47 loc) · 2.57 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import pytorch_lightning as pl
from hydra.utils import instantiate
from omegaconf import DictConfig, OmegaConf
import hydra
import wandb
from pytorch_lightning.loggers import WandbLogger
import torch
from conglude.utils.lightning import init_lightning_callbacks
@hydra.main(config_path="configs", config_name="train", version_base="1.2")
def run(cfg: DictConfig):
torch.set_float32_matmul_precision(cfg.precision)
pl.seed_everything(cfg.seed, workers=True)
# Resolve config to a plain dict for logging (e.g. WandB)
config = OmegaConf.to_container(cfg, resolve=True, throw_on_missing=True)
datamodule = instantiate(cfg.datamodule)
model = instantiate(cfg.model)
# Build run name: [DEBUG_]<task>_<train_mode>_<loss_weight_flags>
debug_flag = "DEBUG_" if cfg.debug else ""
train_mode = "mixed" if len(datamodule.train_datasets) == 2 else datamodule.train_datasets[0].dataset_name.split("_")[0]
loss_weights = f"{int(cfg.model.segmentation_loss_weight)}{int(cfg.model.vn_pos_loss_weight)}{int(cfg.model.confidence_loss_weight)}{int(cfg.model.pocket_ranking_loss_weight)}{int(cfg.model.protein_loss_weight)}{int(cfg.model.SB_virtual_screening_loss_weight)}{int(cfg.model.LB_virtual_screening_loss_weight)}"
run_name = f"{debug_flag}{cfg.task}_{train_mode}_{loss_weights}"
callbacks = init_lightning_callbacks(cfg)
# `~logger` (Hydra config-group removal) drops the key entirely rather
# than setting it to false/null, so fall back to Lightning's own default
# logger in that case — the LearningRateMonitor callback in the
# (non-debug) default callback set requires an active logger, so
# `logger=False` is only viable when it (and model checkpointing) are
# explicitly disabled, as in debug mode.
cfg_logger = cfg.get("logger", True)
if cfg_logger is False or cfg_logger is None:
logger = False
elif cfg_logger is True:
logger = True
else:
logger = instantiate(cfg_logger)(config=config, name=run_name)
if isinstance(logger, WandbLogger):
logger.watch(model, log="all")
trainer = instantiate(cfg.trainer, logger=logger, callbacks=callbacks)
trainer.fit(model, datamodule=datamodule)
# Evaluate on test set using the best checkpoint from training, or the
# in-memory model when checkpointing is disabled (e.g. debug runs)
ckpt_path = "best" if trainer.checkpoint_callback is not None else None
trainer.test(datamodule=datamodule, model=model, ckpt_path=ckpt_path)
if isinstance(logger, WandbLogger):
wandb.finish()
if __name__ == "__main__":
run()