2025-04-02 11:35:20 - root - INFO - Project root: /home/ubuntu/50T/lzy/mars-mcp/.venv/lib/python3.10/site-packages/fairchem
2025-04-02 11:35:21 - root - INFO - amp: true
cmd:
  checkpoint_dir: /home/ubuntu/50T/lzy/mars-mcp/checkpoints/2025-04-02-11-35-28
  commit: core:603304e,experimental:NA
  identifier: ''
  logs_dir: /home/ubuntu/50T/lzy/mars-mcp/logs/wandb/2025-04-02-11-35-28
  print_every: 100
  results_dir: /home/ubuntu/50T/lzy/mars-mcp/results/2025-04-02-11-35-28
  seed: null
  timestamp_id: 2025-04-02-11-35-28
  version: 1.9.0
dataset:
  a2g_args:
    r_energy: true
    r_forces: true
    r_stress: true
  format: ase_db
  transforms:
    decompose_tensor:
      decomposition:
        stress_anisotropic:
          irrep_dim: 2
        stress_isotropic:
          irrep_dim: 0
      rank: 2
      tensor: stress
    element_references:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/element_references.pt
    normalizer:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/normalizers.pt
evaluation_metrics:
  metrics:
    energy:
    - mae
    - mae_density
    forces:
    - mae
    - forcesx_mae
    - forcesy_mae
    - forcesz_mae
    - cosine_similarity
    stress:
    - mae
    - mae_density
    stress_anisotropic:
    - mae
    stress_isotropic:
    - mae
  primary_metric: energy_mae
gp_gpus: null
gpus: 0
logger: wandb
loss_functions:
- energy:
    coefficient: 20
    fn: mae_density
- forces:
    coefficient: 10
    fn: l2mae
- stress_isotropic:
    coefficient: 1
    fn: mae
- stress_anisotropic:
    coefficient: 1
    fn: mae
    reduction: mean_all
model:
  backbone:
    alpha_drop: 0.1
    attn_activation: silu
    attn_alpha_channels: 64
    attn_hidden_channels: 64
    attn_value_channels: 16
    avg_degree: 61.94676351484548
    avg_num_nodes: 31.16592360068011
    distance_function: gaussian
    drop_path_rate: 0.1
    edge_channels: 128
    enforce_max_neighbors_strictly: false
    ffn_activation: silu
    ffn_hidden_channels: 128
    grid_resolution: 18
    lmax_list:
    - 6
    max_neighbors: 20
    max_num_elements: 96
    max_radius: 12.0
    mmax_list:
    - 4
    model: equiformer_v2_backbone
    norm_type: layer_norm_sh
    num_distance_basis: 512
    num_heads: 8
    num_layers: 10
    num_sphere_samples: 128
    otf_graph: true
    proj_drop: 0.0
    share_atom_edge_embedding: false
    sphere_channels: 128
    use_atom_edge_embedding: true
    use_attn_renorm: true
    use_gate_act: false
    use_grid_mlp: true
    use_m_share_rad: false
    use_pbc: true
    use_pbc_single: true
    use_s2_act_attn: false
    use_sep_s2_act: true
    weight_init: uniform
  heads:
    energy:
      module: equiformer_v2_energy_head
    forces:
      module: equiformer_v2_force_head
    stress:
      decompose: true
      module: rank2_symmetric_head
      output_name: stress
      use_source_target_embedding: true
  name: hydra
  otf_graph: true
  pass_through_head_outputs: true
optim:
  batch_size: 8
  clip_grad_norm: 100
  ema_decay: 0.999
  eval_batch_size: 12
  eval_every: 3000
  load_balancing: atoms
  lr_initial: 0.0002
  max_epochs: 16
  num_workers: 8
  optimizer: AdamW
  optimizer_params:
    weight_decay: 0.001
  scheduler: LambdaLR
  scheduler_params:
    epochs: 741904
    lambda_type: cosine
    lr: 0.0002
    lr_min_factor: 0.01
    warmup_epochs: 463
    warmup_factor: 0.2
outputs:
  energy:
    level: system
    property: energy
  forces:
    eval_on_free_atoms: true
    level: atom
    property: forces
    train_on_free_atoms: true
  stress:
    decomposition:
      stress_anisotropic:
        eval_on_free_atoms: true
        irrep_dim: 2
        level: system
        parent: stress
        train_on_free_atoms: true
      stress_isotropic:
        eval_on_free_atoms: true
        irrep_dim: 0
        level: system
        parent: stress
        train_on_free_atoms: true
    level: system
    property: stress
relax_dataset: {}
slurm:
  account: ocp
  cpus_per_task: 9
  folder: /fsx-ocp-med/lbluque/logs/omat-alex-mp/S2EFS/train/4460394
  gpus_per_node: 8
  job_id: '4460394'
  job_name: eqV2_86M_ft_alexmptraj_e20_f10_s1_cos16
  mem: 480GB
  nodes: 4
  ntasks_per_node: 8
  partition: learn
  qos: ocp_high
  time: 4320
task: {}
test_dataset: {}
trainer: ocp
val_dataset: {}

2025-04-02 11:35:21 - root - INFO - Loading model: hydra
2025-04-02 11:35:25 - root - WARNING - equiformerV2_energy_head (EquiformerV2EnergyHead) class is deprecated in favor of equiformerV2_scalar_head  (EqV2ScalarHead)
2025-04-02 11:35:25 - root - WARNING - equiformerV2_force_head (EquiformerV2ForceHead) class is deprecated in favor of equiformerV2_rank1_head  (EqV2Rank1Head)
2025-04-02 11:35:25 - root - INFO - Loaded HydraModel with 86589068 parameters.
2025-04-02 11:35:25 - root - INFO - Loading checkpoint in inference-only mode, not loading keys associated with trainer state!
2025-04-02 11:35:26 - root - WARNING - No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
2025-04-02 11:35:26 - mars_toolkit.compute.structure_opt - INFO - FairChem model initialized successfully
2025-04-02 11:36:59 - root - INFO - Project root: /home/ubuntu/50T/lzy/mars-mcp/.venv/lib/python3.10/site-packages/fairchem
2025-04-02 11:37:01 - root - INFO - amp: true
cmd:
  checkpoint_dir: /home/ubuntu/50T/lzy/mars-mcp/checkpoints/2025-04-02-11-37-36
  commit: core:603304e,experimental:NA
  identifier: ''
  logs_dir: /home/ubuntu/50T/lzy/mars-mcp/logs/wandb/2025-04-02-11-37-36
  print_every: 100
  results_dir: /home/ubuntu/50T/lzy/mars-mcp/results/2025-04-02-11-37-36
  seed: null
  timestamp_id: 2025-04-02-11-37-36
  version: 1.9.0
dataset:
  a2g_args:
    r_energy: true
    r_forces: true
    r_stress: true
  format: ase_db
  transforms:
    decompose_tensor:
      decomposition:
        stress_anisotropic:
          irrep_dim: 2
        stress_isotropic:
          irrep_dim: 0
      rank: 2
      tensor: stress
    element_references:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/element_references.pt
    normalizer:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/normalizers.pt
evaluation_metrics:
  metrics:
    energy:
    - mae
    - mae_density
    forces:
    - mae
    - forcesx_mae
    - forcesy_mae
    - forcesz_mae
    - cosine_similarity
    stress:
    - mae
    - mae_density
    stress_anisotropic:
    - mae
    stress_isotropic:
    - mae
  primary_metric: energy_mae
gp_gpus: null
gpus: 0
logger: wandb
loss_functions:
- energy:
    coefficient: 20
    fn: mae_density
- forces:
    coefficient: 10
    fn: l2mae
- stress_isotropic:
    coefficient: 1
    fn: mae
- stress_anisotropic:
    coefficient: 1
    fn: mae
    reduction: mean_all
model:
  backbone:
    alpha_drop: 0.1
    attn_activation: silu
    attn_alpha_channels: 64
    attn_hidden_channels: 64
    attn_value_channels: 16
    avg_degree: 61.94676351484548
    avg_num_nodes: 31.16592360068011
    distance_function: gaussian
    drop_path_rate: 0.1
    edge_channels: 128
    enforce_max_neighbors_strictly: false
    ffn_activation: silu
    ffn_hidden_channels: 128
    grid_resolution: 18
    lmax_list:
    - 6
    max_neighbors: 20
    max_num_elements: 96
    max_radius: 12.0
    mmax_list:
    - 4
    model: equiformer_v2_backbone
    norm_type: layer_norm_sh
    num_distance_basis: 512
    num_heads: 8
    num_layers: 10
    num_sphere_samples: 128
    otf_graph: true
    proj_drop: 0.0
    share_atom_edge_embedding: false
    sphere_channels: 128
    use_atom_edge_embedding: true
    use_attn_renorm: true
    use_gate_act: false
    use_grid_mlp: true
    use_m_share_rad: false
    use_pbc: true
    use_pbc_single: true
    use_s2_act_attn: false
    use_sep_s2_act: true
    weight_init: uniform
  heads:
    energy:
      module: equiformer_v2_energy_head
    forces:
      module: equiformer_v2_force_head
    stress:
      decompose: true
      module: rank2_symmetric_head
      output_name: stress
      use_source_target_embedding: true
  name: hydra
  otf_graph: true
  pass_through_head_outputs: true
optim:
  batch_size: 8
  clip_grad_norm: 100
  ema_decay: 0.999
  eval_batch_size: 12
  eval_every: 3000
  load_balancing: atoms
  lr_initial: 0.0002
  max_epochs: 16
  num_workers: 8
  optimizer: AdamW
  optimizer_params:
    weight_decay: 0.001
  scheduler: LambdaLR
  scheduler_params:
    epochs: 741904
    lambda_type: cosine
    lr: 0.0002
    lr_min_factor: 0.01
    warmup_epochs: 463
    warmup_factor: 0.2
outputs:
  energy:
    level: system
    property: energy
  forces:
    eval_on_free_atoms: true
    level: atom
    property: forces
    train_on_free_atoms: true
  stress:
    decomposition:
      stress_anisotropic:
        eval_on_free_atoms: true
        irrep_dim: 2
        level: system
        parent: stress
        train_on_free_atoms: true
      stress_isotropic:
        eval_on_free_atoms: true
        irrep_dim: 0
        level: system
        parent: stress
        train_on_free_atoms: true
    level: system
    property: stress
relax_dataset: {}
slurm:
  account: ocp
  cpus_per_task: 9
  folder: /fsx-ocp-med/lbluque/logs/omat-alex-mp/S2EFS/train/4460394
  gpus_per_node: 8
  job_id: '4460394'
  job_name: eqV2_86M_ft_alexmptraj_e20_f10_s1_cos16
  mem: 480GB
  nodes: 4
  ntasks_per_node: 8
  partition: learn
  qos: ocp_high
  time: 4320
task: {}
test_dataset: {}
trainer: ocp
val_dataset: {}

2025-04-02 11:37:01 - root - INFO - Loading model: hydra
2025-04-02 11:37:04 - root - WARNING - equiformerV2_energy_head (EquiformerV2EnergyHead) class is deprecated in favor of equiformerV2_scalar_head  (EqV2ScalarHead)
2025-04-02 11:37:04 - root - WARNING - equiformerV2_force_head (EquiformerV2ForceHead) class is deprecated in favor of equiformerV2_rank1_head  (EqV2Rank1Head)
2025-04-02 11:37:05 - root - INFO - Loaded HydraModel with 86589068 parameters.
2025-04-02 11:37:05 - root - INFO - Loading checkpoint in inference-only mode, not loading keys associated with trainer state!
2025-04-02 11:37:05 - root - WARNING - No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
2025-04-02 11:37:05 - mars_toolkit.compute.structure_opt - INFO - FairChem model initialized successfully
2025-04-02 12:32:21 - root - INFO - Project root: /home/ubuntu/50T/lzy/mars-mcp/.venv/lib/python3.10/site-packages/fairchem
2025-04-02 12:32:22 - root - INFO - amp: true
cmd:
  checkpoint_dir: /home/ubuntu/50T/lzy/mars-mcp/checkpoints/2025-04-02-12-33-04
  commit: core:603304e,experimental:NA
  identifier: ''
  logs_dir: /home/ubuntu/50T/lzy/mars-mcp/logs/wandb/2025-04-02-12-33-04
  print_every: 100
  results_dir: /home/ubuntu/50T/lzy/mars-mcp/results/2025-04-02-12-33-04
  seed: null
  timestamp_id: 2025-04-02-12-33-04
  version: 1.9.0
dataset:
  a2g_args:
    r_energy: true
    r_forces: true
    r_stress: true
  format: ase_db
  transforms:
    decompose_tensor:
      decomposition:
        stress_anisotropic:
          irrep_dim: 2
        stress_isotropic:
          irrep_dim: 0
      rank: 2
      tensor: stress
    element_references:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/element_references.pt
    normalizer:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/normalizers.pt
evaluation_metrics:
  metrics:
    energy:
    - mae
    - mae_density
    forces:
    - mae
    - forcesx_mae
    - forcesy_mae
    - forcesz_mae
    - cosine_similarity
    stress:
    - mae
    - mae_density
    stress_anisotropic:
    - mae
    stress_isotropic:
    - mae
  primary_metric: energy_mae
gp_gpus: null
gpus: 0
logger: wandb
loss_functions:
- energy:
    coefficient: 20
    fn: mae_density
- forces:
    coefficient: 10
    fn: l2mae
- stress_isotropic:
    coefficient: 1
    fn: mae
- stress_anisotropic:
    coefficient: 1
    fn: mae
    reduction: mean_all
model:
  backbone:
    alpha_drop: 0.1
    attn_activation: silu
    attn_alpha_channels: 64
    attn_hidden_channels: 64
    attn_value_channels: 16
    avg_degree: 61.94676351484548
    avg_num_nodes: 31.16592360068011
    distance_function: gaussian
    drop_path_rate: 0.1
    edge_channels: 128
    enforce_max_neighbors_strictly: false
    ffn_activation: silu
    ffn_hidden_channels: 128
    grid_resolution: 18
    lmax_list:
    - 6
    max_neighbors: 20
    max_num_elements: 96
    max_radius: 12.0
    mmax_list:
    - 4
    model: equiformer_v2_backbone
    norm_type: layer_norm_sh
    num_distance_basis: 512
    num_heads: 8
    num_layers: 10
    num_sphere_samples: 128
    otf_graph: true
    proj_drop: 0.0
    share_atom_edge_embedding: false
    sphere_channels: 128
    use_atom_edge_embedding: true
    use_attn_renorm: true
    use_gate_act: false
    use_grid_mlp: true
    use_m_share_rad: false
    use_pbc: true
    use_pbc_single: true
    use_s2_act_attn: false
    use_sep_s2_act: true
    weight_init: uniform
  heads:
    energy:
      module: equiformer_v2_energy_head
    forces:
      module: equiformer_v2_force_head
    stress:
      decompose: true
      module: rank2_symmetric_head
      output_name: stress
      use_source_target_embedding: true
  name: hydra
  otf_graph: true
  pass_through_head_outputs: true
optim:
  batch_size: 8
  clip_grad_norm: 100
  ema_decay: 0.999
  eval_batch_size: 12
  eval_every: 3000
  load_balancing: atoms
  lr_initial: 0.0002
  max_epochs: 16
  num_workers: 8
  optimizer: AdamW
  optimizer_params:
    weight_decay: 0.001
  scheduler: LambdaLR
  scheduler_params:
    epochs: 741904
    lambda_type: cosine
    lr: 0.0002
    lr_min_factor: 0.01
    warmup_epochs: 463
    warmup_factor: 0.2
outputs:
  energy:
    level: system
    property: energy
  forces:
    eval_on_free_atoms: true
    level: atom
    property: forces
    train_on_free_atoms: true
  stress:
    decomposition:
      stress_anisotropic:
        eval_on_free_atoms: true
        irrep_dim: 2
        level: system
        parent: stress
        train_on_free_atoms: true
      stress_isotropic:
        eval_on_free_atoms: true
        irrep_dim: 0
        level: system
        parent: stress
        train_on_free_atoms: true
    level: system
    property: stress
relax_dataset: {}
slurm:
  account: ocp
  cpus_per_task: 9
  folder: /fsx-ocp-med/lbluque/logs/omat-alex-mp/S2EFS/train/4460394
  gpus_per_node: 8
  job_id: '4460394'
  job_name: eqV2_86M_ft_alexmptraj_e20_f10_s1_cos16
  mem: 480GB
  nodes: 4
  ntasks_per_node: 8
  partition: learn
  qos: ocp_high
  time: 4320
task: {}
test_dataset: {}
trainer: ocp
val_dataset: {}

2025-04-02 12:32:22 - root - INFO - Loading model: hydra
2025-04-02 12:32:25 - root - WARNING - equiformerV2_energy_head (EquiformerV2EnergyHead) class is deprecated in favor of equiformerV2_scalar_head  (EqV2ScalarHead)
2025-04-02 12:32:25 - root - WARNING - equiformerV2_force_head (EquiformerV2ForceHead) class is deprecated in favor of equiformerV2_rank1_head  (EqV2Rank1Head)
2025-04-02 12:32:26 - root - INFO - Loaded HydraModel with 86589068 parameters.
2025-04-02 12:32:26 - root - INFO - Loading checkpoint in inference-only mode, not loading keys associated with trainer state!
2025-04-02 12:32:26 - root - WARNING - No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
2025-04-02 12:32:26 - mars_toolkit.compute.structure_opt - INFO - FairChem model initialized successfully
2025-04-02 12:41:48 - root - INFO - Project root: /home/ubuntu/50T/lzy/mars-mcp/.venv/lib/python3.10/site-packages/fairchem
2025-04-02 12:41:49 - root - INFO - amp: true
cmd:
  checkpoint_dir: /home/ubuntu/50T/lzy/mars-mcp/checkpoints/2025-04-02-12-41-36
  commit: core:603304e,experimental:NA
  identifier: ''
  logs_dir: /home/ubuntu/50T/lzy/mars-mcp/logs/wandb/2025-04-02-12-41-36
  print_every: 100
  results_dir: /home/ubuntu/50T/lzy/mars-mcp/results/2025-04-02-12-41-36
  seed: null
  timestamp_id: 2025-04-02-12-41-36
  version: 1.9.0
dataset:
  a2g_args:
    r_energy: true
    r_forces: true
    r_stress: true
  format: ase_db
  transforms:
    decompose_tensor:
      decomposition:
        stress_anisotropic:
          irrep_dim: 2
        stress_isotropic:
          irrep_dim: 0
      rank: 2
      tensor: stress
    element_references:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/element_references.pt
    normalizer:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/normalizers.pt
evaluation_metrics:
  metrics:
    energy:
    - mae
    - mae_density
    forces:
    - mae
    - forcesx_mae
    - forcesy_mae
    - forcesz_mae
    - cosine_similarity
    stress:
    - mae
    - mae_density
    stress_anisotropic:
    - mae
    stress_isotropic:
    - mae
  primary_metric: energy_mae
gp_gpus: null
gpus: 0
logger: wandb
loss_functions:
- energy:
    coefficient: 20
    fn: mae_density
- forces:
    coefficient: 10
    fn: l2mae
- stress_isotropic:
    coefficient: 1
    fn: mae
- stress_anisotropic:
    coefficient: 1
    fn: mae
    reduction: mean_all
model:
  backbone:
    alpha_drop: 0.1
    attn_activation: silu
    attn_alpha_channels: 64
    attn_hidden_channels: 64
    attn_value_channels: 16
    avg_degree: 61.94676351484548
    avg_num_nodes: 31.16592360068011
    distance_function: gaussian
    drop_path_rate: 0.1
    edge_channels: 128
    enforce_max_neighbors_strictly: false
    ffn_activation: silu
    ffn_hidden_channels: 128
    grid_resolution: 18
    lmax_list:
    - 6
    max_neighbors: 20
    max_num_elements: 96
    max_radius: 12.0
    mmax_list:
    - 4
    model: equiformer_v2_backbone
    norm_type: layer_norm_sh
    num_distance_basis: 512
    num_heads: 8
    num_layers: 10
    num_sphere_samples: 128
    otf_graph: true
    proj_drop: 0.0
    share_atom_edge_embedding: false
    sphere_channels: 128
    use_atom_edge_embedding: true
    use_attn_renorm: true
    use_gate_act: false
    use_grid_mlp: true
    use_m_share_rad: false
    use_pbc: true
    use_pbc_single: true
    use_s2_act_attn: false
    use_sep_s2_act: true
    weight_init: uniform
  heads:
    energy:
      module: equiformer_v2_energy_head
    forces:
      module: equiformer_v2_force_head
    stress:
      decompose: true
      module: rank2_symmetric_head
      output_name: stress
      use_source_target_embedding: true
  name: hydra
  otf_graph: true
  pass_through_head_outputs: true
optim:
  batch_size: 8
  clip_grad_norm: 100
  ema_decay: 0.999
  eval_batch_size: 12
  eval_every: 3000
  load_balancing: atoms
  lr_initial: 0.0002
  max_epochs: 16
  num_workers: 8
  optimizer: AdamW
  optimizer_params:
    weight_decay: 0.001
  scheduler: LambdaLR
  scheduler_params:
    epochs: 741904
    lambda_type: cosine
    lr: 0.0002
    lr_min_factor: 0.01
    warmup_epochs: 463
    warmup_factor: 0.2
outputs:
  energy:
    level: system
    property: energy
  forces:
    eval_on_free_atoms: true
    level: atom
    property: forces
    train_on_free_atoms: true
  stress:
    decomposition:
      stress_anisotropic:
        eval_on_free_atoms: true
        irrep_dim: 2
        level: system
        parent: stress
        train_on_free_atoms: true
      stress_isotropic:
        eval_on_free_atoms: true
        irrep_dim: 0
        level: system
        parent: stress
        train_on_free_atoms: true
    level: system
    property: stress
relax_dataset: {}
slurm:
  account: ocp
  cpus_per_task: 9
  folder: /fsx-ocp-med/lbluque/logs/omat-alex-mp/S2EFS/train/4460394
  gpus_per_node: 8
  job_id: '4460394'
  job_name: eqV2_86M_ft_alexmptraj_e20_f10_s1_cos16
  mem: 480GB
  nodes: 4
  ntasks_per_node: 8
  partition: learn
  qos: ocp_high
  time: 4320
task: {}
test_dataset: {}
trainer: ocp
val_dataset: {}

2025-04-02 12:41:49 - root - INFO - Loading model: hydra
2025-04-02 12:41:52 - root - WARNING - equiformerV2_energy_head (EquiformerV2EnergyHead) class is deprecated in favor of equiformerV2_scalar_head  (EqV2ScalarHead)
2025-04-02 12:41:52 - root - WARNING - equiformerV2_force_head (EquiformerV2ForceHead) class is deprecated in favor of equiformerV2_rank1_head  (EqV2Rank1Head)
2025-04-02 12:41:52 - root - INFO - Loaded HydraModel with 86589068 parameters.
2025-04-02 12:41:52 - root - INFO - Loading checkpoint in inference-only mode, not loading keys associated with trainer state!
2025-04-02 12:41:53 - root - WARNING - No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
2025-04-02 12:41:53 - mars_toolkit.compute.structure_opt - INFO - FairChem model initialized successfully
2025-04-02 12:44:38 - root - INFO - Project root: /home/ubuntu/50T/lzy/mars-mcp/.venv/lib/python3.10/site-packages/fairchem
2025-04-02 12:44:39 - root - INFO - amp: true
cmd:
  checkpoint_dir: /home/ubuntu/50T/lzy/mars-mcp/checkpoints/2025-04-02-12-43-44
  commit: core:603304e,experimental:NA
  identifier: ''
  logs_dir: /home/ubuntu/50T/lzy/mars-mcp/logs/wandb/2025-04-02-12-43-44
  print_every: 100
  results_dir: /home/ubuntu/50T/lzy/mars-mcp/results/2025-04-02-12-43-44
  seed: null
  timestamp_id: 2025-04-02-12-43-44
  version: 1.9.0
dataset:
  a2g_args:
    r_energy: true
    r_forces: true
    r_stress: true
  format: ase_db
  transforms:
    decompose_tensor:
      decomposition:
        stress_anisotropic:
          irrep_dim: 2
        stress_isotropic:
          irrep_dim: 0
      rank: 2
      tensor: stress
    element_references:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/element_references.pt
    normalizer:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/normalizers.pt
evaluation_metrics:
  metrics:
    energy:
    - mae
    - mae_density
    forces:
    - mae
    - forcesx_mae
    - forcesy_mae
    - forcesz_mae
    - cosine_similarity
    stress:
    - mae
    - mae_density
    stress_anisotropic:
    - mae
    stress_isotropic:
    - mae
  primary_metric: energy_mae
gp_gpus: null
gpus: 0
logger: wandb
loss_functions:
- energy:
    coefficient: 20
    fn: mae_density
- forces:
    coefficient: 10
    fn: l2mae
- stress_isotropic:
    coefficient: 1
    fn: mae
- stress_anisotropic:
    coefficient: 1
    fn: mae
    reduction: mean_all
model:
  backbone:
    alpha_drop: 0.1
    attn_activation: silu
    attn_alpha_channels: 64
    attn_hidden_channels: 64
    attn_value_channels: 16
    avg_degree: 61.94676351484548
    avg_num_nodes: 31.16592360068011
    distance_function: gaussian
    drop_path_rate: 0.1
    edge_channels: 128
    enforce_max_neighbors_strictly: false
    ffn_activation: silu
    ffn_hidden_channels: 128
    grid_resolution: 18
    lmax_list:
    - 6
    max_neighbors: 20
    max_num_elements: 96
    max_radius: 12.0
    mmax_list:
    - 4
    model: equiformer_v2_backbone
    norm_type: layer_norm_sh
    num_distance_basis: 512
    num_heads: 8
    num_layers: 10
    num_sphere_samples: 128
    otf_graph: true
    proj_drop: 0.0
    share_atom_edge_embedding: false
    sphere_channels: 128
    use_atom_edge_embedding: true
    use_attn_renorm: true
    use_gate_act: false
    use_grid_mlp: true
    use_m_share_rad: false
    use_pbc: true
    use_pbc_single: true
    use_s2_act_attn: false
    use_sep_s2_act: true
    weight_init: uniform
  heads:
    energy:
      module: equiformer_v2_energy_head
    forces:
      module: equiformer_v2_force_head
    stress:
      decompose: true
      module: rank2_symmetric_head
      output_name: stress
      use_source_target_embedding: true
  name: hydra
  otf_graph: true
  pass_through_head_outputs: true
optim:
  batch_size: 8
  clip_grad_norm: 100
  ema_decay: 0.999
  eval_batch_size: 12
  eval_every: 3000
  load_balancing: atoms
  lr_initial: 0.0002
  max_epochs: 16
  num_workers: 8
  optimizer: AdamW
  optimizer_params:
    weight_decay: 0.001
  scheduler: LambdaLR
  scheduler_params:
    epochs: 741904
    lambda_type: cosine
    lr: 0.0002
    lr_min_factor: 0.01
    warmup_epochs: 463
    warmup_factor: 0.2
outputs:
  energy:
    level: system
    property: energy
  forces:
    eval_on_free_atoms: true
    level: atom
    property: forces
    train_on_free_atoms: true
  stress:
    decomposition:
      stress_anisotropic:
        eval_on_free_atoms: true
        irrep_dim: 2
        level: system
        parent: stress
        train_on_free_atoms: true
      stress_isotropic:
        eval_on_free_atoms: true
        irrep_dim: 0
        level: system
        parent: stress
        train_on_free_atoms: true
    level: system
    property: stress
relax_dataset: {}
slurm:
  account: ocp
  cpus_per_task: 9
  folder: /fsx-ocp-med/lbluque/logs/omat-alex-mp/S2EFS/train/4460394
  gpus_per_node: 8
  job_id: '4460394'
  job_name: eqV2_86M_ft_alexmptraj_e20_f10_s1_cos16
  mem: 480GB
  nodes: 4
  ntasks_per_node: 8
  partition: learn
  qos: ocp_high
  time: 4320
task: {}
test_dataset: {}
trainer: ocp
val_dataset: {}

2025-04-02 12:44:39 - root - INFO - Loading model: hydra
2025-04-02 12:44:43 - root - WARNING - equiformerV2_energy_head (EquiformerV2EnergyHead) class is deprecated in favor of equiformerV2_scalar_head  (EqV2ScalarHead)
2025-04-02 12:44:43 - root - WARNING - equiformerV2_force_head (EquiformerV2ForceHead) class is deprecated in favor of equiformerV2_rank1_head  (EqV2Rank1Head)
2025-04-02 12:44:43 - root - INFO - Loaded HydraModel with 86589068 parameters.
2025-04-02 12:44:43 - root - INFO - Loading checkpoint in inference-only mode, not loading keys associated with trainer state!
2025-04-02 12:44:43 - root - WARNING - No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
2025-04-02 12:44:43 - mars_toolkit.compute.structure_opt - INFO - FairChem model initialized successfully
2025-04-02 12:45:04 - mars_toolkit.compute.structure_opt - ERROR - Failed to optimize structure: 'str' object has no attribute 'site_properties'
2025-04-02 12:45:04 - mars_toolkit.core.error_handlers - ERROR - Unexpected error: 'str' object has no attribute 'site_properties'
2025-04-02 12:47:19 - root - INFO - Project root: /home/ubuntu/50T/lzy/mars-mcp/.venv/lib/python3.10/site-packages/fairchem
2025-04-02 12:47:20 - root - INFO - amp: true
cmd:
  checkpoint_dir: /home/ubuntu/50T/lzy/mars-mcp/checkpoints/2025-04-02-12-48-00
  commit: core:603304e,experimental:NA
  identifier: ''
  logs_dir: /home/ubuntu/50T/lzy/mars-mcp/logs/wandb/2025-04-02-12-48-00
  print_every: 100
  results_dir: /home/ubuntu/50T/lzy/mars-mcp/results/2025-04-02-12-48-00
  seed: null
  timestamp_id: 2025-04-02-12-48-00
  version: 1.9.0
dataset:
  a2g_args:
    r_energy: true
    r_forces: true
    r_stress: true
  format: ase_db
  transforms:
    decompose_tensor:
      decomposition:
        stress_anisotropic:
          irrep_dim: 2
        stress_isotropic:
          irrep_dim: 0
      rank: 2
      tensor: stress
    element_references:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/element_references.pt
    normalizer:
      file: /fsx-ocp-med/shared/alex-10M/alex-mp-norms-refs/normalizers.pt
evaluation_metrics:
  metrics:
    energy:
    - mae
    - mae_density
    forces:
    - mae
    - forcesx_mae
    - forcesy_mae
    - forcesz_mae
    - cosine_similarity
    stress:
    - mae
    - mae_density
    stress_anisotropic:
    - mae
    stress_isotropic:
    - mae
  primary_metric: energy_mae
gp_gpus: null
gpus: 0
logger: wandb
loss_functions:
- energy:
    coefficient: 20
    fn: mae_density
- forces:
    coefficient: 10
    fn: l2mae
- stress_isotropic:
    coefficient: 1
    fn: mae
- stress_anisotropic:
    coefficient: 1
    fn: mae
    reduction: mean_all
model:
  backbone:
    alpha_drop: 0.1
    attn_activation: silu
    attn_alpha_channels: 64
    attn_hidden_channels: 64
    attn_value_channels: 16
    avg_degree: 61.94676351484548
    avg_num_nodes: 31.16592360068011
    distance_function: gaussian
    drop_path_rate: 0.1
    edge_channels: 128
    enforce_max_neighbors_strictly: false
    ffn_activation: silu
    ffn_hidden_channels: 128
    grid_resolution: 18
    lmax_list:
    - 6
    max_neighbors: 20
    max_num_elements: 96
    max_radius: 12.0
    mmax_list:
    - 4
    model: equiformer_v2_backbone
    norm_type: layer_norm_sh
    num_distance_basis: 512
    num_heads: 8
    num_layers: 10
    num_sphere_samples: 128
    otf_graph: true
    proj_drop: 0.0
    share_atom_edge_embedding: false
    sphere_channels: 128
    use_atom_edge_embedding: true
    use_attn_renorm: true
    use_gate_act: false
    use_grid_mlp: true
    use_m_share_rad: false
    use_pbc: true
    use_pbc_single: true
    use_s2_act_attn: false
    use_sep_s2_act: true
    weight_init: uniform
  heads:
    energy:
      module: equiformer_v2_energy_head
    forces:
      module: equiformer_v2_force_head
    stress:
      decompose: true
      module: rank2_symmetric_head
      output_name: stress
      use_source_target_embedding: true
  name: hydra
  otf_graph: true
  pass_through_head_outputs: true
optim:
  batch_size: 8
  clip_grad_norm: 100
  ema_decay: 0.999
  eval_batch_size: 12
  eval_every: 3000
  load_balancing: atoms
  lr_initial: 0.0002
  max_epochs: 16
  num_workers: 8
  optimizer: AdamW
  optimizer_params:
    weight_decay: 0.001
  scheduler: LambdaLR
  scheduler_params:
    epochs: 741904
    lambda_type: cosine
    lr: 0.0002
    lr_min_factor: 0.01
    warmup_epochs: 463
    warmup_factor: 0.2
outputs:
  energy:
    level: system
    property: energy
  forces:
    eval_on_free_atoms: true
    level: atom
    property: forces
    train_on_free_atoms: true
  stress:
    decomposition:
      stress_anisotropic:
        eval_on_free_atoms: true
        irrep_dim: 2
        level: system
        parent: stress
        train_on_free_atoms: true
      stress_isotropic:
        eval_on_free_atoms: true
        irrep_dim: 0
        level: system
        parent: stress
        train_on_free_atoms: true
    level: system
    property: stress
relax_dataset: {}
slurm:
  account: ocp
  cpus_per_task: 9
  folder: /fsx-ocp-med/lbluque/logs/omat-alex-mp/S2EFS/train/4460394
  gpus_per_node: 8
  job_id: '4460394'
  job_name: eqV2_86M_ft_alexmptraj_e20_f10_s1_cos16
  mem: 480GB
  nodes: 4
  ntasks_per_node: 8
  partition: learn
  qos: ocp_high
  time: 4320
task: {}
test_dataset: {}
trainer: ocp
val_dataset: {}

2025-04-02 12:47:20 - root - INFO - Loading model: hydra
2025-04-02 12:47:24 - root - WARNING - equiformerV2_energy_head (EquiformerV2EnergyHead) class is deprecated in favor of equiformerV2_scalar_head  (EqV2ScalarHead)
2025-04-02 12:47:24 - root - WARNING - equiformerV2_force_head (EquiformerV2ForceHead) class is deprecated in favor of equiformerV2_rank1_head  (EqV2Rank1Head)
2025-04-02 12:47:24 - root - INFO - Loaded HydraModel with 86589068 parameters.
2025-04-02 12:47:24 - root - INFO - Loading checkpoint in inference-only mode, not loading keys associated with trainer state!
2025-04-02 12:47:24 - root - WARNING - No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
2025-04-02 12:47:24 - mars_toolkit.compute.structure_opt - INFO - FairChem model initialized successfully