# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause

from isaaclab.utils import configclass

from isaaclab_rl.rsl_rl import RslRlOnPolicyRunnerCfg, RslRlPpoActorCriticCfg, RslRlPpoAlgorithmCfg


@configclass
class PPORunnerCfg(RslRlOnPolicyRunnerCfg):
    num_steps_per_env = 48  # 稍微增加每轮步数
    max_iterations = 5000   # 增加迭代次数，给它足够的时间学习
    save_interval = 100
    experiment_name = "mindbot_grasp"  # 建议改个名，不要叫 cartpole 了
    
    policy = RslRlPpoActorCriticCfg(
        init_noise_std=0.7,
        actor_obs_normalization=True,   # 开启归一化
        critic_obs_normalization=True,  # 开启归一化
        # 增加网络容量：三层 MLP，宽度足够
        actor_hidden_dims=[128, 64, 32], 
        critic_hidden_dims=[128, 64, 32],
        activation="elu",
    )
    
    algorithm = RslRlPpoAlgorithmCfg(
        value_loss_coef=1.0,
        use_clipped_value_loss=True,
        clip_param=0.2,
        entropy_coef=0.005, # 保持适度的探索
        num_learning_epochs=5,
        num_mini_batches=8,
        learning_rate=3e-4, # 如果网络变大后不稳定，可以尝试降低到 5.0e-4
        schedule="adaptive",
        gamma=0.99,
        lam=0.95,
        desired_kl=0.01,
        max_grad_norm=1.0,
    )


# @configclass
# class PPORunnerCfg(RslRlOnPolicyRunnerCfg):
#     num_steps_per_env = 16
#     max_iterations = 150
#     save_interval = 50
#     experiment_name = "cartpole_direct"
#     policy = RslRlPpoActorCriticCfg(
#         init_noise_std=1.0,
#         actor_obs_normalization=False,
#         critic_obs_normalization=False,
#         actor_hidden_dims=[32, 32],
#         critic_hidden_dims=[32, 32],
#         activation="elu",
#     )
#     algorithm = RslRlPpoAlgorithmCfg(
#         value_loss_coef=1.0,
#         use_clipped_value_loss=True,
#         clip_param=0.2,
#         entropy_coef=0.005,
#         num_learning_epochs=5,
#         num_mini_batches=4,
#         learning_rate=1.0e-3,
#         schedule="adaptive",
#         gamma=0.99,
#         lam=0.95,
#         desired_kl=0.01,
#         max_grad_norm=1.0,
#     )