Added possiblity to record and replay delta actions during teleoperation rather than absolute actions

Co-authored-by: Adil Zouitine <adilzouitinegm@gmail.com>

lerobot/configs/env/so100_real.yaml

@@ -12,8 +12,10 @@ env:
   
   wrapper:
     crop_params_dict:
-      observation.images.laptop: [58, 89, 357, 455]
-      observation.images.phone: [3, 4, 471, 633]
+      observation.images.front: [126, 43, 329, 518]
+      observation.images.side: [93, 69, 381, 434]
+      # observation.images.front: [135, 59, 331, 527]
+      # observation.images.side: [79, 47, 397, 450]
     resize_size: [128, 128]
     control_time_s: 20
     reset_follower_pos: true

lerobot/configs/policy/hilserl_classifier.yaml

@@ -4,7 +4,9 @@ defaults:
   - _self_
 
 seed: 13
-dataset_repo_id: aractingi/push_green_cube_hf_cropped_resized
+dataset_repo_id: aractingi/push_cube_square_reward_cropped_resized
+dataset_root: data/aractingi/push_cube_square_reward_cropped_resized
+local_files_only: true
 train_split_proportion: 0.8
 
 # Required by logger
@@ -14,7 +16,7 @@ env:
 
 
 training:
-  num_epochs: 5
+  num_epochs: 6
   batch_size: 16
   learning_rate: 1e-4
   num_workers: 4
@@ -25,7 +27,7 @@ training:
   save_freq: 1  # How often to save checkpoints (in epochs)
   save_checkpoint: true
   # image_keys: ["observation.images.top", "observation.images.wrist"]
-  image_keys: ["observation.images.laptop", "observation.images.phone"]
+  image_keys: ["observation.images.front", "observation.images.side"]
   label_key: "next.reward"
   profile_inference_time: false
   profile_inference_time_iters: 20
@@ -35,8 +37,8 @@ eval:
   num_samples_to_log: 30  # Number of validation samples to log in the table
 
 policy:
-  name: "hilserl/classifier/push_green_cube_hf_cropped_resized" #"hilserl/classifier/pick_place_lego_cube_1"
-  model_name: "helper2424/resnet10"
+  name: "hilserl/classifier/push_cube_square_reward_cropped_resized" #"hilserl/classifier/pick_place_lego_cube_120
+  model_name: "helper2424/resnet10" # "facebook/convnext-base-224" #"helper2424/resnet10"
   model_type: "cnn"
   num_cameras: 2 # Has to be len(training.image_keys)
 
@@ -48,4 +50,4 @@ wandb:
 
 device: "mps"
 resume: false
-output_dir: "outputs/classifier"
+output_dir: "outputs/classifier/resnet10_frozen"

lerobot/configs/policy/sac_real.yaml

@@ -57,22 +57,22 @@ policy:
   input_shapes:
     # # TODO(rcadene, alexander-soare): add variables for height and width from the dataset/env?
     observation.state: ["${env.state_dim}"]
-    observation.images.laptop: [3, 128, 128]
-    observation.images.phone: [3, 128, 128]
+    observation.images.front: [3, 128, 128]
+    observation.images.side: [3, 128, 128]
     # observation.image: [3, 128, 128]
   output_shapes:
     action: ["${env.action_dim}"]
 
   # Normalization / Unnormalization
   input_normalization_modes: 
-    observation.images.laptop: mean_std
-    observation.images.phone: mean_std
+    observation.images.front: mean_std
+    observation.images.side: mean_std
     observation.state: min_max
   input_normalization_params:
-    observation.images.laptop:
+    observation.images.front:
       mean: [0.485, 0.456, 0.406]
       std: [0.229, 0.224, 0.225]
-    observation.images.phone:
+    observation.images.side:
       mean: [0.485, 0.456, 0.406]
       std: [0.229, 0.224, 0.225]
     observation.state:

lerobot/configs/robot/so100.yaml

@@ -50,13 +50,13 @@ follower_arms:
       gripper: [6, "sts3215"]
 
 cameras:
-  laptop:
+  front:
     _target_: lerobot.common.robot_devices.cameras.opencv.OpenCVCamera
     camera_index: 0
     fps: 30
     width: 640
     height: 480
-  phone:
+  side:
     _target_: lerobot.common.robot_devices.cameras.opencv.OpenCVCamera
     camera_index: 1
     fps: 30