Fix artifacts with proper latency

examples/advanced/1_train_act_pusht/act_pusht.yaml

@@ -28,7 +28,7 @@ training:
   online_steps_between_rollouts: 1
 
   delta_timestamps:
-    action: "[i / ${fps} for i in range(${policy.chunk_size})]"
+    action: "[i / ${fps} + ${latency} for i in range(${policy.chunk_size})]"
 
 eval:
   n_episodes: 50

lerobot/configs/default.yaml

@@ -57,3 +57,7 @@ wandb:
   disable_artifact: false
   project: lerobot
   notes: ""
+
+# These configs need to be defined in environment yaml files (e.g. `env/aloha.yaml`)
+fps: ???
+latency: ???

lerobot/configs/env/aloha.yaml

@@ -1,6 +1,12 @@
 # @package _global_
 
 fps: 50
+# The latency corresponds to the delay in seconds between the current time at which the observation
+# is fed to the policy to compute the next action, and the next action being sent to the controller.
+# Note: For simulation environments, there could be no latency, but for real-world, latency is usually
+# set to `1 / fps` to account for the runtime of the policy (which is often lower than `1 / fps`) and then
+# a forced sleep to reach `+ 1 / fps` from current observation time.
+latency: 0
 
 env:
   name: aloha

lerobot/configs/env/dora_aloha_real.yaml

@@ -1,6 +1,12 @@
 # @package _global_
 
 fps: 30
+# The latency corresponds to the delay in seconds between the current time at which the observation
+# is fed to the policy to compute the next action, and the next action being sent to the controller.
+# Note: For simulation environments, there could be no latency, but for real-world, latency is usually
+# set to `1 / fps` to account for the runtime of the policy (which is often lower than `1 / fps`) and then
+# a forced sleep to reach `+ 1 / fps` from current observation time.
+latency: 1 / ${fps}
 
 env:
   name: dora

lerobot/configs/env/pusht.yaml

@@ -1,6 +1,12 @@
 # @package _global_
 
 fps: 10
+# The latency corresponds to the delay in seconds between the current time at which the observation
+# is fed to the policy to compute the next action, and the next action being sent to the controller.
+# Note: For simulation environments, there could be no latency, but for real-world, latency is usually
+# set to `1 / fps` to account for the runtime of the policy (which is often lower than `1 / fps`) and then
+# a forced sleep to reach `+ 1 / fps` from current observation time.
+latency: 0
 
 env:
   name: pusht

lerobot/configs/env/xarm.yaml

@@ -1,6 +1,12 @@
 # @package _global_
 
 fps: 15
+# The latency corresponds to the delay in seconds between the current time at which the observation
+# is fed to the policy to compute the next action, and the next action being sent to the controller.
+# Note: For simulation environments, there could be no latency, but for real-world, latency is usually
+# set to `1 / fps` to account for the runtime of the policy (which is often lower than `1 / fps`) and then
+# a forced sleep to reach `+ 1 / fps` from current observation time.
+latency: 0
 
 env:
   name: xarm

lerobot/configs/policy/act.yaml

@@ -25,7 +25,7 @@ training:
   online_steps_between_rollouts: 1
 
   delta_timestamps:
-    action: "[i / ${fps} for i in range(${policy.chunk_size})]"
+    action: "[i / ${fps} + ${latency} for i in range(${policy.chunk_size})]"
 
 eval:
   n_episodes: 50

lerobot/configs/policy/act_real.yaml

@@ -51,7 +51,7 @@ training:
   online_steps_between_rollouts: 1
 
   delta_timestamps:
-    action: "[i / ${fps} for i in range(${policy.chunk_size})]"
+    action: "[i / ${fps} + ${latency} for i in range(${policy.chunk_size})]"
 
 eval:
   n_episodes: 50

lerobot/configs/policy/act_real_no_state.yaml

@@ -49,7 +49,7 @@ training:
   online_steps_between_rollouts: 1
 
   delta_timestamps:
-    action: "[i / ${fps} for i in range(${policy.chunk_size})]"
+    action: "[i / ${fps} + ${latency} for i in range(${policy.chunk_size})]"
 
 eval:
   n_episodes: 50

lerobot/configs/policy/diffusion.yaml

@@ -42,7 +42,7 @@ training:
   delta_timestamps:
     observation.image: "[i / ${fps} for i in range(1 - ${policy.n_obs_steps}, 1)]"
     observation.state: "[i / ${fps} for i in range(1 - ${policy.n_obs_steps}, 1)]"
-    action: "[i / ${fps} for i in range(1 - ${policy.n_obs_steps}, 1 - ${policy.n_obs_steps} + ${policy.horizon})]"
+    action: "[i / ${fps} + ${latency} for i in range(1 - ${policy.n_obs_steps}, 1 - ${policy.n_obs_steps} + ${policy.horizon})]"
 
   # The original implementation doesn't sample frames for the last 7 steps,
   # which avoids excessive padding and leads to improved training results.

lerobot/configs/policy/tdmpc.yaml

@@ -20,7 +20,7 @@ training:
     observation.image: "[i / ${fps} for i in range(${policy.horizon} + 1)]"
     observation.state: "[i / ${fps} for i in range(${policy.horizon} + 1)]"
     action: "[i / ${fps} for i in range(${policy.horizon})]"
-    next.reward: "[i / ${fps} for i in range(${policy.horizon})]"
+    next.reward: "[i / ${fps} + ${latency} for i in range(${policy.horizon})]"
 
 policy:
   name: tdmpc

tests/data/save_policy_to_safetensors/dora_aloha_real_act_real/actions.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2fff6294b94cf42d4dd1249dcc5c3b0269d6d9c697f894e61b867d7ab81a94e4
+oid sha256:7eda418c27d747c11006a7c7d013a897b7f11633118cf124d4a1f17086e370b0
 size 5104

tests/data/save_policy_to_safetensors/dora_aloha_real_act_real/grad_stats.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4aa23e51607604a18b70fa42edbbe1af34f119d985628fc27cc1bbb0efbc8901
+oid sha256:8d32450bf5e21922e0522497636274f37b71cf1676a31608eee920e4e2f65010
 size 31688

tests/data/save_policy_to_safetensors/dora_aloha_real_act_real/output_dict.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6fd368406c93cb562a69ff11cf7adf34a4b223507dcb2b9e9b8f44ee1036988a
+oid sha256:3d0afb5e2c0dabfeb0aa43dc1229756c5f9cc5c98ef4326cbfb9da91e2e0bf75
 size 68

tests/data/save_policy_to_safetensors/dora_aloha_real_act_real/param_stats.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5663ee79a13bb70a1604b887dd21bf89d18482287442419c6cc6c5bf0e753e99
+oid sha256:46b02cacbfea20128a00c599a800babece17ca31f65c8e2527a155cd8fc99291
 size 34928

tests/data/save_policy_to_safetensors/dora_aloha_real_act_real_no_state/actions.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb1a45463efd860af2ca22c16c77d55a18bd96fef080ae77978845a2f22ef716
+oid sha256:fb423e9171e65696d094b8e75de748c6bc42f8b89563fc4398f94fa4ebdeeaa9
 size 5104

tests/data/save_policy_to_safetensors/dora_aloha_real_act_real_no_state/grad_stats.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa5a43e22f01d8e2f8d19f31753608794f1edbd74aaf71660091ab80ea58dc9b
+oid sha256:715d42afe6a1f376a6339781473f621bef1f029b229de6c53fa33230aca70221
 size 30808

tests/data/save_policy_to_safetensors/dora_aloha_real_act_real_no_state/output_dict.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97455b4360748c99905cd103473c1a52da6901d0a73ffbc51b5ea3eb250d1386
+oid sha256:ac5f446d5b6716b05e1ae9b5052f87522adb1252a062d4e236c37df50955f439
 size 68

tests/data/save_policy_to_safetensors/dora_aloha_real_act_real_no_state/param_stats.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54d1f75cf67a7b1d7a7c6865ecb9b1cc86a2f032d1890245f8996789ab6e0df6
+oid sha256:8ec83c376f09b426094f50459f5fa4cfa5f6dc997d381152745214b11eb615e5
 size 33608