Add video decoding to LeRobotDataset (#92)

tests/data/lerobot/aloha_sim_insertion_human/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 50
+    "fps": 50,
+    "video": 1
 }

tests/data/lerobot/aloha_sim_insertion_human/train/dataset_info.json

@@ -3,7 +3,7 @@
   "description": "",
   "features": {
     "observation.images.top": {
-      "_type": "Image"
+      "_type": "VideoFrame"
     },
     "observation.state": {
       "feature": {

tests/data/lerobot/aloha_sim_insertion_human/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "22eeca7a3f4725ee",
+  "_fingerprint": "eb913a2b1a68aa74",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/data/lerobot/aloha_sim_insertion_scripted/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 50
+    "fps": 50,
+    "video": 1
 }

tests/data/lerobot/aloha_sim_insertion_scripted/train/dataset_info.json

@@ -3,7 +3,7 @@
   "description": "",
   "features": {
     "observation.images.top": {
-      "_type": "Image"
+      "_type": "VideoFrame"
     },
     "observation.state": {
       "feature": {

tests/data/lerobot/aloha_sim_insertion_scripted/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "97c28d4ad1536e4c",
+  "_fingerprint": "d20c2acf1e107266",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/data/lerobot/aloha_sim_transfer_cube_human/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 50
+    "fps": 50,
+    "video": 1
 }

tests/data/lerobot/aloha_sim_transfer_cube_human/train/dataset_info.json

@@ -3,7 +3,7 @@
   "description": "",
   "features": {
     "observation.images.top": {
-      "_type": "Image"
+      "_type": "VideoFrame"
     },
     "observation.state": {
       "feature": {

tests/data/lerobot/aloha_sim_transfer_cube_human/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "cb9349b5c92951e8",
+  "_fingerprint": "243b01eb8a4b184e",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/data/lerobot/aloha_sim_transfer_cube_scripted/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 50
+    "fps": 50,
+    "video": 1
 }

tests/data/lerobot/aloha_sim_transfer_cube_scripted/train/dataset_info.json

@@ -3,7 +3,7 @@
   "description": "",
   "features": {
     "observation.images.top": {
-      "_type": "Image"
+      "_type": "VideoFrame"
     },
     "observation.state": {
       "feature": {

tests/data/lerobot/aloha_sim_transfer_cube_scripted/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "e4d7ad2b360db1af",
+  "_fingerprint": "eb759bbf60df7be9",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/data/lerobot/pusht/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 10
+    "fps": 10,
+    "video": 1
 }

tests/data/lerobot/pusht/train/dataset_info.json

@@ -3,7 +3,7 @@
   "description": "",
   "features": {
     "observation.image": {
-      "_type": "Image"
+      "_type": "VideoFrame"
     },
     "observation.state": {
       "feature": {

tests/data/lerobot/pusht/train/meta_data/info.json

@@ -1,3 +0,0 @@
-{
-    "fps": 10
-}

tests/data/lerobot/pusht/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "a04a9ce660122e23",
+  "_fingerprint": "3e02d7879f423c56",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/data/lerobot/umi_cup_in_the_wild/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 10
+    "fps": 10,
+    "video": 1
 }

tests/data/lerobot/umi_cup_in_the_wild/train/dataset_info.json

@@ -2,6 +2,9 @@
   "citation": "",
   "description": "",
   "features": {
+    "observation.image": {
+      "_type": "VideoFrame"
+    },
     "observation.state": {
       "feature": {
         "dtype": "float32",
@@ -57,9 +60,6 @@
     "index": {
       "dtype": "int64",
       "_type": "Value"
-    },
-    "observation.image": {
-      "_type": "Image"
     }
   },
   "homepage": "",

tests/data/lerobot/umi_cup_in_the_wild/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "fd95ee932cb1fce2",
+  "_fingerprint": "c8b78ec1bbf7a579",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/data/lerobot/xarm_lift_medium/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 15
+    "fps": 15,
+    "video": 1
 }

tests/data/lerobot/xarm_lift_medium/train/dataset_info.json

@@ -3,7 +3,7 @@
   "description": "",
   "features": {
     "observation.image": {
-      "_type": "Image"
+      "_type": "VideoFrame"
     },
     "observation.state": {
       "feature": {

tests/data/lerobot/xarm_lift_medium/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "cc6afdfcdd6f63ab",
+  "_fingerprint": "720072274a55db4d",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/data/lerobot/xarm_lift_medium_replay/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 15
+    "fps": 15,
+    "video": 1
 }

tests/data/lerobot/xarm_lift_medium_replay/train/dataset_info.json

@@ -3,7 +3,7 @@
   "description": "",
   "features": {
     "observation.image": {
-      "_type": "Image"
+      "_type": "VideoFrame"
     },
     "observation.state": {
       "feature": {
@@ -18,7 +18,7 @@
         "dtype": "float32",
         "_type": "Value"
       },
-      "length": 4,
+      "length": 3,
       "_type": "Sequence"
     },
     "episode_index": {

tests/data/lerobot/xarm_lift_medium_replay/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "9f8e1a8c1845df55",
+  "_fingerprint": "9f3d8cbb0b2e74a2",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/data/lerobot/xarm_push_medium/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 15
+    "fps": 15,
+    "video": 1
 }

tests/data/lerobot/xarm_push_medium/train/dataset_info.json

@@ -3,7 +3,7 @@
   "description": "",
   "features": {
     "observation.image": {
-      "_type": "Image"
+      "_type": "VideoFrame"
     },
     "observation.state": {
       "feature": {

tests/data/lerobot/xarm_push_medium/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "c900258061dd0b3f",
+  "_fingerprint": "9f3d8cbb0b2e74a2",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/data/lerobot/xarm_push_medium_replay/meta_data/info.json

@@ -1,3 +1,4 @@
 {
-    "fps": 15
+    "fps": 15,
+    "video": 1
 }

tests/data/lerobot/xarm_push_medium_replay/train/dataset_info.json

@@ -3,7 +3,7 @@
   "description": "",
   "features": {
     "observation.image": {
-      "_type": "Image"
+      "_type": "VideoFrame"
     },
     "observation.state": {
       "feature": {

tests/data/lerobot/xarm_push_medium_replay/train/state.json

@@ -4,10 +4,10 @@
       "filename": "data-00000-of-00001.arrow"
     }
   ],
-  "_fingerprint": "e51c80a33c7688c0",
+  "_fingerprint": "9f3d8cbb0b2e74a2",
   "_format_columns": null,
   "_format_kwargs": {},
-  "_format_type": "torch",
+  "_format_type": null,
   "_output_all_columns": false,
   "_split": null
 }

tests/scripts/save_dataset_to_safetensors.py

@@ -8,7 +8,7 @@ If you know that your change will break backward compatibility, you should write
 doesnt need to be merged into the `main` branch. Then you need to run this script and update the tests artifacts.
 
 Example usage:
-    `python tests/script/save_dataset_to_safetensors.py`
+    `python tests/scripts/save_dataset_to_safetensors.py`
 """
 
 import os