WIP v21 to v30

2025-03-31 07:38:01 +00:00
parent 65738f0a80
commit 53ecec5fb2
3 changed files with 141 additions and 1 deletions
--- a/examples/port_datasets/droid_rlds/slurm_upload.py
+++ b/examples/port_datasets/droid_rlds/slurm_upload.py
@@ -117,6 +117,7 @@ class UploadDataset(PipelineStep):
                        commit_message=f"DataTrove upload ({len(chunk)} files)",
                        revision=self.branch,
                    )
+                    # TODO: every 100 chunks super_squach_commits()
                    logging.info("create_commit completed!")
                    break
                except HfHubHTTPError as e: