aggregate works

2025-02-22 15:33:47 +00:00
parent 39ad2d16d4
commit ff0029f84b
2 changed files with 164 additions and 2 deletions
--- a/examples/port_datasets/openx_rlds_datatrove.py
+++ b/examples/port_datasets/openx_rlds_datatrove.py
@@ -69,12 +69,12 @@ def main(slurm=True):
            "job_name": port_job_name,
            "tasks": 2048,
            # "workers": 20,  # 8 * 16,
-            "workers": 1,  # 8 * 16,
+            "workers": 20,  # 8 * 16,
            "time": "08:00:00",
            "partition": "hopper-cpu",
            "cpus_per_task": 24,
            "mem_per_cpu_gb": 2,
-            # "max_array_launch_parallel": True,
+            "max_array_launch_parallel": True,
        }
    else:
        executor_class = LocalPipelineExecutor