From 0a67d46b0d6acd07c20cd44e411dc435c8d7ae0a Mon Sep 17 00:00:00 2001 From: Ury Zhilinsky Date: Thu, 13 Feb 2025 19:12:23 -0800 Subject: [PATCH] Add a few extra connections to prevent exceeding the pool size --- src/openpi/shared/download.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/openpi/shared/download.py b/src/openpi/shared/download.py index d919477..11d09c4 100644 --- a/src/openpi/shared/download.py +++ b/src/openpi/shared/download.py @@ -236,7 +236,8 @@ def _download_boto3( def _get_s3_transfer_manager( session: boto3.Session, workers: int, botocore_config: botocore.config.Config | None = None ) -> s3_transfer.TransferManager: - config = botocore.config.Config(max_pool_connections=workers) + # Add a few extra connections to prevent exceeding the pool size. + config = botocore.config.Config(max_pool_connections=workers + 2) if botocore_config is not None: config = config.merge(botocore_config) s3client = session.client("s3", config=config)