Fix provisioning of S3 credentials.

This commit is contained in:
Ury Zhilinsky
2024-12-24 19:37:02 -08:00
committed by uzhilinsky
parent 5eed870a64
commit 098d387c8e

View File

@@ -75,9 +75,10 @@ def maybe_download(url: str, **kwargs) -> pathlib.Path:
if _is_openpi_url(url):
# Download with openpi credentials.
# TODO(ury): Remove once the bucket becomes public.
boto_session = boto3.Session(
boto_session = boto3.session.Session(
aws_access_key_id="AKIA4MTWIIQIZBO44C62",
aws_secret_access_key="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",
region_name="us-east-1",
)
_download_boto3(url, scratch_path, boto_session=boto_session)
elif url.startswith("s3://"):
@@ -136,13 +137,14 @@ def _download_boto3(
return bucket_name, prefix
bucket_name, prefix = validate_and_parse_url(url)
s3api = boto3.resource("s3")
session = boto_session or boto3.Session()
s3api = session.resource("s3")
bucket = s3api.Bucket(bucket_name)
objects = list(bucket.objects.filter(Prefix=prefix))
total_size = sum(obj.size for obj in objects)
session = boto_session or boto3.Session()
s3t = _get_s3_transfer_manager(session, workers)
def transfer(