import os.path import boto3 from botocore.config import Config class S3Loader: def __init__(self, bucket_name): self.bucket_name = bucket_name endpoint = None if "AWS_ENDPOINT_S3" in os.environ: s3_config = Config( s3={'addressing_style': 'virtual', 'payload_signing_enabled': False}, # Disable chunked encoding which causes issues with OSS signature_version='s3v4' ) endpoint = os.environ["AWS_ENDPOINT_S3"] self.s3 = boto3.resource('s3', endpoint_url=endpoint, config=s3_config) else: self.s3 = boto3.resource('s3') def get_file(self, filename): key = filename return self.s3.Object(self.bucket_name, key).get()['Body'].read() def get_file_list(self, path): bucket = self.s3.Bucket(self.bucket_name) prefix = path return [obj.key for obj in bucket.objects.filter(Prefix=prefix)] def file_exists(self, path): bucket = self.s3.Bucket(self.bucket_name) prefix = path return any([obj.key == path for obj in bucket.objects.filter(Prefix=prefix)]) def save_file(self, filename, content): raise Exception("s3 loader does not support save file") def upload_file(self, filename, s3_key): key = s3_key self.s3.Object(self.bucket_name, key).upload_file(filename)