From fc4d9a5b0c749142b51532f2a46dac9c6776f3d4 Mon Sep 17 00:00:00 2001 From: trhr Date: Thu, 20 Feb 2020 23:44:18 -0600 Subject: [PATCH] Boto3 / AWS contrib plugin --- evennia/contrib/aws-s3-cdn.py | 847 ++++++++++++++++++++++++++++++++++ 1 file changed, 847 insertions(+) create mode 100644 evennia/contrib/aws-s3-cdn.py diff --git a/evennia/contrib/aws-s3-cdn.py b/evennia/contrib/aws-s3-cdn.py new file mode 100644 index 0000000000..dbbdf59063 --- /dev/null +++ b/evennia/contrib/aws-s3-cdn.py @@ -0,0 +1,847 @@ +""" +This plugin migrates the Web-based portion of Evennia, +namely images, javascript, and other items located +inside staticfiles into Amazon AWS (S3) for hosting. + +INSTALLATION: + +1) If you don't have an AWS S3 account, you should create one now. + +Credentials required are an AWS IAM Access Key and Secret Keys, +which can be generated/found in the AWS Console. + +Example IAM Control Policy Permissions, if desired: + +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "evennia", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObjectAcl", + "s3:GetObject", + "s3:ListBucket", + "s3:DeleteObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::YOUR_BUCKET_NAME/*", + "arn:aws:s3:::YOUR_BUCKET_NAME" + ] + } + ], + [ + { + "Sid":"evennia", + "Effect":"Allow", + "Action":[ + "s3:CreateBucket", + ], + "Resource":[ + "arn:aws:s3:::*" + ] + } + ] +} + +Advanced Users: The second IAM statement, CreateBucket, is only needed +for initial installation. You can remove it later, or you can +create the bucket and set the ACL yourself before you continue. + +2) This package requires the dependency "boto3," the official +AWS python package. You can install it with 'pip install boto3' +while inside your evennia virtual environment (or, simply +in your shell if you don't use a virtual environment). + +3) Customize the variables defined below in secret_settings.py, +then run 'evennia stop', 'evennia start', 'evennia collectstatic' + +AWS_ACCESS_KEY_ID = 'EUHUB20BU08AEU7' # CHANGE ME! +AWS_SECRET_ACCESS_KEY = 'a/uoexauodabuq4j;kmw;kvka0d2' # CHANGE ME! +AWS_STORAGE_BUCKET_NAME = 'mygame-evennia' # CHANGE ME! +AWS_S3_REGION_NAME = 'us-east-1' # N. Virginia +AWS_S3_OBJECT_PARAMETERS = { 'Expires': 'Thu, 31 Dec 2099 20:00:00 GMT', 'CacheControl': 'max-age=94608000', } +AWS_DEFAULT_ACL = 'public-read' +AWS_S3_CUSTOM_DOMAIN = '%s.s3.amazonaws.com' % settings.AWS_BUCKET_NAME +AWS_AUTO_CREATE_BUCKET = True +STATICFILES_STORAGE = 'evennia.contrib.aws-s3-cdn.S3Boto3Storage' +You may also store these as environment variables of the same name. + +UNINSTALLATION: + +If you haven't made changes to your static files (uploaded images, etc), +you can simply remove the lines you added to secret_settings.py. If you +have made changes and want to install at a later date, you can export +your files from your S3 bucket and put them in /static/ in the evennia +directory. + +LICENSE: + +aws-s3-cdn contrib is (c) 2020, trhr and released under BSD 3-Clause +License except where this license conflicts with the Evennia license. +Thank you to github.com/jschneier for contributions on django/boto3 classes. + +BSD 3-Clause License + +Copyright (c) 2008 - 2020, See AUTHORS file. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" + +from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation, SuspiciousFileOperation + +try: + from django.conf import settings as ev_settings + if not ev_settings.AWS_ACCESS_KEY_ID or not ev_settings.AWS_SECRET_ACCESS_KEY or not ev_settings.AWS_STORAGE_BUCKET_NAME or not ev_settings.AWS_S3_REGION_NAME: + raise ImproperlyConfigured("You must add AWS-specific settings to mygame/server/conf/secret_settings.py to use this plugin.") + + if 'mygame-evennia' == ev_settings.AWS_STORAGE_BUCKET_NAME: + raise ImproperlyConfigured("You must customize your AWS_STORAGE_BUCKET_NAME in mygame/server/conf/secret_settings.py; it must be unique among ALL other S3 users") + +except Exception as e: + print(e) + +import io +import mimetypes +import os +import posixpath +import threading +import warnings +from gzip import GzipFile +from tempfile import SpooledTemporaryFile +from django.core.files.base import File +from django.core.files.storage import Storage +from django.utils.deconstruct import deconstructible +from django.utils.encoding import (filepath_to_uri, force_bytes, force_text, smart_text) +from django.utils.timezone import is_naive, make_naive + +try: + from django.utils.six.moves.urllib import parse as urlparse +except ImportError: + from urllib import parse as urlparse + +try: + import boto3.session + from boto3 import __version__ as boto3_version + from botocore.client import Config + from botocore.exceptions import ClientError +except ImportError as e: + raise ImproperlyConfigured("Could not load Boto3's S3 bindings. %s Did you run 'pip install boto3?'" % e) + +boto3_version_info = tuple([int(i) for i in boto3_version.split('.')]) + + +def setting(name, default=None): + """ + Helper function to get a Django setting by name. If setting doesn't exists + it will return a default. + :param name: Name of setting + :type name: str + :param default: Value if setting is unfound + :returns: Setting's value + """ + return getattr(ev_settings, name, default) + + +def clean_name(name): + """ + Cleans the name so that Windows style paths work + """ + # Normalize Windows style paths + clean_name = posixpath.normpath(name).replace('\\', '/') + + # os.path.normpath() can strip trailing slashes so we implement + # a workaround here. + if name.endswith('/') and not clean_name.endswith('/'): + # Add a trailing slash as it was stripped. + clean_name = clean_name + '/' + + # Given an empty string, os.path.normpath() will return ., which we don't want + if clean_name == '.': + clean_name = '' + + return clean_name + + +def safe_join(base, *paths): + """ + A version of django.utils._os.safe_join for S3 paths. + Joins one or more path components to the base path component + intelligently. Returns a normalized version of the final path. + The final path must be located inside of the base path component + (otherwise a ValueError is raised). + Paths outside the base path indicate a possible security + sensitive operation. + """ + base_path = force_text(base) + base_path = base_path.rstrip('/') + paths = [force_text(p) for p in paths] + + final_path = base_path + '/' + for path in paths: + _final_path = posixpath.normpath(posixpath.join(final_path, path)) + # posixpath.normpath() strips the trailing /. Add it back. + if path.endswith('/') or _final_path + '/' == final_path: + _final_path += '/' + final_path = _final_path + if final_path == base_path: + final_path += '/' + + # Ensure final_path starts with base_path and that the next character after + # the base path is /. + base_path_len = len(base_path) + if (not final_path.startswith(base_path) or final_path[base_path_len] != '/'): + raise ValueError('the joined path is located outside of the base path' + ' component') + + return final_path.lstrip('/') + + +def check_location(storage): + if storage.location.startswith('/'): + correct = storage.location.lstrip('/') + raise ImproperlyConfigured( + "{}.location cannot begin with a leading slash. Found '{}'. Use '{}' instead.".format( + storage.__class__.__name__, + storage.location, + correct, + ) + ) + + +def lookup_env(names): + """ + Look up for names in environment. Returns the first element + found. + """ + for name in names: + value = os.environ.get(name) + if value: + return value + + +def get_available_overwrite_name(name, max_length): + if max_length is None or len(name) <= max_length: + return name + + # Adapted from Django + dir_name, file_name = os.path.split(name) + file_root, file_ext = os.path.splitext(file_name) + truncation = len(name) - max_length + + file_root = file_root[:-truncation] + if not file_root: + raise SuspiciousFileOperation( + 'aws-s3-cdn tried to truncate away entire filename "%s". ' + 'Please make sure that the corresponding file field ' + 'allows sufficient "max_length".' % name + ) + return os.path.join(dir_name, "{}{}".format(file_root, file_ext)) + + +@deconstructible +class S3Boto3StorageFile(File): + + """ + The default file object used by the S3Boto3Storage backend. + This file implements file streaming using boto's multipart + uploading functionality. The file can be opened in read or + write mode. + This class extends Django's File class. However, the contained + data is only the data contained in the current buffer. So you + should not access the contained file object directly. You should + access the data via this class. + Warning: This file *must* be closed using the close() method in + order to properly write the file to S3. Be sure to close the file + in your application. + """ + buffer_size = setting('AWS_S3_FILE_BUFFER_SIZE', 5242880) + + def __init__(self, name, mode, storage, buffer_size=None): + if 'r' in mode and 'w' in mode: + raise ValueError("Can't combine 'r' and 'w' in mode.") + self._storage = storage + self.name = name[len(self._storage.location):].lstrip('/') + self._mode = mode + self._force_mode = (lambda b: b) if 'b' in mode else force_text + self.obj = storage.bucket.Object(storage._encode_name(name)) + if 'w' not in mode: + # Force early RAII-style exception if object does not exist + self.obj.load() + self._is_dirty = False + self._raw_bytes_written = 0 + self._file = None + self._multipart = None + # 5 MB is the minimum part size (if there is more than one part). + # Amazon allows up to 10,000 parts. The default supports uploads + # up to roughly 50 GB. Increase the part size to accommodate + # for files larger than this. + if buffer_size is not None: + self.buffer_size = buffer_size + self._write_counter = 0 + + @property + def size(self): + return self.obj.content_length + + def _get_file(self): + if self._file is None: + self._file = SpooledTemporaryFile( + max_size=self._storage.max_memory_size, + suffix=".S3Boto3StorageFile", + dir=setting("FILE_UPLOAD_TEMP_DIR") + ) + if 'r' in self._mode: + self._is_dirty = False + self.obj.download_fileobj(self._file) + self._file.seek(0) + if self._storage.gzip and self.obj.content_encoding == 'gzip': + self._file = GzipFile(mode=self._mode, fileobj=self._file, mtime=0.0) + return self._file + + def _set_file(self, value): + self._file = value + + file = property(_get_file, _set_file) + + def read(self, *args, **kwargs): + if 'r' not in self._mode: + raise AttributeError("File was not opened in read mode.") + return self._force_mode(super(S3Boto3StorageFile, self).read(*args, **kwargs)) + + def readline(self, *args, **kwargs): + if 'r' not in self._mode: + raise AttributeError("File was not opened in read mode.") + return self._force_mode(super(S3Boto3StorageFile, self).readline(*args, **kwargs)) + + def write(self, content): + if 'w' not in self._mode: + raise AttributeError("File was not opened in write mode.") + self._is_dirty = True + if self._multipart is None: + self._multipart = self.obj.initiate_multipart_upload( + **self._storage._get_write_parameters(self.obj.key) + ) + if self.buffer_size <= self._buffer_file_size: + self._flush_write_buffer() + bstr = force_bytes(content) + self._raw_bytes_written += len(bstr) + return super(S3Boto3StorageFile, self).write(bstr) + + @property + def _buffer_file_size(self): + pos = self.file.tell() + self.file.seek(0, os.SEEK_END) + length = self.file.tell() + self.file.seek(pos) + return length + + def _flush_write_buffer(self): + """ + Flushes the write buffer. + """ + if self._buffer_file_size: + self._write_counter += 1 + self.file.seek(0) + part = self._multipart.Part(self._write_counter) + part.upload(Body=self.file.read()) + self.file.seek(0) + self.file.truncate() + + def _create_empty_on_close(self): + """ + Attempt to create an empty file for this key when this File is closed if no bytes + have been written and no object already exists on S3 for this key. + This behavior is meant to mimic the behavior of Django's builtin FileSystemStorage, + where files are always created after they are opened in write mode: + f = storage.open("file.txt", mode="w") + f.close() + """ + assert "w" in self._mode + assert self._raw_bytes_written == 0 + + try: + # Check if the object exists on the server; if so, don't do anything + self.obj.load() + except ClientError as err: + if err.response["ResponseMetadata"]["HTTPStatusCode"] == 404: + self.obj.put( + Body=b"", **self._storage._get_write_parameters(self.obj.key) + ) + else: + raise + + def close(self): + if self._is_dirty: + self._flush_write_buffer() + # TODO: Possibly cache the part ids as they're being uploaded + # instead of requesting parts from server. For now, emulating + # s3boto's behavior. + parts = [{'ETag': part.e_tag, 'PartNumber': part.part_number} + for part in self._multipart.parts.all()] + self._multipart.complete( + MultipartUpload={'Parts': parts}) + else: + if self._multipart is not None: + self._multipart.abort() + if 'w' in self._mode and self._raw_bytes_written == 0: + self._create_empty_on_close() + if self._file is not None: + self._file.close() + self._file = None + + +@deconstructible +class S3Boto3Storage(Storage): + """ + Amazon Simple Storage Service using Boto3 + This storage backend supports opening files in read or write + mode and supports streaming(buffering) data in chunks to S3 + when writing. + """ + default_content_type = 'application/octet-stream' + # If config provided in init, signature_version and addressing_style settings/args are ignored. + config = None + + # used for looking up the access and secret key from env vars + access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID'] + secret_key_names = ['AWS_S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY'] + security_token_names = ['AWS_SESSION_TOKEN', 'AWS_SECURITY_TOKEN'] + security_token = None + + access_key = setting('AWS_S3_ACCESS_KEY_ID', setting('AWS_ACCESS_KEY_ID')) + secret_key = setting('AWS_S3_SECRET_ACCESS_KEY', setting('AWS_SECRET_ACCESS_KEY')) + file_overwrite = setting('AWS_S3_FILE_OVERWRITE', True) + object_parameters = setting('AWS_S3_OBJECT_PARAMETERS', {}) + bucket_name = setting('AWS_STORAGE_BUCKET_NAME') + auto_create_bucket = setting('AWS_AUTO_CREATE_BUCKET', False) + default_acl = setting('AWS_DEFAULT_ACL', 'public-read') + bucket_acl = setting('AWS_BUCKET_ACL', default_acl) + querystring_auth = setting('AWS_QUERYSTRING_AUTH', True) + querystring_expire = setting('AWS_QUERYSTRING_EXPIRE', 3600) + signature_version = setting('AWS_S3_SIGNATURE_VERSION') + reduced_redundancy = setting('AWS_REDUCED_REDUNDANCY', False) + location = setting('AWS_LOCATION', '') + encryption = setting('AWS_S3_ENCRYPTION', False) + custom_domain = setting('AWS_S3_CUSTOM_DOMAIN') + addressing_style = setting('AWS_S3_ADDRESSING_STYLE') + secure_urls = setting('AWS_S3_SECURE_URLS', True) + file_name_charset = setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8') + gzip = setting('AWS_IS_GZIPPED', False) + preload_metadata = setting('AWS_PRELOAD_METADATA', False) + gzip_content_types = setting('GZIP_CONTENT_TYPES', ( + 'text/css', + 'text/javascript', + 'application/javascript', + 'application/x-javascript', + 'image/svg+xml', + )) + url_protocol = setting('AWS_S3_URL_PROTOCOL', 'http:') + endpoint_url = setting('AWS_S3_ENDPOINT_URL') + proxies = setting('AWS_S3_PROXIES') + region_name = setting('AWS_S3_REGION_NAME') + use_ssl = setting('AWS_S3_USE_SSL', True) + verify = setting('AWS_S3_VERIFY', None) + max_memory_size = setting('AWS_S3_MAX_MEMORY_SIZE', 0) + + def __init__(self, acl=None, bucket=None, **settings): + # check if some of the settings we've provided as class attributes + # need to be overwritten with values passed in here + for name, value in settings.items(): + if hasattr(self, name): + setattr(self, name, value) + + check_location(self) + + # Backward-compatibility: given the anteriority of the SECURE_URL setting + # we fall back to https if specified in order to avoid the construction + # of unsecure urls. + if self.secure_urls: + self.url_protocol = 'https:' + + self._entries = {} + self._bucket = None + self._connections = threading.local() + + self.access_key, self.secret_key = self._get_access_keys() + self.security_token = self._get_security_token() + + if not self.config: + kwargs = dict( + s3={'addressing_style': self.addressing_style}, + signature_version=self.signature_version, + ) + + if boto3_version_info >= (1, 4, 4): + kwargs['proxies'] = self.proxies + else: + warnings.warn( + "In version 1.10 of django-storages the minimum required version of " + "boto3 will be 1.4.4. You have %s " % boto3_version_info + ) + self.config = Config(**kwargs) + + + def __getstate__(self): + state = self.__dict__.copy() + state.pop('_connections', None) + state.pop('_bucket', None) + return state + + def __setstate__(self, state): + state['_connections'] = threading.local() + state['_bucket'] = None + self.__dict__ = state + + @property + def connection(self): + connection = getattr(self._connections, 'connection', None) + if connection is None: + session = boto3.session.Session() + self._connections.connection = session.resource( + 's3', + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.security_token, + region_name=self.region_name, + use_ssl=self.use_ssl, + endpoint_url=self.endpoint_url, + config=self.config, + verify=self.verify, + ) + return self._connections.connection + + @property + def bucket(self): + """ + Get the current bucket. If there is no current bucket object + create it. + """ + if self._bucket is None: + self._bucket = self._get_or_create_bucket(self.bucket_name) + return self._bucket + + @property + def entries(self): + """ + Get the locally cached files for the bucket. + """ + if self.preload_metadata and not self._entries: + self._entries = { + self._decode_name(entry.key): entry + for entry in self.bucket.objects.filter(Prefix=self.location) + } + return self._entries + + def _get_access_keys(self): + """ + Gets the access keys to use when accessing S3. If none is + provided in the settings then get them from the environment + variables. + """ + access_key = self.access_key or lookup_env(S3Boto3Storage.access_key_names) + secret_key = self.secret_key or lookup_env(S3Boto3Storage.secret_key_names) + return access_key, secret_key + + def _get_security_token(self): + """ + Gets the security token to use when accessing S3. Get it from + the environment variables. + """ + security_token = self.security_token or lookup_env(S3Boto3Storage.security_token_names) + return security_token + + def _get_or_create_bucket(self, name): + """ + Retrieves a bucket if it exists, otherwise creates it. + """ + bucket = self.connection.Bucket(name) + if self.auto_create_bucket: + try: + # Directly call head_bucket instead of bucket.load() because head_bucket() + # fails on wrong region, while bucket.load() does not. + bucket.meta.client.head_bucket(Bucket=name) + except ClientError as err: + if err.response['ResponseMetadata']['HTTPStatusCode'] == 301: + raise ImproperlyConfigured("Bucket %s exists, but in a different " + "region than we are connecting to. Set " + "the region to connect to by setting " + "AWS_S3_REGION_NAME to the correct region." % name) + + elif err.response['ResponseMetadata']['HTTPStatusCode'] == 404: + # Notes: When using the us-east-1 Standard endpoint, you can create + # buckets in other regions. The same is not true when hitting region specific + # endpoints. However, when you create the bucket not in the same region, the + # connection will fail all future requests to the Bucket after the creation + # (301 Moved Permanently). + # + # For simplicity, we enforce in S3Boto3Storage that any auto-created + # bucket must match the region that the connection is for. + # + # Also note that Amazon specifically disallows "us-east-1" when passing bucket + # region names; LocationConstraint *must* be blank to create in US Standard. + + if self.bucket_acl: + bucket_params = {'ACL': self.bucket_acl} + else: + bucket_params = {} + region_name = self.connection.meta.client.meta.region_name + if region_name != 'us-east-1': + bucket_params['CreateBucketConfiguration'] = { + 'LocationConstraint': region_name} + bucket.create(**bucket_params) + else: + raise + return bucket + + def _clean_name(self, name): + """ + Cleans the name so that Windows style paths work + """ + # Normalize Windows style paths + clean_name = posixpath.normpath(name).replace('\\', '/') + + # os.path.normpath() can strip trailing slashes so we implement + # a workaround here. + if name.endswith('/') and not clean_name.endswith('/'): + # Add a trailing slash as it was stripped. + clean_name += '/' + return clean_name + + def _normalize_name(self, name): + """ + Normalizes the name so that paths like /path/to/ignored/../something.txt + work. We check to make sure that the path pointed to is not outside + the directory specified by the LOCATION setting. + """ + try: + return safe_join(self.location, name) + except ValueError: + raise SuspiciousOperation("Attempted access to '%s' denied." % + name) + + def _encode_name(self, name): + return smart_text(name, encoding=self.file_name_charset) + + def _decode_name(self, name): + return force_text(name, encoding=self.file_name_charset) + + def _compress_content(self, content): + """Gzip a given string content.""" + content.seek(0) + zbuf = io.BytesIO() + # The GZIP header has a modification time attribute (see http://www.zlib.org/rfc-gzip.html) + # This means each time a file is compressed it changes even if the other contents don't change + # For S3 this defeats detection of changes using MD5 sums on gzipped files + # Fixing the mtime at 0.0 at compression time avoids this problem + zfile = GzipFile(mode='wb', fileobj=zbuf, mtime=0.0) + try: + zfile.write(force_bytes(content.read())) + finally: + zfile.close() + zbuf.seek(0) + # Boto 2 returned the InMemoryUploadedFile with the file pointer replaced, + # but Boto 3 seems to have issues with that. No need for fp.name in Boto3 + # so just returning the BytesIO directly + return zbuf + + def _open(self, name, mode='rb'): + name = self._normalize_name(self._clean_name(name)) + try: + f = S3Boto3StorageFile(name, mode, self) + except ClientError as err: + if err.response['ResponseMetadata']['HTTPStatusCode'] == 404: + raise IOError('File does not exist: %s' % name) + raise # Let it bubble up if it was some other error + return f + + def _save(self, name, content): + cleaned_name = self._clean_name(name) + name = self._normalize_name(cleaned_name) + params = self._get_write_parameters(name, content) + + if (self.gzip and + params['ContentType'] in self.gzip_content_types and + 'ContentEncoding' not in params): + content = self._compress_content(content) + params['ContentEncoding'] = 'gzip' + + encoded_name = self._encode_name(name) + obj = self.bucket.Object(encoded_name) + if self.preload_metadata: + self._entries[encoded_name] = obj + + content.seek(0, os.SEEK_SET) + obj.upload_fileobj(content, ExtraArgs=params) + return cleaned_name + + def delete(self, name): + name = self._normalize_name(self._clean_name(name)) + self.bucket.Object(self._encode_name(name)).delete() + + if name in self._entries: + del self._entries[name] + + def exists(self, name): + name = self._normalize_name(self._clean_name(name)) + if self.entries: + return name in self.entries + try: + self.connection.meta.client.head_object(Bucket=self.bucket_name, Key=name) + return True + except ClientError: + return False + + def listdir(self, name): + path = self._normalize_name(self._clean_name(name)) + # The path needs to end with a slash, but if the root is empty, leave + # it. + if path and not path.endswith('/'): + path += '/' + + directories = [] + files = [] + paginator = self.connection.meta.client.get_paginator('list_objects') + pages = paginator.paginate(Bucket=self.bucket_name, Delimiter='/', Prefix=path) + for page in pages: + for entry in page.get('CommonPrefixes', ()): + directories.append(posixpath.relpath(entry['Prefix'], path)) + for entry in page.get('Contents', ()): + files.append(posixpath.relpath(entry['Key'], path)) + return directories, files + + def size(self, name): + name = self._normalize_name(self._clean_name(name)) + if self.entries: + entry = self.entries.get(name) + if entry: + return entry.size if hasattr(entry, 'size') else entry.content_length + return 0 + return self.bucket.Object(self._encode_name(name)).content_length + + def _get_write_parameters(self, name, content=None): + params = {} + + if self.encryption: + params['ServerSideEncryption'] = 'AES256' + if self.reduced_redundancy: + params['StorageClass'] = 'REDUCED_REDUNDANCY' + if self.default_acl: + params['ACL'] = self.default_acl + + _type, encoding = mimetypes.guess_type(name) + content_type = getattr(content, 'content_type', None) + content_type = content_type or _type or self.default_content_type + + params['ContentType'] = content_type + if encoding: + params['ContentEncoding'] = encoding + + params.update(self.get_object_parameters(name)) + return params + + def get_object_parameters(self, name): + """ + Returns a dictionary that is passed to file upload. Override this + method to adjust this on a per-object basis to set e.g ContentDisposition. + By default, returns the value of AWS_S3_OBJECT_PARAMETERS. + Setting ContentEncoding will prevent objects from being automatically gzipped. + """ + return self.object_parameters.copy() + + def get_modified_time(self, name): + """ + Returns an (aware) datetime object containing the last modified time if + USE_TZ is True, otherwise returns a naive datetime in the local timezone. + """ + name = self._normalize_name(self._clean_name(name)) + entry = self.entries.get(name) + # only call self.bucket.Object() if the key is not found + # in the preloaded metadata. + if entry is None: + entry = self.bucket.Object(self._encode_name(name)) + if setting('USE_TZ'): + # boto3 returns TZ aware timestamps + return entry.last_modified + else: + return make_naive(entry.last_modified) + + def modified_time(self, name): + """Returns a naive datetime object containing the last modified time.""" + # If USE_TZ=False then get_modified_time will return a naive datetime + # so we just return that, else we have to localize and strip the tz + mtime = self.get_modified_time(name) + return mtime if is_naive(mtime) else make_naive(mtime) + + def _strip_signing_parameters(self, url): + # Boto3 does not currently support generating URLs that are unsigned. Instead we + # take the signed URLs and strip any querystring params related to signing and expiration. + # Note that this may end up with URLs that are still invalid, especially if params are + # passed in that only work with signed URLs, e.g. response header params. + # The code attempts to strip all query parameters that match names of known parameters + # from v2 and v4 signatures, regardless of the actual signature version used. + split_url = urlparse.urlsplit(url) + qs = urlparse.parse_qsl(split_url.query, keep_blank_values=True) + blacklist = { + 'x-amz-algorithm', 'x-amz-credential', 'x-amz-date', + 'x-amz-expires', 'x-amz-signedheaders', 'x-amz-signature', + 'x-amz-security-token', 'awsaccesskeyid', 'expires', 'signature', + } + filtered_qs = ((key, val) for key, val in qs if key.lower() not in blacklist) + # Note: Parameters that did not have a value in the original query string will have + # an '=' sign appended to it, e.g ?foo&bar becomes ?foo=&bar= + joined_qs = ('='.join(keyval) for keyval in filtered_qs) + split_url = split_url._replace(query="&".join(joined_qs)) + return split_url.geturl() + + def url(self, name, parameters=None, expire=None): + # Preserve the trailing slash after normalizing the path. + name = self._normalize_name(self._clean_name(name)) + if self.custom_domain: + return "{}//{}/{}".format(self.url_protocol, + self.custom_domain, filepath_to_uri(name)) + if expire is None: + expire = self.querystring_expire + + params = parameters.copy() if parameters else {} + params['Bucket'] = self.bucket.name + params['Key'] = self._encode_name(name) + url = self.bucket.meta.client.generate_presigned_url('get_object', Params=params, + ExpiresIn=expire) + if self.querystring_auth: + return url + return self._strip_signing_parameters(url) + + def get_available_name(self, name, max_length=None): + """Overwrite existing file with the same name.""" + name = self._clean_name(name) + if self.file_overwrite: + return get_available_overwrite_name(name, max_length) + return super(S3Boto3Storage, self).get_available_name(name, max_length)