file system: Added support for file system

Author: eadwinCode

ellar/core/conf/app_settings_models.py

@@ -1,4 +1,3 @@
-import sys
 import typing as t
 
 from ellar.common.constants import (
@@ -19,10 +18,6 @@
 
 from .mixins import ConfigDefaultTypesMixin
 
-if sys.version_info >= (3, 8):  # pragma: no cover
-    from typing import Literal
-else:  # pragma: no cover
-    from typing_extensions import Literal
 if t.TYPE_CHECKING:  # pragma: no cover
     from ellar.app.main import App
 
@@ -127,7 +122,7 @@ class ConfigValidationSchema(Serializer, ConfigDefaultTypesMixin):
     SESSION_COOKIE_PATH: str = "/"
     SESSION_COOKIE_HTTPONLY: bool = True
     SESSION_COOKIE_SECURE: bool = False
-    SESSION_COOKIE_SAME_SITE: Literal["lax", "strict", "none"] = "lax"
+    SESSION_COOKIE_SAME_SITE: t.Literal["lax", "strict", "none"] = "lax"
     SESSION_COOKIE_MAX_AGE: t.Optional[int] = 14 * 24 * 60 * 60  # 14 days, in seconds
 
     @field_validator("MIDDLEWARE", mode="before")

ellar/core/files/__init__.py

@@ -0,0 +1,12 @@
+from .base import BaseStorage
+
+# from .factory import get_file_storage
+from .interface import Storage
+from .local import FileSystemStorage
+
+__all__ = [
+    "Storage",
+    "BaseStorage",
+    "FileSystemStorage",
+    # 'get_file_storage',
+]

ellar/core/files/storages/__init__.py

@@ -0,0 +1,141 @@
+try:
+    import boto3
+except ImportError as im_ex:  # pragma: no cover
+    raise RuntimeError(
+        "boto3 must be installed to use the 'S3AWSFileStorage' class."
+    ) from im_ex
+import typing as t
+import urllib.parse
+from io import BytesIO
+
+from .base import BaseStorage
+
+
+class S3AWSFileStorage(BaseStorage):
+    def service_name(self) -> str:
+        return "s3_bucket"
+
+    def __init__(
+        self,
+        bucket: str,
+        access_key: str,
+        secret_key: str,
+        region: str,
+        max_age: int = 60 * 60 * 24 * 365,
+        prefix: t.Optional[str] = None,
+        endpoint_url: t.Optional[str] = None,
+        acl: str = "private",
+        enable_cache_control: bool = False,
+        public_link_expiration: int = 3600,
+    ) -> None:
+        self.bucket = self.get_aws_bucket(
+            bucket=bucket,
+            secret_key=secret_key,
+            access_key=access_key,
+            region=region,
+            endpoint_url=endpoint_url,
+        )
+        self._max_age = max_age
+        self._prefix = prefix
+        self._acl = acl
+        self._enable_cache_control = enable_cache_control
+        self._public_link_expiration = public_link_expiration
+
+    def get_aws_bucket(
+        self,
+        bucket: str,
+        access_key: str,
+        secret_key: str,
+        region: str,
+        endpoint_url: t.Optional[str] = None,
+    ) -> t.Any:
+        session = boto3.session.Session()
+        config = boto3.session.Config(
+            s3={"addressing_style": "path"}, signature_version="s3v4"
+        )
+
+        s3 = session.resource(
+            "s3",
+            config=config,
+            endpoint_url=endpoint_url,
+            region_name=region,
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key,
+        )
+        return s3.Bucket(bucket)
+
+    def get_s3_path(self, filename: str) -> str:
+        if self._prefix:
+            return "{0}/{1}".format(self._prefix, filename)
+        return filename
+
+    def _upload_file(
+        self, filename: str, data: str, content_type: t.Optional[str], rrs: bool = False
+    ) -> t.Any:
+        put_object_kwargs = {
+            "Key": filename,
+            "Body": data,
+            "ACL": self._acl,
+            "StorageClass": "REDUCED_REDUNDANCY" if rrs else "STANDARD",
+            "ContentType": content_type or "",
+        }
+        if self._enable_cache_control:
+            put_object_kwargs.update(CacheControl="max-age=" + str(self._max_age))
+
+        return self.bucket.put_object(**put_object_kwargs)
+
+    def put(self, filename: str, stream: t.IO) -> int:
+        path = self.get_s3_path(filename)
+        stream.seek(0)
+        data = stream.read()
+
+        content_type = getattr(stream, "content_type", None)
+        rrs = getattr(stream, "reproducible", False)
+
+        self._upload_file(path, data, content_type, rrs=rrs)
+
+        return len(data)
+
+    def delete(self, filename: str) -> None:
+        path = self.get_s3_path(filename)
+        self.bucket.Object(path).delete()
+
+    def open(self, filename: str, mode: str = "rb") -> t.IO:
+        path = self.get_s3_path(filename)
+        obj = self.bucket.Object(path).get()
+
+        return BytesIO(obj["Body"].read())
+
+    def _strip_signing_parameters(self, url: str) -> str:
+        split_url = urllib.parse.urlsplit(url)
+        qs = urllib.parse.parse_qsl(split_url.query, keep_blank_values=True)
+        blacklist = {
+            "x-amz-algorithm",
+            "x-amz-credential",
+            "x-amz-date",
+            "x-amz-expires",
+            "x-amz-signedheaders",
+            "x-amz-signature",
+            "x-amz-security-token",
+            "awsaccesskeyid",
+            "expires",
+            "signature",
+        }
+        filtered_qs = ((key, val) for key, val in qs if key.lower() not in blacklist)
+        joined_qs = ("=".join(keyval) for keyval in filtered_qs)
+        split_url = split_url._replace(query="&".join(joined_qs))
+        return split_url.geturl()
+
+    def locate(self, filename: str) -> str:
+        path = self.get_s3_path(filename)
+        params = {"Key": path, "Bucket": self.bucket.name}
+        expires = self._public_link_expiration
+
+        url = self.bucket.meta.client.generate_presigned_url(
+            "get_object", Params=params, ExpiresIn=expires
+        )
+
+        if self._acl in ["public-read", "public-read-write"]:
+            url = self._strip_signing_parameters(url)
+
+        return url  # type:ignore[no-any-return]

ellar/core/files/storages/aws_s3.py

@@ -0,0 +1,32 @@
+import os
+import pathlib
+from abc import ABC
+
+from .exceptions import SuspiciousFileOperation
+from .interface import Storage
+from .utils import get_valid_filename, validate_file_name
+
+
+class BaseStorage(ABC, Storage):
+    DEFAULT_CHUNK_SIZE = 64 * 2**10
+
+    def validate_file_name(self, filename: str) -> None:
+        """
+        Return an alternative filename, by adding an underscore and a random 7-character alphanumeric string
+        (before the file extension, if one exists) to the filename.
+        """
+        validate_file_name(name=filename)
+
+    def generate_filename(self, filename: str) -> str:
+        """
+        Validate the filename by calling get_valid_name() and return a filename
+        to be passed to the save() method.
+        """
+        filename = str(filename).replace("\\", "/")
+        # `filename` may include a path as returned by FileField.upload_to.
+        dirname, filename = os.path.split(filename)
+        if ".." in pathlib.PurePath(dirname).parts:
+            raise SuspiciousFileOperation(
+                "Detected path traversal attempt in '%s'" % dirname
+            )
+        return os.path.normpath(os.path.join(dirname, get_valid_filename(filename)))

ellar/core/files/storages/base.py

@@ -0,0 +1,8 @@
+class SuspiciousOperation(Exception):
+    """The user did something suspicious"""
+
+
+class SuspiciousFileOperation(SuspiciousOperation):
+    """A Suspicious filesystem operation was attempted"""
+
+    pass

ellar/core/files/storages/exceptions.py

@@ -0,0 +1,44 @@
+import typing as t
+from abc import abstractmethod
+
+
+class Storage:
+    """The abstract base class for all stores."""
+
+    @abstractmethod
+    def service_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def put(self, filename: str, stream: t.IO) -> int:
+        """
+        Puts the file object as the given filename in the store.
+
+        :param filename: target filename.
+        :param stream: source file-like object
+        :return: length of the stored file.
+        """
+
+    @abstractmethod
+    def delete(self, filename: str) -> None:
+        """
+        deletes a given file.
+
+        :param filename: The filename to delete
+        """
+
+    @abstractmethod
+    def open(self, filename: str, mode: str = "rb") -> t.IO:
+        """
+        Return a file object representing the file in the store.
+        :param filename: The filename to open.
+        :param mode: same as the `mode` in famous :func:`.open` function.
+        """
+
+    @abstractmethod
+    def locate(self, filename: str) -> str:
+        """
+        Returns a shareable public link to the filename.
+
+        :param filename: The filename to locate.
+        """

ellar/core/files/storages/interface.py

@@ -0,0 +1,41 @@
+import os
+import typing as t
+
+from .base import BaseStorage
+from .utils import copy_stream
+
+
+class FileSystemStorage(BaseStorage):
+    def service_name(self) -> str:
+        return "local"
+
+    def __init__(
+        self, location: str, chunk_size: int = BaseStorage.DEFAULT_CHUNK_SIZE
+    ) -> None:
+        self.root_path = os.path.abspath(location)
+        self.chunk_size = chunk_size
+
+    def _get_physical_path(self, filename: str) -> str:
+        return os.path.join(self.root_path, filename)
+
+    def put(self, filename: str, stream: t.IO) -> int:
+        physical_path = self._get_physical_path(filename)
+        physical_directory = os.path.dirname(physical_path)
+
+        if not os.path.exists(physical_directory):
+            os.makedirs(physical_directory, exist_ok=True)
+
+        stream.seek(0)
+
+        with open(physical_path, mode="wb") as target_file:
+            return copy_stream(stream, target_file, chunk_size=self.chunk_size)
+
+    def delete(self, filename: str) -> None:
+        physical_path = self._get_physical_path(filename)
+        os.remove(physical_path)
+
+    def open(self, filename: str, mode: str = "rb") -> t.IO:
+        return open(self._get_physical_path(filename), mode=mode)
+
+    def locate(self, filename: str) -> str:
+        return f"{self.root_path}/{filename}"

ellar/core/files/storages/local.py

@@ -0,0 +1,51 @@
+import os
+import re
+import typing
+from io import BytesIO
+
+from .exceptions import SuspiciousFileOperation
+
+
+def copy_stream(
+    source: typing.IO, target: typing.IO, *, chunk_size: int = 16 * 1024
+) -> int:
+    length = 0
+    while 1:
+        buf = source.read(chunk_size)
+        if not buf:
+            break
+        length += len(buf)
+        target.write(buf)
+    return length
+
+
+def get_length(source: typing.IO) -> int:
+    buffer = BytesIO()
+    return copy_stream(source, buffer)
+
+
+def get_valid_filename(name: str) -> str:
+    """
+    Return the given string converted to a string that can be used for a clean
+    filename. Remove leading and trailing spaces; convert other spaces to
+    underscores; and remove anything that is not an alphanumeric, dash,
+    underscore, or dot.
+    >>> get_valid_filename("john's portrait in 2004.jpg")
+    'johns_portrait_in_2004.jpg'
+    """
+    s = str(name).strip().replace(" ", "_")
+    s = re.sub(r"(?u)[^-\w.]", "", s)
+    if s in {"", ".", ".."}:
+        raise Exception("Could not derive file name from '%s'" % name)
+    return s
+
+
+def validate_file_name(name: str) -> str:
+    # Remove potentially dangerous names
+    if os.path.basename(name) in {"", ".", ".."}:
+        raise SuspiciousFileOperation("Could not derive file name from '%s'" % name)
+
+    if name != os.path.basename(name):
+        raise SuspiciousFileOperation("File name '%s' includes path elements" % name)
+
+    return name

ellar/core/files/storages/utils.py

@@ -34,6 +34,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3 :: Only",
+    "Framework :: Ellar",
     "Framework :: AsyncIO",
     "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
     "Topic :: Internet :: WWW/HTTP",