From 355280861df0aab90290e7675c884958462c69d1 Mon Sep 17 00:00:00 2001 From: Two Dev Date: Thu, 5 Dec 2024 15:08:41 +0700 Subject: [PATCH] chore: fix multipart encoders, cross share auth. --- .github/workflows/ci.yml | 23 +++++++++++++++ .gitignore | 1 + docs/advanced/async_client.md | 4 +-- docs/advanced/authentication.md | 16 +++++++---- docs/advanced/client.md | 16 +++++------ docs/advanced/hooks.md | 7 +++-- docs/advanced/proxies.md | 6 ++-- docs/quickstart.md | 47 +++++++++++++++---------------- docs/tls/index.md | 6 ++-- tls_requests/__version__.py | 2 +- tls_requests/client.py | 50 ++++++++++++++++++++++++--------- tls_requests/models/encoders.py | 2 +- 12 files changed, 115 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e33a25..04d7e40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,3 +28,26 @@ jobs: - name: Run pre-commit uses: pre-commit/action@v3.0.0 + + deploy: + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + steps: + - uses: actions/checkout@v4 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + - run: pip install -r requirements.txt + - run: mkdocs gh-deploy --force diff --git a/.gitignore b/.gitignore index 7b6caf3..015c6bc 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ +tls_requests/bin/*xgo* diff --git a/docs/advanced/async_client.md b/docs/advanced/async_client.md index d2b6037..76a3452 100644 --- a/docs/advanced/async_client.md +++ b/docs/advanced/async_client.md @@ -25,7 +25,7 @@ To send asynchronous HTTP requests, use the `AsyncClient`: >>> import asyncio >>> async def fetch(url): async with tls_requests.AsyncClient() as client: - r = await client.get("https://www.example.com/") + r = await client.get(url) return r >>> r = asyncio.run(fetch("https://httpbin.org/get")) @@ -87,7 +87,7 @@ import asyncio async def fetch(url): client = tls_requests.AsyncClient() try: - response = await client.get("https://www.example.com/") + response = await client.get("https://httpbin.org/get") finally: await client.aclose() ``` diff --git a/docs/advanced/authentication.md b/docs/advanced/authentication.md index 49b82bf..54fc8d1 100644 --- a/docs/advanced/authentication.md +++ b/docs/advanced/authentication.md @@ -29,10 +29,13 @@ To customize how authentication is handled, you can use a function that modifies request.headers["X-Authorization"] = "123456" return request ->>> client = tls_requests.Client(auth=custom_auth) ->>> response = client.get("https://www.example.com/") +>>> response = tls_requests.get("https://httpbin.org/headers", auth=custom_auth) >>> response +>>> response.request.headers["X-Authorization"] +'123456' +>>> response.json()["headers"]["X-Authorization"] +'123456' ``` * * * @@ -53,7 +56,7 @@ class BearerAuth(tls_requests.Auth): self.token = token def build_auth(self, request: tls_requests.Request) -> tls_requests.Request | None: - request.headers['Authorization'] = f"Bearer {self.token}" + request.headers["Authorization"] = f"Bearer {self.token}" return request ``` @@ -65,10 +68,13 @@ To use your custom `BearerAuth` implementation: ```pycon >>> auth = BearerAuth(token="your_jwt_token") ->>> client = tls_requests.Client(auth=auth) ->>> response = client.get("https://www.example.com/secure-endpoint") +>>> response = tls_requests.get("https://httpbin.org/headers", auth=auth) >>> response +>>> response.request.headers["Authorization"] +'Bearer your_jwt_token' +>>> response.json()["headers"]["Authorization"] +'Bearer your_jwt_token' ``` With these approaches, you can integrate various authentication strategies into your `tls_requests` workflow, whether built-in or custom-designed for specific needs. diff --git a/docs/advanced/client.md b/docs/advanced/client.md index e3f878f..a498c69 100644 --- a/docs/advanced/client.md +++ b/docs/advanced/client.md @@ -33,7 +33,7 @@ The best practice is to use a `Client` as a context manager. This ensures connec ```python with tls_requests.Client() as client: - response = client.get("https://example.com") + response = client.get("https://httpbin.org/get") print(response) # ``` @@ -44,7 +44,7 @@ If not using a context manager, ensure to close the client explicitly: ```python client = tls_requests.Client() try: - response = client.get("https://example.com") + response = client.get("https://httpbin.org/get") print(response) # finally: client.close() @@ -59,7 +59,7 @@ A `Client` can send requests using methods like `.get()`, `.post()`, etc.: ```python with tls_requests.Client() as client: - response = client.get("https://example.com") + response = client.get("https://httpbin.org/get") print(response) # ``` @@ -70,7 +70,7 @@ To include custom headers in a request: ```python headers = {'X-Custom': 'value'} with tls_requests.Client() as client: - response = client.get("https://example.com", headers=headers) + response = client.get("https://httpbin.org/get", headers=headers) print(response.request.headers['X-Custom']) # 'value' ``` @@ -101,7 +101,7 @@ When client-level and request-level options overlap: client_headers = {'X-Auth': 'client'} request_headers = {'X-Custom': 'request'} with tls_requests.Client(headers=client_headers) as client: - response = client.get("https://example.com", headers=request_headers) + response = client.get("https://httpbin.org/get", headers=request_headers) print(response.request.headers['X-Auth']) # 'client' print(response.request.headers['X-Custom']) # 'request' ``` @@ -110,7 +110,7 @@ with tls_requests.Client(headers=client_headers) as client: ```python with tls_requests.Client(auth=('user', 'pass')) as client: - response = client.get("https://example.com", auth=('admin', 'adminpass')) + response = client.get("https://httpbin.org/get", auth=('admin', 'adminpass')) print(response.request.headers['Authorization']) # Encoded 'admin:adminpass' ``` @@ -123,7 +123,7 @@ Advanced Request Handling For more control, explicitly build and send `Request` instances: ```python -request = tls_requests.Request("GET", "https://example.com") +request = tls_requests.Request("GET", "https://httpbin.org/get") with tls_requests.Client() as client: response = client.send(request) print(response) # @@ -133,7 +133,7 @@ To combine client- and request-level configurations: ```python with tls_requests.Client(headers={"X-Client-ID": "ABC123"}) as client: - request = client.build_request("GET", "https://api.example.com") + request = client.build_request("GET", "https://httpbin.org/json") del request.headers["X-Client-ID"] # Modify as needed response = client.send(request) print(response) diff --git a/docs/advanced/hooks.md b/docs/advanced/hooks.md index 1b9a830..1687f4a 100644 --- a/docs/advanced/hooks.md +++ b/docs/advanced/hooks.md @@ -100,8 +100,9 @@ client.hooks = { Best Practices -------------- -1. **Always Use Lists:** Hooks must be registered as **lists of callables**, even if you are adding only one function. -2. **Combine Hooks:** You can register multiple hooks for the same event type to handle various concerns, such as logging and error handling. -3. **Order Matters:** Hooks are executed in the order they are registered. +1. **Access Content**: Use `.read()` or `await read()` in asynchronous contexts to access `response.content` before returning it. +2. **Always Use Lists:** Hooks must be registered as **lists of callables**, even if you are adding only one function. +3. **Combine Hooks:** You can register multiple hooks for the same event type to handle various concerns, such as logging and error handling. +4. **Order Matters:** Hooks are executed in the order they are registered. With hooks, TLS Requests provides a flexible mechanism to seamlessly integrate monitoring, logging, or custom behaviors into your HTTP workflows. diff --git a/docs/advanced/proxies.md b/docs/advanced/proxies.md index acdcc07..756c5bd 100644 --- a/docs/advanced/proxies.md +++ b/docs/advanced/proxies.md @@ -19,7 +19,7 @@ To route traffic through an HTTP proxy, specify the proxy URL in the `proxy` par ```python with tls_requests.Client(proxy="http://localhost:8030") as client: - response = client.get("https://example.com") + response = client.get("https://httpbin.org/get") print(response) # ``` @@ -30,7 +30,7 @@ For SOCKS proxies, use the `socks5` scheme in the proxy URL: ```python client = tls_requests.Client(proxy="socks5://user:pass@host:port") -response = client.get("https://example.com") +response = client.get("https://httpbin.org/get") print(response) # ``` @@ -47,7 +47,7 @@ You can include proxy credentials in the `userinfo` section of the URL: ```python with tls_requests.Client(proxy="http://username:password@localhost:8030") as client: - response = client.get("https://example.com") + response = client.get("https://httpbin.org/get") print(response) # ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index d64717c..bdc36a8 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -109,6 +109,28 @@ Include lists or merge parameters with existing query strings: '' ``` +* * * + +Custom Headers +-------------- + +Add custom headers to requests: + +```pycon +>>> url = 'https://httpbin.org/headers' +>>> headers = {'user-agent': 'my-app/1.0.0'} +>>> r = tls_requests.get(url, headers=headers) +>>> r.json() +{ + "headers": { + ... + "Host": "httpbin.org", + "User-Agent": "my-app/1.0.0", + ... + } +} +``` + * * * @@ -163,28 +185,6 @@ Parse JSON responses directly: } ``` -* * * - -Custom Headers --------------- - -Add custom headers to requests: - -```pycon ->>> url = 'https://httpbin.org/headers' ->>> headers = {'user-agent': 'my-app/1.0.0'} ->>> r = tls_requests.get(url, headers=headers) ->>> r.json() -{ - "headers": { - ... - "Host": "httpbin.org", - "User-Agent": "my-app/1.0.0", - ... - } -} -``` - ### Form-Encoded Data Include form data in POST requests: @@ -380,9 +380,6 @@ The `Headers` data type is case-insensitive, so you can use any capitalization. ```pycon >>> r.headers['Content-Type'] 'application/json' - ->>> r.headers.get('content-type') -'application/json' ``` ### Cookies diff --git a/docs/tls/index.md b/docs/tls/index.md index d6d7890..b999565 100644 --- a/docs/tls/index.md +++ b/docs/tls/index.md @@ -48,7 +48,7 @@ Retrieves cookies associated with a session for a specific URL. ```pycon >>> from tls_requests import TLSClient >>> TLSClient.initialize() ->>> cookies = TLSClient.get_cookies(session_id="session123", url="https://example.com") +>>> cookies = TLSClient.get_cookies(session_id="session123", url="https://httpbin.org/get") ``` * * * @@ -74,7 +74,7 @@ Adds cookies to a specific TLS session. "value": "baz2", }], "sessionId": "session123", - "url": "https://example.com", + "url": "https://httpbin.org/", } >>> TLSClient.add_cookies(session_id="session123", payload=payload) ``` @@ -138,7 +138,7 @@ Sends a request using the TLS library. Using [TLSConfig](configuration) to gener ```pycon >>> from tls_requests import TLSClient, TLSConfig >>> TLSClient.initialize() ->>> config = TLSConfig(requestMethod="GET", requestUrl="https://example.com") +>>> config = TLSConfig(requestMethod="GET", requestUrl="https://httpbin.org/get") >>> response = TLSClient.request(config.to_dict()) ``` diff --git a/tls_requests/__version__.py b/tls_requests/__version__.py index 6ed6558..a6eac23 100644 --- a/tls_requests/__version__.py +++ b/tls_requests/__version__.py @@ -3,5 +3,5 @@ __url__ = "https://github.com/thewebscraping/tls-requests" __author__ = "Tu Pham" __author_email__ = "thetwofarm@gmail.com" -__version__ = "1.0.2" +__version__ = "1.0.3" __license__ = "MIT" diff --git a/tls_requests/client.py b/tls_requests/client.py index 0ea6deb..2d39ac5 100644 --- a/tls_requests/client.py +++ b/tls_requests/client.py @@ -9,7 +9,7 @@ TypeVar, Union) from .exceptions import RemoteProtocolError, TooManyRedirects -from .models import (URL, BasicAuth, Cookies, Headers, Proxy, Request, +from .models import (URL, Auth, BasicAuth, Cookies, Headers, Proxy, Request, Response, StatusCodes, TLSClient, TLSConfig, URLParams) from .settings import (DEFAULT_FOLLOW_REDIRECTS, DEFAULT_HEADERS, DEFAULT_MAX_REDIRECTS, DEFAULT_TIMEOUT, @@ -166,12 +166,15 @@ def prepare_auth( ) -> Union[Request, Any]: """Build Auth Request instance""" - if isinstance(self.auth, tuple) and len(self.auth) == 2: - auth = BasicAuth(self.auth[0], self.auth[1]) + if isinstance(auth, tuple) and len(auth) == 2: + auth = BasicAuth(auth[0], auth[1]) return auth.build_auth(request) - if callable(self.auth): - return self.auth(request) + if callable(auth): + return auth(request) + + if isinstance(auth, Auth): + return auth.build_auth(request) def prepare_headers(self, headers: HeaderTypes = None) -> Headers: """Prepare Headers""" @@ -265,7 +268,7 @@ def build_hook_response( if isinstance(request_hooks, Sequence): for hook in request_hooks: if callable(hook): - return hook(request) + return hook(response) def _rebuild_hooks(self, hooks: HookTypes): if isinstance(hooks, dict): @@ -355,10 +358,6 @@ def _send( return self._send(response.next, history=history, start=start) response.history = history - response_ = self.build_hook_response(response) - if isinstance(response_, Response): - response = response_ - return response def close(self) -> None: @@ -476,7 +475,7 @@ def send( self._state = ClientState.OPENED for fn in [self.prepare_auth, self.build_hook_request]: - request_ = fn(request, auth, follow_redirects) + request_ = fn(request, auth or self.auth, follow_redirects) if isinstance(request_, Request): request = request_ @@ -485,7 +484,14 @@ def send( request, start=time.perf_counter(), ) - response.read() + + if self.hooks.get("response"): + response_ = self.build_hook_response(response) + if isinstance(response_, Response): + response = response_ + else: + response.read() + response.close() return response @@ -761,8 +767,24 @@ async def send( raise RuntimeError("Cannot send a request, as the client has been closed.") self._state = ClientState.OPENED - response = self._send(request, start=time.perf_counter()) - await response.aread() + for fn in [self.prepare_auth, self.build_hook_request]: + request_ = fn(request, auth or self.auth, follow_redirects) + if isinstance(request_, Request): + request = request_ + + self.follow_redirects = follow_redirects + response = self._send( + request, + start=time.perf_counter(), + ) + + if self.hooks.get("response"): + response_ = self.build_hook_response(response) + if isinstance(response_, Response): + response = response_ + else: + await response.aread() + await response.aclose() return response diff --git a/tls_requests/models/encoders.py b/tls_requests/models/encoders.py index 33b369e..b226af0 100644 --- a/tls_requests/models/encoders.py +++ b/tls_requests/models/encoders.py @@ -102,7 +102,7 @@ def __init__(self, name: str, value: RequestFileValue): self.filename, self._buffer, self.content_type = self.unpack(value) def unpack(self, value: RequestFileValue) -> tuple[str, BufferTypes, str]: - filename = content_type = None, None + filename, content_type = None, None if isinstance(value, tuple): if len(value) > 1: filename, buffer, *args = value