Arguments (#190)

* starting Arguments

* arguments validator WIP

* working on validate_args_pair

* arguments validator compiling

* starting tests

* WIP JSON args mapping

* fix argument mapping for json

* fix tests, add comment about build_args

* tweaking argument errors

* missing tuple args

* fix tuple tests

* using new errors

* mapping positional arguments

* fix too_long and too_short errors

* linting

* converting kwarg errors

* improve coverage

* improve coverage

* add always_validate_kwargs

* allow *args with zero arguments

* fix extra args with args mapping

* error on repeat arguments

* validator decorator and more coverage

* skip positional only args tests for <3.10

* fix for python <3.10

* separate build_validate_generic_mapping

* Arguments, take 2 (#203)

* WIP moving to lookup item for args

* Arguments take 2

* adding var_args and var_kwargs handling

* new arguments logic working

* fixing tests

* Non-default argument follows default arguments

* test aliases

* test kwargs

* fix unexpected_positional_argument

* improve coverage

* PyArgs use tuple, not list

* coverage and tweaks

Author: samuelcolvin

.gitignore

@@ -29,3 +29,4 @@ node_modules/
 package-lock.json
 /pytest-speed/
 /src/self_schema.py
+/worktree/

Makefile

@@ -63,7 +63,7 @@ format:
 
 .PHONY: lint-python
 lint-python:
-	flake8 --max-complexity 10 --max-line-length 120 --ignore E203,W503 pydantic_core tests
+	flake8 --max-line-length 120 pydantic_core tests
 	$(isort) --check-only --df
 	$(black) --check --diff
 

generate_self_schema.py

@@ -48,7 +48,7 @@ def get_schema(obj):
         return 'any'
 
     origin = get_origin(obj)
-    assert origin is not None, f'origin cannot be None, obj={obj}'
+    assert origin is not None, f'origin cannot be None, obj={obj}, you probably need to fix generate_self_schema.py'
     if origin is Union:
         return union_schema(obj)
     elif obj is Callable or origin is Callable:

pydantic_core/_types.py

@@ -34,10 +34,11 @@ class Config(TypedDict, total=False):
     # settings related to typed_dicts only
     typed_dict_extra_behavior: Literal['allow', 'forbid', 'ignore']
     typed_dict_full: bool  # default: True
-    typed_dict_populate_by_name: bool  # replaces `allow_population_by_field_name` in pydantic v1
     # used on typed-dicts and tagged union keys
     from_attributes: bool
     revalidate_models: bool
+    # used on typed-dicts and arguments
+    populate_by_name: bool  # replaces `allow_population_by_field_name` in pydantic v1
     # fields related to string fields only
     str_max_length: int
     str_min_length: int
@@ -277,6 +278,24 @@ class CallableSchema(TypedDict):
     type: Literal['callable']
 
 
+class ArgumentInfo(TypedDict):
+    name: str
+    mode: Literal['positional_only', 'positional_or_keyword', 'keyword_only']
+    schema: Schema
+    default: NotRequired[Any]
+    default_factory: NotRequired[Callable[[], Any]]
+    alias: NotRequired[Union[str, List[Union[str, int]], List[List[Union[str, int]]]]]
+
+
+class ArgumentsSchema(TypedDict, total=False):
+    type: Required[Literal['arguments']]
+    arguments_schema: Required[List[ArgumentInfo]]
+    populate_by_name: bool
+    var_args_schema: Schema
+    var_kwargs_schema: Schema
+    ref: str
+
+
 # pydantic allows types to be defined via a simple string instead of dict with just `type`, e.g.
 # 'int' is equivalent to {'type': 'int'}, this only applies to schema types which do not have other required fields
 BareType = Literal[
@@ -332,4 +351,5 @@ class CallableSchema(TypedDict):
     TimedeltaSchema,
     IsInstanceSchema,
     CallableSchema,
+    ArgumentsSchema,
 ]

src/build_tools.rs

@@ -81,10 +81,20 @@ where
     }
 }
 
+pub fn schema_or_config_same<'py, T>(
+    schema: &'py PyDict,
+    config: Option<&'py PyDict>,
+    key: &PyString,
+) -> PyResult<Option<T>>
+where
+    T: FromPyObject<'py>,
+{
+    schema_or_config(schema, config, key, key)
+}
+
 pub fn is_strict(schema: &PyDict, config: Option<&PyDict>) -> PyResult<bool> {
     let py = schema.py();
-    let k = intern!(py, "strict");
-    Ok(schema_or_config(schema, config, k, k)?.unwrap_or(false))
+    Ok(schema_or_config_same(schema, config, intern!(py, "strict"))?.unwrap_or(false))
 }
 
 // we could perhaps do clever things here to store each schema error, or have different types for the top

src/errors/kinds.rs

@@ -31,7 +31,7 @@ pub enum ErrorKind {
     Missing,
     #[strum(message = "Extra values are not permitted")]
     ExtraForbidden,
-    #[strum(message = "Model keys must be strings")]
+    #[strum(message = "Keys must be strings")]
     InvalidKey,
     #[strum(message = "Error extracting attribute: {error}")]
     GetAttributeError {
@@ -71,13 +71,19 @@ pub enum ErrorKind {
     },
     // ---------------------
     // generic length errors - used for everything with a length except strings and bytes which need custom messages
-    #[strum(message = "Input must have at least {min_length} items")]
+    #[strum(
+        message = "Input must have at least {min_length} item{expected_plural}, got {input_length} item{input_plural}"
+    )]
     TooShort {
         min_length: usize,
+        input_length: usize,
     },
-    #[strum(message = "Input must have at most {max_length} items")]
+    #[strum(
+        message = "Input must have at most {max_length} item{expected_plural}, got {input_length} item{input_plural}"
+    )]
     TooLong {
         max_length: usize,
+        input_length: usize,
     },
     // ---------------------
     // string errors
@@ -297,6 +303,20 @@ pub enum ErrorKind {
     UnionTagNotFound {
         discriminator: String,
     },
+    // ---------------------
+    // argument errors
+    #[strum(message = "Arguments must be a tuple of (positional arguments, keyword arguments) or a plain dict")]
+    ArgumentsType,
+    #[strum(message = "Unexpected keyword argument")]
+    UnexpectedKeywordArgument,
+    #[strum(message = "Missing required keyword argument")]
+    MissingKeywordArgument,
+    #[strum(message = "Unexpected positional argument")]
+    UnexpectedPositionalArgument,
+    #[strum(message = "Missing required positional argument")]
+    MissingPositionalArgument,
+    #[strum(message = "Got multiple values for argument")]
+    MultipleArgumentValues,
 }
 
 macro_rules! render {
@@ -331,6 +351,14 @@ macro_rules! py_dict {
     }};
 }
 
+fn plural_s(value: &usize) -> &'static str {
+    if *value == 1 {
+        ""
+    } else {
+        "s"
+    }
+}
+
 impl ErrorKind {
     pub fn kind(&self) -> String {
         match self {
@@ -348,8 +376,22 @@ impl ErrorKind {
             Self::GreaterThanEqual { ge } => render!(self, ge),
             Self::LessThan { lt } => render!(self, lt),
             Self::LessThanEqual { le } => render!(self, le),
-            Self::TooShort { min_length } => to_string_render!(self, min_length),
-            Self::TooLong { max_length } => to_string_render!(self, max_length),
+            Self::TooShort {
+                min_length,
+                input_length,
+            } => {
+                let expected_plural = plural_s(min_length);
+                let input_plural = plural_s(input_length);
+                to_string_render!(self, min_length, input_length, expected_plural, input_plural)
+            }
+            Self::TooLong {
+                max_length,
+                input_length,
+            } => {
+                let expected_plural = plural_s(max_length);
+                let input_plural = plural_s(input_length);
+                to_string_render!(self, max_length, input_length, expected_plural, input_plural)
+            }
             Self::StrTooShort { min_length } => to_string_render!(self, min_length),
             Self::StrTooLong { max_length } => to_string_render!(self, max_length),
             Self::StrPatternMismatch { pattern } => render!(self, pattern),
@@ -398,8 +440,14 @@ impl ErrorKind {
             Self::GreaterThanEqual { ge } => py_dict!(py, ge),
             Self::LessThan { lt } => py_dict!(py, lt),
             Self::LessThanEqual { le } => py_dict!(py, le),
-            Self::TooShort { min_length } => py_dict!(py, min_length),
-            Self::TooLong { max_length } => py_dict!(py, max_length),
+            Self::TooShort {
+                min_length,
+                input_length,
+            } => py_dict!(py, min_length, input_length),
+            Self::TooLong {
+                max_length,
+                input_length,
+            } => py_dict!(py, max_length, input_length),
             Self::StrTooShort { min_length } => py_dict!(py, min_length),
             Self::StrTooLong { max_length } => py_dict!(py, max_length),
             Self::StrPatternMismatch { pattern } => py_dict!(py, pattern),

src/input/input_abstract.rs

@@ -8,7 +8,7 @@ use crate::input::datetime::EitherTime;
 
 use super::datetime::{EitherDate, EitherDateTime, EitherTimedelta};
 use super::return_enums::{EitherBytes, EitherString};
-use super::{GenericListLike, GenericMapping};
+use super::{GenericArguments, GenericListLike, GenericMapping};
 
 /// all types have three methods: `validate_*`, `strict_*`, `lax_*`
 /// the convention is to either implement:
@@ -42,6 +42,8 @@ pub trait Input<'a>: fmt::Debug + ToPyObject {
         false
     }
 
+    fn validate_args(&'a self) -> ValResult<'a, GenericArguments<'a>>;
+
     fn validate_str(&'a self, strict: bool) -> ValResult<EitherString<'a>> {
         if strict {
             self.strict_str()

src/input/input_json.rs

@@ -5,7 +5,10 @@ use super::datetime::{
     float_as_time, int_as_datetime, int_as_duration, int_as_time, EitherDate, EitherDateTime, EitherTime,
 };
 use super::shared::{float_as_int, int_as_bool, str_as_bool, str_as_int};
-use super::{EitherBytes, EitherString, EitherTimedelta, GenericListLike, GenericMapping, Input, JsonInput};
+use super::{
+    EitherBytes, EitherString, EitherTimedelta, GenericArguments, GenericListLike, GenericMapping, Input, JsonArgs,
+    JsonInput,
+};
 
 impl<'a> Input<'a> for JsonInput {
     /// This is required by since JSON object keys are always strings, I don't think it can be called
@@ -26,6 +29,30 @@ impl<'a> Input<'a> for JsonInput {
         matches!(self, JsonInput::Null)
     }
 
+    fn validate_args(&'a self) -> ValResult<'a, GenericArguments<'a>> {
+        match self {
+            JsonInput::Object(kwargs) => Ok(JsonArgs::new(None, Some(kwargs)).into()),
+            JsonInput::Array(array) => {
+                if array.len() != 2 {
+                    Err(ValError::new(ErrorKind::ArgumentsType, self))
+                } else {
+                    let args = match unsafe { array.get_unchecked(0) } {
+                        JsonInput::Null => None,
+                        JsonInput::Array(args) => Some(args.as_slice()),
+                        _ => return Err(ValError::new(ErrorKind::ArgumentsType, self)),
+                    };
+                    let kwargs = match unsafe { array.get_unchecked(1) } {
+                        JsonInput::Null => None,
+                        JsonInput::Object(kwargs) => Some(kwargs),
+                        _ => return Err(ValError::new(ErrorKind::ArgumentsType, self)),
+                    };
+                    Ok(JsonArgs::new(args, kwargs).into())
+                }
+            }
+            _ => Err(ValError::new(ErrorKind::ArgumentsType, self)),
+        }
+    }
+
     fn strict_str(&'a self) -> ValResult<EitherString<'a>> {
         match self {
             JsonInput::String(s) => Ok(s.as_str().into()),
@@ -245,6 +272,11 @@ impl<'a> Input<'a> for String {
         false
     }
 
+    #[cfg_attr(has_no_coverage, no_coverage)]
+    fn validate_args(&'a self) -> ValResult<'a, GenericArguments<'a>> {
+        Err(ValError::new(ErrorKind::ArgumentsType, self))
+    }
+
     fn validate_str(&'a self, _strict: bool) -> ValResult<EitherString<'a>> {
         Ok(self.as_str().into())
     }

src/input/input_python.rs

@@ -16,7 +16,10 @@ use super::datetime::{
     EitherTime,
 };
 use super::shared::{float_as_int, int_as_bool, str_as_bool, str_as_int};
-use super::{repr_string, EitherBytes, EitherString, EitherTimedelta, GenericListLike, GenericMapping, Input};
+use super::{
+    repr_string, EitherBytes, EitherString, EitherTimedelta, GenericArguments, GenericListLike, GenericMapping, Input,
+    PyArgs,
+};
 
 impl<'a> Input<'a> for PyAny {
     fn as_loc_item(&self) -> LocItem {
@@ -60,6 +63,32 @@ impl<'a> Input<'a> for PyAny {
         self.is_callable()
     }
 
+    fn validate_args(&'a self) -> ValResult<'a, GenericArguments<'a>> {
+        if let Ok(kwargs) = self.cast_as::<PyDict>() {
+            Ok(PyArgs::new(None, Some(kwargs)).into())
+        } else if let Ok((args, kwargs)) = self.extract::<(&PyAny, &PyAny)>() {
+            let args = if let Ok(tuple) = args.cast_as::<PyTuple>() {
+                Some(tuple)
+            } else if args.is_none() {
+                None
+            } else if let Ok(list) = args.cast_as::<PyList>() {
+                Some(PyTuple::new(self.py(), list.iter().collect::<Vec<_>>()))
+            } else {
+                return Err(ValError::new(ErrorKind::ArgumentsType, self));
+            };
+            let kwargs = if let Ok(dict) = kwargs.cast_as::<PyDict>() {
+                Some(dict)
+            } else if kwargs.is_none() {
+                None
+            } else {
+                return Err(ValError::new(ErrorKind::ArgumentsType, self));
+            };
+            Ok(PyArgs::new(args, kwargs).into())
+        } else {
+            Err(ValError::new(ErrorKind::ArgumentsType, self))
+        }
+    }
+
     fn strict_str(&'a self) -> ValResult<EitherString<'a>> {
         if let Ok(py_str) = self.cast_as::<PyString>() {
             Ok(py_str.into())

src/input/mod.rs

@@ -11,7 +11,9 @@ mod shared;
 pub use datetime::{EitherDate, EitherDateTime, EitherTime, EitherTimedelta};
 pub use input_abstract::Input;
 pub use parse_json::{JsonInput, JsonObject};
-pub use return_enums::{EitherBytes, EitherString, GenericListLike, GenericMapping};
+pub use return_enums::{
+    EitherBytes, EitherString, GenericArguments, GenericListLike, GenericMapping, JsonArgs, PyArgs,
+};
 
 pub fn repr_string(v: &PyAny) -> PyResult<String> {
     v.repr()?.extract()