Rename skip-defaults, and add ability to exclude actual defaults (pyd…

…antic#915)

* Rename skip-defaults, and add ability to exclude actual defaults

* Add __defaults__ and deprecation warnings

* Add note about `skip_defaults` to docs

* Incorporate feedback

* Add tests and changes

* Fix reference to .json()

changes/915-dmontagu.md

@@ -0,0 +1 @@
+**Breaking change:** Rename `skip_defaults` to `exclude_unset`, and add ability to exclude actual defaults 

docs/usage/exporting_models.md

@@ -10,7 +10,10 @@ Arguments:
 * `include`: fields to include in the returned dictionary; see [below](#advanced-include-exclude)
 * `exclude`: fields to exclude from the returned dictionary; see [below](#advanced-include-exclude)
 * `by_alias`: whether field aliases should be used as keys in the returned dictionary; default `False`
-* `skip_defaults`: whether fields which were not explicitly set when creating the model should
+* `exclude_unset`: whether fields which were not explicitly set when creating the model should
+  be excluded from the returned dictionary; default `False`.
+  Prior to **v1.0**, `exclude_unset` was known as `skip_defaults`; use of `skip_defaults` is now deprecated
+* `exclude_defaults`: whether fields which are equal to their default values (whether set or otherwise) should
   be excluded from the returned dictionary; default `False`
 
 Example:
@@ -65,7 +68,10 @@ Arguments:
 * `include`: fields to include in the returned dictionary; see [below](#advanced-include-exclude)
 * `exclude`: fields to exclude from the returned dictionary; see [below](#advanced-include-exclude)
 * `by_alias`: whether field aliases should be used as keys in the returned dictionary; default `False`
-* `skip_defaults`: whether fields which were not set when creating the model and have their default values should
+* `exclude_unset`: whether fields which were not set when creating the model and have their default values should
+  be excluded from the returned dictionary; default `False`.
+  Prior to **v1.0**, `exclude_unset` was known as `skip_defaults`; use of `skip_defaults` is now deprecated
+* `exclude_defaults`: whether fields which are equal to their default values (whether set or otherwise) should
   be excluded from the returned dictionary; default `False`
 * `encoder`: a custom encoder function passed to the `default` argument of `json.dumps()`; defaults to a custom
   encoder designed to take care of all common types

pydantic/main.py

@@ -231,6 +231,7 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
         new_namespace = {
             '__config__': config,
             '__fields__': fields,
+            '__field_defaults__': {n: f.default for n, f in fields.items() if not f.required},
             '__validators__': vg.validators,
             '__pre_root_validators__': pre_root_validators + pre_rv_new,
             '__post_root_validators__': post_root_validators + post_rv_new,
@@ -252,6 +253,7 @@ class BaseModel(metaclass=ModelMetaclass):
     if TYPE_CHECKING:
         # populated by the metaclass, defined here to help IDEs only
         __fields__: Dict[str, ModelField] = {}
+        __field_defaults__: Dict[str, Any] = {}
         __validators__: Dict[str, AnyCallable] = {}
         __pre_root_validators__: List[AnyCallable]
         __post_root_validators__: List[AnyCallable]
@@ -303,15 +305,23 @@ def dict(
         include: Union['SetIntStr', 'DictIntStrAny'] = None,
         exclude: Union['SetIntStr', 'DictIntStrAny'] = None,
         by_alias: bool = False,
-        skip_defaults: bool = False,
+        skip_defaults: bool = None,
+        exclude_unset: bool = False,
+        exclude_defaults: bool = False,
     ) -> 'DictStrAny':
         """
         Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.
         """
+        if skip_defaults is not None:
+            warnings.warn(
+                f'{self.__class__.__name__}.dict(): "skip_defaults" is deprecated and replaced by "exclude_unset"',
+                DeprecationWarning,
+            )
+            exclude_unset = skip_defaults
         get_key = self._get_key_factory(by_alias)
         get_key = partial(get_key, self.__fields__)
 
-        allowed_keys = self._calculate_keys(include=include, exclude=exclude, skip_defaults=skip_defaults)
+        allowed_keys = self._calculate_keys(include=include, exclude=exclude, exclude_unset=exclude_unset)
         return {
             get_key(k): v
             for k, v in self._iter(
@@ -320,7 +330,8 @@ def dict(
                 allowed_keys=allowed_keys,
                 include=include,
                 exclude=exclude,
-                skip_defaults=skip_defaults,
+                exclude_unset=exclude_unset,
+                exclude_defaults=exclude_defaults,
             )
         }
 
@@ -336,7 +347,9 @@ def json(
         include: Union['SetIntStr', 'DictIntStrAny'] = None,
         exclude: Union['SetIntStr', 'DictIntStrAny'] = None,
         by_alias: bool = False,
-        skip_defaults: bool = False,
+        skip_defaults: bool = None,
+        exclude_unset: bool = False,
+        exclude_defaults: bool = False,
         encoder: Optional[Callable[[Any], Any]] = None,
         **dumps_kwargs: Any,
     ) -> str:
@@ -345,8 +358,20 @@ def json(
 
         `encoder` is an optional function to supply as `default` to json.dumps(), other arguments as per `json.dumps()`.
         """
+        if skip_defaults is not None:
+            warnings.warn(
+                f'{self.__class__.__name__}.json(): "skip_defaults" is deprecated and replaced by "exclude_unset"',
+                DeprecationWarning,
+            )
+            exclude_unset = skip_defaults
         encoder = cast(Callable[[Any], Any], encoder or self.__json_encoder__)
-        data = self.dict(include=include, exclude=exclude, by_alias=by_alias, skip_defaults=skip_defaults)
+        data = self.dict(
+            include=include,
+            exclude=exclude,
+            by_alias=by_alias,
+            exclude_unset=exclude_unset,
+            exclude_defaults=exclude_defaults,
+        )
         if self.__custom_root_type__:
             data = data[ROOT_KEY]
         return self.__config__.json_dumps(data, default=encoder, **dumps_kwargs)
@@ -446,7 +471,7 @@ def copy(
             # skip constructing values if no arguments are passed
             v = self.__dict__
         else:
-            allowed_keys = self._calculate_keys(include=include, exclude=exclude, skip_defaults=False, update=update)
+            allowed_keys = self._calculate_keys(include=include, exclude=exclude, exclude_unset=False, update=update)
             if allowed_keys is None:
                 v = {**self.__dict__, **(update or {})}
             else:
@@ -457,7 +482,7 @@ def copy(
                             by_alias=False,
                             include=include,
                             exclude=exclude,
-                            skip_defaults=False,
+                            exclude_unset=False,
                             allowed_keys=allowed_keys,
                         )
                     ),
@@ -516,12 +541,19 @@ def _get_value(
         by_alias: bool,
         include: Optional[Union['SetIntStr', 'DictIntStrAny']],
         exclude: Optional[Union['SetIntStr', 'DictIntStrAny']],
-        skip_defaults: bool,
+        exclude_unset: bool,
+        exclude_defaults: bool,
     ) -> Any:
 
         if isinstance(v, BaseModel):
             if to_dict:
-                return v.dict(by_alias=by_alias, skip_defaults=skip_defaults, include=include, exclude=exclude)
+                return v.dict(
+                    by_alias=by_alias,
+                    exclude_unset=exclude_unset,
+                    exclude_defaults=exclude_defaults,
+                    include=include,
+                    exclude=exclude,
+                )
             else:
                 return v.copy(include=include, exclude=exclude)
 
@@ -534,7 +566,8 @@ def _get_value(
                     v_,
                     to_dict=to_dict,
                     by_alias=by_alias,
-                    skip_defaults=skip_defaults,
+                    exclude_unset=exclude_unset,
+                    exclude_defaults=exclude_defaults,
                     include=value_include and value_include.for_element(k_),
                     exclude=value_exclude and value_exclude.for_element(k_),
                 )
@@ -549,7 +582,8 @@ def _get_value(
                     v_,
                     to_dict=to_dict,
                     by_alias=by_alias,
-                    skip_defaults=skip_defaults,
+                    exclude_unset=exclude_unset,
+                    exclude_defaults=exclude_defaults,
                     include=value_include and value_include.for_element(i),
                     exclude=value_exclude and value_exclude.for_element(i),
                 )
@@ -584,12 +618,20 @@ def _iter(
         allowed_keys: Optional['SetStr'] = None,
         include: Union['SetIntStr', 'DictIntStrAny'] = None,
         exclude: Union['SetIntStr', 'DictIntStrAny'] = None,
-        skip_defaults: bool = False,
+        exclude_unset: bool = False,
+        exclude_defaults: bool = False,
     ) -> 'TupleGenerator':
 
         value_exclude = ValueItems(self, exclude) if exclude else None
         value_include = ValueItems(self, include) if include else None
 
+        if exclude_defaults:
+            if allowed_keys is None:
+                allowed_keys = set(self.__fields__)
+            for k, v in self.__field_defaults__.items():
+                if self.__dict__[k] == v:
+                    allowed_keys.discard(k)
+
         for k, v in self.__dict__.items():
             if allowed_keys is None or k in allowed_keys:
                 yield k, self._get_value(
@@ -598,20 +640,21 @@ def _iter(
                     by_alias=by_alias,
                     include=value_include and value_include.for_element(k),
                     exclude=value_exclude and value_exclude.for_element(k),
-                    skip_defaults=skip_defaults,
+                    exclude_unset=exclude_unset,
+                    exclude_defaults=exclude_defaults,
                 )
 
     def _calculate_keys(
         self,
         include: Optional[Union['SetIntStr', 'DictIntStrAny']],
         exclude: Optional[Union['SetIntStr', 'DictIntStrAny']],
-        skip_defaults: bool,
+        exclude_unset: bool,
         update: Optional['DictStrAny'] = None,
     ) -> Optional['SetStr']:
-        if include is None and exclude is None and skip_defaults is False:
+        if include is None and exclude is None and exclude_unset is False:
             return None
 
-        if skip_defaults:
+        if exclude_unset:
             keys = self.__fields_set__.copy()
         else:
             keys = set(self.__dict__.keys())

tests/test_construction.py

@@ -181,8 +181,8 @@ def test_copy_set_fields():
     m = ModelTwo(a=24, d=Model(a='12'))
     m2 = m.copy()
 
-    assert m.dict(skip_defaults=True) == {'a': 24.0, 'd': {'a': 12}}
-    assert m.dict(skip_defaults=True) == m2.dict(skip_defaults=True)
+    assert m.dict(exclude_unset=True) == {'a': 24.0, 'd': {'a': 12}}
+    assert m.dict(exclude_unset=True) == m2.dict(exclude_unset=True)
 
 
 def test_simple_pickle():
@@ -227,9 +227,9 @@ class Config:
 
 def test_pickle_fields_set():
     m = Model(a=24)
-    assert m.dict(skip_defaults=True) == {'a': 24}
+    assert m.dict(exclude_unset=True) == {'a': 24}
     m2 = pickle.loads(pickle.dumps(m))
-    assert m2.dict(skip_defaults=True) == {'a': 24}
+    assert m2.dict(exclude_unset=True) == {'a': 24}
 
 
 def test_copy_update_exclude():
@@ -246,5 +246,5 @@ class Model(BaseModel):
     assert m.copy(exclude={'c'}).dict() == {'d': {'a': 'ax', 'b': 'bx'}}
     assert m.copy(exclude={'c'}, update={'c': 42}).dict() == {'c': 42, 'd': {'a': 'ax', 'b': 'bx'}}
 
-    assert m._calculate_keys(exclude={'x'}, include=None, skip_defaults=False) == {'c', 'd'}
-    assert m._calculate_keys(exclude={'x'}, include=None, skip_defaults=False, update={'c': 42}) == {'d'}
+    assert m._calculate_keys(exclude={'x'}, include=None, exclude_unset=False) == {'c', 'd'}
+    assert m._calculate_keys(exclude={'x'}, include=None, exclude_unset=False, update={'c': 42}) == {'d'}

tests/test_edge_cases.py

@@ -373,26 +373,70 @@ class Model(BaseModel):
     assert m.dict(include={'a', 'b'}, exclude={'a'}) == {'b': 2}
 
 
-def test_include_exclude_default():
+def test_include_exclude_unset():
     class Model(BaseModel):
         a: int
         b: int
         c: int = 3
         d: int = 4
+        e: int = 5
+        f: int = 6
 
-    m = Model(a=1, b=2)
-    assert m.dict() == {'a': 1, 'b': 2, 'c': 3, 'd': 4}
-    assert m.__fields_set__ == {'a', 'b'}
-    assert m.dict(skip_defaults=True) == {'a': 1, 'b': 2}
+    m = Model(a=1, b=2, e=5, f=7)
+    assert m.dict() == {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 7}
+    assert m.__fields_set__ == {'a', 'b', 'e', 'f'}
+    assert m.dict(exclude_unset=True) == {'a': 1, 'b': 2, 'e': 5, 'f': 7}
+
+    assert m.dict(include={'a'}, exclude_unset=True) == {'a': 1}
+    assert m.dict(include={'c'}, exclude_unset=True) == {}
+
+    assert m.dict(exclude={'a'}, exclude_unset=True) == {'b': 2, 'e': 5, 'f': 7}
+    assert m.dict(exclude={'c'}, exclude_unset=True) == {'a': 1, 'b': 2, 'e': 5, 'f': 7}
+
+    assert m.dict(include={'a', 'b', 'c'}, exclude={'b'}, exclude_unset=True) == {'a': 1}
+    assert m.dict(include={'a', 'b', 'c'}, exclude={'a', 'c'}, exclude_unset=True) == {'b': 2}
 
-    assert m.dict(include={'a'}, skip_defaults=True) == {'a': 1}
-    assert m.dict(include={'c'}, skip_defaults=True) == {}
 
-    assert m.dict(exclude={'a'}, skip_defaults=True) == {'b': 2}
-    assert m.dict(exclude={'c'}, skip_defaults=True) == {'a': 1, 'b': 2}
+def test_include_exclude_defaults():
+    class Model(BaseModel):
+        a: int
+        b: int
+        c: int = 3
+        d: int = 4
+        e: int = 5
+        f: int = 6
+
+    m = Model(a=1, b=2, e=5, f=7)
+    assert m.dict() == {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 7}
+    assert m.__fields_set__ == {'a', 'b', 'e', 'f'}
+    assert m.dict(exclude_defaults=True) == {'a': 1, 'b': 2, 'f': 7}
+
+    assert m.dict(include={'a'}, exclude_defaults=True) == {'a': 1}
+    assert m.dict(include={'c'}, exclude_defaults=True) == {}
+
+    assert m.dict(exclude={'a'}, exclude_defaults=True) == {'b': 2, 'f': 7}
+    assert m.dict(exclude={'c'}, exclude_defaults=True) == {'a': 1, 'b': 2, 'f': 7}
+
+    assert m.dict(include={'a', 'b', 'c'}, exclude={'b'}, exclude_defaults=True) == {'a': 1}
+    assert m.dict(include={'a', 'b', 'c'}, exclude={'a', 'c'}, exclude_defaults=True) == {'b': 2}
+
+
+def test_skip_defaults_deprecated():
+    class Model(BaseModel):
+        x: int
+
+    m = Model(x=1)
+    match = r'Model.dict\(\): "skip_defaults" is deprecated and replaced by "exclude_unset"'
+    with pytest.warns(DeprecationWarning, match=match):
+        assert m.dict(skip_defaults=True)
+    with pytest.warns(DeprecationWarning, match=match):
+        assert m.dict(skip_defaults=False)
 
-    assert m.dict(include={'a', 'b', 'c'}, exclude={'b'}, skip_defaults=True) == {'a': 1}
-    assert m.dict(include={'a', 'b', 'c'}, exclude={'a', 'c'}, skip_defaults=True) == {'b': 2}
+    match = r'Model.json\(\): "skip_defaults" is deprecated and replaced by "exclude_unset"'
+    with pytest.warns(DeprecationWarning, match=match):
+        assert m.json(skip_defaults=True)
+    with pytest.warns(DeprecationWarning, match=match):
+        assert m.json(skip_defaults=False)
 
 
 def test_advanced_exclude():
@@ -474,12 +518,12 @@ class Config:
     m = Model(a=1, b=2)
     assert m.dict() == {'a': 1, 'b': 2, 'c': 3}
     assert m.__fields_set__ == {'a', 'b'}
-    assert m.dict(skip_defaults=True) == {'a': 1, 'b': 2}
+    assert m.dict(exclude_unset=True) == {'a': 1, 'b': 2}
 
     m2 = Model(a=1, b=2, d=4)
     assert m2.dict() == {'a': 1, 'b': 2, 'c': 3}
     assert m2.__fields_set__ == {'a', 'b'}
-    assert m2.dict(skip_defaults=True) == {'a': 1, 'b': 2}
+    assert m2.dict(exclude_unset=True) == {'a': 1, 'b': 2}
 
 
 def test_field_set_allow_extra():
@@ -494,12 +538,12 @@ class Config:
     m = Model(a=1, b=2)
     assert m.dict() == {'a': 1, 'b': 2, 'c': 3}
     assert m.__fields_set__ == {'a', 'b'}
-    assert m.dict(skip_defaults=True) == {'a': 1, 'b': 2}
+    assert m.dict(exclude_unset=True) == {'a': 1, 'b': 2}
 
     m2 = Model(a=1, b=2, d=4)
     assert m2.dict() == {'a': 1, 'b': 2, 'c': 3, 'd': 4}
     assert m2.__fields_set__ == {'a', 'b', 'd'}
-    assert m2.dict(skip_defaults=True) == {'a': 1, 'b': 2, 'd': 4}
+    assert m2.dict(exclude_unset=True) == {'a': 1, 'b': 2, 'd': 4}
 
 
 def test_field_set_field_name():
@@ -509,7 +553,7 @@ class Model(BaseModel):
         b: int = 3
 
     assert Model(a=1, field_set=2).dict() == {'a': 1, 'field_set': 2, 'b': 3}
-    assert Model(a=1, field_set=2).dict(skip_defaults=True) == {'a': 1, 'field_set': 2}
+    assert Model(a=1, field_set=2).dict(exclude_unset=True) == {'a': 1, 'field_set': 2}
     assert Model.construct(dict(a=1, field_set=3), {'a', 'field_set'}).dict() == {'a': 1, 'field_set': 3}