Fix Field mandatory & normalize in row save typecast

src/Field.php

@@ -164,7 +164,7 @@ public function __construct()
             $value = $persistence->typecastLoadField($this, $value);
 
             if ($value === null) {
-                if ($this->required/* known bug, see https://github.com/atk4/data/issues/575, fix in https://github.com/atk4/data/issues/576 || $this->mandatory*/) {
+                if ($this->required || $this->mandatory) {
                     throw new Exception('Must not be null');
                 }
 

src/Model.php

@@ -31,7 +31,6 @@ class Model implements \IteratorAggregate
         add as _add;
     }
     use DiContainerTrait {
-        warnPropertyDoesNotExist as private __di_warnPropertyDoesNotExist;
         DiContainerTrait::__isset as private __di_isset;
         DiContainerTrait::__get as private __di_get;
         DiContainerTrait::__set as private __di_set;
@@ -379,7 +378,12 @@ public function createEntity(): self
             $this->_model = null;
         }
         $model->_entityId = null;
-        $model->scope = null; // @phpstan-ignore-line
+
+        // TODO unset properties that should work only on model,
+        // they will emit undefined warning then if accessed then
+        // unset($model->table);
+        // unset($model->table_alias);
+        unset($model->{'scope'});
 
         return $model;
     }
@@ -1641,8 +1645,7 @@ protected function _rawInsert(array $row): void
      */
     public function insert(array $row)
     {
-        $model = ($this->isEntity() ? $this->getModel() : $this)
-            ->createEntity();
+        $model = $this->createEntity();
         $model->_rawInsert($row);
 
         return $this->id_field ? $model->getId() : null;
@@ -1953,37 +1956,46 @@ public function addCalculatedField(string $name, $expression)
 
     // }}}
 
-    protected function warnPropertyDoesNotExist(string $name): void
+    public function __isset(string $name): bool
     {
-        if (!isset($this->getHintableProps()[$name])) {
-            $this->__di_warnPropertyDoesNotExist($name);
+        if (isset($this->getHintableProps()[$name])) {
+            return $this->__hintable_isset($name);
         }
-    }
 
-    public function __isset(string $name): bool
-    {
-        return $this->__hintable_isset($name);
+        return $this->__di_isset($name);
     }
 
     /**
      * @return mixed
      */
     public function &__get(string $name)
     {
-        return $this->__hintable_get($name);
+        if (isset($this->getHintableProps()[$name])) {
+            return $this->__hintable_get($name);
+        }
+
+        return $this->__di_get($name);
     }
 
     /**
      * @param mixed $value
      */
     public function __set(string $name, $value): void
     {
-        $this->__hintable_set($name, $value);
+        if (isset($this->getHintableProps()[$name])) {
+            $this->__hintable_set($name, $value);
+        }
+
+        $this->__di_set($name, $value);
     }
 
     public function __unset(string $name): void
     {
-        $this->__hintable_unset($name);
+        if (isset($this->getHintableProps()[$name])) {
+            $this->__hintable_unset($name);
+        }
+
+        $this->__di_unset($name);
     }
 
     // {{{ Debug Methods

src/Persistence.php

@@ -143,11 +143,7 @@ public function tryLoad(Model $model, $id): ?array
      */
     public function load(Model $model, $id): array
     {
-        $data = $this->tryLoad(
-            $model,
-            $id,
-            ...array_slice(func_get_args(), 2, null, true)
-        );
+        $data = $this->tryLoad($model, $id);
 
         if (!$data) {
             $noId = $id === self::ID_LOAD_ONE || $id === self::ID_LOAD_ANY;
@@ -174,14 +170,7 @@ public function typecastSaveRow(Model $model, array $row): array
         foreach ($row as $fieldName => $value) {
             $field = $model->getField($fieldName);
 
-            $value = $this->typecastSaveField($field, $value);
-
-            // check null values for mandatory fields
-            if ($value === null && $field->mandatory) {
-                throw new ValidationException([$field->short_name => 'Mandatory field value cannot be null'], $field->getOwner());
-            }
-
-            $result[$field->getPersistenceName()] = $value;
+            $result[$field->getPersistenceName()] = $this->typecastSaveField($field, $value);
         }
 
         return $result;
@@ -219,8 +208,13 @@ public function typecastLoadRow(Model $model, array $row): array
      *
      * @return scalar|Persistence\Sql\Expressionable|null
      */
-    public function typecastSaveField(Field $field, $value)
+    final public function typecastSaveField(Field $field, $value)
     {
+        $prevFrame = debug_backtrace(\DEBUG_BACKTRACE_PROVIDE_OBJECT | \DEBUG_BACKTRACE_IGNORE_ARGS, 2)[1] ?? [];
+        if (($prevFrame['object'] ?? null) !== $field || ($prevFrame['function'] ?? null) !== 'normalize') {
+            $value = $field->normalize($value);
+        }
+
         if ($value === null) {
             return null;
         }

src/Persistence/Array_.php

@@ -235,6 +235,9 @@ public function insert(Model $model, array $data)
     {
         $this->seedData($model);
 
+        if ($model->id_field && ($data[$model->id_field] ?? null) === null) {
+            unset($data[$model->id_field]);
+        }
         $data = $this->typecastSaveRow($model, $data);
 
         $id = $data[$model->id_field] ?? $this->generateNewId($model);

src/Persistence/Array_/Join.php

@@ -58,7 +58,7 @@ public function afterLoad(): void
 
         try {
             $data = Persistence\Array_::assertInstanceOf($model->persistence)
-                ->load($this->makeFakeModelWithForeignTable(), $this->id, $this->foreign_table);
+                ->load($this->makeFakeModelWithForeignTable(), $this->id);
         } catch (Exception $e) {
             throw (new Exception('Unable to load joined record', $e->getCode(), $e))
                 ->addMoreInfo('table', $this->foreign_table)

src/Persistence/Csv.php

@@ -219,7 +219,7 @@ public function typecastLoadRow(Model $model, array $row): array
         }
 
         $row = array_combine($this->header, $row);
-        if ($model->id_field && isset($id)) {
+        if ($model->id_field && $id !== null) {
             $row[$model->id_field] = $id;
         }
 
@@ -302,6 +302,8 @@ public function insert(Model $model, array $data)
             throw new Exception('Currently reading records, so writing is not possible.');
         }
 
+        $data = $this->typecastSaveRow($model, $data);
+
         if (!$this->handle) {
             $this->saveHeader($model);
         }

src/Persistence/Sql.php

@@ -559,7 +559,7 @@ public function insert(Model $model, array $data): string
     {
         $insert = $model->action('insert');
 
-        if ($model->id_field && !isset($data[$model->id_field])) {
+        if ($model->id_field && ($data[$model->id_field] ?? null) === null) {
             unset($data[$model->id_field]);
         }
 
@@ -577,7 +577,7 @@ public function insert(Model $model, array $data): string
                 ->addMoreInfo('scope', $model->getModel(true)->scope()->toWords());
         }
 
-        if ($model->id_field && isset($data[$model->id_field])) {
+        if ($model->id_field && ($data[$model->id_field] ?? null) !== null) {
             $id = (string) $data[$model->id_field];
         } else {
             $id = $this->lastInsertId($model);

tests/ContainsManyTest.php

@@ -123,7 +123,7 @@ public function testContainsMany(): void
 
         // now let's delete line with id=2 and add one more line
         $i->lines
-            ->load(2)->delete()
+            ->load(2)->delete()->getModel()
             ->insert([
                 $l->fieldName()->vat_rate_id => 2,
                 $l->fieldName()->price => 50,

tests/FieldTest.php

@@ -65,11 +65,8 @@ public function testMandatory1(): void
         $m->set('foo', 'abc');
         $m->set('foo', '');
 
-        /* known bug, see https://github.com/atk4/data/issues/575, fix in https://github.com/atk4/data/issues/576
-        $this->expectException(ValidationException::class);*/
+        $this->expectException(ValidationException::class);
         $m->set('foo', null);
-
-        $this->assertTrue(true); // no exceptions
     }
 
     public function testRequired1(): void
@@ -257,7 +254,7 @@ public function testEnum4(): void
         $m = new Model();
         $m->addField('foo', ['enum' => [1, 'bar'], 'default' => 1]);
         $m = $m->createEntity();
-        $m->set('foo', null);
+        $m->setNull('foo');
 
         $this->assertNull($m->get('foo'));
     }
@@ -389,7 +386,7 @@ public function testTitle(): void
         $this->assertSame('John', $m->get('name'));
         $this->assertSame('Programmer', $m->get('category'));
 
-        $m->insert(['name' => 'Peter', 'category' => 'Sales']);
+        $m->getModel()->insert(['name' => 'Peter', 'category' => 'Sales']);
 
         $this->assertEquals([
             'user' => [

tests/JoinArrayTest.php

@@ -122,7 +122,7 @@ public function testJoinSaving2(): void
         $m_u->addField('name');
         $j = $m_u->join('contact.test_id');
         $j->addField('contact_phone');
-        $j->addField('test_id');
+        $j->addField('test_id', ['type' => 'integer']);
 
         $m_u2 = $m_u->createEntity();
         $m_u2->set('name', 'John');
@@ -174,7 +174,7 @@ public function testJoinSaving3(): void
         $db = new Persistence\Array_(['user' => [], 'contact' => []]);
         $m_u = new Model($db, ['table' => 'user']);
         $m_u->addField('name');
-        $m_u->addField('test_id');
+        $m_u->addField('test_id', ['type' => 'integer']);
         $j = $m_u->join('contact', ['master_field' => 'test_id']);
         $j->addField('contact_phone');
         $m_u = $m_u->createEntity();
@@ -225,7 +225,7 @@ public function testJoinLoading(): void
             ],
         ]);
         $m_u = new Model($db, ['table' => 'user']);
-        $m_u->addField('contact_id');
+        $m_u->addField('contact_id', ['type' => 'integer']);
         $m_u->addField('name');
         $j = $m_u->join('contact');
         $j->addField('contact_phone');
@@ -259,7 +259,7 @@ public function testJoinUpdate(): void
             ],
         ]);
         $m_u = new Model($db, ['table' => 'user']);
-        $m_u->addField('contact_id');
+        $m_u->addField('contact_id', ['type' => 'integer']);
         $m_u->addField('name');
         $j = $m_u->join('contact');
         $j->addField('contact_phone');
@@ -330,7 +330,7 @@ public function testJoinDelete(): void
             ],
         ]);
         $m_u = new Model($db, ['table' => 'user']);
-        $m_u->addField('contact_id');
+        $m_u->addField('contact_id', ['type' => 'integer']);
         $m_u->addField('name');
         $j = $m_u->join('contact');
         $j->addField('contact_phone');
@@ -363,7 +363,7 @@ public function testLoadMissing(): void
             ],
         ]);
         $m_u = new Model($db, ['table' => 'user']);
-        $m_u->addField('contact_id');
+        $m_u->addField('contact_id', ['type' => 'integer']);
         $m_u->addField('name');
         $j = $m_u->join('contact');
         $j->addField('contact_phone');

tests/RandomTest.php

@@ -258,8 +258,7 @@ public function testUpdateCondition(): void
                 throw (new \Atk4\Core\Exception('Update didn\'t affect any records'))
                     ->addMoreInfo('query', $update->getDebugQuery())
                     ->addMoreInfo('statement', $st)
-                    ->addMoreInfo('model', $m)
-                    ->addMoreInfo('conditions', $m->conditions);
+                    ->addMoreInfo('model', $m);
             }
         });
 

tests/SerializeTest.php

@@ -19,10 +19,10 @@ public function testBasicSerialize(): void
         $f = $m->addField('data', ['type' => 'object']);
 
         $this->assertSame(
-            ['data' => 'a:1:{s:3:"foo";s:3:"bar";}'],
+            ['data' => 'O:8:"stdClass":1:{s:3:"foo";s:3:"bar";}'],
             $db->typecastSaveRow(
                 $m,
-                ['data' => ['foo' => 'bar']]
+                ['data' => (object) ['foo' => 'bar']]
             )
         );
         $this->assertSame(

tests/Util/DeepCopyTest.php

@@ -332,16 +332,15 @@ public function testDeepError(): void
         $dc = new DeepCopy();
 
         $this->expectException(DeepCopyException::class);
-
         try {
             $invoice = $dc
                 ->from($quote)
                 ->excluding(['Lines' => ['qty']])
                 ->to($invoice)
                 ->with(['Lines'])
                 ->copy();
-        } catch (\Atk4\Data\Util\DeepCopyException $e) {
-            $this->assertSame('Mandatory field value cannot be null', $e->getPrevious()->getMessage());
+        } catch (DeepCopyException $e) {
+            $this->assertSame('Must not be null', $e->getPrevious()->getMessage());
 
             throw $e;
         }