zpl_inode.c: Fix SMACK interoperability

SMACK needs to have the ZFS dentry security field setup before
SMACK's d_instantiate() hook is called as it requires functioning
'__vfs_getxattr()' calls to properly set the labels.

Fxes:
1) file instantiation properly setting the object label to the
   subject's label
2) proper file labeling in a transmutable directory

Functions Updated:
1) zpl_create()
2) zpl_mknod()
3) zpl_mkdir()
4) zpl_symlink()

External-issue: cschaufler/smack-next#1
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: TerraTech <TerraTech@users.noreply.github.com>
Closes #11646 
Closes #11839

module/os/linux/zfs/zpl_inode.c

@@ -149,14 +149,17 @@ zpl_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool flag)
 	error = -zfs_create(ITOZ(dir), dname(dentry), vap, 0,
 	    mode, &zp, cr, 0, NULL);
 	if (error == 0) {
-		d_instantiate(dentry, ZTOI(zp));
-
 		error = zpl_xattr_security_init(ZTOI(zp), dir, &dentry->d_name);
 		if (error == 0)
 			error = zpl_init_acl(ZTOI(zp), dir);
 
-		if (error)
+		if (error) {
 			(void) zfs_remove(ITOZ(dir), dname(dentry), cr, 0);
+			remove_inode_hash(ZTOI(zp));
+			iput(ZTOI(zp));
+		} else {
+			d_instantiate(dentry, ZTOI(zp));
+		}
 	}
 
 	spl_fstrans_unmark(cookie);
@@ -198,14 +201,17 @@ zpl_mknod(struct inode *dir, struct dentry *dentry, umode_t mode,
 	error = -zfs_create(ITOZ(dir), dname(dentry), vap, 0,
 	    mode, &zp, cr, 0, NULL);
 	if (error == 0) {
-		d_instantiate(dentry, ZTOI(zp));
-
 		error = zpl_xattr_security_init(ZTOI(zp), dir, &dentry->d_name);
 		if (error == 0)
 			error = zpl_init_acl(ZTOI(zp), dir);
 
-		if (error)
+		if (error) {
 			(void) zfs_remove(ITOZ(dir), dname(dentry), cr, 0);
+			remove_inode_hash(ZTOI(zp));
+			iput(ZTOI(zp));
+		} else {
+			d_instantiate(dentry, ZTOI(zp));
+		}
 	}
 
 	spl_fstrans_unmark(cookie);
@@ -308,14 +314,17 @@ zpl_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
 	cookie = spl_fstrans_mark();
 	error = -zfs_mkdir(ITOZ(dir), dname(dentry), vap, &zp, cr, 0, NULL);
 	if (error == 0) {
-		d_instantiate(dentry, ZTOI(zp));
-
 		error = zpl_xattr_security_init(ZTOI(zp), dir, &dentry->d_name);
 		if (error == 0)
 			error = zpl_init_acl(ZTOI(zp), dir);
 
-		if (error)
+		if (error) {
 			(void) zfs_rmdir(ITOZ(dir), dname(dentry), NULL, cr, 0);
+			remove_inode_hash(ZTOI(zp));
+			iput(ZTOI(zp));
+		} else {
+			d_instantiate(dentry, ZTOI(zp));
+		}
 	}
 
 	spl_fstrans_unmark(cookie);
@@ -488,11 +497,14 @@ zpl_symlink(struct inode *dir, struct dentry *dentry, const char *name)
 	error = -zfs_symlink(ITOZ(dir), dname(dentry), vap,
 	    (char *)name, &zp, cr, 0);
 	if (error == 0) {
-		d_instantiate(dentry, ZTOI(zp));
-
 		error = zpl_xattr_security_init(ZTOI(zp), dir, &dentry->d_name);
-		if (error)
+		if (error) {
 			(void) zfs_remove(ITOZ(dir), dname(dentry), cr, 0);
+			remove_inode_hash(ZTOI(zp));
+			iput(ZTOI(zp));
+		} else {
+			d_instantiate(dentry, ZTOI(zp));
+		}
 	}
 
 	spl_fstrans_unmark(cookie);