os: do not follow dangling symlinks in Root when O_CREATE|O_EXCL on AIX

OpenFile with O_CREATE|O_EXCL should not follow dangling symlinks.
On AIX it does, because AIX's openat(2) apparently returns ELOOP
in this case. Most Unices return EEXIST.

Ensure that we never follow symlinks in the final component of
the path when opening a file with O_CREATE|O_EXCL.

Fixes #73924

Change-Id: I869afb7faefccb0bb29d155553a7d7e5be80467d
Reviewed-on: https://go-review.googlesource.com/c/go/+/677735
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Damien Neil <dneil@google.com>
Reviewed-by: Alan Donovan <adonovan@google.com>

Author: neild

src/os/root_unix.go

@@ -83,8 +83,18 @@ func rootOpenFileNolog(root *Root, name string, flag int, perm FileMode) (*File,
 	fd, err := doInRoot(root, name, nil, func(parent int, name string) (fd int, err error) {
 		ignoringEINTR(func() error {
 			fd, err = unix.Openat(parent, name, syscall.O_NOFOLLOW|syscall.O_CLOEXEC|flag, uint32(perm))
-			if isNoFollowErr(err) || err == syscall.ENOTDIR {
-				err = checkSymlink(parent, name, err)
+			if err != nil {
+				// Never follow symlinks when O_CREATE|O_EXCL, no matter
+				// what error the OS returns.
+				isCreateExcl := flag&(O_CREATE|O_EXCL) == (O_CREATE | O_EXCL)
+				if !isCreateExcl && (isNoFollowErr(err) || err == syscall.ENOTDIR) {
+					err = checkSymlink(parent, name, err)
+				}
+				// AIX returns ELOOP instead of EEXIST for a dangling symlink.
+				// Convert this to EEXIST so it matches ErrExists.
+				if isCreateExcl && err == syscall.ELOOP {
+					err = syscall.EEXIST
+				}
 			}
 			return err
 		})