exec: remove legacy custom binfmt modules autoloading

Problem: The search binary handler logic contains legacy code
to handle automatically loading kernel modules of unsupported
binary formats.
This logic is a leftover from a.out-to-ELF transition.
After removal of a.out support, this code has no use anymore.

Solution: Clean up this code from the search binary handler,
also remove the line initialising retval to -ENOENT and instead
just return -ENOEXEC if the flow has reached the end of the func.

Note: Anyone who might find future uses for this legacy code
would be better off using binfmt_misc to trigger whatever
module loading they might need - would be more flexible that way.

Suggested-by: Alexander Viro <viro@zeniv.linux.org.uk>
Signed-off-by: Nir Lichtman <nir@lichtman.org>
Link: https://lore.kernel.org/r/20241116231323.GA225987@lichtman.org
Signed-off-by: Kees Cook <kees@kernel.org>

Author: nir9

fs/exec.c

@@ -1723,13 +1723,11 @@ int remove_arg_zero(struct linux_binprm *bprm)
 }
 EXPORT_SYMBOL(remove_arg_zero);
 
-#define printable(c) (((c)=='\t') || ((c)=='\n') || (0x20<=(c) && (c)<=0x7e))
 /*
  * cycle the list of binary formats handler, until one recognizes the image
  */
 static int search_binary_handler(struct linux_binprm *bprm)
 {
-	bool need_retry = IS_ENABLED(CONFIG_MODULES);
 	struct linux_binfmt *fmt;
 	int retval;
 
@@ -1741,8 +1739,6 @@ static int search_binary_handler(struct linux_binprm *bprm)
 	if (retval)
 		return retval;
 
-	retval = -ENOENT;
- retry:
 	read_lock(&binfmt_lock);
 	list_for_each_entry(fmt, &formats, lh) {
 		if (!try_module_get(fmt->module))
@@ -1760,17 +1756,7 @@ static int search_binary_handler(struct linux_binprm *bprm)
 	}
 	read_unlock(&binfmt_lock);
 
-	if (need_retry) {
-		if (printable(bprm->buf[0]) && printable(bprm->buf[1]) &&
-		    printable(bprm->buf[2]) && printable(bprm->buf[3]))
-			return retval;
-		if (request_module("binfmt-%04x", *(ushort *)(bprm->buf + 2)) < 0)
-			return retval;
-		need_retry = false;
-		goto retry;
-	}
-
-	return retval;
+	return -ENOEXEC;
 }
 
 /* binfmt handlers will call back into begin_new_exec() on success. */