fix bugs in fnmatch(), see issue #55186

all tests in tests/posix/c_lib_ext/src/fnmatch.c seem to pass
except one (which is still commented)

Signed-off-by: Nimish Tambe <nimisht@gmail.com>

Author: nimisht

lib/posix/options/fnmatch.c

@@ -84,12 +84,15 @@ static const char *rangematch(const char *pattern, int test, int flags)
 	for (need = true, ok = false, c = FOLDCASE(*pattern++, flags); c != ']' || need;
 	     c = FOLDCASE(*pattern++, flags)) {
 		need = false;
-		if (c == '/') {
-			return (void *)-1;
-		}
+
+		if (c == '/' && (flags & FNM_PATHNAME)) {
+                         return (void *)-1;
+                }
 
 		if (c == '\\' && !(flags & FNM_NOESCAPE)) {
-			c = FOLDCASE(*pattern++, flags);
+			if (*pattern != ']' && *pattern != EOS) {
+                                c = FOLDCASE(*pattern++, flags);
+                        }
 		}
 
 		if (c == EOS) {
@@ -221,7 +224,23 @@ static int fnmatchx(const char *pattern, const char *string, int flags, size_t r
 				return FNM_NOMATCH;
 			}
 
+			if (*string == '.' &&
+                           (flags & FNM_PERIOD) &&
+                           (string == stringstart ||
+                           ((flags & FNM_PATHNAME) && *(string - 1) == '/'))) {
+                                return FNM_NOMATCH;
+                        }
+
 			r = rangematch(pattern, FOLDCASE(*string, flags), flags);
+
+                        if (r == NULL) {
+                                if (FOLDCASE('[', flags) != FOLDCASE(*string, flags)) {
+                                        return FNM_NOMATCH;
+                                }
+                                ++string;
+                                break;
+                        }
+
 			if (r == NULL) {
 				return FNM_NOMATCH;
 			}

tests/posix/c_lib_ext/src/fnmatch.c

@@ -30,14 +30,14 @@ ZTEST(posix_c_lib_ext, test_fnmatch)
 	zassert_equal(fnmatch("a*.c", "a/x.c", FNM_PATHNAME), FNM_NOMATCH);
 	zassert_ok(fnmatch("*/foo", "/foo", FNM_PATHNAME));
 	zassert_ok(fnmatch("-O[01]", "-O1", 0));
-	/* zassert_ok(fnmatch("[[?*\\]", "\\", 0)); */
-	/* zassert_ok(fnmatch("[]?*\\]", "]", 0)); */
+        zassert_ok(fnmatch("[[?*\\]", "\\", 0));
+	zassert_ok(fnmatch("[]?*\\]", "]", 0));
 	zassert_ok(fnmatch("[!]a-]", "b", 0));
 	zassert_ok(fnmatch("[]-_]", "^", 0));
 	zassert_ok(fnmatch("[!]-_]", "X", 0));
 	zassert_equal(fnmatch("??", "-", 0), FNM_NOMATCH);
 	zassert_equal(fnmatch("*LIB*", "lib", FNM_PERIOD), FNM_NOMATCH);
-	/* zassert_ok(fnmatch("a[/]b", "a/b", 0)); */
+	zassert_ok(fnmatch("a[/]b", "a/b", 0));
 	zassert_equal(fnmatch("a[/]b", "a/b", FNM_PATHNAME), FNM_NOMATCH);
 	zassert_ok(fnmatch("[a-z]/[a-z]", "a/b", 0));
 	zassert_equal(fnmatch("*", "a/b", FNM_PATHNAME), FNM_NOMATCH);
@@ -64,8 +64,8 @@ ZTEST(posix_c_lib_ext, test_fnmatch)
 	zassert_equal(fnmatch("a/?b", "a/.b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
 	zassert_equal(fnmatch("*a/b", ".a/b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
 	zassert_equal(fnmatch("a/*b", "a/.b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
-	/* zassert_equal(fnmatch("[.]a/b", ".a/b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH); */
-	/* zassert_equal(fnmatch("a/[.]b", "a/.b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH); */
+	zassert_equal(fnmatch("[.]a/b", ".a/b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
+	zassert_equal(fnmatch("a/[.]b", "a/.b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
 	zassert_ok(fnmatch("*/?", "a/b", FNM_PATHNAME | FNM_PERIOD));
 	zassert_ok(fnmatch("?/*", "a/b", FNM_PATHNAME | FNM_PERIOD));
 	zassert_ok(fnmatch(".*/?", ".a/b", FNM_PATHNAME | FNM_PERIOD));