Support brackets in URL detection (#273)

Author: pavlovicisidora

src/main/scala/io/lambdaworks/detection/UrlDetector.scala

@@ -39,7 +39,7 @@ final class UrlDetector private (
       .detect()
       .asScala
       .toList
-      .map(lUrl => AbsoluteUrl.parse(sanitize(lUrl.toString)))
+      .map(url => AbsoluteUrl.parse(sanitize(cleanUrlForBracketMatch(content, url.toString))))
       .filter(url => allowedUrl(url) && notEmail(url) && validTopLevelDomain(url))
       .toSet
   }
@@ -86,6 +86,19 @@ final class UrlDetector private (
   private def allowedUrl(url: AbsoluteUrl): Boolean =
     allowedWithoutWww.forall(containsHost(_, url)) && deniedWithoutWww.forall(!containsHost(_, url))
 
+  private def cleanUrlForBracketMatch(content: String, url: String): String = {
+    def isAllowedUrlChar(c: Char): Boolean =
+      c.isLetterOrDigit || AllowedSpecialChars.contains(c)
+
+    Option(content.indexOf(url)).filter(_ >= 0).fold(url) { from =>
+      val extendedUrl = content.substring(from).takeWhile(isAllowedUrlChar)
+
+      EmptyParensRegex
+        .findFirstMatchIn(extendedUrl)
+        .fold(extendedUrl)(m => extendedUrl.substring(0, m.start))
+    }
+  }
+
   private def containsHost(hosts: NonEmptySet[Host], url: AbsoluteUrl): Boolean =
     hosts.exists(host => host.subdomain.fold(host.apexDomain.exists(url.apexDomain.contains))(_ => host == url.host))
 
@@ -131,7 +144,12 @@ object UrlDetector {
    */
   lazy val default: UrlDetector = UrlDetector(UrlDetectorOptions.Default)
 
-  private final val SanitizeRegex: Regex = "[,!-.`/]+$".r
+  private final val AllowedSpecialChars: Set[Char] = Set(
+    '-', '.', '_', '~', ':', '/', '?', '#', '[', ']', '@', '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', '%'
+  )
+
+  private final val EmptyParensRegex: Regex = "\\(\\)[^()]*".r
+  private final val SanitizeRegex: Regex    = "[,!-.`/]+$".r
 
   implicit private[detection] val orderingHost: Ordering[Host] = orderHost.toOrdering
 

src/test/scala/io/lambdaworks/detection/UrlDetectorSpec.scala

@@ -43,6 +43,14 @@ final class UrlDetectorSpec extends AnyFlatSpec with Matchers {
             Url.parse("http://test.link/aaa")
           )
         ),
+        (
+          "Parse https://learn.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/platform-apis/aa752574(v=vs.85)?redirectedfrom=MSDN",
+          Set(
+            Url.parse(
+              "https://learn.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/platform-apis/aa752574(v=vs.85)?redirectedfrom=MSDN"
+            )
+          )
+        ),
         (
           "192.168.1.3 255.255.1.34 1234.34.34.5 0.0.0.0 192.168.1.257 2.3.4.5",
           Set(
@@ -286,6 +294,22 @@ final class UrlDetectorSpec extends AnyFlatSpec with Matchers {
             Url.parse("http://test.link/HWRqhq"),
             Url.parse("http://test.link/KeKy")
           )
+        ),
+        (
+          "Parse https://site.com/(v=1.2)",
+          Set(Url.parse("https://site.com/(v=1.2"))
+        ),
+        (
+          "Parse https://learn.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/platform-apis/aa752574(v=vs.85)?redirectedfrom=MSDN " +
+            "and http://www.website.com/?utm_source=google%5BB%2B%5D&utm_medium=cpc&utm_content=google_ad(B)&utm_campaign=product",
+          Set(
+            Url.parse(
+              "https://learn.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/platform-apis/aa752574(v=vs.85)?redirectedfrom=MSDN"
+            ),
+            Url.parse(
+              "http://www.website.com/?utm_source=google%5BB%2B%5D&utm_medium=cpc&utm_content=google_ad(B)&utm_campaign=product"
+            )
+          )
         )
       )