fix compound inheritance penalty (#24775)

fixes #24773

`c.inheritancePenalty` is supposed to be used for the entire match, but
in these places the inheritance penalty of a single argument overrides
the entire match penalty. The `+ ord(c.inheritancePenalty < 0)` is
copied from other places that use the same idiom, the intent is that the
existing penalty changes from -1 to 0 first to mark that it participates
in inheritance before adding the inheritance depth.

---------

Co-authored-by: Andreas Rumpf <araq4k@proton.me>

Author: metagn

compiler/sigmatch.nim

@@ -1541,12 +1541,14 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
         reduceToBase(a)
     if effectiveArgType.kind == tyObject:
       if sameObjectTypes(f, effectiveArgType):
-        c.inheritancePenalty = if tfFinal in f.flags: -1 else: 0
+        if tfFinal notin f.flags:
+          inc c.inheritancePenalty, ord(c.inheritancePenalty < 0)
         result = isEqual
         # elif tfHasMeta in f.flags: result = recordRel(c, f, a)
       elif trIsOutParam notin flags:
-        c.inheritancePenalty = isObjectSubtype(c, effectiveArgType, f, nil)
-        if c.inheritancePenalty > 0:
+        let depth = isObjectSubtype(c, effectiveArgType, f, nil)
+        if depth > 0:
+          inc c.inheritancePenalty, depth + ord(c.inheritancePenalty < 0)
           result = isSubtype
   of tyDistinct:
     a = a.skipTypes({tyOwned, tyGenericInst, tyRange})
@@ -1846,9 +1848,10 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
           if  c.inheritancePenalty > -1:
             minInheritance = min(minInheritance, c.inheritancePenalty)
           result = x
+      c.inheritancePenalty = oldInheritancePenalty
       if result >= isIntConv:
         if minInheritance < maxInheritancePenalty:
-          c.inheritancePenalty = oldInheritancePenalty + minInheritance
+          inc c.inheritancePenalty, minInheritance + ord(c.inheritancePenalty < 0)
         if result > isGeneric: result = isGeneric
         bindingRet result
       else:

tests/overload/tcompoundinheritance.nim

@@ -0,0 +1,26 @@
+# issue #24773
+
+import std/assertions
+
+type
+  A {.inheritable.} = object
+  B = object of A
+  C = object of B
+
+proc add1(v: B) =
+  doAssert true
+
+proc add1(v: A) =
+  doAssert false
+
+proc add2(v: B, v2: A) =
+  doAssert true
+
+proc add2(v: A, v2: A) =
+  doAssert false
+
+var x: C
+var y: B
+
+add1(x)
+add2(x, y)