chore: Handle forward references, repeatable annotations, and use str enums (#43)

- Previously, we eagerly compile the class as soon as `@planning_entity`
  or `@planning_solution` is reached. This cause problem if the class
  contains forward references, since:

  - The referenced class does not exist when `@planning_entity` or
    `@planning_solution` is reached, since it is defined later.

  - This causes `get_type_hints` to raise a NameError, since it
    cannot find the type name in locals or globals

  Now, the classes are compiled when the SolverConfig is read (and
  thus, all the referenced classes should be defined).

- In order to handle forward references in annotations and the class
  translator, usage of getJavaClass (which may throw an exception if the
  class is in the middle of being defined)  need to be changed to
  getJavaClassInternalName (which will never throw). Thus:

  - ArgumentSpec now take the return type and parameters of a function
    as String instead of their Class. There could be a Class overload
    that calls the String version, but I decided against it to prevent
    the temptation to pass getJavaClass to it.

  - PythonDefaultArgumentImplementor now sets its constants inside
    <clinit>, which means ArgumentSpec must store itself in a static
    field so clinit can access it. This is because if a Class cannot
    be loaded until all the Classes it references are defined (and
    thus, we cannot set the field values from Java, since
    PythonDefaultArgumentImplementor might reference a class still
    being defined).

  - AnnotationMetadata now store Class values as Type instead of
    Class.

- In order for Timefold to discover the "true" type of annotated
  getters, we remove NoneType from the Union of the getter
  return type (but keep it in the actual field type). That is,
  if a type is annotated `Value | None`, the getter type will
  be `Value`, and the field type will be
  `get_common_ancestor(Value, NoneType) = object`.

- Use PythonClassWriter instead of ClassWriter in the
  ClassTranslator, so ASM will not complain about missing
  classes when computing frames.

- In order to properly visit a repeatable annotation, we need
  to group the repeated annotation by type and put them as the
  value of their container annotation class.

- Make VersionMapping throw an exception if it gets a null version
  mapping (otherwise, an undescriptive NPE will happen if the
  unsupported opcode is in the bytecode).

- Added the missing PreviousElementShadowVariable and NextElementShadowVariable
  annotations.

- Use str enums, so the enum serializes to their names instead of a
  number.

Author: Christopher-Chianelli

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/AnnotationMetadata.java

@@ -1,7 +1,11 @@
 package ai.timefold.jpyinterpreter;
 
 import java.lang.annotation.Annotation;
+import java.lang.annotation.Repeatable;
 import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.objectweb.asm.AnnotationVisitor;
@@ -23,6 +27,30 @@ public void addAnnotationTo(MethodVisitor methodVisitor) {
         visitAnnotation(methodVisitor.visitAnnotation(Type.getDescriptor(annotationType), true));
     }
 
+    public static List<AnnotationMetadata> getAnnotationListWithoutRepeatable(List<AnnotationMetadata> metadata) {
+        List<AnnotationMetadata> out = new ArrayList<>();
+        Map<Class<? extends Annotation>, List<AnnotationMetadata>> repeatableAnnotationMap = new LinkedHashMap<>();
+        for (AnnotationMetadata annotation : metadata) {
+            Repeatable repeatable = annotation.annotationType().getAnnotation(Repeatable.class);
+            if (repeatable == null) {
+                out.add(annotation);
+                continue;
+            }
+            var annotationContainer = repeatable.value();
+            repeatableAnnotationMap.computeIfAbsent(annotationContainer,
+                    ignored -> new ArrayList<>()).add(annotation);
+        }
+        for (var entry : repeatableAnnotationMap.entrySet()) {
+            out.add(new AnnotationMetadata(entry.getKey(),
+                    Map.of("value", entry.getValue().toArray(AnnotationMetadata[]::new))));
+        }
+        return out;
+    }
+
+    public static Type getValueAsType(String className) {
+        return Type.getType("L" + className.replace('.', '/') + ";");
+    }
+
     private void visitAnnotation(AnnotationVisitor annotationVisitor) {
         for (var entry : annotationValueMap.entrySet()) {
             var annotationAttributeName = entry.getKey();
@@ -42,8 +70,8 @@ private void visitAnnotationAttribute(AnnotationVisitor annotationVisitor, Strin
             return;
         }
 
-        if (attributeValue instanceof Class<?> clazz) {
-            annotationVisitor.visit(attributeName, Type.getType(clazz));
+        if (attributeValue instanceof Type type) {
+            annotationVisitor.visit(attributeName, type);
             return;
         }
 

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/PythonClassTranslator.java

@@ -46,6 +46,7 @@
 import ai.timefold.jpyinterpreter.types.collections.PythonLikeTuple;
 import ai.timefold.jpyinterpreter.types.wrappers.JavaObjectWrapper;
 import ai.timefold.jpyinterpreter.types.wrappers.OpaquePythonReference;
+import ai.timefold.jpyinterpreter.util.JavaPythonClassWriter;
 import ai.timefold.jpyinterpreter.util.arguments.ArgumentSpec;
 
 import org.objectweb.asm.ClassWriter;
@@ -122,7 +123,8 @@ public static PythonLikeType translatePythonClass(PythonCompiledClass pythonComp
 
         for (Class<?> javaInterface : pythonCompiledClass.javaInterfaces) {
             javaInterfaceImplementorSet.add(
-                    new DelegatingInterfaceImplementor(internalClassName, javaInterface, instanceMethodNameToMethodDescriptor));
+                    new DelegatingInterfaceImplementor(internalClassName, javaInterface,
+                            instanceMethodNameToMethodDescriptor));
         }
 
         if (pythonCompiledClass.superclassList.isEmpty()) {
@@ -173,14 +175,14 @@ public static PythonLikeType translatePythonClass(PythonCompiledClass pythonComp
             interfaces[i] = Type.getInternalName(nonObjectInterfaceImplementors.get(i).getInterfaceClass());
         }
 
-        ClassWriter classWriter = new ClassWriter(ClassWriter.COMPUTE_MAXS | ClassWriter.COMPUTE_FRAMES);
+        ClassWriter classWriter = new JavaPythonClassWriter(ClassWriter.COMPUTE_MAXS | ClassWriter.COMPUTE_FRAMES);
 
         classWriter.visit(Opcodes.V11, Modifier.PUBLIC, internalClassName, null,
                 superClassType.getJavaTypeInternalName(), interfaces);
 
         classWriter.visitSource(pythonCompiledClass.moduleFilePath, null);
 
-        for (var annotation : pythonCompiledClass.annotations) {
+        for (var annotation : AnnotationMetadata.getAnnotationListWithoutRepeatable(pythonCompiledClass.annotations)) {
             annotation.addAnnotationTo(classWriter);
         }
 
@@ -208,19 +210,23 @@ public static PythonLikeType translatePythonClass(PythonCompiledClass pythonComp
             var typeHint = pythonCompiledClass.typeAnnotations.getOrDefault(attributeName,
                     TypeHint.withoutAnnotations(BuiltinTypes.BASE_TYPE));
             PythonLikeType type = typeHint.type();
+            PythonLikeType javaGetterType = typeHint.javaGetterType();
             if (type == null) { // null might be in __annotations__
                 type = BuiltinTypes.BASE_TYPE;
             }
 
             attributeNameToTypeMap.put(attributeName, type);
             FieldVisitor fieldVisitor;
             String javaFieldTypeDescriptor;
+            String getterTypeDescriptor;
             String signature = null;
             boolean isJavaType;
             if (type.getJavaTypeInternalName().equals(Type.getInternalName(JavaObjectWrapper.class))) {
                 javaFieldTypeDescriptor = Type.getDescriptor(type.getJavaObjectWrapperType());
-                fieldVisitor = classWriter.visitField(Modifier.PUBLIC, getJavaFieldName(attributeName), javaFieldTypeDescriptor,
-                        null, null);
+                getterTypeDescriptor = javaFieldTypeDescriptor;
+                fieldVisitor =
+                        classWriter.visitField(Modifier.PUBLIC, getJavaFieldName(attributeName), javaFieldTypeDescriptor,
+                                null, null);
                 isJavaType = true;
             } else {
                 if (typeHint.genericArgs() != null) {
@@ -229,16 +235,21 @@ public static PythonLikeType translatePythonClass(PythonCompiledClass pythonComp
                     signature = signatureWriter.toString();
                 }
                 javaFieldTypeDescriptor = 'L' + type.getJavaTypeInternalName() + ';';
-                fieldVisitor = classWriter.visitField(Modifier.PUBLIC, getJavaFieldName(attributeName), javaFieldTypeDescriptor,
-                        signature, null);
+                getterTypeDescriptor = javaGetterType.getJavaTypeDescriptor();
+                fieldVisitor =
+                        classWriter.visitField(Modifier.PUBLIC, getJavaFieldName(attributeName), javaFieldTypeDescriptor,
+                                signature, null);
                 isJavaType = false;
             }
             fieldVisitor.visitEnd();
+
             createJavaGetterSetter(classWriter, preparedClassInfo,
                     attributeName,
                     Type.getType(javaFieldTypeDescriptor),
+                    Type.getType(getterTypeDescriptor),
                     signature,
                     typeHint);
+
             FieldDescriptor fieldDescriptor =
                     new FieldDescriptor(attributeName, getJavaFieldName(attributeName), internalClassName,
                             javaFieldTypeDescriptor, type, true, isJavaType);
@@ -344,7 +355,8 @@ public static PythonLikeType translatePythonClass(PythonCompiledClass pythonComp
         pythonLikeType.$setAttribute("__module__", PythonString.valueOf(pythonCompiledClass.module));
 
         PythonLikeDict annotations = new PythonLikeDict();
-        pythonCompiledClass.typeAnnotations.forEach((name, type) -> annotations.put(PythonString.valueOf(name), type.type()));
+        pythonCompiledClass.typeAnnotations
+                .forEach((name, type) -> annotations.put(PythonString.valueOf(name), type.type()));
         pythonLikeType.$setAttribute("__annotations__", annotations);
 
         PythonLikeTuple mro = new PythonLikeTuple();
@@ -552,7 +564,7 @@ private static Class<?> createPythonWrapperMethod(String methodName, PythonCompi
         String className = maybeClassName;
         String internalClassName = className.replace('.', '/');
 
-        ClassWriter classWriter = new ClassWriter(ClassWriter.COMPUTE_MAXS | ClassWriter.COMPUTE_FRAMES);
+        ClassWriter classWriter = new JavaPythonClassWriter(ClassWriter.COMPUTE_MAXS | ClassWriter.COMPUTE_FRAMES);
 
         classWriter.visit(Opcodes.V11, Modifier.PUBLIC, internalClassName, null,
                 Type.getInternalName(Object.class), new String[] { interfaceDeclaration.interfaceName });
@@ -663,7 +675,7 @@ private static PythonLikeFunction createConstructor(String classInternalName,
         String constructorClassName = maybeClassName;
         String constructorInternalClassName = constructorClassName.replace('.', '/');
 
-        ClassWriter classWriter = new ClassWriter(ClassWriter.COMPUTE_MAXS | ClassWriter.COMPUTE_FRAMES);
+        ClassWriter classWriter = new JavaPythonClassWriter(ClassWriter.COMPUTE_MAXS | ClassWriter.COMPUTE_FRAMES);
         classWriter.visit(Opcodes.V11, Modifier.PUBLIC, constructorInternalClassName, null, Type.getInternalName(Object.class),
                 new String[] {
                         Type.getInternalName(PythonLikeFunction.class)
@@ -775,24 +787,24 @@ private static PythonLikeFunction createConstructor(String classInternalName,
 
     private static void createJavaGetterSetter(ClassWriter classWriter,
             PreparedClassInfo preparedClassInfo,
-            String attributeName, Type attributeType,
+            String attributeName, Type attributeType, Type getterType,
             String signature,
             TypeHint typeHint) {
-        createJavaGetter(classWriter, preparedClassInfo, attributeName, attributeType, signature, typeHint);
-        createJavaSetter(classWriter, preparedClassInfo, attributeName, attributeType, signature, typeHint);
+        createJavaGetter(classWriter, preparedClassInfo, attributeName, attributeType, getterType, signature, typeHint);
+        createJavaSetter(classWriter, preparedClassInfo, attributeName, attributeType, getterType, signature, typeHint);
     }
 
     private static void createJavaGetter(ClassWriter classWriter, PreparedClassInfo preparedClassInfo, String attributeName,
-            Type attributeType, String signature, TypeHint typeHint) {
+            Type attributeType, Type getterType, String signature, TypeHint typeHint) {
         var getterName = "get" + attributeName.substring(0, 1).toUpperCase() + attributeName.substring(1);
-        if (signature != null) {
+        if (signature != null && Objects.equals(attributeType, getterType)) {
             signature = "()" + signature;
         }
-        var getterVisitor = classWriter.visitMethod(Modifier.PUBLIC, getterName, Type.getMethodDescriptor(attributeType),
+        var getterVisitor = classWriter.visitMethod(Modifier.PUBLIC, getterName, Type.getMethodDescriptor(getterType),
                 signature, null);
         var maxStack = 1;
 
-        for (var annotation : typeHint.annotationList()) {
+        for (var annotation : AnnotationMetadata.getAnnotationListWithoutRepeatable(typeHint.annotationList())) {
             annotation.addAnnotationTo(getterVisitor);
         }
 
@@ -813,19 +825,22 @@ private static void createJavaGetter(ClassWriter classWriter, PreparedClassInfo
             // If branch is taken, stack is field
             // If branch is not taken, stack is null
         }
+        if (!Objects.equals(attributeType, getterType)) {
+            getterVisitor.visitTypeInsn(Opcodes.CHECKCAST, getterType.getInternalName());
+        }
         getterVisitor.visitInsn(Opcodes.ARETURN);
         getterVisitor.visitMaxs(maxStack, 0);
         getterVisitor.visitEnd();
     }
 
     private static void createJavaSetter(ClassWriter classWriter, PreparedClassInfo preparedClassInfo, String attributeName,
-            Type attributeType, String signature, TypeHint typeHint) {
+            Type attributeType, Type setterType, String signature, TypeHint typeHint) {
         var setterName = "set" + attributeName.substring(0, 1).toUpperCase() + attributeName.substring(1);
-        if (signature != null) {
+        if (signature != null && Objects.equals(attributeType, setterType)) {
             signature = "(" + signature + ")V";
         }
         var setterVisitor = classWriter.visitMethod(Modifier.PUBLIC, setterName, Type.getMethodDescriptor(Type.VOID_TYPE,
-                attributeType),
+                setterType),
                 signature, null);
         var maxStack = 2;
         setterVisitor.visitCode();
@@ -845,6 +860,9 @@ private static void createJavaSetter(ClassWriter classWriter, PreparedClassInfo
             // If branch is taken, stack is (non-null instance)
             // If branch is not taken, stack is None
         }
+        if (!Objects.equals(attributeType, setterType)) {
+            setterVisitor.visitTypeInsn(Opcodes.CHECKCAST, attributeType.getInternalName());
+        }
         setterVisitor.visitFieldInsn(Opcodes.PUTFIELD, preparedClassInfo.classInternalName,
                 attributeName, attributeType.getDescriptor());
         setterVisitor.visitInsn(Opcodes.RETURN);
@@ -855,7 +873,7 @@ private static void createJavaSetter(ClassWriter classWriter, PreparedClassInfo
     private static void addAnnotationsToMethod(PythonCompiledFunction function, MethodVisitor methodVisitor) {
         var returnTypeHint = function.typeAnnotations.get("return");
         if (returnTypeHint != null) {
-            for (var annotation : returnTypeHint.annotationList()) {
+            for (var annotation : AnnotationMetadata.getAnnotationListWithoutRepeatable(returnTypeHint.annotationList())) {
                 annotation.addAnnotationTo(methodVisitor);
             }
         }
@@ -1444,7 +1462,7 @@ public static InterfaceDeclaration createInterfaceForFunctionSignature(FunctionS
 
         String internalClassName = className.replace('.', '/');
 
-        ClassWriter classWriter = new ClassWriter(ClassWriter.COMPUTE_MAXS | ClassWriter.COMPUTE_FRAMES);
+        ClassWriter classWriter = new JavaPythonClassWriter(ClassWriter.COMPUTE_MAXS | ClassWriter.COMPUTE_FRAMES);
         classWriter.visit(Opcodes.V11, Modifier.PUBLIC | Modifier.INTERFACE | Modifier.ABSTRACT, internalClassName, null,
                 Type.getInternalName(Object.class), null);
 

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/PythonCompiledFunction.java

@@ -208,12 +208,16 @@ private static <T> Class<T> getParameterJavaClass(List<PythonLikeType> parameter
         return (Class) parameterTypeList.get(variableIndex).getJavaClassOrDefault(PythonLikeObject.class);
     }
 
+    private static String getParameterJavaClassName(List<PythonLikeType> parameterTypeList, int variableIndex) {
+        return parameterTypeList.get(variableIndex).getJavaTypeInternalName();
+    }
+
     @SuppressWarnings({ "unchecked", "rawtypes" })
     public BiFunction<PythonLikeTuple, PythonLikeDict, ArgumentSpec<PythonLikeObject>> getArgumentSpecMapper() {
         return (defaultPositionalArguments, defaultKeywordArguments) -> {
             ArgumentSpec<PythonLikeObject> out = ArgumentSpec.forFunctionReturning(qualifiedName, getReturnType()
-                    .map(type -> (Class) type.getJavaClassOrDefault(PythonLikeObject.class))
-                    .orElse(PythonLikeObject.class));
+                    .map(PythonLikeType::getJavaTypeInternalName)
+                    .orElse(PythonLikeObject.class.getName()));
 
             int variableIndex = 0;
             int defaultPositionalStartIndex = co_argcount - defaultPositionalArguments.size();
@@ -226,23 +230,23 @@ public BiFunction<PythonLikeTuple, PythonLikeDict, ArgumentSpec<PythonLikeObject
             for (; variableIndex < co_posonlyargcount; variableIndex++) {
                 if (variableIndex >= defaultPositionalStartIndex) {
                     out = out.addPositionalOnlyArgument(co_varnames.get(variableIndex),
-                            getParameterJavaClass(parameterTypeList, variableIndex),
+                            getParameterJavaClassName(parameterTypeList, variableIndex),
                             defaultPositionalArguments.get(
                                     variableIndex - defaultPositionalStartIndex));
                 } else {
                     out = out.addPositionalOnlyArgument(co_varnames.get(variableIndex),
-                            getParameterJavaClass(parameterTypeList, variableIndex));
+                            getParameterJavaClassName(parameterTypeList, variableIndex));
                 }
             }
 
             for (; variableIndex < co_argcount; variableIndex++) {
                 if (variableIndex >= defaultPositionalStartIndex) {
                     out = out.addArgument(co_varnames.get(variableIndex),
-                            getParameterJavaClass(parameterTypeList, variableIndex),
+                            getParameterJavaClassName(parameterTypeList, variableIndex),
                             defaultPositionalArguments.get(variableIndex - defaultPositionalStartIndex));
                 } else {
                     out = out.addArgument(co_varnames.get(variableIndex),
-                            getParameterJavaClass(parameterTypeList, variableIndex));
+                            getParameterJavaClassName(parameterTypeList, variableIndex));
                 }
             }
 
@@ -251,11 +255,11 @@ public BiFunction<PythonLikeTuple, PythonLikeDict, ArgumentSpec<PythonLikeObject
                         defaultKeywordArguments.get(PythonString.valueOf(co_varnames.get(variableIndex)));
                 if (maybeDefault != null) {
                     out = out.addKeywordOnlyArgument(co_varnames.get(variableIndex),
-                            getParameterJavaClass(parameterTypeList, variableIndex),
+                            getParameterJavaClassName(parameterTypeList, variableIndex),
                             maybeDefault);
                 } else {
                     out = out.addKeywordOnlyArgument(co_varnames.get(variableIndex),
-                            getParameterJavaClass(parameterTypeList, variableIndex));
+                            getParameterJavaClassName(parameterTypeList, variableIndex));
                 }
                 variableIndex++;
             }