Fix #1543

Author: cowtowncoder

release-notes/CREDITS

@@ -587,6 +587,10 @@ Stephan Schroevers (Stephan202@github)
   * Reported #1505: @JsonEnumDefaultValue should take precedence over FAIL_ON_NUMBERS_FOR_ENUMS
    (2.8.7)
 
+Alex Panchenko (panchenko@github)
+  * Reported #1543: JsonFormat.Shape.NUMBER_INT does not work when defined on enum type in 2.8
+   (2.8.8)
+
 Connor Kuhn (ckuhn@github)
   * Contributed #1341: FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY
    (2.9.0)

release-notes/VERSION

@@ -6,6 +6,8 @@ Project: jackson-databind
 2.8.8 (not yet released)
 
 #1533: `AsPropertyTypeDeserializer` ignores `DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT`
+#1543: JsonFormat.Shape.NUMBER_INT does not work when defined on enum type in 2.8
+ (reported by Alex P)
 - Minor fix to creation of `PropertyMetadata`, had one path that could lead to NPE
 
 2.8.7 (21-Feb-2017)

src/main/java/com/fasterxml/jackson/databind/ser/std/EnumSerializer.java

@@ -67,7 +67,7 @@ public EnumSerializer(EnumValues v, Boolean serializeAsIndex)
         _values = v;
         _serializeAsIndex = serializeAsIndex;
     }
-    
+
     /**
      * Factory method used by {@link com.fasterxml.jackson.databind.ser.BasicSerializerFactory}
      * for constructing serializer instance of Enum types.
@@ -83,7 +83,7 @@ public static EnumSerializer construct(Class<?> enumClass, SerializationConfig c
          *   handle toString() case dynamically (for example)
          */
         EnumValues v = EnumValues.constructFromName(config, (Class<Enum<?>>) enumClass);
-        Boolean serializeAsIndex = _isShapeWrittenUsingIndex(enumClass, format, true);
+        Boolean serializeAsIndex = _isShapeWrittenUsingIndex(enumClass, format, true, null);
         return new EnumSerializer(v, serializeAsIndex);
     }
 
@@ -100,7 +100,8 @@ public JsonSerializer<?> createContextual(SerializerProvider serializers,
             JsonFormat.Value format = findFormatOverrides(serializers,
                     property, handledType());
             if (format != null) {
-                Boolean serializeAsIndex = _isShapeWrittenUsingIndex(property.getType().getRawClass(), format, false);
+                Boolean serializeAsIndex = _isShapeWrittenUsingIndex(property.getType().getRawClass(),
+                        format, false, _serializeAsIndex);
                 if (serializeAsIndex != _serializeAsIndex) {
                     return new EnumSerializer(_values, serializeAsIndex);
                 }
@@ -209,18 +210,20 @@ protected final boolean _serializeAsIndex(SerializerProvider serializers)
     }
 
     /**
-     * Helper method called to check whether 
+     * Helper method called to check whether serialization should be done using
+     * index (number) or not.
      */
     protected static Boolean _isShapeWrittenUsingIndex(Class<?> enumClass,
-            JsonFormat.Value format, boolean fromClass)
+            JsonFormat.Value format, boolean fromClass,
+            Boolean defaultValue)
     {
         JsonFormat.Shape shape = (format == null) ? null : format.getShape();
         if (shape == null) {
-            return null;
+            return defaultValue;
         }
         // i.e. "default", check dynamically
         if (shape == Shape.ANY || shape == Shape.SCALAR) {
-            return null;
+            return defaultValue;
         }
         // 19-May-2016, tatu: also consider "natural" shape
         if (shape == Shape.STRING || shape == Shape.NATURAL) {
@@ -230,9 +233,9 @@ protected static Boolean _isShapeWrittenUsingIndex(Class<?> enumClass,
         if (shape.isNumeric() || (shape == Shape.ARRAY)) {
             return Boolean.TRUE;
         }
-        throw new IllegalArgumentException("Unsupported serialization shape ("+shape+") for Enum "+enumClass.getName()
-                    +", not supported as "
-                    + (fromClass? "class" : "property")
-                    +" annotation");
+        // 07-Mar-2017, tatu: Also means `OBJECT` not available as property annotation...
+        throw new IllegalArgumentException(String.format(
+                "Unsupported serialization shape (%s) for Enum %s, not supported as %s annotation",
+                    shape, enumClass.getName(), (fromClass? "class" : "property")));
     }
 }

src/test/java/com/fasterxml/jackson/databind/struct/EnumFormatShapeTest.java

@@ -104,10 +104,13 @@ public void testOverrideEnumAsNumber() throws Exception {
     }
 
     // for [databind#1543]
-    public void testEnumAsNumber() throws Exception {
+    public void testEnumValueAsNumber() throws Exception {
         assertEquals(String.valueOf(Color.GREEN.ordinal()),
                 MAPPER.writeValueAsString(Color.GREEN));
-        assertEquals(String.format(aposToQuotes("{'color':'%s'}"), Color.GREEN.ordinal()),
+    }
+
+    public void testEnumPropertyAsNumber() throws Exception {
+        assertEquals(String.format(aposToQuotes("{'color':%s}"), Color.GREEN.ordinal()),
                 MAPPER.writeValueAsString(new ColorWrapper(Color.GREEN)));
     }
 }