diff --git a/src/annotation.js b/src/annotation.js
index 7568c06d4..6a82cb074 100644
--- a/src/annotation.js
+++ b/src/annotation.js
@@ -108,14 +108,16 @@ export default {
       },
     },
     clip: true,
-    drawTime: 'afterDatasetsDraw',
-    interaction: {
-      mode: undefined,
-      axis: undefined,
-      intersect: undefined
-    },
-    label: {
-      drawTime: null
+    dblClickSpeed: 350, // ms
+    common: {
+      drawTime: 'afterDatasetsDraw',
+      interaction: {
+        mode: undefined,
+        axis: undefined,
+        intersect: undefined
+      },
+      label: {
+      }
     }
   },
 
@@ -124,10 +126,15 @@ export default {
     _scriptable: (prop) => !hooks.includes(prop),
     annotations: {
       _allKeys: false,
-      _fallback: (prop, opts) => `elements.${annotationTypes[resolveType(opts.type)].id}`,
+      _fallback: (prop, opts) => `elements.${annotationTypes[resolveType(opts.type)].id}`
     },
-    interaction: {
-      _fallback: true,
+    common: {
+      interaction: {
+        _fallback: true
+      },
+      label: {
+        _fallback: true
+      }
     }
   },
 
@@ -155,7 +162,7 @@ function draw(chart, caller, clip) {
       return;
     }
     const label = el.options.label;
-    if (label && label.display && label.content && (label.drawTime || el.options.drawTime) === caller) {
+    if (label && label.display && label.content && label.drawTime === caller) {
       el.drawLabel(ctx, chartArea);
     }
   });
diff --git a/src/types/index.js b/src/types/index.js
index 79fce3a19..e7879da25 100644
--- a/src/types/index.js
+++ b/src/types/index.js
@@ -34,6 +34,6 @@ export {
  */
 Object.keys(annotationTypes).forEach(key => {
   defaults.describe(`elements.${annotationTypes[key].id}`, {
-    _fallback: 'plugins.annotation'
+    _fallback: 'plugins.annotation.common'
   });
 });
diff --git a/test/fixtures/line/label_drawTime.js b/test/fixtures/line/label_drawTime.js
index ca57021bb..5d33d3a5d 100644
--- a/test/fixtures/line/label_drawTime.js
+++ b/test/fixtures/line/label_drawTime.js
@@ -1,6 +1,13 @@
 module.exports = {
   config: {
     type: 'scatter',
+    data: {
+      datasets: [{
+        backgroundColor: 'rgba(255,165,0,0.9)',
+        radius: 20,
+        data: [{x: 15, y: 25}, {x: 35, y: 50}, {x: 50, y: 95}, {x: 82, y: 75}, {x: 82, y: 5}]
+      }]
+    },
     options: {
       scales: {
         x: {
@@ -15,8 +22,11 @@ module.exports = {
         }
       },
       plugins: {
+        legend: false,
         annotation: {
-          drawTime: 'beforeDraw',
+          common: {
+            drawTime: 'beforeDraw',
+          },
           annotations: {
             left: {
               drawTime: 'afterDraw',
@@ -39,7 +49,7 @@ module.exports = {
               borderColor: 'black',
               borderWidth: 5,
               label: {
-                drawTime: 'beforeDraw',
+                drawTime: 'afterDraw',
                 position: 'center',
                 backgroundColor: 'red',
                 content: 'beforeDraw/afterDraw',
@@ -56,7 +66,7 @@ module.exports = {
               label: {
                 drawTime: 'beforeDraw',
                 position: 'end',
-                backgroundColor: 'black',
+                backgroundColor: 'green',
                 content: 'afterDraw/beforeDraw',
                 display: true
               },
diff --git a/test/fixtures/line/label_drawTime.png b/test/fixtures/line/label_drawTime.png
index b112c74b0..eead1ff7b 100644
Binary files a/test/fixtures/line/label_drawTime.png and b/test/fixtures/line/label_drawTime.png differ
diff --git a/test/specs/annotation.spec.js b/test/specs/annotation.spec.js
index 9a7200e65..1fb7c884b 100644
--- a/test/specs/annotation.spec.js
+++ b/test/specs/annotation.spec.js
@@ -142,4 +142,76 @@ describe('Annotation plugin', function() {
     console.warn = origWarn;
   });
 
+  describe('Annotation option resolution', function() {
+    it('should resolve from plugin common options', function() {
+      const chart = acquireChart({
+        type: 'line',
+        options: {
+          plugins: {
+            annotation: {
+              common: {
+                drawTime: 'fallback',
+              },
+              annotations: {
+                test: {
+                  type: 'line'
+                }
+              }
+            }
+          }
+        }
+      });
+      const state = window['chartjs-plugin-annotation']._getState(chart);
+      const element = state.elements[0];
+      expect(element.options.drawTime).toBe('fallback');
+    });
+
+    it('should not resolve from same sub key in plugin options', function() {
+      // https://github.com/chartjs/chartjs-plugin-annotation/issues/625
+      // https://github.com/chartjs/chartjs-plugin-annotation/pull/626#issuecomment-1012960850
+      const chart = acquireChart({
+        type: 'line',
+        options: {
+          plugins: {
+            annotation: {
+              label: {
+                drawTime: 'this should not be read'
+              },
+              annotations: {
+                label: {
+                  type: 'line'
+                }
+              }
+            }
+          }
+        }
+      });
+      const state = window['chartjs-plugin-annotation']._getState(chart);
+      const element = state.elements[0];
+      expect(element.options.drawTime).toBe('afterDatasetsDraw');
+    });
+
+    it('should resolve to same options through chart options', function() {
+      const chart = acquireChart({
+        type: 'line',
+        options: {
+          plugins: {
+            annotation: {
+              label: {
+                drawTime: 'this should not be read'
+              },
+              annotations: {
+                label: {
+                  type: 'line'
+                }
+              }
+            }
+          }
+        }
+      });
+      const state = window['chartjs-plugin-annotation']._getState(chart);
+      const element = state.elements[0];
+      expect(element.options.drawTime).toBe(chart.options.plugins.annotation.annotations.label.drawTime);
+    });
+  });
 });