From 4c2a711f0d866e34bbbb6e3323640e1a9397aa08 Mon Sep 17 00:00:00 2001 From: Henry Walshaw Date: Wed, 28 Aug 2024 12:04:44 +1000 Subject: [PATCH 1/7] Add the croniter library to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 4968d1e..8b443ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ APScheduler==3.6.1 passlib==1.7.1 Werkzeug==0.16.1 tzlocal<3.0 # Fix based on https://github.com/Yelp/elastalert/issues/2968 +croniter From 1e1d7966a0df99d04790c87b31908d442450945f Mon Sep 17 00:00:00 2001 From: Henry Walshaw Date: Wed, 28 Aug 2024 11:48:58 +1000 Subject: [PATCH 2/7] Add a cron column with a validator. The goal is that the user can optionally store a cron value against a Resource to set the exact time a particular probe is run rather than the frequency with a random starting point. --- GeoHealthCheck/models.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index 2c09b51..dcd030c 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -35,7 +35,7 @@ from itsdangerous import TimedJSONWebSignatureSerializer as Serializer from sqlalchemy import func, and_ -from sqlalchemy.orm import deferred +from sqlalchemy.orm import deferred, validates from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound import util @@ -45,6 +45,7 @@ from resourceauth import ResourceAuth from wtforms.validators import Email, ValidationError from owslib.util import bind_url +from croniter import croniter, CroniterBadCronError APP = App.get_app() DB = App.get_db() @@ -404,6 +405,20 @@ class Resource(DB.Model): tags = DB.relationship('Tag', secondary=resource_tags, backref='resource') run_frequency = DB.Column(DB.Integer, default=60) _auth = DB.Column('auth', DB.Text, nullable=True, default=None) + cron = DB.Column(DB.Text, nullable=True, default=None) + + @validates('cron') + def validate_cron(self, key, cron): + if cron == "": + # set null over an empty string + return None + + try: + croniter(cron) + except CroniterBadCronError as error: + raise ValueError(f"Bad cron pattern '{cron}': {str(error)}") from error + + return cron def __init__(self, owner, resource_type, title, url, tags, auth=None): self.resource_type = resource_type @@ -653,6 +668,7 @@ def for_json(self): 'owner': self.owner.username, 'owner_identifier': self.owner.identifier, 'run_frequency': self.run_frequency, + 'cron': self.cron, 'reliability': round(self.reliability, 1) } From 00aa2f1b9f357e3a3894e3cfd1502485da1d222a Mon Sep 17 00:00:00 2001 From: Henry Walshaw Date: Wed, 28 Aug 2024 12:02:19 +1000 Subject: [PATCH 3/7] ``add_job`` in scheduler preferences the cron formula if available If the resource has a ``cron`` set then use it to determine the trigger for the job rather than the ``run_frequency`` (which is ignored). --- GeoHealthCheck/scheduler.py | 42 ++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index 37d56d6..92cb884 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -40,6 +40,8 @@ from apscheduler.events import \ EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, \ EVENT_JOB_MISSED, EVENT_JOB_ERROR +from apscheduler.triggers.cron import CronTrigger +from croniter import croniter from init import App LOGGER = logging.getLogger(__name__) @@ -243,14 +245,38 @@ def update_job(resource): def add_job(resource): LOGGER.info('Starting job for resource=%d' % resource.identifier) - freq = resource.run_frequency - next_run_time = datetime.now() + timedelta( - seconds=random.randint(0, freq * 60)) - scheduler.add_job( - run_job, 'interval', args=[resource.identifier, freq], - minutes=freq, next_run_time=next_run_time, max_instances=1, - misfire_grace_time=round((freq * 60) / 2), coalesce=True, - id=str(resource.identifier)) + + if resource.cron: + # determine the frequency. Used as input to ``run_job`` and to determine misfire + # grace time + cron = croniter(resource.cron) + # Use the next two execution times to determine time difference + seconds = abs(cron.get_next() - cron.get_next()) + + LOGGER.info(f'scheduling job with cron pattern ({seconds})') + + scheduler.add_job( + run_job, + CronTrigger.from_crontab(resource.cron), + args=[resource.identifier, round(seconds / 60)], + max_instances=1, + misfire_grace_time=round(seconds / 2), + coalesce=True, + id=str(resource.identifier) + ) + + else: + freq = resource.run_frequency + next_run_time = datetime.now() + timedelta( + seconds=random.randint(0, freq * 60)) + + LOGGER.info('scheduling job with frequency') + + scheduler.add_job( + run_job, 'interval', args=[resource.identifier, freq], + minutes=freq, next_run_time=next_run_time, max_instances=1, + misfire_grace_time=round((freq * 60) / 2), coalesce=True, + id=str(resource.identifier)) def stop_job(resource_id): From dae7b26d7858c7c26fd2cab5f78974740915e290 Mon Sep 17 00:00:00 2001 From: Henry Walshaw Date: Wed, 28 Aug 2024 14:01:14 +1000 Subject: [PATCH 4/7] Add migration to add the ``cron`` column to the ``resource`` table --- ...ebc78_add_cron_column_to_resource_table.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 GeoHealthCheck/migrations/versions/2f9f07aebc78_add_cron_column_to_resource_table.py diff --git a/GeoHealthCheck/migrations/versions/2f9f07aebc78_add_cron_column_to_resource_table.py b/GeoHealthCheck/migrations/versions/2f9f07aebc78_add_cron_column_to_resource_table.py new file mode 100644 index 0000000..a20d1db --- /dev/null +++ b/GeoHealthCheck/migrations/versions/2f9f07aebc78_add_cron_column_to_resource_table.py @@ -0,0 +1,25 @@ +"""Add cron column to Resource table + +Revision ID: 2f9f07aebc78 +Revises: 933717a14052 +Create Date: 2024-08-28 04:00:34.863188 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '2f9f07aebc78' +down_revision = '933717a14052' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('resource', sa.Column('cron', sa.Text(), nullable=True)) + + +def downgrade(): + op.drop_column('resource', 'cron') From a84f7247c4a103fb1a742ed9e3bce0d9f2189769 Mon Sep 17 00:00:00 2001 From: Henry Walshaw Date: Wed, 28 Aug 2024 14:02:15 +1000 Subject: [PATCH 5/7] Set ``nullable=False`` to match migrations ``Resource.run_frequency`` and ``ResourceLock.resource_identifier`` are not nullable according to the migrations, so that's now reflected in the model code. --- GeoHealthCheck/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index dcd030c..ed5c7c1 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -403,7 +403,7 @@ class Resource(DB.Model): owner = DB.relationship('User', backref=DB.backref('username2', lazy='dynamic')) tags = DB.relationship('Tag', secondary=resource_tags, backref='resource') - run_frequency = DB.Column(DB.Integer, default=60) + run_frequency = DB.Column(DB.Integer, default=60, nullable=False) _auth = DB.Column('auth', DB.Text, nullable=True, default=None) cron = DB.Column(DB.Text, nullable=True, default=None) @@ -680,7 +680,8 @@ class ResourceLock(DB.Model): primary_key=True, autoincrement=False, unique=True) resource_identifier = DB.Column(DB.Integer, DB.ForeignKey('resource.identifier'), - unique=True) + unique=True, + nullable=False) resource = DB.relationship('Resource', backref=DB.backref('locks', lazy='dynamic', cascade="all,delete")) From 74b9c376d463f9332cb960faf5c6bde41b71bfa6 Mon Sep 17 00:00:00 2001 From: Henry Walshaw Date: Wed, 28 Aug 2024 14:20:22 +1000 Subject: [PATCH 6/7] Add the crontab fields to the form --- GeoHealthCheck/templates/edit_resource.html | 15 ++++++++++++--- GeoHealthCheck/templates/resource.html | 11 ++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/GeoHealthCheck/templates/edit_resource.html b/GeoHealthCheck/templates/edit_resource.html index 83bb156..d3bab9b 100644 --- a/GeoHealthCheck/templates/edit_resource.html +++ b/GeoHealthCheck/templates/edit_resource.html @@ -138,6 +138,11 @@

[{{ _('Edit') }}] {{ resour + + {{ _('crontab pattern') }} + + + Probes @@ -317,7 +322,7 @@

[{{ _('Edit') }}] {{ resour $('#save').click(function() { $('#toggle-status-detail').hide(); - + // Resource auth: get and map input elements for auth type var auth_type = $('select[name="select_resource_auth_type_val"]').val(); var new_auth = { @@ -335,7 +340,7 @@

[{{ _('Edit') }}] {{ resour } new_auth_data[elm.data('auth-param')] = elm.val(); }); - + // Collect title var new_title = $('input[name="resource_title_value"]').val(); @@ -345,6 +350,9 @@

[{{ _('Edit') }}] {{ resour // Collect test_frequency var new_frequency = $('input[name="resource_frequency_value"]').val(); + // Collect cron pattern + var new_cron = $('input[name="resource_cron"]').val(); + // Collect active var new_active = $('#input_resource_active').prop('checked'); @@ -425,6 +433,7 @@

[{{ _('Edit') }}] {{ resour active: new_active, tags: new_tags, run_frequency: new_frequency, + cron: new_cron, probes: new_probes, notify_emails: new_notify_emails, notify_webhooks: new_notify_webhooks @@ -543,7 +552,7 @@

[{{ _('Edit') }}] {{ resour 'focus': function(){return false}} $('#input_resource_notify_emails').autocomplete(ac_params); - // quick'n'dirty autoheight for text fields, + // quick'n'dirty autoheight for text fields, // https://stackoverflow.com/a/10080841/35735 // http://jsfiddle.net/SpYk3/m8Qk9/ $(function() { diff --git a/GeoHealthCheck/templates/resource.html b/GeoHealthCheck/templates/resource.html index c949a16..266b7e3 100644 --- a/GeoHealthCheck/templates/resource.html +++ b/GeoHealthCheck/templates/resource.html @@ -21,13 +21,13 @@ /*displays pop-up when "active" class is present*/ visibility:visible; } - + .run-popup-content { /*Hides pop-up content when there is no "active" class */ margin: 12px; visibility:hidden; } - + .run-popup-content.active { /*Shows pop-up content when "active" class is present */ visibility:visible; @@ -83,10 +83,15 @@

+ {% if resource.cron %} + {{ _('crontab pattern') }} + {{ resource.cron }} + {% else %} {{ _('Run Every') }} {{ resource.run_frequency }} {{ _('minutes') }} + {% endif %} Probes @@ -263,7 +268,7 @@

{{ _('Download') }}: