From 47be6f2178969abcfd62a8d648efdfef7912a939 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 27 Jan 2025 09:21:50 -0500 Subject: [PATCH 01/12] Add origin & origin_ptr to Branch model --- docs/models/branch.md | 8 +++++ netbox_branching/api/serializers.py | 30 +++++++++++++++++-- netbox_branching/filtersets.py | 10 +++++++ .../migrations/0003_branch_cloning.py | 22 ++++++++++++++ netbox_branching/models/branches.py | 13 ++++++++ netbox_branching/tables/tables.py | 13 ++++++-- .../templates/netbox_branching/branch.html | 10 +++++++ 7 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 netbox_branching/migrations/0003_branch_cloning.py diff --git a/docs/models/branch.md b/docs/models/branch.md index 1ecbf59..ec46be9 100644 --- a/docs/models/branch.md +++ b/docs/models/branch.md @@ -32,6 +32,14 @@ The current status of the branch. This must be one of the following values. | Archived | A merged branch which has been deprovisioned in the database | | Failed | Provisioning the schema for this branch has failed | +### Origin + +The branch from which this branch was cloned (if any). + +### Origin Pointer + +The last change record belonging to the origin branch successfully applied to this branch. + ### Last Sync The time at which this branch was most recently synchronized with main. This value will be null if the branch has never been synchronized. diff --git a/netbox_branching/api/serializers.py b/netbox_branching/api/serializers.py index d7b24c6..9c7dbab 100644 --- a/netbox_branching/api/serializers.py +++ b/netbox_branching/api/serializers.py @@ -4,7 +4,7 @@ from core.choices import ObjectChangeActionChoices from netbox.api.exceptions import SerializerNotFound from netbox.api.fields import ChoiceField, ContentTypeField -from netbox.api.serializers import NetBoxModelSerializer +from netbox.api.serializers import NetBoxModelSerializer, WritableNestedSerializer from netbox_branching.choices import BranchEventTypeChoices, BranchStatusChoices from netbox_branching.models import ChangeDiff, Branch, BranchEvent from users.api.serializers import UserSerializer @@ -15,9 +15,27 @@ 'BranchEventSerializer', 'ChangeDiffSerializer', 'CommitSerializer', + 'NestedBranchSerializer', ) +class NestedBranchSerializer(WritableNestedSerializer): + url = serializers.HyperlinkedIdentityField( + view_name='plugins-api:netbox_branching-api:branch-detail' + ) + status = ChoiceField( + choices=BranchStatusChoices + ) + owner = UserSerializer( + nested=True, + read_only=True + ) + + class Meta: + model = Branch + fields = ['id', 'url', 'display_url', 'display', 'name', 'status', 'owner', 'description'] + + class BranchSerializer(NetBoxModelSerializer): url = serializers.HyperlinkedIdentityField( view_name='plugins-api:netbox_branching-api:branch-detail' @@ -26,6 +44,12 @@ class BranchSerializer(NetBoxModelSerializer): nested=True, read_only=True ) + origin = NestedBranchSerializer( + read_only=True + ) + origin_ptr = serializers.IntegerField( + read_only=True + ) merged_by = UserSerializer( nested=True, read_only=True @@ -37,8 +61,8 @@ class BranchSerializer(NetBoxModelSerializer): class Meta: model = Branch fields = [ - 'id', 'url', 'display', 'name', 'status', 'owner', 'description', 'schema_id', 'last_sync', 'merged_time', - 'merged_by', 'comments', 'tags', 'custom_fields', 'created', 'last_updated', + 'id', 'url', 'display', 'name', 'origin', 'origin_ptr', 'status', 'owner', 'description', 'schema_id', + 'last_sync', 'merged_time', 'merged_by', 'comments', 'tags', 'custom_fields', 'created', 'last_updated', ] brief_fields = ('id', 'url', 'display', 'name', 'status', 'description') diff --git a/netbox_branching/filtersets.py b/netbox_branching/filtersets.py index fa6cbd9..6a158cc 100644 --- a/netbox_branching/filtersets.py +++ b/netbox_branching/filtersets.py @@ -21,6 +21,16 @@ class BranchFilterSet(NetBoxModelFilterSet): choices=BranchStatusChoices, null_value=None ) + origin_id = django_filters.ModelMultipleChoiceFilter( + queryset=Branch.objects.all(), + label=_('Origin (ID)'), + ) + origin = django_filters.ModelMultipleChoiceFilter( + field_name='origin__schema_id', + queryset=Branch.objects.all(), + to_field_name='schema_id', + label=_('Origin (schema ID)'), + ) last_sync = filters.MultiValueDateTimeFilter() class Meta: diff --git a/netbox_branching/migrations/0003_branch_cloning.py b/netbox_branching/migrations/0003_branch_cloning.py new file mode 100644 index 0000000..5906854 --- /dev/null +++ b/netbox_branching/migrations/0003_branch_cloning.py @@ -0,0 +1,22 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('netbox_branching', '0002_branch_schema_id_unique'), + ] + + operations = [ + migrations.AddField( + model_name='branch', + name='origin', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='clones', to='netbox_branching.branch'), + ), + migrations.AddField( + model_name='branch', + name='origin_ptr', + field=models.PositiveBigIntegerField(blank=True, null=True), + ), + ] diff --git a/netbox_branching/models/branches.py b/netbox_branching/models/branches.py index 517b0cc..5ab63d3 100644 --- a/netbox_branching/models/branches.py +++ b/netbox_branching/models/branches.py @@ -50,6 +50,19 @@ class Branch(JobsMixin, PrimaryModel): null=True, related_name='branches' ) + origin = models.ForeignKey( + to='self', + on_delete=models.PROTECT, + blank=True, + null=True, + related_name='clones', + help_text=_("The branch from which this branch was cloned.") + ) + origin_ptr = models.PositiveBigIntegerField( + blank=True, + null=True, + help_text=_("The last successfully applied change from the original branch.") + ) schema_id = models.CharField( max_length=8, unique=True, diff --git a/netbox_branching/tables/tables.py b/netbox_branching/tables/tables.py index b1165b3..831db4a 100644 --- a/netbox_branching/tables/tables.py +++ b/netbox_branching/tables/tables.py @@ -59,6 +59,13 @@ class BranchTable(NetBoxTable): status = columns.ChoiceFieldColumn( verbose_name=_('Status'), ) + origin = tables.Column( + linkify=True, + verbose_name=_('Origin') + ) + origin_ptr = tables.Column( + verbose_name=_('Origin Pointer') + ) conflicts = ConflictsColumn( verbose_name=_('Conflicts') ) @@ -72,11 +79,11 @@ class BranchTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = Branch fields = ( - 'pk', 'id', 'name', 'is_active', 'status', 'conflicts', 'schema_id', 'description', 'owner', 'tags', - 'created', 'last_updated', + 'pk', 'id', 'name', 'is_active', 'status', 'origin', 'origin_ptr', 'conflicts', 'schema_id', 'description', + 'owner', 'tags', 'created', 'last_updated', ) default_columns = ( - 'pk', 'name', 'is_active', 'status', 'owner', 'conflicts', 'schema_id', 'description', + 'pk', 'name', 'is_active', 'status', 'origin', 'owner', 'conflicts', 'schema_id', 'description', ) def render_is_active(self, value): diff --git a/netbox_branching/templates/netbox_branching/branch.html b/netbox_branching/templates/netbox_branching/branch.html index 75eff0c..91dc6d8 100644 --- a/netbox_branching/templates/netbox_branching/branch.html +++ b/netbox_branching/templates/netbox_branching/branch.html @@ -72,6 +72,16 @@
{% trans "Branch" %}
{% endif %} + + {% trans "Origin" %} + + {% if object.origin %} + {{ object.origin|linkify }} ({{ object.origin_ptr }}) + {% else %} + {{ ''|placeholder }} + {% endif %} + + {% trans "Owner" %} {{ object.owner }} From 55833942d1f199b7928e1dcd56b2e75d1c0963a5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 27 Jan 2025 13:46:47 -0500 Subject: [PATCH 02/12] Introduce the ability to replay changes from one branch onto another --- netbox_branching/forms/model_forms.py | 21 +++++++-- netbox_branching/jobs.py | 7 ++- netbox_branching/models/branches.py | 64 +++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/netbox_branching/forms/model_forms.py b/netbox_branching/forms/model_forms.py index 69fab95..eeec4ee 100644 --- a/netbox_branching/forms/model_forms.py +++ b/netbox_branching/forms/model_forms.py @@ -1,7 +1,8 @@ -from netbox_branching.models import Branch +from django.utils.translation import gettext_lazy as _ from netbox.forms import NetBoxModelForm -from utilities.forms.fields import CommentField +from netbox_branching.models import Branch +from utilities.forms.fields import CommentField, DynamicModelChoiceField from utilities.forms.rendering import FieldSet __all__ = ( @@ -11,10 +12,22 @@ class BranchForm(NetBoxModelForm): fieldsets = ( - FieldSet('name', 'description', 'tags'), + FieldSet('name', 'origin', 'description', 'tags'), + ) + origin = DynamicModelChoiceField( + label=_('Origin'), + queryset=Branch.objects.all(), + required=False ) comments = CommentField() class Meta: model = Branch - fields = ('name', 'description', 'comments', 'tags') + fields = ('name', 'origin', 'description', 'comments', 'tags') + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if self.instance.pk: + # Originating branch is cannot be modified + self.fields['origin'].disabled = True diff --git a/netbox_branching/jobs.py b/netbox_branching/jobs.py index 13f0199..a5f8872 100644 --- a/netbox_branching/jobs.py +++ b/netbox_branching/jobs.py @@ -38,9 +38,14 @@ def run(self, *args, **kwargs): logger.setLevel(logging.DEBUG) logger.addHandler(ListHandler(queue=get_job_log(self.job))) - # Provision the Branch + # Provision the Branch by copying the main schema branch = self.job.object branch.provision(user=self.job.user) + branch.refresh_from_db() + + # If the Branch specifies an origin, replay changes from it + if branch.origin: + branch.replay(logger=logger) class SyncBranchJob(JobRunner): diff --git a/netbox_branching/models/branches.py b/netbox_branching/models/branches.py index 5ab63d3..697e6f9 100644 --- a/netbox_branching/models/branches.py +++ b/netbox_branching/models/branches.py @@ -115,6 +115,16 @@ def get_absolute_url(self): def get_status_color(self): return BranchStatusChoices.colors.get(self.status) + def clone(self): + """ + Override CloningMixin's clone() method to nullify active branch and set origin ID. + """ + return { + '_branch': '', + 'origin': self.pk, + **super().clone(), + } + @cached_property def is_active(self): return self == active_branch.get() @@ -365,6 +375,60 @@ def sync(self, user, commit=True): sync.alters_data = True + def replay(self, logger=None): + """ + Replay changes from the originating Branch onto this Branch. + """ + logger = logger or logging.getLogger('netbox_branching.branch.replay') + + if not self.origin: + raise Exception(f"Cannot replay changes onto branch {self}: No origin branch is defined.") + logger.info(f'Replaying changes from branch {self.origin} onto branch {self} ({self.schema_name})') + + if not self.ready: + raise Exception(f"Branch {self} is not ready for replay") + + # Fetch changes to apply from originating Branch + changes = ObjectChange.objects.using(self.origin.connection_name).filter( + changed_object_type__in=get_branchable_object_types(), + pk__gt=self.origin_ptr or 0 + ).order_by('pk') + if changes: + logger.info(f"Found {len(changes)} changes to replay") + else: + logger.info(f"No changes found; aborting.") + return + + # Create a dummy request for the event_tracking() context manager + request = RequestFactory().get(reverse('home')) + + # Apply each change from the origin schema + try: + with activate_branch(self): + with event_tracking(request): + for change in changes: + request.id = change.request_id + request.user = change.user + change.apply(using=self.connection_name, logger=logger) + self.origin_ptr = change.pk + + except Exception as e: + if err_message := str(e): + logger.error(err_message) + # Restore original branch status + active_branch.set(None) + Branch.objects.filter(pk=self.pk).update(status=BranchStatusChoices.READY, origin_ptr=self.origin_ptr) + + # Record the branch's last_synced time & update its status + logger.debug(f"Setting branch status to {BranchStatusChoices.READY}") + self.last_sync = timezone.now() + self.status = BranchStatusChoices.READY + self.save() + + logger.info('Replay completed') + + replay.alters_data = True + def merge(self, user, commit=True): """ Apply all changes in the Branch to the main schema by replaying them in From 00adcd2b2d3c5464e9e9522ea11857251c93c6fd Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 28 Jan 2025 09:22:27 -0500 Subject: [PATCH 03/12] Log replay failures --- netbox_branching/choices.py | 2 ++ netbox_branching/models/branches.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/netbox_branching/choices.py b/netbox_branching/choices.py index 90c6f46..07b915b 100644 --- a/netbox_branching/choices.py +++ b/netbox_branching/choices.py @@ -46,6 +46,7 @@ class BranchEventTypeChoices(ChoiceSet): MERGED = 'merged' REVERTED = 'reverted' ARCHIVED = 'archived' + REPLAY_FAILED = 'replay-failed' CHOICES = ( (PROVISIONED, _('Provisioned'), 'green'), @@ -53,4 +54,5 @@ class BranchEventTypeChoices(ChoiceSet): (MERGED, _('Merged'), 'blue'), (REVERTED, _('Reverted'), 'orange'), (ARCHIVED, _('Archived'), 'gray'), + (REPLAY_FAILED, _('Replay failed'), 'red'), ) diff --git a/netbox_branching/models/branches.py b/netbox_branching/models/branches.py index 697e6f9..6cbea29 100644 --- a/netbox_branching/models/branches.py +++ b/netbox_branching/models/branches.py @@ -413,8 +413,11 @@ def replay(self, logger=None): self.origin_ptr = change.pk except Exception as e: - if err_message := str(e): - logger.error(err_message) + # Record the replay failure + logger.error(str(e)) + logger.debug(f"Recording branch event: {BranchEventTypeChoices.REPLAY_FAILED}") + BranchEvent.objects.create(branch=self, type=BranchEventTypeChoices.REPLAY_FAILED) + # Restore original branch status active_branch.set(None) Branch.objects.filter(pk=self.pk).update(status=BranchStatusChoices.READY, origin_ptr=self.origin_ptr) From 6fe9de9d6d939cdb1ce24d24dda094ff9e6a8d51 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 28 Jan 2025 15:22:24 -0500 Subject: [PATCH 04/12] Add replay functionality detached from provisioning flow --- netbox_branching/forms/misc.py | 15 ++++++ netbox_branching/jobs.py | 24 ++++++++- netbox_branching/models/branches.py | 44 ++++++++++++++--- netbox_branching/models/changes.py | 8 +++ netbox_branching/tables/tables.py | 32 ++++++++++++ netbox_branching/tables/template_code.py | 5 ++ .../templates/netbox_branching/branch.html | 4 +- .../netbox_branching/branch_replay.html | 33 +++++++++++++ .../buttons/branch_replay.html | 6 +++ .../templatetags/branch_buttons.py | 9 ++++ netbox_branching/views.py | 49 ++++++++++++++++--- 11 files changed, 214 insertions(+), 15 deletions(-) create mode 100644 netbox_branching/tables/template_code.py create mode 100644 netbox_branching/templates/netbox_branching/branch_replay.html create mode 100644 netbox_branching/templates/netbox_branching/buttons/branch_replay.html diff --git a/netbox_branching/forms/misc.py b/netbox_branching/forms/misc.py index 1bbc590..ba28e2d 100644 --- a/netbox_branching/forms/misc.py +++ b/netbox_branching/forms/misc.py @@ -1,10 +1,12 @@ from django import forms from django.utils.translation import gettext_lazy as _ +from core.models import ObjectChange from netbox_branching.models import ChangeDiff __all__ = ( 'BranchActionForm', + 'BranchReplayForm', 'ConfirmationForm', ) @@ -42,6 +44,19 @@ def clean(self): return self.cleaned_data +class BranchReplayForm(BranchActionForm): + # TODO: Populate via REST API + start = forms.ModelChoiceField( + queryset=ObjectChange.objects.all(), + required=False + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.fields['start'].queryset = self.branch.get_replay_queue() + + class ConfirmationForm(forms.Form): confirm = forms.BooleanField( required=True, diff --git a/netbox_branching/jobs.py b/netbox_branching/jobs.py index a5f8872..f6d9853 100644 --- a/netbox_branching/jobs.py +++ b/netbox_branching/jobs.py @@ -10,6 +10,7 @@ __all__ = ( 'MergeBranchJob', 'ProvisionBranchJob', + 'ReplayBranchJob', 'RevertBranchJob', 'SyncBranchJob', ) @@ -45,7 +46,7 @@ def run(self, *args, **kwargs): # If the Branch specifies an origin, replay changes from it if branch.origin: - branch.replay(logger=logger) + branch.replay(user=self.job.user, logger=logger) class SyncBranchJob(JobRunner): @@ -96,6 +97,27 @@ def run(self, commit=True, *args, **kwargs): self._reconnect_signal_receivers() +class ReplayBranchJob(JobRunner): + """ + Replay changes from an origin branch onto a Branch. + """ + class Meta: + name = 'Replay branch' + + def run(self, commit=True, start=None, *args, **kwargs): + # Initialize logging + logger = logging.getLogger('netbox_branching.branch.replay') + logger.setLevel(logging.DEBUG) + logger.addHandler(ListHandler(queue=get_job_log(self.job))) + + # Replay changes + try: + branch = self.job.object + branch.replay(user=self.job.user, commit=commit, start=start) + except AbortTransaction: + logger.info("Dry run completed; rolling back changes") + + class MergeBranchJob(JobRunner): """ Merge changes from a Branch into main. diff --git a/netbox_branching/models/branches.py b/netbox_branching/models/branches.py index 6cbea29..004f1da 100644 --- a/netbox_branching/models/branches.py +++ b/netbox_branching/models/branches.py @@ -235,6 +235,17 @@ def get_unsynced_changes(self): time__gt=self.synced_time ) + def get_replay_queue(self): + """ + Return a queryset of all ObjectChange records from the origin Branch which have yet to be replayed onto + this Branch. + """ + if not self.origin: + return ObjectChange.objects.none() + return ObjectChange.objects.using(self.origin.connection_name).filter( + pk__gt=self.origin_ptr or 0 + ).order_by('pk') + def get_unmerged_changes(self): """ Return a queryset of all unmerged ObjectChange records within the Branch schema. @@ -291,6 +302,14 @@ def can_sync(self): """ return self._can_do_action('sync') + @property + def can_replay(self): + """ + Indicates whether changes from origin can be replayed. + """ + # TODO: Determine how to evaluate this + return True + @property def can_merge(self): """ @@ -375,7 +394,7 @@ def sync(self, user, commit=True): sync.alters_data = True - def replay(self, logger=None): + def replay(self, user, commit=True, start=None, logger=None): """ Replay changes from the originating Branch onto this Branch. """ @@ -389,9 +408,15 @@ def replay(self, logger=None): raise Exception(f"Branch {self} is not ready for replay") # Fetch changes to apply from originating Branch + if start: + start_pk = start.pk + elif self.origin_ptr: + start_pk = self.origin_ptr + 1 + else: + start_pk = 0 changes = ObjectChange.objects.using(self.origin.connection_name).filter( changed_object_type__in=get_branchable_object_types(), - pk__gt=self.origin_ptr or 0 + pk__gte=start_pk ).order_by('pk') if changes: logger.info(f"Found {len(changes)} changes to replay") @@ -411,17 +436,22 @@ def replay(self, logger=None): request.user = change.user change.apply(using=self.connection_name, logger=logger) self.origin_ptr = change.pk + if not commit: + raise AbortTransaction() except Exception as e: - # Record the replay failure - logger.error(str(e)) - logger.debug(f"Recording branch event: {BranchEventTypeChoices.REPLAY_FAILED}") - BranchEvent.objects.create(branch=self, type=BranchEventTypeChoices.REPLAY_FAILED) - # Restore original branch status active_branch.set(None) Branch.objects.filter(pk=self.pk).update(status=BranchStatusChoices.READY, origin_ptr=self.origin_ptr) + if type(e) is AbortTransaction: + raise e + + # Record the replay failure + logger.error(str(e)) + logger.debug(f"Recording branch event: {BranchEventTypeChoices.REPLAY_FAILED}") + BranchEvent.objects.create(branch=self, user=user, type=BranchEventTypeChoices.REPLAY_FAILED) + # Record the branch's last_synced time & update its status logger.debug(f"Setting branch status to {BranchStatusChoices.READY}") self.last_sync = timezone.now() diff --git a/netbox_branching/models/changes.py b/netbox_branching/models/changes.py index 1890823..74abfb9 100644 --- a/netbox_branching/models/changes.py +++ b/netbox_branching/models/changes.py @@ -27,6 +27,14 @@ class ObjectChange(ObjectChange_): class Meta: proxy = True + def __str__(self): + return '{}: {} {} by {}'.format( + self.pk, + self.object_repr, + self.get_action_display().lower(), + self.user_name + ) + def apply(self, using=DEFAULT_DB_ALIAS, logger=None): """ Apply the change using the specified database connection. diff --git a/netbox_branching/tables/tables.py b/netbox_branching/tables/tables.py index 831db4a..670542a 100644 --- a/netbox_branching/tables/tables.py +++ b/netbox_branching/tables/tables.py @@ -7,11 +7,13 @@ from netbox_branching.models import Branch, ChangeDiff from utilities.templatetags.builtins.filters import placeholder from .columns import ConflictsColumn, DiffColumn +from .template_code import * __all__ = ( 'ChangeDiffTable', 'BranchTable', 'ChangesTable', + 'ReplayTable', ) @@ -182,3 +184,33 @@ class Meta(NetBoxTable.Meta): fields = ( 'pk', 'time', 'action', 'model', 'changed_object_type', 'object_repr', 'request_id', 'before', 'after', ) + + +class ReplayTable(NetBoxTable): + time = columns.DateTimeColumn( + verbose_name=_('Time'), + timespec='minutes', + linkify=True + ) + action = columns.ChoiceFieldColumn( + verbose_name=_('Action'), + ) + changed_object_type = columns.ContentTypeColumn( + verbose_name=_('Type') + ) + object_repr = tables.TemplateColumn( + accessor=tables.A('changed_object'), + template_code=OBJECTCHANGE_OBJECT, + verbose_name=_('Object'), + orderable=False + ) + actions = columns.ActionsColumn( + actions=(), + extra_buttons=REPLAY_CHANGE + ) + + class Meta(NetBoxTable.Meta): + model = ObjectChange + fields = ( + 'pk', 'time', 'action', 'changed_object_type', 'object_repr', + ) diff --git a/netbox_branching/tables/template_code.py b/netbox_branching/tables/template_code.py new file mode 100644 index 0000000..6a3ee95 --- /dev/null +++ b/netbox_branching/tables/template_code.py @@ -0,0 +1,5 @@ +REPLAY_CHANGE = """ + + + +""" diff --git a/netbox_branching/templates/netbox_branching/branch.html b/netbox_branching/templates/netbox_branching/branch.html index 91dc6d8..d231e8c 100644 --- a/netbox_branching/templates/netbox_branching/branch.html +++ b/netbox_branching/templates/netbox_branching/branch.html @@ -24,6 +24,7 @@ {% endif %} {% if object.ready %} {% branch_sync_button object %} + {% branch_replay_button object %} {% branch_merge_button object %} {% endif %} {% if object.merged %} @@ -76,7 +77,8 @@
{% trans "Branch" %}
{% trans "Origin" %} {% if object.origin %} - {{ object.origin|linkify }} ({{ object.origin_ptr }}) + {{ object.origin|linkify }} ({{ object.origin_ptr }})
+ {{ object.get_replay_queue.count }} changes to replay {% else %} {{ ''|placeholder }} {% endif %} diff --git a/netbox_branching/templates/netbox_branching/branch_replay.html b/netbox_branching/templates/netbox_branching/branch_replay.html new file mode 100644 index 0000000..5521790 --- /dev/null +++ b/netbox_branching/templates/netbox_branching/branch_replay.html @@ -0,0 +1,33 @@ +{% extends 'netbox_branching/branch_action.html' %} +{% load form_helpers %} +{% load i18n %} + +{% block content %} + {# Form tab #} +
+ {% if not action_permitted %} +
+ + {% blocktrans %} + This action is disallowed per policy, however dry runs are permitted. + {% endblocktrans %} +
+ {% endif %} +
+ {% csrf_token %} +
+
+ {% render_field form.start %} + {% render_field form.commit %} +
+ {% trans "Cancel" %} + +
+
+
+
+
+ {# /Form tab #} +{% endblock content %} diff --git a/netbox_branching/templates/netbox_branching/buttons/branch_replay.html b/netbox_branching/templates/netbox_branching/buttons/branch_replay.html new file mode 100644 index 0000000..d7bc5bd --- /dev/null +++ b/netbox_branching/templates/netbox_branching/buttons/branch_replay.html @@ -0,0 +1,6 @@ +{% load i18n %} +{% if branch.origin %} + + {% trans "Replay" %} + +{% endif %} diff --git a/netbox_branching/templatetags/branch_buttons.py b/netbox_branching/templatetags/branch_buttons.py index 2fa62f2..41161ae 100644 --- a/netbox_branching/templatetags/branch_buttons.py +++ b/netbox_branching/templatetags/branch_buttons.py @@ -3,6 +3,7 @@ __all__ = ( 'branch_sync_button', 'branch_merge_button', + 'branch_replay_button', 'branch_revert_button', 'branch_archive_button', ) @@ -18,6 +19,14 @@ def branch_sync_button(context, branch): } +@register.inclusion_tag('netbox_branching/buttons/branch_replay.html', takes_context=True) +def branch_replay_button(context, branch): + return { + 'branch': branch, + 'perms': context.get('perms'), + } + + @register.inclusion_tag('netbox_branching/buttons/branch_merge.html', takes_context=True) def branch_merge_button(context, branch): return { diff --git a/netbox_branching/views.py b/netbox_branching/views.py index bd0ca69..6f3481b 100644 --- a/netbox_branching/views.py +++ b/netbox_branching/views.py @@ -11,7 +11,7 @@ from utilities.views import ViewTab, register_model_view from . import filtersets, forms, tables from .choices import BranchStatusChoices -from .jobs import MergeBranchJob, RevertBranchJob, SyncBranchJob +from .jobs import MergeBranchJob, ReplayBranchJob, RevertBranchJob, SyncBranchJob from .models import Branch, ChangeDiff @@ -85,6 +85,24 @@ def _get_diff_count(obj): return ChangeDiff.objects.filter(branch=obj).count() +@register_model_view(Branch, name='replay_queue', path='replay-queue') +class BranchReplayQueueView(generic.ObjectChildrenView): + queryset = Branch.objects.all() + child_model = ObjectChange + filterset = ObjectChangeFilterSet + table = tables.ReplayTable + actions = {} + tab = ViewTab( + label=_('Replay Queue'), + badge=lambda obj: obj.get_replay_queue().count(), + permission='netbox_branching.view_branch', + hide_if_empty=True + ) + + def get_children(self, request, parent): + return parent.get_replay_queue().order_by('time') + + @register_model_view(Branch, 'diff') class BranchDiffView(generic.ObjectChildrenView): queryset = Branch.objects.all() @@ -136,10 +154,6 @@ def get_children(self, request, parent): return parent.get_unmerged_changes().order_by('time') -def _get_change_count(obj): - return obj.get_unmerged_changes().count() - - @register_model_view(Branch, 'changes-merged') class BranchChangesMergedView(generic.ObjectChildrenView): queryset = Branch.objects.all() @@ -187,7 +201,7 @@ def do_action(self, branch, request, form): def get(self, request, **kwargs): branch = self.get_object(**kwargs) action_permitted = getattr(branch, f'can_{self.action}') - form = self.form(branch, allow_commit=action_permitted) + form = self.form(branch, allow_commit=action_permitted, initial=request.GET) return render(request, self.template_name, { 'branch': branch, @@ -195,6 +209,7 @@ def get(self, request, **kwargs): 'form': form, 'action_permitted': action_permitted, 'conflicts_table': self._get_conflicts_table(branch), + **self.get_extra_context(request, branch), }) def post(self, request, **kwargs): @@ -215,6 +230,7 @@ def post(self, request, **kwargs): 'form': form, 'action_permitted': action_permitted, 'conflicts_table': self._get_conflicts_table(branch), + **self.get_extra_context(request, branch), }) @@ -234,6 +250,27 @@ def do_action(self, branch, request, form): return redirect(branch.get_absolute_url()) +@register_model_view(Branch, 'replay') +class BranchReplayView(BaseBranchActionView): + action = 'replay' + form = forms.BranchReplayForm + template_name = 'netbox_branching/branch_replay.html' + + def do_action(self, branch, request, form): + # Enqueue a background job to replay changes from origin onto the Branch + ReplayBranchJob.enqueue( + instance=branch, + user=request.user, + start=form.cleaned_data['start'], + commit=form.cleaned_data['commit'] + ) + messages.success(request, _("Replaying changes from branch {branch.origin} onto {branch}").format( + branch=branch + )) + + return redirect(branch.get_absolute_url()) + + @register_model_view(Branch, 'merge') class BranchMergeView(BaseBranchActionView): action = 'merge' From 10089d5800cd13056aad90b39983bb675cb56393 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2025 09:57:49 -0500 Subject: [PATCH 05/12] Revert introduction of replay logic --- docs/models/branch.md | 8 - netbox_branching/api/serializers.py | 10 +- netbox_branching/choices.py | 2 - netbox_branching/filtersets.py | 10 - netbox_branching/forms/misc.py | 4 +- netbox_branching/jobs.py | 28 +-- .../migrations/0003_branch_cloning.py | 22 --- netbox_branching/models/branches.py | 175 ++++++++---------- netbox_branching/tables/tables.py | 71 ++++--- netbox_branching/tables/template_code.py | 5 - .../templates/netbox_branching/branch.html | 12 -- .../netbox_branching/branch_action.html | 1 + .../netbox_branching/branch_replay.html | 33 ---- .../buttons/branch_replay.html | 6 - .../templatetags/branch_buttons.py | 9 - netbox_branching/views.py | 42 +---- 16 files changed, 118 insertions(+), 320 deletions(-) delete mode 100644 netbox_branching/migrations/0003_branch_cloning.py delete mode 100644 netbox_branching/tables/template_code.py delete mode 100644 netbox_branching/templates/netbox_branching/branch_replay.html delete mode 100644 netbox_branching/templates/netbox_branching/buttons/branch_replay.html diff --git a/docs/models/branch.md b/docs/models/branch.md index ec46be9..1ecbf59 100644 --- a/docs/models/branch.md +++ b/docs/models/branch.md @@ -32,14 +32,6 @@ The current status of the branch. This must be one of the following values. | Archived | A merged branch which has been deprovisioned in the database | | Failed | Provisioning the schema for this branch has failed | -### Origin - -The branch from which this branch was cloned (if any). - -### Origin Pointer - -The last change record belonging to the origin branch successfully applied to this branch. - ### Last Sync The time at which this branch was most recently synchronized with main. This value will be null if the branch has never been synchronized. diff --git a/netbox_branching/api/serializers.py b/netbox_branching/api/serializers.py index 9c7dbab..ba6af63 100644 --- a/netbox_branching/api/serializers.py +++ b/netbox_branching/api/serializers.py @@ -44,12 +44,6 @@ class BranchSerializer(NetBoxModelSerializer): nested=True, read_only=True ) - origin = NestedBranchSerializer( - read_only=True - ) - origin_ptr = serializers.IntegerField( - read_only=True - ) merged_by = UserSerializer( nested=True, read_only=True @@ -61,8 +55,8 @@ class BranchSerializer(NetBoxModelSerializer): class Meta: model = Branch fields = [ - 'id', 'url', 'display', 'name', 'origin', 'origin_ptr', 'status', 'owner', 'description', 'schema_id', - 'last_sync', 'merged_time', 'merged_by', 'comments', 'tags', 'custom_fields', 'created', 'last_updated', + 'id', 'url', 'display', 'name', 'status', 'owner', 'description', 'schema_id', 'last_sync', 'merged_time', + 'merged_by', 'comments', 'tags', 'custom_fields', 'created', 'last_updated', ] brief_fields = ('id', 'url', 'display', 'name', 'status', 'description') diff --git a/netbox_branching/choices.py b/netbox_branching/choices.py index 07b915b..90c6f46 100644 --- a/netbox_branching/choices.py +++ b/netbox_branching/choices.py @@ -46,7 +46,6 @@ class BranchEventTypeChoices(ChoiceSet): MERGED = 'merged' REVERTED = 'reverted' ARCHIVED = 'archived' - REPLAY_FAILED = 'replay-failed' CHOICES = ( (PROVISIONED, _('Provisioned'), 'green'), @@ -54,5 +53,4 @@ class BranchEventTypeChoices(ChoiceSet): (MERGED, _('Merged'), 'blue'), (REVERTED, _('Reverted'), 'orange'), (ARCHIVED, _('Archived'), 'gray'), - (REPLAY_FAILED, _('Replay failed'), 'red'), ) diff --git a/netbox_branching/filtersets.py b/netbox_branching/filtersets.py index 6a158cc..fa6cbd9 100644 --- a/netbox_branching/filtersets.py +++ b/netbox_branching/filtersets.py @@ -21,16 +21,6 @@ class BranchFilterSet(NetBoxModelFilterSet): choices=BranchStatusChoices, null_value=None ) - origin_id = django_filters.ModelMultipleChoiceFilter( - queryset=Branch.objects.all(), - label=_('Origin (ID)'), - ) - origin = django_filters.ModelMultipleChoiceFilter( - field_name='origin__schema_id', - queryset=Branch.objects.all(), - to_field_name='schema_id', - label=_('Origin (schema ID)'), - ) last_sync = filters.MultiValueDateTimeFilter() class Meta: diff --git a/netbox_branching/forms/misc.py b/netbox_branching/forms/misc.py index ba28e2d..7ff9ecd 100644 --- a/netbox_branching/forms/misc.py +++ b/netbox_branching/forms/misc.py @@ -6,7 +6,7 @@ __all__ = ( 'BranchActionForm', - 'BranchReplayForm', + 'BranchMergeForm', 'ConfirmationForm', ) @@ -44,7 +44,7 @@ def clean(self): return self.cleaned_data -class BranchReplayForm(BranchActionForm): +class BranchMergeForm(BranchActionForm): # TODO: Populate via REST API start = forms.ModelChoiceField( queryset=ObjectChange.objects.all(), diff --git a/netbox_branching/jobs.py b/netbox_branching/jobs.py index f6d9853..9542e24 100644 --- a/netbox_branching/jobs.py +++ b/netbox_branching/jobs.py @@ -10,7 +10,6 @@ __all__ = ( 'MergeBranchJob', 'ProvisionBranchJob', - 'ReplayBranchJob', 'RevertBranchJob', 'SyncBranchJob', ) @@ -33,7 +32,7 @@ class ProvisionBranchJob(JobRunner): class Meta: name = 'Provision branch' - def run(self, *args, **kwargs): + def run(self, origin=None, *args, **kwargs): # Initialize logging logger = logging.getLogger('netbox_branching.branch.provision') logger.setLevel(logging.DEBUG) @@ -45,8 +44,8 @@ def run(self, *args, **kwargs): branch.refresh_from_db() # If the Branch specifies an origin, replay changes from it - if branch.origin: - branch.replay(user=self.job.user, logger=logger) + if origin: + origin.merge(target=branch, user=self.job.user) class SyncBranchJob(JobRunner): @@ -97,27 +96,6 @@ def run(self, commit=True, *args, **kwargs): self._reconnect_signal_receivers() -class ReplayBranchJob(JobRunner): - """ - Replay changes from an origin branch onto a Branch. - """ - class Meta: - name = 'Replay branch' - - def run(self, commit=True, start=None, *args, **kwargs): - # Initialize logging - logger = logging.getLogger('netbox_branching.branch.replay') - logger.setLevel(logging.DEBUG) - logger.addHandler(ListHandler(queue=get_job_log(self.job))) - - # Replay changes - try: - branch = self.job.object - branch.replay(user=self.job.user, commit=commit, start=start) - except AbortTransaction: - logger.info("Dry run completed; rolling back changes") - - class MergeBranchJob(JobRunner): """ Merge changes from a Branch into main. diff --git a/netbox_branching/migrations/0003_branch_cloning.py b/netbox_branching/migrations/0003_branch_cloning.py deleted file mode 100644 index 5906854..0000000 --- a/netbox_branching/migrations/0003_branch_cloning.py +++ /dev/null @@ -1,22 +0,0 @@ -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('netbox_branching', '0002_branch_schema_id_unique'), - ] - - operations = [ - migrations.AddField( - model_name='branch', - name='origin', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='clones', to='netbox_branching.branch'), - ), - migrations.AddField( - model_name='branch', - name='origin_ptr', - field=models.PositiveBigIntegerField(blank=True, null=True), - ), - ] diff --git a/netbox_branching/models/branches.py b/netbox_branching/models/branches.py index 004f1da..dc06219 100644 --- a/netbox_branching/models/branches.py +++ b/netbox_branching/models/branches.py @@ -50,19 +50,6 @@ class Branch(JobsMixin, PrimaryModel): null=True, related_name='branches' ) - origin = models.ForeignKey( - to='self', - on_delete=models.PROTECT, - blank=True, - null=True, - related_name='clones', - help_text=_("The branch from which this branch was cloned.") - ) - origin_ptr = models.PositiveBigIntegerField( - blank=True, - null=True, - help_text=_("The last successfully applied change from the original branch.") - ) schema_id = models.CharField( max_length=8, unique=True, @@ -235,16 +222,16 @@ def get_unsynced_changes(self): time__gt=self.synced_time ) - def get_replay_queue(self): - """ - Return a queryset of all ObjectChange records from the origin Branch which have yet to be replayed onto - this Branch. - """ - if not self.origin: - return ObjectChange.objects.none() - return ObjectChange.objects.using(self.origin.connection_name).filter( - pk__gt=self.origin_ptr or 0 - ).order_by('pk') + # def get_replay_queue(self): + # """ + # Return a queryset of all ObjectChange records from the origin Branch which have yet to be replayed onto + # this Branch. + # """ + # if not self.origin: + # return ObjectChange.objects.none() + # return ObjectChange.objects.using(self.origin.connection_name).filter( + # pk__gt=self.origin_ptr or 0 + # ).order_by('pk') def get_unmerged_changes(self): """ @@ -302,14 +289,6 @@ def can_sync(self): """ return self._can_do_action('sync') - @property - def can_replay(self): - """ - Indicates whether changes from origin can be replayed. - """ - # TODO: Determine how to evaluate this - return True - @property def can_merge(self): """ @@ -394,73 +373,73 @@ def sync(self, user, commit=True): sync.alters_data = True - def replay(self, user, commit=True, start=None, logger=None): - """ - Replay changes from the originating Branch onto this Branch. - """ - logger = logger or logging.getLogger('netbox_branching.branch.replay') - - if not self.origin: - raise Exception(f"Cannot replay changes onto branch {self}: No origin branch is defined.") - logger.info(f'Replaying changes from branch {self.origin} onto branch {self} ({self.schema_name})') - - if not self.ready: - raise Exception(f"Branch {self} is not ready for replay") - - # Fetch changes to apply from originating Branch - if start: - start_pk = start.pk - elif self.origin_ptr: - start_pk = self.origin_ptr + 1 - else: - start_pk = 0 - changes = ObjectChange.objects.using(self.origin.connection_name).filter( - changed_object_type__in=get_branchable_object_types(), - pk__gte=start_pk - ).order_by('pk') - if changes: - logger.info(f"Found {len(changes)} changes to replay") - else: - logger.info(f"No changes found; aborting.") - return - - # Create a dummy request for the event_tracking() context manager - request = RequestFactory().get(reverse('home')) - - # Apply each change from the origin schema - try: - with activate_branch(self): - with event_tracking(request): - for change in changes: - request.id = change.request_id - request.user = change.user - change.apply(using=self.connection_name, logger=logger) - self.origin_ptr = change.pk - if not commit: - raise AbortTransaction() - - except Exception as e: - # Restore original branch status - active_branch.set(None) - Branch.objects.filter(pk=self.pk).update(status=BranchStatusChoices.READY, origin_ptr=self.origin_ptr) - - if type(e) is AbortTransaction: - raise e - - # Record the replay failure - logger.error(str(e)) - logger.debug(f"Recording branch event: {BranchEventTypeChoices.REPLAY_FAILED}") - BranchEvent.objects.create(branch=self, user=user, type=BranchEventTypeChoices.REPLAY_FAILED) - - # Record the branch's last_synced time & update its status - logger.debug(f"Setting branch status to {BranchStatusChoices.READY}") - self.last_sync = timezone.now() - self.status = BranchStatusChoices.READY - self.save() - - logger.info('Replay completed') - - replay.alters_data = True + # def replay(self, user, commit=True, start=None, logger=None): + # """ + # Replay changes from the originating Branch onto this Branch. + # """ + # logger = logger or logging.getLogger('netbox_branching.branch.replay') + # + # if not self.origin: + # raise Exception(f"Cannot replay changes onto branch {self}: No origin branch is defined.") + # logger.info(f'Replaying changes from branch {self.origin} onto branch {self} ({self.schema_name})') + # + # if not self.ready: + # raise Exception(f"Branch {self} is not ready for replay") + # + # # Fetch changes to apply from originating Branch + # if start: + # start_pk = start.pk + # elif self.origin_ptr: + # start_pk = self.origin_ptr + 1 + # else: + # start_pk = 0 + # changes = ObjectChange.objects.using(self.origin.connection_name).filter( + # changed_object_type__in=get_branchable_object_types(), + # pk__gte=start_pk + # ).order_by('pk') + # if changes: + # logger.info(f"Found {len(changes)} changes to replay") + # else: + # logger.info(f"No changes found; aborting.") + # return + # + # # Create a dummy request for the event_tracking() context manager + # request = RequestFactory().get(reverse('home')) + # + # # Apply each change from the origin schema + # try: + # with activate_branch(self): + # with event_tracking(request): + # for change in changes: + # request.id = change.request_id + # request.user = change.user + # change.apply(using=self.connection_name, logger=logger) + # self.origin_ptr = change.pk + # if not commit: + # raise AbortTransaction() + # + # except Exception as e: + # # Restore original branch status + # active_branch.set(None) + # Branch.objects.filter(pk=self.pk).update(status=BranchStatusChoices.READY, origin_ptr=self.origin_ptr) + # + # if type(e) is AbortTransaction: + # raise e + # + # # Record the replay failure + # logger.error(str(e)) + # logger.debug(f"Recording branch event: {BranchEventTypeChoices.REPLAY_FAILED}") + # BranchEvent.objects.create(branch=self, user=user, type=BranchEventTypeChoices.REPLAY_FAILED) + # + # # Record the branch's last_synced time & update its status + # logger.debug(f"Setting branch status to {BranchStatusChoices.READY}") + # self.last_sync = timezone.now() + # self.status = BranchStatusChoices.READY + # self.save() + # + # logger.info('Replay completed') + # + # replay.alters_data = True def merge(self, user, commit=True): """ diff --git a/netbox_branching/tables/tables.py b/netbox_branching/tables/tables.py index 670542a..83f2492 100644 --- a/netbox_branching/tables/tables.py +++ b/netbox_branching/tables/tables.py @@ -7,13 +7,11 @@ from netbox_branching.models import Branch, ChangeDiff from utilities.templatetags.builtins.filters import placeholder from .columns import ConflictsColumn, DiffColumn -from .template_code import * __all__ = ( 'ChangeDiffTable', 'BranchTable', 'ChangesTable', - 'ReplayTable', ) @@ -61,13 +59,6 @@ class BranchTable(NetBoxTable): status = columns.ChoiceFieldColumn( verbose_name=_('Status'), ) - origin = tables.Column( - linkify=True, - verbose_name=_('Origin') - ) - origin_ptr = tables.Column( - verbose_name=_('Origin Pointer') - ) conflicts = ConflictsColumn( verbose_name=_('Conflicts') ) @@ -81,11 +72,11 @@ class BranchTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = Branch fields = ( - 'pk', 'id', 'name', 'is_active', 'status', 'origin', 'origin_ptr', 'conflicts', 'schema_id', 'description', - 'owner', 'tags', 'created', 'last_updated', + 'pk', 'id', 'name', 'is_active', 'status', 'conflicts', 'schema_id', 'description', 'owner', 'tags', + 'created', 'last_updated', ) default_columns = ( - 'pk', 'name', 'is_active', 'status', 'origin', 'owner', 'conflicts', 'schema_id', 'description', + 'pk', 'name', 'is_active', 'status', 'owner', 'conflicts', 'schema_id', 'description', ) def render_is_active(self, value): @@ -186,31 +177,31 @@ class Meta(NetBoxTable.Meta): ) -class ReplayTable(NetBoxTable): - time = columns.DateTimeColumn( - verbose_name=_('Time'), - timespec='minutes', - linkify=True - ) - action = columns.ChoiceFieldColumn( - verbose_name=_('Action'), - ) - changed_object_type = columns.ContentTypeColumn( - verbose_name=_('Type') - ) - object_repr = tables.TemplateColumn( - accessor=tables.A('changed_object'), - template_code=OBJECTCHANGE_OBJECT, - verbose_name=_('Object'), - orderable=False - ) - actions = columns.ActionsColumn( - actions=(), - extra_buttons=REPLAY_CHANGE - ) - - class Meta(NetBoxTable.Meta): - model = ObjectChange - fields = ( - 'pk', 'time', 'action', 'changed_object_type', 'object_repr', - ) +# class ReplayTable(NetBoxTable): +# time = columns.DateTimeColumn( +# verbose_name=_('Time'), +# timespec='minutes', +# linkify=True +# ) +# action = columns.ChoiceFieldColumn( +# verbose_name=_('Action'), +# ) +# changed_object_type = columns.ContentTypeColumn( +# verbose_name=_('Type') +# ) +# object_repr = tables.TemplateColumn( +# accessor=tables.A('changed_object'), +# template_code=OBJECTCHANGE_OBJECT, +# verbose_name=_('Object'), +# orderable=False +# ) +# actions = columns.ActionsColumn( +# actions=(), +# extra_buttons=REPLAY_CHANGE +# ) +# +# class Meta(NetBoxTable.Meta): +# model = ObjectChange +# fields = ( +# 'pk', 'time', 'action', 'changed_object_type', 'object_repr', +# ) diff --git a/netbox_branching/tables/template_code.py b/netbox_branching/tables/template_code.py deleted file mode 100644 index 6a3ee95..0000000 --- a/netbox_branching/tables/template_code.py +++ /dev/null @@ -1,5 +0,0 @@ -REPLAY_CHANGE = """ - - - -""" diff --git a/netbox_branching/templates/netbox_branching/branch.html b/netbox_branching/templates/netbox_branching/branch.html index d231e8c..75eff0c 100644 --- a/netbox_branching/templates/netbox_branching/branch.html +++ b/netbox_branching/templates/netbox_branching/branch.html @@ -24,7 +24,6 @@ {% endif %} {% if object.ready %} {% branch_sync_button object %} - {% branch_replay_button object %} {% branch_merge_button object %} {% endif %} {% if object.merged %} @@ -73,17 +72,6 @@
{% trans "Branch" %}
{% endif %} - - {% trans "Origin" %} - - {% if object.origin %} - {{ object.origin|linkify }} ({{ object.origin_ptr }})
- {{ object.get_replay_queue.count }} changes to replay - {% else %} - {{ ''|placeholder }} - {% endif %} - - {% trans "Owner" %} {{ object.owner }} diff --git a/netbox_branching/templates/netbox_branching/branch_action.html b/netbox_branching/templates/netbox_branching/branch_action.html index 885e2d0..ba59f7b 100644 --- a/netbox_branching/templates/netbox_branching/branch_action.html +++ b/netbox_branching/templates/netbox_branching/branch_action.html @@ -50,6 +50,7 @@ {% endif %}
+ {% render_field form.start %} {% render_field form.commit %}
{% trans "Cancel" %} diff --git a/netbox_branching/templates/netbox_branching/branch_replay.html b/netbox_branching/templates/netbox_branching/branch_replay.html deleted file mode 100644 index 5521790..0000000 --- a/netbox_branching/templates/netbox_branching/branch_replay.html +++ /dev/null @@ -1,33 +0,0 @@ -{% extends 'netbox_branching/branch_action.html' %} -{% load form_helpers %} -{% load i18n %} - -{% block content %} - {# Form tab #} -
- {% if not action_permitted %} -
- - {% blocktrans %} - This action is disallowed per policy, however dry runs are permitted. - {% endblocktrans %} -
- {% endif %} -
- {% csrf_token %} -
-
- {% render_field form.start %} - {% render_field form.commit %} -
- {% trans "Cancel" %} - -
-
-
-
-
- {# /Form tab #} -{% endblock content %} diff --git a/netbox_branching/templates/netbox_branching/buttons/branch_replay.html b/netbox_branching/templates/netbox_branching/buttons/branch_replay.html deleted file mode 100644 index d7bc5bd..0000000 --- a/netbox_branching/templates/netbox_branching/buttons/branch_replay.html +++ /dev/null @@ -1,6 +0,0 @@ -{% load i18n %} -{% if branch.origin %} - - {% trans "Replay" %} - -{% endif %} diff --git a/netbox_branching/templatetags/branch_buttons.py b/netbox_branching/templatetags/branch_buttons.py index 41161ae..2fa62f2 100644 --- a/netbox_branching/templatetags/branch_buttons.py +++ b/netbox_branching/templatetags/branch_buttons.py @@ -3,7 +3,6 @@ __all__ = ( 'branch_sync_button', 'branch_merge_button', - 'branch_replay_button', 'branch_revert_button', 'branch_archive_button', ) @@ -19,14 +18,6 @@ def branch_sync_button(context, branch): } -@register.inclusion_tag('netbox_branching/buttons/branch_replay.html', takes_context=True) -def branch_replay_button(context, branch): - return { - 'branch': branch, - 'perms': context.get('perms'), - } - - @register.inclusion_tag('netbox_branching/buttons/branch_merge.html', takes_context=True) def branch_merge_button(context, branch): return { diff --git a/netbox_branching/views.py b/netbox_branching/views.py index 6f3481b..56c6276 100644 --- a/netbox_branching/views.py +++ b/netbox_branching/views.py @@ -11,7 +11,7 @@ from utilities.views import ViewTab, register_model_view from . import filtersets, forms, tables from .choices import BranchStatusChoices -from .jobs import MergeBranchJob, ReplayBranchJob, RevertBranchJob, SyncBranchJob +from .jobs import MergeBranchJob, RevertBranchJob, SyncBranchJob from .models import Branch, ChangeDiff @@ -85,24 +85,6 @@ def _get_diff_count(obj): return ChangeDiff.objects.filter(branch=obj).count() -@register_model_view(Branch, name='replay_queue', path='replay-queue') -class BranchReplayQueueView(generic.ObjectChildrenView): - queryset = Branch.objects.all() - child_model = ObjectChange - filterset = ObjectChangeFilterSet - table = tables.ReplayTable - actions = {} - tab = ViewTab( - label=_('Replay Queue'), - badge=lambda obj: obj.get_replay_queue().count(), - permission='netbox_branching.view_branch', - hide_if_empty=True - ) - - def get_children(self, request, parent): - return parent.get_replay_queue().order_by('time') - - @register_model_view(Branch, 'diff') class BranchDiffView(generic.ObjectChildrenView): queryset = Branch.objects.all() @@ -250,30 +232,10 @@ def do_action(self, branch, request, form): return redirect(branch.get_absolute_url()) -@register_model_view(Branch, 'replay') -class BranchReplayView(BaseBranchActionView): - action = 'replay' - form = forms.BranchReplayForm - template_name = 'netbox_branching/branch_replay.html' - - def do_action(self, branch, request, form): - # Enqueue a background job to replay changes from origin onto the Branch - ReplayBranchJob.enqueue( - instance=branch, - user=request.user, - start=form.cleaned_data['start'], - commit=form.cleaned_data['commit'] - ) - messages.success(request, _("Replaying changes from branch {branch.origin} onto {branch}").format( - branch=branch - )) - - return redirect(branch.get_absolute_url()) - - @register_model_view(Branch, 'merge') class BranchMergeView(BaseBranchActionView): action = 'merge' + form = forms.BranchMergeForm def do_action(self, branch, request, form): # Enqueue a background job to merge the Branch From 1c848ab26e37e3b9d029db0f9b7d0a3b17a3a48a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2025 12:24:30 -0500 Subject: [PATCH 06/12] Introduce branch pulling --- docs/models/branchevent.md | 4 + netbox_branching/__init__.py | 1 + netbox_branching/api/serializers.py | 7 +- netbox_branching/choices.py | 2 + netbox_branching/constants.py | 1 + netbox_branching/forms/misc.py | 23 ++- netbox_branching/forms/model_forms.py | 18 +- netbox_branching/jobs.py | 26 ++- .../0003_branchevent_related_branch.py | 19 ++ netbox_branching/models/branches.py | 190 ++++++++++-------- netbox_branching/signals.py | 4 + .../templates/netbox_branching/branch.html | 11 +- .../netbox_branching/branch_action.html | 3 +- .../netbox_branching/buttons/branch_pull.html | 4 + .../templatetags/branch_buttons.py | 13 +- netbox_branching/views.py | 26 ++- 16 files changed, 239 insertions(+), 113 deletions(-) create mode 100644 netbox_branching/migrations/0003_branchevent_related_branch.py create mode 100644 netbox_branching/templates/netbox_branching/buttons/branch_pull.html diff --git a/docs/models/branchevent.md b/docs/models/branchevent.md index 6180b2b..b08853b 100644 --- a/docs/models/branchevent.md +++ b/docs/models/branchevent.md @@ -12,6 +12,10 @@ The time at which the event occurred. The [branch](./branch.md) to which this event pertains. +### Related Branch + +The related branch affected by this event, where applicable. (This is relevant only when one branch is merged into another.) + ### User The NetBox user responsible for triggering this event. This field may be null if the event was triggered by an internal process. diff --git a/netbox_branching/__init__.py b/netbox_branching/__init__.py index 13c9012..0975e39 100644 --- a/netbox_branching/__init__.py +++ b/netbox_branching/__init__.py @@ -33,6 +33,7 @@ class AppConfig(PluginConfig): # Branch action validators 'sync_validators': [], + 'pull_validators': [], 'merge_validators': [], 'revert_validators': [], 'archive_validators': [], diff --git a/netbox_branching/api/serializers.py b/netbox_branching/api/serializers.py index ba6af63..492bc63 100644 --- a/netbox_branching/api/serializers.py +++ b/netbox_branching/api/serializers.py @@ -76,6 +76,11 @@ class BranchEventSerializer(NetBoxModelSerializer): nested=True, read_only=True ) + related_branch = BranchSerializer( + nested=True, + read_only=True, + allow_null=True + ) user = UserSerializer( nested=True, read_only=True @@ -88,7 +93,7 @@ class BranchEventSerializer(NetBoxModelSerializer): class Meta: model = BranchEvent fields = [ - 'id', 'url', 'display', 'time', 'branch', 'user', 'type', + 'id', 'url', 'display', 'time', 'branch', 'related_branch', 'user', 'type', ] brief_fields = ('id', 'url', 'display') diff --git a/netbox_branching/choices.py b/netbox_branching/choices.py index 90c6f46..e3745d0 100644 --- a/netbox_branching/choices.py +++ b/netbox_branching/choices.py @@ -43,6 +43,7 @@ class BranchStatusChoices(ChoiceSet): class BranchEventTypeChoices(ChoiceSet): PROVISIONED = 'provisioned' SYNCED = 'synced' + PULLED = 'pulled' MERGED = 'merged' REVERTED = 'reverted' ARCHIVED = 'archived' @@ -50,6 +51,7 @@ class BranchEventTypeChoices(ChoiceSet): CHOICES = ( (PROVISIONED, _('Provisioned'), 'green'), (SYNCED, _('Synced'), 'cyan'), + (PULLED, _('Pulled'), 'blue'), (MERGED, _('Merged'), 'blue'), (REVERTED, _('Reverted'), 'orange'), (ARCHIVED, _('Archived'), 'gray'), diff --git a/netbox_branching/constants.py b/netbox_branching/constants.py index 5b3fc7f..7649801 100644 --- a/netbox_branching/constants.py +++ b/netbox_branching/constants.py @@ -10,6 +10,7 @@ # Branch actions BRANCH_ACTIONS = ( 'sync', + 'pull', 'merge', 'revert', 'archive', diff --git a/netbox_branching/forms/misc.py b/netbox_branching/forms/misc.py index 7ff9ecd..09d2246 100644 --- a/netbox_branching/forms/misc.py +++ b/netbox_branching/forms/misc.py @@ -1,12 +1,11 @@ from django import forms from django.utils.translation import gettext_lazy as _ -from core.models import ObjectChange -from netbox_branching.models import ChangeDiff +from netbox_branching.models import Branch, ChangeDiff __all__ = ( 'BranchActionForm', - 'BranchMergeForm', + 'BranchPullForm', 'ConfirmationForm', ) @@ -14,7 +13,8 @@ class BranchActionForm(forms.Form): pk = forms.ModelMultipleChoiceField( queryset=ChangeDiff.objects.all(), - required=False + required=False, + widget=forms.HiddenInput() ) commit = forms.BooleanField( required=False, @@ -44,17 +44,20 @@ def clean(self): return self.cleaned_data -class BranchMergeForm(BranchActionForm): - # TODO: Populate via REST API - start = forms.ModelChoiceField( - queryset=ObjectChange.objects.all(), - required=False +class BranchPullForm(BranchActionForm): + source = forms.ModelChoiceField( + queryset=Branch.objects.all() ) + # start = forms.ModelChoiceField( + # queryset=ObjectChange.objects.all(), + # required=False + # ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['start'].queryset = self.branch.get_replay_queue() + self.fields['source'].queryset = Branch.objects.exclude(pk=self.branch.pk) + # self.fields['start'].queryset = self.branch.get_replay_queue() class ConfirmationForm(forms.Form): diff --git a/netbox_branching/forms/model_forms.py b/netbox_branching/forms/model_forms.py index eeec4ee..a1c2119 100644 --- a/netbox_branching/forms/model_forms.py +++ b/netbox_branching/forms/model_forms.py @@ -12,10 +12,10 @@ class BranchForm(NetBoxModelForm): fieldsets = ( - FieldSet('name', 'origin', 'description', 'tags'), + FieldSet('name', 'clone_from', 'description', 'tags'), ) - origin = DynamicModelChoiceField( - label=_('Origin'), + clone_from = DynamicModelChoiceField( + label=_('Clone from'), queryset=Branch.objects.all(), required=False ) @@ -23,11 +23,11 @@ class BranchForm(NetBoxModelForm): class Meta: model = Branch - fields = ('name', 'origin', 'description', 'comments', 'tags') + fields = ('name', 'clone_from', 'description', 'comments', 'tags') - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def save(self, *args, **kwargs): - if self.instance.pk: - # Originating branch is cannot be modified - self.fields['origin'].disabled = True + if clone_from := self.cleaned_data.get('clone_from'): + self.instance._clone_from = clone_from + + return super().save(*args, **kwargs) diff --git a/netbox_branching/jobs.py b/netbox_branching/jobs.py index 9542e24..3820370 100644 --- a/netbox_branching/jobs.py +++ b/netbox_branching/jobs.py @@ -10,6 +10,7 @@ __all__ = ( 'MergeBranchJob', 'ProvisionBranchJob', + 'PullBranchJob', 'RevertBranchJob', 'SyncBranchJob', ) @@ -43,10 +44,6 @@ def run(self, origin=None, *args, **kwargs): branch.provision(user=self.job.user) branch.refresh_from_db() - # If the Branch specifies an origin, replay changes from it - if origin: - origin.merge(target=branch, user=self.job.user) - class SyncBranchJob(JobRunner): """ @@ -117,6 +114,27 @@ def run(self, commit=True, *args, **kwargs): logger.info("Dry run completed; rolling back changes") +class PullBranchJob(JobRunner): + """ + Pull changes from one Branch into another. + """ + class Meta: + name = 'Pull branch' + + def run(self, source, commit=True, *args, **kwargs): + # Initialize logging + logger = logging.getLogger('netbox_branching.branch.pull') + logger.setLevel(logging.DEBUG) + logger.addHandler(ListHandler(queue=get_job_log(self.job))) + + # Pull changes from the source Branch + try: + branch = self.job.object + branch.pull(source, user=self.job.user, commit=commit) + except AbortTransaction: + logger.info("Dry run completed; rolling back changes") + + class RevertBranchJob(JobRunner): """ Revert changes from a merged Branch. diff --git a/netbox_branching/migrations/0003_branchevent_related_branch.py b/netbox_branching/migrations/0003_branchevent_related_branch.py new file mode 100644 index 0000000..5260a91 --- /dev/null +++ b/netbox_branching/migrations/0003_branchevent_related_branch.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.5 on 2025-01-29 15:06 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('netbox_branching', '0002_branch_schema_id_unique'), + ] + + operations = [ + migrations.AddField( + model_name='branchevent', + name='related_branch', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='netbox_branching.branch'), + ), + ] diff --git a/netbox_branching/models/branches.py b/netbox_branching/models/branches.py index dc06219..2d0b22d 100644 --- a/netbox_branching/models/branches.py +++ b/netbox_branching/models/branches.py @@ -108,7 +108,7 @@ def clone(self): """ return { '_branch': '', - 'origin': self.pk, + 'clone_from': self.pk, **super().clone(), } @@ -167,7 +167,7 @@ def save(self, provision=True, *args, **kwargs): provision: If True, automatically enqueue a background Job to provision the Branch. (Set this to False if you will call provision() on the instance manually.) """ - from netbox_branching.jobs import ProvisionBranchJob + from netbox_branching.jobs import ProvisionBranchJob, PullBranchJob _provision = provision and self.pk is None @@ -184,6 +184,15 @@ def save(self, provision=True, *args, **kwargs): user=request.user if request else None ) + if clone_from := getattr(self, '_clone_from', None): + PullBranchJob.enqueue( + instance=self, + source=clone_from, + user=request.user, + commit=True + ) + + def delete(self, *args, **kwargs): if active_branch.get(): raise AbortRequest(_("Cannot delete a branch while a branch is active.")) @@ -222,17 +231,6 @@ def get_unsynced_changes(self): time__gt=self.synced_time ) - # def get_replay_queue(self): - # """ - # Return a queryset of all ObjectChange records from the origin Branch which have yet to be replayed onto - # this Branch. - # """ - # if not self.origin: - # return ObjectChange.objects.none() - # return ObjectChange.objects.using(self.origin.connection_name).filter( - # pk__gt=self.origin_ptr or 0 - # ).order_by('pk') - def get_unmerged_changes(self): """ Return a queryset of all unmerged ObjectChange records within the Branch schema. @@ -251,6 +249,26 @@ def get_merged_changes(self): application__branch=self ) + def get_unpulled_changes(self, source): + """ + Return a queryset of all ObjectChange records from the source Branch which have yet to be replayed onto + this Branch. + """ + if source.status not in BranchStatusChoices.WORKING: + return ObjectChange.objects.none() + + changes = ObjectChange.objects.using(source.connection_name).order_by('time') + + # Return only the changes from this Branch which have not yet been pulled from the source Branch + last_pull = BranchEvent.objects.filter( + branch=self, + related_branch=source, + type=BranchEventTypeChoices.PULLED + ).order_by('-time').first() + if last_pull: + return changes.filter(time__gt=last_pull.time) + return changes + def get_event_history(self): history = [] last_time = timezone.now() @@ -289,6 +307,13 @@ def can_sync(self): """ return self._can_do_action('sync') + @property + def can_pull(self): + """ + Indicates whether changes can be pulled in from another Branch. + """ + return self._can_do_action('pull') + @property def can_merge(self): """ @@ -373,77 +398,73 @@ def sync(self, user, commit=True): sync.alters_data = True - # def replay(self, user, commit=True, start=None, logger=None): - # """ - # Replay changes from the originating Branch onto this Branch. - # """ - # logger = logger or logging.getLogger('netbox_branching.branch.replay') - # - # if not self.origin: - # raise Exception(f"Cannot replay changes onto branch {self}: No origin branch is defined.") - # logger.info(f'Replaying changes from branch {self.origin} onto branch {self} ({self.schema_name})') - # - # if not self.ready: - # raise Exception(f"Branch {self} is not ready for replay") - # - # # Fetch changes to apply from originating Branch - # if start: - # start_pk = start.pk - # elif self.origin_ptr: - # start_pk = self.origin_ptr + 1 - # else: - # start_pk = 0 - # changes = ObjectChange.objects.using(self.origin.connection_name).filter( - # changed_object_type__in=get_branchable_object_types(), - # pk__gte=start_pk - # ).order_by('pk') - # if changes: - # logger.info(f"Found {len(changes)} changes to replay") - # else: - # logger.info(f"No changes found; aborting.") - # return - # - # # Create a dummy request for the event_tracking() context manager - # request = RequestFactory().get(reverse('home')) - # - # # Apply each change from the origin schema - # try: - # with activate_branch(self): - # with event_tracking(request): - # for change in changes: - # request.id = change.request_id - # request.user = change.user - # change.apply(using=self.connection_name, logger=logger) - # self.origin_ptr = change.pk - # if not commit: - # raise AbortTransaction() - # - # except Exception as e: - # # Restore original branch status - # active_branch.set(None) - # Branch.objects.filter(pk=self.pk).update(status=BranchStatusChoices.READY, origin_ptr=self.origin_ptr) - # - # if type(e) is AbortTransaction: - # raise e - # - # # Record the replay failure - # logger.error(str(e)) - # logger.debug(f"Recording branch event: {BranchEventTypeChoices.REPLAY_FAILED}") - # BranchEvent.objects.create(branch=self, user=user, type=BranchEventTypeChoices.REPLAY_FAILED) - # - # # Record the branch's last_synced time & update its status - # logger.debug(f"Setting branch status to {BranchStatusChoices.READY}") - # self.last_sync = timezone.now() - # self.status = BranchStatusChoices.READY - # self.save() - # - # logger.info('Replay completed') - # - # replay.alters_data = True + def pull(self, source, user, commit=True): + """ + Replicate all unpulled changes from the source branch into this one. + """ + logger = logging.getLogger('netbox_branching.branch.pull') + logger.info(f'Pulling changes from branch {source} into {self.name}') + + if not self.ready: + raise Exception(f"Branch {self} is not ready for changes.") + if not source.ready: + raise Exception(f"Changes cannot be pulled from branch {source} at this time.") + # if commit and not self.can_pull: + # raise Exception(f"Pulling this branch is not permitted.") + + # Emit pre-pull signal + pre_pull.send(sender=self.__class__, branch=self, user=user) + + # Retrieve staged changes before we update the Branch's status + if changes := self.get_unpulled_changes(source): + logger.info(f"Found {len(changes)} changes to pull") + else: + logger.info(f"No changes found; aborting.") + return + + # Create a dummy request for the event_tracking() context manager + request = RequestFactory().get(reverse('home')) + + # Prep & connect the signal receiver for recording AppliedChanges + handler = partial(record_applied_change, branch=self) + post_save.connect(handler, sender=ObjectChange_, weak=False) + + try: + with transaction.atomic(): + # Apply each change from the Branch + for change in changes: + with event_tracking(request): + request.id = change.request_id + request.user = change.user + change.apply(using=self.connection_name, logger=logger) + if not commit: + raise AbortTransaction() + + except Exception as e: + if err_message := str(e): + logger.error(err_message) + # Disconnect signal receiver & restore original branch status + post_save.disconnect(handler, sender=ObjectChange_) + Branch.objects.filter(pk=self.pk).update(status=BranchStatusChoices.READY) + raise e + + # Record a branch event for the merge + logger.debug(f"Recording branch event: {BranchEventTypeChoices.PULLED}") + BranchEvent.objects.create(branch=self, related_branch=source, user=user, type=BranchEventTypeChoices.PULLED) + + # Emit post-pull signal + post_pull.send(sender=self.__class__, branch=self, user=user) + + logger.info('Pull completed') + + # Disconnect the signal receiver + post_save.disconnect(handler, sender=ObjectChange_) + + pull.alters_data = True def merge(self, user, commit=True): """ - Apply all changes in the Branch to the main schema by replaying them in + Apply all changes from this Branch to the target Branch (or to the main schema) by replaying them in chronological order. """ logger = logging.getLogger('netbox_branching.branch.merge') @@ -736,6 +757,13 @@ class BranchEvent(models.Model): on_delete=models.CASCADE, related_name='events' ) + related_branch = models.ForeignKey( + to='netbox_branching.branch', + on_delete=models.CASCADE, + related_name='+', + blank=True, + null=True + ) user = models.ForeignKey( to=get_user_model(), on_delete=models.SET_NULL, diff --git a/netbox_branching/signals.py b/netbox_branching/signals.py index 063f6c4..fe34d6e 100644 --- a/netbox_branching/signals.py +++ b/netbox_branching/signals.py @@ -4,11 +4,13 @@ 'post_deprovision', 'post_merge', 'post_provision', + 'post_pull', 'post_revert', 'post_sync', 'pre_deprovision', 'pre_merge', 'pre_provision', + 'pre_pull', 'pre_revert', 'pre_sync', ) @@ -17,6 +19,7 @@ pre_provision = Signal() pre_deprovision = Signal() pre_sync = Signal() +pre_pull = Signal() pre_merge = Signal() pre_revert = Signal() @@ -24,5 +27,6 @@ post_provision = Signal() post_deprovision = Signal() post_sync = Signal() +post_pull = Signal() post_merge = Signal() post_revert = Signal() diff --git a/netbox_branching/templates/netbox_branching/branch.html b/netbox_branching/templates/netbox_branching/branch.html index 75eff0c..dc2a3b8 100644 --- a/netbox_branching/templates/netbox_branching/branch.html +++ b/netbox_branching/templates/netbox_branching/branch.html @@ -24,6 +24,7 @@ {% endif %} {% if object.ready %} {% branch_sync_button object %} + {% branch_pull_button object %} {% branch_merge_button object %} {% endif %} {% if object.merged %} @@ -128,8 +129,14 @@
{% trans "Event History" %}
{% badge event.get_type_display bg_color=event.get_type_color %}
- {{ event.get_type_display }}{% if event.user %} by {{ event.user }}{% endif %} - {{ event.time|isodatetime }} + + {{ event.get_type_display }} + {% if event.related_branch %} {{ event.related_branch|linkify }}{% endif %} + + + {{ event.time|isodatetime }} + {% if event.user %}· {{ event.user }}{% endif %} +
{% else %} {# Change summary #} diff --git a/netbox_branching/templates/netbox_branching/branch_action.html b/netbox_branching/templates/netbox_branching/branch_action.html index ba59f7b..aad06c3 100644 --- a/netbox_branching/templates/netbox_branching/branch_action.html +++ b/netbox_branching/templates/netbox_branching/branch_action.html @@ -50,8 +50,7 @@ {% endif %}
- {% render_field form.start %} - {% render_field form.commit %} + {% render_form form %}
{% trans "Cancel" %}