Skip to content

Commit 43cd9e3

Browse files
gjonggjong
authored
Only fetch transactions once per import job (#13)
* Only run the import of transactions once, preventing potential issues when importing with APIs. * Fix an issue in the expense not being recognized in the rule engine. * Correct issues in the account reconciliation flow. * Update name of PR template. * Add additional tests to validate some edge cases.
1 parent b980ffb commit 43cd9e3

File tree

29 files changed

+197
-252
lines changed

29 files changed

+197
-252
lines changed

‎.github/pull_request_template.md renamed to ‎.github/PULL_REQUEST_TEMPLATE.md

File renamed without changes.

‎bpmn-process/src/.camunda/element-templates/importer.json

Lines changed: 9 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,15 @@
4949
"source": "${storageTokens}"
5050
}
5151
},
52+
{
53+
"label": "Account locatable",
54+
"type": "String",
55+
"description": "A set containing all account information in this import job.",
56+
"binding": {
57+
"type": "camunda:outputParameter",
58+
"source": "${locatable}"
59+
}
60+
},
5261
{
5362
"label": "Account generation",
5463
"type": "String",
@@ -108,58 +117,6 @@
108117
}
109118
]
110119
},
111-
{
112-
"$schema": "https://unpkg.com/@camunda/element-templates-json-schema/resources/schema.json",
113-
"name": "Importer: Extract account information",
114-
"description": "Extract the account information from the CSV file.",
115-
"id": "com.jongsoft.finance.bpmn.delegate.importer.ExtractAccountDetailsDelegate",
116-
"version": 1,
117-
"appliesTo": [
118-
"bpmn:ServiceTask"
119-
],
120-
"properties": [
121-
{
122-
"label": "Implementation",
123-
"type": "String",
124-
"value": "${extractAccountDetailsDelegate}",
125-
"editable": false,
126-
"binding": {
127-
"type": "property",
128-
"name": "camunda:delegateExpression"
129-
}
130-
},
131-
{
132-
"label": "Import Job Slug",
133-
"type": "String",
134-
"binding": {
135-
"type": "camunda:inputParameter",
136-
"name": "batchImportSlug"
137-
},
138-
"constraint": {
139-
"notEmpty": true
140-
}
141-
},
142-
{
143-
"label": "Configuration JSON",
144-
"type": "String",
145-
"binding": {
146-
"type": "camunda:inputParameter",
147-
"name": "importConfig"
148-
},
149-
"constraint": {
150-
"notEmpty": true
151-
}
152-
},
153-
{
154-
"label": "Account extracts",
155-
"type": "String",
156-
"binding": {
157-
"type": "camunda:outputParameter",
158-
"source": "${locatable}"
159-
}
160-
}
161-
]
162-
},
163120
{
164121
"$schema": "https://unpkg.com/@camunda/element-templates-json-schema/resources/schema.json",
165122
"name": "Importer: Lookup account with rules",

‎bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public ProcessEngineConfiguration(
4040
public ProcessEngine processEngine() throws IOException {
4141
var configuration = new StandaloneProcessEngineConfiguration();
4242

43-
configuration.setHistory(org.camunda.bpm.engine.ProcessEngineConfiguration.HISTORY_AUTO)
43+
configuration.setHistory(camundaDatasourceConfiguration.getHistoryLevel())
4444
.setJobExecutorActivate(true)
4545
.setMetricsEnabled(true)
4646
.setJdbcDriver(camundaDatasourceConfiguration.getDriverClassName())

‎bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/CamundaDatasourceConfiguration.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,7 @@ public interface CamundaDatasourceConfiguration {
2828
@NotBlank
2929
String getAutoUpdate();
3030

31+
@Bindable(defaultValue = "auto")
32+
String getHistoryLevel();
33+
3134
}

‎bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java

Lines changed: 0 additions & 69 deletions
This file was deleted.

‎bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ReadTransactionLogDelegate.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.jongsoft.finance.importer.ImporterProvider;
66
import com.jongsoft.finance.importer.api.ImporterConfiguration;
77
import com.jongsoft.finance.providers.ImportProvider;
8+
import com.jongsoft.finance.serialized.ExtractedAccountLookup;
89
import com.jongsoft.finance.serialized.ImportJobSettings;
910
import jakarta.inject.Inject;
1011
import jakarta.inject.Singleton;
@@ -14,7 +15,9 @@
1415

1516
import java.nio.charset.StandardCharsets;
1617
import java.util.ArrayList;
18+
import java.util.HashSet;
1719
import java.util.List;
20+
import java.util.Set;
1821

1922
/**
2023
* Delegate to trigger the actual {@link ImporterProvider} to start the job of fetching and converting the transactions.
@@ -60,22 +63,35 @@ public void execute(DelegateExecution execution) throws Exception {
6063

6164
var importJob = importProvider.lookup(batchImportSlug).get();
6265
List<String> storageTokens = new ArrayList<>();
66+
Set<ExtractedAccountLookup> locatable = new HashSet<>();
6367

6468
importerProviders.stream()
6569
.filter(provider -> provider.supports(importJobSettings.importConfiguration()))
6670
.findFirst()
6771
.ifPresentOrElse(
6872
provider -> provider.readTransactions(
6973
transactionDTO -> {
74+
// write the serialized transaction to storage and store the token
7075
var serialized = mapper.writeSafe(transactionDTO)
7176
.getBytes(StandardCharsets.UTF_8);
7277
storageTokens.add(storageService.store(serialized));
78+
79+
// write the extracted account lookup to the locatable set
80+
locatable.add(new ExtractedAccountLookup(
81+
transactionDTO.opposingName(),
82+
transactionDTO.opposingIBAN(),
83+
transactionDTO.description()));
7384
},
7485
importJobSettings.importConfiguration(),
7586
importJob),
7687
() -> log.warn("No importer provider found for configuration: {}", importJobSettings.importConfiguration())
7788
);
7889

90+
if (locatable.isEmpty()) {
91+
log.warn("No accounts found for import job {}", batchImportSlug);
92+
}
93+
94+
execution.setVariableLocal("locatable", locatable);
7995
execution.setVariableLocal("generateAccounts", importJobSettings.generateAccounts());
8096
execution.setVariableLocal("applyRules", importJobSettings.applyRules());
8197
execution.setVariableLocal("targetAccountId", importJobSettings.accountId());

‎bpmn-process/src/main/resources/bpmn/transaction/import-job.bpmn

Lines changed: 48 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ When calling the flow the following variables must be set:
1717
</bpmn:extensionElements>
1818
<bpmn:incoming>arrow_to_config</bpmn:incoming>
1919
<bpmn:incoming>arrow_account_missing</bpmn:incoming>
20+
<bpmn:incoming>no_data_found</bpmn:incoming>
2021
<bpmn:outgoing>Flow_088kfgl</bpmn:outgoing>
2122
</bpmn:userTask>
2223
<bpmn:sequenceFlow id="Flow_1nv7ilo" sourceRef="startJob" targetRef="read_import_config" />
@@ -52,7 +53,7 @@ When calling the flow the following variables must be set:
5253
</bpmn:exclusiveGateway>
5354
<bpmn:sequenceFlow id="Flow_0xcxk57" name="with account id" sourceRef="find_target_account" targetRef="target_account_exists" />
5455
<bpmn:sequenceFlow id="arrow_account_missing" name="reconfigure" sourceRef="target_account_exists" targetRef="task_configure" />
55-
<bpmn:sequenceFlow id="arrow_target_account_exists" name="yes" sourceRef="target_account_exists" targetRef="extract_accounts">
56+
<bpmn:sequenceFlow id="arrow_target_account_exists" name="yes" sourceRef="target_account_exists" targetRef="read_csv_file">
5657
<bpmn:conditionExpression xsi:type="bpmn:tFormalExpression">${targetAccountId != null}</bpmn:conditionExpression>
5758
</bpmn:sequenceFlow>
5859
<bpmn:serviceTask id="read_csv_file" name="Read in CSV file" camunda:modelerTemplate="com.jongsoft.finance.bpmn.delegate.importer.ReadTransactionLogDelegate" camunda:modelerTemplateVersion="1" camunda:delegateExpression="${readTransactionLogDelegate}">
@@ -63,13 +64,14 @@ When calling the flow the following variables must be set:
6364
<camunda:outputParameter name="transactionTokens">${storageTokens}</camunda:outputParameter>
6465
<camunda:outputParameter name="allowGenerate">${generateAccounts}</camunda:outputParameter>
6566
<camunda:outputParameter name="applyRules">${applyRules}</camunda:outputParameter>
67+
<camunda:outputParameter name="accounts_data">${locatable}</camunda:outputParameter>
6668
</camunda:inputOutput>
6769
</bpmn:extensionElements>
68-
<bpmn:incoming>Flow_11g50b7</bpmn:incoming>
69-
<bpmn:outgoing>Flow_02v0o7o</bpmn:outgoing>
70+
<bpmn:incoming>arrow_target_account_exists</bpmn:incoming>
71+
<bpmn:outgoing>Flow_03qkx35</bpmn:outgoing>
7072
</bpmn:serviceTask>
7173
<bpmn:subProcess id="process_account_mapping" name="Map accounts">
72-
<bpmn:incoming>Flow_0ow7pbb</bpmn:incoming>
74+
<bpmn:incoming>transactions_found</bpmn:incoming>
7375
<bpmn:outgoing>Flow_0pngnev</bpmn:outgoing>
7476
<bpmn:multiInstanceLoopCharacteristics isSequential="true" camunda:collection="${accounts_data}" camunda:elementVariable="account_extract" />
7577
<bpmn:startEvent id="start_match_accounts">
@@ -147,7 +149,6 @@ When calling the flow the following variables must be set:
147149
<bpmn:outgoing>Flow_0czgnsv</bpmn:outgoing>
148150
</bpmn:serviceTask>
149151
</bpmn:subProcess>
150-
<bpmn:sequenceFlow id="Flow_0ow7pbb" name="extracts" sourceRef="extract_accounts" targetRef="process_account_mapping" />
151152
<bpmn:sequenceFlow id="Flow_0pngnev" name="mappings" sourceRef="process_account_mapping" targetRef="confirm_mappings" />
152153
<bpmn:userTask id="confirm_mappings" name="Confirm account mapping">
153154
<bpmn:extensionElements>
@@ -159,20 +160,9 @@ When calling the flow the following variables must be set:
159160
<bpmn:incoming>Flow_0pngnev</bpmn:incoming>
160161
<bpmn:outgoing>Flow_11g50b7</bpmn:outgoing>
161162
</bpmn:userTask>
162-
<bpmn:sequenceFlow id="Flow_11g50b7" sourceRef="confirm_mappings" targetRef="read_csv_file" />
163-
<bpmn:serviceTask id="extract_accounts" name="Extract accounts from CSV" camunda:modelerTemplate="com.jongsoft.finance.bpmn.delegate.importer.ExtractAccountDetailsDelegate" camunda:modelerTemplateVersion="1" camunda:delegateExpression="${extractAccountDetailsDelegate}">
164-
<bpmn:extensionElements>
165-
<camunda:inputOutput>
166-
<camunda:inputParameter name="batchImportSlug">${importJobSlug}</camunda:inputParameter>
167-
<camunda:inputParameter name="importConfig">${importConfiguration}</camunda:inputParameter>
168-
<camunda:outputParameter name="accounts_data">${locatable}</camunda:outputParameter>
169-
</camunda:inputOutput>
170-
</bpmn:extensionElements>
171-
<bpmn:incoming>arrow_target_account_exists</bpmn:incoming>
172-
<bpmn:outgoing>Flow_0ow7pbb</bpmn:outgoing>
173-
</bpmn:serviceTask>
163+
<bpmn:sequenceFlow id="Flow_11g50b7" sourceRef="confirm_mappings" targetRef="process_create_transactions" />
174164
<bpmn:subProcess id="process_create_transactions" name="Process transactions">
175-
<bpmn:incoming>Flow_02v0o7o</bpmn:incoming>
165+
<bpmn:incoming>Flow_11g50b7</bpmn:incoming>
176166
<bpmn:outgoing>Flow_16v2prx</bpmn:outgoing>
177167
<bpmn:multiInstanceLoopCharacteristics isSequential="true" camunda:collection="${transactionTokens}" camunda:elementVariable="transactionToken" />
178168
<bpmn:startEvent id="start_create_transaction">
@@ -315,7 +305,6 @@ When calling the flow the following variables must be set:
315305
<bpmn:outgoing>Flow_0rz5nze</bpmn:outgoing>
316306
</bpmn:serviceTask>
317307
</bpmn:subProcess>
318-
<bpmn:sequenceFlow id="Flow_02v0o7o" sourceRef="read_csv_file" targetRef="process_create_transactions" />
319308
<bpmn:sequenceFlow id="Flow_16v2prx" sourceRef="process_create_transactions" targetRef="mark_import_done" />
320309
<bpmn:endEvent id="endJob">
321310
<bpmn:incoming>Flow_06dyjk4</bpmn:incoming>
@@ -331,6 +320,16 @@ When calling the flow the following variables must be set:
331320
<bpmn:incoming>Flow_16v2prx</bpmn:incoming>
332321
<bpmn:outgoing>Flow_06dyjk4</bpmn:outgoing>
333322
</bpmn:serviceTask>
323+
<bpmn:sequenceFlow id="Flow_03qkx35" sourceRef="read_csv_file" targetRef="no_transactions" />
324+
<bpmn:exclusiveGateway id="no_transactions" name="empty import" default="transactions_found">
325+
<bpmn:incoming>Flow_03qkx35</bpmn:incoming>
326+
<bpmn:outgoing>transactions_found</bpmn:outgoing>
327+
<bpmn:outgoing>no_data_found</bpmn:outgoing>
328+
</bpmn:exclusiveGateway>
329+
<bpmn:sequenceFlow id="transactions_found" sourceRef="no_transactions" targetRef="process_account_mapping" />
330+
<bpmn:sequenceFlow id="no_data_found" name="Empty import file" sourceRef="no_transactions" targetRef="task_configure">
331+
<bpmn:conditionExpression xsi:type="bpmn:tFormalExpression">${accounts_data.isEmpty()}</bpmn:conditionExpression>
332+
</bpmn:sequenceFlow>
334333
</bpmn:process>
335334
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
336335
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="import_job">
@@ -357,7 +356,13 @@ When calling the flow the following variables must be set:
357356
</bpmndi:BPMNLabel>
358357
</bpmndi:BPMNShape>
359358
<bpmndi:BPMNShape id="Activity_1y5d1uo_di" bpmnElement="read_csv_file">
360-
<dc:Bounds x="1480" y="250" width="100" height="80" />
359+
<dc:Bounds x="1060" y="260" width="100" height="80" />
360+
</bpmndi:BPMNShape>
361+
<bpmndi:BPMNShape id="Gateway_02tmvr8_di" bpmnElement="no_transactions" isMarkerVisible="true">
362+
<dc:Bounds x="1085" y="365" width="50" height="50" />
363+
<bpmndi:BPMNLabel>
364+
<dc:Bounds x="1010" y="383" width="65" height="14" />
365+
</bpmndi:BPMNLabel>
361366
</bpmndi:BPMNShape>
362367
<bpmndi:BPMNShape id="Activity_0bn3ft1_di" bpmnElement="process_account_mapping" isExpanded="true">
363368
<dc:Bounds x="810" y="450" width="780" height="330" />
@@ -445,9 +450,6 @@ When calling the flow the following variables must be set:
445450
<bpmndi:BPMNShape id="Activity_0vbcm2j_di" bpmnElement="confirm_mappings">
446451
<dc:Bounds x="1280" y="250" width="100" height="80" />
447452
</bpmndi:BPMNShape>
448-
<bpmndi:BPMNShape id="Activity_0cfirls_di" bpmnElement="extract_accounts">
449-
<dc:Bounds x="1070" y="250" width="100" height="80" />
450-
</bpmndi:BPMNShape>
451453
<bpmndi:BPMNShape id="Activity_0nxa7la_di" bpmnElement="process_create_transactions" isExpanded="true">
452454
<dc:Bounds x="1710" y="85" width="1220" height="410" />
453455
<bpmndi:BPMNLabel />
@@ -624,24 +626,17 @@ When calling the flow the following variables must be set:
624626
<bpmndi:BPMNEdge id="Flow_1q5av68_di" bpmnElement="arrow_account_missing">
625627
<di:waypoint x="930" y="265" />
626628
<di:waypoint x="930" y="160" />
627-
<di:waypoint x="540" y="160" />
628-
<di:waypoint x="540" y="250" />
629+
<di:waypoint x="560" y="160" />
630+
<di:waypoint x="560" y="250" />
629631
<bpmndi:BPMNLabel>
630-
<dc:Bounds x="708" y="142" width="56" height="14" />
632+
<dc:Bounds x="718" y="142" width="56" height="14" />
631633
</bpmndi:BPMNLabel>
632634
</bpmndi:BPMNEdge>
633635
<bpmndi:BPMNEdge id="Flow_1fbjukg_di" bpmnElement="arrow_target_account_exists">
634636
<di:waypoint x="955" y="290" />
635-
<di:waypoint x="1070" y="290" />
636-
<bpmndi:BPMNLabel>
637-
<dc:Bounds x="1005" y="272" width="17" height="14" />
638-
</bpmndi:BPMNLabel>
639-
</bpmndi:BPMNEdge>
640-
<bpmndi:BPMNEdge id="Flow_0ow7pbb_di" bpmnElement="Flow_0ow7pbb">
641-
<di:waypoint x="1120" y="330" />
642-
<di:waypoint x="1120" y="450" />
637+
<di:waypoint x="1060" y="290" />
643638
<bpmndi:BPMNLabel>
644-
<dc:Bounds x="1130" y="383" width="40" height="14" />
639+
<dc:Bounds x="985" y="272" width="17" height="14" />
645640
</bpmndi:BPMNLabel>
646641
</bpmndi:BPMNEdge>
647642
<bpmndi:BPMNEdge id="Flow_0pngnev_di" bpmnElement="Flow_0pngnev">
@@ -653,10 +648,6 @@ When calling the flow the following variables must be set:
653648
</bpmndi:BPMNEdge>
654649
<bpmndi:BPMNEdge id="Flow_11g50b7_di" bpmnElement="Flow_11g50b7">
655650
<di:waypoint x="1380" y="290" />
656-
<di:waypoint x="1480" y="290" />
657-
</bpmndi:BPMNEdge>
658-
<bpmndi:BPMNEdge id="Flow_02v0o7o_di" bpmnElement="Flow_02v0o7o">
659-
<di:waypoint x="1580" y="290" />
660651
<di:waypoint x="1710" y="290" />
661652
</bpmndi:BPMNEdge>
662653
<bpmndi:BPMNEdge id="Flow_16v2prx_di" bpmnElement="Flow_16v2prx">
@@ -667,6 +658,24 @@ When calling the flow the following variables must be set:
667658
<di:waypoint x="3110" y="290" />
668659
<di:waypoint x="3192" y="290" />
669660
</bpmndi:BPMNEdge>
661+
<bpmndi:BPMNEdge id="Flow_03qkx35_di" bpmnElement="Flow_03qkx35">
662+
<di:waypoint x="1110" y="340" />
663+
<di:waypoint x="1110" y="365" />
664+
</bpmndi:BPMNEdge>
665+
<bpmndi:BPMNEdge id="Flow_03qrqg6_di" bpmnElement="transactions_found">
666+
<di:waypoint x="1110" y="415" />
667+
<di:waypoint x="1110" y="450" />
668+
</bpmndi:BPMNEdge>
669+
<bpmndi:BPMNEdge id="Flow_1wlgrgi_di" bpmnElement="no_data_found">
670+
<di:waypoint x="1135" y="390" />
671+
<di:waypoint x="1200" y="390" />
672+
<di:waypoint x="1200" y="130" />
673+
<di:waypoint x="540" y="130" />
674+
<di:waypoint x="540" y="250" />
675+
<bpmndi:BPMNLabel>
676+
<dc:Bounds x="1109" y="133" width="82" height="14" />
677+
</bpmndi:BPMNLabel>
678+
</bpmndi:BPMNEdge>
670679
</bpmndi:BPMNPlane>
671680
</bpmndi:BPMNDiagram>
672681
</bpmn:definitions>

0 commit comments

Comments
 (0)