Skip to content

Commit 005b266

Browse files
strheastrhea
authored
Merge pull request #225 from adv12/ImportPerformance
Improve Import Performance
2 parents 4bda9d2 + edd679e commit 005b266

File tree

14 files changed

+286
-81
lines changed

14 files changed

+286
-81
lines changed

‎ISOv4Plugin/ExtensionMethods/ExtensionMethods.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,34 @@ public static string AsHexDDI(this int n)
6767
return n.ToString("X4");
6868
}
6969

70+
public static bool ReverseEquals(this string s1, string s2)
71+
{
72+
if (ReferenceEquals(s1, s2))
73+
{
74+
return true;
75+
}
76+
77+
if (s1 == null || s2 == null)
78+
{
79+
return false;
80+
}
81+
82+
if (s1.Length != s2.Length)
83+
{
84+
return false;
85+
}
86+
87+
for (int i = s1.Length - 1; i >= 0; i--)
88+
{
89+
if (s1[i] != s2[i])
90+
{
91+
return false;
92+
}
93+
}
94+
95+
return true;
96+
}
97+
7098

7199
/// <summary>
72100
/// Looks up unit, converts and loads representation

‎ISOv4Plugin/ISOModels/ISODataLogValue.cs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,17 @@ namespace AgGateway.ADAPT.ISOv4Plugin.ISOModels
1414
{
1515
public class ISODataLogValue : ISOElement
1616
{
17-
//Attributes
18-
public string ProcessDataDDI { get; set; }
17+
private string _processDataDDI;
18+
public string ProcessDataDDI
19+
{
20+
get => _processDataDDI;
21+
set
22+
{
23+
_processDataDDI = value;
24+
ProcessDataIntDDI = value.AsInt32DDI();
25+
}
26+
}
27+
public int ProcessDataIntDDI { get; private set; }
1928
public int? ProcessDataValue { get; set; }
2029
public string DeviceElementIdRef { get; set; }
2130
public uint? DataLogPGN { get; set; }

‎ISOv4Plugin/ISOModels/ISODevice.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,36 @@ public ISODevice()
3636
public List<ISODeviceProperty> DeviceProperties { get; set; }
3737
public List<ISODeviceValuePresentation> DeviceValuePresentations { get; set; }
3838

39+
//Performance optimizations
40+
private Dictionary<int, ISODeviceProcessData> _deviceProcessDataByIntDDI;
41+
42+
public ISODeviceProcessData FirstOrDefaultDeviceProcessData(int intDDI)
43+
{
44+
if (DeviceProcessDatas == null)
45+
{
46+
return null;
47+
}
48+
if (_deviceProcessDataByIntDDI == null)
49+
{
50+
_deviceProcessDataByIntDDI = new Dictionary<int, ISODeviceProcessData>();
51+
foreach (var dpd2 in DeviceProcessDatas)
52+
{
53+
if (!_deviceProcessDataByIntDDI.TryGetValue(dpd2.IntDDI, out _))
54+
{
55+
_deviceProcessDataByIntDDI[dpd2.IntDDI] = dpd2;
56+
}
57+
}
58+
}
59+
60+
ISODeviceProcessData dpd;
61+
if (_deviceProcessDataByIntDDI.TryGetValue(intDDI, out dpd))
62+
{
63+
return dpd;
64+
}
65+
66+
return null;
67+
}
68+
3969
public override XmlWriter WriteXML(XmlWriter xmlBuilder)
4070
{
4171
xmlBuilder.WriteStartElement("DVC");

‎ISOv4Plugin/ISOModels/ISODeviceProcessData.cs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,17 @@ public class ISODeviceProcessData : ISOElement
1717
{
1818
//Attributes
1919
public uint ObjectID { get; set; }
20-
public string DDI { get; set; }
20+
private string _ddi;
21+
public string DDI
22+
{
23+
get => _ddi;
24+
set
25+
{
26+
_ddi = value;
27+
IntDDI = value.AsInt32DDI();
28+
}
29+
}
30+
public int IntDDI { get; private set; }
2131
public int Property { get; set; }
2232
public int TriggerMethods { get; set; }
2333
public string Designator { get; set; }

‎ISOv4Plugin/Mappers/Factories/TimeLogMapperFactory.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ private List<TimeLogWrapperGroup> HandleDuplicateDataLogValues(List<TimeLogWrapp
9393
foreach (var timeLogGroup in timeLogGroups)
9494
{
9595
var duplicatesByElementAndDDI = timeLogGroup.SelectMany(x => x.DataLogValues)
96-
.Where(x => x.DataLogPGN == null && x.ProcessDataDDI != "0000")
96+
.Where(x => x.DataLogPGN == null && x.ProcessDataIntDDI != 0x0000)
9797
.GroupBy(x => new { x.DeviceElementIdRef, x.ProcessDataDDI })
9898
.Where(x => x.Count() > 1);
9999
if (!timeLogGroup.KeepAsGroup && duplicatesByElementAndDDI.Any())

‎ISOv4Plugin/Mappers/LoggedDataMappers/Import/CondensedWorkStateMeterCreator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,13 +106,13 @@ public abstract class CondensedStateMeterCreator : IEnumeratedMeterCreator
106106
public List<ISOEnumeratedMeter> CreateMeters(IEnumerable<ISOSpatialRow> spatialRows, ISODataLogValue dlv)
107107
{
108108
//We need to find a row of data with the value in order to create the correct number of meters.
109-
var spatialRowWithDdi = spatialRows.FirstOrDefault(x => x.SpatialValues.Any(y => y.DataLogValue.ProcessDataDDI.AsInt32DDI() == DDI
109+
var spatialRowWithDdi = spatialRows.FirstOrDefault(x => x.SpatialValues.Any(y => y.DataLogValue.ProcessDataIntDDI == DDI
110110
&& y.DataLogValue.DeviceElementIdRef == dlv.DeviceElementIdRef));
111111

112112
int numberOfSections = 0;
113113
if (spatialRowWithDdi != null)
114114
{
115-
var spatialValue = spatialRowWithDdi.SpatialValues.First(x => x.DataLogValue.ProcessDataDDI.AsInt32DDI() == DDI &&
115+
var spatialValue = spatialRowWithDdi.SpatialValues.First(x => x.DataLogValue.ProcessDataIntDDI == DDI &&
116116
x.DataLogValue.DeviceElementIdRef == dlv.DeviceElementIdRef);
117117
numberOfSections = GetNumberOfInstalledSections(spatialValue);
118118
}

‎ISOv4Plugin/Mappers/LoggedDataMappers/Import/SectionMapper.cs‎

Lines changed: 52 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Linq;
44
using AgGateway.ADAPT.ApplicationDataModel.Equipment;
55
using AgGateway.ADAPT.ApplicationDataModel.LoggedData;
6+
using AgGateway.ADAPT.ISOv4Plugin.ExtensionMethods;
67
using AgGateway.ADAPT.ISOv4Plugin.ISOModels;
78
using AgGateway.ADAPT.ISOv4Plugin.ObjectModel;
89

@@ -30,17 +31,50 @@ public List<DeviceElementUse> Map(ISOTime time,
3031
IEnumerable<string> isoDeviceElementIDs,
3132
Dictionary<string, List<ISOProductAllocation>> isoProductAllocations)
3233
{
34+
var usedDataLogValues = new List<ISODataLogValue>();
35+
36+
foreach (string isoDeviceElementID in isoDeviceElementIDs)
37+
{
38+
DeviceHierarchyElement hierarchyElement = TaskDataMapper.DeviceElementHierarchies.GetMatchingElement(isoDeviceElementID);
39+
int? adaptDeviceElementId = TaskDataMapper.InstanceIDMap.GetADAPTID(isoDeviceElementID);
40+
if (hierarchyElement != null &&
41+
adaptDeviceElementId.HasValue &&
42+
DataModel.Catalog.DeviceElements.Any(d => d.Id.ReferenceId == adaptDeviceElementId.Value))
43+
{
44+
usedDataLogValues.AddRange(_workingDataMapper.GetDataLogValuesForDeviceElement(time, hierarchyElement));
45+
}
46+
}
47+
48+
List<ISOSpatialRow> isoRecordsWithData = new List<ISOSpatialRow>();
49+
if (usedDataLogValues.Any())
50+
{
51+
foreach (var isoRecord in isoRecords)
52+
{
53+
int beforeCount = usedDataLogValues.Count;
54+
var notReferencedDataLogValues = usedDataLogValues.Where(x =>
55+
!isoRecord.SpatialValues.Any(y => y.DataLogValue.ProcessDataIntDDI == x.ProcessDataIntDDI &&
56+
y.DataLogValue.DeviceElementIdRef.ReverseEquals(x.DeviceElementIdRef)));
57+
usedDataLogValues = notReferencedDataLogValues.ToList();
58+
if (beforeCount != usedDataLogValues.Count)
59+
{
60+
isoRecordsWithData.Add(isoRecord);
61+
}
62+
if (usedDataLogValues.Count == 0)
63+
{
64+
break;
65+
}
66+
}
67+
}
68+
69+
3370
var sections = new List<DeviceElementUse>();
3471
foreach (string isoDeviceElementID in isoDeviceElementIDs)
3572
{
3673
DeviceHierarchyElement hierarchyElement = TaskDataMapper.DeviceElementHierarchies.GetMatchingElement(isoDeviceElementID);
3774
if (hierarchyElement != null)
3875
{
39-
DeviceElementUse deviceElementUse = null;
40-
List<WorkingData> workingDatas = new List<WorkingData>();
41-
4276
//Get the relevant DeviceElementConfiguration
43-
int adaptDeviceElementId = TaskDataMapper.InstanceIDMap.GetADAPTID(isoDeviceElementID).Value;
77+
int? adaptDeviceElementId = TaskDataMapper.InstanceIDMap.GetADAPTID(isoDeviceElementID);
4478
DeviceElement adaptDeviceElement = DataModel.Catalog.DeviceElements.SingleOrDefault(d => d.Id.ReferenceId == adaptDeviceElementId);
4579
if (adaptDeviceElement != null)
4680
{
@@ -55,33 +89,29 @@ public List<DeviceElementUse> Map(ISOTime time,
5589
order = hierarchyElement.Parent.Order;
5690
}
5791

58-
deviceElementUse = sections.FirstOrDefault(d => d.DeviceConfigurationId == config.Id.ReferenceId);
92+
List<WorkingData> workingDatas = new List<WorkingData>();
93+
DeviceElementUse deviceElementUse = sections.FirstOrDefault(d => d.DeviceConfigurationId == config.Id.ReferenceId);
5994
if (deviceElementUse == null)
6095
{
6196
//Create the DeviceElementUse
62-
deviceElementUse = new DeviceElementUse();
63-
deviceElementUse.Depth = depth;
64-
deviceElementUse.Order = order;
65-
deviceElementUse.OperationDataId = operationDataId;
66-
deviceElementUse.DeviceConfigurationId = config.Id.ReferenceId;
67-
68-
//Add Working Data for any data on this device element
69-
List<WorkingData> data = _workingDataMapper.Map(time, isoRecords, deviceElementUse, hierarchyElement, sections, isoProductAllocations);
70-
if (data.Any())
97+
deviceElementUse = new DeviceElementUse
7198
{
72-
workingDatas.AddRange(data);
73-
}
99+
Depth = depth,
100+
Order = order,
101+
OperationDataId = operationDataId,
102+
DeviceConfigurationId = config.Id.ReferenceId
103+
};
74104
}
75105
else
76106
{
77107
workingDatas = deviceElementUse.GetWorkingDatas().ToList();
108+
}
78109

79-
//Add Additional Working Data
80-
List<WorkingData> data = _workingDataMapper.Map(time, isoRecords, deviceElementUse, hierarchyElement, sections, isoProductAllocations);
81-
if (data.Any())
82-
{
83-
workingDatas.AddRange(data);
84-
}
110+
//Add Working Data for any data on this device element
111+
List<WorkingData> data = _workingDataMapper.Map(time, isoRecordsWithData, deviceElementUse, hierarchyElement, sections, isoProductAllocations);
112+
if (data.Any())
113+
{
114+
workingDatas.AddRange(data);
85115
}
86116

87117
deviceElementUse.GetWorkingDatas = () => workingDatas;

‎ISOv4Plugin/Mappers/LoggedDataMappers/Import/SkyConditionsMeterCreator.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public List<ISOEnumeratedMeter> CreateMeters(IEnumerable<ISOSpatialRow> spatialR
3535

3636
public EnumeratedValue GetValueForMeter(SpatialValue value, ISOEnumeratedMeter meter)
3737
{
38-
if (value.DataLogValue.ProcessDataDDI.AsInt32DDI() != DDI)
38+
if (value.DataLogValue.ProcessDataIntDDI != DDI)
3939
return null;
4040

4141
const int clear = 0x20524C43;

‎ISOv4Plugin/Mappers/LoggedDataMappers/Import/SpatialRecordMapper.cs‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Linq;
55
using AgGateway.ADAPT.ApplicationDataModel.LoggedData;
66
using AgGateway.ADAPT.ApplicationDataModel.Representations;
7+
using AgGateway.ADAPT.ISOv4Plugin.ExtensionMethods;
78
using AgGateway.ADAPT.ISOv4Plugin.ObjectModel;
89
using AgGateway.ADAPT.ISOv4Plugin.ISOModels;
910
using AgGateway.ADAPT.Representation.UnitSystem;
@@ -91,8 +92,9 @@ private void SetEnumeratedMeterValue(ISOSpatialRow isoSpatialRow, EnumeratedWork
9192
{
9293
var isoDataLogValue = _workingDataMapper.DataLogValuesByWorkingDataID[meter.Id.ReferenceId];
9394
var isoValue = isoSpatialRow.SpatialValues.FirstOrDefault(v =>
94-
v.DataLogValue.DeviceElementIdRef == isoDataLogValue.DeviceElementIdRef &&
95-
v.DataLogValue.ProcessDataDDI == isoDataLogValue.ProcessDataDDI);
95+
v.DataLogValue.ProcessDataIntDDI == isoDataLogValue.ProcessDataIntDDI &&
96+
v.DataLogValue.DeviceElementIdRef.ReverseEquals(isoDataLogValue.DeviceElementIdRef)
97+
);
9698
if (isoValue != null)
9799
{
98100
var isoEnumeratedMeter = meter as ISOEnumeratedMeter;
@@ -112,14 +114,13 @@ private void SetNumericMeterValue(ISOSpatialRow isoSpatialRow, NumericWorkingDat
112114
var dataLogValue = _workingDataMapper.DataLogValuesByWorkingDataID.ContainsKey(meter.Id.ReferenceId)
113115
? _workingDataMapper.DataLogValuesByWorkingDataID[meter.Id.ReferenceId]
114116
: null;
115-
var isoValue = dataLogValue != null
117+
var isoValue = dataLogValue != null && dataLogValue.ProcessDataIntDDI != 0xDFEE
116118
? isoSpatialRow.SpatialValues.FirstOrDefault(v =>
117-
v.DataLogValue.ProcessDataDDI != "DFFE" &&
118-
v.DataLogValue.DeviceElementIdRef == dataLogValue.DeviceElementIdRef &&
119-
v.DataLogValue.ProcessDataDDI == dataLogValue.ProcessDataDDI)
119+
v.DataLogValue.ProcessDataIntDDI == dataLogValue.ProcessDataIntDDI &&
120+
v.DataLogValue.DeviceElementIdRef.ReverseEquals(dataLogValue.DeviceElementIdRef)
121+
)
120122
: null;
121123

122-
123124
if (isoValue != null)
124125
{
125126
ADAPT.ApplicationDataModel.Common.UnitOfMeasure userProvidedUnitOfMeasure = meter.UnitOfMeasure; //Default; no display uom provided.
@@ -218,7 +219,7 @@ private DateTime ToUtc(DateTime dateTime)
218219
}
219220
else if (dateTime.Kind == DateTimeKind.Unspecified && _taskDataMapper.GPSToLocalDelta.HasValue)
220221
{
221-
utc = new DateTime(dateTime.AddHours(- _taskDataMapper.GPSToLocalDelta.Value).Ticks, DateTimeKind.Utc);
222+
utc = new DateTime(dateTime.AddHours(-_taskDataMapper.GPSToLocalDelta.Value).Ticks, DateTimeKind.Utc);
222223
}
223224
else
224225
{

‎ISOv4Plugin/Mappers/LoggedDataMappers/Import/WorkingDataMapper.cs‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ public interface IWorkingDataMapper
2020
WorkingData ConvertToBaseType(WorkingData meter);
2121
Dictionary<int, ISODataLogValue> DataLogValuesByWorkingDataID { get; set; }
2222
Dictionary<int, string> ISODeviceElementIDsByWorkingDataID { get; set; }
23+
24+
IEnumerable<ISODataLogValue> GetDataLogValuesForDeviceElement(ISOTime time, DeviceHierarchyElement deviceElementHierarchy);
2325
}
2426

2527
public class WorkingDataMapper : BaseMapper, IWorkingDataMapper
@@ -40,6 +42,13 @@ public WorkingDataMapper(IEnumeratedMeterFactory enumeratedMeterCreatorFactory,
4042
ISODeviceElementIDsByWorkingDataID = new Dictionary<int, string>();
4143
}
4244

45+
public IEnumerable<ISODataLogValue> GetDataLogValuesForDeviceElement(ISOTime time, DeviceHierarchyElement deviceElementHierarchy)
46+
{
47+
return time.DataLogValues.Where(dlv => dlv.DeviceElementIdRef == deviceElementHierarchy.DeviceElement.DeviceElementId || //DLV DET reference matches the primary DET for the ADAPT element
48+
deviceElementHierarchy.MergedElements.Any(e => e.DeviceElementId == dlv.DeviceElementIdRef)) //DLV DET reference matches one of the merged DETs on the ADAPT element
49+
.ToList();
50+
}
51+
4352
public List<WorkingData> Map(ISOTime time,
4453
IEnumerable<ISOSpatialRow> isoSpatialRows,
4554
DeviceElementUse deviceElementUse,
@@ -73,7 +82,7 @@ public List<WorkingData> Map(ISOTime time,
7382
isoDeviceElementHierarchy);
7483
if (newWorkingDatas.Count() > 0)
7584
{
76-
int ddi = dlv.ProcessDataDDI.AsInt32DDI();
85+
int ddi = dlv.ProcessDataIntDDI;
7786
if (!EnumeratedMeterFactory.IsCondensedMeter(ddi))
7887
{
7988
//We skip adding Condensed WorkingDatas to this DeviceElementUse since they were added separately below to their specific DeviceElementUse
@@ -126,7 +135,7 @@ private IEnumerable<WorkingData> Map(ISODataLogValue dlv,
126135
DeviceHierarchyElement isoDeviceElementHierarchy)
127136
{
128137
var workingDatas = new List<WorkingData>();
129-
if (_ddis.ContainsKey(dlv.ProcessDataDDI.AsInt32DDI()))
138+
if (_ddis.ContainsKey(dlv.ProcessDataIntDDI))
130139
{
131140
//Numeric Representations
132141
NumericWorkingData numericMeter = MapNumericMeter(dlv, deviceElementUse.Id.ReferenceId);
@@ -135,7 +144,7 @@ private IEnumerable<WorkingData> Map(ISODataLogValue dlv,
135144
workingDatas.Add(numericMeter);
136145
return workingDatas;
137146
}
138-
var meterCreator = _enumeratedMeterCreatorFactory.GetMeterCreator(dlv.ProcessDataDDI.AsInt32DDI());
147+
var meterCreator = _enumeratedMeterCreatorFactory.GetMeterCreator(dlv.ProcessDataIntDDI);
139148
if (meterCreator != null)
140149
{
141150
//Enumerated Representations
@@ -187,9 +196,9 @@ private NumericWorkingData MapNumericMeter(ISODataLogValue dlv, int deviceElemen
187196
{
188197
var meter = new NumericWorkingData
189198
{
190-
UnitOfMeasure = RepresentationMapper.GetUnitForDdi(dlv.ProcessDataDDI.AsInt32DDI()),
199+
UnitOfMeasure = RepresentationMapper.GetUnitForDdi(dlv.ProcessDataIntDDI),
191200
DeviceElementUseId = deviceElementUseId,
192-
Representation = RepresentationMapper.Map(dlv.ProcessDataDDI.AsInt32DDI())
201+
Representation = RepresentationMapper.Map(dlv.ProcessDataIntDDI)
193202
};
194203
return meter;
195204
}

0 commit comments

Comments
 (0)