diff --git a/Px.Abstractions/Interfaces/IDataSource.cs b/Px.Abstractions/Interfaces/IDataSource.cs
index 661ea03f..dedc82f3 100644
--- a/Px.Abstractions/Interfaces/IDataSource.cs
+++ b/Px.Abstractions/Interfaces/IDataSource.cs
@@ -43,5 +43,16 @@ public interface IDataSource
/// codelist with texts for a specific language
///
Codelist? GetCodelist(string id, string language);
+
+ /// Intended for the (Lucene) indexing.
+ /// Returns a list of tableid (url-type not datasource-type) for tables where published is in the intervall [from,to]
+ /// For cnmm the tableid is the maintable.tableid column and time of publication is the content.published column.
+ /// Does not restrict the output-list to things like PresCategory = public or table "is running", since
+ /// we want to run it with internal DBs.
+ ///
+ /// Earliest. MinDate. Inclusive
+ /// Lastest. MaxDate. Inclusive
+ /// A list of (url-type) tableids (cnmm:maintable.tableid) which may be empty
+ List GetTablesPublishedBetween(DateTime from, DateTime to);
}
}
diff --git a/PxWeb.UnitTests/DataSource/PXDataSourceTest.cs b/PxWeb.UnitTests/DataSource/PXDataSourceTest.cs
index 54e753ea..ddb8cd06 100644
--- a/PxWeb.UnitTests/DataSource/PXDataSourceTest.cs
+++ b/PxWeb.UnitTests/DataSource/PXDataSourceTest.cs
@@ -1,5 +1,7 @@
+using System;
+
namespace PxWeb.UnitTests.DataSource
{
[TestClass]
@@ -224,6 +226,37 @@ public void ShouldNotResolveTablePath()
Assert.IsFalse(selectionExists);
}
+ [TestMethod]
+ public void GetTablesPublishedBetween_WhenCalled_ReturnNonNull()
+ {
+ string pathRunning = Directory.GetCurrentDirectory();
+ int index = pathRunning.IndexOf("PxWeb.UnitTests");
+
+ if (index == -1)
+ {
+ throw new System.Exception("Hmm, Directory.GetCurrentDirectory() does not contain string:PxWeb.UnitTests , so unable to find wwwroot path.");
+ }
+ string repoRoot = pathRunning.Substring(0, index);
+ string wwwPath = Path.Combine(repoRoot, "PxWeb", "wwwroot");
+
+ var hostingEnvironmentMock = new Mock();
+ hostingEnvironmentMock
+ .Setup(m => m.RootPath)
+ .Returns(wwwPath);
+
+ var dataSource = new PxFileDataSource(
+ new Mock().Object,
+ new Mock().Object,
+ new Mock().Object,
+ hostingEnvironmentMock.Object,
+ new Mock().Object);
+
+ var updataedTables = dataSource.GetTablesPublishedBetween(new DateTime(2023, 8, 1), new DateTime(2023, 9, 1));
+
+ Assert.IsNotNull(updataedTables);
+
+ }
+
private TablePathResolverPxFile GetTablePathResolver()
{
var testFactory = new TestFactory();
@@ -260,5 +293,7 @@ private TablePathResolverPxFile GetTablePathResolver()
return resolver;
}
+
+
}
}
diff --git a/PxWeb/Code/Api2/DataSource/Cnmm/CnmmDataSource.cs b/PxWeb/Code/Api2/DataSource/Cnmm/CnmmDataSource.cs
index 25d6ea92..b0808784 100644
--- a/PxWeb/Code/Api2/DataSource/Cnmm/CnmmDataSource.cs
+++ b/PxWeb/Code/Api2/DataSource/Cnmm/CnmmDataSource.cs
@@ -249,5 +249,10 @@ private static bool IsInteger(string value)
return codelist;
}
+
+ public List GetTablesPublishedBetween(DateTime from, DateTime to)
+ {
+ return PCAxis.Sql.ApiUtils.ApiUtilStatic.GetTablesPublishedBetween(from, to);
+ }
}
}
diff --git a/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs b/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs
index 52dae0c5..3f210f9c 100644
--- a/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs
+++ b/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs
@@ -166,5 +166,40 @@ private string GetIdentifierWithoutPath(string id)
}
}
+ public List GetTablesPublishedBetween(DateTime from, DateTime to)
+ {
+ MenuXmlFile menuXmlFile = new MenuXmlFile(_hostingEnvironment);
+ var doc = menuXmlFile.GetAsXmlDocument();
+
+ var tableIds = new List();
+ var nodes = doc.SelectNodes("//Link[LastUpdated]");
+
+ if (nodes is null)
+ {
+ return tableIds;
+ }
+
+ foreach (XmlNode link in nodes)
+ {
+ var lastUpdatedNode = link.SelectSingleNode("LastUpdated");
+ if (lastUpdatedNode != null)
+ {
+ if (DateTime.TryParse(lastUpdatedNode.InnerText, out DateTime lastUpdated))
+ {
+ if (lastUpdated >= from && lastUpdated <= to)
+ {
+ string? tableId = link.Attributes?["tableId"]?.Value;
+
+ if (tableId is not null && !tableIds.Contains(tableId))
+ {
+ tableIds.Add(tableId);
+ }
+ }
+ }
+ }
+ }
+
+ return tableIds;
+ }
}
}
diff --git a/PxWeb/Controllers/Api2/Admin/SearchindexController.cs b/PxWeb/Controllers/Api2/Admin/SearchindexController.cs
index a363aa54..4010728a 100644
--- a/PxWeb/Controllers/Api2/Admin/SearchindexController.cs
+++ b/PxWeb/Controllers/Api2/Admin/SearchindexController.cs
@@ -1,6 +1,7 @@
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Text.RegularExpressions;
+using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
@@ -47,32 +48,53 @@ public SearchindexController(BackgroundWorkerQueue backgroundWorkerQueue, IContr
[SwaggerOperation("IndexDatabase")]
[SwaggerResponse(statusCode: 202, description: "Accepted")]
[SwaggerResponse(statusCode: 401, description: "Unauthorized")]
- public IActionResult IndexDatabase()
+ public IActionResult IndexDatabase(int? pastHours)
{
_backgroundWorkerQueue.QueueBackgroundWorkItem(async token =>
{
- try
+ if (pastHours is not null)
{
- List languages = new List();
+ try
+ {
+ DateTime to = DateTime.Now;
+ DateTime from = to - TimeSpan.FromHours(pastHours.Value);
+ List tableList = _dataSource.GetTablesPublishedBetween(from, to);
- var config = _pxApiConfigurationService.GetConfiguration();
+ string message = $"Looked for tables published between {from:yyyy-MM-dd HH:mm:ss} and {to:yyyy-MM-dd HH:mm:ss}. Found {tableList.Count()}";
- if (config.Languages.Count == 0)
- {
- _logger.LogError("No languages configured for PxApi. New index will not be created.");
- return;
+ _responseState.AddEvent(new Event("Information", message));
+ _logger.LogDebug(message);
+ if (tableList.Count > 0)
+ {
+
+ await UpdateFromTableList(tableList, token);
+ }
}
- foreach (var lang in config.Languages)
+ catch (System.Exception ex)
{
- languages.Add(lang.Id);
+ _responseState.AddEvent(new Event("Error", ex.Message));
+ _logger.LogError(ex, "Error when building search index");
}
-
- Indexer indexer = new Indexer(_dataSource, _backend, _logger);
- await Task.Run(() => indexer.IndexDatabase(languages), token);
}
- catch (System.Exception ex)
+ else
{
- _logger.LogError(ex, "Error when building search index");
+ try
+ {
+ //TODO: this factoring adds _responseState.AddEvent(new Event("Error", message)); Good thing, likp?
+ List languages = GetLangaugesFromConfig();
+ if (languages.Count == 0)
+ {
+ return;
+ }
+
+ Indexer indexer = new Indexer(_dataSource, _backend, _logger);
+ await Task.Run(() => indexer.IndexDatabase(languages), token);
+ }
+ catch (System.Exception ex)
+ {
+ _responseState.AddEvent(new Event("Error", ex.Message));
+ _logger.LogError(ex, "Error when building search index");
+ }
}
});
return new AcceptedResult();
@@ -94,46 +116,43 @@ public IActionResult IndexDatabase([FromBody, Required] string[] tables)
{
try
{
- List languages = new List();
List tableList = tables
.Select(table => Regex.Replace(table.Trim(), @"[^0-9a-zA-Z]+", "", RegexOptions.None, TimeSpan.FromMilliseconds(100)))
.ToList();
- if (tableList.Count == 0)
- {
- string message = "Incoming list with table id's to be updated is empty. Index will not be updated.";
- _logger.LogError(message);
- _responseState.AddEvent(new Event("Error", message));
- return;
- }
-
- var config = _pxApiConfigurationService.GetConfiguration();
-
- if (config.Languages.Count == 0)
- {
- string message = "No languages configured for PxApi. Index will not be updated.";
- _logger.LogError(message);
- _responseState.AddEvent(new Event("Error", message));
- return;
- }
-
- foreach (var lang in config.Languages)
- {
- languages.Add(lang.Id);
- }
-
- Indexer indexer = new Indexer(_dataSource, _backend, _logger);
- await Task.Run(() => indexer.UpdateTableEntries(tableList, languages), token);
+ await UpdateFromTableList(tableList, token);
}
catch (System.Exception ex)
{
_responseState.AddEvent(new Event("Error", ex.Message));
- _logger.LogError(ex.Message);
+ _logger.LogError(ex, ex.Message);
}
+
+
});
return new AcceptedResult();
}
+ private async Task UpdateFromTableList(List tableList, CancellationToken token)
+ {
+ if (tableList.Count == 0)
+ {
+ string message = "Incoming list with table id's to be updated is empty. Index will not be updated.";
+ _logger.LogError(message);
+ _responseState.AddEvent(new Event("Error", message));
+ return;
+ }
+
+ List languages = GetLangaugesFromConfig();
+ if (languages.Count == 0)
+ {
+ return;
+ }
+
+ Indexer indexer = new Indexer(_dataSource, _backend, _logger);
+ await Task.Run(() => indexer.UpdateTableEntries(tableList, languages), token);
+ }
+
[HttpGet]
[Route("/admin/searchindex")]
[SwaggerOperation("IndexDatabase")]
@@ -143,5 +162,25 @@ public IActionResult GetState()
{
return new JsonResult(_responseState.Data);
}
+
+ private List GetLangaugesFromConfig()
+ {
+ List languages = new List();
+ var config = _pxApiConfigurationService.GetConfiguration();
+
+ if (config.Languages.Count == 0)
+ {
+ string message = "No languages configured for PxApi. Index will not be updated.";
+ _logger.LogError(message);
+ _responseState.AddEvent(new Event("Error", message));
+ return languages;
+ }
+
+ foreach (var lang in config.Languages)
+ {
+ languages.Add(lang.Id);
+ }
+ return languages;
+ }
}
}