diff --git a/src/ContentRepository.InMemory/InMemoryDataProvider.cs b/src/ContentRepository.InMemory/InMemoryDataProvider.cs index 63b2a92ae..cbde69662 100644 --- a/src/ContentRepository.InMemory/InMemoryDataProvider.cs +++ b/src/ContentRepository.InMemory/InMemoryDataProvider.cs @@ -1447,7 +1447,10 @@ public override STT.Task UpdateIndexingActivityRunningStateAsync(int indexingAct { var activity = DB.IndexingActivities.FirstOrDefault(r => r.IndexingActivityId == indexingActivityId); if (activity != null) + { activity.RunningState = runningState; + activity.LockTime = DateTime.UtcNow; + } } return STT.Task.CompletedTask; } @@ -1468,11 +1471,15 @@ public override STT.Task RefreshIndexingActivityLockTimeAsync(int[] waitingIds, return STT.Task.CompletedTask; } - public override STT.Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken) + public override STT.Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + var timeLimit = DateTime.UtcNow.AddMinutes(maxAgeInMinutes); lock (DB.IndexingActivities) - foreach(var existing in DB.IndexingActivities.Where(x => x.RunningState == IndexingActivityRunningState.Done).ToArray()) + foreach(var existing in DB.IndexingActivities + .Where(x => x.RunningState == IndexingActivityRunningState.Done && + (x.LockTime == null || x.LockTime < timeLimit)) + .ToArray()) DB.IndexingActivities.Remove(existing); return STT.Task.CompletedTask; } diff --git a/src/ContentRepository.MsSql/MsSqlDataProviderScripts.cs b/src/ContentRepository.MsSql/MsSqlDataProviderScripts.cs index 07a270689..c9ebbc3f2 100644 --- a/src/ContentRepository.MsSql/MsSqlDataProviderScripts.cs +++ b/src/ContentRepository.MsSql/MsSqlDataProviderScripts.cs @@ -1149,7 +1149,7 @@ DECLARE @IdTable AS TABLE(Id INT) #region DeleteFinishedIndexingActivitiesScript protected override string DeleteFinishedIndexingActivitiesScript => @"-- MsSqlDataProvider.DeleteFinishedIndexingActivities -DELETE FROM IndexingActivities WHERE RunningState = 'Done' AND (LockTime < DATEADD(MINUTE, -23, GETUTCDATE()) OR LockTime IS NULL) +DELETE FROM IndexingActivities WHERE RunningState = 'Done' AND (LockTime < DATEADD(MINUTE, -@Minutes, GETUTCDATE()) OR LockTime IS NULL) "; #endregion diff --git a/src/ContentRepository/Search/Indexing/CentralizedIndexingActivityQueue.cs b/src/ContentRepository/Search/Indexing/CentralizedIndexingActivityQueue.cs index 7a7755a61..81fd03aaa 100644 --- a/src/ContentRepository/Search/Indexing/CentralizedIndexingActivityQueue.cs +++ b/src/ContentRepository/Search/Indexing/CentralizedIndexingActivityQueue.cs @@ -22,7 +22,8 @@ internal class CentralizedIndexingActivityQueue private readonly TimeSpan _waitingPollingPeriod = TimeSpan.FromSeconds(2); private readonly TimeSpan _healthCheckPeriod = TimeSpan.FromMinutes(2); - private readonly TimeSpan _deleteFinishedPeriod = TimeSpan.FromMinutes(23); + private readonly TimeSpan _deleteFinishedPeriod = TimeSpan.FromMinutes(Configuration.Indexing.IndexingActivityDeletionPeriodInMinutes); + private readonly int _maxAgeInMinutes = Configuration.Indexing.IndexingActivityMaxAgeInMinutes; private const int ActiveTaskLimit = 43; private System.Timers.Timer _timer; @@ -176,7 +177,7 @@ private void DeleteFinishedActivitiesOccasionally() { using (var op = SnTrace.IndexQueue.StartOperation("CIAQ: DeleteFinishedActivities")) { - DataStore.DeleteFinishedIndexingActivitiesAsync(CancellationToken.None) + DataStore.DeleteFinishedIndexingActivitiesAsync(_maxAgeInMinutes, CancellationToken.None) .GetAwaiter().GetResult(); _lastDeleteFinishedTime = DateTime.UtcNow; op.Successful = true; diff --git a/src/Storage/Configuration/Indexing.cs b/src/Storage/Configuration/Indexing.cs index 95e7ee4d2..7b2f85533 100644 --- a/src/Storage/Configuration/Indexing.cs +++ b/src/Storage/Configuration/Indexing.cs @@ -2,9 +2,35 @@ // ReSharper disable RedundantTypeArgumentsOfMethod using System; +using SenseNet.Tools.Configuration; namespace SenseNet.Configuration { + [OptionsClass("sensenet:indexing")] + public class IndexingOptions + { + public static string IndexDirectoryPath { get; set; } = ""; + /// + /// Gets or sets the periodicity of executing lost indexing tasks in seconds. Default: 60. + /// + public static int IndexHealthMonitorRunningPeriod { get; set; } = 60; + //public static int IndexHistoryItemLimit { get; set; } = 1000000; + //public static double CommitDelayInSeconds { get; set; } = 2.0d; + //public static int DelayedCommitCycleMaxCount { get; set; } = 10; + //public static int IndexingPausedTimeout { get; set; } = 60; + public static int IndexingActivityTimeoutInSeconds { get; set; } = 120; + public static int IndexingActivityQueueMaxLength { get; set; } = 500; + public static int TextExtractTimeout { get; set; } = 300; + /// + /// Gets or sets the periodicity of deleting old IndexingActivities. Default: 10 minutes. + /// + public static int IndexingActivityDeletionPeriodInMinutes { get; set; } = 10; + /// + /// Gets or sets the age threshold for IndexingActivities that are periodically deleted. + /// The default age threshold is set to 480 (8 hours). + /// + public static int IndexingActivityMaxAgeInMinutes { get; set; } = 480; + } public class Indexing : SnConfig { public static readonly string DefaultLocalIndexDirectory = "App_Data\\LocalIndex"; @@ -78,6 +104,9 @@ public static string IndexDirectoryFullPath public static int IndexingActivityQueueMaxLength { get; internal set; } = GetInt(SectionName, "IndexingActivityQueueMaxLength", 500); public static int TextExtractTimeout { get; internal set; } = GetInt(SectionName, "TextExtractTimeout", 300); + public static int IndexingActivityDeletionPeriodInMinutes { get; internal set; } = GetInt(SectionName, "IndexingActivityDeletionPeriodInMinutes", 10); + public static int IndexingActivityMaxAgeInMinutes { get; internal set; } = GetInt(SectionName, "IndexingActivityMaxAgeInMinutes", 8 * 60); + private static bool? GetNullableBool(string key) { var textValue = GetString(SectionName, key); diff --git a/src/Storage/Data/DataProvider.cs b/src/Storage/Data/DataProvider.cs index df5920b94..7d02c5b1e 100644 --- a/src/Storage/Data/DataProvider.cs +++ b/src/Storage/Data/DataProvider.cs @@ -714,12 +714,14 @@ public abstract Task UpdateIndexingActivityRunningStateAsync(int indexingActivit /// The token to monitor for cancellation requests. /// A Task that represents the asynchronous operation. public abstract Task RefreshIndexingActivityLockTimeAsync(int[] waitingIds, CancellationToken cancellationToken); + /// /// Deletes finished activities. Called by a cleanup background process. /// + /// Age of the IndexingActivities that will be deleted periodically. /// The token to monitor for cancellation requests. /// A Task that represents the asynchronous operation. - public abstract Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken); + public abstract Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken); /// /// Deletes all activities from the database. /// diff --git a/src/Storage/Data/DataStore.cs b/src/Storage/Data/DataStore.cs index 5fe0db371..887d288f0 100644 --- a/src/Storage/Data/DataStore.cs +++ b/src/Storage/Data/DataStore.cs @@ -698,9 +698,9 @@ public Task RefreshIndexingActivityLockTimeAsync(int[] waitingIds, CancellationT { return DataProvider.RefreshIndexingActivityLockTimeAsync(waitingIds, cancellationToken); } - public Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken) + public Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken) { - return DataProvider.DeleteFinishedIndexingActivitiesAsync(cancellationToken); + return DataProvider.DeleteFinishedIndexingActivitiesAsync(maxAgeInMinutes, cancellationToken); } public Task DeleteAllIndexingActivitiesAsync(CancellationToken cancellationToken) { diff --git a/src/Storage/Data/IDataStore.cs b/src/Storage/Data/IDataStore.cs index 6f35172d0..7cf2e0829 100644 --- a/src/Storage/Data/IDataStore.cs +++ b/src/Storage/Data/IDataStore.cs @@ -613,9 +613,10 @@ public Task UpdateIndexingActivityRunningStateAsync(int indexingActivityId, /// /// Deletes finished activities. Called by a cleanup background process. /// + /// Age of the IndexingActivities that will be deleted periodically. /// The token to monitor for cancellation requests. /// A Task that represents the asynchronous operation. - public Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken); + public Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken); /// /// Deletes all activities from the database. diff --git a/src/Storage/Data/RelationalDataProviderBase.cs b/src/Storage/Data/RelationalDataProviderBase.cs index 478aeb041..91ac83ace 100644 --- a/src/Storage/Data/RelationalDataProviderBase.cs +++ b/src/Storage/Data/RelationalDataProviderBase.cs @@ -2044,11 +2044,14 @@ await ctx.ExecuteNonQueryAsync(RefreshIndexingActivityLockTimeScript, cmd => } protected abstract string RefreshIndexingActivityLockTimeScript { get; } - public override async Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken) + public override async Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken) { using var op = SnTrace.Database.StartOperation("RelationalDataProviderBase: DeleteFinishedIndexingActivities()"); using var ctx = CreateDataContext(cancellationToken); - await ctx.ExecuteNonQueryAsync(DeleteFinishedIndexingActivitiesScript).ConfigureAwait(false); + await ctx.ExecuteNonQueryAsync(DeleteFinishedIndexingActivitiesScript, cmd => + { + cmd.Parameters.Add(ctx.CreateParameter("@Minutes", DbType.Int32, maxAgeInMinutes)); + }).ConfigureAwait(false); op.Successful = true; } protected abstract string DeleteFinishedIndexingActivitiesScript { get; } diff --git a/src/Tests/SenseNet.IntegrationTests/TestCases/CentralizedIndexingTestCases.cs b/src/Tests/SenseNet.IntegrationTests/TestCases/CentralizedIndexingTestCases.cs index 8b2287b4e..476a77f98 100644 --- a/src/Tests/SenseNet.IntegrationTests/TestCases/CentralizedIndexingTestCases.cs +++ b/src/Tests/SenseNet.IntegrationTests/TestCases/CentralizedIndexingTestCases.cs @@ -457,7 +457,7 @@ await RegisterActivityAsync(IndexingActivityType.AddDocument, IndexingActivit await RegisterActivityAsync(IndexingActivityType.AddDocument, IndexingActivityRunningState.Waiting, 5, 5, "/Root/Path5"), }; - await DP.DeleteFinishedIndexingActivitiesAsync(CancellationToken.None); + await DP.DeleteFinishedIndexingActivitiesAsync(23, CancellationToken.None); var indexingActivityFactory = Providers.Instance.Services.GetRequiredService(); var loaded = await DP.LoadIndexingActivitiesAsync(0, int.MaxValue, 9999, false, indexingActivityFactory, CancellationToken.None); diff --git a/src/Tests/SenseNet.IntegrationTests/TestCases/DataProviderTestCases.cs b/src/Tests/SenseNet.IntegrationTests/TestCases/DataProviderTestCases.cs index ef07cabed..8096a2d37 100644 --- a/src/Tests/SenseNet.IntegrationTests/TestCases/DataProviderTestCases.cs +++ b/src/Tests/SenseNet.IntegrationTests/TestCases/DataProviderTestCases.cs @@ -1639,7 +1639,7 @@ public async Task DP_IA_DeleteFinished() await IndexingActivityTest(async (firstId, lastId) => { // ACTION - await DP.DeleteFinishedIndexingActivitiesAsync(CancellationToken.None); + await DP.DeleteFinishedIndexingActivitiesAsync(23, CancellationToken.None); // ASSERT var result = await DP.LoadIndexingActivitiesAsync(firstId, lastId, 100, false, new TestIndexingActivityFactory(), CancellationToken.None);