Configurable IndexingActivity cleanup (#2028)

* New config keys in sensenet/indexing: IndexingActivityDeletionPeriodInMinutes, IndexingActivityMaxAgeInMinutes

* Option doc clarification for indexing.

* Implement maxAgeInMinutes in the InMem dataprovider.

---------

Co-authored-by: tusmester <tusmester@gmail.com>

src/ContentRepository.InMemory/InMemoryDataProvider.cs

@@ -1447,7 +1447,10 @@ public override STT.Task UpdateIndexingActivityRunningStateAsync(int indexingAct
             {
                 var activity = DB.IndexingActivities.FirstOrDefault(r => r.IndexingActivityId == indexingActivityId);
                 if (activity != null)
+                {
                     activity.RunningState = runningState;
+                    activity.LockTime = DateTime.UtcNow;
+                }
             }
             return STT.Task.CompletedTask;
         }
@@ -1468,11 +1471,15 @@ public override STT.Task RefreshIndexingActivityLockTimeAsync(int[] waitingIds,
             return STT.Task.CompletedTask;
         }
 
-        public override STT.Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken)
+        public override STT.Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken)
         {
             cancellationToken.ThrowIfCancellationRequested();
+            var timeLimit = DateTime.UtcNow.AddMinutes(maxAgeInMinutes);
             lock (DB.IndexingActivities)
-                foreach(var existing in DB.IndexingActivities.Where(x => x.RunningState == IndexingActivityRunningState.Done).ToArray())
+                foreach(var existing in DB.IndexingActivities
+                            .Where(x => x.RunningState == IndexingActivityRunningState.Done &&
+                                        (x.LockTime == null || x.LockTime < timeLimit))
+                            .ToArray())
                     DB.IndexingActivities.Remove(existing);
             return STT.Task.CompletedTask;
         }

src/ContentRepository.MsSql/MsSqlDataProviderScripts.cs

@@ -1149,7 +1149,7 @@ DECLARE @IdTable AS TABLE(Id INT)
 
         #region DeleteFinishedIndexingActivitiesScript
         protected override string DeleteFinishedIndexingActivitiesScript => @"-- MsSqlDataProvider.DeleteFinishedIndexingActivities
-DELETE FROM IndexingActivities WHERE RunningState = 'Done' AND (LockTime < DATEADD(MINUTE, -23, GETUTCDATE()) OR LockTime IS NULL)
+DELETE FROM IndexingActivities WHERE RunningState = 'Done' AND (LockTime < DATEADD(MINUTE, -@Minutes, GETUTCDATE()) OR LockTime IS NULL)
 ";
         #endregion
 

src/ContentRepository/Search/Indexing/CentralizedIndexingActivityQueue.cs

@@ -22,7 +22,8 @@ internal class CentralizedIndexingActivityQueue
 
         private readonly TimeSpan _waitingPollingPeriod = TimeSpan.FromSeconds(2);
         private readonly TimeSpan _healthCheckPeriod = TimeSpan.FromMinutes(2);
-        private readonly TimeSpan _deleteFinishedPeriod = TimeSpan.FromMinutes(23);
+        private readonly TimeSpan _deleteFinishedPeriod = TimeSpan.FromMinutes(Configuration.Indexing.IndexingActivityDeletionPeriodInMinutes);
+        private readonly int _maxAgeInMinutes = Configuration.Indexing.IndexingActivityMaxAgeInMinutes;
         private const int ActiveTaskLimit = 43;
 
         private System.Timers.Timer _timer;
@@ -176,7 +177,7 @@ private void DeleteFinishedActivitiesOccasionally()
             {
                 using (var op = SnTrace.IndexQueue.StartOperation("CIAQ: DeleteFinishedActivities"))
                 {
-                    DataStore.DeleteFinishedIndexingActivitiesAsync(CancellationToken.None)
+                    DataStore.DeleteFinishedIndexingActivitiesAsync(_maxAgeInMinutes, CancellationToken.None)
                         .GetAwaiter().GetResult();
                     _lastDeleteFinishedTime = DateTime.UtcNow;
                     op.Successful = true;

src/Storage/Configuration/Indexing.cs

@@ -2,9 +2,35 @@
 // ReSharper disable RedundantTypeArgumentsOfMethod
 
 using System;
+using SenseNet.Tools.Configuration;
 
 namespace SenseNet.Configuration
 {
+    [OptionsClass("sensenet:indexing")]
+    public class IndexingOptions
+    {
+        public static string IndexDirectoryPath { get; set; } = "";
+        /// <summary>
+        /// Gets or sets the periodicity of executing lost indexing tasks in seconds. Default: 60.
+        /// </summary>
+        public static int IndexHealthMonitorRunningPeriod { get; set; } = 60;
+        //public static int IndexHistoryItemLimit { get; set; } = 1000000;
+        //public static double CommitDelayInSeconds { get; set; } = 2.0d;
+        //public static int DelayedCommitCycleMaxCount { get; set; } = 10;
+        //public static int IndexingPausedTimeout { get; set; } = 60;
+        public static int IndexingActivityTimeoutInSeconds { get; set; } = 120;
+        public static int IndexingActivityQueueMaxLength { get; set; } = 500;
+        public static int TextExtractTimeout { get; set; } = 300;
+        /// <summary>
+        /// Gets or sets the periodicity of deleting old IndexingActivities. Default: 10 minutes.
+        /// </summary>
+        public static int IndexingActivityDeletionPeriodInMinutes { get; set; } = 10;
+        /// <summary>
+        /// Gets or sets the age threshold for IndexingActivities that are periodically deleted.
+        /// The default age threshold is set to 480 (8 hours).
+        /// </summary>
+        public static int IndexingActivityMaxAgeInMinutes { get; set; } = 480;
+    }
     public class Indexing : SnConfig
     {
         public static readonly string DefaultLocalIndexDirectory = "App_Data\\LocalIndex";
@@ -78,6 +104,9 @@ public static string IndexDirectoryFullPath
         public static int IndexingActivityQueueMaxLength { get; internal set; } = GetInt(SectionName, "IndexingActivityQueueMaxLength", 500);
         public static int TextExtractTimeout { get; internal set; } = GetInt(SectionName, "TextExtractTimeout", 300);
 
+        public static int IndexingActivityDeletionPeriodInMinutes { get; internal set; } = GetInt(SectionName, "IndexingActivityDeletionPeriodInMinutes", 10);
+        public static int IndexingActivityMaxAgeInMinutes { get; internal set; } = GetInt(SectionName, "IndexingActivityMaxAgeInMinutes", 8 * 60);
+
         private static bool? GetNullableBool(string key)
         {
             var textValue = GetString(SectionName, key);

src/Storage/Data/DataProvider.cs

@@ -714,12 +714,14 @@ public abstract Task UpdateIndexingActivityRunningStateAsync(int indexingActivit
         /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
         /// <returns>A Task that represents the asynchronous operation.</returns>
         public abstract Task RefreshIndexingActivityLockTimeAsync(int[] waitingIds, CancellationToken cancellationToken);
+
         /// <summary>
         /// Deletes finished activities. Called by a cleanup background process.
         /// </summary>
+        /// <param name="maxAgeInMinutes">Age of the IndexingActivities that will be deleted periodically.</param>
         /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
         /// <returns>A Task that represents the asynchronous operation.</returns>
-        public abstract Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken);
+        public abstract Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken);
         /// <summary>
         /// Deletes all activities from the database.
         /// </summary>

src/Storage/Data/DataStore.cs

@@ -698,9 +698,9 @@ public Task RefreshIndexingActivityLockTimeAsync(int[] waitingIds, CancellationT
         {
             return DataProvider.RefreshIndexingActivityLockTimeAsync(waitingIds, cancellationToken);
         }
-        public Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken)
+        public Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken)
         {
-            return DataProvider.DeleteFinishedIndexingActivitiesAsync(cancellationToken);
+            return DataProvider.DeleteFinishedIndexingActivitiesAsync(maxAgeInMinutes, cancellationToken);
         }
         public Task DeleteAllIndexingActivitiesAsync(CancellationToken cancellationToken)
         {

src/Storage/Data/IDataStore.cs

@@ -613,9 +613,10 @@ public Task UpdateIndexingActivityRunningStateAsync(int indexingActivityId,
         /// <summary>
         /// Deletes finished activities. Called by a cleanup background process.
         /// </summary>
+        /// <param name="maxAgeInMinutes">Age of the IndexingActivities that will be deleted periodically.</param>
         /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
         /// <returns>A Task that represents the asynchronous operation.</returns>
-        public Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken);
+        public Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken);
 
         /// <summary>
         /// Deletes all activities from the database.

src/Storage/Data/RelationalDataProviderBase.cs

@@ -2044,11 +2044,14 @@ await ctx.ExecuteNonQueryAsync(RefreshIndexingActivityLockTimeScript, cmd =>
         }
         protected abstract string RefreshIndexingActivityLockTimeScript { get; }
 
-        public override async Task DeleteFinishedIndexingActivitiesAsync(CancellationToken cancellationToken)
+        public override async Task DeleteFinishedIndexingActivitiesAsync(int maxAgeInMinutes, CancellationToken cancellationToken)
         {
             using var op = SnTrace.Database.StartOperation("RelationalDataProviderBase: DeleteFinishedIndexingActivities()");
             using var ctx = CreateDataContext(cancellationToken);
-            await ctx.ExecuteNonQueryAsync(DeleteFinishedIndexingActivitiesScript).ConfigureAwait(false);
+            await ctx.ExecuteNonQueryAsync(DeleteFinishedIndexingActivitiesScript, cmd =>
+            {
+                cmd.Parameters.Add(ctx.CreateParameter("@Minutes", DbType.Int32, maxAgeInMinutes));
+            }).ConfigureAwait(false);
             op.Successful = true;
         }
         protected abstract string DeleteFinishedIndexingActivitiesScript { get; }

src/Tests/SenseNet.IntegrationTests/TestCases/CentralizedIndexingTestCases.cs

@@ -457,7 +457,7 @@ await RegisterActivityAsync(IndexingActivityType.AddDocument,    IndexingActivit
                     await RegisterActivityAsync(IndexingActivityType.AddDocument,    IndexingActivityRunningState.Waiting, 5, 5, "/Root/Path5"),
                 };
 
-                await DP.DeleteFinishedIndexingActivitiesAsync(CancellationToken.None);
+                await DP.DeleteFinishedIndexingActivitiesAsync(23, CancellationToken.None);
 
                 var indexingActivityFactory = Providers.Instance.Services.GetRequiredService<IIndexingActivityFactory>();
                 var loaded = await DP.LoadIndexingActivitiesAsync(0, int.MaxValue, 9999, false, indexingActivityFactory, CancellationToken.None);

src/Tests/SenseNet.IntegrationTests/TestCases/DataProviderTestCases.cs

@@ -1639,7 +1639,7 @@ public async Task DP_IA_DeleteFinished()
             await IndexingActivityTest(async (firstId, lastId) =>
             {
                 // ACTION
-                await DP.DeleteFinishedIndexingActivitiesAsync(CancellationToken.None);
+                await DP.DeleteFinishedIndexingActivitiesAsync(23, CancellationToken.None);
 
                 // ASSERT
                 var result = await DP.LoadIndexingActivitiesAsync(firstId, lastId, 100, false, new TestIndexingActivityFactory(), CancellationToken.None);