60 lines
2.1 KiB
C#
60 lines
2.1 KiB
C#
|
using Microsoft.Extensions.Options;
|
||
|
|
|
||
|
|
namespace JobsMedical.Web.Services.Scraping;
|
||
|
|
|
||
|
|
public class IngestionOptions
|
||
|
|
{
|
||
|
|
public bool Enabled { get; set; } = false; // off by default — opt in via config
|
||
|
|
public int IntervalMinutes { get; set; } = 30;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// Periodically runs the ingestion engine when enabled (Ingestion:Enabled=true). Off by default
|
||
|
|
/// so nothing scrapes uninvited; admins can also trigger a run on demand from the admin UI.
|
||
|
|
/// </summary>
|
||
|
|
public class IngestionWorker : BackgroundService
|
||
|
|
{
|
||
|
|
private readonly IServiceScopeFactory _scopes;
|
||
|
|
private readonly IngestionOptions _opts;
|
||
|
|
private readonly ILogger<IngestionWorker> _log;
|
||
|
|
|
||
|
|
public IngestionWorker(IServiceScopeFactory scopes, IOptions<IngestionOptions> opts,
|
||
|
|
ILogger<IngestionWorker> log)
|
||
|
|
{
|
||
|
|
_scopes = scopes;
|
||
|
|
_opts = opts.Value;
|
||
|
|
_log = log;
|
||
|
|
}
|
||
|
|
|
||
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||
|
|
{
|
||
|
|
if (!_opts.Enabled)
|
||
|
|
{
|
||
|
|
_log.LogInformation("Ingestion worker disabled (Ingestion:Enabled=false).");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
var interval = TimeSpan.FromMinutes(Math.Max(1, _opts.IntervalMinutes));
|
||
|
|
_log.LogInformation("Ingestion worker on; every {Min} min.", _opts.IntervalMinutes);
|
||
|
|
|
||
|
|
while (!stoppingToken.IsCancellationRequested)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
using var scope = _scopes.CreateScope();
|
||
|
|
var svc = scope.ServiceProvider.GetRequiredService<IngestionService>();
|
||
|
|
var summary = await svc.RunAsync(stoppingToken);
|
||
|
|
_log.LogInformation("Scheduled ingestion: queued={Q} flagged={F} spam={S} dupes={D}",
|
||
|
|
summary.TotalQueued, summary.TotalFlagged, summary.TotalSpam, summary.TotalDuplicates);
|
||
|
|
}
|
||
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
||
|
|
{
|
||
|
|
_log.LogError(ex, "Scheduled ingestion run failed");
|
||
|
|
}
|
||
|
|
|
||
|
|
try { await Task.Delay(interval, stoppingToken); }
|
||
|
|
catch (OperationCanceledException) { break; }
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|