hamkadr/src/JobsMedical.Web/Services/Scraping/IngestionService.cs

using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using JobsMedical.Web.Data;
using JobsMedical.Web.Models;
using Microsoft.EntityFrameworkCore;

namespace JobsMedical.Web.Services.Scraping;

public record SourceResult(string Source, int Fetched, int Queued, int Published, int Flagged, int Spam, int Duplicates);

public record IngestionSummary(List<SourceResult> Sources)
{
    public int TotalFetched => Sources.Sum(s => s.Fetched);
    public int TotalQueued => Sources.Sum(s => s.Queued);
    public int TotalPublished => Sources.Sum(s => s.Published);
    public int TotalFlagged => Sources.Sum(s => s.Flagged);
    public int TotalSpam => Sources.Sum(s => s.Spam);
    public int TotalDuplicates => Sources.Sum(s => s.Duplicates);
}

/// <summary>
/// The scrape engine. For every enabled source: dedupe by content hash → parse → rule-validate →
/// (optional) AI audit → decide. Decision depends on admin settings:
///   • spam → Discarded
///   • AI on: AI verdict drives approve/reject/review; approve + Automatic + AiAutoApprove → publish
///   • AI off: Automatic + confidence ≥ threshold → publish; else queue/flag
/// "Publish" resolves-or-creates an (unverified) facility and creates the Shift/JobOpening.
/// </summary>
public class IngestionService
{
    private readonly AppDbContext _db;
    private readonly IEnumerable<IListingSource> _sources;
    private readonly IListingParser _parser;
    private readonly ListingValidator _validator;
    private readonly IAiAuditor _ai;
    private readonly SettingsService _settings;
    private readonly ILogger<IngestionService> _log;

    public IngestionService(AppDbContext db, IEnumerable<IListingSource> sources, IListingParser parser,
        ListingValidator validator, IAiAuditor ai, SettingsService settings, ILogger<IngestionService> log)
    {
        _db = db; _sources = sources; _parser = parser; _validator = validator;
        _ai = ai; _settings = settings; _log = log;
    }

    public IReadOnlyList<string> SourceNames => _sources.Select(s => s.Name).ToList();

    public async Task<IngestionSummary> RunAsync(CancellationToken ct = default)
    {
        var settings = await _settings.GetAsync();
        var roles = await _db.Roles.ToListAsync(ct);
        var cities = await _db.Cities.ToListAsync(ct);
        var districts = await _db.Districts.ToListAsync(ct);
        var facilities = await _db.Facilities.ToListAsync(ct);   // fuzzy-matched + grown as we create
        var roleNames = roles.Select(r => r.Name).ToList();
        var cityNames = cities.Select(c => c.Name).ToList();
        var districtNames = districts.Select(d => d.Name).ToList();

        var results = new List<SourceResult>();

        foreach (var source in _sources)
        {
            int fetched = 0, queued = 0, published = 0, flagged = 0, spam = 0, dupes = 0;
            IReadOnlyList<ScrapedItem> items;
            try { items = await source.FetchAsync(settings, ct); }
            catch (Exception ex) { _log.LogError(ex, "Source {Source} failed", source.Name); continue; }
            if (items.Count == 0) continue;   // disabled/unconfigured source

            foreach (var item in items)
            {
                fetched++;
                var hash = Hash(item.RawText);
                if (await _db.RawListings.AnyAsync(r => r.ContentHash == hash, ct)) { dupes++; continue; }

                var parsed = _parser.Parse(item.RawText, roleNames, cityNames, districtNames);
                var val = _validator.Validate(item.RawText, parsed);

                AiAuditResult? ai = null;
                if (settings.AiEnabled && !val.IsSpam)
                    ai = await _ai.AuditAsync(item.RawText, settings, ct);

                var (status, reason, confidence) = Decide(settings, val, ai);

                var raw = new RawListing
                {
                    SourceChannel = item.Source,
                    SourceUrl = item.SourceUrl,
                    RawText = item.RawText.Trim(),
                    ContentHash = hash,
                    Confidence = confidence,
                    ValidationNotes = reason,
                    Status = status,
                };
                _db.RawListings.Add(raw);

                if (status == RawListingStatus.Normalized)
                {
                    try { Publish(parsed, ai, raw, roles, cities, districts, facilities); published++; }
                    catch (Exception ex) { _log.LogWarning(ex, "Auto-publish failed; queueing instead"); raw.Status = RawListingStatus.New; queued++; }
                }
                else if (status == RawListingStatus.New) queued++;
                else if (status == RawListingStatus.Flagged) flagged++;
                else spam++;
            }

            await _db.SaveChangesAsync(ct);
            results.Add(new SourceResult(source.Name, fetched, queued, published, flagged, spam, dupes));
            _log.LogInformation("Ingest {S}: fetched={F} queued={Q} published={P} flagged={Fl} spam={Sp} dupes={D}",
                source.Name, fetched, queued, published, flagged, spam, dupes);
        }

        var summary = new IngestionSummary(results);

        // Persist a run-log row so admins get a crawl history (with a per-source breakdown).
        if (results.Count > 0)
        {
            var detail = string.Join("؛ ", results.Select(r =>
                $"{r.Source}: یافت {r.Fetched}، صف {r.Queued}، منتشر {r.Published}، پرچم {r.Flagged}، اسپم {r.Spam}، تکراری {r.Duplicates}"));
            _db.IngestionRuns.Add(new IngestionRun
            {
                Fetched = summary.TotalFetched,
                Queued = summary.TotalQueued,
                Published = summary.TotalPublished,
                Flagged = summary.TotalFlagged,
                Spam = summary.TotalSpam,
                Duplicates = summary.TotalDuplicates,
                Detail = detail.Length > 2000 ? detail[..2000] : detail,
            });
            await _db.SaveChangesAsync(ct);
        }

        return summary;
    }

    private static (RawListingStatus status, string? reason, int confidence) Decide(
        AppSetting s, ValidationResult val, AiAuditResult? ai)
    {
        var notes = val.Issues.Count > 0 ? string.Join("؛ ", val.Issues) : null;

        if (val.IsSpam)
            return (RawListingStatus.Discarded, Join("اسپم", notes), val.Confidence);

        if (ai is not null)
        {
            var aiNote = Join($"AI: {ai.Decision} ({ai.Confidence}٪)" + (ai.Reason is null ? "" : $" — {ai.Reason}"), notes);
            if (ai.Reject) return (RawListingStatus.Discarded, aiNote, ai.Confidence);
            if (ai.Approve)
                return (s.Mode == IngestionMode.Automatic && s.AiAutoApprove
                    ? RawListingStatus.Normalized : RawListingStatus.New, aiNote, ai.Confidence);
            return (RawListingStatus.Flagged, aiNote, ai.Confidence);  // review
        }

        if (!val.IsValid) return (RawListingStatus.Flagged, notes, val.Confidence);
        if (s.Mode == IngestionMode.Automatic && val.Confidence >= s.AutoPublishMinConfidence)
            return (RawListingStatus.Normalized, notes, val.Confidence);
        return (RawListingStatus.New, notes, val.Confidence);
    }

    private void Publish(ParsedListing parsed, AiAuditResult? ai, RawListing raw,
        List<Role> roles, List<City> cities, List<District> districts, List<Facility> facilities)
    {
        var d = ai?.Data;
        var roleName = d?.Role ?? parsed.RoleName;
        var cityName = d?.City ?? parsed.CityName;
        var districtName = d?.District ?? parsed.DistrictName;

        var role = roles.FirstOrDefault(r => r.Name == roleName) ?? roles.First();
        var city = cities.FirstOrDefault(c => c.Name == cityName)
                   ?? cities.FirstOrDefault(c => c.IsActive) ?? cities.First();
        var district = districts.FirstOrDefault(x => x.Name == districtName && x.CityId == city.Id);

        var kindStr = (d?.Kind ?? parsed.Kind.ToString()).ToLowerInvariant();

        // «آماده به کار» — a worker offering themselves. No facility involved.
        if (parsed.Kind == ListingKind.Talent || kindStr.Contains("talent") || kindStr.Contains("آماده"))
        {
            // Prefer the AI's tags when present, else the heuristic parser.
            var tPay = d?.PayAmount ?? parsed.PayAmount;
            var tShare = d?.SharePercent ?? parsed.SharePercent;
            _db.TalentListings.Add(new TalentListing
            {
                Role = role, City = city, DistrictId = district?.Id,
                PersonName = !string.IsNullOrWhiteSpace(d?.PersonName) ? d!.PersonName!.Trim() : parsed.PersonName,
                YearsExperience = d?.YearsExperience ?? parsed.YearsExperience,
                IsLicensed = d?.IsLicensed ?? parsed.IsLicensed,
                AreaNote = parsed.AreaNote,
                Availability = MapEmployment(d?.EmploymentType, parsed.EmploymentType),
                Gender = parsed.Gender,
                PayType = tShare is not null && tPay is null ? PayType.Percentage
                          : tPay is null ? PayType.Negotiable : PayType.PerShift,
                PayAmount = tPay, SharePercent = tShare,
                Phone = !string.IsNullOrWhiteSpace(d?.Phone) ? d!.Phone!.Trim() : parsed.Phone,
                Description = raw.RawText,
                Status = ShiftStatus.Open, Source = ShiftSource.Aggregated, SourceUrl = raw.SourceUrl,
            });
            raw.Status = RawListingStatus.Normalized;
            return;
        }

        // Never surface the crawl source (e.g. «مدجابز») in a public facility name.
        var facilityName = !string.IsNullOrWhiteSpace(d?.FacilityName) ? d!.FacilityName!.Trim()
            : !string.IsNullOrWhiteSpace(parsed.FacilityName) ? parsed.FacilityName!.Trim()
            : "مرکز درمانی (نامشخص)";
        // Reuse an existing facility (exact or Persian-aware fuzzy match) before creating a new one.
        var facility = FacilityMatcher.FindBest(facilities, facilityName, city.Id);
        if (facility is null)
        {
            facility = new Facility
            {
                Name = facilityName, Type = FacilityType.Clinic, City = city, DistrictId = district?.Id,
                Phone = !string.IsNullOrWhiteSpace(d?.Phone) ? d!.Phone!.Trim() : parsed.Phone, IsVerified = false,
            };
            _db.Facilities.Add(facility);
            facilities.Add(facility);   // so later listings in this run match it too
        }

        if (kindStr.Contains("job") || kindStr.Contains("استخدام"))
        {
            _db.JobOpenings.Add(new JobOpening
            {
                Facility = facility, Role = role,
                Title = !string.IsNullOrWhiteSpace(d?.Title) ? d!.Title!.Trim() : $"استخدام {role.Name}",
                EmploymentType = MapEmployment(d?.EmploymentType, parsed.EmploymentType),
                SalaryMin = parsed.PayAmount,
                Description = raw.RawText, Status = ShiftStatus.Open, Source = ShiftSource.Aggregated,
                SourceUrl = raw.SourceUrl,
            });
        }
        else
        {
            var st = MapShiftType(d?.ShiftType, parsed.ShiftType);
            var (start, end) = DefaultTimes(st);
            _db.Shifts.Add(new Shift
            {
                Facility = facility, Role = role,
                Date = DateOnly.FromDateTime(DateTime.UtcNow).AddDays(1),
                StartTime = start, EndTime = end, ShiftType = st,
                SpecialtyRequired = role.Name, Description = raw.RawText,
                PayType = parsed.SharePercent is not null && parsed.PayAmount is null ? PayType.Percentage
                          : parsed.PayAmount is null ? PayType.Negotiable : PayType.PerShift,
                PayAmount = parsed.PayAmount, SharePercent = parsed.SharePercent,
                Status = ShiftStatus.Open, Source = ShiftSource.Aggregated, SourceUrl = raw.SourceUrl,
            });
        }
        raw.Status = RawListingStatus.Normalized;
    }

    private static ShiftType MapShiftType(string? ai, ShiftType? parsed) => (ai?.ToLowerInvariant()) switch
    {
        "day" => ShiftType.Day, "evening" => ShiftType.Evening, "night" => ShiftType.Night, "oncall" => ShiftType.OnCall,
        _ => parsed ?? ShiftType.Day,
    };

    private static EmploymentType MapEmployment(string? ai, EmploymentType? parsed) => (ai?.ToLowerInvariant()) switch
    {
        "parttime" => EmploymentType.PartTime, "contract" => EmploymentType.Contract,
        "plan" => EmploymentType.Plan, "fulltime" => EmploymentType.FullTime,
        _ => parsed ?? EmploymentType.FullTime,
    };

    private static (TimeOnly, TimeOnly) DefaultTimes(ShiftType t) => t switch
    {
        ShiftType.Day => (new TimeOnly(8, 0), new TimeOnly(14, 0)),
        ShiftType.Evening => (new TimeOnly(14, 0), new TimeOnly(20, 0)),
        ShiftType.Night => (new TimeOnly(20, 0), new TimeOnly(8, 0)),
        _ => (new TimeOnly(8, 0), new TimeOnly(8, 0)),
    };

    private static string? Join(string a, string? b) => string.IsNullOrEmpty(b) ? a : $"{a} | {b}";

    private static string Hash(string text)
    {
        var normalized = Regex.Replace((text ?? "").Trim(), @"\s+", " ");
        return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(normalized))).ToLowerInvariant();
    }
}