hamkadr/src/JobsMedical.Web/Services/Scraping/DivarListingSource.cs

using System.Text;
using System.Text.Json;
using JobsMedical.Web.Models;

namespace JobsMedical.Web.Services.Scraping;

/// <summary>
/// Best-effort Divar fetch: queries Divar's web-search JSON for each term and harvests post
/// titles + descriptions. Enabled + city + queries come from admin settings (DB). Divar's
/// private API shifts shape, so we walk JSON tolerantly and fail soft.
/// </summary>
public class DivarListingSource : IListingSource
{
    private const string BaseUrl = "https://api.divar.ir/v8/web-search";
    private readonly ScrapeHttpClients _clients;
    private readonly ILogger<DivarListingSource> _log;

    public DivarListingSource(ScrapeHttpClients clients, ILogger<DivarListingSource> log)
    {
        _clients = clients;
        _log = log;
    }

    public string Name => "دیوار";

    public async Task<IReadOnlyList<ScrapedItem>> FetchAsync(AppSetting s, CancellationToken ct = default)
    {
        var queries = AppSetting.SplitList(s.DivarQueries);
        if (!s.DivarEnabled || queries.Count == 0) return Array.Empty<ScrapedItem>();
        var city = string.IsNullOrWhiteSpace(s.DivarCity) ? "tehran" : s.DivarCity.Trim();

        var client = _clients.For(s, s.DivarUseProxy);
        var items = new List<ScrapedItem>();
        foreach (var q in queries)
        {
            try
            {
                var url = $"{BaseUrl}/{city}/jobs?q={Uri.EscapeDataString(q)}";
                var body = await client.GetStringAsync(url, ct);
                using var doc = JsonDocument.Parse(body);
                foreach (var text in Harvest(doc.RootElement).Take(25))
                    items.Add(new ScrapedItem("دیوار", text, "https://divar.ir"));
            }
            catch (Exception ex) { _log.LogWarning(ex, "Divar fetch failed for query {Query}", q); }
        }
        return items;
    }

    private static readonly string[] DescKeys =
        { "description", "middle_description_text", "subtitle", "bottom_description_text", "normal_text" };

    private static IEnumerable<string> Harvest(JsonElement el)
    {
        if (el.ValueKind == JsonValueKind.Object)
        {
            if (el.TryGetProperty("title", out var t) && t.ValueKind == JsonValueKind.String)
            {
                var sb = new StringBuilder(t.GetString());
                foreach (var k in DescKeys)
                    if (el.TryGetProperty(k, out var d) && d.ValueKind == JsonValueKind.String)
                    { sb.Append(" — ").Append(d.GetString()); break; }
                var text = sb.ToString().Trim();
                if (text.Length >= 15) yield return text;
            }
            foreach (var p in el.EnumerateObject())
                foreach (var x in Harvest(p.Value)) yield return x;
        }
        else if (el.ValueKind == JsonValueKind.Array)
        {
            foreach (var item in el.EnumerateArray())
                foreach (var x in Harvest(item)) yield return x;
        }
    }
}