hamkadr/src/JobsMedical.Web/Services/Scraping/DivarListingSource.cs

using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Options;

namespace JobsMedical.Web.Services.Scraping;

public class DivarOptions
{
    public bool Enabled { get; set; }
    public string City { get; set; } = "tehran";
    public string Category { get; set; } = "jobs";
    public List<string> Queries { get; set; } = new();    // e.g. "پرستار", "پزشک عمومی", "درمانگاه"
    public string BaseUrl { get; set; } = "https://api.divar.ir/v8/web-search";
    public int PerQuery { get; set; } = 25;
}

/// <summary>
/// Best-effort Divar fetch: queries Divar's web-search JSON for each term and harvests post
/// titles + descriptions. Divar's private API shifts shape over time, so we walk the JSON
/// tolerantly for any object carrying a "title" plus a nearby description field, and fail soft.
/// </summary>
public class DivarListingSource : IListingSource
{
    private readonly DivarOptions _opts;
    private readonly IHttpClientFactory _http;
    private readonly ILogger<DivarListingSource> _log;

    public DivarListingSource(IOptions<DivarOptions> opts, IHttpClientFactory http,
        ILogger<DivarListingSource> log)
    {
        _opts = opts.Value;
        _http = http;
        _log = log;
    }

    public string Name => "دیوار";
    public bool Enabled => _opts.Enabled && _opts.Queries.Count > 0;

    public async Task<IReadOnlyList<ScrapedItem>> FetchAsync(CancellationToken ct = default)
    {
        if (!Enabled) { _log.LogInformation("Divar source disabled/unconfigured."); return Array.Empty<ScrapedItem>(); }

        var client = _http.CreateClient("scrape");
        var items = new List<ScrapedItem>();
        foreach (var q in _opts.Queries.Where(q => q.Trim().Length > 0))
        {
            try
            {
                var url = $"{_opts.BaseUrl.TrimEnd('/')}/{_opts.City}/{_opts.Category}?q={Uri.EscapeDataString(q)}";
                var body = await client.GetStringAsync(url, ct);
                using var doc = JsonDocument.Parse(body);
                foreach (var text in Harvest(doc.RootElement).Take(_opts.PerQuery))
                    items.Add(new ScrapedItem("دیوار", text, "https://divar.ir"));
            }
            catch (Exception ex) { _log.LogWarning(ex, "Divar fetch failed for query {Query}", q); }
        }
        return items;
    }

    private static readonly string[] DescKeys =
        { "description", "middle_description_text", "subtitle", "bottom_description_text", "normal_text" };

    /// <summary>Walk the JSON; for each object with a string "title", emit title + first description.</summary>
    private static IEnumerable<string> Harvest(JsonElement el)
    {
        if (el.ValueKind == JsonValueKind.Object)
        {
            if (el.TryGetProperty("title", out var t) && t.ValueKind == JsonValueKind.String)
            {
                var sb = new StringBuilder(t.GetString());
                foreach (var k in DescKeys)
                    if (el.TryGetProperty(k, out var d) && d.ValueKind == JsonValueKind.String)
                    { sb.Append(" — ").Append(d.GetString()); break; }
                var text = sb.ToString().Trim();
                if (text.Length >= 15) yield return text;
            }
            foreach (var p in el.EnumerateObject())
                foreach (var s in Harvest(p.Value)) yield return s;
        }
        else if (el.ValueKind == JsonValueKind.Array)
        {
            foreach (var item in el.EnumerateArray())
                foreach (var s in Harvest(item)) yield return s;
        }
    }
}