[Ingest] Per-source proxy toggle instead of one global switch
CI/CD / CI · dotnet build (push) Successful in 56s
CI/CD / Deploy · hamkadr (push) Successful in 1m6s

Each ingestion source now decides independently whether to route through the proxy: added TelegramUseProxy/BaleUseProxy/DivarUseProxy/MedjobsUseProxy/WebsitesUseProxy flags (migration). ScrapeHttpClients.For(s, useProxy) takes the source's own flag; a source is proxied only when its flag is on AND a proxy URL is set. Settings 'sources' tab: removed the global enable checkbox, kept the proxy address field, and added an «از پروکسی استفاده شود» checkbox under each source. Old IngestProxyEnabled column kept for compatibility but no longer gates routing.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
soroush.asadi
2026-06-04 18:46:48 +03:30
parent cde6b68a39
commit b1e474ba33
14 changed files with 1370 additions and 23 deletions
@@ -27,7 +27,7 @@ public class BaleListingSource : IListingSource
try
{
var client = _clients.For(s);
var client = _clients.For(s, s.BaleUseProxy);
var body = await client.GetStringAsync($"{BaseUrl}/bot{s.BaleBotToken}/getUpdates", ct);
using var doc = JsonDocument.Parse(body);
if (!doc.RootElement.TryGetProperty("result", out var result) || result.ValueKind != JsonValueKind.Array)
@@ -29,7 +29,7 @@ public class DivarListingSource : IListingSource
if (!s.DivarEnabled || queries.Count == 0) return Array.Empty<ScrapedItem>();
var city = string.IsNullOrWhiteSpace(s.DivarCity) ? "tehran" : s.DivarCity.Trim();
var client = _clients.For(s);
var client = _clients.For(s, s.DivarUseProxy);
var items = new List<ScrapedItem>();
foreach (var q in queries)
{
@@ -28,7 +28,7 @@ public class MedjobsListingSource : IListingSource
{
if (!s.MedjobsEnabled) return Array.Empty<ScrapedItem>();
var max = Math.Clamp(s.MedjobsMaxAds, 1, 500);
var client = _clients.For(s);
var client = _clients.For(s, s.MedjobsUseProxy);
try
{
@@ -19,10 +19,11 @@ public sealed class ScrapeHttpClients : IDisposable
{
private readonly ConcurrentDictionary<string, HttpClient> _cache = new();
/// <summary>The HttpClient for the given settings — proxied when enabled, direct otherwise.</summary>
public HttpClient For(AppSetting s)
/// <summary>The HttpClient for a source — proxied only when that source opts in AND a proxy
/// URL is configured; otherwise a direct client. Pass the source's own per-source flag.</summary>
public HttpClient For(AppSetting s, bool useProxy)
{
var key = (s.IngestProxyEnabled && !string.IsNullOrWhiteSpace(s.IngestProxyUrl))
var key = (useProxy && !string.IsNullOrWhiteSpace(s.IngestProxyUrl))
? s.IngestProxyUrl.Trim()
: "direct";
@@ -44,8 +44,12 @@ public class SettingsService
s.DemoMode = incoming.DemoMode;
s.WebsitesEnabled = incoming.WebsitesEnabled;
s.WebsiteUrls = incoming.WebsiteUrls?.Trim();
s.IngestProxyEnabled = incoming.IngestProxyEnabled;
s.IngestProxyUrl = incoming.IngestProxyUrl?.Trim();
s.TelegramUseProxy = incoming.TelegramUseProxy;
s.BaleUseProxy = incoming.BaleUseProxy;
s.DivarUseProxy = incoming.DivarUseProxy;
s.MedjobsUseProxy = incoming.MedjobsUseProxy;
s.WebsitesUseProxy = incoming.WebsitesUseProxy;
s.DivarEnabled = incoming.DivarEnabled;
s.DivarCity = string.IsNullOrWhiteSpace(incoming.DivarCity) ? "tehran" : incoming.DivarCity.Trim();
s.DivarQueries = incoming.DivarQueries?.Trim();
@@ -26,7 +26,7 @@ public class TelegramListingSource : IListingSource
var channels = AppSetting.SplitList(s.TelegramChannels);
if (!s.TelegramEnabled || channels.Count == 0) return Array.Empty<ScrapedItem>();
var client = _clients.For(s);
var client = _clients.For(s, s.TelegramUseProxy);
var items = new List<ScrapedItem>();
foreach (var ch in channels.Select(c => c.TrimStart('@')).Where(c => c.Length > 0))
{
@@ -27,7 +27,7 @@ public class WebsiteListingSource : IListingSource
var urls = AppSetting.SplitList(s.WebsiteUrls);
if (!s.WebsitesEnabled || urls.Count == 0) return Array.Empty<ScrapedItem>();
var client = _clients.For(s);
var client = _clients.For(s, s.WebsitesUseProxy);
var items = new List<ScrapedItem>();
foreach (var url in urls.Where(u => u.StartsWith("http")))
{