[Ingest] Route scraping through an optional V2Ray/Xray proxy (Telegram in Iran)
Telegram and some sources are filtered in Iran. .NET cannot speak vmess/vless/trojan, so add an Xray sidecar (compose service 'xray', behind the 'proxy' profile) that converts the admin's config into a local SOCKS5 proxy (xray:10808). New ScrapeHttpClients provider builds a proxied or direct HttpClient (WebProxy supports socks5/socks4/http) cached per proxy URL; all five ingestion sources (Telegram/Bale/Divar/Medjobs/Websites) now use it. Admin settings gain IngestProxyEnabled + IngestProxyUrl (migration; UI under sources). Added deploy/xray/config.json template + README with vmess/vless/trojan examples. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -13,12 +13,12 @@ namespace JobsMedical.Web.Services.Scraping;
|
||||
public class MedjobsListingSource : IListingSource
|
||||
{
|
||||
private const string SitemapIndex = "https://medjobs.ir/sitemap_index.xml";
|
||||
private readonly IHttpClientFactory _http;
|
||||
private readonly ScrapeHttpClients _clients;
|
||||
private readonly ILogger<MedjobsListingSource> _log;
|
||||
|
||||
public MedjobsListingSource(IHttpClientFactory http, ILogger<MedjobsListingSource> log)
|
||||
public MedjobsListingSource(ScrapeHttpClients clients, ILogger<MedjobsListingSource> log)
|
||||
{
|
||||
_http = http;
|
||||
_clients = clients;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ public class MedjobsListingSource : IListingSource
|
||||
{
|
||||
if (!s.MedjobsEnabled) return Array.Empty<ScrapedItem>();
|
||||
var max = Math.Clamp(s.MedjobsMaxAds, 1, 500);
|
||||
var client = _http.CreateClient("scrape");
|
||||
var client = _clients.For(s);
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user