Add scrape/ingestion engine + validation, and 24h shift hour-range visualization
Scrape engine (Services/Scraping/): pluggable IListingSource (working sample + Telegram/Divar credential-ready stubs) → IngestionService (content-hash dedupe → parse → validate → review queue) → ListingValidator (completeness score + spam screen) → IngestionWorker (config-gated hosted service). RawListing gains ContentHash/Confidence/ValidationNotes; RawListingStatus.Flagged. Admin /Admin gets run-now, source list, confidence + flagged queue. Hour-range viz: _HourBar 24h timeline bar (colored by type, overnight wrap) on shift cards, recommendation cards, and detail. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
soroush.asadi
@@ -6,29 +6,55 @@
|
||||
|
||||
<div class="page-head">
|
||||
<div class="container">
|
||||
<h1>پنل مدیریت — صف آگهیهای خام</h1>
|
||||
<h1>پنل مدیریت — جمعآوری و صف آگهیها</h1>
|
||||
<p class="muted">
|
||||
آگهیهای جمعآوریشده از کانالها را اینجا بررسی، ساختارمند و منتشر کن.
|
||||
(@JalaliDate.ToPersianDigits(Model.Queue.Count.ToString()) در انتظار بررسی)
|
||||
آگهیهای جمعآوریشده از منابع را بررسی، ساختارمند و منتشر کن.
|
||||
(@JalaliDate.ToPersianDigits(Model.Queue.Count.ToString()) در صف،
|
||||
@JalaliDate.ToPersianDigits(Model.Flagged.Count.ToString()) پرچمخورده)
|
||||
· <a asp-page="/Admin/Facilities">تأیید مراکز درمانی</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="container section">
|
||||
@if (Model.IngestMessage is not null)
|
||||
{
|
||||
<div class="alert alert-success">✓ @Model.IngestMessage</div>
|
||||
}
|
||||
|
||||
<div class="layout-2">
|
||||
<aside class="card card-pad filter-card">
|
||||
<h3>افزودن آگهی خام</h3>
|
||||
<h3>موتور جمعآوری</h3>
|
||||
<p class="muted" style="font-size:13px;">منابع متصل:</p>
|
||||
<ul style="margin:0 0 12px; padding-inline-start:18px; font-size:13.5px;">
|
||||
@foreach (var src in Model.Sources)
|
||||
{
|
||||
<li>@src.Name —
|
||||
@if (src.Enabled) { <span style="color:var(--primary-dark);">فعال</span> }
|
||||
else { <span class="muted">غیرفعال (نیازمند تنظیمات)</span> }
|
||||
</li>
|
||||
}
|
||||
</ul>
|
||||
<form method="post">
|
||||
<button type="submit" asp-page-handler="RunIngestion" class="btn btn-accent btn-block">اجرای جمعآوری اکنون</button>
|
||||
</form>
|
||||
<p class="muted" style="font-size:11px; margin:8px 0 0;">
|
||||
موتور: واکشی ← حذف تکراری ← تجزیه ← اعتبارسنجی ← صف بررسی.
|
||||
</p>
|
||||
|
||||
<hr style="border:none; border-top:1px solid var(--line); margin:16px 0;" />
|
||||
|
||||
<h3>افزودن دستی</h3>
|
||||
<form method="post">
|
||||
<div class="filter-group">
|
||||
<label>منبع (کانال/سایت)</label>
|
||||
<label>منبع</label>
|
||||
<input type="text" name="SourceChannel" placeholder="مثلاً کانال شیفت تهران" />
|
||||
</div>
|
||||
<div class="filter-group">
|
||||
<label>متن آگهی</label>
|
||||
<textarea name="RawText" rows="6" placeholder="متن کپیشده از تلگرام/بله/دیوار را اینجا بچسبان..."></textarea>
|
||||
<textarea name="RawText" rows="5" placeholder="متن کپیشده را بچسبان..."></textarea>
|
||||
</div>
|
||||
<button type="submit" asp-page-handler="Add" class="btn btn-primary btn-block">افزودن به صف</button>
|
||||
<button type="submit" asp-page-handler="Add" class="btn btn-outline btn-block">افزودن به صف</button>
|
||||
</form>
|
||||
<p class="muted" style="font-size:12px; margin-bottom:0;">
|
||||
منتشرشده: @JalaliDate.ToPersianDigits(Model.PublishedShifts.ToString()) شیفت،
|
||||
@@ -37,22 +63,26 @@
|
||||
</aside>
|
||||
|
||||
<div>
|
||||
<h2 style="font-size:20px; margin-top:0;">صف بررسی</h2>
|
||||
@if (Model.Queue.Count == 0)
|
||||
{
|
||||
<div class="card empty-state">صف خالی است. آگهی جدیدی برای بررسی وجود ندارد.</div>
|
||||
<div class="card empty-state">صف خالی است. «اجرای جمعآوری» را بزن یا آگهی اضافه کن.</div>
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (var r in Model.Queue)
|
||||
{
|
||||
<div class="card card-pad" style="margin-bottom:14px;">
|
||||
<div class="row" style="display:flex; justify-content:space-between;">
|
||||
<strong>@r.SourceChannel</strong>
|
||||
<span class="muted" style="font-size:12px;">@JalaliDate.ToLongDate(DateOnly.FromDateTime(r.FetchedAt))</span>
|
||||
</div>
|
||||
<p style="margin:10px 0; white-space:pre-wrap;">@r.RawText</p>
|
||||
<a class="btn btn-accent" asp-page="/Admin/Review" asp-route-id="@r.Id">بررسی و انتشار ←</a>
|
||||
</div>
|
||||
<partial name="_RawListingRow" model="r" />
|
||||
}
|
||||
}
|
||||
|
||||
@if (Model.Flagged.Count > 0)
|
||||
{
|
||||
<h2 style="font-size:20px; margin-top:28px;">پرچمخورده (ناقص/مشکوک)</h2>
|
||||
<p class="muted" style="font-size:13px;">اعتبارسنجی اینها را کامل ندانست؛ در صورت صحت میتوانی منتشرشان کنی.</p>
|
||||
foreach (var r in Model.Flagged)
|
||||
{
|
||||
<partial name="_RawListingRow" model="r" />
|
||||
}
|
||||
}
|
||||
</div>
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using JobsMedical.Web.Data;
|
||||
using JobsMedical.Web.Models;
|
||||
using JobsMedical.Web.Services.Scraping;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.AspNetCore.Mvc.RazorPages;
|
||||
@@ -7,19 +8,29 @@ using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace JobsMedical.Web.Pages.Admin;
|
||||
|
||||
[Authorize(Roles = "Admin")] // secured by the OTP-auth Admin role
|
||||
[Authorize(Roles = "Admin")]
|
||||
public class IndexModel : PageModel
|
||||
{
|
||||
private readonly AppDbContext _db;
|
||||
public IndexModel(AppDbContext db) => _db = db;
|
||||
private readonly IngestionService _ingest;
|
||||
|
||||
public IndexModel(AppDbContext db, IngestionService ingest)
|
||||
{
|
||||
_db = db;
|
||||
_ingest = ingest;
|
||||
}
|
||||
|
||||
public List<RawListing> Queue { get; private set; } = new();
|
||||
public List<RawListing> Flagged { get; private set; } = new();
|
||||
public IReadOnlyList<(string Name, bool Enabled)> Sources { get; private set; } = new List<(string, bool)>();
|
||||
public int PublishedShifts { get; private set; }
|
||||
public int PublishedJobs { get; private set; }
|
||||
|
||||
[BindProperty] public string? SourceChannel { get; set; }
|
||||
[BindProperty] public string? RawText { get; set; }
|
||||
|
||||
[TempData] public string? IngestMessage { get; set; }
|
||||
|
||||
public async Task OnGetAsync() => await LoadAsync();
|
||||
|
||||
public async Task<IActionResult> OnPostAddAsync()
|
||||
@@ -37,11 +48,23 @@ public class IndexModel : PageModel
|
||||
return RedirectToPage();
|
||||
}
|
||||
|
||||
public async Task<IActionResult> OnPostRunIngestionAsync()
|
||||
{
|
||||
var s = await _ingest.RunAsync();
|
||||
IngestMessage = $"جمعآوری انجام شد — {s.TotalQueued} در صف، {s.TotalFlagged} پرچمخورده، " +
|
||||
$"{s.TotalSpam} اسپم، {s.TotalDuplicates} تکراری.";
|
||||
return RedirectToPage();
|
||||
}
|
||||
|
||||
private async Task LoadAsync()
|
||||
{
|
||||
Queue = await _db.RawListings
|
||||
.Where(r => r.Status == RawListingStatus.New)
|
||||
.OrderByDescending(r => r.Confidence).ThenByDescending(r => r.FetchedAt).ToListAsync();
|
||||
Flagged = await _db.RawListings
|
||||
.Where(r => r.Status == RawListingStatus.Flagged)
|
||||
.OrderByDescending(r => r.FetchedAt).ToListAsync();
|
||||
Sources = _ingest.Sources;
|
||||
PublishedShifts = await _db.Shifts.CountAsync(s => s.Source != ShiftSource.Direct);
|
||||
PublishedJobs = await _db.JobOpenings.CountAsync();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user