Før nytt oppfriskingsgrensesnitt
This commit is contained in:
parent
e5b76a7477
commit
b8adf6b365
4 changed files with 449 additions and 38 deletions
|
|
@ -175,17 +175,27 @@ async def queue_scrape_job(job_type: str, facility_ids: List[int], requested_by:
|
|||
if not facility_ids:
|
||||
raise HTTPException(status_code=400, detail="Ingen anleggs-IDer ble oppgitt.")
|
||||
|
||||
job, was_created = await enqueue_scrape_job(app.state.pool, job_type, facility_ids, requested_by=requested_by)
|
||||
status = "queued" if was_created else "already_queued"
|
||||
requested_ids = sorted({int(facility_id) for facility_id in facility_ids if str(facility_id).strip()})
|
||||
job, status = await enqueue_scrape_job(app.state.pool, job_type, requested_ids, requested_by=requested_by)
|
||||
was_created = status == "queued"
|
||||
overlapping_ids = sorted({int(facility_id) for facility_id in (job.get("overlapping_facility_ids") or [])})
|
||||
available_ids = [facility_id for facility_id in requested_ids if facility_id not in overlapping_ids]
|
||||
message = (
|
||||
f"{job_type.capitalize()}-skraping for {len(job['facility_ids'])} anlegg ble lagt i kø."
|
||||
if was_created
|
||||
else f"Fant allerede en aktiv {job_type}-jobb for samme anlegg."
|
||||
)
|
||||
if status == "conflict":
|
||||
message = (
|
||||
f"Kunne ikke legge jobben i kø fordi {len(overlapping_ids)} valgt"
|
||||
f" anlegg allerede inngår i en aktiv {job_type}-jobb."
|
||||
)
|
||||
return {
|
||||
"status": status,
|
||||
"message": message,
|
||||
"job": job,
|
||||
"conflicting_facility_ids": overlapping_ids,
|
||||
"idle_facility_ids": available_ids,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import Any, Iterable
|
|||
|
||||
SCRAPE_JOB_TYPES = ("banestatus", "medlemskap", "greenfee", "vtg")
|
||||
SCRAPE_JOB_STATUSES = ("pending", "running", "completed", "failed")
|
||||
DEFAULT_MAX_ATTEMPTS = 3
|
||||
|
||||
|
||||
def normalize_facility_ids(facility_ids: Iterable[int]) -> list[int]:
|
||||
|
|
@ -15,6 +16,16 @@ def normalize_facility_ids(facility_ids: Iterable[int]) -> list[int]:
|
|||
return sorted(facility_id for facility_id in cleaned if facility_id > 0)
|
||||
|
||||
|
||||
def _normalize_int_list(values: Iterable[Any]) -> list[int]:
|
||||
cleaned: set[int] = set()
|
||||
for value in values:
|
||||
try:
|
||||
cleaned.add(int(value))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return sorted(item for item in cleaned if item > 0)
|
||||
|
||||
|
||||
def _parse_json(value: Any, fallback: Any) -> Any:
|
||||
if value is None:
|
||||
return fallback
|
||||
|
|
@ -32,7 +43,7 @@ def format_scrape_job_row(row: Any) -> dict[str, Any] | None:
|
|||
|
||||
data = dict(row)
|
||||
|
||||
for key in ("created_at", "started_at", "finished_at", "updated_at", "last_heartbeat_at"):
|
||||
for key in ("created_at", "started_at", "finished_at", "updated_at", "last_heartbeat_at", "next_retry_at", "last_error_at"):
|
||||
if isinstance(data.get(key), (date, datetime)):
|
||||
data[key] = data[key].isoformat()
|
||||
|
||||
|
|
@ -41,10 +52,52 @@ def format_scrape_job_row(row: Any) -> dict[str, Any] | None:
|
|||
|
||||
result_summary = _parse_json(data.get("result_summary"), {})
|
||||
data["result_summary"] = result_summary if isinstance(result_summary, dict) else {}
|
||||
overlapping_facility_ids = _parse_json(data.get("overlapping_facility_ids"), [])
|
||||
data["overlapping_facility_ids"] = _normalize_int_list(overlapping_facility_ids if isinstance(overlapping_facility_ids, list) else [])
|
||||
data["retryable"] = bool(data.get("retryable", False))
|
||||
data["attempt_count"] = int(data.get("attempt_count") or 0)
|
||||
data["max_attempts"] = int(data.get("max_attempts") or DEFAULT_MAX_ATTEMPTS)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def classify_scrape_error(exc: Exception) -> tuple[str, bool]:
|
||||
message = str(exc).lower()
|
||||
type_name = type(exc).__name__.lower()
|
||||
module_name = type(exc).__module__.lower()
|
||||
|
||||
if "json" in type_name or "jsondecodeerror" in type_name or "expecting value" in message:
|
||||
return "json_parse", False
|
||||
if "gemini_api_key" in message or "api_key" in message or "mangler i .env" in message:
|
||||
return "configuration", False
|
||||
if "permission denied" in message or "not found" in message and "module" in message:
|
||||
return "configuration", False
|
||||
if "timeout" in message or "timed out" in message or "timeouterror" in type_name:
|
||||
return "timeout", True
|
||||
if "playwright" in module_name or "browser" in message or "page.goto" in message:
|
||||
return "browser", True
|
||||
if "connection" in message or "connectionreseterror" in type_name or "dns" in message or "network" in message:
|
||||
return "network", True
|
||||
if "asyncpg" in module_name or "postgres" in message or "database" in message:
|
||||
return "database", True
|
||||
if "valueerror" in type_name:
|
||||
return "validation", False
|
||||
return "unknown", False
|
||||
|
||||
|
||||
def compute_retry_delay_seconds(attempt_count: int, error_code: str) -> int:
|
||||
base_delay = {
|
||||
"timeout": 30,
|
||||
"network": 45,
|
||||
"browser": 60,
|
||||
"database": 30,
|
||||
"unknown": 90,
|
||||
}.get(error_code, 0)
|
||||
if base_delay <= 0:
|
||||
return 0
|
||||
return min(300, base_delay * max(1, attempt_count))
|
||||
|
||||
|
||||
async def ensure_scrape_jobs_table(conn) -> None:
|
||||
await conn.execute(
|
||||
"""
|
||||
|
|
@ -57,16 +110,26 @@ async def ensure_scrape_jobs_table(conn) -> None:
|
|||
requested_by TEXT,
|
||||
worker_name TEXT,
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 3,
|
||||
error_message TEXT,
|
||||
error_code TEXT,
|
||||
retryable BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
result_summary JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
started_at TIMESTAMPTZ,
|
||||
finished_at TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
last_heartbeat_at TIMESTAMPTZ
|
||||
last_heartbeat_at TIMESTAMPTZ,
|
||||
next_retry_at TIMESTAMPTZ,
|
||||
last_error_at TIMESTAMPTZ
|
||||
)
|
||||
"""
|
||||
)
|
||||
await conn.execute("ALTER TABLE scrape_jobs ADD COLUMN IF NOT EXISTS max_attempts INTEGER NOT NULL DEFAULT 3")
|
||||
await conn.execute("ALTER TABLE scrape_jobs ADD COLUMN IF NOT EXISTS error_code TEXT")
|
||||
await conn.execute("ALTER TABLE scrape_jobs ADD COLUMN IF NOT EXISTS retryable BOOLEAN NOT NULL DEFAULT FALSE")
|
||||
await conn.execute("ALTER TABLE scrape_jobs ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ")
|
||||
await conn.execute("ALTER TABLE scrape_jobs ADD COLUMN IF NOT EXISTS last_error_at TIMESTAMPTZ")
|
||||
await conn.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_scrape_jobs_status_created_at
|
||||
|
|
@ -79,9 +142,52 @@ async def ensure_scrape_jobs_table(conn) -> None:
|
|||
ON scrape_jobs (job_type, created_at DESC)
|
||||
"""
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_scrape_jobs_status_next_retry_at
|
||||
ON scrape_jobs (status, next_retry_at)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
async def enqueue_scrape_job(pool, job_type: str, facility_ids: Iterable[int], requested_by: str | None = None) -> tuple[dict[str, Any], bool]:
|
||||
async def _find_active_job_conflict(conn, job_type: str, normalized_ids: list[int]) -> tuple[dict[str, Any] | None, str | None]:
|
||||
if not normalized_ids:
|
||||
return None, None
|
||||
|
||||
requested_set = set(normalized_ids)
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM scrape_jobs
|
||||
WHERE job_type = $1
|
||||
AND status IN ('pending', 'running')
|
||||
ORDER BY created_at DESC
|
||||
""",
|
||||
job_type,
|
||||
)
|
||||
|
||||
partial_conflict: dict[str, Any] | None = None
|
||||
for row in rows:
|
||||
formatted = format_scrape_job_row(row)
|
||||
if not formatted:
|
||||
continue
|
||||
existing_ids = normalize_facility_ids(formatted.get("facility_ids", []))
|
||||
overlap = sorted(requested_set.intersection(existing_ids))
|
||||
if not overlap:
|
||||
continue
|
||||
|
||||
formatted["overlapping_facility_ids"] = overlap
|
||||
if existing_ids == normalized_ids:
|
||||
return formatted, "already_queued"
|
||||
if partial_conflict is None:
|
||||
partial_conflict = formatted
|
||||
|
||||
if partial_conflict:
|
||||
return partial_conflict, "conflict"
|
||||
return None, None
|
||||
|
||||
|
||||
async def enqueue_scrape_job(pool, job_type: str, facility_ids: Iterable[int], requested_by: str | None = None) -> tuple[dict[str, Any], str]:
|
||||
if job_type not in SCRAPE_JOB_TYPES:
|
||||
raise ValueError(f"Ugyldig job_type: {job_type}")
|
||||
|
||||
|
|
@ -90,21 +196,9 @@ async def enqueue_scrape_job(pool, job_type: str, facility_ids: Iterable[int], r
|
|||
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
existing = await conn.fetchrow(
|
||||
"""
|
||||
SELECT *
|
||||
FROM scrape_jobs
|
||||
WHERE job_type = $1
|
||||
AND status IN ('pending', 'running')
|
||||
AND facility_ids = $2::jsonb
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
job_type,
|
||||
facility_ids_json,
|
||||
)
|
||||
if existing:
|
||||
return format_scrape_job_row(existing), False
|
||||
existing_job, queue_status = await _find_active_job_conflict(conn, job_type, normalized_ids)
|
||||
if existing_job and queue_status:
|
||||
return existing_job, queue_status
|
||||
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
|
|
@ -122,7 +216,7 @@ async def enqueue_scrape_job(pool, job_type: str, facility_ids: Iterable[int], r
|
|||
len(normalized_ids),
|
||||
requested_by,
|
||||
)
|
||||
return format_scrape_job_row(row), True
|
||||
return format_scrape_job_row(row), "queued"
|
||||
|
||||
|
||||
async def list_scrape_jobs(pool, job_type: str | None = None, limit: int = 10) -> list[dict[str, Any]]:
|
||||
|
|
@ -154,15 +248,53 @@ async def list_scrape_jobs(pool, job_type: str | None = None, limit: int = 10) -
|
|||
return [format_scrape_job_row(row) for row in rows]
|
||||
|
||||
|
||||
async def claim_next_scrape_job(pool, worker_name: str) -> dict[str, Any] | None:
|
||||
async def claim_next_scrape_job(pool, worker_name: str, stale_heartbeat_seconds: int = 120) -> dict[str, Any] | None:
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE scrape_jobs
|
||||
SET status = 'pending',
|
||||
worker_name = NULL,
|
||||
updated_at = NOW(),
|
||||
retryable = TRUE,
|
||||
error_code = COALESCE(error_code, 'worker_stale'),
|
||||
error_message = COALESCE(error_message, 'Forrige worker mistet heartbeat før jobben ble ferdig.'),
|
||||
last_error_at = NOW(),
|
||||
next_retry_at = NOW() + INTERVAL '15 seconds'
|
||||
WHERE status = 'running'
|
||||
AND attempt_count < max_attempts
|
||||
AND last_heartbeat_at IS NOT NULL
|
||||
AND last_heartbeat_at < NOW() - ($1 * INTERVAL '1 second')
|
||||
""",
|
||||
stale_heartbeat_seconds,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE scrape_jobs
|
||||
SET status = 'failed',
|
||||
worker_name = NULL,
|
||||
updated_at = NOW(),
|
||||
finished_at = NOW(),
|
||||
retryable = FALSE,
|
||||
error_code = COALESCE(error_code, 'worker_stale'),
|
||||
error_message = COALESCE(error_message, 'Forrige worker mistet heartbeat og maks forsok er oppbrukt.'),
|
||||
last_error_at = NOW(),
|
||||
next_retry_at = NULL
|
||||
WHERE status = 'running'
|
||||
AND attempt_count >= max_attempts
|
||||
AND last_heartbeat_at IS NOT NULL
|
||||
AND last_heartbeat_at < NOW() - ($1 * INTERVAL '1 second')
|
||||
""",
|
||||
stale_heartbeat_seconds,
|
||||
)
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
WITH next_job AS (
|
||||
SELECT id
|
||||
FROM scrape_jobs
|
||||
WHERE status = 'pending'
|
||||
AND COALESCE(next_retry_at, created_at) <= NOW()
|
||||
ORDER BY created_at ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
|
|
@ -174,7 +306,8 @@ async def claim_next_scrape_job(pool, worker_name: str) -> dict[str, Any] | None
|
|||
started_at = COALESCE(job.started_at, NOW()),
|
||||
updated_at = NOW(),
|
||||
last_heartbeat_at = NOW(),
|
||||
error_message = NULL
|
||||
error_message = NULL,
|
||||
next_retry_at = NULL
|
||||
FROM next_job
|
||||
WHERE job.id = next_job.id
|
||||
RETURNING job.*
|
||||
|
|
@ -208,6 +341,9 @@ async def complete_scrape_job(pool, job_id: int, result_summary: dict[str, Any]
|
|||
updated_at = NOW(),
|
||||
last_heartbeat_at = NOW(),
|
||||
error_message = NULL,
|
||||
error_code = NULL,
|
||||
retryable = FALSE,
|
||||
next_retry_at = NULL,
|
||||
result_summary = $2::jsonb
|
||||
WHERE id = $1
|
||||
""",
|
||||
|
|
@ -216,21 +352,52 @@ async def complete_scrape_job(pool, job_id: int, result_summary: dict[str, Any]
|
|||
)
|
||||
|
||||
|
||||
async def fail_scrape_job(pool, job_id: int, error_message: str, result_summary: dict[str, Any] | None = None) -> None:
|
||||
async def fail_scrape_job(
|
||||
pool,
|
||||
job_id: int,
|
||||
error_message: str,
|
||||
result_summary: dict[str, Any] | None = None,
|
||||
*,
|
||||
error_code: str = "unknown",
|
||||
retryable: bool = False,
|
||||
retry_delay_seconds: int = 0,
|
||||
) -> dict[str, Any] | None:
|
||||
payload = json.dumps(result_summary or {})
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
UPDATE scrape_jobs
|
||||
SET status = 'failed',
|
||||
finished_at = NOW(),
|
||||
SET status = CASE
|
||||
WHEN $4 AND attempt_count < max_attempts THEN 'pending'
|
||||
ELSE 'failed'
|
||||
END,
|
||||
finished_at = CASE
|
||||
WHEN $4 AND attempt_count < max_attempts THEN NULL
|
||||
ELSE NOW()
|
||||
END,
|
||||
updated_at = NOW(),
|
||||
last_heartbeat_at = NOW(),
|
||||
error_message = $2,
|
||||
result_summary = $3::jsonb
|
||||
error_code = $3,
|
||||
retryable = $4,
|
||||
result_summary = $5::jsonb,
|
||||
last_error_at = NOW(),
|
||||
worker_name = CASE
|
||||
WHEN $4 AND attempt_count < max_attempts THEN NULL
|
||||
ELSE worker_name
|
||||
END,
|
||||
next_retry_at = CASE
|
||||
WHEN $4 AND attempt_count < max_attempts THEN NOW() + ($6 * INTERVAL '1 second')
|
||||
ELSE NULL
|
||||
END
|
||||
WHERE id = $1
|
||||
RETURNING *
|
||||
""",
|
||||
job_id,
|
||||
error_message[:1000],
|
||||
error_code,
|
||||
retryable,
|
||||
payload,
|
||||
max(0, retry_delay_seconds),
|
||||
)
|
||||
return format_scrape_job_row(row)
|
||||
|
|
|
|||
|
|
@ -7,8 +7,10 @@ from dotenv import load_dotenv
|
|||
|
||||
from scrape_job_runner import run_scrape_job
|
||||
from scrape_jobs import (
|
||||
classify_scrape_error,
|
||||
claim_next_scrape_job,
|
||||
complete_scrape_job,
|
||||
compute_retry_delay_seconds,
|
||||
ensure_scrape_jobs_table,
|
||||
fail_scrape_job,
|
||||
heartbeat_scrape_job,
|
||||
|
|
@ -20,6 +22,9 @@ DB_URL = os.getenv("DATABASE_URL", "postgresql://teeoff_admin:teeoff_secret_pass
|
|||
WORKER_NAME = os.getenv("SCRAPE_WORKER_NAME", f"scrape-worker-{os.getpid()}")
|
||||
POLL_INTERVAL_SECONDS = int(os.getenv("SCRAPE_WORKER_POLL_INTERVAL", "5"))
|
||||
HEARTBEAT_INTERVAL_SECONDS = int(os.getenv("SCRAPE_WORKER_HEARTBEAT_INTERVAL", "15"))
|
||||
STALE_HEARTBEAT_SECONDS = int(
|
||||
os.getenv("SCRAPE_WORKER_STALE_HEARTBEAT_SECONDS", str(max(HEARTBEAT_INTERVAL_SECONDS * 4, 60)))
|
||||
)
|
||||
|
||||
|
||||
async def heartbeat_loop(pool, job_id: int, stop_event: asyncio.Event) -> None:
|
||||
|
|
@ -30,11 +35,11 @@ async def heartbeat_loop(pool, job_id: int, stop_event: asyncio.Event) -> None:
|
|||
try:
|
||||
await heartbeat_scrape_job(pool, job_id)
|
||||
except Exception as exc:
|
||||
print(f"⚠️ Klarte ikke å oppdatere heartbeat for jobb {job_id}: {exc}")
|
||||
print(f"Warning: Klarte ikke å oppdatere heartbeat for jobb {job_id}: {exc}")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
print(f"🚀 Starter scrape worker: {WORKER_NAME}")
|
||||
print(f"Starter scrape worker: {WORKER_NAME}")
|
||||
pool = await asyncpg.create_pool(DB_URL, min_size=1, max_size=5, command_timeout=60)
|
||||
|
||||
try:
|
||||
|
|
@ -42,13 +47,16 @@ async def main() -> None:
|
|||
await ensure_scrape_jobs_table(conn)
|
||||
|
||||
while True:
|
||||
job = await claim_next_scrape_job(pool, WORKER_NAME)
|
||||
job = await claim_next_scrape_job(pool, WORKER_NAME, stale_heartbeat_seconds=STALE_HEARTBEAT_SECONDS)
|
||||
if not job:
|
||||
await asyncio.sleep(POLL_INTERVAL_SECONDS)
|
||||
continue
|
||||
|
||||
job_id = job["id"]
|
||||
print(f"🎯 Worker plukket jobb #{job_id} ({job['job_type']}) for {len(job.get('facility_ids', []))} anlegg")
|
||||
print(
|
||||
f"Worker plukket jobb #{job_id} ({job['job_type']}) "
|
||||
f"for {len(job.get('facility_ids', []))} anlegg"
|
||||
)
|
||||
|
||||
stop_event = asyncio.Event()
|
||||
heartbeat_task = asyncio.create_task(heartbeat_loop(pool, job_id, stop_event))
|
||||
|
|
@ -56,16 +64,35 @@ async def main() -> None:
|
|||
try:
|
||||
result_summary = await run_scrape_job(job)
|
||||
await complete_scrape_job(pool, job_id, result_summary)
|
||||
print(f"✅ Jobb #{job_id} fullført")
|
||||
print(f"Jobb #{job_id} fullført")
|
||||
except Exception as exc:
|
||||
trace = traceback.format_exc(limit=5)
|
||||
print(f"🔥 Jobb #{job_id} feilet: {exc}\n{trace}")
|
||||
await fail_scrape_job(
|
||||
error_code, retryable = classify_scrape_error(exc)
|
||||
retry_delay_seconds = (
|
||||
compute_retry_delay_seconds(int(job.get("attempt_count") or 1), error_code)
|
||||
if retryable
|
||||
else 0
|
||||
)
|
||||
updated_job = await fail_scrape_job(
|
||||
pool,
|
||||
job_id,
|
||||
str(exc),
|
||||
{"traceback": trace},
|
||||
{
|
||||
"error_code": error_code,
|
||||
"traceback": trace,
|
||||
},
|
||||
error_code=error_code,
|
||||
retryable=retryable,
|
||||
retry_delay_seconds=retry_delay_seconds,
|
||||
)
|
||||
if updated_job and updated_job.get("status") == "pending":
|
||||
print(
|
||||
f"Jobb #{job_id} feilet midlertidig ({error_code}). "
|
||||
f"Nytt forsøk {updated_job.get('attempt_count', 0)}/"
|
||||
f"{updated_job.get('max_attempts', 0)} om {retry_delay_seconds} sek."
|
||||
)
|
||||
else:
|
||||
print(f"Jobb #{job_id} feilet permanent ({error_code}): {exc}\n{trace}")
|
||||
finally:
|
||||
stop_event.set()
|
||||
await heartbeat_task
|
||||
|
|
|
|||
|
|
@ -20,12 +20,24 @@ type ScrapeJob = {
|
|||
facility_ids: number[];
|
||||
total_facilities: number;
|
||||
error_message?: string | null;
|
||||
error_code?: string | null;
|
||||
retryable?: boolean;
|
||||
attempt_count?: number;
|
||||
max_attempts?: number;
|
||||
next_retry_at?: string | null;
|
||||
last_error_at?: string | null;
|
||||
result_summary?: Record<string, number | string | null>;
|
||||
created_at?: string | null;
|
||||
started_at?: string | null;
|
||||
finished_at?: string | null;
|
||||
};
|
||||
|
||||
type QueueFeedback = {
|
||||
tone: 'success' | 'warning' | 'info' | 'error';
|
||||
title: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
type TwoFactorSetupResponse = {
|
||||
issuer: string;
|
||||
account_name: string;
|
||||
|
|
@ -48,6 +60,18 @@ const JOB_STATUS_LABELS: Record<ScrapeJobStatus, string> = {
|
|||
failed: 'Feilet',
|
||||
};
|
||||
|
||||
const JOB_ERROR_LABELS: Record<string, string> = {
|
||||
json_parse: 'JSON-tolkning',
|
||||
configuration: 'Konfigurasjon',
|
||||
timeout: 'Tidsavbrudd',
|
||||
browser: 'Nettleser / Playwright',
|
||||
network: 'Nettverk',
|
||||
database: 'Database',
|
||||
validation: 'Validering',
|
||||
unknown: 'Ukjent feil',
|
||||
worker_stale: 'Worker mistet heartbeat',
|
||||
};
|
||||
|
||||
const InlineEdit = ({ facilityId, field, initialValue, onSave }: { facilityId: number, field: string, initialValue: string, onSave: (id: number, field: string, val: string) => void }) => {
|
||||
const [isEditing, setIsEditing] = useState(false);
|
||||
const [value, setValue] = useState(initialValue || '');
|
||||
|
|
@ -101,6 +125,7 @@ export default function AdminDashboard() {
|
|||
const [isLoadingTwoFactor, setIsLoadingTwoFactor] = useState(false);
|
||||
const [twoFactorSetup, setTwoFactorSetup] = useState<TwoFactorSetupResponse | null>(null);
|
||||
const [copiedTwoFactorField, setCopiedTwoFactorField] = useState<'secret' | 'uri' | null>(null);
|
||||
const [queueFeedback, setQueueFeedback] = useState<QueueFeedback | null>(null);
|
||||
|
||||
const fetchFacilities = () => {
|
||||
fetch(`${API_URL}/facilities`)
|
||||
|
|
@ -140,6 +165,7 @@ export default function AdminDashboard() {
|
|||
|
||||
useEffect(() => {
|
||||
setSelectedFacilities([]);
|
||||
setQueueFeedback(null);
|
||||
fetchScrapeJobs(activeTab);
|
||||
}, [activeTab]);
|
||||
|
||||
|
|
@ -150,7 +176,9 @@ export default function AdminDashboard() {
|
|||
}, [isScraping]);
|
||||
|
||||
useEffect(() => {
|
||||
const currentState = latestJob ? `${latestJob.id}:${latestJob.status}` : null;
|
||||
const currentState = latestJob
|
||||
? `${latestJob.id}:${latestJob.status}:${latestJob.attempt_count ?? 0}:${latestJob.next_retry_at ?? ''}:${latestJob.finished_at ?? ''}`
|
||||
: null;
|
||||
const previousState = latestJobStateRef.current;
|
||||
latestJobStateRef.current = currentState;
|
||||
|
||||
|
|
@ -216,6 +244,29 @@ export default function AdminDashboard() {
|
|||
.join(' • ');
|
||||
}, [latestJob]);
|
||||
|
||||
const latestJobAttemptLabel = useMemo(() => {
|
||||
if (!latestJob) return '';
|
||||
const attempts = latestJob.attempt_count ?? 0;
|
||||
const maxAttempts = latestJob.max_attempts ?? 0;
|
||||
if (!attempts || !maxAttempts) return '';
|
||||
return `Forsøk ${attempts} av ${maxAttempts}`;
|
||||
}, [latestJob]);
|
||||
|
||||
const latestJobRetryLabel = useMemo(() => {
|
||||
if (!latestJob?.next_retry_at || latestJob.status !== 'pending') return '';
|
||||
return `Nytt forsøk planlagt ${new Date(latestJob.next_retry_at).toLocaleString('nb-NO')}`;
|
||||
}, [latestJob]);
|
||||
|
||||
const latestJobErrorLabel = useMemo(() => {
|
||||
if (!latestJob?.error_code) return '';
|
||||
return JOB_ERROR_LABELS[latestJob.error_code] || latestJob.error_code.replaceAll('_', ' ');
|
||||
}, [latestJob]);
|
||||
|
||||
const recentJobs = useMemo(() => {
|
||||
if (!latestJob) return scrapeJobs.slice(0, 4);
|
||||
return scrapeJobs.filter(job => job.id !== latestJob.id).slice(0, 4);
|
||||
}, [latestJob, scrapeJobs]);
|
||||
|
||||
const handleSelectAll = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
if (e.target.checked) setSelectedFacilities(filteredFacilities.map(f => f.id));
|
||||
else setSelectedFacilities([]);
|
||||
|
|
@ -244,6 +295,7 @@ export default function AdminDashboard() {
|
|||
const handleRunScrapers = async () => {
|
||||
if (selectedFacilities.length === 0) return;
|
||||
setIsQueueing(true);
|
||||
setQueueFeedback(null);
|
||||
const endpoint = activeTab === 'banestatus' ? '/admin/run-scraper' :
|
||||
activeTab === 'medlemskap' ? '/admin/run-membership-scraper' :
|
||||
activeTab === 'greenfee' ? '/admin/run-greenfee-scraper' :
|
||||
|
|
@ -256,9 +308,39 @@ export default function AdminDashboard() {
|
|||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.detail || "Kunne ikke starte skraping");
|
||||
setSelectedFacilities([]);
|
||||
if (data.status === 'queued') {
|
||||
setQueueFeedback({
|
||||
tone: 'success',
|
||||
title: 'Jobb lagt i kø',
|
||||
message: data.message || 'Skrapejobben ble lagt i kø.'
|
||||
});
|
||||
setSelectedFacilities([]);
|
||||
} else if (data.status === 'already_queued') {
|
||||
setQueueFeedback({
|
||||
tone: 'info',
|
||||
title: 'Jobben finnes allerede',
|
||||
message: data.message || 'Det finnes allerede en aktiv jobb for dette utvalget.'
|
||||
});
|
||||
setSelectedFacilities([]);
|
||||
} else if (data.status === 'conflict') {
|
||||
const conflictingCount = Array.isArray(data.conflicting_facility_ids) ? data.conflicting_facility_ids.length : 0;
|
||||
const idleCount = Array.isArray(data.idle_facility_ids) ? data.idle_facility_ids.length : 0;
|
||||
setQueueFeedback({
|
||||
tone: 'warning',
|
||||
title: 'Noen anlegg er allerede i arbeid',
|
||||
message:
|
||||
conflictingCount > 0
|
||||
? `${data.message || 'Utvalget overlapper med en aktiv jobb.'}${idleCount > 0 ? ` ${idleCount} andre anlegg i valget er fortsatt ledige hvis du vil justere utvalget.` : ''}`
|
||||
: (data.message || 'Utvalget overlapper med en aktiv jobb.')
|
||||
});
|
||||
}
|
||||
fetchScrapeJobs(activeTab);
|
||||
} catch (error) {
|
||||
setQueueFeedback({
|
||||
tone: 'error',
|
||||
title: 'Kunne ikke starte jobben',
|
||||
message: `Det oppstod en feil ved start av ${activeTab}-skraperen.`
|
||||
});
|
||||
alert(`Feil ved start av ${activeTab}-skraperen.`);
|
||||
} finally {
|
||||
setIsQueueing(false);
|
||||
|
|
@ -716,6 +798,32 @@ export default function AdminDashboard() {
|
|||
</div>
|
||||
</header>
|
||||
|
||||
{queueFeedback && (
|
||||
<div className={`mb-6 rounded-[1.5rem] border px-5 py-4 md:px-6 ${
|
||||
queueFeedback.tone === 'success'
|
||||
? 'border-[#d8e8c8] bg-[#f1f7ed]'
|
||||
: queueFeedback.tone === 'warning'
|
||||
? 'border-amber-200 bg-amber-50'
|
||||
: queueFeedback.tone === 'error'
|
||||
? 'border-red-100 bg-red-50'
|
||||
: 'border-slate-200 bg-slate-50'
|
||||
}`}>
|
||||
<div className="flex flex-col gap-2 md:flex-row md:items-start md:justify-between">
|
||||
<div className="space-y-1">
|
||||
<p className="text-[10px] font-black uppercase tracking-[0.2em] text-gray-500">Køstatus</p>
|
||||
<p className="text-sm font-black text-[#11280f]">{queueFeedback.title}</p>
|
||||
<p className="text-sm leading-relaxed text-gray-600">{queueFeedback.message}</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => setQueueFeedback(null)}
|
||||
className="self-start rounded-xl border border-white/70 bg-white/80 px-3 py-2 text-[10px] font-black uppercase tracking-widest text-gray-500 transition-colors hover:text-[#11280f]"
|
||||
>
|
||||
Lukk
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{latestJob && latestJob.job_type === activeTab && (
|
||||
<div className={`mb-8 rounded-[1.75rem] border p-5 md:p-6 animate-fade-in ${
|
||||
latestJob.status === 'failed'
|
||||
|
|
@ -742,12 +850,27 @@ export default function AdminDashboard() {
|
|||
<span className="text-xs font-bold text-[#11280f]">
|
||||
{latestJob.total_facilities} anlegg
|
||||
</span>
|
||||
{latestJobAttemptLabel && (
|
||||
<span className="inline-flex rounded-xl bg-white/80 px-3 py-1 text-[10px] font-black uppercase tracking-widest text-gray-600 border border-white/70">
|
||||
{latestJobAttemptLabel}
|
||||
</span>
|
||||
)}
|
||||
{latestJobErrorLabel && (
|
||||
<span className="inline-flex rounded-xl bg-white/80 px-3 py-1 text-[10px] font-black uppercase tracking-widest text-gray-600 border border-white/70">
|
||||
{latestJobErrorLabel}
|
||||
</span>
|
||||
)}
|
||||
{latestJob.created_at && (
|
||||
<span className="text-xs text-gray-500">
|
||||
Opprettet {new Date(latestJob.created_at).toLocaleString('nb-NO')}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{latestJobRetryLabel && (
|
||||
<p className="text-xs font-bold text-amber-700 leading-relaxed">
|
||||
{latestJobRetryLabel}
|
||||
</p>
|
||||
)}
|
||||
{latestJobSummary && (
|
||||
<p className="text-xs text-gray-600 leading-relaxed">{latestJobSummary}</p>
|
||||
)}
|
||||
|
|
@ -765,6 +888,90 @@ export default function AdminDashboard() {
|
|||
</div>
|
||||
)}
|
||||
|
||||
{recentJobs.length > 0 && (
|
||||
<section className="mb-8 rounded-[1.75rem] border border-gray-100 bg-[#fbfcf8] p-5 md:p-6">
|
||||
<div className="mb-4 flex flex-col gap-2 md:flex-row md:items-end md:justify-between">
|
||||
<div>
|
||||
<p className="text-[10px] font-black uppercase tracking-[0.2em] text-[#7ca982]">Jobbhistorikk</p>
|
||||
<h3 className="text-lg font-black tracking-tight text-[#11280f]">Siste jobber i {JOB_LABELS[activeTab].toLowerCase()}</h3>
|
||||
</div>
|
||||
<p className="text-xs text-gray-500">Kortene viser status, forsøk og siste kjente resultat uten sideveis scrolling.</p>
|
||||
</div>
|
||||
<div className="grid gap-4 xl:grid-cols-2">
|
||||
{recentJobs.map(job => {
|
||||
const historyErrorLabel = job.error_code ? (JOB_ERROR_LABELS[job.error_code] || job.error_code.replaceAll('_', ' ')) : '';
|
||||
const historyAttemptLabel = job.attempt_count && job.max_attempts ? `Forsøk ${job.attempt_count} av ${job.max_attempts}` : '';
|
||||
const historySummary = job.result_summary
|
||||
? Object.entries(job.result_summary)
|
||||
.filter(([, value]) => value !== null && value !== undefined && value !== '')
|
||||
.map(([key, value]) => `${key.replaceAll('_', ' ')}: ${value}`)
|
||||
.join(' • ')
|
||||
: '';
|
||||
|
||||
return (
|
||||
<article
|
||||
key={job.id}
|
||||
className={`rounded-[1.5rem] border p-4 shadow-sm ${
|
||||
job.status === 'failed'
|
||||
? 'border-red-100 bg-white'
|
||||
: job.status === 'completed'
|
||||
? 'border-[#d8e8c8] bg-white'
|
||||
: 'border-amber-100 bg-white'
|
||||
}`}
|
||||
>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className={`inline-flex rounded-xl px-3 py-1 text-[10px] font-black uppercase tracking-widest ${
|
||||
job.status === 'failed'
|
||||
? 'bg-red-100 text-red-700'
|
||||
: job.status === 'completed'
|
||||
? 'bg-[#d8e8c8] text-[#11280f]'
|
||||
: 'bg-amber-100 text-amber-700'
|
||||
}`}>
|
||||
{JOB_STATUS_LABELS[job.status]}
|
||||
</span>
|
||||
<span className="text-[10px] font-black uppercase tracking-[0.18em] text-gray-400">Jobb #{job.id}</span>
|
||||
<span className="text-xs font-bold text-[#11280f]">{job.total_facilities} anlegg</span>
|
||||
</div>
|
||||
<div className="mt-3 flex flex-wrap gap-2">
|
||||
{historyAttemptLabel && (
|
||||
<span className="inline-flex rounded-xl bg-[#f4f7ef] px-3 py-1 text-[10px] font-black uppercase tracking-widest text-gray-600">
|
||||
{historyAttemptLabel}
|
||||
</span>
|
||||
)}
|
||||
{historyErrorLabel && (
|
||||
<span className="inline-flex rounded-xl bg-[#f4f7ef] px-3 py-1 text-[10px] font-black uppercase tracking-widest text-gray-600">
|
||||
{historyErrorLabel}
|
||||
</span>
|
||||
)}
|
||||
{job.finished_at && (
|
||||
<span className="text-xs text-gray-500">
|
||||
Ferdig {new Date(job.finished_at).toLocaleString('nb-NO')}
|
||||
</span>
|
||||
)}
|
||||
{!job.finished_at && job.created_at && (
|
||||
<span className="text-xs text-gray-500">
|
||||
Opprettet {new Date(job.created_at).toLocaleString('nb-NO')}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{job.next_retry_at && job.status === 'pending' && (
|
||||
<p className="mt-3 text-xs font-bold text-amber-700">
|
||||
Nytt forsøk planlagt {new Date(job.next_retry_at).toLocaleString('nb-NO')}
|
||||
</p>
|
||||
)}
|
||||
{historySummary && (
|
||||
<p className="mt-3 text-xs leading-relaxed text-gray-600">{historySummary}</p>
|
||||
)}
|
||||
{job.error_message && (
|
||||
<p className="mt-3 text-xs leading-relaxed text-red-600">{job.error_message}</p>
|
||||
)}
|
||||
</article>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
|
||||
{/* VELDIG SYNLIGE FANER */}
|
||||
<div className="mb-8 flex flex-wrap gap-2 border-b-2 border-gray-100 pb-3">
|
||||
<button onClick={() => setActiveTab('banestatus')} className={`px-6 py-3 text-xs font-black uppercase tracking-widest rounded-t-xl transition-all whitespace-nowrap ${activeTab === 'banestatus' ? 'bg-[#8bc34a] text-white shadow-md' : 'bg-gray-50 text-gray-500 hover:bg-gray-200'}`}>Banestatus</button>
|
||||
|
|
|
|||
Loading…
Reference in a new issue