Move KYC polling to locked worker
This commit is contained in:
@@ -21,8 +21,9 @@ COPY --from=builder /app/src /app/src
|
|||||||
ENV PATH="/app/.venv/bin:$PATH" \
|
ENV PATH="/app/.venv/bin:$PATH" \
|
||||||
PYTHONUNBUFFERED=1 \
|
PYTHONUNBUFFERED=1 \
|
||||||
PYTHONDONTWRITEBYTECODE=1 \
|
PYTHONDONTWRITEBYTECODE=1 \
|
||||||
PYTHONPATH=/app
|
PYTHONPATH=/app \
|
||||||
|
PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_multiproc
|
||||||
|
|
||||||
EXPOSE 8000
|
EXPOSE 8000
|
||||||
|
|
||||||
CMD ["sh", "-c", "granian --interface asgi ${APP_MODULE:-src.main:app} --host ${APP_HOST:-0.0.0.0} --port ${APP_PORT:-8000} --workers ${APP_WORKERS:-1} --loop uvloop"]
|
CMD ["sh", "-c", "mkdir -p "$PROMETHEUS_MULTIPROC_DIR" && rm -f "$PROMETHEUS_MULTIPROC_DIR"/* && granian --interface asgi ${APP_MODULE:-src.main:app} --host ${APP_HOST:-0.0.0.0} --port ${APP_PORT:-8000} --workers ${APP_WORKERS:-1} --loop uvloop"]
|
||||||
|
|||||||
@@ -15,3 +15,15 @@ services:
|
|||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
restart: no
|
restart: no
|
||||||
|
|
||||||
|
kyc-poller:
|
||||||
|
container_name: kyc-poller
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
command: ["python", "-m", "src.worker"]
|
||||||
|
environment:
|
||||||
|
PYTHONUNBUFFERED: "1"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
restart: no
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ dependencies = [
|
|||||||
"hvac==2.4.0",
|
"hvac==2.4.0",
|
||||||
"itsdangerous==2.2.0",
|
"itsdangerous==2.2.0",
|
||||||
"orjson==3.11.7",
|
"orjson==3.11.7",
|
||||||
|
"prometheus-client==0.23.1",
|
||||||
"pydantic-settings==2.12.0",
|
"pydantic-settings==2.12.0",
|
||||||
"python-jose==3.5.0",
|
"python-jose==3.5.0",
|
||||||
"python-ulid==3.1.0",
|
"python-ulid==3.1.0",
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
import asyncio
|
||||||
from datetime import datetime,timezone,timedelta
|
from datetime import datetime,timezone,timedelta
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
from src.application.abstractions import IUnitOfWork
|
from src.application.abstractions import IUnitOfWork
|
||||||
from src.application.contracts import IBeorgService,ILogger
|
from src.application.contracts import IBeorgService,ILogger
|
||||||
from src.application.domain.dto import BeorgKycCreateResponse,BeorgKycResultResponse,KycPersonalData,KycSessionResponse
|
from src.application.domain.dto import BeorgKycCreateResponse,BeorgKycResultResponse,KycPersonalData,KycSessionResponse
|
||||||
@@ -165,6 +168,7 @@ class GetKycSessionCommand:
|
|||||||
|
|
||||||
|
|
||||||
class PollKycSessionsCommand:
|
class PollKycSessionsCommand:
|
||||||
|
_ADVISORY_LOCK_ID = 50291076
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -173,30 +177,59 @@ class PollKycSessionsCommand:
|
|||||||
logger: ILogger,
|
logger: ILogger,
|
||||||
beorg_service: IBeorgService,
|
beorg_service: IBeorgService,
|
||||||
batch_size: int,
|
batch_size: int,
|
||||||
|
concurrency: int = 4,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._unit_of_work = unit_of_work
|
self._unit_of_work = unit_of_work
|
||||||
self._logger = logger
|
self._logger = logger
|
||||||
self._beorg_service = beorg_service
|
self._beorg_service = beorg_service
|
||||||
self._batch_size = batch_size
|
self._batch_size = batch_size
|
||||||
|
self._concurrency = max(1,concurrency)
|
||||||
|
|
||||||
|
|
||||||
async def __call__(self) -> None:
|
async def __call__(self) -> None:
|
||||||
|
session_factory = getattr(self._unit_of_work,'session_factory',None)
|
||||||
|
if session_factory is None:
|
||||||
|
await self._poll_batch()
|
||||||
|
return
|
||||||
|
async with session_factory() as lock_session:
|
||||||
|
locked = await lock_session.scalar(
|
||||||
|
text('select pg_try_advisory_lock(:lock_id)'),
|
||||||
|
{'lock_id': self._ADVISORY_LOCK_ID},
|
||||||
|
)
|
||||||
|
if not locked:
|
||||||
|
self._logger.debug('KYC polling skipped: another worker holds advisory lock')
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
await self._poll_batch()
|
||||||
|
finally:
|
||||||
|
await lock_session.execute(
|
||||||
|
text('select pg_advisory_unlock(:lock_id)'),
|
||||||
|
{'lock_id': self._ADVISORY_LOCK_ID},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _poll_batch(self) -> None:
|
||||||
now = _utc_now()
|
now = _utc_now()
|
||||||
async with self._unit_of_work as unit_of_work:
|
async with self._unit_of_work as unit_of_work:
|
||||||
await unit_of_work.kyc_repository.expire_all_started_sessions(now=now)
|
await unit_of_work.kyc_repository.expire_all_started_sessions(now=now)
|
||||||
sessions = await unit_of_work.kyc_repository.get_started_sessions(now=now,limit=self._batch_size)
|
sessions = await unit_of_work.kyc_repository.get_started_sessions(now=now,limit=self._batch_size)
|
||||||
|
|
||||||
for session in sessions:
|
semaphore = asyncio.Semaphore(self._concurrency)
|
||||||
if not session.user_id or not session.user_token:
|
tasks = [self._poll_session_safe(session,semaphore) for session in sessions if session.user_id and session.user_token]
|
||||||
continue
|
if tasks:
|
||||||
|
await asyncio.gather(*tasks)
|
||||||
|
|
||||||
|
|
||||||
|
async def _poll_session_safe(self,session: KycEntity,semaphore: asyncio.Semaphore) -> None:
|
||||||
|
async with semaphore:
|
||||||
try:
|
try:
|
||||||
await self._poll_session(user_id=session.user_id,user_token=session.user_token)
|
await self._poll_session(user_id=str(session.user_id),user_token=str(session.user_token))
|
||||||
except ApplicationException as exc:
|
except ApplicationException as exc:
|
||||||
if exc.status_code in (400,403,422):
|
if exc.status_code in (400,403,422):
|
||||||
async with self._unit_of_work as unit_of_work:
|
async with self._unit_of_work as unit_of_work:
|
||||||
await unit_of_work.kyc_repository.update_session_result(
|
await unit_of_work.kyc_repository.update_session_result(
|
||||||
user_id=session.user_id,
|
user_id=str(session.user_id),
|
||||||
user_token=session.user_token,
|
user_token=str(session.user_token),
|
||||||
status='failed',
|
status='failed',
|
||||||
done_state=True,
|
done_state=True,
|
||||||
set_id=None,
|
set_id=None,
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ class Settings(BaseSettings):
|
|||||||
DATABASE_ECHO: bool = False
|
DATABASE_ECHO: bool = False
|
||||||
KYC_POLL_SECONDS: int = 10
|
KYC_POLL_SECONDS: int = 10
|
||||||
KYC_POLL_BATCH_SIZE: int = 20
|
KYC_POLL_BATCH_SIZE: int = 20
|
||||||
|
KYC_POLL_CONCURRENCY: int = 4
|
||||||
KYC_SESSION_TTL_SECONDS: int = 900
|
KYC_SESSION_TTL_SECONDS: int = 900
|
||||||
EXCLUDED_PATHS: tuple[str,...] = ('/docs','/redoc','/openapi.json','/ping')
|
EXCLUDED_PATHS: tuple[str,...] = ('/docs','/redoc','/openapi.json','/ping')
|
||||||
BEORG_TIMEOUT: int = 120
|
BEORG_TIMEOUT: int = 120
|
||||||
|
|||||||
@@ -1,52 +1,40 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
import time
|
import time
|
||||||
from collections import defaultdict
|
from collections.abc import Awaitable, Callable
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from typing import Iterable
|
from typing import Any
|
||||||
|
|
||||||
from fastapi import Request, Response
|
from fastapi import Request, Response
|
||||||
|
from prometheus_client import CollectorRegistry, Counter, Histogram, CONTENT_TYPE_LATEST, generate_latest, multiprocess
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
|
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
|
||||||
from starlette.responses import Response as StarletteResponse
|
from starlette.responses import Response as StarletteResponse
|
||||||
|
|
||||||
from src.infrastructure.database.context import async_session_maker
|
from src.infrastructure.database.context import async_session_maker
|
||||||
|
from src.infrastructure.database.models.kyc import KycModel
|
||||||
|
|
||||||
SERVICE_NAME = "kyc"
|
SERVICE_NAME = "kyc"
|
||||||
WINDOWS: tuple[tuple[str, timedelta | None], ...] = (
|
BUSINESS_CACHE_TTL_SECONDS = 30
|
||||||
("all", None),
|
|
||||||
("30d", timedelta(days=30)),
|
|
||||||
("7d", timedelta(days=7)),
|
|
||||||
("24h", timedelta(days=1)),
|
|
||||||
("1h", timedelta(hours=1)),
|
|
||||||
)
|
|
||||||
LATENCY_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
|
LATENCY_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
|
||||||
|
WINDOWS = (("all", None), ("30d", timedelta(days=30)), ("7d", timedelta(days=7)), ("24h", timedelta(days=1)), ("1h", timedelta(hours=1)))
|
||||||
|
|
||||||
_http_requests: dict[tuple[str, str, str], int] = defaultdict(int)
|
HTTP_REQUESTS = Counter("kyc_http_requests_total", "Total HTTP requests handled by kyc service.", ("service", "method", "route", "status_code"))
|
||||||
_http_latency_sum: dict[tuple[str, str], float] = defaultdict(float)
|
HTTP_LATENCY = Histogram("kyc_http_request_duration_seconds", "HTTP request duration in seconds for kyc service.", ("service", "method", "route"), buckets=LATENCY_BUCKETS)
|
||||||
_http_latency_count: dict[tuple[str, str], int] = defaultdict(int)
|
|
||||||
_http_latency_buckets: dict[tuple[str, str, float], int] = defaultdict(int)
|
|
||||||
|
|
||||||
|
|
||||||
def _labels(labels: dict[str, str]) -> str:
|
|
||||||
return ",".join(f'{key}="{value}"' for key, value in labels.items())
|
|
||||||
|
|
||||||
|
|
||||||
def _sample(name: str, value: int | float, labels: dict[str, str] | None = None) -> str:
|
|
||||||
return f"{name}{{{_labels(labels)}}} {value}" if labels else f"{name} {value}"
|
|
||||||
|
|
||||||
|
|
||||||
def _route_template(request: Request) -> str:
|
def _route_template(request: Request) -> str:
|
||||||
route = request.scope.get("route")
|
route = request.scope.get("route")
|
||||||
return str(getattr(route, "path", None) or request.url.path)
|
path = getattr(route, "path", None)
|
||||||
|
return str(path) if path else "__unmatched__"
|
||||||
|
|
||||||
|
|
||||||
class PrometheusMetricsMiddleware(BaseHTTPMiddleware):
|
class PrometheusMetricsMiddleware(BaseHTTPMiddleware):
|
||||||
async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> StarletteResponse:
|
async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> StarletteResponse:
|
||||||
if request.url.path == "/metrics":
|
if request.url.path == "/metrics":
|
||||||
return await call_next(request)
|
return await call_next(request)
|
||||||
|
|
||||||
started = time.perf_counter()
|
started = time.perf_counter()
|
||||||
status_code = "500"
|
status_code = "500"
|
||||||
try:
|
try:
|
||||||
@@ -56,91 +44,52 @@ class PrometheusMetricsMiddleware(BaseHTTPMiddleware):
|
|||||||
finally:
|
finally:
|
||||||
elapsed = time.perf_counter() - started
|
elapsed = time.perf_counter() - started
|
||||||
route = _route_template(request)
|
route = _route_template(request)
|
||||||
method = request.method
|
HTTP_REQUESTS.labels(SERVICE_NAME, request.method, route, status_code).inc()
|
||||||
_http_requests[(method, route, status_code)] += 1
|
HTTP_LATENCY.labels(SERVICE_NAME, request.method, route).observe(elapsed)
|
||||||
_http_latency_sum[(method, route)] += elapsed
|
|
||||||
_http_latency_count[(method, route)] += 1
|
|
||||||
for bucket in LATENCY_BUCKETS:
|
|
||||||
if elapsed <= bucket:
|
|
||||||
_http_latency_buckets[(method, route, bucket)] += 1
|
|
||||||
|
|
||||||
|
|
||||||
async def _count(statement) -> int:
|
class CachedBusinessMetrics:
|
||||||
async with async_session_maker() as session:
|
def __init__(self, ttl_seconds: int, collector: Callable[[], Awaitable[list[str]]]) -> None:
|
||||||
result = await session.execute(statement)
|
self._ttl_seconds = ttl_seconds
|
||||||
return int(result.scalar_one() or 0)
|
self._collector = collector
|
||||||
|
self._expires_at = 0.0
|
||||||
|
self._lines: list[str] = []
|
||||||
|
self._lock = asyncio.Lock()
|
||||||
|
|
||||||
|
async def get(self) -> list[str]:
|
||||||
|
now = time.monotonic()
|
||||||
|
if now < self._expires_at:
|
||||||
|
return self._lines
|
||||||
|
async with self._lock:
|
||||||
|
now = time.monotonic()
|
||||||
|
if now < self._expires_at:
|
||||||
|
return self._lines
|
||||||
|
self._lines = await self._collector()
|
||||||
|
self._expires_at = now + self._ttl_seconds
|
||||||
|
return self._lines
|
||||||
|
|
||||||
|
|
||||||
async def _rows(statement):
|
def _labels(labels: dict[str, str]) -> str:
|
||||||
async with async_session_maker() as session:
|
escaped = {key: value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") for key, value in labels.items()}
|
||||||
result = await session.execute(statement)
|
return ",".join(f'{key}="{value}"' for key, value in escaped.items())
|
||||||
return result.all()
|
|
||||||
|
|
||||||
|
|
||||||
def _http_metrics(prefix: str) -> Iterable[str]:
|
def _sample(name: str, value: int | float, labels: dict[str, str] | None = None) -> str:
|
||||||
yield f"# TYPE {prefix}_http_requests_total counter"
|
return f"{name}{{{_labels(labels)}}} {value}" if labels else f"{name} {value}"
|
||||||
for (method, route, status_code), value in sorted(_http_requests.items()):
|
|
||||||
yield _sample(
|
|
||||||
f"{prefix}_http_requests_total",
|
|
||||||
value,
|
|
||||||
{"service": SERVICE_NAME, "method": method, "route": route, "status_code": status_code},
|
|
||||||
)
|
|
||||||
|
|
||||||
yield f"# TYPE {prefix}_http_request_duration_seconds histogram"
|
|
||||||
for method, route in sorted(_http_latency_count):
|
|
||||||
for bucket in LATENCY_BUCKETS:
|
|
||||||
yield _sample(
|
|
||||||
f"{prefix}_http_request_duration_seconds_bucket",
|
|
||||||
_http_latency_buckets.get((method, route, bucket), 0),
|
|
||||||
{"service": SERVICE_NAME, "method": method, "route": route, "le": str(bucket)},
|
|
||||||
)
|
|
||||||
yield _sample(
|
|
||||||
f"{prefix}_http_request_duration_seconds_bucket",
|
|
||||||
_http_latency_count[(method, route)],
|
|
||||||
{"service": SERVICE_NAME, "method": method, "route": route, "le": "+Inf"},
|
|
||||||
)
|
|
||||||
yield _sample(
|
|
||||||
f"{prefix}_http_request_duration_seconds_sum",
|
|
||||||
_http_latency_sum[(method, route)],
|
|
||||||
{"service": SERVICE_NAME, "method": method, "route": route},
|
|
||||||
)
|
|
||||||
yield _sample(
|
|
||||||
f"{prefix}_http_request_duration_seconds_count",
|
|
||||||
_http_latency_count[(method, route)],
|
|
||||||
{"service": SERVICE_NAME, "method": method, "route": route},
|
|
||||||
)
|
|
||||||
|
|
||||||
from src.infrastructure.database.models.kyc import KycModel
|
|
||||||
|
|
||||||
|
|
||||||
async def _business_metrics() -> list[str]:
|
async def _scalar(session: Any, statement: Any) -> int:
|
||||||
lines: list[str] = []
|
result = await session.execute(statement)
|
||||||
total = await _count(select(func.count()).select_from(KycModel))
|
return int(result.scalar_one() or 0)
|
||||||
lines.append(_sample("kyc_requests_total", total, {"service": SERVICE_NAME}))
|
|
||||||
for status, count in await _rows(select(KycModel.status, func.count()).group_by(KycModel.status)):
|
|
||||||
status_value = str(status)
|
|
||||||
lines.append(_sample("kyc_requests_by_status_total", int(count), {"service": SERVICE_NAME, "status": status_value}))
|
|
||||||
approved = await _count(select(func.count()).select_from(KycModel).where(KycModel.done_state.is_(True)))
|
|
||||||
failed = await _count(select(func.count()).select_from(KycModel).where(KycModel.done_state.is_(False)))
|
|
||||||
lines.append(_sample("kyc_approvals_total", approved, {"service": SERVICE_NAME}))
|
|
||||||
lines.append(_sample("kyc_fails_total", failed, {"service": SERVICE_NAME}))
|
|
||||||
for window_name, delta in WINDOWS:
|
|
||||||
conditions = []
|
|
||||||
completed_conditions = []
|
|
||||||
if delta is not None:
|
|
||||||
conditions.append(KycModel.created_at >= func.now() - delta)
|
|
||||||
completed_conditions.append(KycModel.completed_at >= func.now() - delta)
|
|
||||||
created = await _count(select(func.count()).select_from(KycModel).where(*conditions))
|
|
||||||
approvals = await _count(select(func.count()).select_from(KycModel).where(KycModel.done_state.is_(True), *completed_conditions))
|
|
||||||
fails = await _count(select(func.count()).select_from(KycModel).where(KycModel.done_state.is_(False), *completed_conditions))
|
|
||||||
lines.append(_sample("kyc_requests_created_total", created, {"service": SERVICE_NAME, "window": window_name}))
|
|
||||||
lines.append(_sample("kyc_approvals_window_total", approvals, {"service": SERVICE_NAME, "window": window_name}))
|
|
||||||
lines.append(_sample("kyc_fails_window_total", fails, {"service": SERVICE_NAME, "window": window_name}))
|
|
||||||
return lines
|
|
||||||
|
|
||||||
|
|
||||||
async def metrics_response() -> Response:
|
async def _rows(session: Any, statement: Any) -> list[Any]:
|
||||||
lines = [
|
result = await session.execute(statement)
|
||||||
|
return list(result.all())
|
||||||
|
|
||||||
|
|
||||||
|
async def _collect_business_metrics() -> list[str]:
|
||||||
|
lines: list[str] = [
|
||||||
"# TYPE kyc_requests_total gauge",
|
"# TYPE kyc_requests_total gauge",
|
||||||
"# TYPE kyc_requests_by_status_total gauge",
|
"# TYPE kyc_requests_by_status_total gauge",
|
||||||
"# TYPE kyc_requests_created_total gauge",
|
"# TYPE kyc_requests_created_total gauge",
|
||||||
@@ -149,10 +98,48 @@ async def metrics_response() -> Response:
|
|||||||
"# TYPE kyc_approvals_window_total gauge",
|
"# TYPE kyc_approvals_window_total gauge",
|
||||||
"# TYPE kyc_fails_window_total gauge",
|
"# TYPE kyc_fails_window_total gauge",
|
||||||
]
|
]
|
||||||
|
async with async_session_maker() as session:
|
||||||
|
total = await _scalar(session, select(func.count()).select_from(KycModel))
|
||||||
|
lines.append(_sample("kyc_requests_total", total, {"service": SERVICE_NAME}))
|
||||||
|
for status, count in await _rows(session, select(KycModel.status, func.count()).group_by(KycModel.status)):
|
||||||
|
lines.append(_sample("kyc_requests_by_status_total", int(count), {"service": SERVICE_NAME, "status": str(status)}))
|
||||||
|
approved = await _scalar(session, select(func.count()).select_from(KycModel).where(KycModel.done_state.is_(True)))
|
||||||
|
failed = await _scalar(session, select(func.count()).select_from(KycModel).where(KycModel.done_state.is_(False)))
|
||||||
|
lines.append(_sample("kyc_approvals_total", approved, {"service": SERVICE_NAME}))
|
||||||
|
lines.append(_sample("kyc_fails_total", failed, {"service": SERVICE_NAME}))
|
||||||
|
for window_name, delta in WINDOWS:
|
||||||
|
conditions = []
|
||||||
|
completed_conditions = []
|
||||||
|
if delta is not None:
|
||||||
|
conditions.append(KycModel.created_at >= func.now() - delta)
|
||||||
|
completed_conditions.append(KycModel.completed_at >= func.now() - delta)
|
||||||
|
created = await _scalar(session, select(func.count()).select_from(KycModel).where(*conditions))
|
||||||
|
approvals = await _scalar(session, select(func.count()).select_from(KycModel).where(KycModel.done_state.is_(True), *completed_conditions))
|
||||||
|
fails = await _scalar(session, select(func.count()).select_from(KycModel).where(KycModel.done_state.is_(False), *completed_conditions))
|
||||||
|
labels = {"service": SERVICE_NAME, "window": window_name}
|
||||||
|
lines.append(_sample("kyc_requests_created_total", created, labels))
|
||||||
|
lines.append(_sample("kyc_approvals_window_total", approvals, labels))
|
||||||
|
lines.append(_sample("kyc_fails_window_total", fails, labels))
|
||||||
|
lines.append(_sample("kyc_metrics_scrape_success", 1, {"service": SERVICE_NAME}))
|
||||||
|
return lines
|
||||||
|
|
||||||
|
|
||||||
|
_business_metrics_cache = CachedBusinessMetrics(BUSINESS_CACHE_TTL_SECONDS, _collect_business_metrics)
|
||||||
|
|
||||||
|
|
||||||
|
def _client_metrics() -> bytes:
|
||||||
|
if os.environ.get("PROMETHEUS_MULTIPROC_DIR"):
|
||||||
|
registry = CollectorRegistry()
|
||||||
|
multiprocess.MultiProcessCollector(registry)
|
||||||
|
return generate_latest(registry)
|
||||||
|
return generate_latest()
|
||||||
|
|
||||||
|
|
||||||
|
async def metrics_response() -> Response:
|
||||||
|
lines: list[str] = []
|
||||||
try:
|
try:
|
||||||
lines.extend(await _business_metrics())
|
lines.extend(await _business_metrics_cache.get())
|
||||||
lines.append(_sample("kyc_metrics_scrape_success", 1, {"service": SERVICE_NAME}))
|
|
||||||
except Exception:
|
except Exception:
|
||||||
lines.append(_sample("kyc_metrics_scrape_success", 0, {"service": SERVICE_NAME}))
|
lines.append(_sample("kyc_metrics_scrape_success", 0, {"service": SERVICE_NAME}))
|
||||||
lines.extend(_http_metrics("kyc"))
|
body = _client_metrics().decode("utf-8") + "\n".join(lines) + "\n"
|
||||||
return Response("\n".join(lines) + "\n", media_type="text/plain; version=0.0.4; charset=utf-8")
|
return Response(body, media_type=CONTENT_TYPE_LATEST)
|
||||||
|
|||||||
27
src/main.py
27
src/main.py
@@ -2,20 +2,15 @@ from __future__ import annotations
|
|||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
import secrets
|
import secrets
|
||||||
from typing import AsyncGenerator
|
from typing import AsyncGenerator
|
||||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
||||||
from fastapi import Depends,FastAPI
|
from fastapi import Depends,FastAPI
|
||||||
from fastapi.exceptions import RequestValidationError
|
from fastapi.exceptions import RequestValidationError
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
|
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
|
||||||
from fastapi.responses import HTMLResponse
|
from fastapi.responses import HTMLResponse
|
||||||
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
||||||
from src.application.commands import PollKycSessionsCommand
|
|
||||||
from src.application.domain.enums import LogFormat,LogLevel
|
from src.application.domain.enums import LogFormat,LogLevel
|
||||||
from src.application.domain.exceptions import ApplicationException,UnauthorizedException
|
from src.application.domain.exceptions import ApplicationException,UnauthorizedException
|
||||||
from src.infrastructure.beorg import BeorgService
|
|
||||||
from src.infrastructure.config.settings import get_settings
|
from src.infrastructure.config.settings import get_settings
|
||||||
from src.infrastructure.database.context import async_session_maker
|
|
||||||
from src.infrastructure.database.unit_of_work import UnitOfWork
|
|
||||||
from src.infrastructure.vault import JwtKeyStore, start_jwt_keys_scheduler
|
from src.infrastructure.vault import JwtKeyStore, start_jwt_keys_scheduler
|
||||||
from src.infrastructure.utils import generate_instance_id
|
from src.infrastructure.utils import generate_instance_id
|
||||||
from src.infrastructure.logger import logger
|
from src.infrastructure.logger import logger
|
||||||
@@ -61,34 +56,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
|||||||
await jwt_store.refresh()
|
await jwt_store.refresh()
|
||||||
|
|
||||||
jwt_scheduler = start_jwt_keys_scheduler(jwt_store, refresh_seconds=settings.JWT_KEYS_REFRESH_SECONDS)
|
jwt_scheduler = start_jwt_keys_scheduler(jwt_store, refresh_seconds=settings.JWT_KEYS_REFRESH_SECONDS)
|
||||||
kyc_poll_command = PollKycSessionsCommand(
|
|
||||||
unit_of_work=UnitOfWork(session_factory=async_session_maker,logger=logger),
|
|
||||||
logger=logger,
|
|
||||||
beorg_service=BeorgService(
|
|
||||||
project_id=settings.BEORG_PROJECT_ID,
|
|
||||||
machine_uid=settings.BEORG_MACHINE_UID,
|
|
||||||
token=settings.BEORG_TOKEN,
|
|
||||||
process_info=settings.BEORG_PROCESS_INFO,
|
|
||||||
timeout=settings.BEORG_TIMEOUT,
|
|
||||||
),
|
|
||||||
batch_size=settings.KYC_POLL_BATCH_SIZE,
|
|
||||||
)
|
|
||||||
kyc_scheduler = AsyncIOScheduler()
|
|
||||||
kyc_scheduler.add_job(
|
|
||||||
kyc_poll_command.__call__,
|
|
||||||
'interval',
|
|
||||||
seconds=settings.KYC_POLL_SECONDS,
|
|
||||||
max_instances=1,
|
|
||||||
)
|
|
||||||
kyc_scheduler.start()
|
|
||||||
|
|
||||||
app.state.jwt_key_store = jwt_store
|
app.state.jwt_key_store = jwt_store
|
||||||
app.state.jwt_keys_scheduler = jwt_scheduler
|
app.state.jwt_keys_scheduler = jwt_scheduler
|
||||||
app.state.kyc_scheduler = kyc_scheduler
|
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
app.state.kyc_scheduler.shutdown(wait=False)
|
|
||||||
app.state.jwt_keys_scheduler.shutdown(wait=False)
|
app.state.jwt_keys_scheduler.shutdown(wait=False)
|
||||||
logger.info(f'KYC service instance ended with id {instance_id}')
|
logger.info(f'KYC service instance ended with id {instance_id}')
|
||||||
|
|
||||||
|
|||||||
47
src/worker.py
Normal file
47
src/worker.py
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from src.application.commands import PollKycSessionsCommand
|
||||||
|
from src.application.domain.enums import LogFormat, LogLevel
|
||||||
|
from src.infrastructure.beorg import BeorgService
|
||||||
|
from src.infrastructure.config import settings
|
||||||
|
from src.infrastructure.database.context import async_session_maker
|
||||||
|
from src.infrastructure.database.unit_of_work import UnitOfWork
|
||||||
|
from src.infrastructure.logger import logger
|
||||||
|
from src.infrastructure.utils import generate_instance_id
|
||||||
|
|
||||||
|
|
||||||
|
def _build_poll_command() -> PollKycSessionsCommand:
|
||||||
|
return PollKycSessionsCommand(
|
||||||
|
unit_of_work=UnitOfWork(session_factory=async_session_maker,logger=logger),
|
||||||
|
logger=logger,
|
||||||
|
beorg_service=BeorgService(
|
||||||
|
project_id=settings.BEORG_PROJECT_ID,
|
||||||
|
machine_uid=settings.BEORG_MACHINE_UID,
|
||||||
|
token=settings.BEORG_TOKEN,
|
||||||
|
process_info=settings.BEORG_PROCESS_INFO,
|
||||||
|
timeout=settings.BEORG_TIMEOUT,
|
||||||
|
),
|
||||||
|
batch_size=settings.KYC_POLL_BATCH_SIZE,
|
||||||
|
concurrency=settings.KYC_POLL_CONCURRENCY,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def main() -> None:
|
||||||
|
instance_id = generate_instance_id()
|
||||||
|
logger.set_format(LogFormat(settings.LOG_FORMAT.lower()))
|
||||||
|
logger.set_min_level(LogLevel[settings.LOG_LEVEL.upper()])
|
||||||
|
logger.set_instance_id(instance_id)
|
||||||
|
logger.info(f'KYC poller started with id {instance_id}')
|
||||||
|
command = _build_poll_command()
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
await command()
|
||||||
|
await asyncio.sleep(settings.KYC_POLL_SECONDS)
|
||||||
|
finally:
|
||||||
|
logger.info(f'KYC poller stopped with id {instance_id}')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
asyncio.run(main())
|
||||||
13
uv.lock
generated
13
uv.lock
generated
@@ -335,9 +335,7 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/ef/32/f2ce6d4cac3e55bc6173f92dbe627e782e1850f89d986c3606feb63aafa7/greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f", size = 286228, upload-time = "2026-04-27T12:20:34.421Z" },
|
{ url = "https://files.pythonhosted.org/packages/ef/32/f2ce6d4cac3e55bc6173f92dbe627e782e1850f89d986c3606feb63aafa7/greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f", size = 286228, upload-time = "2026-04-27T12:20:34.421Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/b7/aa/caed9e5adf742315fc7be2a84196373aab4816e540e38ba0d76cb7584d68/greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628", size = 601775, upload-time = "2026-04-27T12:52:41.045Z" },
|
{ url = "https://files.pythonhosted.org/packages/b7/aa/caed9e5adf742315fc7be2a84196373aab4816e540e38ba0d76cb7584d68/greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628", size = 601775, upload-time = "2026-04-27T12:52:41.045Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/c7/af/90ae08497400a941595d12774447f752d3dfe0fbb012e35b76bc5c0ff37e/greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b", size = 614436, upload-time = "2026-04-27T12:59:41.595Z" },
|
{ url = "https://files.pythonhosted.org/packages/c7/af/90ae08497400a941595d12774447f752d3dfe0fbb012e35b76bc5c0ff37e/greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b", size = 614436, upload-time = "2026-04-27T12:59:41.595Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/3f/e9/4eeadf8cb3403ac274245ba75f07844abc7fa5f6787583fc9156ba741e0f/greenlet-3.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:41353ec2ecedf7aa8f682753a41919f8718031a6edac46b8d3dc7ed9e1ceb136", size = 620610, upload-time = "2026-04-27T13:02:39.194Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/2b/e0/2e13df68f367e2f9960616927d60857dd7e56aaadd59a47c644216b2f920/greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c", size = 611388, upload-time = "2026-04-27T12:25:28.008Z" },
|
{ url = "https://files.pythonhosted.org/packages/2b/e0/2e13df68f367e2f9960616927d60857dd7e56aaadd59a47c644216b2f920/greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c", size = 611388, upload-time = "2026-04-27T12:25:28.008Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/ee/ef/f913b3c0eb7d26d86a2401c5e1546c9d46b657efee724b06f6f4ac5d8824/greenlet-3.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:58c1c374fe2b3d852f9b6b11a7dff4c85404e51b9a596fd9e89cf904eb09866d", size = 422775, upload-time = "2026-04-27T13:05:14.261Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/82/f7/393c64055132ac0d488ef6be549253b7e6274194863967ddc0bc8f5b87b8/greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588", size = 1570768, upload-time = "2026-04-27T12:53:28.099Z" },
|
{ url = "https://files.pythonhosted.org/packages/82/f7/393c64055132ac0d488ef6be549253b7e6274194863967ddc0bc8f5b87b8/greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588", size = 1570768, upload-time = "2026-04-27T12:53:28.099Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/b8/4b/eaf7735253522cf56d1b74d672a58f54fc114702ceaf05def59aae72f6e1/greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e", size = 1635983, upload-time = "2026-04-27T12:25:26.903Z" },
|
{ url = "https://files.pythonhosted.org/packages/b8/4b/eaf7735253522cf56d1b74d672a58f54fc114702ceaf05def59aae72f6e1/greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e", size = 1635983, upload-time = "2026-04-27T12:25:26.903Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/4c/fe/4fb3a0805bd5165da5ebf858da7cc01cce8061674106d2cf5bdab32cbfde/greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8", size = 238840, upload-time = "2026-04-27T12:23:54.806Z" },
|
{ url = "https://files.pythonhosted.org/packages/4c/fe/4fb3a0805bd5165da5ebf858da7cc01cce8061674106d2cf5bdab32cbfde/greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8", size = 238840, upload-time = "2026-04-27T12:23:54.806Z" },
|
||||||
@@ -390,6 +388,7 @@ dependencies = [
|
|||||||
{ name = "hvac" },
|
{ name = "hvac" },
|
||||||
{ name = "itsdangerous" },
|
{ name = "itsdangerous" },
|
||||||
{ name = "orjson" },
|
{ name = "orjson" },
|
||||||
|
{ name = "prometheus-client" },
|
||||||
{ name = "pydantic-settings" },
|
{ name = "pydantic-settings" },
|
||||||
{ name = "python-jose" },
|
{ name = "python-jose" },
|
||||||
{ name = "python-ulid" },
|
{ name = "python-ulid" },
|
||||||
@@ -410,6 +409,7 @@ requires-dist = [
|
|||||||
{ name = "hvac", specifier = "==2.4.0" },
|
{ name = "hvac", specifier = "==2.4.0" },
|
||||||
{ name = "itsdangerous", specifier = "==2.2.0" },
|
{ name = "itsdangerous", specifier = "==2.2.0" },
|
||||||
{ name = "orjson", specifier = "==3.11.7" },
|
{ name = "orjson", specifier = "==3.11.7" },
|
||||||
|
{ name = "prometheus-client", specifier = "==0.23.1" },
|
||||||
{ name = "pydantic-settings", specifier = "==2.12.0" },
|
{ name = "pydantic-settings", specifier = "==2.12.0" },
|
||||||
{ name = "python-jose", specifier = "==3.5.0" },
|
{ name = "python-jose", specifier = "==3.5.0" },
|
||||||
{ name = "python-ulid", specifier = "==3.1.0" },
|
{ name = "python-ulid", specifier = "==3.1.0" },
|
||||||
@@ -467,6 +467,15 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/68/40/c2051bd19fc467610fed469dc29e43ac65891571138f476834ca192bc290/orjson-3.11.7-cp312-cp312-win_arm64.whl", hash = "sha256:26c3b9132f783b7d7903bf1efb095fed8d4a3a85ec0d334ee8beff3d7a4749d5", size = 126089, upload-time = "2026-02-02T15:38:05.297Z" },
|
{ url = "https://files.pythonhosted.org/packages/68/40/c2051bd19fc467610fed469dc29e43ac65891571138f476834ca192bc290/orjson-3.11.7-cp312-cp312-win_arm64.whl", hash = "sha256:26c3b9132f783b7d7903bf1efb095fed8d4a3a85ec0d334ee8beff3d7a4749d5", size = 126089, upload-time = "2026-02-02T15:38:05.297Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "prometheus-client"
|
||||||
|
version = "0.23.1"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/23/53/3edb5d68ecf6b38fcbcc1ad28391117d2a322d9a1a3eff04bfdb184d8c3b/prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce", size = 80481, upload-time = "2025-09-18T20:47:25.043Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "propcache"
|
name = "propcache"
|
||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
|
|||||||
Reference in New Issue
Block a user