Performance Metrics Tracking

              | Online (live)         | Offline (batch)
--------------|----------------------|------------------------
System        | latency, QPS, errors | resource cost, build time
ML / Business | CTR, accuracy*,      | holdout accuracy,
              | revenue/req          | fairness, calibration

* with delayed label

from prometheus_client import Histogram

LATENCY = Histogram(
    "predict_latency_seconds",
    "Inference latency",
    ["model_version"],
    buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
)

with LATENCY.labels("v1").time():
    pred = model.predict(x)

# Grafana query
histogram_quantile(0.99,
  sum by (le) (rate(predict_latency_seconds_bucket{model_version="v1"}[5m]))
)

# truth arrives in clickstream T+1h
def evaluate_predictions_with_label(window="1h"):
    df = (
        warehouse.query(f"""
          SELECT p.prediction, l.label
          FROM predictions p JOIN labels l USING (request_id)
          WHERE l.ts >= now() - INTERVAL '{window}'
        """)
    )
    acc = (df.prediction == df.label).mean()
    push_to_prometheus("rolling_accuracy_1h", acc)

import mlflow

mlflow.set_experiment("iris-prod")

with mlflow.start_run(run_name=f"deploy-{version}"):
    mlflow.log_param("n_estimators", 200)
    mlflow.log_metric("offline_accuracy", 0.96)
    mlflow.log_metric("online_p95_ms", 38.2)
    mlflow.log_metric("rolling_accuracy_7d", 0.93)
    mlflow.sklearn.log_model(model, "model")