tftsr_ai/templates/ollama/override.conf.j2

[Service]
# ── Ollama API & model settings ────────────────────────────────────────────
Environment="OLLAMA_API_KEY={{ ollama_api_key }}"
Environment="OLLAMA_HOST=0.0.0.0:11434"
Environment="OLLAMA_MODELS=/mnt/ai_data/ollama_models"
Environment="OLLAMA_KEEP_ALIVE=-1"

# ── Inference performance ──────────────────────────────────────────────────
# Flash attention: fused softmax, ~20% less memory bandwidth, faster on AVX2
Environment="OLLAMA_FLASH_ATTENTION=1"

# Threads: 28 logical CPUs on NUMA node 1 (14 physical cores × 2 HT)
# Covers all threads on the socket so no cross-socket migrations occur
Environment="OLLAMA_NUM_THREADS={{ ollama_num_threads }}"

# Parallel inference streams — 4 simultaneous requests, 7 threads each
Environment="OLLAMA_NUM_PARALLEL={{ ollama_num_parallel }}"

# Keep 4 models warm in RAM (KEEP_ALIVE=-1 means never unload)
Environment="OLLAMA_MAX_LOADED_MODELS={{ ollama_max_loaded_models }}"

# ── NUMA / CPU binding ────────────────────────────────────────────────────
# Pin all Ollama threads to NUMA node 1 CPUs (odd: 1,3,5,...,55).
# Node 1 has ~120 GB free RAM vs node 0's ~75 GB.
# CPUAffinity prevents cross-NUMA thread migration; Linux will naturally
# allocate memory from the local node when all threads are on that node.
CPUAffinity={{ ollama_cpu_affinity }}

# ── Memory hardening ───────────────────────────────────────────────────────
# Prevent model weights from being paged out under memory pressure
LimitMEMLOCK=infinity

# Sufficient file descriptors for parallel connections and mmap'd model files
LimitNOFILE=65535

# Disable OOM kill — losing a loaded model mid-inference is worse than
# the kernel reclaiming other memory first
OOMScoreAdjust=-500