My config for daily beta llaama.cpp vulcan on 7900xtx/ubuntu. 262k inf, qwen3.6 35b a3b iq4_xs. sits about 22k MiB. crazy fast token generation and compacting. Twice as fast as optimized rocm 7.14 and lower memory usage/footprint.

#!/usr/bin/env bash

# llama-server (Vulkan) — Qwen3.6-35B-A3B IQ4_XS. Set paths in CONFIG, then run.

# Needs a Vulkan-enabled llama.cpp build and a Vulkan driver (vulkaninfo --summary).

set -euo pipefail

# ---- CONFIG ----

SERVER_PATH="${LLAMA_SERVER:-$HOME/llama.cpp/build/bin/llama-server}"

MODEL="${LLAMA_MODEL:-$HOME/models/Qwen3.6-35B-A3B-IQ4_XS.gguf}"

MMPROJ="${LLAMA_MMPROJ:-}" # empty = text-only

DATA_DIR="${LLAMA_DATA_DIR:-$HOME/.cache/llama-server}"

VK_DEVICE="${LLAMA_VK_DEVICE:-0}"

HOST="${LLAMA_HOST:-127.0.0.1}" # 0.0.0.0 = reachable on the network, no auth

PORT="${LLAMA_PORT:-8081}"

CTX_SIZE="${LLAMA_CTX:-262144}"

CACHE_TYPE="${LLAMA_CACHE_TYPE:-q4_0}"

CACHE_RAM_MB="${LLAMA_CACHE_RAM_MB:-16384}"

NGL="${LLAMA_NGL:-99}"

THREADS="${LLAMA_THREADS:-20}"

BATCH_SIZE="${LLAMA_BATCH:-512}"

IMAGE_MIN_TOKENS="${LLAMA_IMAGE_MIN_TOKENS:-1024}"

REASONING="${LLAMA_REASONING:-off}"

# ----------------

export GGML_VK_VISIBLE_DEVICES="$VK_DEVICE"

export LD_LIBRARY_PATH="$(dirname "$SERVER_PATH"):${LD_LIBRARY_PATH:-}"

[ -x "$SERVER_PATH" ] || { echo "[ABORT] llama-server not found/executable: $SERVER_PATH"; exit 1; }

[ -f "$MODEL" ] || { echo "[ABORT] model not found: $MODEL"; exit 1; }

if [ -n "$MMPROJ" ] && [ ! -f "$MMPROJ" ]; then

echo "[ABORT] MMPROJ set but not found: $MMPROJ (use MMPROJ=\"\" for text-only)"; exit 1

fi

mkdir -p "$DATA_DIR"

[ "$HOST" = "0.0.0.0" ] && echo "[WARN] bound to 0.0.0.0 — no auth, reachable from the network"

ARGS=(

-m "$MODEL"

-c "$CTX_SIZE"

-np 1

--n-gpu-layers "$NGL"

--flash-attn on

--cache-type-k "$CACHE_TYPE"

--cache-type-v "$CACHE_TYPE"

--cache-ram "$CACHE_RAM_MB"

--jinja

--reasoning "$REASONING"

--cont-batching

--no-warmup

--batch-size "$BATCH_SIZE"

--threads "$THREADS"

--host "$HOST"

--port "$PORT"

--slot-save-path "$DATA_DIR"

--log-file "$DATA_DIR/llama-server.log"

)

[ -n "$MMPROJ" ] && ARGS+=( --mmproj "$MMPROJ" --image-min-tokens "$IMAGE_MIN_TOKENS" )

exec "$SERVER_PATH" "${ARGS[@]}"