# Development candidate. Model import and lifecycle acceptance are pending. app: id: llama-server name: Llama Inference Runtime version: b9860 upstream: kind: github repo: ggml-org/llama.cpp description: >- Private CPU inference runtime for an explicitly imported GGUF model. No model downloads or host API ports. Intended as a Routstr upstream. container: build: context: /opt/archipelago/docker/llama-server dockerfile: Dockerfile tag: localhost/archy-llama-server:b9860-archy1 network: archy-net network_aliases: [llama-server] generated_secrets: - name: llama-server-api-key kind: hex32 secret_env: - key: LLAMA_API_KEY secret_file: llama-server-api-key custom_args: - --model - /models/model.gguf - --alias - archy-llama # Qwen2.5-1.5B-Instruct (trained on 32k) at 8k: room for AIUI's # ~2.4k-token instructions plus a conversation. A model trained on less # is silently capped by llama-server; keep ARCHY_MODEL_CONTEXT in # apps/routstr in step. Threads default to the physical core count. - --ctx-size - "8192" - --parallel - "1" - --n-predict - "1024" - --no-webui dependencies: - storage: 10Gi resources: cpu_limit: 4 memory_limit: 8Gi security: capabilities: [] readonly_root: true no_new_privileges: true network_policy: isolated # No host listener: only authenticated app-network callers can use the API. ports: [] volumes: - type: bind source: /var/lib/archipelago/llama-server/models target: /models options: [ro] - type: tmpfs target: /tmp tmpfs_options: rw,noexec,nosuid,size=128m health_check: type: http endpoint: http://localhost:8080 path: /health interval: 30s timeout: 5s retries: 3 metadata: category: tools tier: optional author: ggml-org repo: https://github.com/ggml-org/llama.cpp