Private AI for sensitive data: a local LLM setup that passes the security review

{"id": "contract-017", "file": "contracts/017.pdf",
 "question": "What is the notice period?",
 "expected": "3 months to the end of a quarter",
 "must_cite": "section 9.2", "data_class": "confidential"}

# OpenAI-compatible endpoint on an internal GPU node (example model;
# pick yours with the test set)
vllm serve Qwen/Qwen2.5-14B-Instruct-AWQ \
  --quantization awq \
  --max-model-len 32768 \
  --gpu-memory-utilization 0.90 \
  --host 0.0.0.0 --port 8000

# Any OpenAI client can now talk to it, inside your network only
curl http://llm.internal:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model": "Qwen/Qwen2.5-14B-Instruct-AWQ",
       "messages": [{"role": "user", "content": "Summarise clause 9.2"}]}'

-- Chunks with their source and who may read them
create extension if not exists vector;
create table doc_chunks (
  id bigserial primary key,
  source text not null,
  allowed_groups text[] not null,
  content text not null,
  embedding vector(1024)
);

-- Nearest chunks the user is allowed to see
select source, content
from doc_chunks
where allowed_groups && $1      -- the user's groups
order by embedding <=> $2       -- cosine distance
limit 8;