Local LLMs in an air-gapped network: AI for your most sensitive data

# Connected zone: fetch an open-weight model pinned to one commit, no pickle files
set -euo pipefail
MODEL=Qwen/Qwen2.5-14B-Instruct      # example, pick yours by licence and fit
REV=<commit-sha-from-the-model-page>

hf download "$MODEL" --revision "$REV" --exclude "*.bin" --local-dir models/qwen2.5-14b-instruct

cd models
find qwen2.5-14b-instruct -type f -exec sha256sum {} + > qwen2.5-14b-instruct.SHA256SUMS
cosign sign-blob --key cosign.key --bundle qwen2.5-14b-instruct.bundle qwen2.5-14b-instruct.SHA256SUMS

# Enclave: serve the approved model from the offline registry
export HF_HUB_OFFLINE=1                      # never try to reach the Hub

vllm serve /models/qwen2.5-14b-instruct \
  --served-model-name internal-14b \
  --api-key "$INTERNAL_API_KEY" \
  --max-model-len 16384

# Small team alternative: Ollama with a local Modelfile
#   FROM /models/qwen2.5-14b-instruct
#   ollama create internal-14b -f Modelfile

CREATE EXTENSION IF NOT EXISTS vector;

CREATE TABLE chunks (
  id             bigserial PRIMARY KEY,
  doc_id         text      NOT NULL,
  allowed_groups text[]    NOT NULL,   -- who may see this passage
  content        text      NOT NULL,
  embedding      vector(1024) NOT NULL -- size of your local embedding model
);
CREATE INDEX ON chunks USING hnsw (embedding vector_cosine_ops);

-- Only search what the asking user is allowed to read
SELECT doc_id, content
FROM chunks
WHERE allowed_groups && $1::text[]
ORDER BY embedding <=> $2
LIMIT 8;