Prompt research at scale with an open model

# All URLs from a sitemap (for a sitemap index, run it again on each listed sitemap)
curl -s https://www.example.com/sitemap.xml \
  | grep -o '<loc>[^<]*</loc>' | sed 's/<\/\?loc>//g' > urls.txt

# Count URLs per template (first path segment)
cut -d/ -f4 urls.txt | sort | uniq -c | sort -rn | head

# Laptop: Ollama with a 9B Qwen model (OpenAI-compatible API on :11434/v1)
ollama pull qwen3.5:9b

# One GPU server: vLLM with a 27B Qwen model (OpenAI-compatible API on :8000/v1)
vllm serve Qwen/Qwen3.6-27B --max-model-len 16384

# pip install openai
# Generates customer-style prompts per page with a local model; nothing leaves your machine.
import csv, json
from openai import OpenAI

client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")  # vLLM: http://localhost:8000/v1
MODEL = "qwen3.5:9b"

BRIEF = """You write questions that real customers ask an AI assistant.
Page facts: {facts}
Write {n} prompts in {lang}. Each prompt: a real situation plus at least one constraint
(budget, date, group size, accessibility, location...). Never mention the brand.
Vary intent: discover, compare, decide, practical.
Return JSON only: [{{"intent": "...", "prompt": "..."}}]"""

pages = [  # in practice: built from your sitemap and page titles
    {"url": "https://www.example.com/berlin/italian", "facts": "Italian restaurant, Berlin Mitte, mid-price, terrace, step-free access, groups up to 12"},
]

seen, rows = set(), []
for page in pages:
    for lang in ["English", "German", "Spanish"]:
        out = client.chat.completions.create(
            model=MODEL, temperature=0.7,
            messages=[{"role": "user", "content": BRIEF.format(facts=page["facts"], n=6, lang=lang)}],
        ).choices[0].message.content
        start, end = out.find("["), out.rfind("]") + 1
        for item in json.loads(out[start:end]):
            key = item["prompt"].lower().strip()
            if key not in seen:  # drop exact duplicates across pages
                seen.add(key)
                rows.append([page["url"], lang, item["intent"], item["prompt"]])

with open("prompts.csv", "w", newline="") as f:
    csv.writer(f).writerows([["url", "language", "intent", "prompt"], *rows])
print(len(rows), "prompts written to prompts.csv")