Ollama Tutorial: Run LLMs Locally on Your Machine

ollama pull llama3.2
ollama run llama3.2

import json
import re
import subprocess
import urllib.request
from pathlib import Path

BASE = "http://localhost:11434"
TERMINAL_CODES = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]|[⠀-⣿]|\r")   # spinners and cursor codes


def cli(*args):
    out = subprocess.run(["ollama", *args], capture_output=True, text=True, encoding="utf-8")
    text = TERMINAL_CODES.sub("", out.stdout + out.stderr)
    return "\n".join(line.rstrip() for line in text.strip("\n").splitlines())


def post(path, body, headers=None):
    req = urllib.request.Request(BASE + path, json.dumps(body).encode(),
                                 {"Content-Type": "application/json", **(headers or {})})
    with urllib.request.urlopen(req, timeout=900) as r:
        return json.load(r)


# 1. The command line
print(cli("--version"))
print(cli("show", "llama3.2"))

# 2. The models on this machine, through the REST API
with urllib.request.urlopen(BASE + "/api/tags") as r:
    for m in sorted(json.load(r)["models"], key=lambda m: m["name"]):
        d = m["details"]
        print(f"{m['name']:<26} {m['size'] / 1e9:>5.1f} GB  {d['parameter_size']:>6}  {d['quantization_level']}")

# 3. One prompt, one answer, with the timing Ollama reports
settings = {"temperature": 0, "seed": 42}
r = post("/api/generate", {"model": "llama3.2", "prompt": "Name three uses of a local language model, in one line.",
                           "stream": False, "options": settings})
print("\n" + r["response"])
print(f"{r['prompt_eval_count']} prompt tokens, {r['eval_count']} tokens generated")
print(f"generation speed on this laptop: {r['eval_count'] / r['eval_duration'] * 1e9:.1f} tokens per second")

# 4. A conversation: the model remembers only what you send back to it
history = [{"role": "user", "content": "My name is Asha and I teach statistics. Reply in one sentence."}]
history.append(post("/api/chat", {"model": "llama3.2", "messages": history, "stream": False, "options": settings})["message"])
history.append({"role": "user", "content": "What subject do I teach?"})
print("\nwith history:   ", post("/api/chat", {"model": "llama3.2", "messages": history, "stream": False,
                                               "options": settings})["message"]["content"])
print("without history:", post("/api/chat", {"model": "llama3.2", "messages": history[-1:], "stream": False,
                                               "options": settings})["message"]["content"])

# 5. The OpenAI-compatible endpoint: code written for OpenAI's API, pointed at this machine
r = post("/v1/chat/completions", {"model": "llama3.2", "temperature": 0, "seed": 42,
                                  "messages": [{"role": "user", "content": "Say hello in French, one word."}]},
         {"Authorization": "Bearer ollama"})
print("\nOpenAI-style reply:", r["choices"][0]["message"]["content"], "| usage:", r["usage"])

# 6. A custom model from a Modelfile: same weights, a fixed system prompt and settings
Path("Modelfile").write_text(
    "FROM llama3.2\n"
    "PARAMETER temperature 0\n"
    "PARAMETER seed 42\n"
    'SYSTEM """You are a glossary for data science students. Define the term you are given in one plain sentence."""\n',
    encoding="utf-8")
print("\n" + cli("create", "glossary", "-f", "Modelfile").splitlines()[-1])
print(post("/api/generate", {"model": "glossary", "prompt": "overfitting", "stream": False})["response"])
print(cli("rm", "glossary"))

ollama version is 0.12.9
  Model
    architecture        llama
    parameters          3.2B
    context length      131072
    embedding length    3072
    quantization        Q4_K_M

  Capabilities
    completion
    tools

  Parameters
    stop    "<|start_header_id|>"
    stop    "<|end_header_id|>"
    stop    "<|eot_id|>"

  License
    LLAMA 3.2 COMMUNITY LICENSE AGREEMENT
    Llama 3.2 Version Release Date: September 25, 2024
    ...
llama3.2:3b                  2.0 GB    3.2B  Q4_K_M
llama3.2:latest              2.0 GB    3.2B  Q4_K_M
nomic-embed-text:latest      0.3 GB    137M  F16
phi3:mini                    2.2 GB    3.8B  Q4_0
phi:latest                   1.6 GB      3B  Q4_0
qwen2.5-coder:1.5b-base      1.0 GB    1.5B  Q4_K_M

A local language model can be used for personalized content recommendation, language translation, and speech-to-text transcription tailored to the specific dialect or regional accent of a user's location.
38 prompt tokens, 35 tokens generated
generation speed on this laptop: 9.4 tokens per second

with history:    You teach Statistics.
without history: I don't have any information about you or your teaching position. Can you please tell me what subject you teach?

OpenAI-style reply: Bonjour! | usage: {'prompt_tokens': 33, 'completion_tokens': 3, 'total_tokens': 36}

success
Overfitting refers to when a model is too complex and learns the noise in the training data, resulting in poor performance on new, unseen data.
deleted 'glossary'