Modal
Installation
# Install Modal client and authenticate
pip install modal
modal setup # Opens browser for authentication
Hello World — Run a Function on GPU
# hello_gpu.py — Run a simple function on a cloud GPU
import modal
app = modal.App("hello-gpu")
@app.function(gpu="T4")
def gpu_info():
import subprocess
result = subprocess.run(["nvidia-smi"], capture_output=True, text=True)
return result.stdout
@app.local_entrypoint()
def main():
print(gpu_info.remote())
# Run it
modal run hello_gpu.py
Custom Container Images
# custom_image.py — Define a custom container with ML dependencies
import modal
image = modal.Image.debian_slim(python_version="3.11").pip_install(
"torch==2.1.0",
"transformers==4.36.0",
"accelerate",
"bitsandbytes",
)
app = modal.App("ml-inference", image=image)
@app.function(gpu="A100", timeout=300)
def generate(prompt: str) -> str:
from transformers import pipeline
pipe = pipeline("text-generation", model="mistralai/Mistral-7B-v0.1", device=0)
result = pipe(prompt, max_new_tokens=200)
return result[0]["generated_text"]
@app.local_entrypoint()
def main():
print(generate.remote("The meaning of life is"))
Model Serving with Web Endpoints
# serve_model.py — Deploy an inference API with automatic scaling
import modal
app = modal.App("llm-server")
image = modal.Image.debian_slim().pip_install("vllm")
@app.cls(gpu="A100", image=image, container_idle_timeout=300)
class LLMServer:
@modal.enter()
def load_model(self):
from vllm import LLM
self.llm = LLM(model="mistralai/Mistral-7B-Instruct-v0.2")
@modal.web_endpoint(method="POST")
def generate(self, request: dict):
from vllm import SamplingParams
params = SamplingParams(temperature=0.7, max_tokens=request.get("max_tokens", 200))
outputs = self.llm.generate([request["prompt"]], params)
return {"text": outputs[0].outputs[0].text}
# Deploy the endpoint
modal deploy serve_model.py
# Returns: https://your-username--llm-server-llmserver-generate.modal.run
Volumes for Persistent Storage
# volume_cache.py — Cache model weights across function invocations
import modal
volume = modal.Volume.from_name("model-cache", create_if_missing=True)
app = modal.App("cached-model")
image = modal.Image.debian_slim().pip_install("huggingface_hub", "torch", "transformers")
@app.function(gpu="A100", volumes={"/models": volume}, image=image, timeout=600)
def run_inference(prompt: str) -> str:
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
model_path = "/models/mistral-7b"
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype=torch.float16, device_map="auto")
inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
outputs = model.generate(**inputs, max_new_tokens=100)
return tokenizer.decode(outputs[0], skip_special_tokens=True)
Parallel Map (Batch Processing)
# batch_process.py — Process many items in parallel across GPUs
import modal
app = modal.App("batch-embeddings")
image = modal.Image.debian_slim().pip_install("sentence-transformers")
@app.function(gpu="T4", image=image, concurrency_limit=10)
def embed(text: str) -> list[float]:
from sentence_transformers import SentenceTransformer
model = SentenceTransformer("all-MiniLM-L6-v2")
return model.encode(text).tolist()
@app.local_entrypoint()
def main():
texts = [f"Document {i}: This is sample text." for i in range(100)]
# Process all texts in parallel
results = list(embed.map(texts))
print(f"Generated {len(results)} embeddings of dim {len(results[0])}")
Scheduled Jobs (Cron)
# scheduled_job.py — Run a function on a schedule
import modal
app = modal.App("daily-training")
@app.function(schedule=modal.Cron("0 2 * * *"), gpu="A100", timeout=3600)
def nightly_finetune():
"""Runs every night at 2 AM UTC"""
print("Starting nightly fine-tuning job...")
# training logic here
Secrets Management
# secrets.py — Access API keys and secrets securely
import modal
app = modal.App("with-secrets")
@app.function(secrets=[modal.Secret.from_name("my-openai-secret")])
def call_openai():
import os
from openai import OpenAI
# OPENAI_API_KEY is injected from the secret
client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
return client.chat.completions.create(
model="gpt-4",
messages=[{"role": "user", "content": "Hello"}],
).choices[0].message.content
Key Concepts
- Scales to zero: No cost when idle; containers spin up on demand
- GPU selection:
"T4", "A10G", "A100", "H100" — pick by workload
@modal.enter(): Runs once when container starts — ideal for loading models- Volumes: Persistent storage shared across function calls for model caching
.map(): Fan out work across many containers in parallel- Web endpoints: Auto-generated HTTPS URLs with authentication
modal deploy for persistent endpoints, modal run for one-off execution
eliferjunior