diff --git a/optional-skills/mlops/guidance/SKILL.md b/optional-skills/mlops/guidance/SKILL.md index 3fa841270049..157eb98a40f0 100644 --- a/optional-skills/mlops/guidance/SKILL.md +++ b/optional-skills/mlops/guidance/SKILL.md @@ -1,7 +1,7 @@ --- name: guidance description: Constrain LLM output with grammars; guarantee valid JSON. -version: 1.0.0 +version: 1.0.1 author: Orchestra Research license: MIT dependencies: [guidance, transformers] @@ -53,13 +53,19 @@ result = lm + "The capital of France is " + gen("capital", max_tokens=5) print(result["capital"]) # "Paris" ``` -### With Anthropic Claude +### Chat format with a local model + +> **Constraint support requires local logit access.** Regex, `select()`, and +> grammar-based constrained generation only work with local backends +> (`Transformers`, `LlamaCpp`). Remote API backends (`OpenAI`, and Azure +> variants) support unconstrained `gen()` / chat only — they cannot enforce +> token-level constraints. guidance 0.3.x has no `models.Anthropic` class. ```python from guidance import models, gen, system, user, assistant -# Configure Claude -lm = models.Anthropic("claude-sonnet-4-5-20250929") +# Local model (supports constrained generation) +lm = models.Transformers("microsoft/Phi-4-mini-instruct") # Use context managers for chat format with system(): @@ -81,7 +87,7 @@ Guidance uses Pythonic context managers for chat-style interactions. ```python from guidance import system, user, assistant, gen -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") # System message with system(): @@ -112,7 +118,7 @@ Guidance ensures outputs match specified patterns using regex or grammars. ```python from guidance import models, gen -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") # Constrain to valid email format lm += "Email: " + gen("email", regex=r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}") @@ -137,7 +143,7 @@ print(lm["date"]) # Guaranteed YYYY-MM-DD format ```python from guidance import models, gen, select -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") # Constrain to specific choices lm += "Sentiment: " + select(["positive", "negative", "neutral"], name="sentiment") @@ -171,7 +177,7 @@ prompt = "The capital of France is " ```python from guidance import models, gen -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") # Token healing enabled by default lm += "The capital of France is " + gen("capital", max_tokens=5) @@ -185,26 +191,30 @@ lm += "The capital of France is " + gen("capital", max_tokens=5) ### 4. Grammar-Based Generation -Define complex structures using context-free grammars. +Define complex structures by composing grammar functions. The template-string +`grammar=` form is not part of current guidance — build grammars from +composable functions, or use `guidance.json()` for JSON. ```python from guidance import models, gen +from guidance import json as gen_json +from pydantic import BaseModel, Field -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") -# JSON grammar (simplified) -json_grammar = """ -{ - "name": , - "age": , - "email": -} -""" +# JSON via a Pydantic schema (guidance.json compiles the schema to a grammar) +class Person(BaseModel): + name: str = Field(pattern=r"[A-Za-z ]+") + age: int + email: str = Field(pattern=r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}") -# Generate valid JSON -lm += gen("person", grammar=json_grammar) +lm += gen_json(name="person", schema=Person) -print(lm["person"]) # Guaranteed valid JSON structure +print(lm["person"]) # Guaranteed valid JSON matching the schema + +# Or compose grammar functions directly: +grammar = "name=" + gen("name", regex=r"[A-Za-z ]+") + " age=" + gen("age", regex=r"[0-9]+") +lm += grammar ``` **Use cases:** @@ -228,7 +238,7 @@ def generate_person(lm): return lm # Use the function -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") lm = generate_person(lm) print(lm["name"]) @@ -266,20 +276,14 @@ def react_agent(lm, question, tools, max_rounds=5): ## Backend Configuration -### Anthropic Claude +### OpenAI (remote — unconstrained only) + +> Remote API backends cannot do constrained generation (regex/select/grammar); +> use them only for plain chat/`gen()`. For constraints, use a local backend. ```python from guidance import models -lm = models.Anthropic( - model="claude-sonnet-4-5-20250929", - api_key="your-api-key" # Or set ANTHROPIC_API_KEY env var -) -``` - -### OpenAI - -```python lm = models.OpenAI( model="gpt-4o-mini", api_key="your-api-key" # Or set OPENAI_API_KEY env var @@ -316,7 +320,7 @@ lm = LlamaCpp( ```python from guidance import models, gen, system, user, assistant -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") with system(): lm += "You generate valid JSON." @@ -339,7 +343,7 @@ print(lm) # Valid JSON guaranteed ```python from guidance import models, gen, select -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") text = "This product is amazing! I love it." @@ -370,7 +374,7 @@ def chain_of_thought(lm, question): return lm -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") lm = chain_of_thought(lm, "What is 15% of 200?") print(lm["answer"]) @@ -412,7 +416,7 @@ def react_agent(lm, question): return lm -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") lm = react_agent(lm, "What is 25 * 4 + 10?") print(lm["answer"]) ``` @@ -443,7 +447,7 @@ def extract_entities(lm, text): text = "Tim Cook announced at Apple Park on 2024-09-15 in Cupertino." -lm = models.Anthropic("claude-sonnet-4-5-20250929") +lm = models.Transformers("microsoft/Phi-4-mini-instruct") lm = extract_entities(lm, text) print(f"Person: {lm['person']}") diff --git a/optional-skills/mlops/inference/outlines/SKILL.md b/optional-skills/mlops/inference/outlines/SKILL.md index 148a28fa6926..16744e9a61c7 100644 --- a/optional-skills/mlops/inference/outlines/SKILL.md +++ b/optional-skills/mlops/inference/outlines/SKILL.md @@ -1,7 +1,7 @@ --- name: outlines description: "Outlines: structured JSON/regex/Pydantic LLM generation." -version: 1.0.0 +version: 1.0.1 author: Orchestra Research license: MIT dependencies: [outlines, transformers, vllm, pydantic] @@ -24,7 +24,15 @@ Use Outlines when you need to: - **Generate against JSON schemas** automatically - **Control token sampling** at the grammar level -**GitHub Stars**: 8,000+ | **From**: dottxt.ai (formerly .txt) +**GitHub Stars**: 12,000+ | **From**: dottxt.ai (formerly .txt) + +> **API note (Outlines 1.x):** This skill targets the current v1 API. +> The pre-1.0 helpers (`outlines.models.transformers(...)`, +> `outlines.generate.json/choice/regex/...`) have been **removed**. In v1 you +> create a model with `outlines.from_transformers(...)` (or `from_vllm`, +> `from_llamacpp`, `from_openai`) and then **call the model directly** with an +> output type: `model(prompt, output_type)`. JSON/Pydantic outputs are returned +> as a **JSON string** — validate with `YourModel.model_validate_json(result)`. ## Installation @@ -45,14 +53,19 @@ pip install outlines vllm # vLLM for high-throughput ```python import outlines from typing import Literal +from transformers import AutoModelForCausalLM, AutoTokenizer -# Load model -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") +MODEL_NAME = "microsoft/Phi-3-mini-4k-instruct" -# Generate with type constraint +# v1: wrap a Transformers model + tokenizer +model = outlines.from_transformers( + AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="auto"), + AutoTokenizer.from_pretrained(MODEL_NAME), +) + +# Call the model directly with an output type prompt = "Sentiment of 'This product is amazing!': " -generator = outlines.generate.choice(model, ["positive", "negative", "neutral"]) -sentiment = generator(prompt) +sentiment = model(prompt, Literal["positive", "negative", "neutral"]) print(sentiment) # "positive" (guaranteed one of these) ``` @@ -62,19 +75,24 @@ print(sentiment) # "positive" (guaranteed one of these) ```python from pydantic import BaseModel import outlines +from transformers import AutoModelForCausalLM, AutoTokenizer class User(BaseModel): name: str age: int email: str -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") +MODEL_NAME = "microsoft/Phi-3-mini-4k-instruct" +model = outlines.from_transformers( + AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="auto"), + AutoTokenizer.from_pretrained(MODEL_NAME), +) -# Generate structured output +# Generate structured output (returns a JSON string) prompt = "Extract user: John Doe, 30 years old, john@example.com" -generator = outlines.generate.json(model, User) -user = generator(prompt) +result = model(prompt, User, max_new_tokens=200) +user = User.model_validate_json(result) # parse into the Pydantic model print(user.name) # "John Doe" print(user.age) # 30 print(user.email) # "john@example.com" @@ -84,11 +102,12 @@ print(user.email) # "john@example.com" ### 1. Constrained Token Sampling -Outlines uses Finite State Machines (FSM) to constrain token generation at the logit level. +Outlines constrains token generation at the logit level using a compiled +automaton derived from your output type. **How it works:** -1. Convert schema (JSON/Pydantic/regex) to context-free grammar (CFG) -2. Transform CFG into Finite State Machine (FSM) +1. Convert the output type (JSON/Pydantic/regex/`Literal`) to a schema/grammar +2. Compile the grammar into a token-level automaton 3. Filter invalid tokens at each step during generation 4. Fast-forward when only one valid token exists @@ -99,42 +118,36 @@ Outlines uses Finite State Machines (FSM) to constrain token generation at the l ```python import outlines +from pydantic import BaseModel +from transformers import AutoModelForCausalLM, AutoTokenizer -# Pydantic model -> JSON schema -> CFG -> FSM class Person(BaseModel): name: str age: int -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") - -# Behind the scenes: -# 1. Person -> JSON schema -# 2. JSON schema -> CFG -# 3. CFG -> FSM -# 4. FSM filters tokens during generation - -generator = outlines.generate.json(model, Person) -result = generator("Generate person: Alice, 25") -``` - -### 2. Structured Generators - -Outlines provides specialized generators for different output types. - -#### Choice Generator - -```python -# Multiple choice selection -generator = outlines.generate.choice( - model, - ["positive", "negative", "neutral"] +model = outlines.from_transformers( + AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct", device_map="auto"), + AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct"), ) -sentiment = generator("Review: This is great!") -# Result: One of the three choices +result = model("Generate person: Alice, 25", Person) +person = Person.model_validate_json(result) ``` -#### JSON Generator +### 2. Output Types + +In v1 you pass the desired **output type** directly as the second argument. + +#### Multiple choice (`Literal`) + +```python +from typing import Literal + +sentiment = model("Review: This is great!", Literal["positive", "negative", "neutral"]) +# Result: one of the three choices +``` + +#### JSON via Pydantic ```python from pydantic import BaseModel @@ -144,97 +157,85 @@ class Product(BaseModel): price: float in_stock: bool -# Generate valid JSON matching schema -generator = outlines.generate.json(model, Product) -product = generator("Extract: iPhone 15, $999, available") - -# Guaranteed valid Product instance -print(type(product)) # +result = model("Extract: iPhone 15, $999, available", Product) +product = Product.model_validate_json(result) # valid Product instance ``` -#### Regex Generator +#### Regex (pass a regex string) ```python -# Generate text matching regex -generator = outlines.generate.regex( - model, - r"[0-9]{3}-[0-9]{3}-[0-9]{4}" # Phone number pattern -) - -phone = generator("Generate phone number:") -# Result: "555-123-4567" (guaranteed to match pattern) +# Generate text matching a regex pattern +phone = model("Generate phone number:", r"[0-9]{3}-[0-9]{3}-[0-9]{4}") +# Result: "555-123-4567" (guaranteed to match the pattern) ``` -#### Integer/Float Generators +#### Numeric types ```python -# Generate specific numeric types -int_generator = outlines.generate.integer(model) -age = int_generator("Person's age:") # Guaranteed integer - -float_generator = outlines.generate.float(model) -price = float_generator("Product price:") # Guaranteed float +# Pass the Python type directly +age = model("Person's age:", int) # guaranteed integer +price = model("Product price:", float) # guaranteed float ``` ### 3. Model Backends -Outlines supports multiple local and API-based backends. +Outlines supports multiple local and API-based backends via `from_*` factories. #### Transformers (Hugging Face) ```python import outlines +from transformers import AutoModelForCausalLM, AutoTokenizer -# Load from Hugging Face -model = outlines.models.transformers( - "microsoft/Phi-3-mini-4k-instruct", - device="cuda" # Or "cpu" +model = outlines.from_transformers( + AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct", device_map="auto"), + AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct"), ) -# Use with any generator -generator = outlines.generate.json(model, YourModel) +result = model(prompt, YourModel) ``` #### llama.cpp ```python -# Load GGUF model -model = outlines.models.llamacpp( - "./models/llama-3.1-8b-instruct.Q4_K_M.gguf", - n_gpu_layers=35 -) +import outlines +from llama_cpp import Llama -generator = outlines.generate.json(model, YourModel) +llm = Llama("./models/llama-3.1-8b-instruct.Q4_K_M.gguf", n_gpu_layers=35, n_ctx=4096) +model = outlines.from_llamacpp(llm) + +result = model(prompt, YourModel) ``` #### vLLM (High Throughput) ```python -# For production deployments -model = outlines.models.vllm( - "meta-llama/Llama-3.1-8B-Instruct", - tensor_parallel_size=2 # Multi-GPU -) +import outlines +from vllm import LLM -generator = outlines.generate.json(model, YourModel) +llm = LLM("meta-llama/Llama-3.1-8B-Instruct", tensor_parallel_size=2) +model = outlines.from_vllm(llm) + +result = model(prompt, YourModel) ``` -#### OpenAI (Limited Support) +#### OpenAI (server-side constrained JSON) ```python -# Basic OpenAI support -model = outlines.models.openai( - "gpt-4o-mini", - api_key="your-api-key" -) +import outlines +from openai import OpenAI -# Note: Some features limited with API models -generator = outlines.generate.json(model, YourModel) +client = OpenAI() +model = outlines.from_openai(client, "gpt-4o-mini") + +# API backends support JSON-schema style structured output +result = model(prompt, YourModel) ``` ### 4. Pydantic Integration Outlines has first-class Pydantic support with automatic schema translation. +Generation returns a JSON string; call `model_validate_json` to get an instance. #### Basic Models @@ -247,10 +248,8 @@ class Article(BaseModel): word_count: int = Field(description="Number of words", gt=0) tags: list[str] = Field(description="List of tags") -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") -generator = outlines.generate.json(model, Article) - -article = generator("Generate article about AI") +result = model("Generate article about AI", Article, max_new_tokens=300) +article = Article.model_validate_json(result) print(article.title) print(article.word_count) # Guaranteed > 0 ``` @@ -268,9 +267,8 @@ class Person(BaseModel): age: int address: Address # Nested model -generator = outlines.generate.json(model, Person) -person = generator("Generate person in New York") - +result = model("Generate person in New York", Person) +person = Person.model_validate_json(result) print(person.address.city) # "New York" ``` @@ -290,9 +288,8 @@ class Application(BaseModel): status: Status # Must be one of enum values priority: Literal["low", "medium", "high"] # Must be one of literals -generator = outlines.generate.json(model, Application) -app = generator("Generate application") - +result = model("Generate application", Application) +app = Application.model_validate_json(result) print(app.status) # Status.PENDING (or APPROVED/REJECTED) ``` @@ -303,6 +300,7 @@ print(app.status) # Status.PENDING (or APPROVED/REJECTED) ```python from pydantic import BaseModel import outlines +from transformers import AutoModelForCausalLM, AutoTokenizer class CompanyInfo(BaseModel): name: str @@ -310,8 +308,10 @@ class CompanyInfo(BaseModel): industry: str employees: int -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") -generator = outlines.generate.json(model, CompanyInfo) +model = outlines.from_transformers( + AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct", device_map="auto"), + AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct"), +) text = """ Apple Inc. was founded in 1976 in the technology industry. @@ -319,7 +319,7 @@ The company employs approximately 164,000 people worldwide. """ prompt = f"Extract company information:\n{text}\n\nCompany:" -company = generator(prompt) +company = CompanyInfo.model_validate_json(model(prompt, CompanyInfo, max_new_tokens=200)) print(f"Name: {company.name}") print(f"Founded: {company.founded_year}") @@ -331,26 +331,24 @@ print(f"Employees: {company.employees}") ```python from typing import Literal -import outlines - -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") +from pydantic import BaseModel # Binary classification -generator = outlines.generate.choice(model, ["spam", "not_spam"]) -result = generator("Email: Buy now! 50% off!") +result = model("Email: Buy now! 50% off!", Literal["spam", "not_spam"]) # Multi-class classification -categories = ["technology", "business", "sports", "entertainment"] -category_gen = outlines.generate.choice(model, categories) -category = category_gen("Article: Apple announces new iPhone...") +category = model( + "Article: Apple announces new iPhone...", + Literal["technology", "business", "sports", "entertainment"], +) # With confidence class Classification(BaseModel): label: Literal["positive", "negative", "neutral"] confidence: float -classifier = outlines.generate.json(model, Classification) -result = classifier("Review: This product is okay, nothing special") +out = model("Review: This product is okay, nothing special", Classification) +result = Classification.model_validate_json(out) ``` ### Pattern 3: Structured Forms @@ -364,9 +362,6 @@ class UserProfile(BaseModel): country: str interests: list[str] -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") -generator = outlines.generate.json(model, UserProfile) - prompt = """ Extract user profile from: Name: Alice Johnson @@ -377,7 +372,7 @@ Country: USA Interests: hiking, photography, cooking """ -profile = generator(prompt) +profile = UserProfile.model_validate_json(model(prompt, UserProfile, max_new_tokens=250)) print(profile.full_name) print(profile.interests) # ["hiking", "photography", "cooking"] ``` @@ -385,6 +380,8 @@ print(profile.interests) # ["hiking", "photography", "cooking"] ### Pattern 4: Multi-Entity Extraction ```python +from typing import Literal + class Entity(BaseModel): name: str type: Literal["PERSON", "ORGANIZATION", "LOCATION"] @@ -392,13 +389,10 @@ class Entity(BaseModel): class DocumentEntities(BaseModel): entities: list[Entity] -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") -generator = outlines.generate.json(model, DocumentEntities) - text = "Tim Cook met with Satya Nadella at Microsoft headquarters in Redmond." prompt = f"Extract entities from: {text}" -result = generator(prompt) +result = DocumentEntities.model_validate_json(model(prompt, DocumentEntities, max_new_tokens=300)) for entity in result.entities: print(f"{entity.name} ({entity.type})") ``` @@ -412,11 +406,8 @@ class PythonFunction(BaseModel): docstring: str body: str -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") -generator = outlines.generate.json(model, PythonFunction) - prompt = "Generate a Python function to calculate factorial" -func = generator(prompt) +func = PythonFunction.model_validate_json(model(prompt, PythonFunction, max_new_tokens=300)) print(f"def {func.function_name}({', '.join(func.parameters)}):") print(f' """{func.docstring}"""') @@ -426,29 +417,29 @@ print(f" {func.body}") ### Pattern 6: Batch Processing ```python -def batch_extract(texts: list[str], schema: type[BaseModel]): - """Extract structured data from multiple texts.""" - model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") - generator = outlines.generate.json(model, schema) - - results = [] - for text in texts: - result = generator(f"Extract from: {text}") - results.append(result) - - return results +import outlines +from transformers import AutoModelForCausalLM, AutoTokenizer +from pydantic import BaseModel class Person(BaseModel): name: str age: int +model = outlines.from_transformers( + AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct", device_map="auto"), + AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct"), +) + texts = [ "John is 30 years old", "Alice is 25 years old", - "Bob is 40 years old" + "Bob is 40 years old", ] -people = batch_extract(texts, Person) +# v1 accepts a list of prompts for batched generation +prompts = [f"Extract from: {t}" for t in texts] +outputs = model(prompts, Person, max_new_tokens=100) +people = [Person.model_validate_json(o) for o in outputs] for person in people: print(f"{person.name}: {person.age}") ``` @@ -459,58 +450,67 @@ for person in people: ```python import outlines +from transformers import AutoModelForCausalLM, AutoTokenizer + +MODEL_NAME = "microsoft/Phi-3-mini-4k-instruct" # Basic usage -model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct") +model = outlines.from_transformers( + AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="auto"), + AutoTokenizer.from_pretrained(MODEL_NAME), +) -# GPU configuration -model = outlines.models.transformers( - "microsoft/Phi-3-mini-4k-instruct", - device="cuda", - model_kwargs={"torch_dtype": "float16"} +# GPU + dtype configuration is set on the HF model itself +import torch +model = outlines.from_transformers( + AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="cuda", torch_dtype=torch.float16), + AutoTokenizer.from_pretrained(MODEL_NAME), ) # Popular models -model = outlines.models.transformers("meta-llama/Llama-3.1-8B-Instruct") -model = outlines.models.transformers("mistralai/Mistral-7B-Instruct-v0.3") -model = outlines.models.transformers("Qwen/Qwen2.5-7B-Instruct") +for name in [ + "meta-llama/Llama-3.1-8B-Instruct", + "mistralai/Mistral-7B-Instruct-v0.3", + "Qwen/Qwen2.5-7B-Instruct", +]: + model = outlines.from_transformers( + AutoModelForCausalLM.from_pretrained(name, device_map="auto"), + AutoTokenizer.from_pretrained(name), + ) ``` ### llama.cpp ```python -# Load GGUF model -model = outlines.models.llamacpp( - "./models/llama-3.1-8b.Q4_K_M.gguf", - n_ctx=4096, # Context window - n_gpu_layers=35, # GPU layers - n_threads=8 # CPU threads -) +import outlines +from llama_cpp import Llama -# Full GPU offload -model = outlines.models.llamacpp( - "./models/model.gguf", - n_gpu_layers=-1 # All layers on GPU +# Load GGUF model +llm = Llama( + "./models/llama-3.1-8b.Q4_K_M.gguf", + n_ctx=4096, # Context window + n_gpu_layers=35, # GPU layers + n_threads=8, # CPU threads ) +model = outlines.from_llamacpp(llm) + +# Full GPU offload: set n_gpu_layers=-1 on the Llama object ``` ### vLLM (Production) ```python +import outlines +from vllm import LLM + # Single GPU -model = outlines.models.vllm("meta-llama/Llama-3.1-8B-Instruct") +model = outlines.from_vllm(LLM("meta-llama/Llama-3.1-8B-Instruct")) # Multi-GPU -model = outlines.models.vllm( - "meta-llama/Llama-3.1-70B-Instruct", - tensor_parallel_size=4 # 4 GPUs -) +model = outlines.from_vllm(LLM("meta-llama/Llama-3.1-70B-Instruct", tensor_parallel_size=4)) # With quantization -model = outlines.models.vllm( - "meta-llama/Llama-3.1-8B-Instruct", - quantization="awq" # Or "gptq" -) +model = outlines.from_vllm(LLM("meta-llama/Llama-3.1-8B-Instruct", quantization="awq")) ``` ## Best Practices @@ -598,15 +598,23 @@ class Article(BaseModel): # Can succeed even if author/date missing ``` +### 6. Always Validate JSON Output + +```python +# v1 returns a JSON string for Pydantic/JSON output types. +result = model(prompt, Article) # str +article = Article.model_validate_json(result) # Article instance +``` + ## Comparison to Alternatives | Feature | Outlines | Instructor | Guidance | LMQL | |---------|----------|------------|----------|------| -| Pydantic Support | ✅ Native | ✅ Native | ❌ No | ❌ No | -| JSON Schema | ✅ Yes | ✅ Yes | ⚠️ Limited | ✅ Yes | +| Pydantic Support | ✅ Native | ✅ Native | ✅ Yes | ❌ No | +| JSON Schema | ✅ Yes | ✅ Yes | ✅ Yes | ✅ Yes | | Regex Constraints | ✅ Yes | ❌ No | ✅ Yes | ✅ Yes | | Local Models | ✅ Full | ⚠️ Limited | ✅ Full | ✅ Full | -| API Models | ⚠️ Limited | ✅ Full | ✅ Full | ✅ Full | +| API Models | ✅ Yes | ✅ Full | ✅ Yes | ✅ Full | | Zero Overhead | ✅ Yes | ❌ No | ⚠️ Partial | ✅ Yes | | Automatic Retrying | ❌ No | ✅ Yes | ❌ No | ❌ No | | Learning Curve | Low | Low | Low | High | @@ -631,19 +639,19 @@ class Article(BaseModel): - **1.2-2x faster** than post-generation validation approaches **Memory:** -- FSM compiled once per schema (cached) +- Automaton compiled once per output type (cached) - Minimal runtime overhead - Efficient with vLLM for high throughput **Accuracy:** -- **100% valid outputs** (guaranteed by FSM) +- **100% valid outputs** (guaranteed by the constrained automaton) - No retry loops needed - Deterministic token filtering ## Resources -- **Documentation**: https://outlines-dev.github.io/outlines -- **GitHub**: https://github.com/outlines-dev/outlines (8k+ stars) +- **Documentation**: https://dottxt-ai.github.io/outlines/ +- **GitHub**: https://github.com/dottxt-ai/outlines (12k+ stars) - **Discord**: https://discord.gg/R9DSu34mGd - **Blog**: https://blog.dottxt.co @@ -652,5 +660,3 @@ class Article(BaseModel): - `references/json_generation.md` - Comprehensive JSON and Pydantic patterns - `references/backends.md` - Backend-specific configuration - `references/examples.md` - Production-ready examples - - diff --git a/optional-skills/mlops/modal/SKILL.md b/optional-skills/mlops/modal/SKILL.md index 1f8a553387d9..66d754fdc6f4 100644 --- a/optional-skills/mlops/modal/SKILL.md +++ b/optional-skills/mlops/modal/SKILL.md @@ -1,10 +1,10 @@ --- name: modal-serverless-gpu description: Serverless GPU cloud for ML jobs and model APIs. -version: 1.0.0 +version: 1.0.1 author: Orchestra Research license: MIT -dependencies: [modal>=0.64.0] +dependencies: [modal>=1.0] platforms: [linux, macos, windows] metadata: hermes: @@ -227,7 +227,6 @@ async def batch_predict(inputs: list[str]) -> list[dict]: # Inputs automatically batched return model.batch_predict(inputs) ``` - ## Secrets management ```bash @@ -259,10 +258,10 @@ def hourly_job(): ### Cold start mitigation ```python -@app.function( - container_idle_timeout=300, # Keep warm 5 min - allow_concurrent_inputs=10, # Handle concurrent requests -) +# Modal 1.0 autoscaler params: scaledown_window (was container_idle_timeout). +# Input concurrency moved to the @modal.concurrent decorator. +@app.function(scaledown_window=300) # Keep warm 5 min +@modal.concurrent(max_inputs=10) # Handle concurrent requests per container def inference(): pass ``` @@ -304,14 +303,21 @@ def run_parallel(): memory=32768, # 32GB RAM cpu=4, # 4 CPU cores timeout=3600, # 1 hour max - container_idle_timeout=120,# Keep warm 2 min + scaledown_window=120, # Keep warm 2 min (was container_idle_timeout) retries=3, # Retry on failure - concurrency_limit=10, # Max concurrent containers + max_containers=10, # Max concurrent containers (was concurrency_limit) + min_containers=1, # Keep N containers warm (was keep_warm) ) def my_function(): pass ``` +> **Modal 1.0 autoscaler renames** (see the [migration guide](https://modal.com/docs/guide/modal-1-0-migration)): +> - `container_idle_timeout` → `scaledown_window` +> - `concurrency_limit` → `max_containers` +> - `keep_warm` → `min_containers` +> - `allow_concurrent_inputs=N` → the `@modal.concurrent(max_inputs=N)` decorator + ## Debugging ```python @@ -327,7 +333,7 @@ if __name__ == "__main__": | Issue | Solution | |-------|----------| -| Cold start latency | Increase `container_idle_timeout`, use `@modal.enter()` | +| Cold start latency | Increase `scaledown_window`, use `@modal.enter()` | | GPU OOM | Use larger GPU (`A100-80GB`), enable gradient checkpointing | | Image build fails | Pin dependency versions, check CUDA compatibility | | Timeout errors | Increase `timeout`, add checkpointing | diff --git a/optional-skills/mlops/pinecone/SKILL.md b/optional-skills/mlops/pinecone/SKILL.md index aca609ebe370..b32dcddadfb9 100644 --- a/optional-skills/mlops/pinecone/SKILL.md +++ b/optional-skills/mlops/pinecone/SKILL.md @@ -1,10 +1,10 @@ --- name: pinecone description: Managed vector DB for production RAG and search. -version: 1.0.0 +version: 1.0.1 author: Orchestra Research license: MIT -dependencies: [pinecone-client] +dependencies: [pinecone] platforms: [linux, macos, windows] metadata: hermes: @@ -42,9 +42,11 @@ The vector database for production AI applications. ### Installation ```bash -pip install pinecone-client +pip install pinecone ``` +> Note: the old `pinecone-client` package is deprecated. Install `pinecone` (v5+; current 9.x). The import stays `from pinecone import Pinecone`. + ### Basic usage ```python @@ -226,14 +228,31 @@ index.upsert(vectors=[ ]) # Hybrid query +# NOTE: index.query() does NOT accept an `alpha` kwarg. Pinecone stores a +# single sparse-dense vector, so weighting must be applied by pre-scaling the +# query vectors before sending them. Use the hybrid_score_norm helper below +# (alpha * dense + (1 - alpha) * sparse; alpha=1 → pure dense, 0 → pure sparse). + +def hybrid_score_norm(dense, sparse, alpha: float): + """Scale dense/sparse query vectors for weighted hybrid search.""" + if not 0 <= alpha <= 1: + raise ValueError("alpha must be between 0 and 1") + scaled_sparse = { + "indices": sparse["indices"], + "values": [v * (1 - alpha) for v in sparse["values"]], + } + return [v * alpha for v in dense], scaled_sparse + +hdense, hsparse = hybrid_score_norm( + dense=[0.1, 0.2, ...], + sparse={"indices": [10, 45], "values": [0.5, 0.3]}, + alpha=0.5, # 0=sparse, 1=dense, 0.5=balanced +) + results = index.query( - vector=[0.1, 0.2, ...], - sparse_vector={ - "indices": [10, 45], - "values": [0.5, 0.3] - }, + vector=hdense, + sparse_vector=hsparse, top_k=5, - alpha=0.5 # 0=sparse, 1=dense, 0.5=hybrid ) ``` diff --git a/optional-skills/mlops/qdrant/SKILL.md b/optional-skills/mlops/qdrant/SKILL.md index 29786d3d0540..4507d89a1c93 100644 --- a/optional-skills/mlops/qdrant/SKILL.md +++ b/optional-skills/mlops/qdrant/SKILL.md @@ -1,10 +1,10 @@ --- name: qdrant-vector-search description: Vector search engine for production RAG systems. -version: 1.0.0 +version: 1.0.1 author: Orchestra Research license: MIT -dependencies: [qdrant-client>=1.12.0] +dependencies: [qdrant-client>=1.14.0] platforms: [linux, macos, windows] metadata: hermes: @@ -89,17 +89,17 @@ client.upsert( ] ) -# Search with filtering -results = client.search( +# Search with filtering (query_points is the current API; client.search is removed in qdrant-client 1.14+) +response = client.query_points( collection_name="documents", - query_vector=[0.15, 0.25, ...], + query=[0.15, 0.25, ...], query_filter={ "must": [{"key": "category", "match": {"value": "tech"}}] }, limit=10 ) -for point in results: +for point in response.points: print(f"ID: {point.id}, Score: {point.score}, Payload: {point.payload}") ``` @@ -169,14 +169,15 @@ print(f"Points: {info.points_count}, Vectors: {info.vectors_count}") ### Basic search ```python -# Simple nearest neighbor search -results = client.search( +# Simple nearest neighbor search (returns a QueryResponse; use .points) +response = client.query_points( collection_name="documents", - query_vector=[0.1, 0.2, ...], + query=[0.1, 0.2, ...], limit=10, with_payload=True, with_vectors=False # Don't return vectors (faster) ) +results = response.points ``` ### Filtered search @@ -185,9 +186,9 @@ results = client.search( from qdrant_client.models import Filter, FieldCondition, MatchValue, Range # Complex filtering -results = client.search( +response = client.query_points( collection_name="documents", - query_vector=query_embedding, + query=query_embedding, query_filter=Filter( must=[ FieldCondition(key="category", match=MatchValue(value="tech")), @@ -198,12 +199,12 @@ results = client.search( ] ), limit=10 -) +).points # Shorthand filter syntax -results = client.search( +response = client.query_points( collection_name="documents", - query_vector=query_embedding, + query=query_embedding, query_filter={ "must": [ {"key": "category", "match": {"value": "tech"}}, @@ -211,23 +212,27 @@ results = client.search( ] }, limit=10 -) +).points ``` ### Batch search ```python -from qdrant_client.models import SearchRequest +from qdrant_client.models import QueryRequest -# Multiple queries in one request -results = client.search_batch( +# Multiple queries in one request (search_batch is replaced by query_batch_points) +responses = client.query_batch_points( collection_name="documents", requests=[ - SearchRequest(vector=[0.1, ...], limit=5), - SearchRequest(vector=[0.2, ...], limit=5, filter={"must": [...]}), - SearchRequest(vector=[0.3, ...], limit=10) + QueryRequest(query=[0.1, ...], limit=5), + QueryRequest(query=[0.2, ...], limit=5, filter={"must": [...]}), + QueryRequest(query=[0.3, ...], limit=10) ] ) +# Each element is a QueryResponse; use .points +for resp in responses: + for point in resp.points: + print(point.id, point.score) ``` ## RAG integration @@ -268,12 +273,12 @@ client.upsert(collection_name="knowledge_base", points=points) # RAG retrieval def retrieve(query: str, top_k: int = 5) -> list[dict]: query_vector = encoder.encode(query).tolist() - results = client.search( + response = client.query_points( collection_name="knowledge_base", - query_vector=query_vector, + query=query_vector, limit=top_k ) - return [{"text": r.payload["text"], "score": r.score} for r in results] + return [{"text": r.payload["text"], "score": r.score} for r in response.points] # Use in RAG pipeline context = retrieve("What is Python?") @@ -334,12 +339,14 @@ client.upsert( ] ) -# Search specific vector -results = client.search( +# Search specific named vector (pass the vector name via `using`) +response = client.query_points( collection_name="hybrid_search", - query_vector=("dense", query_dense), # Specify which vector + query=query_dense, + using="dense", # Specify which named vector to search limit=10 ) +results = response.points ``` ### Sparse vectors (BM25, SPLADE) @@ -380,12 +387,13 @@ client.create_collection( ) # Search with rescoring -results = client.search( +response = client.query_points( collection_name="quantized", - query_vector=query, + query=query, search_params={"quantization": {"rescore": True}}, # Rescore top results limit=10 ) +results = response.points ``` ## Payload indexing @@ -493,5 +501,5 @@ client = QdrantClient( - **Docs**: https://qdrant.tech/documentation/ - **Python Client**: https://github.com/qdrant/qdrant-client - **Cloud**: https://cloud.qdrant.io -- **Version**: 1.12.0+ +- **Version**: 1.14.0+ - **License**: Apache 2.0