mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-24 16:54:43 +00:00
fix(skills): sync mlops structured-output/vectordb skills to current APIs
Five optional mlops skills documented removed pre-major-version APIs. Verified each against upstream and rewrote to the current form: - outlines: pre-1.0 outlines.generate.*/models.transformers -> v1 from_transformers + model(prompt, output_type) - guidance: models.Anthropic (nonexistent in 0.3.x) -> Transformers backend; grammar-string -> guidance.json(); noted constrained gen needs local logits - pinecone: pip install pinecone-client (deprecated) -> pinecone; removed bogus alpha= query kwarg, pre-scale hybrid vectors - qdrant: client.search()/search_batch() (removed) -> query_points()/query_batch_points() - modal: container_idle_timeout/concurrency_limit/allow_concurrent_inputs -> scaledown_window/max_containers/@modal.concurrent; floor bumped to modal>=1.0
This commit is contained in:
teknium1
Teknium
parent
80e575dfba
commit
8a2b288462
5 changed files with 305 additions and 262 deletions
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
name: guidance
|
||||
description: Constrain LLM output with grammars; guarantee valid JSON.
|
||||
version: 1.0.0
|
||||
version: 1.0.1
|
||||
author: Orchestra Research
|
||||
license: MIT
|
||||
dependencies: [guidance, transformers]
|
||||
|
|
@ -53,13 +53,19 @@ result = lm + "The capital of France is " + gen("capital", max_tokens=5)
|
|||
print(result["capital"]) # "Paris"
|
||||
```
|
||||
|
||||
### With Anthropic Claude
|
||||
### Chat format with a local model
|
||||
|
||||
> **Constraint support requires local logit access.** Regex, `select()`, and
|
||||
> grammar-based constrained generation only work with local backends
|
||||
> (`Transformers`, `LlamaCpp`). Remote API backends (`OpenAI`, and Azure
|
||||
> variants) support unconstrained `gen()` / chat only — they cannot enforce
|
||||
> token-level constraints. guidance 0.3.x has no `models.Anthropic` class.
|
||||
|
||||
```python
|
||||
from guidance import models, gen, system, user, assistant
|
||||
|
||||
# Configure Claude
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
# Local model (supports constrained generation)
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
|
||||
# Use context managers for chat format
|
||||
with system():
|
||||
|
|
@ -81,7 +87,7 @@ Guidance uses Pythonic context managers for chat-style interactions.
|
|||
```python
|
||||
from guidance import system, user, assistant, gen
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
|
||||
# System message
|
||||
with system():
|
||||
|
|
@ -112,7 +118,7 @@ Guidance ensures outputs match specified patterns using regex or grammars.
|
|||
```python
|
||||
from guidance import models, gen
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
|
||||
# Constrain to valid email format
|
||||
lm += "Email: " + gen("email", regex=r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
|
||||
|
|
@ -137,7 +143,7 @@ print(lm["date"]) # Guaranteed YYYY-MM-DD format
|
|||
```python
|
||||
from guidance import models, gen, select
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
|
||||
# Constrain to specific choices
|
||||
lm += "Sentiment: " + select(["positive", "negative", "neutral"], name="sentiment")
|
||||
|
|
@ -171,7 +177,7 @@ prompt = "The capital of France is "
|
|||
```python
|
||||
from guidance import models, gen
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
|
||||
# Token healing enabled by default
|
||||
lm += "The capital of France is " + gen("capital", max_tokens=5)
|
||||
|
|
@ -185,26 +191,30 @@ lm += "The capital of France is " + gen("capital", max_tokens=5)
|
|||
|
||||
### 4. Grammar-Based Generation
|
||||
|
||||
Define complex structures using context-free grammars.
|
||||
Define complex structures by composing grammar functions. The template-string
|
||||
`grammar=` form is not part of current guidance — build grammars from
|
||||
composable functions, or use `guidance.json()` for JSON.
|
||||
|
||||
```python
|
||||
from guidance import models, gen
|
||||
from guidance import json as gen_json
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
|
||||
# JSON grammar (simplified)
|
||||
json_grammar = """
|
||||
{
|
||||
"name": <gen name regex="[A-Za-z ]+" max_tokens=20>,
|
||||
"age": <gen age regex="[0-9]+" max_tokens=3>,
|
||||
"email": <gen email regex="[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}" max_tokens=50>
|
||||
}
|
||||
"""
|
||||
# JSON via a Pydantic schema (guidance.json compiles the schema to a grammar)
|
||||
class Person(BaseModel):
|
||||
name: str = Field(pattern=r"[A-Za-z ]+")
|
||||
age: int
|
||||
email: str = Field(pattern=r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
|
||||
|
||||
# Generate valid JSON
|
||||
lm += gen("person", grammar=json_grammar)
|
||||
lm += gen_json(name="person", schema=Person)
|
||||
|
||||
print(lm["person"]) # Guaranteed valid JSON structure
|
||||
print(lm["person"]) # Guaranteed valid JSON matching the schema
|
||||
|
||||
# Or compose grammar functions directly:
|
||||
grammar = "name=" + gen("name", regex=r"[A-Za-z ]+") + " age=" + gen("age", regex=r"[0-9]+")
|
||||
lm += grammar
|
||||
```
|
||||
|
||||
**Use cases:**
|
||||
|
|
@ -228,7 +238,7 @@ def generate_person(lm):
|
|||
return lm
|
||||
|
||||
# Use the function
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
lm = generate_person(lm)
|
||||
|
||||
print(lm["name"])
|
||||
|
|
@ -266,20 +276,14 @@ def react_agent(lm, question, tools, max_rounds=5):
|
|||
|
||||
## Backend Configuration
|
||||
|
||||
### Anthropic Claude
|
||||
### OpenAI (remote — unconstrained only)
|
||||
|
||||
> Remote API backends cannot do constrained generation (regex/select/grammar);
|
||||
> use them only for plain chat/`gen()`. For constraints, use a local backend.
|
||||
|
||||
```python
|
||||
from guidance import models
|
||||
|
||||
lm = models.Anthropic(
|
||||
model="claude-sonnet-4-5-20250929",
|
||||
api_key="your-api-key" # Or set ANTHROPIC_API_KEY env var
|
||||
)
|
||||
```
|
||||
|
||||
### OpenAI
|
||||
|
||||
```python
|
||||
lm = models.OpenAI(
|
||||
model="gpt-4o-mini",
|
||||
api_key="your-api-key" # Or set OPENAI_API_KEY env var
|
||||
|
|
@ -316,7 +320,7 @@ lm = LlamaCpp(
|
|||
```python
|
||||
from guidance import models, gen, system, user, assistant
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
|
||||
with system():
|
||||
lm += "You generate valid JSON."
|
||||
|
|
@ -339,7 +343,7 @@ print(lm) # Valid JSON guaranteed
|
|||
```python
|
||||
from guidance import models, gen, select
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
|
||||
text = "This product is amazing! I love it."
|
||||
|
||||
|
|
@ -370,7 +374,7 @@ def chain_of_thought(lm, question):
|
|||
|
||||
return lm
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
lm = chain_of_thought(lm, "What is 15% of 200?")
|
||||
|
||||
print(lm["answer"])
|
||||
|
|
@ -412,7 +416,7 @@ def react_agent(lm, question):
|
|||
|
||||
return lm
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
lm = react_agent(lm, "What is 25 * 4 + 10?")
|
||||
print(lm["answer"])
|
||||
```
|
||||
|
|
@ -443,7 +447,7 @@ def extract_entities(lm, text):
|
|||
|
||||
text = "Tim Cook announced at Apple Park on 2024-09-15 in Cupertino."
|
||||
|
||||
lm = models.Anthropic("claude-sonnet-4-5-20250929")
|
||||
lm = models.Transformers("microsoft/Phi-4-mini-instruct")
|
||||
lm = extract_entities(lm, text)
|
||||
|
||||
print(f"Person: {lm['person']}")
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
name: outlines
|
||||
description: "Outlines: structured JSON/regex/Pydantic LLM generation."
|
||||
version: 1.0.0
|
||||
version: 1.0.1
|
||||
author: Orchestra Research
|
||||
license: MIT
|
||||
dependencies: [outlines, transformers, vllm, pydantic]
|
||||
|
|
@ -24,7 +24,15 @@ Use Outlines when you need to:
|
|||
- **Generate against JSON schemas** automatically
|
||||
- **Control token sampling** at the grammar level
|
||||
|
||||
**GitHub Stars**: 8,000+ | **From**: dottxt.ai (formerly .txt)
|
||||
**GitHub Stars**: 12,000+ | **From**: dottxt.ai (formerly .txt)
|
||||
|
||||
> **API note (Outlines 1.x):** This skill targets the current v1 API.
|
||||
> The pre-1.0 helpers (`outlines.models.transformers(...)`,
|
||||
> `outlines.generate.json/choice/regex/...`) have been **removed**. In v1 you
|
||||
> create a model with `outlines.from_transformers(...)` (or `from_vllm`,
|
||||
> `from_llamacpp`, `from_openai`) and then **call the model directly** with an
|
||||
> output type: `model(prompt, output_type)`. JSON/Pydantic outputs are returned
|
||||
> as a **JSON string** — validate with `YourModel.model_validate_json(result)`.
|
||||
|
||||
## Installation
|
||||
|
||||
|
|
@ -45,14 +53,19 @@ pip install outlines vllm # vLLM for high-throughput
|
|||
```python
|
||||
import outlines
|
||||
from typing import Literal
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
# Load model
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
MODEL_NAME = "microsoft/Phi-3-mini-4k-instruct"
|
||||
|
||||
# Generate with type constraint
|
||||
# v1: wrap a Transformers model + tokenizer
|
||||
model = outlines.from_transformers(
|
||||
AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="auto"),
|
||||
AutoTokenizer.from_pretrained(MODEL_NAME),
|
||||
)
|
||||
|
||||
# Call the model directly with an output type
|
||||
prompt = "Sentiment of 'This product is amazing!': "
|
||||
generator = outlines.generate.choice(model, ["positive", "negative", "neutral"])
|
||||
sentiment = generator(prompt)
|
||||
sentiment = model(prompt, Literal["positive", "negative", "neutral"])
|
||||
|
||||
print(sentiment) # "positive" (guaranteed one of these)
|
||||
```
|
||||
|
|
@ -62,19 +75,24 @@ print(sentiment) # "positive" (guaranteed one of these)
|
|||
```python
|
||||
from pydantic import BaseModel
|
||||
import outlines
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
class User(BaseModel):
|
||||
name: str
|
||||
age: int
|
||||
email: str
|
||||
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
MODEL_NAME = "microsoft/Phi-3-mini-4k-instruct"
|
||||
model = outlines.from_transformers(
|
||||
AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="auto"),
|
||||
AutoTokenizer.from_pretrained(MODEL_NAME),
|
||||
)
|
||||
|
||||
# Generate structured output
|
||||
# Generate structured output (returns a JSON string)
|
||||
prompt = "Extract user: John Doe, 30 years old, john@example.com"
|
||||
generator = outlines.generate.json(model, User)
|
||||
user = generator(prompt)
|
||||
result = model(prompt, User, max_new_tokens=200)
|
||||
|
||||
user = User.model_validate_json(result) # parse into the Pydantic model
|
||||
print(user.name) # "John Doe"
|
||||
print(user.age) # 30
|
||||
print(user.email) # "john@example.com"
|
||||
|
|
@ -84,11 +102,12 @@ print(user.email) # "john@example.com"
|
|||
|
||||
### 1. Constrained Token Sampling
|
||||
|
||||
Outlines uses Finite State Machines (FSM) to constrain token generation at the logit level.
|
||||
Outlines constrains token generation at the logit level using a compiled
|
||||
automaton derived from your output type.
|
||||
|
||||
**How it works:**
|
||||
1. Convert schema (JSON/Pydantic/regex) to context-free grammar (CFG)
|
||||
2. Transform CFG into Finite State Machine (FSM)
|
||||
1. Convert the output type (JSON/Pydantic/regex/`Literal`) to a schema/grammar
|
||||
2. Compile the grammar into a token-level automaton
|
||||
3. Filter invalid tokens at each step during generation
|
||||
4. Fast-forward when only one valid token exists
|
||||
|
||||
|
|
@ -99,42 +118,36 @@ Outlines uses Finite State Machines (FSM) to constrain token generation at the l
|
|||
|
||||
```python
|
||||
import outlines
|
||||
from pydantic import BaseModel
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
# Pydantic model -> JSON schema -> CFG -> FSM
|
||||
class Person(BaseModel):
|
||||
name: str
|
||||
age: int
|
||||
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
|
||||
# Behind the scenes:
|
||||
# 1. Person -> JSON schema
|
||||
# 2. JSON schema -> CFG
|
||||
# 3. CFG -> FSM
|
||||
# 4. FSM filters tokens during generation
|
||||
|
||||
generator = outlines.generate.json(model, Person)
|
||||
result = generator("Generate person: Alice, 25")
|
||||
```
|
||||
|
||||
### 2. Structured Generators
|
||||
|
||||
Outlines provides specialized generators for different output types.
|
||||
|
||||
#### Choice Generator
|
||||
|
||||
```python
|
||||
# Multiple choice selection
|
||||
generator = outlines.generate.choice(
|
||||
model,
|
||||
["positive", "negative", "neutral"]
|
||||
model = outlines.from_transformers(
|
||||
AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct", device_map="auto"),
|
||||
AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct"),
|
||||
)
|
||||
|
||||
sentiment = generator("Review: This is great!")
|
||||
# Result: One of the three choices
|
||||
result = model("Generate person: Alice, 25", Person)
|
||||
person = Person.model_validate_json(result)
|
||||
```
|
||||
|
||||
#### JSON Generator
|
||||
### 2. Output Types
|
||||
|
||||
In v1 you pass the desired **output type** directly as the second argument.
|
||||
|
||||
#### Multiple choice (`Literal`)
|
||||
|
||||
```python
|
||||
from typing import Literal
|
||||
|
||||
sentiment = model("Review: This is great!", Literal["positive", "negative", "neutral"])
|
||||
# Result: one of the three choices
|
||||
```
|
||||
|
||||
#### JSON via Pydantic
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -144,97 +157,85 @@ class Product(BaseModel):
|
|||
price: float
|
||||
in_stock: bool
|
||||
|
||||
# Generate valid JSON matching schema
|
||||
generator = outlines.generate.json(model, Product)
|
||||
product = generator("Extract: iPhone 15, $999, available")
|
||||
|
||||
# Guaranteed valid Product instance
|
||||
print(type(product)) # <class '__main__.Product'>
|
||||
result = model("Extract: iPhone 15, $999, available", Product)
|
||||
product = Product.model_validate_json(result) # valid Product instance
|
||||
```
|
||||
|
||||
#### Regex Generator
|
||||
#### Regex (pass a regex string)
|
||||
|
||||
```python
|
||||
# Generate text matching regex
|
||||
generator = outlines.generate.regex(
|
||||
model,
|
||||
r"[0-9]{3}-[0-9]{3}-[0-9]{4}" # Phone number pattern
|
||||
)
|
||||
|
||||
phone = generator("Generate phone number:")
|
||||
# Result: "555-123-4567" (guaranteed to match pattern)
|
||||
# Generate text matching a regex pattern
|
||||
phone = model("Generate phone number:", r"[0-9]{3}-[0-9]{3}-[0-9]{4}")
|
||||
# Result: "555-123-4567" (guaranteed to match the pattern)
|
||||
```
|
||||
|
||||
#### Integer/Float Generators
|
||||
#### Numeric types
|
||||
|
||||
```python
|
||||
# Generate specific numeric types
|
||||
int_generator = outlines.generate.integer(model)
|
||||
age = int_generator("Person's age:") # Guaranteed integer
|
||||
|
||||
float_generator = outlines.generate.float(model)
|
||||
price = float_generator("Product price:") # Guaranteed float
|
||||
# Pass the Python type directly
|
||||
age = model("Person's age:", int) # guaranteed integer
|
||||
price = model("Product price:", float) # guaranteed float
|
||||
```
|
||||
|
||||
### 3. Model Backends
|
||||
|
||||
Outlines supports multiple local and API-based backends.
|
||||
Outlines supports multiple local and API-based backends via `from_*` factories.
|
||||
|
||||
#### Transformers (Hugging Face)
|
||||
|
||||
```python
|
||||
import outlines
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
# Load from Hugging Face
|
||||
model = outlines.models.transformers(
|
||||
"microsoft/Phi-3-mini-4k-instruct",
|
||||
device="cuda" # Or "cpu"
|
||||
model = outlines.from_transformers(
|
||||
AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct", device_map="auto"),
|
||||
AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct"),
|
||||
)
|
||||
|
||||
# Use with any generator
|
||||
generator = outlines.generate.json(model, YourModel)
|
||||
result = model(prompt, YourModel)
|
||||
```
|
||||
|
||||
#### llama.cpp
|
||||
|
||||
```python
|
||||
# Load GGUF model
|
||||
model = outlines.models.llamacpp(
|
||||
"./models/llama-3.1-8b-instruct.Q4_K_M.gguf",
|
||||
n_gpu_layers=35
|
||||
)
|
||||
import outlines
|
||||
from llama_cpp import Llama
|
||||
|
||||
generator = outlines.generate.json(model, YourModel)
|
||||
llm = Llama("./models/llama-3.1-8b-instruct.Q4_K_M.gguf", n_gpu_layers=35, n_ctx=4096)
|
||||
model = outlines.from_llamacpp(llm)
|
||||
|
||||
result = model(prompt, YourModel)
|
||||
```
|
||||
|
||||
#### vLLM (High Throughput)
|
||||
|
||||
```python
|
||||
# For production deployments
|
||||
model = outlines.models.vllm(
|
||||
"meta-llama/Llama-3.1-8B-Instruct",
|
||||
tensor_parallel_size=2 # Multi-GPU
|
||||
)
|
||||
import outlines
|
||||
from vllm import LLM
|
||||
|
||||
generator = outlines.generate.json(model, YourModel)
|
||||
llm = LLM("meta-llama/Llama-3.1-8B-Instruct", tensor_parallel_size=2)
|
||||
model = outlines.from_vllm(llm)
|
||||
|
||||
result = model(prompt, YourModel)
|
||||
```
|
||||
|
||||
#### OpenAI (Limited Support)
|
||||
#### OpenAI (server-side constrained JSON)
|
||||
|
||||
```python
|
||||
# Basic OpenAI support
|
||||
model = outlines.models.openai(
|
||||
"gpt-4o-mini",
|
||||
api_key="your-api-key"
|
||||
)
|
||||
import outlines
|
||||
from openai import OpenAI
|
||||
|
||||
# Note: Some features limited with API models
|
||||
generator = outlines.generate.json(model, YourModel)
|
||||
client = OpenAI()
|
||||
model = outlines.from_openai(client, "gpt-4o-mini")
|
||||
|
||||
# API backends support JSON-schema style structured output
|
||||
result = model(prompt, YourModel)
|
||||
```
|
||||
|
||||
### 4. Pydantic Integration
|
||||
|
||||
Outlines has first-class Pydantic support with automatic schema translation.
|
||||
Generation returns a JSON string; call `model_validate_json` to get an instance.
|
||||
|
||||
#### Basic Models
|
||||
|
||||
|
|
@ -247,10 +248,8 @@ class Article(BaseModel):
|
|||
word_count: int = Field(description="Number of words", gt=0)
|
||||
tags: list[str] = Field(description="List of tags")
|
||||
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
generator = outlines.generate.json(model, Article)
|
||||
|
||||
article = generator("Generate article about AI")
|
||||
result = model("Generate article about AI", Article, max_new_tokens=300)
|
||||
article = Article.model_validate_json(result)
|
||||
print(article.title)
|
||||
print(article.word_count) # Guaranteed > 0
|
||||
```
|
||||
|
|
@ -268,9 +267,8 @@ class Person(BaseModel):
|
|||
age: int
|
||||
address: Address # Nested model
|
||||
|
||||
generator = outlines.generate.json(model, Person)
|
||||
person = generator("Generate person in New York")
|
||||
|
||||
result = model("Generate person in New York", Person)
|
||||
person = Person.model_validate_json(result)
|
||||
print(person.address.city) # "New York"
|
||||
```
|
||||
|
||||
|
|
@ -290,9 +288,8 @@ class Application(BaseModel):
|
|||
status: Status # Must be one of enum values
|
||||
priority: Literal["low", "medium", "high"] # Must be one of literals
|
||||
|
||||
generator = outlines.generate.json(model, Application)
|
||||
app = generator("Generate application")
|
||||
|
||||
result = model("Generate application", Application)
|
||||
app = Application.model_validate_json(result)
|
||||
print(app.status) # Status.PENDING (or APPROVED/REJECTED)
|
||||
```
|
||||
|
||||
|
|
@ -303,6 +300,7 @@ print(app.status) # Status.PENDING (or APPROVED/REJECTED)
|
|||
```python
|
||||
from pydantic import BaseModel
|
||||
import outlines
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
class CompanyInfo(BaseModel):
|
||||
name: str
|
||||
|
|
@ -310,8 +308,10 @@ class CompanyInfo(BaseModel):
|
|||
industry: str
|
||||
employees: int
|
||||
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
generator = outlines.generate.json(model, CompanyInfo)
|
||||
model = outlines.from_transformers(
|
||||
AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct", device_map="auto"),
|
||||
AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct"),
|
||||
)
|
||||
|
||||
text = """
|
||||
Apple Inc. was founded in 1976 in the technology industry.
|
||||
|
|
@ -319,7 +319,7 @@ The company employs approximately 164,000 people worldwide.
|
|||
"""
|
||||
|
||||
prompt = f"Extract company information:\n{text}\n\nCompany:"
|
||||
company = generator(prompt)
|
||||
company = CompanyInfo.model_validate_json(model(prompt, CompanyInfo, max_new_tokens=200))
|
||||
|
||||
print(f"Name: {company.name}")
|
||||
print(f"Founded: {company.founded_year}")
|
||||
|
|
@ -331,26 +331,24 @@ print(f"Employees: {company.employees}")
|
|||
|
||||
```python
|
||||
from typing import Literal
|
||||
import outlines
|
||||
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
from pydantic import BaseModel
|
||||
|
||||
# Binary classification
|
||||
generator = outlines.generate.choice(model, ["spam", "not_spam"])
|
||||
result = generator("Email: Buy now! 50% off!")
|
||||
result = model("Email: Buy now! 50% off!", Literal["spam", "not_spam"])
|
||||
|
||||
# Multi-class classification
|
||||
categories = ["technology", "business", "sports", "entertainment"]
|
||||
category_gen = outlines.generate.choice(model, categories)
|
||||
category = category_gen("Article: Apple announces new iPhone...")
|
||||
category = model(
|
||||
"Article: Apple announces new iPhone...",
|
||||
Literal["technology", "business", "sports", "entertainment"],
|
||||
)
|
||||
|
||||
# With confidence
|
||||
class Classification(BaseModel):
|
||||
label: Literal["positive", "negative", "neutral"]
|
||||
confidence: float
|
||||
|
||||
classifier = outlines.generate.json(model, Classification)
|
||||
result = classifier("Review: This product is okay, nothing special")
|
||||
out = model("Review: This product is okay, nothing special", Classification)
|
||||
result = Classification.model_validate_json(out)
|
||||
```
|
||||
|
||||
### Pattern 3: Structured Forms
|
||||
|
|
@ -364,9 +362,6 @@ class UserProfile(BaseModel):
|
|||
country: str
|
||||
interests: list[str]
|
||||
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
generator = outlines.generate.json(model, UserProfile)
|
||||
|
||||
prompt = """
|
||||
Extract user profile from:
|
||||
Name: Alice Johnson
|
||||
|
|
@ -377,7 +372,7 @@ Country: USA
|
|||
Interests: hiking, photography, cooking
|
||||
"""
|
||||
|
||||
profile = generator(prompt)
|
||||
profile = UserProfile.model_validate_json(model(prompt, UserProfile, max_new_tokens=250))
|
||||
print(profile.full_name)
|
||||
print(profile.interests) # ["hiking", "photography", "cooking"]
|
||||
```
|
||||
|
|
@ -385,6 +380,8 @@ print(profile.interests) # ["hiking", "photography", "cooking"]
|
|||
### Pattern 4: Multi-Entity Extraction
|
||||
|
||||
```python
|
||||
from typing import Literal
|
||||
|
||||
class Entity(BaseModel):
|
||||
name: str
|
||||
type: Literal["PERSON", "ORGANIZATION", "LOCATION"]
|
||||
|
|
@ -392,13 +389,10 @@ class Entity(BaseModel):
|
|||
class DocumentEntities(BaseModel):
|
||||
entities: list[Entity]
|
||||
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
generator = outlines.generate.json(model, DocumentEntities)
|
||||
|
||||
text = "Tim Cook met with Satya Nadella at Microsoft headquarters in Redmond."
|
||||
prompt = f"Extract entities from: {text}"
|
||||
|
||||
result = generator(prompt)
|
||||
result = DocumentEntities.model_validate_json(model(prompt, DocumentEntities, max_new_tokens=300))
|
||||
for entity in result.entities:
|
||||
print(f"{entity.name} ({entity.type})")
|
||||
```
|
||||
|
|
@ -412,11 +406,8 @@ class PythonFunction(BaseModel):
|
|||
docstring: str
|
||||
body: str
|
||||
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
generator = outlines.generate.json(model, PythonFunction)
|
||||
|
||||
prompt = "Generate a Python function to calculate factorial"
|
||||
func = generator(prompt)
|
||||
func = PythonFunction.model_validate_json(model(prompt, PythonFunction, max_new_tokens=300))
|
||||
|
||||
print(f"def {func.function_name}({', '.join(func.parameters)}):")
|
||||
print(f' """{func.docstring}"""')
|
||||
|
|
@ -426,29 +417,29 @@ print(f" {func.body}")
|
|||
### Pattern 6: Batch Processing
|
||||
|
||||
```python
|
||||
def batch_extract(texts: list[str], schema: type[BaseModel]):
|
||||
"""Extract structured data from multiple texts."""
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
generator = outlines.generate.json(model, schema)
|
||||
|
||||
results = []
|
||||
for text in texts:
|
||||
result = generator(f"Extract from: {text}")
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
import outlines
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
from pydantic import BaseModel
|
||||
|
||||
class Person(BaseModel):
|
||||
name: str
|
||||
age: int
|
||||
|
||||
model = outlines.from_transformers(
|
||||
AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct", device_map="auto"),
|
||||
AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct"),
|
||||
)
|
||||
|
||||
texts = [
|
||||
"John is 30 years old",
|
||||
"Alice is 25 years old",
|
||||
"Bob is 40 years old"
|
||||
"Bob is 40 years old",
|
||||
]
|
||||
|
||||
people = batch_extract(texts, Person)
|
||||
# v1 accepts a list of prompts for batched generation
|
||||
prompts = [f"Extract from: {t}" for t in texts]
|
||||
outputs = model(prompts, Person, max_new_tokens=100)
|
||||
people = [Person.model_validate_json(o) for o in outputs]
|
||||
for person in people:
|
||||
print(f"{person.name}: {person.age}")
|
||||
```
|
||||
|
|
@ -459,58 +450,67 @@ for person in people:
|
|||
|
||||
```python
|
||||
import outlines
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
MODEL_NAME = "microsoft/Phi-3-mini-4k-instruct"
|
||||
|
||||
# Basic usage
|
||||
model = outlines.models.transformers("microsoft/Phi-3-mini-4k-instruct")
|
||||
model = outlines.from_transformers(
|
||||
AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="auto"),
|
||||
AutoTokenizer.from_pretrained(MODEL_NAME),
|
||||
)
|
||||
|
||||
# GPU configuration
|
||||
model = outlines.models.transformers(
|
||||
"microsoft/Phi-3-mini-4k-instruct",
|
||||
device="cuda",
|
||||
model_kwargs={"torch_dtype": "float16"}
|
||||
# GPU + dtype configuration is set on the HF model itself
|
||||
import torch
|
||||
model = outlines.from_transformers(
|
||||
AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="cuda", torch_dtype=torch.float16),
|
||||
AutoTokenizer.from_pretrained(MODEL_NAME),
|
||||
)
|
||||
|
||||
# Popular models
|
||||
model = outlines.models.transformers("meta-llama/Llama-3.1-8B-Instruct")
|
||||
model = outlines.models.transformers("mistralai/Mistral-7B-Instruct-v0.3")
|
||||
model = outlines.models.transformers("Qwen/Qwen2.5-7B-Instruct")
|
||||
for name in [
|
||||
"meta-llama/Llama-3.1-8B-Instruct",
|
||||
"mistralai/Mistral-7B-Instruct-v0.3",
|
||||
"Qwen/Qwen2.5-7B-Instruct",
|
||||
]:
|
||||
model = outlines.from_transformers(
|
||||
AutoModelForCausalLM.from_pretrained(name, device_map="auto"),
|
||||
AutoTokenizer.from_pretrained(name),
|
||||
)
|
||||
```
|
||||
|
||||
### llama.cpp
|
||||
|
||||
```python
|
||||
# Load GGUF model
|
||||
model = outlines.models.llamacpp(
|
||||
"./models/llama-3.1-8b.Q4_K_M.gguf",
|
||||
n_ctx=4096, # Context window
|
||||
n_gpu_layers=35, # GPU layers
|
||||
n_threads=8 # CPU threads
|
||||
)
|
||||
import outlines
|
||||
from llama_cpp import Llama
|
||||
|
||||
# Full GPU offload
|
||||
model = outlines.models.llamacpp(
|
||||
"./models/model.gguf",
|
||||
n_gpu_layers=-1 # All layers on GPU
|
||||
# Load GGUF model
|
||||
llm = Llama(
|
||||
"./models/llama-3.1-8b.Q4_K_M.gguf",
|
||||
n_ctx=4096, # Context window
|
||||
n_gpu_layers=35, # GPU layers
|
||||
n_threads=8, # CPU threads
|
||||
)
|
||||
model = outlines.from_llamacpp(llm)
|
||||
|
||||
# Full GPU offload: set n_gpu_layers=-1 on the Llama object
|
||||
```
|
||||
|
||||
### vLLM (Production)
|
||||
|
||||
```python
|
||||
import outlines
|
||||
from vllm import LLM
|
||||
|
||||
# Single GPU
|
||||
model = outlines.models.vllm("meta-llama/Llama-3.1-8B-Instruct")
|
||||
model = outlines.from_vllm(LLM("meta-llama/Llama-3.1-8B-Instruct"))
|
||||
|
||||
# Multi-GPU
|
||||
model = outlines.models.vllm(
|
||||
"meta-llama/Llama-3.1-70B-Instruct",
|
||||
tensor_parallel_size=4 # 4 GPUs
|
||||
)
|
||||
model = outlines.from_vllm(LLM("meta-llama/Llama-3.1-70B-Instruct", tensor_parallel_size=4))
|
||||
|
||||
# With quantization
|
||||
model = outlines.models.vllm(
|
||||
"meta-llama/Llama-3.1-8B-Instruct",
|
||||
quantization="awq" # Or "gptq"
|
||||
)
|
||||
model = outlines.from_vllm(LLM("meta-llama/Llama-3.1-8B-Instruct", quantization="awq"))
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
|
@ -598,15 +598,23 @@ class Article(BaseModel):
|
|||
# Can succeed even if author/date missing
|
||||
```
|
||||
|
||||
### 6. Always Validate JSON Output
|
||||
|
||||
```python
|
||||
# v1 returns a JSON string for Pydantic/JSON output types.
|
||||
result = model(prompt, Article) # str
|
||||
article = Article.model_validate_json(result) # Article instance
|
||||
```
|
||||
|
||||
## Comparison to Alternatives
|
||||
|
||||
| Feature | Outlines | Instructor | Guidance | LMQL |
|
||||
|---------|----------|------------|----------|------|
|
||||
| Pydantic Support | ✅ Native | ✅ Native | ❌ No | ❌ No |
|
||||
| JSON Schema | ✅ Yes | ✅ Yes | ⚠️ Limited | ✅ Yes |
|
||||
| Pydantic Support | ✅ Native | ✅ Native | ✅ Yes | ❌ No |
|
||||
| JSON Schema | ✅ Yes | ✅ Yes | ✅ Yes | ✅ Yes |
|
||||
| Regex Constraints | ✅ Yes | ❌ No | ✅ Yes | ✅ Yes |
|
||||
| Local Models | ✅ Full | ⚠️ Limited | ✅ Full | ✅ Full |
|
||||
| API Models | ⚠️ Limited | ✅ Full | ✅ Full | ✅ Full |
|
||||
| API Models | ✅ Yes | ✅ Full | ✅ Yes | ✅ Full |
|
||||
| Zero Overhead | ✅ Yes | ❌ No | ⚠️ Partial | ✅ Yes |
|
||||
| Automatic Retrying | ❌ No | ✅ Yes | ❌ No | ❌ No |
|
||||
| Learning Curve | Low | Low | Low | High |
|
||||
|
|
@ -631,19 +639,19 @@ class Article(BaseModel):
|
|||
- **1.2-2x faster** than post-generation validation approaches
|
||||
|
||||
**Memory:**
|
||||
- FSM compiled once per schema (cached)
|
||||
- Automaton compiled once per output type (cached)
|
||||
- Minimal runtime overhead
|
||||
- Efficient with vLLM for high throughput
|
||||
|
||||
**Accuracy:**
|
||||
- **100% valid outputs** (guaranteed by FSM)
|
||||
- **100% valid outputs** (guaranteed by the constrained automaton)
|
||||
- No retry loops needed
|
||||
- Deterministic token filtering
|
||||
|
||||
## Resources
|
||||
|
||||
- **Documentation**: https://outlines-dev.github.io/outlines
|
||||
- **GitHub**: https://github.com/outlines-dev/outlines (8k+ stars)
|
||||
- **Documentation**: https://dottxt-ai.github.io/outlines/
|
||||
- **GitHub**: https://github.com/dottxt-ai/outlines (12k+ stars)
|
||||
- **Discord**: https://discord.gg/R9DSu34mGd
|
||||
- **Blog**: https://blog.dottxt.co
|
||||
|
||||
|
|
@ -652,5 +660,3 @@ class Article(BaseModel):
|
|||
- `references/json_generation.md` - Comprehensive JSON and Pydantic patterns
|
||||
- `references/backends.md` - Backend-specific configuration
|
||||
- `references/examples.md` - Production-ready examples
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
---
|
||||
name: modal-serverless-gpu
|
||||
description: Serverless GPU cloud for ML jobs and model APIs.
|
||||
version: 1.0.0
|
||||
version: 1.0.1
|
||||
author: Orchestra Research
|
||||
license: MIT
|
||||
dependencies: [modal>=0.64.0]
|
||||
dependencies: [modal>=1.0]
|
||||
platforms: [linux, macos, windows]
|
||||
metadata:
|
||||
hermes:
|
||||
|
|
@ -227,7 +227,6 @@ async def batch_predict(inputs: list[str]) -> list[dict]:
|
|||
# Inputs automatically batched
|
||||
return model.batch_predict(inputs)
|
||||
```
|
||||
|
||||
## Secrets management
|
||||
|
||||
```bash
|
||||
|
|
@ -259,10 +258,10 @@ def hourly_job():
|
|||
### Cold start mitigation
|
||||
|
||||
```python
|
||||
@app.function(
|
||||
container_idle_timeout=300, # Keep warm 5 min
|
||||
allow_concurrent_inputs=10, # Handle concurrent requests
|
||||
)
|
||||
# Modal 1.0 autoscaler params: scaledown_window (was container_idle_timeout).
|
||||
# Input concurrency moved to the @modal.concurrent decorator.
|
||||
@app.function(scaledown_window=300) # Keep warm 5 min
|
||||
@modal.concurrent(max_inputs=10) # Handle concurrent requests per container
|
||||
def inference():
|
||||
pass
|
||||
```
|
||||
|
|
@ -304,14 +303,21 @@ def run_parallel():
|
|||
memory=32768, # 32GB RAM
|
||||
cpu=4, # 4 CPU cores
|
||||
timeout=3600, # 1 hour max
|
||||
container_idle_timeout=120,# Keep warm 2 min
|
||||
scaledown_window=120, # Keep warm 2 min (was container_idle_timeout)
|
||||
retries=3, # Retry on failure
|
||||
concurrency_limit=10, # Max concurrent containers
|
||||
max_containers=10, # Max concurrent containers (was concurrency_limit)
|
||||
min_containers=1, # Keep N containers warm (was keep_warm)
|
||||
)
|
||||
def my_function():
|
||||
pass
|
||||
```
|
||||
|
||||
> **Modal 1.0 autoscaler renames** (see the [migration guide](https://modal.com/docs/guide/modal-1-0-migration)):
|
||||
> - `container_idle_timeout` → `scaledown_window`
|
||||
> - `concurrency_limit` → `max_containers`
|
||||
> - `keep_warm` → `min_containers`
|
||||
> - `allow_concurrent_inputs=N` → the `@modal.concurrent(max_inputs=N)` decorator
|
||||
|
||||
## Debugging
|
||||
|
||||
```python
|
||||
|
|
@ -327,7 +333,7 @@ if __name__ == "__main__":
|
|||
|
||||
| Issue | Solution |
|
||||
|-------|----------|
|
||||
| Cold start latency | Increase `container_idle_timeout`, use `@modal.enter()` |
|
||||
| Cold start latency | Increase `scaledown_window`, use `@modal.enter()` |
|
||||
| GPU OOM | Use larger GPU (`A100-80GB`), enable gradient checkpointing |
|
||||
| Image build fails | Pin dependency versions, check CUDA compatibility |
|
||||
| Timeout errors | Increase `timeout`, add checkpointing |
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
---
|
||||
name: pinecone
|
||||
description: Managed vector DB for production RAG and search.
|
||||
version: 1.0.0
|
||||
version: 1.0.1
|
||||
author: Orchestra Research
|
||||
license: MIT
|
||||
dependencies: [pinecone-client]
|
||||
dependencies: [pinecone]
|
||||
platforms: [linux, macos, windows]
|
||||
metadata:
|
||||
hermes:
|
||||
|
|
@ -42,9 +42,11 @@ The vector database for production AI applications.
|
|||
### Installation
|
||||
|
||||
```bash
|
||||
pip install pinecone-client
|
||||
pip install pinecone
|
||||
```
|
||||
|
||||
> Note: the old `pinecone-client` package is deprecated. Install `pinecone` (v5+; current 9.x). The import stays `from pinecone import Pinecone`.
|
||||
|
||||
### Basic usage
|
||||
|
||||
```python
|
||||
|
|
@ -226,14 +228,31 @@ index.upsert(vectors=[
|
|||
])
|
||||
|
||||
# Hybrid query
|
||||
# NOTE: index.query() does NOT accept an `alpha` kwarg. Pinecone stores a
|
||||
# single sparse-dense vector, so weighting must be applied by pre-scaling the
|
||||
# query vectors before sending them. Use the hybrid_score_norm helper below
|
||||
# (alpha * dense + (1 - alpha) * sparse; alpha=1 → pure dense, 0 → pure sparse).
|
||||
|
||||
def hybrid_score_norm(dense, sparse, alpha: float):
|
||||
"""Scale dense/sparse query vectors for weighted hybrid search."""
|
||||
if not 0 <= alpha <= 1:
|
||||
raise ValueError("alpha must be between 0 and 1")
|
||||
scaled_sparse = {
|
||||
"indices": sparse["indices"],
|
||||
"values": [v * (1 - alpha) for v in sparse["values"]],
|
||||
}
|
||||
return [v * alpha for v in dense], scaled_sparse
|
||||
|
||||
hdense, hsparse = hybrid_score_norm(
|
||||
dense=[0.1, 0.2, ...],
|
||||
sparse={"indices": [10, 45], "values": [0.5, 0.3]},
|
||||
alpha=0.5, # 0=sparse, 1=dense, 0.5=balanced
|
||||
)
|
||||
|
||||
results = index.query(
|
||||
vector=[0.1, 0.2, ...],
|
||||
sparse_vector={
|
||||
"indices": [10, 45],
|
||||
"values": [0.5, 0.3]
|
||||
},
|
||||
vector=hdense,
|
||||
sparse_vector=hsparse,
|
||||
top_k=5,
|
||||
alpha=0.5 # 0=sparse, 1=dense, 0.5=hybrid
|
||||
)
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
---
|
||||
name: qdrant-vector-search
|
||||
description: Vector search engine for production RAG systems.
|
||||
version: 1.0.0
|
||||
version: 1.0.1
|
||||
author: Orchestra Research
|
||||
license: MIT
|
||||
dependencies: [qdrant-client>=1.12.0]
|
||||
dependencies: [qdrant-client>=1.14.0]
|
||||
platforms: [linux, macos, windows]
|
||||
metadata:
|
||||
hermes:
|
||||
|
|
@ -89,17 +89,17 @@ client.upsert(
|
|||
]
|
||||
)
|
||||
|
||||
# Search with filtering
|
||||
results = client.search(
|
||||
# Search with filtering (query_points is the current API; client.search is removed in qdrant-client 1.14+)
|
||||
response = client.query_points(
|
||||
collection_name="documents",
|
||||
query_vector=[0.15, 0.25, ...],
|
||||
query=[0.15, 0.25, ...],
|
||||
query_filter={
|
||||
"must": [{"key": "category", "match": {"value": "tech"}}]
|
||||
},
|
||||
limit=10
|
||||
)
|
||||
|
||||
for point in results:
|
||||
for point in response.points:
|
||||
print(f"ID: {point.id}, Score: {point.score}, Payload: {point.payload}")
|
||||
```
|
||||
|
||||
|
|
@ -169,14 +169,15 @@ print(f"Points: {info.points_count}, Vectors: {info.vectors_count}")
|
|||
### Basic search
|
||||
|
||||
```python
|
||||
# Simple nearest neighbor search
|
||||
results = client.search(
|
||||
# Simple nearest neighbor search (returns a QueryResponse; use .points)
|
||||
response = client.query_points(
|
||||
collection_name="documents",
|
||||
query_vector=[0.1, 0.2, ...],
|
||||
query=[0.1, 0.2, ...],
|
||||
limit=10,
|
||||
with_payload=True,
|
||||
with_vectors=False # Don't return vectors (faster)
|
||||
)
|
||||
results = response.points
|
||||
```
|
||||
|
||||
### Filtered search
|
||||
|
|
@ -185,9 +186,9 @@ results = client.search(
|
|||
from qdrant_client.models import Filter, FieldCondition, MatchValue, Range
|
||||
|
||||
# Complex filtering
|
||||
results = client.search(
|
||||
response = client.query_points(
|
||||
collection_name="documents",
|
||||
query_vector=query_embedding,
|
||||
query=query_embedding,
|
||||
query_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(key="category", match=MatchValue(value="tech")),
|
||||
|
|
@ -198,12 +199,12 @@ results = client.search(
|
|||
]
|
||||
),
|
||||
limit=10
|
||||
)
|
||||
).points
|
||||
|
||||
# Shorthand filter syntax
|
||||
results = client.search(
|
||||
response = client.query_points(
|
||||
collection_name="documents",
|
||||
query_vector=query_embedding,
|
||||
query=query_embedding,
|
||||
query_filter={
|
||||
"must": [
|
||||
{"key": "category", "match": {"value": "tech"}},
|
||||
|
|
@ -211,23 +212,27 @@ results = client.search(
|
|||
]
|
||||
},
|
||||
limit=10
|
||||
)
|
||||
).points
|
||||
```
|
||||
|
||||
### Batch search
|
||||
|
||||
```python
|
||||
from qdrant_client.models import SearchRequest
|
||||
from qdrant_client.models import QueryRequest
|
||||
|
||||
# Multiple queries in one request
|
||||
results = client.search_batch(
|
||||
# Multiple queries in one request (search_batch is replaced by query_batch_points)
|
||||
responses = client.query_batch_points(
|
||||
collection_name="documents",
|
||||
requests=[
|
||||
SearchRequest(vector=[0.1, ...], limit=5),
|
||||
SearchRequest(vector=[0.2, ...], limit=5, filter={"must": [...]}),
|
||||
SearchRequest(vector=[0.3, ...], limit=10)
|
||||
QueryRequest(query=[0.1, ...], limit=5),
|
||||
QueryRequest(query=[0.2, ...], limit=5, filter={"must": [...]}),
|
||||
QueryRequest(query=[0.3, ...], limit=10)
|
||||
]
|
||||
)
|
||||
# Each element is a QueryResponse; use .points
|
||||
for resp in responses:
|
||||
for point in resp.points:
|
||||
print(point.id, point.score)
|
||||
```
|
||||
|
||||
## RAG integration
|
||||
|
|
@ -268,12 +273,12 @@ client.upsert(collection_name="knowledge_base", points=points)
|
|||
# RAG retrieval
|
||||
def retrieve(query: str, top_k: int = 5) -> list[dict]:
|
||||
query_vector = encoder.encode(query).tolist()
|
||||
results = client.search(
|
||||
response = client.query_points(
|
||||
collection_name="knowledge_base",
|
||||
query_vector=query_vector,
|
||||
query=query_vector,
|
||||
limit=top_k
|
||||
)
|
||||
return [{"text": r.payload["text"], "score": r.score} for r in results]
|
||||
return [{"text": r.payload["text"], "score": r.score} for r in response.points]
|
||||
|
||||
# Use in RAG pipeline
|
||||
context = retrieve("What is Python?")
|
||||
|
|
@ -334,12 +339,14 @@ client.upsert(
|
|||
]
|
||||
)
|
||||
|
||||
# Search specific vector
|
||||
results = client.search(
|
||||
# Search specific named vector (pass the vector name via `using`)
|
||||
response = client.query_points(
|
||||
collection_name="hybrid_search",
|
||||
query_vector=("dense", query_dense), # Specify which vector
|
||||
query=query_dense,
|
||||
using="dense", # Specify which named vector to search
|
||||
limit=10
|
||||
)
|
||||
results = response.points
|
||||
```
|
||||
|
||||
### Sparse vectors (BM25, SPLADE)
|
||||
|
|
@ -380,12 +387,13 @@ client.create_collection(
|
|||
)
|
||||
|
||||
# Search with rescoring
|
||||
results = client.search(
|
||||
response = client.query_points(
|
||||
collection_name="quantized",
|
||||
query_vector=query,
|
||||
query=query,
|
||||
search_params={"quantization": {"rescore": True}}, # Rescore top results
|
||||
limit=10
|
||||
)
|
||||
results = response.points
|
||||
```
|
||||
|
||||
## Payload indexing
|
||||
|
|
@ -493,5 +501,5 @@ client = QdrantClient(
|
|||
- **Docs**: https://qdrant.tech/documentation/
|
||||
- **Python Client**: https://github.com/qdrant/qdrant-client
|
||||
- **Cloud**: https://cloud.qdrant.io
|
||||
- **Version**: 1.12.0+
|
||||
- **Version**: 1.14.0+
|
||||
- **License**: Apache 2.0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue