Embedding Product Catalogs

The quality of your RAG system depends on how well you embed your products. A poorly structured embedding means "wireless earbuds for running" won't match "Bluetooth sports headphones" - even though they're the same thing.

Product Data Structure

A typical e-commerce product has multiple fields. The key is knowing what to embed vs. what to store as metadata.

# Example product from your database
product = {
    "id": "SKU-12345",
    "name": "Sony WH-1000XM5 Headphones",
    "brand": "Sony",
    "category": "Electronics > Audio > Headphones",
    "price": 349.99,
    "rating": 4.7,
    "review_count": 2847,
    "in_stock": True,
    "description": """
        Industry-leading noise cancellation with Auto NC Optimizer.
        Crystal clear hands-free calling with 4 beamforming microphones.
        Up to 30-hour battery life with quick charging (3 min charge = 3 hours).
        Multipoint connection pairs with two Bluetooth devices simultaneously.
        Ultra-comfortable with new synthetic leather design.
    """,
    "features": [
        "Active Noise Cancellation",
        "30-hour battery",
        "Bluetooth 5.2",
        "Touch controls",
        "Foldable design"
    ],
    "color": "Black",
    "weight": "250g"
}

What to Embed vs. Store as Metadata

Creating Rich Product Embeddings

from openai import OpenAI
from typing import Dict, List

client = OpenAI()

def create_product_embedding_text(product: Dict) -> str:
    """Create optimized text for embedding a product"""

    # Combine name and brand
    text_parts = [f"{product['brand']} {product['name']}"]

    # Add category for context
    if product.get('category'):
        text_parts.append(f"Category: {product['category']}")

    # Add description
    if product.get('description'):
        text_parts.append(product['description'].strip())

    # Add features as a list
    if product.get('features'):
        features_text = "Features: " + ", ".join(product['features'])
        text_parts.append(features_text)

    # Add any specific use cases or target audience
    if product.get('use_cases'):
        text_parts.append(f"Great for: {', '.join(product['use_cases'])}")

    return "\n".join(text_parts)


def embed_product(product: Dict) -> Dict:
    """Embed a product and return vector with metadata"""

    # Create text for embedding
    embedding_text = create_product_embedding_text(product)

    # Generate embedding
    response = client.embeddings.create(
        model="text-embedding-3-small",
        input=embedding_text
    )

    return {
        "id": product["id"],
        "values": response.data[0].embedding,
        "metadata": {
            "name": product["name"],
            "brand": product["brand"],
            "category": product["category"],
            "price": product["price"],
            "rating": product.get("rating", 0),
            "in_stock": product.get("in_stock", True),
            "color": product.get("color"),
            "embedding_text": embedding_text[:500]  # Store truncated for debugging
        }
    }


# Example usage
product = {
    "id": "SKU-12345",
    "name": "WH-1000XM5 Headphones",
    "brand": "Sony",
    "category": "Electronics > Audio > Headphones",
    "price": 349.99,
    "rating": 4.7,
    "in_stock": True,
    "description": "Industry-leading noise cancellation with up to 30-hour battery life.",
    "features": ["Active Noise Cancellation", "30-hour battery", "Bluetooth 5.2"],
    "use_cases": ["Travel", "Work from home", "Music production"]
}

embedded = embed_product(product)
print(f"Embedding dimensions: {len(embedded['values'])}")
print(f"Metadata: {embedded['metadata']}")

Handling Long Product Descriptions

Some products have extensive descriptions. Use chunking to capture all content.

from langchain.text_splitter import RecursiveCharacterTextSplitter

def chunk_product_description(product: Dict, chunk_size: int = 500) -> List[Dict]:
    """Split long product descriptions into chunks"""

    # Create base text
    base_text = f"{product['brand']} {product['name']}\nCategory: {product['category']}\n\n"

    # Combine description and features
    full_description = product.get('description', '')
    if product.get('features'):
        full_description += "\n\nFeatures:\n" + "\n".join(f"- {f}" for f in product['features'])

    # If short enough, return single chunk
    if len(base_text + full_description) <= chunk_size:
        return [{"id": product["id"], "text": base_text + full_description, "chunk": 0}]

    # Split into chunks
    splitter = RecursiveCharacterTextSplitter(
        chunk_size=chunk_size,
        chunk_overlap=50,
        separators=["\n\n", "\n", ". ", " "]
    )

    chunks = splitter.split_text(full_description)

    return [
        {
            "id": f"{product['id']}_chunk_{i}",
            "parent_id": product["id"],
            "text": base_text + chunk,
            "chunk": i
        }
        for i, chunk in enumerate(chunks)
    ]


# Example: Long product description
product_with_long_desc = {
    "id": "CHAIR-001",
    "name": "Ergonomic Office Chair",
    "brand": "Herman Miller",
    "category": "Furniture > Office > Chairs",
    "price": 1395.00,
    "description": """
        The Aeron Chair has been remastered with over 30 years of research
        and development. Updated with new 8Z Pellicle zones that provide
        varying tension across different parts of your body...

        [Imagine 2000 more characters of detailed specs]
    """,
    "features": ["8Z Pellicle suspension", "PostureFit SL", "Adjustable arms"]
}

chunks = chunk_product_description(product_with_long_desc, chunk_size=400)
print(f"Split into {len(chunks)} chunks")

Batch Embedding for Large Catalogs

import time
from typing import List, Dict
from tqdm import tqdm

def batch_embed_products(products: List[Dict], batch_size: int = 100) -> List[Dict]:
    """Efficiently embed a large product catalog"""

    all_vectors = []

    # Process in batches
    for i in tqdm(range(0, len(products), batch_size)):
        batch = products[i:i + batch_size]

        # Create embedding texts for batch
        texts = [create_product_embedding_text(p) for p in batch]

        # Batch embed (OpenAI supports up to 2048 items per request)
        response = client.embeddings.create(
            model="text-embedding-3-small",
            input=texts
        )

        # Combine embeddings with metadata
        for product, embedding_data in zip(batch, response.data):
            all_vectors.append({
                "id": product["id"],
                "values": embedding_data.embedding,
                "metadata": {
                    "name": product["name"],
                    "brand": product["brand"],
                    "category": product["category"],
                    "price": product["price"],
                    "rating": product.get("rating", 0),
                    "in_stock": product.get("in_stock", True)
                }
            })

        # Rate limiting (OpenAI allows 3000 RPM for embeddings)
        time.sleep(0.1)

    return all_vectors


# Index a catalog
products = load_products_from_database()  # Your function
vectors = batch_embed_products(products)

# Upsert to Pinecone
index.upsert(vectors=vectors, batch_size=100)
print(f"Indexed {len(vectors)} products")

Metadata Filtering in Action

def search_products_with_filters(
    query: str,
    min_price: float = None,
    max_price: float = None,
    category: str = None,
    brand: str = None,
    in_stock_only: bool = True,
    min_rating: float = None,
    top_k: int = 10
) -> List[Dict]:
    """Search with semantic query and metadata filters"""

    # Build filter
    filters = {}

    if in_stock_only:
        filters["in_stock"] = True

    if min_price is not None or max_price is not None:
        filters["price"] = {}
        if min_price:
            filters["price"]["$gte"] = min_price
        if max_price:
            filters["price"]["$lte"] = max_price

    if category:
        filters["category"] = {"$eq": category}

    if brand:
        filters["brand"] = {"$eq": brand}

    if min_rating:
        filters["rating"] = {"$gte": min_rating}

    # Embed query
    query_embedding = client.embeddings.create(
        model="text-embedding-3-small",
        input=query
    ).data[0].embedding

    # Search with filters
    results = index.query(
        vector=query_embedding,
        top_k=top_k,
        include_metadata=True,
        filter=filters if filters else None
    )

    return results.matches


# Example: Complex filtered search
results = search_products_with_filters(
    query="comfortable headphones for long flights",
    max_price=300,
    category="Electronics > Audio > Headphones",
    in_stock_only=True,
    min_rating=4.0
)

for match in results:
    print(f"{match.metadata['name']} - ${match.metadata['price']} ({match.score:.3f})")

Sony WH-1000XM4 - $248.00 (0.891)
Bose QuietComfort 45 - $279.00 (0.874)
Sennheiser Momentum 4 - $299.00 (0.856)

Handling Product Variants

Products often have variants (size, color). Here's how to handle them:

def embed_product_with_variants(base_product: Dict, variants: List[Dict]) -> List[Dict]:
    """Embed a product and its variants"""

    vectors = []

    # Create base embedding text
    base_text = create_product_embedding_text(base_product)

    # Embed base product
    base_embedding = client.embeddings.create(
        model="text-embedding-3-small",
        input=base_text
    ).data[0].embedding

    vectors.append({
        "id": base_product["id"],
        "values": base_embedding,
        "metadata": {
            "name": base_product["name"],
            "brand": base_product["brand"],
            "category": base_product["category"],
            "price": base_product["price"],
            "is_variant": False,
            "parent_id": None
        }
    })

    # Embed variants with additional context
    for variant in variants:
        variant_text = f"{base_text}\nVariant: {variant['color']}, Size {variant['size']}"

        variant_embedding = client.embeddings.create(
            model="text-embedding-3-small",
            input=variant_text
        ).data[0].embedding

        vectors.append({
            "id": variant["id"],
            "values": variant_embedding,
            "metadata": {
                "name": f"{base_product['name']} - {variant['color']}",
                "brand": base_product["brand"],
                "category": base_product["category"],
                "price": variant.get("price", base_product["price"]),
                "color": variant["color"],
                "size": variant["size"],
                "in_stock": variant["in_stock"],
                "is_variant": True,
                "parent_id": base_product["id"]
            }
        })

    return vectors

Key Takeaways

Next up: Building Product Search with RAG - Putting embeddings to work for semantic product discovery.