Task 3: Generate Structured Feedback

Real interview feedback needs structure:

• Numerical scores
• Specific strengths
• Areas for improvement
• Actionable suggestions

Let's use LangChain's Output Parsers to get structured JSON instead of free text.

The Problem with Unstructured Output

# Free-form feedback is hard to use programmatically
feedback = "The candidate showed good understanding of Python basics,
            particularly with list comprehensions. However, they struggled
            with explaining decorators. I'd rate them 7/10..."

# How do you extract the score? The strengths? 🤷

Solution: Pydantic Output Parser

Define your output structure with Pydantic, and LangChain ensures the LLM returns valid JSON.

Step 1: Define the Feedback Schema

# chains/evaluator.py
from pydantic import BaseModel, Field
from typing import List, Optional

class AnswerFeedback(BaseModel):
    """Structured feedback for a single answer."""

    score: int = Field(
        description="Score from 1-10 (1=poor, 10=excellent)",
        ge=1,
        le=10
    )
    understanding: str = Field(
        description="Assessment of conceptual understanding"
    )
    communication: str = Field(
        description="How well they explained their answer"
    )
    strengths: List[str] = Field(
        description="Specific things the candidate did well"
    )
    improvements: List[str] = Field(
        description="Specific areas to improve"
    )
    follow_up_question: Optional[str] = Field(
        description="A follow-up question to probe deeper",
        default=None
    )

class InterviewReport(BaseModel):
    """Final interview evaluation report."""

    overall_score: int = Field(ge=1, le=10)
    recommendation: str = Field(
        description="hire / maybe / no_hire"
    )
    summary: str = Field(
        description="2-3 sentence overall assessment"
    )
    technical_skills: int = Field(ge=1, le=10)
    communication_skills: int = Field(ge=1, le=10)
    problem_solving: int = Field(ge=1, le=10)
    strengths: List[str]
    areas_to_improve: List[str]
    suggested_topics_to_study: List[str]

Step 2: Create the Evaluation Chain

# chains/evaluator.py (continued)
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import PydanticOutputParser

def create_evaluator_chain():
    """Create a chain that returns structured feedback."""

    # Create parser from our Pydantic model
    parser = PydanticOutputParser(pydantic_object=AnswerFeedback)

    prompt = ChatPromptTemplate.from_messages([
        ("system", """You are an expert interview evaluator.

Evaluate the candidate's answer to the given question.
Be specific and constructive in your feedback.

Question context: {question}
Position level: {level}

{format_instructions}
"""),
        ("human", "Candidate's answer: {answer}")
    ])

    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.3)

    chain = prompt | llm | parser

    return chain, parser

# Usage
evaluator, parser = create_evaluator_chain()

feedback = evaluator.invoke({
    "question": "Explain the difference between lists and tuples in Python",
    "level": "senior",
    "answer": "Lists are mutable and tuples are immutable. You use lists when you need to modify the collection.",
    "format_instructions": parser.get_format_instructions()
})

print(f"Score: {feedback.score}/10")
print(f"Strengths: {feedback.strengths}")
print(f"Improvements: {feedback.improvements}")

AnswerFeedback(
    score=6,
    understanding="Shows basic understanding but lacks depth",
    communication="Clear but too brief for a senior position",
    strengths=["Correctly identified mutability difference"],
    improvements=[
        "Explain when immutability matters (hashability, dict keys)",
        "Discuss performance implications",
        "Give concrete use case examples"
    ],
    follow_up_question="Can you explain why tuples can be used as dictionary keys but lists cannot?"
)

Step 3: Using with_structured_output (Simpler!)

LangChain now has a cleaner approach:

# chains/evaluator.py (modern approach)
from langchain_openai import ChatOpenAI

def create_evaluator_simple():
    """Use with_structured_output for cleaner code."""

    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.3)

    # Automatically handles the output parsing
    structured_llm = llm.with_structured_output(AnswerFeedback)

    prompt = ChatPromptTemplate.from_messages([
        ("system", """You are an expert interview evaluator.
Evaluate the candidate's answer to the given question.
Be specific and constructive in your feedback.

Question: {question}
Position level: {level}"""),
        ("human", "{answer}")
    ])

    chain = prompt | structured_llm

    return chain

Step 4: Generate Final Report

# chains/evaluator.py (continued)
def create_report_generator():
    """Generate final interview report."""

    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.3)
    structured_llm = llm.with_structured_output(InterviewReport)

    prompt = ChatPromptTemplate.from_messages([
        ("system", """You are generating a final interview evaluation report.

Based on the interview transcript below, provide a comprehensive assessment.
Be fair, specific, and constructive.

Position: {position}
Level: {level}
Interview type: {interview_type}
"""),
        ("human", """Interview transcript:
{transcript}

Individual question scores: {scores}

Generate the final report.""")
    ])

    return prompt | structured_llm

# Usage
report_generator = create_report_generator()

report = report_generator.invoke({
    "position": "Senior Python Developer",
    "level": "senior",
    "interview_type": "technical",
    "transcript": """
Q1: Explain Python decorators.
A1: Decorators are functions that wrap other functions...

Q2: What are generators?
A2: Generators use yield to produce values lazily...
""",
    "scores": [7, 8, 6, 7]
})

print(f"Overall: {report.overall_score}/10")
print(f"Recommendation: {report.recommendation}")
print(f"Summary: {report.summary}")

Step 5: Real-time Feedback Integration

Combine with our interviewer:

# main.py
from chains.interviewer import create_interviewer_with_history
from chains.evaluator import create_evaluator_simple, create_report_generator

def run_interview_with_feedback():
    interviewer = create_interviewer_with_history()
    evaluator = create_evaluator_simple()

    session_id = "interview_001"
    config = {"configurable": {"session_id": session_id}}

    interview_config = {
        "interview_type": "technical Python",
        "level": "senior",
        "focus_area": "Python internals"
    }

    scores = []
    transcript = []

    # Start interview
    question = interviewer.invoke(
        {**interview_config, "input": "Start the interview"},
        config=config
    )
    print(f"\nInterviewer: {question}\n")
    current_question = question

    for i in range(5):  # 5 questions
        answer = input("You: ")
        if answer.lower() == 'quit':
            break

        # Evaluate the answer
        feedback = evaluator.invoke({
            "question": current_question,
            "level": "senior",
            "answer": answer
        })

        scores.append(feedback.score)
        transcript.append(f"Q: {current_question}\nA: {answer}")

        # Show score
        print(f"\n[Score: {feedback.score}/10 - {feedback.understanding}]")
        if feedback.improvements:
            print(f"[Tip: {feedback.improvements[0]}]")

        # Get next question
        next_input = f"The candidate answered: {answer}"
        if feedback.follow_up_question:
            next_input += f"\n\nConsider asking this follow-up: {feedback.follow_up_question}"

        question = interviewer.invoke(
            {**interview_config, "input": next_input},
            config=config
        )
        print(f"\nInterviewer: {question}\n")
        current_question = question

    # Generate final report
    print("\n" + "=" * 50)
    print("INTERVIEW REPORT")
    print("=" * 50)

    report_gen = create_report_generator()
    report = report_gen.invoke({
        "position": "Senior Python Developer",
        "level": "senior",
        "interview_type": "technical",
        "transcript": "\n\n".join(transcript),
        "scores": scores
    })

    print(f"\nOverall Score: {report.overall_score}/10")
    print(f"Recommendation: {report.recommendation.upper()}")
    print(f"\nSummary: {report.summary}")
    print(f"\nStrengths:")
    for s in report.strengths:
        print(f"  ✓ {s}")
    print(f"\nAreas to Improve:")
    for a in report.areas_to_improve:
        print(f"  • {a}")

Handling Parsing Errors

Sometimes the LLM returns malformed output:

from langchain_core.exceptions import OutputParserException

def safe_evaluate(evaluator, question, answer, level):
    """Evaluate with error handling."""
    try:
        return evaluator.invoke({
            "question": question,
            "level": level,
            "answer": answer
        })
    except OutputParserException as e:
        # Return a default feedback
        return AnswerFeedback(
            score=5,
            understanding="Could not evaluate - please try again",
            communication="N/A",
            strengths=["Answer was received"],
            improvements=["Please provide more detail"]
        )

Key Takeaways

• Pydantic models define the structure you want
• PydanticOutputParser instructs the LLM and parses output
• with_structured_output() is the modern, cleaner approach
• Structured output enables programmatic use of LLM responses
• Always handle parsing errors gracefully

Next Up