Chapter 9: Feedback

async function evaluateOnly(task: string) {
  const output = await generate(task);
  const score = await evaluate(task, output); // e.g. number from 0 to 1

  // Score is logged, but not used to change anything
  console.log('Score:', score);
  return output;
}

type Issue = {
  description: string;
  location?: string;      // e.g. "line 23" or "paragraph 2"
  suggestion?: string;    // what to change
};

type EvalResult = {
  score: number;
  issues: Issue[];
};

async function generateWithFeedback(task: string, maxIters = 3) {
  let output = await generate(task);

  for (let i = 0; i < maxIters; i++) {
    const evalResult: EvalResult = await evaluateDetailed(task, output);

    if (evalResult.score >= 0.9 || evalResult.issues.length === 0) {
      return output; // good enough
    }

    output = await improveWithIssues(task, output, evalResult.issues);
  }

  return output;
}

async function improveWithIssues(task: string, output: string, issues: Issue[]) {
  const issueText = issues.map((issue, i) => {
    return `${i + 1}. ${issue.description}
Location: ${issue.location ?? 'unspecified'}
Suggested fix: ${issue.suggestion ?? 'not specified'}`;
  }).join('\n\n');

  const prompt = `
You previously attempted this task:

TASK:
${task}

PREVIOUS OUTPUT:
${output}

IDENTIFIED ISSUES:
${issueText}

INSTRUCTION:
Produce a revised version that fixes these issues.
Preserve everything that is already correct.
Only change what is necessary to address the issues.`;

  return await llmComplete({
    prompt,
    maxTokens: 1024,
    temperature: 0.2
  });
}

type FeedbackSource = 'tests' | 'facts' | 'model-accuracy' | 'model-clarity';

type TrustedIssue = Issue & {
  source: FeedbackSource;
  severity: 'critical' | 'major' | 'minor';
};

type MultiEvalResult = {
  score: number;
  issues: TrustedIssue[];
};

async function evaluateWithSources(task: string, output: string): Promise<MultiEvalResult> {
  const [testIssues, factIssues, modelIssues] = await Promise.all([
    runTestSuite(task, output),          // external tests, if any
    checkFacts(task, output),            // external knowledge, if any
    modelReview(task, output)            // LLM-based critique
  ]);

  const allIssues: TrustedIssue[] = [
    ...testIssues.map(i => ({ ...i, source: 'tests', severity: 'critical' })),
    ...factIssues.map(i => ({ ...i, source: 'facts', severity: 'critical' })),
    ...modelIssues.map(i => ({
      ...i,
      source: i.type === 'accuracy' ? 'model-accuracy' : 'model-clarity',
      severity: i.type === 'accuracy' ? 'major' : 'minor'
    }))
  ];

  const score = scoreFromIssues(allIssues); // e.g. lower with more severe issues

  return { score, issues: prioritizeIssues(allIssues) };
}

async function modelReview(task: string, output: string): Promise<Issue[]> {
  const reviewPrompt = `
You are reviewing an output for the following task:

TASK:
${task}

OUTPUT TO REVIEW:
${output}

Identify any issues with:
1. Factual accuracy
2. Logical consistency
3. Clarity and structure

For each issue, provide:
- category: "accuracy" | "logic" | "clarity"
- description
- location (if specific)
- suggested fix
Respond as JSON array of issues.`;

  const reviewText = await llmComplete(reviewPrompt);
  return JSON.parse(reviewText);
}

function prioritizeIssues(issues: TrustedIssue[]): TrustedIssue[] {
  const order = { critical: 0, major: 1, minor: 2 } as const;

  return issues
    .sort((a, b) => order[a.severity] - order[b.severity]);
}

type IterationRecord = {
  iteration: number;
  score: number;
  output: string;
};

type IterationResult = {
  output: string;
  converged: boolean;
  reason: 'quality_threshold' | 'no_improvement' | 'max_iterations';
  history: IterationRecord[];
};

async function iterateToQuality(
  task: string,
  initialOutput?: string,
  options = { targetScore: 0.9, maxIterations: 4, minDelta: 0.02 }
): Promise<IterationResult> {
  let currentOutput = initialOutput ?? await generate(task);
  let history: IterationRecord[] = [];
  let bestOutput = currentOutput;
  let bestScore = 0;

  for (let i = 0; i < options.maxIterations; i++) {
    const evalResult = await evaluateWithSources(task, currentOutput);
    const { score, issues } = evalResult;

    history.push({ iteration: i, score, output: currentOutput });

    if (score > bestScore) {
      bestScore = score;
      bestOutput = currentOutput;
    }

    // 1. Stop if good enough
    if (score >= options.targetScore) {
      return {
        output: currentOutput,
        converged: true,
        reason: 'quality_threshold',
        history
      };
    }

    // 2. Stop if improvement stalls
    const prev = history[history.length - 2];
    if (prev && score - prev.score < options.minDelta) {
      return {
        output: bestOutput,
        converged: true,
        reason: 'no_improvement',
        history
      };
    }

    // 3. Otherwise, improve using feedback and continue
    currentOutput = await improveWithIssues(task, currentOutput, issues);
  }

  // 4. Stop if out of iterations
  return {
    output: bestOutput,
    converged: false,
    reason: 'max_iterations',
    history
  };
}

type Cost = {
  promptTokens: number;
  completionTokens: number;
  wallTimeMs: number;
};

type EvalWithCost = {
  score: number;
  issues: Issue[];
  cost: Cost;
};

type IterationOptions = {
  targetScore: number;
  maxIterations: number;
  maxTotalTokens: number;
};

async function iterateWithBudget(
  task: string,
  options: IterationOptions
) {
  let output = await generate(task);
  let totalTokens = 0;
  let bestScore = 0;
  let bestOutput = output;

  for (let i = 0; i < options.maxIterations; i++) {
    const evalResult: EvalWithCost = await evaluateWithCost(task, output);
    const { score, issues, cost } = evalResult;

    totalTokens += cost.promptTokens + cost.completionTokens;

    if (score > bestScore) {
      bestScore = score;
      bestOutput = output;
    }

    if (score >= options.targetScore) {
      break; // quality threshold reached
    }

    if (totalTokens >= options.maxTotalTokens) {
      break; // budget exhausted
    }

    // Optional: if remaining budget is tiny, expect negligible improvement
    const remainingTokens = options.maxTotalTokens - totalTokens;
    if (remainingTokens < estimatedTokensForOneMoreIteration(task)) {
      break;
    }

    output = await improveWithIssues(task, output, issues);
  }

  return { output: bestOutput, score: bestScore, totalTokens };
}