Feedback from review

spong · spong · commit 893d1fabd3bf · 2026-01-09T10:21:57.000-07:00
diff --git a/x-pack/platform/packages/shared/agent-builder/kbn-evals-suite-agent-builder/evals/product_documentation/product_documentation.spec.ts b/x-pack/platform/packages/shared/agent-builder/kbn-evals-suite-agent-builder/evals/product_documentation/product_documentation.spec.ts
@@ -182,7 +182,6 @@ evaluate.describe('AgentBuilder product documentation tool', { tag: '@svlOblt' }
               metadata: {
                 agentId: productDocAgentId,
                 expectedOnlyToolId: platformCoreTools.productDocumentation,
-                requireOnlyFromToolOutput: true,
                 product: 'kibana',
               },
             },
diff --git a/x-pack/platform/packages/shared/agent-builder/kbn-evals-suite-agent-builder/src/evaluate_dataset.ts b/x-pack/platform/packages/shared/agent-builder/kbn-evals-suite-agent-builder/src/evaluate_dataset.ts
@@ -23,18 +23,16 @@ import type { ExperimentTask } from '@kbn/evals/src/types';
 import type { TaskOutput } from '@arizeai/phoenix-client/dist/esm/types/experiments';
 import type { EsClient } from '@kbn/scout';
 import type { ToolingLog } from '@kbn/tooling-log';
-import type { AgentBuilderEvaluationChatClient } from './chat_client';
 import {
-  containsAllTerms,
   extractAllStrings,
   extractMaxSemver,
   extractReleaseDateNearVersion,
   getBooleanMeta,
   getFinalAssistantMessage,
   getStringMeta,
   getToolCallSteps,
-  includesOneOf,
-} from './evaluate_dataset_utils';
+} from '@kbn/evals';
+import type { AgentBuilderEvaluationChatClient } from './chat_client';
 
 interface DatasetExample extends Example {
   input: {
@@ -185,52 +183,6 @@ export function createEvaluateDataset({
             };
           },
         },
-        {
-          name: 'OnlyFromToolOutputHeuristic',
-          kind: 'CODE' as const,
-          evaluate: async ({ output, metadata }) => {
-            if (!getBooleanMeta(metadata, 'requireOnlyFromToolOutput')) return { score: 1 };
-
-            const expectedOnlyToolId = getStringMeta(metadata, 'expectedOnlyToolId');
-            const toolCalls = getToolCallSteps(output as TaskOutput);
-            const matching = expectedOnlyToolId
-              ? toolCalls.filter((t) => t.tool_id === expectedOnlyToolId)
-              : toolCalls;
-
-            const strings: string[] = [];
-            for (const call of matching) {
-              extractAllStrings(call.results, strings);
-            }
-            const toolText = strings.join('\n');
-            const answer = getFinalAssistantMessage(output as TaskOutput);
-
-            // The prompt explicitly asks about the relationship between Elasticsearch, Kibana, and Logstash.
-            // If the retrieved docs don't mention all three, the agent should explicitly state insufficiency.
-            const requiredTerms = ['elasticsearch', 'kibana', 'logstash'];
-            const hasAllRequiredTerms = containsAllTerms(toolText, requiredTerms);
-            if (hasAllRequiredTerms) return { score: 1 };
-
-            const explicitlyInsufficient = includesOneOf(answer, [
-              'insufficient',
-              'not enough information',
-              "don't have enough",
-              'do not have enough',
-              "couldn't find",
-              'could not find',
-              "didn't find",
-              'did not find',
-            ]);
-
-            return {
-              score: explicitlyInsufficient ? 1 : 0,
-              metadata: {
-                requiredTerms,
-                hasAllRequiredTerms,
-                answerPreview: answer.slice(0, 500),
-              },
-            };
-          },
-        },
         {
           name: 'DocVersionReleaseDate',
           kind: 'CODE' as const,
diff --git a/x-pack/platform/packages/shared/ai-infra/kbn-evals-suite-llm-tasks/evals/retrieve_documentation/retrieve_documentation.spec.ts b/x-pack/platform/packages/shared/ai-infra/kbn-evals-suite-llm-tasks/evals/retrieve_documentation/retrieve_documentation.spec.ts
@@ -10,7 +10,7 @@ import type { TaskOutput } from '@arizeai/phoenix-client/dist/esm/types/experime
 import type { ElasticsearchClient, KibanaRequest } from '@kbn/core/server';
 import type { Logger } from '@kbn/logging';
 import { defaultInferenceEndpoints } from '@kbn/inference-common';
-import { evaluate, selectEvaluators } from '@kbn/evals';
+import { containsAllTerms, evaluate, selectEvaluators } from '@kbn/evals';
 import { SearchService } from '@kbn/product-doc-base-plugin/server/services/search/search_service';
 import { retrieveDocumentation } from '@kbn/llm-tasks-plugin/server/tasks/retrieve_documentation';
 import type { ProductName } from '@kbn/product-doc-common';
@@ -57,11 +57,6 @@ type RetrieveDocumentationTaskOutput = TaskOutput & {
   }>;
 };
 
-const containsAllTerms = (text: string, terms: string[]) => {
-  const lower = text.toLowerCase();
-  return terms.every((t) => lower.includes(t.toLowerCase()));
-};
-
 const createNoopLogger = (): Logger =>
   ({
     trace: () => {},
diff --git a/x-pack/platform/packages/shared/kbn-evals/index.ts b/x-pack/platform/packages/shared/kbn-evals/index.ts
@@ -13,6 +13,17 @@ export { createQuantitativeCorrectnessEvaluators } from './src/evaluators/correc
 export { createQuantitativeGroundednessEvaluator } from './src/evaluators/groundedness';
 export type { EvaluationDataset, EvaluationWorkerFixtures, EvaluationReport } from './src/types';
 export { withEvaluatorSpan } from './src/utils/tracing';
+export {
+  containsAllTerms,
+  extractAllStrings,
+  extractMaxSemver,
+  extractReleaseDateNearVersion,
+  getBooleanMeta,
+  getFinalAssistantMessage,
+  getStringMeta,
+  getToolCallSteps,
+  includesOneOf,
+} from './src/utils/evaluation_helpers';
 export {
   type EvaluationReporter,
   createDefaultTerminalReporter,
diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluation_helpers.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluation_helpers.ts
