From 272f1f08d07e778dcf88bfb54b17dd9d5bc32795 Mon Sep 17 00:00:00 2001 From: Kelsey Myers <52179263+kelsey-myers@users.noreply.github.com> Date: Thu, 23 Jul 2026 04:49:37 -0700 Subject: [PATCH 1/2] [eval only] Reword search_issues for natural-language semantic queries Draft branch for the copilot-offline-eval MCP tool-selection harness. Rewrites the search_issues tool + query param descriptions to steer models toward natural-language queries instead of qualifier/boolean syntax, matching the end-state of the issue-search experiment (plan-track-agentic-toolkit#415). Not intended to merge: it exists so the eval can build a container from this branch and measure query-formation against the current lexical wording. Toolsnaps regenerated. --- pkg/github/__toolsnaps__/search_issues.snap | 4 ++-- pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap | 4 ++-- pkg/github/issues.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/github/__toolsnaps__/search_issues.snap b/pkg/github/__toolsnaps__/search_issues.snap index a2ec55b91..5ceda5a5b 100644 --- a/pkg/github/__toolsnaps__/search_issues.snap +++ b/pkg/github/__toolsnaps__/search_issues.snap @@ -4,7 +4,7 @@ "readOnlyHint": true, "title": "Search issues" }, - "description": "Search for issues in GitHub repositories using issues search syntax already scoped to is:issue", + "description": "Search for issues in GitHub repositories using natural-language semantic matching. Write the query as a natural-language description of what you're looking for (e.g. \"login fails after password reset\"); best for conceptual or paraphrased queries. Already scoped to is:issue. Avoid boolean OR operators, which fall back to lexical keyword search.", "inputSchema": { "properties": { "order": { @@ -31,7 +31,7 @@ "type": "number" }, "query": { - "description": "Search query using GitHub issues search syntax", + "description": "The search query. Write it as a natural-language description of what you're looking for (e.g. \"login fails after password reset\") rather than exact keywords or boolean operators.", "type": "string" }, "repo": { diff --git a/pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap b/pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap index f705f1472..82af4d851 100644 --- a/pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap +++ b/pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap @@ -4,7 +4,7 @@ "readOnlyHint": true, "title": "Search issues" }, - "description": "Search for issues in GitHub repositories using issues search syntax already scoped to is:issue", + "description": "Search for issues in GitHub repositories using natural-language semantic matching. Write the query as a natural-language description of what you're looking for (e.g. \"login fails after password reset\"); best for conceptual or paraphrased queries. Already scoped to is:issue. Avoid boolean OR operators, which fall back to lexical keyword search.", "inputSchema": { "properties": { "fields": { @@ -64,7 +64,7 @@ "type": "number" }, "query": { - "description": "Search query using GitHub issues search syntax", + "description": "The search query. Write it as a natural-language description of what you're looking for (e.g. \"login fails after password reset\") rather than exact keywords or boolean operators.", "type": "string" }, "repo": { diff --git a/pkg/github/issues.go b/pkg/github/issues.go index 0f12804a5..61c6020e3 100644 --- a/pkg/github/issues.go +++ b/pkg/github/issues.go @@ -1628,7 +1628,7 @@ func searchIssuesTool(t translations.TranslationHelperFunc, includeFields bool) Properties: map[string]*jsonschema.Schema{ "query": { Type: "string", - Description: "Search query using GitHub issues search syntax", + Description: "The search query. Write it as a natural-language description of what you're looking for (e.g. \"login fails after password reset\") rather than exact keywords or boolean operators.", }, "owner": { Type: "string", @@ -1675,7 +1675,7 @@ func searchIssuesTool(t translations.TranslationHelperFunc, includeFields bool) ToolsetMetadataIssues, mcp.Tool{ Name: "search_issues", - Description: t("TOOL_SEARCH_ISSUES_DESCRIPTION", "Search for issues in GitHub repositories using issues search syntax already scoped to is:issue"), + Description: t("TOOL_SEARCH_ISSUES_DESCRIPTION", "Search for issues in GitHub repositories using natural-language semantic matching. Write the query as a natural-language description of what you're looking for (e.g. \"login fails after password reset\"); best for conceptual or paraphrased queries. Already scoped to is:issue. Avoid boolean OR operators, which fall back to lexical keyword search."), Annotations: &mcp.ToolAnnotations{ Title: t("TOOL_SEARCH_ISSUES_USER_TITLE", "Search issues"), ReadOnlyHint: true, From 01e71b965d2df8e4add83cd2aaf73d1fc3b6d41e Mon Sep 17 00:00:00 2001 From: Kelsey Myers <52179263+kelsey-myers@users.noreply.github.com> Date: Thu, 23 Jul 2026 06:08:16 -0700 Subject: [PATCH 2/2] [eval only] Match search_issues wording to remote #911 semantic variant Realign the eval stand-in to the exact strings shipped by the experiment PR (github-mcp-server-remote#911) semantic variant, rather than the stronger end-state wording, so the eval measures what the experiment actually ships: - tool desc: 'Best for conceptual or paraphrased queries...' (not the imperative 'Write the query as a natural-language description...') - query param: 'Write it as natural language for semantic and hybrid variants.' (drops 'rather than exact keywords or boolean operators') Toolsnaps regenerated. --- pkg/github/__toolsnaps__/search_issues.snap | 4 ++-- pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap | 4 ++-- pkg/github/issues.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/github/__toolsnaps__/search_issues.snap b/pkg/github/__toolsnaps__/search_issues.snap index 5ceda5a5b..2c056c0cf 100644 --- a/pkg/github/__toolsnaps__/search_issues.snap +++ b/pkg/github/__toolsnaps__/search_issues.snap @@ -4,7 +4,7 @@ "readOnlyHint": true, "title": "Search issues" }, - "description": "Search for issues in GitHub repositories using natural-language semantic matching. Write the query as a natural-language description of what you're looking for (e.g. \"login fails after password reset\"); best for conceptual or paraphrased queries. Already scoped to is:issue. Avoid boolean OR operators, which fall back to lexical keyword search.", + "description": "Search issues using natural-language semantic matching. Best for conceptual or paraphrased queries (e.g. \"login fails after password reset\"). Already scoped to is:issue. Avoid boolean OR operators, which fall back to lexical search.", "inputSchema": { "properties": { "order": { @@ -31,7 +31,7 @@ "type": "number" }, "query": { - "description": "The search query. Write it as a natural-language description of what you're looking for (e.g. \"login fails after password reset\") rather than exact keywords or boolean operators.", + "description": "The search query. Write it as natural language for semantic and hybrid variants.", "type": "string" }, "repo": { diff --git a/pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap b/pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap index 82af4d851..2c1637e17 100644 --- a/pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap +++ b/pkg/github/__toolsnaps__/search_issues_ff_fields_param.snap @@ -4,7 +4,7 @@ "readOnlyHint": true, "title": "Search issues" }, - "description": "Search for issues in GitHub repositories using natural-language semantic matching. Write the query as a natural-language description of what you're looking for (e.g. \"login fails after password reset\"); best for conceptual or paraphrased queries. Already scoped to is:issue. Avoid boolean OR operators, which fall back to lexical keyword search.", + "description": "Search issues using natural-language semantic matching. Best for conceptual or paraphrased queries (e.g. \"login fails after password reset\"). Already scoped to is:issue. Avoid boolean OR operators, which fall back to lexical search.", "inputSchema": { "properties": { "fields": { @@ -64,7 +64,7 @@ "type": "number" }, "query": { - "description": "The search query. Write it as a natural-language description of what you're looking for (e.g. \"login fails after password reset\") rather than exact keywords or boolean operators.", + "description": "The search query. Write it as natural language for semantic and hybrid variants.", "type": "string" }, "repo": { diff --git a/pkg/github/issues.go b/pkg/github/issues.go index 61c6020e3..fef52de7d 100644 --- a/pkg/github/issues.go +++ b/pkg/github/issues.go @@ -1628,7 +1628,7 @@ func searchIssuesTool(t translations.TranslationHelperFunc, includeFields bool) Properties: map[string]*jsonschema.Schema{ "query": { Type: "string", - Description: "The search query. Write it as a natural-language description of what you're looking for (e.g. \"login fails after password reset\") rather than exact keywords or boolean operators.", + Description: "The search query. Write it as natural language for semantic and hybrid variants.", }, "owner": { Type: "string", @@ -1675,7 +1675,7 @@ func searchIssuesTool(t translations.TranslationHelperFunc, includeFields bool) ToolsetMetadataIssues, mcp.Tool{ Name: "search_issues", - Description: t("TOOL_SEARCH_ISSUES_DESCRIPTION", "Search for issues in GitHub repositories using natural-language semantic matching. Write the query as a natural-language description of what you're looking for (e.g. \"login fails after password reset\"); best for conceptual or paraphrased queries. Already scoped to is:issue. Avoid boolean OR operators, which fall back to lexical keyword search."), + Description: t("TOOL_SEARCH_ISSUES_DESCRIPTION", "Search issues using natural-language semantic matching. Best for conceptual or paraphrased queries (e.g. \"login fails after password reset\"). Already scoped to is:issue. Avoid boolean OR operators, which fall back to lexical search."), Annotations: &mcp.ToolAnnotations{ Title: t("TOOL_SEARCH_ISSUES_USER_TITLE", "Search issues"), ReadOnlyHint: true,