Coverage for tests / unit / compiler / resolvers / test_llm_resolver.py: 0%

1"""Unit tests for the LLM-based dependency resolvers. 

2 

3All LLM calls are mocked — no real API keys or network needed. 

4Tests cover: 

5 - Happy path: LLM returns valid JSON, result is merged correctly 

6 - Validation: hallucinated object names and endpoint names are stripped 

7 - Fallback: when LLM raises an exception and LLM_RESOLVER_FALLBACK_TO_ID=True 

8 - Fallback disabled: exception propagates when LLM_RESOLVER_FALLBACK_TO_ID=False 

9 - Comparison: correct diff structure is built 

10 - Schema context builder: compact representation of objects 

11 - Simplified endpoint builder: compact representation of inputs 

12 - ResolverComparison: summary counts and JSON output 

13 - JSON extraction: strips markdown fences, finds embedded JSON 

14 - JSON mode detection: only passes response_format for supported models 

15 - Retry logic: re-prompts with correction message on non-JSON response 

16""" 

17 

18import json 

19import unittest 

20from unittest.mock import MagicMock, patch 

21 

22from graphqler.compiler.resolvers.llm.llm_resolver import LLMResolver 

23from graphqler.compiler.resolvers.llm.llm_mutation_object_resolver import LLMMutationObjectResolver 

24from graphqler.compiler.resolvers.llm.llm_query_object_resolver import LLMQueryObjectResolver 

25from graphqler.compiler.resolvers.llm.comparison import ResolverComparison 

26 

27 

28# ── Shared fixtures ─────────────────────────────────────────────────────────── 

29 

30OBJECTS = { 

31 "User": { 

32 "fields": [ 

33 {"name": "id", "kind": "NON_NULL", "type": None, "ofType": {"kind": "SCALAR", "name": "ID", "type": "ID", "ofType": None}}, 

34 {"name": "email", "kind": "NON_NULL", "type": None, "ofType": {"kind": "SCALAR", "name": "String", "type": "String", "ofType": None}}, 

35 {"name": "name", "kind": "SCALAR", "type": "String", "ofType": None}, 

36 ], 

37 "hardDependsOn": [], 

38 "softDependsOn": [], 

39 "associatedQueries": [], 

40 "associatedMutatations": [], 

41 }, 

42 "Post": { 

43 "fields": [ 

44 {"name": "id", "kind": "NON_NULL", "type": None, "ofType": {"kind": "SCALAR", "name": "ID", "type": "ID", "ofType": None}}, 

45 {"name": "title", "kind": "SCALAR", "type": "String", "ofType": None}, 

46 {"name": "author", "kind": "OBJECT", "type": "User", "ofType": None}, 

47 ], 

48 "hardDependsOn": [], 

49 "softDependsOn": [], 

50 "associatedQueries": [], 

51 "associatedMutatations": [], 

52 }, 

53} 

54 

55MUTATIONS = { 

56 "createPost": { 

57 "name": "createPost", 

58 "description": "Create a new blog post", 

59 "inputs": { 

60 "title": {"kind": "NON_NULL", "type": None, "name": "title", "ofType": {"kind": "SCALAR", "name": "String", "type": "String", "ofType": None}}, 

61 "authorEmail": {"kind": "NON_NULL", "type": None, "name": "authorEmail", "ofType": {"kind": "SCALAR", "name": "String", "type": "String", "ofType": None}}, 

62 }, 

63 "output": {"kind": "OBJECT", "name": "Post", "type": "Post", "ofType": None}, 

64 "isDepracated": False, 

65 }, 

66 "deletePost": { 

67 "name": "deletePost", 

68 "description": None, 

69 "inputs": { 

70 "id": {"kind": "NON_NULL", "type": None, "name": "id", "ofType": {"kind": "SCALAR", "name": "ID", "type": "ID", "ofType": None}}, 

71 }, 

72 "output": {"kind": "SCALAR", "name": "Boolean", "type": "Boolean", "ofType": None}, 

73 "isDepracated": False, 

74 }, 

75} 

76 

77QUERIES = { 

78 "getPost": { 

79 "name": "getPost", 

80 "description": None, 

81 "inputs": { 

82 "id": {"kind": "NON_NULL", "type": None, "name": "id", "ofType": {"kind": "SCALAR", "name": "ID", "type": "ID", "ofType": None}}, 

83 }, 

84 "output": {"kind": "OBJECT", "name": "Post", "type": "Post", "ofType": None}, 

85 }, 

86 "searchPosts": { 

87 "name": "searchPosts", 

88 "description": "Search posts by author email", 

89 "inputs": { 

90 "authorEmail": {"kind": "NON_NULL", "type": None, "name": "authorEmail", "ofType": {"kind": "SCALAR", "name": "String", "type": "String", "ofType": None}}, 

91 }, 

92 "output": {"kind": "LIST", "name": None, "type": None, "ofType": {"kind": "OBJECT", "name": "Post", "type": "Post", "ofType": None}}, 

93 }, 

94} 

95 

96INPUT_OBJECTS: dict = {} 

97 

98# ── LLMResolver (base) ──────────────────────────────────────────────────────── 

99 

100 

101class TestLLMResolverBase(unittest.TestCase): 

102 def setUp(self): 

103 self.resolver = LLMResolver() 

104 

105 def test_schema_context_contains_object_names(self): 

106 ctx = self.resolver.build_schema_context(OBJECTS) 

107 self.assertIn("User", ctx) 

108 self.assertIn("Post", ctx) 

109 

110 def test_schema_context_contains_field_names(self): 

111 ctx = self.resolver.build_schema_context(OBJECTS) 

112 self.assertIn("email", ctx) 

113 self.assertIn("title", ctx) 

114 

115 def test_simplify_endpoints_returns_all_names(self): 

116 simplified = self.resolver.simplify_endpoints(MUTATIONS) 

117 self.assertIn("createPost", simplified) 

118 self.assertIn("deletePost", simplified) 

119 

120 def test_simplify_endpoints_readable_types(self): 

121 simplified = self.resolver.simplify_endpoints(MUTATIONS) 

122 self.assertIn("String!", simplified["createPost"]["inputs"]["title"]) 

123 

124 def test_validate_mutation_strips_hallucinated_endpoints(self): 

125 raw = { 

126 "createPost": {"mutationType": "CREATE", "hardDependsOn": {"authorEmail": "User"}, "softDependsOn": {}}, 

127 "nonExistentMutation": {"mutationType": "DELETE", "hardDependsOn": {}, "softDependsOn": {}}, 

128 } 

129 validated = self.resolver.validate_llm_mutation_result(raw, list(MUTATIONS.keys()), OBJECTS) 

130 self.assertIn("createPost", validated) 

131 self.assertNotIn("nonExistentMutation", validated) 

132 

133 def test_validate_mutation_strips_unknown_object_deps(self): 

134 raw = { 

135 "createPost": {"mutationType": "CREATE", "hardDependsOn": {"authorEmail": "FakeObject"}, "softDependsOn": {}}, 

136 } 

137 validated = self.resolver.validate_llm_mutation_result(raw, list(MUTATIONS.keys()), OBJECTS) 

138 self.assertEqual(validated["createPost"]["hardDependsOn"], {}) 

139 

140 def test_validate_mutation_normalises_bad_mutation_type(self): 

141 raw = { 

142 "createPost": {"mutationType": "UPSERT", "hardDependsOn": {}, "softDependsOn": {}}, 

143 } 

144 validated = self.resolver.validate_llm_mutation_result(raw, list(MUTATIONS.keys()), OBJECTS) 

145 self.assertEqual(validated["createPost"]["mutationType"], "UNKNOWN") 

146 

147 def test_validate_query_strips_hallucinated_endpoints(self): 

148 raw = { 

149 "getPost": {"hardDependsOn": {"id": "Post"}, "softDependsOn": {}}, 

150 "ghostQuery": {"hardDependsOn": {}, "softDependsOn": {}}, 

151 } 

152 validated = self.resolver.validate_llm_query_result(raw, list(QUERIES.keys()), OBJECTS) 

153 self.assertIn("getPost", validated) 

154 self.assertNotIn("ghostQuery", validated) 

155 

156 def test_merge_with_classic_llm_takes_precedence(self): 

157 classic = { 

158 "createPost": {"mutationType": "UNKNOWN", "hardDependsOn": {}, "softDependsOn": {}, "other": "x"}, 

159 } 

160 llm = { 

161 "createPost": {"mutationType": "CREATE", "hardDependsOn": {"authorEmail": "User"}, "softDependsOn": {}}, 

162 } 

163 merged = self.resolver.merge_with_classic(llm, classic, ["createPost"]) 

164 self.assertEqual(merged["createPost"]["mutationType"], "CREATE") 

165 self.assertEqual(merged["createPost"]["hardDependsOn"], {"authorEmail": "User"}) 

166 self.assertEqual(merged["createPost"]["other"], "x") # non-dep fields preserved 

167 

168 def test_merge_with_classic_fills_missing_llm_entries(self): 

169 classic = { 

170 "createPost": {"mutationType": "CREATE", "hardDependsOn": {}, "softDependsOn": {}}, 

171 "deletePost": {"mutationType": "DELETE", "hardDependsOn": {"id": "Post"}, "softDependsOn": {}}, 

172 } 

173 llm = { 

174 "createPost": {"mutationType": "CREATE", "hardDependsOn": {}, "softDependsOn": {}}, 

175 # deletePost absent from LLM response 

176 } 

177 merged = self.resolver.merge_with_classic(llm, classic, ["createPost", "deletePost"]) 

178 self.assertIn("deletePost", merged) 

179 self.assertEqual(merged["deletePost"]["mutationType"], "DELETE") 

180 

181 # ── _extract_json_from_text ──────────────────────────────────────────────── 

182 

183 def test_extract_json_raw(self): 

184 data = {"foo": "bar"} 

185 result = self.resolver._extract_json_from_text(json.dumps(data)) 

186 self.assertEqual(result, data) 

187 

188 def test_extract_json_with_backtick_fence(self): 

189 data = {"createPost": {"mutationType": "CREATE"}} 

190 text = "```json\n" + json.dumps(data) + "\n```" 

191 result = self.resolver._extract_json_from_text(text) 

192 self.assertEqual(result, data) 

193 

194 def test_extract_json_with_plain_fence(self): 

195 data = {"a": 1} 

196 text = "```\n" + json.dumps(data) + "\n```" 

197 result = self.resolver._extract_json_from_text(text) 

198 self.assertEqual(result, data) 

199 

200 def test_extract_json_embedded_in_prose(self): 

201 data = {"x": "y"} 

202 text = "Here is the result: " + json.dumps(data) + " Hope that helps!" 

203 result = self.resolver._extract_json_from_text(text) 

204 self.assertEqual(result, data) 

205 

206 def test_extract_json_raises_on_garbage(self): 

207 with self.assertRaises(ValueError): 

208 self.resolver._extract_json_from_text("this is not json at all") 

209 

210# ── Retry logic ─────────────────────────────────────────────────────────────── 

211 

212 

213class TestCallLLMRetry(unittest.TestCase): 

214 """call_llm should retry up to LLM_MAX_RETRIES times on non-JSON responses.""" 

215 

216 def _make_response(self, content: str) -> MagicMock: 

217 mock = MagicMock() 

218 mock.choices[0].message.content = content 

219 return mock 

220 

221 @patch("graphqler.compiler.resolvers.llm.llm_resolver.config") 

222 def test_succeeds_on_second_attempt(self, mock_cfg): 

223 mock_cfg.LLM_MODEL = "gpt-4o-mini" 

224 mock_cfg.LLM_API_KEY = "" 

225 mock_cfg.LLM_BASE_URL = "" 

226 mock_cfg.LLM_MAX_RETRIES = 1 

227 

228 bad = self._make_response("not json") 

229 good = self._make_response('{"ok": true}') 

230 

231 import litellm 

232 with patch.object(litellm, "completion", side_effect=[bad, good]): 

233 resolver = LLMResolver() 

234 result = resolver.call_llm("sys", "user") 

235 

236 self.assertEqual(result, {"ok": True}) 

237 

238 @patch("graphqler.compiler.resolvers.llm.llm_resolver.config") 

239 def test_raises_after_all_retries_exhausted(self, mock_cfg): 

240 mock_cfg.LLM_MODEL = "gpt-4o-mini" 

241 mock_cfg.LLM_API_KEY = "" 

242 mock_cfg.LLM_BASE_URL = "" 

243 mock_cfg.LLM_MAX_RETRIES = 1 

244 

245 bad = self._make_response("still not json") 

246 

247 import litellm 

248 with patch.object(litellm, "completion", return_value=bad): 

249 resolver = LLMResolver() 

250 with self.assertRaises(ValueError): 

251 resolver.call_llm("sys", "user") 

252 

253 @patch("graphqler.compiler.resolvers.llm.llm_resolver.config") 

254 def test_correction_turn_appended_to_messages(self, mock_cfg): 

255 mock_cfg.LLM_MODEL = "gpt-4o-mini" 

256 mock_cfg.LLM_API_KEY = "" 

257 mock_cfg.LLM_BASE_URL = "" 

258 mock_cfg.LLM_MAX_RETRIES = 1 

259 

260 bad = self._make_response("not json") 

261 good = self._make_response('{"fixed": true}') 

262 

263 import litellm 

264 with patch.object(litellm, "completion", side_effect=[bad, good]) as mock_completion: 

265 resolver = LLMResolver() 

266 resolver.call_llm("system_prompt", "user_prompt") 

267 

268 # Second call should have 4 messages: system + user + bad_assistant + correction 

269 second_call_messages = mock_completion.call_args_list[1][1]["messages"] 

270 self.assertEqual(len(second_call_messages), 4) 

271 self.assertEqual(second_call_messages[2]["role"], "assistant") 

272 self.assertEqual(second_call_messages[3]["role"], "user") 

273 self.assertIn("JSON", second_call_messages[3]["content"]) 

274 

275 

276# ── LLMMutationObjectResolver ───────────────────────────────────────────────── 

277 

278 

279class TestLLMMutationObjectResolver(unittest.TestCase): 

280 def _make_llm_response(self, payload: dict) -> MagicMock: 

281 mock_response = MagicMock() 

282 mock_response.choices[0].message.content = json.dumps(payload) 

283 return mock_response 

284 

285 @patch("graphqler.compiler.resolvers.llm.llm_resolver.config") 

286 def test_happy_path_llm_result_is_used(self, mock_cfg): 

287 mock_cfg.LLM_MODEL = "gpt-4o-mini" 

288 mock_cfg.LLM_API_KEY = "" 

289 mock_cfg.LLM_BASE_URL = "" 

290 mock_cfg.LLM_RESOLVER_FALLBACK_TO_ID = True 

291 mock_cfg.LLM_MAX_RETRIES = 0 

292 

293 llm_payload = { 

294 "createPost": {"mutationType": "CREATE", "hardDependsOn": {"authorEmail": "User"}, "softDependsOn": {}}, 

295 "deletePost": {"mutationType": "DELETE", "hardDependsOn": {"id": "Post"}, "softDependsOn": {}}, 

296 } 

297 

298 import litellm 

299 with patch.object(litellm, "completion", return_value=self._make_llm_response(llm_payload)): 

300 resolver = LLMMutationObjectResolver() 

301 result = resolver.resolve(OBJECTS, MUTATIONS, INPUT_OBJECTS) 

302 

303 self.assertEqual(result["createPost"]["mutationType"], "CREATE") 

304 self.assertEqual(result["createPost"]["hardDependsOn"], {"authorEmail": "User"}) 

305 self.assertEqual(result["deletePost"]["mutationType"], "DELETE") 

306 self.assertTrue(resolver.comparison) 

307 

308 def test_fallback_on_llm_exception(self): 

309 from graphqler import config as graphqler_config 

310 original = graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID 

311 graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID = True 

312 try: 

313 import litellm 

314 with patch("graphqler.compiler.resolvers.llm.llm_resolver.config") as mock_cfg: 

315 mock_cfg.LLM_MODEL = "gpt-4o-mini" 

316 mock_cfg.LLM_API_KEY = "" 

317 mock_cfg.LLM_BASE_URL = "" 

318 mock_cfg.LLM_MAX_RETRIES = 0 

319 with patch.object(litellm, "completion", side_effect=RuntimeError("API down")): 

320 resolver = LLMMutationObjectResolver() 

321 result = resolver.resolve(OBJECTS, MUTATIONS, INPUT_OBJECTS) 

322 

323 # Should still get a valid result from classic resolver 

324 self.assertIn("createPost", result) 

325 self.assertIn("deletePost", result) 

326 self.assertEqual(resolver.comparison, {}) 

327 finally: 

328 graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID = original 

329 

330 def test_fallback_disabled_propagates_exception(self): 

331 from graphqler import config as graphqler_config 

332 original = graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID 

333 graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID = False 

334 try: 

335 import litellm 

336 with patch("graphqler.compiler.resolvers.llm.llm_resolver.config") as mock_cfg: 

337 mock_cfg.LLM_MODEL = "gpt-4o-mini" 

338 mock_cfg.LLM_API_KEY = "" 

339 mock_cfg.LLM_BASE_URL = "" 

340 mock_cfg.LLM_MAX_RETRIES = 0 

341 with patch.object(litellm, "completion", side_effect=RuntimeError("API down")): 

342 resolver = LLMMutationObjectResolver() 

343 with self.assertRaises(RuntimeError): 

344 resolver.resolve(OBJECTS, MUTATIONS, INPUT_OBJECTS) 

345 finally: 

346 graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID = original 

347 

348 def test_comparison_detects_differences(self): 

349 from graphqler import config as graphqler_config 

350 original = graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID 

351 graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID = True 

352 try: 

353 # LLM catches authorEmail → User (classic would miss this, it's not an ID type) 

354 llm_payload = { 

355 "createPost": {"mutationType": "CREATE", "hardDependsOn": {"authorEmail": "User"}, "softDependsOn": {}}, 

356 "deletePost": {"mutationType": "DELETE", "hardDependsOn": {"id": "Post"}, "softDependsOn": {}}, 

357 } 

358 

359 import litellm 

360 with patch("graphqler.compiler.resolvers.llm.llm_resolver.config") as mock_cfg: 

361 mock_cfg.LLM_MODEL = "gpt-4o-mini" 

362 mock_cfg.LLM_API_KEY = "" 

363 mock_cfg.LLM_BASE_URL = "" 

364 mock_cfg.LLM_MAX_RETRIES = 0 

365 with patch.object(litellm, "completion", return_value=self._make_llm_response(llm_payload)): 

366 resolver = LLMMutationObjectResolver() 

367 resolver.resolve(OBJECTS, MUTATIONS, INPUT_OBJECTS) 

368 

369 # createPost: classic gets {} for hardDependsOn (authorEmail is String not ID), 

370 # LLM gets {"authorEmail": "User"} → should differ 

371 self.assertTrue(resolver.comparison["createPost"]["differs"]) 

372 self.assertIn("hardDependsOn", resolver.comparison["createPost"]["diff"]) 

373 finally: 

374 graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID = original 

375 

376 

377# ── LLMQueryObjectResolver ──────────────────────────────────────────────────── 

378 

379 

380class TestLLMQueryObjectResolver(unittest.TestCase): 

381 def _make_llm_response(self, payload: dict) -> MagicMock: 

382 mock_response = MagicMock() 

383 mock_response.choices[0].message.content = json.dumps(payload) 

384 return mock_response 

385 

386 @patch("graphqler.compiler.resolvers.llm.llm_resolver.config") 

387 def test_happy_path_query_resolved(self, mock_cfg): 

388 mock_cfg.LLM_MODEL = "gpt-4o-mini" 

389 mock_cfg.LLM_API_KEY = "" 

390 mock_cfg.LLM_BASE_URL = "" 

391 mock_cfg.LLM_RESOLVER_FALLBACK_TO_ID = True 

392 mock_cfg.LLM_MAX_RETRIES = 0 

393 

394 llm_payload = { 

395 "getPost": {"hardDependsOn": {"id": "Post"}, "softDependsOn": {}}, 

396 "searchPosts": {"hardDependsOn": {"authorEmail": "User"}, "softDependsOn": {}}, 

397 } 

398 

399 import litellm 

400 with patch.object(litellm, "completion", return_value=self._make_llm_response(llm_payload)): 

401 resolver = LLMQueryObjectResolver() 

402 result = resolver.resolve(OBJECTS, QUERIES, INPUT_OBJECTS) 

403 

404 self.assertEqual(result["getPost"]["hardDependsOn"], {"id": "Post"}) 

405 # Classic misses authorEmail (String, not ID); LLM catches it 

406 self.assertEqual(result["searchPosts"]["hardDependsOn"], {"authorEmail": "User"}) 

407 

408 def test_fallback_on_bad_json(self): 

409 from graphqler import config as graphqler_config 

410 original = graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID 

411 graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID = True 

412 try: 

413 bad_response = MagicMock() 

414 bad_response.choices[0].message.content = "not json at all" 

415 

416 import litellm 

417 with patch("graphqler.compiler.resolvers.llm.llm_resolver.config") as mock_cfg: 

418 mock_cfg.LLM_MODEL = "gpt-4o-mini" 

419 mock_cfg.LLM_API_KEY = "" 

420 mock_cfg.LLM_BASE_URL = "" 

421 mock_cfg.LLM_MAX_RETRIES = 0 

422 with patch.object(litellm, "completion", return_value=bad_response): 

423 resolver = LLMQueryObjectResolver() 

424 result = resolver.resolve(OBJECTS, QUERIES, INPUT_OBJECTS) 

425 

426 # Should fall back gracefully 

427 self.assertIn("getPost", result) 

428 self.assertIn("searchPosts", result) 

429 finally: 

430 graphqler_config.LLM_RESOLVER_FALLBACK_TO_ID = original 

431 

432 

433# ── ResolverComparison ──────────────────────────────────────────────────────── 

434 

435 

436class TestResolverComparison(unittest.TestCase): 

437 def _make_mutation_comparison(self) -> dict: 

438 return { 

439 "createPost": { 

440 "classic": {"mutationType": "UNKNOWN", "hardDependsOn": {}, "softDependsOn": {}}, 

441 "llm": {"mutationType": "CREATE", "hardDependsOn": {"authorEmail": "User"}, "softDependsOn": {}}, 

442 "differs": True, 

443 "diff": { 

444 "mutationType": {"classic": "UNKNOWN", "llm": "CREATE"}, 

445 "hardDependsOn": {"classic": {}, "llm": {"authorEmail": "User"}}, 

446 }, 

447 }, 

448 "deletePost": { 

449 "classic": {"mutationType": "DELETE", "hardDependsOn": {"id": "Post"}, "softDependsOn": {}}, 

450 "llm": {"mutationType": "DELETE", "hardDependsOn": {"id": "Post"}, "softDependsOn": {}}, 

451 "differs": False, 

452 "diff": {}, 

453 }, 

454 } 

455 

456 def _make_query_comparison(self) -> dict: 

457 return { 

458 "getPost": { 

459 "classic": {"hardDependsOn": {"id": "Post"}, "softDependsOn": {}}, 

460 "llm": {"hardDependsOn": {"id": "Post"}, "softDependsOn": {}}, 

461 "differs": False, 

462 "diff": {}, 

463 }, 

464 } 

465 

466 def test_summary_counts_correct(self): 

467 comp = ResolverComparison(self._make_mutation_comparison(), self._make_query_comparison()) 

468 doc = comp.build() 

469 self.assertEqual(doc["summary"]["total_mutations"], 2) 

470 self.assertEqual(doc["summary"]["mutations_that_differ"], 1) 

471 self.assertEqual(doc["summary"]["total_queries"], 1) 

472 self.assertEqual(doc["summary"]["queries_that_differ"], 0) 

473 

474 def test_save_writes_valid_json(self): 

475 import tempfile 

476 import os 

477 comp = ResolverComparison(self._make_mutation_comparison(), self._make_query_comparison()) 

478 with tempfile.TemporaryDirectory() as tmpdir: 

479 comp.save(tmpdir) 

480 json_path = os.path.join(tmpdir, "eval", "resolver_comparison.json") 

481 self.assertTrue(os.path.exists(json_path)) 

482 with open(json_path) as f: 

483 data = json.load(f) 

484 self.assertIn("summary", data) 

485 self.assertIn("mutations", data) 

486 self.assertIn("queries", data) 

487 

488 def test_save_summary_values_in_file(self): 

489 import tempfile 

490 comp = ResolverComparison(self._make_mutation_comparison(), self._make_query_comparison()) 

491 with tempfile.TemporaryDirectory() as tmpdir: 

492 comp.save(tmpdir) 

493 import os 

494 with open(os.path.join(tmpdir, "eval", "resolver_comparison.json")) as f: 

495 data = json.load(f) 

496 self.assertEqual(data["summary"]["mutations_that_differ"], 1) 

497 self.assertEqual(data["summary"]["queries_that_differ"], 0) 

498 

499 

500if __name__ == "__main__": 

501 unittest.main()