test_cfmail_rotation.py 40 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058
  1. import json
  2. from curl_cffi import requests as cffi_requests
  3. from core.cfmail_domain_rotation import DomainHealthTracker, classify_domain_attempt
  4. from core.cfmail_provisioner import CfmailProvisioner, ProvisionResult, ProvisioningSettings
  5. def test_classify_domain_attempt_recognizes_blacklist_codes() -> None:
  6. attempt = classify_domain_attempt(
  7. {
  8. "success": False,
  9. "stage": "create_account",
  10. "error_message": "create account failed",
  11. "metadata": {
  12. "email_domain": "nova.example.test",
  13. "create_account_error_code": "registration_disallowed",
  14. "create_account_error_message": "blocked",
  15. },
  16. },
  17. proxy_key="sg-node",
  18. )
  19. assert attempt is not None
  20. assert attempt.domain == "nova.example.test"
  21. assert attempt.blacklist_code == "registration_disallowed"
  22. assert attempt.proxy_key == "sg-node"
  23. assert attempt.backend_failure is False
  24. def test_classify_domain_attempt_recognizes_mailbox_reused_signal() -> None:
  25. attempt = classify_domain_attempt(
  26. {
  27. "success": False,
  28. "stage": "create_account",
  29. "error_message": "create account failed",
  30. "metadata": {
  31. "email_domain": "nova.example.test",
  32. "create_account_error_code": "user_already_exists",
  33. },
  34. },
  35. proxy_key="tw-node",
  36. )
  37. assert attempt is not None
  38. assert attempt.domain == "nova.example.test"
  39. assert attempt.blacklist_code == "user_already_exists"
  40. assert attempt.proxy_key == "tw-node"
  41. def test_domain_health_tracker_requires_threshold_before_rotation() -> None:
  42. tracker = DomainHealthTracker(window_size=3, blacklist_threshold=3, rotation_cooldown_seconds=1)
  43. payload = {
  44. "success": False,
  45. "stage": "create_account",
  46. "error_message": "create account failed",
  47. "metadata": {
  48. "email_domain": "nova.example.test",
  49. "create_account_error_code": "unsupported_email",
  50. "create_account_error_message": "unsupported",
  51. },
  52. }
  53. for idx in range(2):
  54. attempt = classify_domain_attempt(payload, proxy_key=f"proxy-{idx}")
  55. assert attempt is not None
  56. decision = tracker.record(attempt)
  57. assert decision.should_rotate is False
  58. final_attempt = classify_domain_attempt(payload, proxy_key="proxy-3")
  59. assert final_attempt is not None
  60. decision = tracker.record(final_attempt)
  61. assert decision.should_rotate is True
  62. assert decision.domain == "nova.example.test"
  63. def test_domain_health_tracker_rotates_on_mailbox_reused_threshold() -> None:
  64. tracker = DomainHealthTracker(
  65. window_size=8,
  66. blacklist_threshold=6,
  67. rotation_cooldown_seconds=1,
  68. mailbox_reused_threshold=2,
  69. )
  70. payload = {
  71. "success": False,
  72. "stage": "create_account",
  73. "error_message": "create account failed",
  74. "metadata": {
  75. "email_domain": "nova.example.test",
  76. "create_account_error_code": "user_already_exists",
  77. },
  78. }
  79. tracker.record(classify_domain_attempt(payload, proxy_key="proxy-1")) # type: ignore[arg-type]
  80. decision = tracker.record(classify_domain_attempt(payload, proxy_key="proxy-2")) # type: ignore[arg-type]
  81. assert decision.should_rotate is True
  82. assert decision.reason == "mailbox_reused threshold reached"
  83. def test_domain_health_tracker_rotates_fast_on_registration_disallowed_streak() -> None:
  84. tracker = DomainHealthTracker(
  85. window_size=10,
  86. blacklist_threshold=6,
  87. rotation_cooldown_seconds=1,
  88. )
  89. payload = {
  90. "success": False,
  91. "stage": "create_account",
  92. "error_message": "create account failed",
  93. "metadata": {
  94. "email_domain": "nova.example.test",
  95. "create_account_error_code": "registration_disallowed",
  96. },
  97. }
  98. first = tracker.record(classify_domain_attempt(payload, proxy_key="proxy-1")) # type: ignore[arg-type]
  99. second = tracker.record(classify_domain_attempt(payload, proxy_key="proxy-2")) # type: ignore[arg-type]
  100. assert first.should_rotate is False
  101. assert second.should_rotate is True
  102. assert second.reason == "registration_disallowed threshold reached"
  103. def test_domain_health_tracker_fast_registration_disallowed_threshold_is_default() -> None:
  104. tracker = DomainHealthTracker()
  105. assert tracker.registration_disallowed_threshold == 2
  106. def test_domain_health_tracker_allows_small_number_of_successes_before_rotation() -> None:
  107. tracker = DomainHealthTracker(
  108. window_size=4,
  109. blacklist_threshold=3,
  110. rotation_cooldown_seconds=1,
  111. max_successes_in_window=1,
  112. )
  113. blacklist_payload = {
  114. "success": False,
  115. "stage": "create_account",
  116. "error_message": "create account failed",
  117. "metadata": {
  118. "email_domain": "nova.example.test",
  119. "create_account_error_code": "registration_disallowed",
  120. },
  121. }
  122. success_payload = {
  123. "success": True,
  124. "stage": "completed",
  125. "email": "ok@nova.example.test",
  126. "metadata": {"email_domain": "nova.example.test"},
  127. }
  128. tracker.record(classify_domain_attempt(blacklist_payload, proxy_key="p1")) # type: ignore[arg-type]
  129. tracker.record(classify_domain_attempt(blacklist_payload, proxy_key="p2")) # type: ignore[arg-type]
  130. tracker.record(classify_domain_attempt(success_payload, proxy_key="p3")) # type: ignore[arg-type]
  131. decision = tracker.record(classify_domain_attempt(blacklist_payload, proxy_key="p4")) # type: ignore[arg-type]
  132. assert decision.should_rotate is True
  133. def test_cfmail_provisioner_switch_active_domain_updates_config(tmp_path) -> None:
  134. config_path = tmp_path / "cfmail.json"
  135. config_path.write_text(
  136. json.dumps(
  137. {
  138. "accounts": [
  139. {
  140. "name": "old-active",
  141. "worker_domain": "email-api.example.test",
  142. "email_domain": "nova.example.test",
  143. "admin_password": "pw",
  144. "enabled": True,
  145. }
  146. ]
  147. },
  148. ensure_ascii=False,
  149. indent=2,
  150. )
  151. + "\n",
  152. encoding="utf-8",
  153. )
  154. provisioner = CfmailProvisioner(
  155. config_path=config_path,
  156. settings=ProvisioningSettings(
  157. auth_email="demo@example.com",
  158. auth_key="demo-key",
  159. account_id="acct",
  160. zone_id="zone",
  161. worker_name="worker",
  162. zone_name="example.test",
  163. ),
  164. )
  165. removed_domains = provisioner.switch_active_domain(
  166. old_domain="nova.example.test",
  167. new_domain="auto0322.example.test",
  168. worker_domain="email-api.example.test",
  169. admin_password="pw",
  170. )
  171. payload = json.loads(config_path.read_text(encoding="utf-8"))
  172. accounts = payload["accounts"]
  173. assert removed_domains == []
  174. assert [item["email_domain"] for item in accounts] == [
  175. "nova.example.test",
  176. "auto0322.example.test",
  177. ]
  178. assert accounts[0]["enabled"] is False
  179. assert accounts[1]["enabled"] is True
  180. def test_cfmail_provisioner_provision_additional_domain_keeps_existing_active_domains(monkeypatch, tmp_path) -> None:
  181. config_path = tmp_path / "cfmail.json"
  182. config_path.write_text(
  183. json.dumps(
  184. {
  185. "accounts": [
  186. {
  187. "name": "active-a",
  188. "worker_domain": "email-api.example.test",
  189. "email_domain": "auto-live-a.example.test",
  190. "admin_password": "pw",
  191. "enabled": True,
  192. },
  193. {
  194. "name": "active-b",
  195. "worker_domain": "email-api.example.test",
  196. "email_domain": "auto-live-b.example.test",
  197. "admin_password": "pw",
  198. "enabled": True,
  199. },
  200. ]
  201. }
  202. )
  203. + "\n",
  204. encoding="utf-8",
  205. )
  206. provisioner = CfmailProvisioner(
  207. config_path=config_path,
  208. settings=ProvisioningSettings(
  209. auth_email="demo@example.com",
  210. auth_key="demo-key",
  211. account_id="acct",
  212. zone_id="zone",
  213. worker_name="worker",
  214. zone_name="example.test",
  215. ),
  216. )
  217. worker_domains: list[list[str]] = []
  218. monkeypatch.setattr(provisioner, "_make_new_label", lambda: "auto-next")
  219. monkeypatch.setattr(provisioner, "_create_email_routing_rule", lambda domain, label: None)
  220. monkeypatch.setattr(provisioner, "_create_dns_records", lambda domain: None)
  221. monkeypatch.setattr(provisioner, "_set_worker_domains", lambda domains: worker_domains.append(list(domains)))
  222. monkeypatch.setattr(provisioner, "smoke_test", lambda *args: None)
  223. result = provisioner.provision_additional_domain()
  224. payload = json.loads(config_path.read_text(encoding="utf-8"))
  225. assert result.success is True
  226. assert result.new_domain == "auto-next.example.test"
  227. assert worker_domains == [[
  228. "auto-live-a.example.test",
  229. "auto-live-b.example.test",
  230. "auto-next.example.test",
  231. ]]
  232. assert [item["email_domain"] for item in payload["accounts"]][-1] == "auto-next.example.test"
  233. assert payload["accounts"][-1]["enabled"] is True
  234. def test_cfmail_provisioner_normalize_to_domain_pool_provisions_missing_domains(monkeypatch, tmp_path) -> None:
  235. config_path = tmp_path / "cfmail.json"
  236. config_path.write_text(
  237. json.dumps(
  238. {
  239. "accounts": [
  240. {
  241. "name": "active-a",
  242. "worker_domain": "email-api.example.test",
  243. "email_domain": "auto-live-a.example.test",
  244. "admin_password": "pw",
  245. "enabled": True,
  246. }
  247. ]
  248. }
  249. )
  250. + "\n",
  251. encoding="utf-8",
  252. )
  253. provisioner = CfmailProvisioner(
  254. config_path=config_path,
  255. settings=ProvisioningSettings(
  256. auth_email="demo@example.com",
  257. auth_key="demo-key",
  258. account_id="acct",
  259. zone_id="zone",
  260. worker_name="worker",
  261. zone_name="example.test",
  262. ),
  263. )
  264. monkeypatch.setattr(
  265. provisioner,
  266. "provision_additional_domain",
  267. lambda: ProvisionResult(success=True, step="provision_additional_domain", new_domain="auto-next.example.test"),
  268. )
  269. monkeypatch.setattr(
  270. provisioner,
  271. "current_active_accounts",
  272. lambda: [
  273. {"name": "active-a", "email_domain": "auto-live-a.example.test", "enabled": True},
  274. {"name": "active-b", "email_domain": "auto-next.example.test", "enabled": True},
  275. ],
  276. )
  277. worker_domains: list[list[str]] = []
  278. monkeypatch.setattr(provisioner, "_set_worker_domains", lambda domains: worker_domains.append(list(domains)))
  279. result = provisioner.normalize_to_domain_pool(2)
  280. assert result["active_domains"] == ["auto-live-a.example.test", "auto-next.example.test"]
  281. assert result["changed"] is True
  282. assert worker_domains == [["auto-live-a.example.test", "auto-next.example.test"]]
  283. def test_cfmail_provisioner_normalize_accounts_keeps_active_and_previous_auto_domain(tmp_path, monkeypatch) -> None:
  284. config_path = tmp_path / "cfmail.json"
  285. config_path.write_text(
  286. json.dumps(
  287. {
  288. "accounts": [
  289. {
  290. "name": "cfmail-auto-old1",
  291. "worker_domain": "email-api.example.test",
  292. "email_domain": "auto-old1.example.test",
  293. "admin_password": "pw",
  294. "enabled": True,
  295. },
  296. {
  297. "name": "manual-disabled",
  298. "worker_domain": "email-api.example.test",
  299. "email_domain": "manual.example.test",
  300. "admin_password": "pw",
  301. "enabled": False,
  302. },
  303. {
  304. "name": "cfmail-auto-live",
  305. "worker_domain": "email-api.example.test",
  306. "email_domain": "auto-live.example.test",
  307. "admin_password": "pw",
  308. "enabled": True,
  309. },
  310. ]
  311. },
  312. ensure_ascii=False,
  313. indent=2,
  314. )
  315. + "\n",
  316. encoding="utf-8",
  317. )
  318. provisioner = CfmailProvisioner(
  319. config_path=config_path,
  320. settings=ProvisioningSettings(
  321. auth_email="demo@example.com",
  322. auth_key="demo-key",
  323. account_id="acct",
  324. zone_id="zone",
  325. worker_name="worker",
  326. zone_name="example.test",
  327. ),
  328. )
  329. deleted_domains: list[str] = []
  330. monkeypatch.setattr(provisioner, "_delete_domain_artifacts", lambda domain: deleted_domains.append(domain))
  331. result = provisioner.normalize_accounts_to_single_active_domain()
  332. payload = json.loads(config_path.read_text(encoding="utf-8"))
  333. accounts = payload["accounts"]
  334. assert result == {
  335. "active_domain": "auto-live.example.test",
  336. "removed_domains": [],
  337. }
  338. assert [item["email_domain"] for item in accounts] == [
  339. "auto-old1.example.test",
  340. "manual.example.test",
  341. "auto-live.example.test",
  342. ]
  343. assert deleted_domains == []
  344. assert accounts[0]["enabled"] is False
  345. assert accounts[1]["enabled"] is False
  346. assert accounts[2]["enabled"] is True
  347. def test_cfmail_provisioner_normalize_accounts_keeps_only_latest_previous_auto_domain(tmp_path, monkeypatch) -> None:
  348. config_path = tmp_path / "cfmail.json"
  349. config_path.write_text(
  350. json.dumps(
  351. {
  352. "accounts": [
  353. {
  354. "name": "cfmail-auto-old1",
  355. "worker_domain": "email-api.example.test",
  356. "email_domain": "auto-old1.example.test",
  357. "admin_password": "pw",
  358. "enabled": False,
  359. },
  360. {
  361. "name": "cfmail-auto-old2",
  362. "worker_domain": "email-api.example.test",
  363. "email_domain": "auto-old2.example.test",
  364. "admin_password": "pw",
  365. "enabled": False,
  366. },
  367. {
  368. "name": "cfmail-auto-live",
  369. "worker_domain": "email-api.example.test",
  370. "email_domain": "auto-live.example.test",
  371. "admin_password": "pw",
  372. "enabled": True,
  373. },
  374. ]
  375. },
  376. ensure_ascii=False,
  377. indent=2,
  378. )
  379. + "\n",
  380. encoding="utf-8",
  381. )
  382. provisioner = CfmailProvisioner(
  383. config_path=config_path,
  384. settings=ProvisioningSettings(
  385. auth_email="demo@example.com",
  386. auth_key="demo-key",
  387. account_id="acct",
  388. zone_id="zone",
  389. worker_name="worker",
  390. zone_name="example.test",
  391. ),
  392. )
  393. deleted_domains: list[str] = []
  394. monkeypatch.setattr(provisioner, "_delete_domain_artifacts", lambda domain: deleted_domains.append(domain))
  395. result = provisioner.normalize_accounts_to_single_active_domain()
  396. payload = json.loads(config_path.read_text(encoding="utf-8"))
  397. accounts = payload["accounts"]
  398. assert result == {
  399. "active_domain": "auto-live.example.test",
  400. "removed_domains": ["auto-old1.example.test"],
  401. }
  402. assert deleted_domains == ["auto-old1.example.test"]
  403. assert [item["email_domain"] for item in accounts] == [
  404. "auto-old2.example.test",
  405. "auto-live.example.test",
  406. ]
  407. assert accounts[0]["enabled"] is False
  408. assert accounts[1]["enabled"] is True
  409. def test_cfmail_provisioner_smoke_test_retries_non_json_then_succeeds(monkeypatch) -> None:
  410. provisioner = CfmailProvisioner(
  411. settings=ProvisioningSettings(
  412. auth_email="demo@example.com",
  413. auth_key="demo-key",
  414. account_id="acct",
  415. zone_id="zone",
  416. worker_name="worker",
  417. zone_name="example.test",
  418. ),
  419. proxy_url="http://127.0.0.1:7899",
  420. )
  421. class FakeResponse:
  422. def __init__(self, status_code: int, text: str, payload=None):
  423. self.status_code = status_code
  424. self.text = text
  425. self._payload = payload
  426. self.content = text.encode("utf-8")
  427. def json(self):
  428. if self._payload is None:
  429. raise ValueError("not json")
  430. return self._payload
  431. responses = [
  432. FakeResponse(200, "<html>pending</html>", None),
  433. FakeResponse(200, '{"jwt":"x","address":"y"}', {"jwt": "x", "address": "y"}),
  434. FakeResponse(200, '{"jwt":"x","address":"y"}', {"jwt": "x", "address": "y"}),
  435. FakeResponse(200, '{"jwt":"x","address":"y"}', {"jwt": "x", "address": "y"}),
  436. ]
  437. def fake_post(*args, **kwargs):
  438. return responses.pop(0)
  439. monkeypatch.setattr(cffi_requests, "post", fake_post)
  440. monkeypatch.setattr("core.cfmail_provisioner.time.sleep", lambda *_args, **_kwargs: None)
  441. provisioner.smoke_test("email-api.example.test", "pw", "auto.example.test")
  442. def test_cfmail_provisioner_smoke_test_requires_multiple_successful_creates(monkeypatch) -> None:
  443. provisioner = CfmailProvisioner(
  444. settings=ProvisioningSettings(
  445. auth_email="demo@example.com",
  446. auth_key="demo-key",
  447. account_id="acct",
  448. zone_id="zone",
  449. worker_name="worker",
  450. zone_name="example.test",
  451. ),
  452. )
  453. class FakeResponse:
  454. def __init__(self) -> None:
  455. self.status_code = 200
  456. self.text = '{"jwt":"x","address":"y"}'
  457. self._payload = {"jwt": "x", "address": "y"}
  458. self.content = self.text.encode("utf-8")
  459. def json(self):
  460. return self._payload
  461. calls: list[tuple[tuple[object, ...], dict[str, object]]] = []
  462. def fake_post(*args, **kwargs):
  463. calls.append((args, kwargs))
  464. return FakeResponse()
  465. monkeypatch.setattr(cffi_requests, "post", fake_post)
  466. provisioner.smoke_test("email-api.example.test", "pw", "auto.example.test")
  467. assert len(calls) == 3
  468. def test_cfmail_provisioner_cleanup_stale_domains_removes_old_auto_resources(tmp_path, monkeypatch) -> None:
  469. config_path = tmp_path / 'cfmail.json'
  470. config_path.write_text(
  471. json.dumps({
  472. 'accounts': [
  473. {'name': 'old-1', 'worker_domain': 'email-api.demo', 'email_domain': 'auto-old1.example.test', 'admin_password': 'pw', 'enabled': False},
  474. {'name': 'old-2', 'worker_domain': 'email-api.demo', 'email_domain': 'auto-old2.example.test', 'admin_password': 'pw', 'enabled': False},
  475. {'name': 'keep', 'worker_domain': 'email-api.demo', 'email_domain': 'auto-keep.example.test', 'admin_password': 'pw', 'enabled': False},
  476. {'name': 'active', 'worker_domain': 'email-api.demo', 'email_domain': 'auto-live.example.test', 'admin_password': 'pw', 'enabled': True},
  477. {'name': 'base', 'worker_domain': 'email-api.demo', 'email_domain': 'inbox.example.test', 'admin_password': 'pw', 'enabled': False},
  478. ]
  479. }, ensure_ascii=False, indent=2) + "\n",
  480. encoding='utf-8',
  481. )
  482. provisioner = CfmailProvisioner(
  483. config_path=config_path,
  484. settings=ProvisioningSettings(
  485. auth_email='demo@example.com', auth_key='demo-key', account_id='acct', zone_id='zone', worker_name='worker', zone_name='example.test'
  486. ),
  487. )
  488. deleted_dns = []
  489. deleted_rules = []
  490. monkeypatch.setattr(provisioner, '_list_dns_records', lambda: [
  491. {'id': 'dns-old1', 'type': 'MX', 'name': 'auto-old1.example.test'},
  492. {'id': 'dns-old2', 'type': 'TXT', 'name': 'auto-old2.example.test'},
  493. {'id': 'dns-keep', 'type': 'MX', 'name': 'auto-keep.example.test'},
  494. {'id': 'dns-live', 'type': 'MX', 'name': 'auto-live.example.test'},
  495. ])
  496. monkeypatch.setattr(provisioner, '_list_email_routing_rules', lambda: [
  497. {'id': 'rule-old1', 'matchers': [{'field': 'to', 'value': '*@auto-old1.example.test'}]},
  498. {'id': 'rule-old2', 'matchers': [{'field': 'to', 'value': '*@auto-old2.example.test'}]},
  499. {'id': 'rule-keep', 'matchers': [{'field': 'to', 'value': '*@auto-keep.example.test'}]},
  500. ])
  501. monkeypatch.setattr(provisioner, '_delete_dns_record', lambda record_id: deleted_dns.append(record_id))
  502. monkeypatch.setattr(provisioner, '_delete_email_routing_rule', lambda rule_id: deleted_rules.append(rule_id))
  503. result = provisioner.cleanup_stale_domains()
  504. assert sorted(result['removed_domains']) == ['auto-keep.example.test', 'auto-old1.example.test', 'auto-old2.example.test']
  505. assert sorted(deleted_dns) == ['dns-keep', 'dns-old1', 'dns-old2']
  506. assert sorted(deleted_rules) == ['rule-keep', 'rule-old1', 'rule-old2']
  507. def test_cfmail_provisioner_cleanup_stale_domains_skips_read_only_artifacts(tmp_path, monkeypatch) -> None:
  508. config_path = tmp_path / "cfmail.json"
  509. config_path.write_text(
  510. json.dumps(
  511. {
  512. "accounts": [
  513. {
  514. "name": "old-1",
  515. "worker_domain": "email-api.demo",
  516. "email_domain": "auto-old1.example.test",
  517. "admin_password": "pw",
  518. "enabled": False,
  519. },
  520. {
  521. "name": "old-2",
  522. "worker_domain": "email-api.demo",
  523. "email_domain": "auto-old2.example.test",
  524. "admin_password": "pw",
  525. "enabled": False,
  526. },
  527. {
  528. "name": "active",
  529. "worker_domain": "email-api.demo",
  530. "email_domain": "auto-live.example.test",
  531. "admin_password": "pw",
  532. "enabled": True,
  533. },
  534. ]
  535. },
  536. ensure_ascii=False,
  537. indent=2,
  538. )
  539. + "\n",
  540. encoding="utf-8",
  541. )
  542. provisioner = CfmailProvisioner(
  543. config_path=config_path,
  544. settings=ProvisioningSettings(
  545. auth_email="demo@example.com",
  546. auth_key="demo-key",
  547. account_id="acct",
  548. zone_id="zone",
  549. worker_name="worker",
  550. zone_name="example.test",
  551. ),
  552. )
  553. deleted_dns: list[str] = []
  554. deleted_rules: list[str] = []
  555. monkeypatch.setattr(
  556. provisioner,
  557. "_list_dns_records",
  558. lambda: [
  559. {"id": "dns-old1", "type": "MX", "name": "auto-old1.example.test"},
  560. {"id": "dns-old2", "type": "TXT", "name": "auto-old2.example.test"},
  561. ],
  562. )
  563. monkeypatch.setattr(
  564. provisioner,
  565. "_list_email_routing_rules",
  566. lambda: [
  567. {"id": "rule-old1", "matchers": [{"field": "to", "value": "*@auto-old1.example.test"}]},
  568. {"id": "rule-old2", "matchers": [{"field": "to", "value": "*@auto-old2.example.test"}]},
  569. ],
  570. )
  571. def fake_delete_dns(record_id: str) -> None:
  572. deleted_dns.append(record_id)
  573. if record_id == "dns-old1":
  574. raise RuntimeError('HTTP 400 {"errors":[{"code":1043,"message":"DNS record is read only"}]}')
  575. def fake_delete_rule(rule_id: str) -> None:
  576. deleted_rules.append(rule_id)
  577. if rule_id == "rule-old1":
  578. raise RuntimeError('HTTP 400 {"errors":[{"code":1043,"message":"routing rule is read only"}]}')
  579. monkeypatch.setattr(provisioner, "_delete_dns_record", fake_delete_dns)
  580. monkeypatch.setattr(provisioner, "_delete_email_routing_rule", fake_delete_rule)
  581. result = provisioner.cleanup_stale_domains()
  582. assert sorted(result["removed_domains"]) == ["auto-old1.example.test", "auto-old2.example.test"]
  583. assert result["removed_dns_records"] == ["dns-old2"]
  584. assert result["removed_routing_rules"] == ["rule-old2"]
  585. def test_cfmail_provisioner_cleanup_stale_cf_resources_discovers_cf_side_orphans(tmp_path, monkeypatch) -> None:
  586. config_path = tmp_path / "cfmail.json"
  587. config_path.write_text(
  588. json.dumps(
  589. {
  590. "accounts": [
  591. {
  592. "name": "active",
  593. "worker_domain": "email-api.demo",
  594. "email_domain": "auto-live.example.test",
  595. "admin_password": "pw",
  596. "enabled": True,
  597. }
  598. ]
  599. },
  600. ensure_ascii=False,
  601. indent=2,
  602. )
  603. + "\n",
  604. encoding="utf-8",
  605. )
  606. provisioner = CfmailProvisioner(
  607. config_path=config_path,
  608. settings=ProvisioningSettings(
  609. auth_email="demo@example.com",
  610. auth_key="demo-key",
  611. account_id="acct",
  612. zone_id="zone",
  613. worker_name="worker",
  614. zone_name="example.test",
  615. ),
  616. )
  617. deleted_dns: list[str] = []
  618. deleted_rules: list[str] = []
  619. monkeypatch.setattr(
  620. provisioner,
  621. "_list_dns_records",
  622. lambda: [
  623. {"id": "dns-old1-mx", "type": "MX", "name": "auto-old1.example.test"},
  624. {"id": "dns-old1-txt", "type": "TXT", "name": "auto-old1.example.test"},
  625. {"id": "dns-old2-mx", "type": "MX", "name": "auto-old2.example.test"},
  626. {"id": "dns-active", "type": "MX", "name": "auto-live.example.test"},
  627. {"id": "dns-nova", "type": "MX", "name": "nova.example.test"},
  628. {"id": "dns-ignore", "type": "A", "name": "auto-old1.example.test"},
  629. {"id": "dns-outside", "type": "TXT", "name": "auto-old1.other.test"},
  630. ],
  631. )
  632. monkeypatch.setattr(
  633. provisioner,
  634. "_list_email_routing_rules",
  635. lambda: [
  636. {
  637. "id": "rule-old1",
  638. "name": "old1 subdomain catch-all",
  639. "matchers": [{"field": "to", "value": "*@auto-old1.example.test"}],
  640. },
  641. {
  642. "id": "rule-old2",
  643. "name": "old2 subdomain catch-all",
  644. "matchers": [{"field": "to", "value": "*@auto-old2.example.test"}],
  645. },
  646. {
  647. "id": "rule-active",
  648. "name": "active subdomain catch-all",
  649. "matchers": [{"field": "to", "value": "*@auto-live.example.test"}],
  650. },
  651. {
  652. "id": "rule-nova",
  653. "name": "nova keep",
  654. "matchers": [{"field": "to", "value": "*@nova.example.test"}],
  655. },
  656. ],
  657. )
  658. def fake_delete_dns(record_id: str) -> None:
  659. if record_id == "dns-old2-mx":
  660. raise RuntimeError("dns delete failed")
  661. deleted_dns.append(record_id)
  662. def fake_delete_rule(rule_id: str) -> None:
  663. if rule_id == "rule-old2":
  664. raise RuntimeError("rule delete failed")
  665. deleted_rules.append(rule_id)
  666. monkeypatch.setattr(provisioner, "_delete_dns_record", fake_delete_dns)
  667. monkeypatch.setattr(provisioner, "_delete_email_routing_rule", fake_delete_rule)
  668. result = provisioner.cleanup_stale_cf_resources()
  669. assert sorted(result["removed_dns_records"]) == ["dns-old1-mx", "dns-old1-txt"]
  670. assert result["removed_routing_rules"] == ["rule-old1"]
  671. assert len(result["errors"]) == 2
  672. assert any("dns-old2-mx" in error for error in result["errors"])
  673. assert any("rule-old2" in error for error in result["errors"])
  674. assert deleted_dns == ["dns-old1-mx", "dns-old1-txt"]
  675. assert deleted_rules == ["rule-old1"]
  676. def test_cfmail_provisioner_delete_domain_artifacts_skips_read_only_records(monkeypatch, tmp_path) -> None:
  677. provisioner = CfmailProvisioner(
  678. config_path=tmp_path / "cfmail.json",
  679. settings=ProvisioningSettings(
  680. auth_email="demo@example.com",
  681. auth_key="demo-key",
  682. account_id="acct",
  683. zone_id="zone",
  684. worker_name="worker",
  685. zone_name="example.test",
  686. ),
  687. )
  688. deleted_dns: list[str] = []
  689. deleted_rules: list[str] = []
  690. monkeypatch.setattr(
  691. provisioner,
  692. "_list_dns_records",
  693. lambda: [
  694. {"id": "dns-1", "name": "auto-old.example.test"},
  695. {"id": "dns-2", "name": "auto-old.example.test"},
  696. ],
  697. )
  698. monkeypatch.setattr(
  699. provisioner,
  700. "_list_email_routing_rules",
  701. lambda: [
  702. {"id": "rule-1", "matchers": [{"field": "to", "value": "*@auto-old.example.test"}]},
  703. {"id": "rule-2", "matchers": [{"field": "to", "value": "*@auto-old.example.test"}]},
  704. ],
  705. )
  706. def fake_delete_dns(record_id: str) -> None:
  707. deleted_dns.append(record_id)
  708. if record_id == "dns-1":
  709. raise RuntimeError("read only")
  710. def fake_delete_rule(rule_id: str) -> None:
  711. deleted_rules.append(rule_id)
  712. if rule_id == "rule-1":
  713. raise RuntimeError("read only")
  714. monkeypatch.setattr(provisioner, "_delete_dns_record", fake_delete_dns)
  715. monkeypatch.setattr(provisioner, "_delete_email_routing_rule", fake_delete_rule)
  716. provisioner._delete_domain_artifacts("auto-old.example.test")
  717. assert deleted_dns == ["dns-1", "dns-2"]
  718. assert deleted_rules == ["rule-1", "rule-2"]
  719. def test_cfmail_provisioner_rotate_retries_after_record_quota_cleanup(monkeypatch, tmp_path) -> None:
  720. config_path = tmp_path / 'cfmail.json'
  721. config_path.write_text(json.dumps({'accounts':[{'name':'active','worker_domain':'email-api.demo','email_domain':'auto-live.example.test','admin_password':'pw','enabled':True}]}) + "\n", encoding='utf-8')
  722. provisioner = CfmailProvisioner(
  723. config_path=config_path,
  724. settings=ProvisioningSettings(
  725. auth_email='demo@example.com', auth_key='demo-key', account_id='acct', zone_id='zone', worker_name='worker', zone_name='example.test'
  726. ),
  727. )
  728. labels = iter(['auto-old', 'auto-new'])
  729. monkeypatch.setattr(provisioner, '_make_new_label', lambda: next(labels))
  730. create_calls = []
  731. cleanup_calls = []
  732. switch_calls = []
  733. monkeypatch.setattr(provisioner, '_create_email_routing_rule', lambda domain, label: create_calls.append(('rule', domain, label)))
  734. def fake_create_dns(domain):
  735. create_calls.append(('dns', domain))
  736. if domain == 'auto-old.example.test':
  737. raise RuntimeError('POST dns failed: HTTP 400 {"errors":[{"code":81045,"message":"Record quota exceeded."}]}')
  738. monkeypatch.setattr(provisioner, '_create_dns_records', fake_create_dns)
  739. monkeypatch.setattr(provisioner, '_update_worker_domains', lambda domain, old_domain=None: create_calls.append(('worker', domain, old_domain)))
  740. monkeypatch.setattr(provisioner, 'smoke_test', lambda *args: create_calls.append(('smoke', args[2])))
  741. monkeypatch.setattr(provisioner, 'switch_active_domain', lambda **kwargs: switch_calls.append(kwargs) or [])
  742. monkeypatch.setattr(
  743. provisioner,
  744. 'cleanup_stale_cf_resources',
  745. lambda keep_domains=None: cleanup_calls.append(keep_domains) or {'removed_dns_records':['dns-old'], 'removed_routing_rules':['rule-old'], 'errors': []},
  746. )
  747. monkeypatch.setattr(provisioner, '_delete_domain_artifacts', lambda domain: create_calls.append(('cleanup_domain', domain)))
  748. result = provisioner.rotate_active_domain()
  749. assert result.success is True
  750. assert result.new_domain == 'auto-new.example.test'
  751. assert cleanup_calls == [None, {'auto-live.example.test'}]
  752. assert ('cleanup_domain', 'auto-old.example.test') in create_calls
  753. assert switch_calls[0]['new_domain'] == 'auto-new.example.test'
  754. def test_cfmail_provisioner_rotate_keeps_success_when_cleanup_after_switch_fails(monkeypatch, tmp_path) -> None:
  755. config_path = tmp_path / "cfmail.json"
  756. config_path.write_text(
  757. json.dumps(
  758. {
  759. "accounts": [
  760. {
  761. "name": "active",
  762. "worker_domain": "email-api.demo",
  763. "email_domain": "auto-live.example.test",
  764. "admin_password": "pw",
  765. "enabled": True,
  766. }
  767. ]
  768. }
  769. )
  770. + "\n",
  771. encoding="utf-8",
  772. )
  773. provisioner = CfmailProvisioner(
  774. config_path=config_path,
  775. settings=ProvisioningSettings(
  776. auth_email="demo@example.com",
  777. auth_key="demo-key",
  778. account_id="acct",
  779. zone_id="zone",
  780. worker_name="worker",
  781. zone_name="example.test",
  782. ),
  783. )
  784. switch_calls: list[dict[str, str]] = []
  785. monkeypatch.setattr(provisioner, "_make_new_label", lambda: "auto-next")
  786. monkeypatch.setattr(provisioner, "_create_email_routing_rule", lambda domain, label: None)
  787. monkeypatch.setattr(provisioner, "_create_dns_records", lambda domain: None)
  788. monkeypatch.setattr(provisioner, "_update_worker_domains", lambda domain, old_domain=None: None)
  789. monkeypatch.setattr(provisioner, "smoke_test", lambda *args: None)
  790. monkeypatch.setattr(
  791. provisioner,
  792. "switch_active_domain",
  793. lambda **kwargs: switch_calls.append(kwargs) or [],
  794. )
  795. def fake_cleanup(keep_domains=None): # type: ignore[no-untyped-def]
  796. raise RuntimeError("HTTP 400 read only")
  797. monkeypatch.setattr(provisioner, "cleanup_stale_cf_resources", fake_cleanup)
  798. monkeypatch.setattr(provisioner, "_delete_domain_artifacts", lambda domain: (_ for _ in ()).throw(AssertionError("should not rollback")))
  799. result = provisioner.rotate_active_domain()
  800. assert result.success is True
  801. assert result.new_domain == "auto-next.example.test"
  802. assert switch_calls[0]["new_domain"] == "auto-next.example.test"
  803. def test_cfmail_provisioner_patch_worker_settings_uses_multipart_request_and_total_timeout(tmp_path, monkeypatch) -> None:
  804. provisioner = CfmailProvisioner(
  805. config_path=tmp_path / "cfmail.json",
  806. settings=ProvisioningSettings(
  807. auth_email="demo@example.com",
  808. auth_key="demo-key",
  809. account_id="acct",
  810. zone_id="zone",
  811. worker_name="worker",
  812. zone_name="example.test",
  813. ),
  814. proxy_url="http://127.0.0.1:7890",
  815. )
  816. captured: dict[str, object] = {}
  817. class FakeResponse:
  818. status_code = 200
  819. content = b'{"success":true}'
  820. def json(self): # type: ignore[no-untyped-def]
  821. return {"success": True}
  822. def fake_patch(url, **kwargs): # type: ignore[no-untyped-def]
  823. captured["url"] = url
  824. captured["kwargs"] = kwargs
  825. return FakeResponse()
  826. monkeypatch.setattr(cffi_requests, "patch", fake_patch)
  827. provisioner._patch_worker_settings([{"name": "DOMAINS", "type": "json", "json": ["auto-live.example.test"]}])
  828. kwargs = captured["kwargs"]
  829. assert captured["url"] == "https://api.cloudflare.com/client/v4/accounts/acct/workers/scripts/worker/settings"
  830. assert kwargs["timeout"] == 30
  831. assert kwargs["proxies"] == {"http": "http://127.0.0.1:7890", "https": "http://127.0.0.1:7890"}
  832. assert kwargs["headers"]["Content-Type"].startswith("multipart/form-data; boundary=")
  833. assert b'"bindings":[{"name":"DOMAINS","type":"json","json":["auto-live.example.test"]}]' in kwargs["data"]
  834. def test_cfmail_provisioner_update_worker_domains_keeps_new_and_previous_domain_only(tmp_path, monkeypatch) -> None:
  835. provisioner = CfmailProvisioner(
  836. config_path=tmp_path / "cfmail.json",
  837. settings=ProvisioningSettings(
  838. auth_email="demo@example.com",
  839. auth_key="demo-key",
  840. account_id="acct",
  841. zone_id="zone",
  842. worker_name="worker",
  843. zone_name="example.test",
  844. ),
  845. )
  846. patched_bindings: list[dict[str, object]] = []
  847. monkeypatch.setattr(
  848. provisioner,
  849. "_get_worker_settings",
  850. lambda: {
  851. "bindings": [
  852. {"name": "DOMAINS", "type": "json", "json": ["auto-old1.example.test", "auto-old2.example.test"]},
  853. {"name": "DEFAULT_DOMAINS", "type": "json", "json": ["auto-old1.example.test", "auto-old2.example.test"]},
  854. {"name": "UNRELATED", "type": "plain_text", "text": "keep"},
  855. ]
  856. },
  857. )
  858. monkeypatch.setattr(provisioner, "_patch_worker_settings", lambda bindings: patched_bindings.extend(bindings))
  859. provisioner._update_worker_domains("auto-live.example.test", old_domain="auto-old2.example.test")
  860. assert patched_bindings == [
  861. {"name": "DOMAINS", "type": "json", "json": ["auto-live.example.test", "auto-old2.example.test"]},
  862. {"name": "DEFAULT_DOMAINS", "type": "json", "json": ["auto-live.example.test", "auto-old2.example.test"]},
  863. {"name": "UNRELATED", "type": "plain_text", "text": "keep"},
  864. ]
  865. def test_cfmail_provisioner_rotate_keeps_previous_domain_artifacts_for_dual_domain_grace(monkeypatch, tmp_path) -> None:
  866. config_path = tmp_path / "cfmail.json"
  867. config_path.write_text(
  868. json.dumps(
  869. {
  870. "accounts": [
  871. {
  872. "name": "active",
  873. "worker_domain": "email-api.demo",
  874. "email_domain": "auto-live.example.test",
  875. "admin_password": "pw",
  876. "enabled": True,
  877. }
  878. ]
  879. }
  880. )
  881. + "\n",
  882. encoding="utf-8",
  883. )
  884. provisioner = CfmailProvisioner(
  885. config_path=config_path,
  886. settings=ProvisioningSettings(
  887. auth_email="demo@example.com",
  888. auth_key="demo-key",
  889. account_id="acct",
  890. zone_id="zone",
  891. worker_name="worker",
  892. zone_name="example.test",
  893. ),
  894. )
  895. patched_worker_domains: list[tuple[str, str | None]] = []
  896. deleted_domains: list[str] = []
  897. cleanup_keep_domains: list[set[str]] = []
  898. monkeypatch.setattr(provisioner, "_make_new_label", lambda: "auto-next")
  899. monkeypatch.setattr(provisioner, "_create_email_routing_rule", lambda domain, label: None)
  900. monkeypatch.setattr(provisioner, "_create_dns_records", lambda domain: None)
  901. monkeypatch.setattr(
  902. provisioner,
  903. "_update_worker_domains",
  904. lambda domain, old_domain=None: patched_worker_domains.append((domain, old_domain)),
  905. )
  906. monkeypatch.setattr(provisioner, "smoke_test", lambda *args: None)
  907. monkeypatch.setattr(
  908. provisioner,
  909. "_delete_domain_artifacts",
  910. lambda domain: deleted_domains.append(domain),
  911. )
  912. monkeypatch.setattr(
  913. provisioner,
  914. "cleanup_stale_cf_resources",
  915. lambda keep_domains=None: cleanup_keep_domains.append(set(keep_domains or [])) or {
  916. "removed_domains": [],
  917. "removed_dns_records": [],
  918. "removed_routing_rules": [],
  919. "errors": [],
  920. },
  921. )
  922. result = provisioner.rotate_active_domain()
  923. assert result.success is True
  924. assert patched_worker_domains == [("auto-next.example.test", "auto-live.example.test")]
  925. assert deleted_domains == []
  926. assert cleanup_keep_domains == [{"auto-live.example.test"}]
  927. def test_domain_health_tracker_default_thresholds_are_aggressive(monkeypatch) -> None:
  928. monkeypatch.delenv("ZHUCE6_CFMAIL_ROTATION_WINDOW", raising=False)
  929. monkeypatch.delenv("ZHUCE6_CFMAIL_ROTATION_BLACKLIST_THRESHOLD", raising=False)
  930. monkeypatch.delenv("ZHUCE6_CFMAIL_REGISTRATION_DISALLOWED_THRESHOLD", raising=False)
  931. monkeypatch.delenv("ZHUCE6_CFMAIL_ROTATION_MAX_SUCCESSES", raising=False)
  932. tracker = DomainHealthTracker()
  933. assert tracker.window_size == 10
  934. assert tracker.blacklist_threshold == 6
  935. assert tracker.registration_disallowed_threshold == 2
  936. assert tracker.max_successes_in_window == 2