support underscore-prefixed domains for running core-tests, allowing for faster and independent python test runs

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -11,6 +11,7 @@
 import subprocess
 import sys
 import time
+import urllib.parse
 from typing import AsyncGenerator, Optional
 
 import pytest
@@ -305,10 +306,16 @@ def factory(core_version):
 
         accounts_dir = str(tmp_path.joinpath("account1_venv1"))
         channel = gw.remote_exec(remote_bob_loop)
-        cm = os.environ.get("CHATMAIL_DOMAIN")
+
+        # old cores need "ic=3" to accept
+        # the self-signed cert of an underscore domain
+        addr, password = acfactory.get_credentials()
+        dclogin_qr = f"dclogin://{urllib.parse.quote(addr, safe='@')}?p={urllib.parse.quote(password)}&v=1"
+        if os.environ["CHATMAIL_DOMAIN"].startswith("_"):
+            dclogin_qr += "&ic=3"
 
         # trigger getting an online account on bob's side
-        channel.send((accounts_dir, str(rpc_server_path), cm))
+        channel.send((accounts_dir, str(rpc_server_path), dclogin_qr))
 
         # meanwhile get a local alice account
         alice = acfactory.get_online_account()
@@ -340,10 +347,8 @@ def remote_bob_loop(channel):
     import os
 
     from deltachat_rpc_client import DeltaChat, Rpc
-    from deltachat_rpc_client.pytestplugin import ACFactory
 
-    accounts_dir, rpc_server_path, chatmail_domain = channel.receive()
-    os.environ["CHATMAIL_DOMAIN"] = chatmail_domain
+    accounts_dir, rpc_server_path, dclogin_qr = channel.receive()
 
     # older core versions don't support specifying rpc_server_path
     # so we can't just pass `rpc_server_path` argument to Rpc constructor
@@ -354,8 +359,13 @@ def remote_bob_loop(channel):
     with rpc:
         dc = DeltaChat(rpc)
         channel.send(dc.rpc.get_system_info()["deltachat_core_version"])
-        acfactory = ACFactory(dc)
-        bob = acfactory.get_online_account()
+
+        # ACFactory would configure from a "dcaccount" QR,
+        # which old cores cannot use on underscore domains
+        bob = dc.add_account()
+        bob.add_transport_from_qr(dclogin_qr)
+        bob.bring_online()
+
         alice_vcard = channel.receive()
         [alice_contact] = bob.import_vcard(alice_vcard)
         ns = {"bob": bob, "bob_contact_alice": alice_contact}

deltachat-rpc-client/tests/conftest.py

@@ -37,7 +37,11 @@ def connect(self):
         host = user.rsplit("@")[-1]
         pw = self.account.get_config("mail_pw")
 
-        self.conn = MailBox(host, port, ssl_context=ssl.create_default_context())
+        ssl_context = ssl.create_default_context()
+        if host.startswith("_"):
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+        self.conn = MailBox(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")

deltachat-rpc-client/tests/test_chatlist_events.py

@@ -110,7 +110,7 @@ def test_delivery_status_failed(acfactory: ACFactory) -> None:
     assert failing_message.get_snapshot().state == const.MessageState.OUT_FAILED
 
 
-def test_download_on_demand(acfactory: ACFactory) -> None:
+def test_download_on_demand(acfactory: ACFactory, data) -> None:
     """
     Test if download on demand emits chatlist update events.
     This is only needed for last message in chat, but finding that out is too expensive, so it's always emitted
@@ -128,7 +128,7 @@ def test_download_on_demand(acfactory: ACFactory) -> None:
     msg.get_snapshot().chat.accept()
     bob.get_chat_by_id(chat_id).send_message(
         "Hello World, this message is bigger than 5 bytes",
-        file="../test-data/image/screenshot.jpg",
+        file=data.get_path("image/screenshot.jpg"),
     )
 
     message = alice.wait_for_incoming_msg()

deltachat-rpc-client/tests/test_cross_core.py

@@ -45,7 +45,7 @@ def test_qr_setup_contact(acfactory, alice_and_remote_bob, version) -> None:
 
 def test_send_and_receive_message(alice_and_remote_bob) -> None:
     """Test other-core Bob profile can send a message to Alice on current core."""
-    alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.20.0")
+    alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.23.0")
 
     remote_eval("bob_contact_alice.create_chat().send_text('hello')")
 
@@ -55,7 +55,7 @@ def test_send_and_receive_message(alice_and_remote_bob) -> None:
 
 def test_second_device(acfactory, alice_and_remote_bob) -> None:
     """Test setting up current version as a second device for old version."""
-    _alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.20.0")
+    _alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.23.0")
 
     remote_eval("locals().setdefault('future', bob._rpc.provide_backup.future(bob.id))")
     qr = remote_eval("bob._rpc.get_backup_qr(bob.id)")

deltachat-rpc-client/tests/test_iroh_webxdc.py

@@ -17,6 +17,15 @@
 
 from deltachat_rpc_client import EventType
 
+# Relays on underscore domains advertise themselves as iroh relay
+# but serve a self-signed certificate that iroh's TLS stack rejects.
+# Skipping instead of xfailing keeps the run fast:
+# these tests only fail after waiting for realtime connections to time out.
+pytestmark = pytest.mark.skipif(
+    os.environ.get("CHATMAIL_DOMAIN", "").startswith("_"),
+    reason="iroh does not accept the self-signed certificate of an underscore domain",
+)
+
 
 @pytest.fixture
 def path_to_webxdc(request):

deltachat-rpc-client/tests/test_multitransport.py

@@ -76,7 +76,7 @@ def test_change_address(acfactory) -> None:
     assert sender_addr2 == new_alice_addr
 
 
-def test_download_on_demand(acfactory) -> None:
+def test_download_on_demand(acfactory, data) -> None:
     alice, bob = acfactory.get_online_accounts(2)
     alice.set_config("download_limit", "1")
 
@@ -87,7 +87,7 @@ def test_download_on_demand(acfactory) -> None:
 
     alice.create_chat(bob)
     chat_bob_alice = bob.create_chat(alice)
-    chat_bob_alice.send_message(file="../test-data/image/screenshot.jpg")
+    chat_bob_alice.send_message(file=data.get_path("image/screenshot.jpg"))
     msg = alice.wait_for_incoming_msg()
     snapshot = msg.get_snapshot()
     assert snapshot.download_state == DownloadState.AVAILABLE

deltachat-rpc-client/tests/test_something.py

@@ -1412,7 +1412,7 @@ def test_synchronize_member_list_on_group_rejoin(acfactory, log):
     assert msg.get_snapshot().chat.num_contacts() == 2
 
 
-def test_large_message(acfactory) -> None:
+def test_large_message(acfactory, data) -> None:
     """
     Test sending large message without download limit set,
     so it is sent with pre-message but downloaded without user interaction.
@@ -1422,7 +1422,7 @@ def test_large_message(acfactory) -> None:
     alice_chat_bob = alice.create_chat(bob)
     alice_chat_bob.send_message(
         "Hello World, this message is bigger than 5 bytes",
-        file="../test-data/image/screenshot.jpg",
+        file=data.get_path("image/screenshot.jpg"),
     )
 
     msg = bob.wait_for_incoming_msg()

deltachat-rpc-client/tests/test_webxdc.py

@@ -1,9 +1,9 @@
-def test_webxdc(acfactory) -> None:
+def test_webxdc(acfactory, data) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
     alice_contact_bob = alice.create_contact(bob, "Bob")
     alice_chat_bob = alice_contact_bob.create_chat()
-    alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
+    alice_chat_bob.send_message(text="Let's play chess!", file=data.get_path("webxdc/chess.xdc"))
 
     event = bob.wait_for_incoming_msg_event()
     bob_chat_alice = bob.get_chat_by_id(event.chat_id)
@@ -43,12 +43,12 @@ def test_webxdc(acfactory) -> None:
     ]
 
 
-def test_webxdc_insert_lots_of_updates(acfactory) -> None:
+def test_webxdc_insert_lots_of_updates(acfactory, data) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
     alice_contact_bob = alice.create_contact(bob, "Bob")
     alice_chat_bob = alice_contact_bob.create_chat()
-    message = alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
+    message = alice_chat_bob.send_message(text="Let's play chess!", file=data.get_path("webxdc/chess.xdc"))
 
     for i in range(2000):
         message.send_webxdc_status_update({"payload": str(i)}, "description")

python/src/deltachat/direct_imap.py

@@ -43,9 +43,14 @@ def connect(self):
         host = user.rsplit("@")[-1]
         pw = self.account.get_config("mail_pw")
 
+        ssl_context = ssl.create_default_context()
+        if host.startswith("_"):
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+
         while True:
             try:
-                self.conn = MailBox(host, port, ssl_context=ssl.create_default_context())
+                self.conn = MailBox(host, port, ssl_context=ssl_context)
                 self.conn.login(user, pw)
                 self.select_folder("INBOX")
                 return

src/net/http.rs

@@ -258,6 +258,15 @@ pub(crate) async fn http_cache_cleanup(context: &Context) -> Result<()> {
     Ok(())
 }
 
+/// Returns the request target in origin form, i.e. the path and query of `url`.
+///
+/// The absolute form is only for proxy requests and
+/// nginx rejects it if the host starts contains an underscore.
+fn origin_form(url: &hyper::Uri) -> &str {
+    url.path_and_query()
+        .map_or("/", |path_and_query| path_and_query.as_str())
+}
+
 /// Fetches URL and updates the cache.
 ///
 /// URL is fetched regardless of whether there is an existing result in the cache.
@@ -276,7 +285,7 @@ async fn fetch_url(context: &Context, original_url: &str, strict_tls: bool) -> R
             .context("URL has no authority")?
             .clone();
 
-        let req = hyper::Request::builder().uri(parsed_url);
+        let req = hyper::Request::builder().uri(origin_form(&parsed_url));
 
         // OSM usage policy requires
         // that User-Agent is set for HTTP GET requests
@@ -409,7 +418,7 @@ pub(crate) async fn post_empty(context: &Context, url: &str) -> Result<(String,
         .authority()
         .context("URL has no authority")?
         .clone();
-    let req = hyper::Request::post(parsed_url)
+    let req = hyper::Request::post(origin_form(&parsed_url))
         .header(hyper::header::HOST, authority.as_str())
         .body(http_body_util::Empty::<Bytes>::new())?;
 
@@ -432,6 +441,20 @@ mod tests {
     use crate::test_utils::TestContext;
     use crate::tools::SystemTime;
 
+    #[test]
+    fn test_origin_form() {
+        let url = "https://_cm0.localchat/autoconfig?emailaddress=x%40_cm0.localchat"
+            .parse()
+            .unwrap();
+        assert_eq!(
+            origin_form(&url),
+            "/autoconfig?emailaddress=x%40_cm0.localchat"
+        );
+
+        let url = "https://example.org".parse().unwrap();
+        assert_eq!(origin_form(&url), "/");
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_http_cache() -> Result<()> {
         let t = &TestContext::new().await;