feat(search): the server picks the Messages list's order and names its terms

CHANGELOG.md

@@ -45,7 +45,17 @@ released versions carry their date on the heading.
   is a sentence that names the item, such as "The file sms-2.xml could not
   be read in full: …", where it was `error: sms-2.xml: This file could not
   be read in full: …` before.
-
+- 2026-10-05: **The server picks the order of the Messages list.** With no
+  `sort`, `GET /v1/messages` now puts the best match first when the search
+  has a free-text word to rank by, and the newest message first when it has
+  none. It used to list the oldest first. Each page says which order it
+  applied and which words it ranks by, in a new `search` key beside `items`,
+  and the web app reads both from there: the sort menu offers Relevance only
+  when words came back, and the list draws those words in bold. Nothing
+  changes on screen, except that a search the web app and the server once
+  read differently now shows the server's reading. Picking Relevance now
+  leaves `sort` out of the address, so the next search starts in the
+  server's order.
 - 2026-10-05: **A message's reply count is counted when it is read.** It is
   the number of replies Message Crate shows that quote the message, so a
   reply hidden as a duplicate no longer counts, and a reply that quotes a

crates/server/server/src/db/conversation_messages.rs

@@ -189,18 +189,59 @@ pub enum MessageListSort {
     Relevance,
 }
 
+impl MessageListSort {
+    /// The key as `sort=` spells it.
+    #[must_use]
+    pub const fn name(self) -> &'static str {
+        match self {
+            Self::Date => "date",
+            Self::Relevance => "relevance",
+        }
+    }
+}
+
 /// The Messages list's keys, as `sort=` spells them.
 pub const MESSAGE_LIST_SORT_KEYS: [(&str, MessageListSort); 2] = [
-    ("date", MessageListSort::Date),
-    ("relevance", MessageListSort::Relevance),
+    (MessageListSort::Date.name(), MessageListSort::Date),
+    (
+        MessageListSort::Relevance.name(),
+        MessageListSort::Relevance,
+    ),
 ];
 
-/// Oldest first, as [`DEFAULT_MESSAGE_SORT`] reads a conversation when `sort`
-/// is absent.
-pub const DEFAULT_MESSAGE_LIST_SORT: [SortKey<MessageListSort>; 1] = [SortKey {
-    key: MessageListSort::Date,
-    direction: Direction::Asc,
-}];
+/// The order the Messages list applies when the request names no `sort`:
+/// best match first when the query has a free-text word to rank by
+/// (`ranked`), and newest first otherwise. A search with words is looking for
+/// the messages that hold them, and one with only field words is browsing,
+/// where the latest messages come first (#1538).
+#[must_use]
+pub fn default_message_list_sort(ranked: bool) -> [SortKey<MessageListSort>; 1] {
+    if ranked {
+        [SortKey {
+            key: MessageListSort::Relevance,
+            direction: Direction::Asc,
+        }]
+    } else {
+        [SortKey {
+            key: MessageListSort::Date,
+            direction: Direction::Desc,
+        }]
+    }
+}
+
+/// `order` spelled as `sort=` takes it: `relevance`, `date`, `-date`, or
+/// keys joined by commas, such as `relevance,-date`.
+#[must_use]
+pub fn message_list_sort_text(order: &[SortKey<MessageListSort>]) -> String {
+    order
+        .iter()
+        .map(|k| match k.direction {
+            Direction::Asc => k.key.name().to_string(),
+            Direction::Desc => format!("-{}", k.key.name()),
+        })
+        .collect::<Vec<_>>()
+        .join(",")
+}
 
 /// The join a relevance order ranks by: every message the rank query
 /// matches, with its `bm25()`, keyed by message id. Its one `?` is the rank
@@ -360,7 +401,7 @@ pub(crate) fn message_list_page_sql(
             ));
         };
         from_sql.push_str(RANK_JOIN_SQL);
-        params.push(SqlParam::Text(rank_query.to_string()));
+        params.push(SqlParam::Text(rank_query));
     }
     params.extend_from_slice(filter.params());
 

crates/server/server/src/messages_api.rs

@@ -9,15 +9,79 @@
 
 use crate::extract::{Json, Path, Query};
 use axum::extract::State;
+use serde::Serialize;
 
 use crate::db::conversation_messages::{
-    DEFAULT_MESSAGE_LIST_SORT, DEFAULT_MESSAGE_SORT, MESSAGE_LIST_SORT_KEYS, Message,
-    count_matching_messages, load_message_list_page, load_messages,
+    DEFAULT_MESSAGE_SORT, MESSAGE_LIST_SORT_KEYS, Message, count_matching_messages,
+    default_message_list_sort, load_message_list_page, load_messages, message_list_sort_text,
 };
 use crate::db::sql::SqlParam;
-use crate::paging::{ListRequest, Page, PageQuery};
+use crate::paging::{ListRequest, PageQuery};
+use crate::search::parse::TextTerm;
 use crate::server::{ApiError, AppState, FullAccess};
 
+/// One page of the Messages list and how the server read its search: the
+/// four keys of every page, and `search`, the one key a page carries beside
+/// them (`docs/architecture/http-api.md`, "Lists").
+// The four keys are written out rather than taken from `Page<Message>` with
+// `#[serde(flatten)]`: utoipa describes a flattened field as an `allOf` of
+// two schemas, which gives the page no `properties` of its own for the page
+// rules (`openapi/document_rules.rs`) and the generated web types to read.
+#[derive(Debug, Serialize, utoipa::ToSchema)]
+pub(crate) struct ListMessagesResponse {
+    /// The rows on this page.
+    pub(crate) items: Vec<Message>,
+    /// Rows matching the query across every page.
+    pub(crate) total: u64,
+    /// Page size used.
+    pub(crate) limit: usize,
+    /// Page offset used.
+    pub(crate) offset: usize,
+    /// How the server read `q` and `sort`: the order it applied and the
+    /// free-text terms it ranks by. It describes the query, not the rows, so
+    /// every page of one query carries the same value.
+    pub(crate) search: MessageSearch,
+}
+
+/// A Messages search as the server read it.
+#[derive(Debug, Serialize, utoipa::ToSchema)]
+pub(crate) struct MessageSearch {
+    /// The order the page is in, spelled as `sort` takes it: the `sort` the
+    /// request named, or with none, `relevance` when `terms` is not empty and
+    /// `-date` (newest first) when it is.
+    pub(crate) sort: String,
+    /// The free-text terms the search ranks by, in the order they were typed:
+    /// every word and quoted phrase of `q` that is not a field word and not
+    /// behind `-` or `not`, alone or in a negated group. Empty when `q` has
+    /// none, and then `relevance` is refused.
+    pub(crate) terms: Vec<FreeTextTerm>,
+}
+
+/// One free-text term of a search: a word, or a quoted phrase.
+#[derive(Debug, Serialize, utoipa::ToSchema)]
+pub(crate) struct FreeTextTerm {
+    /// The word, or the phrase without its quotes, as typed.
+    pub(crate) text: String,
+    /// True when the word ended in `*` and matches any word it begins; the
+    /// `*` is not in `text`. Always false for a phrase.
+    pub(crate) prefix: bool,
+}
+
+impl From<&TextTerm> for FreeTextTerm {
+    fn from(term: &TextTerm) -> Self {
+        match term {
+            TextTerm::Term { text, prefix } => Self {
+                text: text.clone(),
+                prefix: *prefix,
+            },
+            TextTerm::Phrase(text) => Self {
+                text: text.clone(),
+                prefix: false,
+            },
+        }
+    }
+}
+
 /// Compile a query against the Messages list of the search language.
 ///
 /// # Errors
@@ -39,9 +103,14 @@ pub(crate) fn message_filter(
     })?)
 }
 
-/// Messages matching `q`, oldest first unless `sort` says otherwise: the same
-/// rows an Export Run with a `query` scope would hand over, behind a logged-in
-/// session with the list defaults and the list's offset ceiling.
+/// Messages matching `q`: the same rows an Export Run with a `query` scope
+/// would hand over, behind a logged-in session with the list defaults and the
+/// list's offset ceiling.
+///
+/// With no `sort`, the best match comes first when `q` has a free-text word
+/// to rank by, and the newest message first when it has none. The page's
+/// `search` says which order it applied and which terms it ranks by, so a
+/// client never parses `q` itself.
 ///
 /// `sort=relevance` puts the best match first, ranked by the full-text
 /// index's `bm25()` on the query's free-text words: the words not behind `-`
@@ -58,42 +127,57 @@ pub(crate) fn message_filter(
         ("q" = Option<String>, Query, description = "Search query in the Messages list's words; empty matches every message"),
         ("limit" = Option<usize>, Query, description = "Page size, default 40, max 500"),
         ("offset" = Option<usize>, Query, description = "Page offset, max 50000"),
-        ("sort" = Option<String>, Query, description = "`date`, `-date` or `relevance` (best match first; needs a free-text word in `q`). Default `date`, oldest first.")
+        ("sort" = Option<String>, Query, description = "`date`, `-date` or `relevance` (best match first; needs a free-text word in `q`). Default `relevance` when `q` has a free-text word, and `-date`, newest first, when it has none.")
     ),
     responses(
-        (status = 200, body = crate::paging::Page<Message>),
+        (status = 200, body = ListMessagesResponse),
         crate::problem::openapi::SearchQueryInvalid
     )
 )]
 pub(crate) async fn list_messages(
     State(state): State<AppState>,
     FullAccess(auth): FullAccess,
     Query(query): Query<PageQuery>,
-) -> Result<Json<Page<Message>>, ApiError> {
+) -> Result<Json<ListMessagesResponse>, ApiError> {
     let mut conn = state.db.acquire().await?;
+    // No default here: which one applies depends on the query, which is
+    // compiled only after the sort has been checked.
     let list = ListRequest::read(
         &mut conn,
         auth.account_id,
         query,
         &MESSAGE_LIST_SORT_KEYS,
-        &DEFAULT_MESSAGE_LIST_SORT,
+        &[],
     )
     .await?;
     let filter = message_filter(auth.account_id, &list.q, list.clock)?;
+    let order = if list.order.is_empty() {
+        default_message_list_sort(!filter.ranked_terms().is_empty()).to_vec()
+    } else {
+        list.order
+    };
     let items = load_message_list_page(
         &mut conn,
         &filter,
-        &list.order,
+        &order,
         list.page.limit,
         list.page.offset,
     )
     .await?;
     let total = count_matching_messages(&mut conn, &filter).await?;
-    Ok(Json(Page {
+    Ok(Json(ListMessagesResponse {
         items,
         total,
         limit: list.page.limit,
         offset: list.page.offset,
+        search: MessageSearch {
+            sort: message_list_sort_text(&order),
+            terms: filter
+                .ranked_terms()
+                .iter()
+                .map(FreeTextTerm::from)
+                .collect(),
+        },
     }))
 }
 

crates/server/server/src/messages_api/tests.rs

@@ -101,20 +101,22 @@ async fn the_messages_route_is_a_page_across_every_conversation() {
     assert_eq!(page["limit"], serde_json::json!(40));
     assert_eq!(page["offset"], serde_json::json!(0));
     assert_eq!(page["items"].as_array().unwrap().len(), 3);
-    // `docs/architecture/http-api.md`: a list is {items, total, limit, offset} and nothing else.
+    // `docs/architecture/http-api.md`, "Lists": a list is {items, total,
+    // limit, offset}, and the Messages list adds `search`, its one exception.
     let keys: Vec<&str> = page
         .as_object()
         .unwrap()
         .keys()
         .map(String::as_str)
         .collect();
-    assert_eq!(keys, ["items", "limit", "offset", "total"]);
+    assert_eq!(keys, ["items", "limit", "offset", "search", "total"]);
 }
 
 #[tokio::test]
 async fn a_page_across_two_conversations_names_each_conversations_own_participants() {
     let (fixture, alice, direct, group) = seeded().await;
-    let page: serde_json::Value = get_json(&fixture.state, "/v1/messages", &alice.token).await;
+    let page: serde_json::Value =
+        get_json(&fixture.state, "/v1/messages?sort=date", &alice.token).await;
 
     let handles: Vec<(i64, &str)> = page["items"]
         .as_array()
@@ -1970,6 +1972,102 @@ async fn relevance_ranks_by_the_positive_words_only() {
     assert_eq!(texts(&page)[0], "the dentist moved it", "{page}");
 }
 
+/// With no `sort`, a search with a free-text word puts the best match first
+/// and says so, and a search with only field words lists the newest message
+/// first and says so (#1538). The client reads the order back rather than
+/// parsing `q` to guess it.
+#[tokio::test]
+async fn with_no_sort_the_server_picks_the_order_and_reports_it() {
+    let (fixture, alice) = seeded_for_relevance().await;
+    let page: serde_json::Value =
+        get_json(&fixture.state, "/v1/messages?q=dentist", &alice.token).await;
+    assert_eq!(
+        texts(&page),
+        [
+            "dentist dentist dentist",
+            "the dentist moved it",
+            "after work I will call the office of the dentist about next week",
+        ],
+        "{page}"
+    );
+    assert_eq!(
+        page["search"],
+        serde_json::json!({"sort": "relevance", "terms": [{"text": "dentist", "prefix": false}]})
+    );
+
+    let page: serde_json::Value =
+        get_json(&fixture.state, "/v1/messages?q=date%3A2024", &alice.token).await;
+    assert_eq!(
+        texts(&page),
+        [
+            "nothing to see",
+            "the dentist moved it",
+            "after work I will call the office of the dentist about next week",
+            "dentist dentist dentist",
+        ],
+        "{page}"
+    );
+    assert_eq!(
+        page["search"],
+        serde_json::json!({"sort": "-date", "terms": []})
+    );
+}
+
+/// A `sort` the request names is applied as it is, and reported in the
+/// spelling `sort` takes, keys and signs included.
+#[tokio::test]
+async fn a_named_sort_is_applied_and_reported() {
+    let (fixture, alice) = seeded_for_relevance().await;
+    for (sort, reported) in [
+        ("date", "date"),
+        ("-DATE", "-date"),
+        ("relevance,date", "relevance,date"),
+    ] {
+        let page: serde_json::Value = get_json(
+            &fixture.state,
+            &format!("/v1/messages?q=dentist&sort={sort}"),
+            &alice.token,
+        )
+        .await;
+        assert_eq!(
+            page["search"]["sort"],
+            serde_json::json!(reported),
+            "{sort}: {page}"
+        );
+    }
+    let page: serde_json::Value = get_json(
+        &fixture.state,
+        "/v1/messages?q=dentist&sort=date",
+        &alice.token,
+    )
+    .await;
+    assert_eq!(texts(&page)[0], "dentist dentist dentist", "{page}");
+}
+
+/// The terms a search ranks by are the free-text words and phrases a match
+/// must or may have, in the order typed: a word behind `-` or `not`, a word
+/// in a negated group, and a field word's value are none of them. A prefix
+/// keeps its `*` as a flag, and a phrase comes back without its quotes.
+#[tokio::test]
+async fn the_page_names_the_terms_the_search_ranks_by() {
+    let (fixture, alice) = seeded_for_relevance().await;
+    // dentist -office not moved -("next week" or call) body:work date:2024
+    // "the dentist" or den*
+    let q = "dentist%20-office%20not%20moved%20-(%22next%20week%22%20or%20call)%20body%3Awork%20date%3A2024%20%22the%20dentist%22%20or%20den*";
+    let page: serde_json::Value =
+        get_json(&fixture.state, &format!("/v1/messages?q={q}"), &alice.token).await;
+    assert_eq!(
+        page["search"]["terms"],
+        serde_json::json!([
+            {"text": "dentist", "prefix": false},
+            {"text": "the dentist", "prefix": false},
+            {"text": "den", "prefix": true},
+        ]),
+        "{page}"
+    );
+    assert_eq!(page["search"]["sort"], serde_json::json!("relevance"));
+}
+
 /// Relevance has one direction, best match first: `-relevance` would list the
 /// unranked messages first and then the worst match, which nobody asks for.
 #[tokio::test]