Fix Arabic, Hebrew and Persian text in mpv chat and the chat log

syncplay/constants.py

@@ -27,6 +27,10 @@ def getValueForOS(constantDict):
 MPC_OSD_POSITION = 1  # Right corner, 1 for left
 MPLAYER_OSD_LEVEL = 1
 UI_TIME_FORMAT = "[%X] "
+BIDI_ISOLATE_FORMAT = "\u2068{}\u2069"  # Unicode first-strong isolate: RTL text (e.g. Arabic) keeps its own direction
+BIDI_ISOLATE_LTR_FORMAT = "\u2066{}\u2069"  # Unicode left-to-right isolate
+BIDI_ISOLATE_RTL_FORMAT = "\u2067{}\u2069"  # Unicode right-to-left isolate
+BIDI_ISOLATE_MARKS = "\u2066\u2067\u2068\u2069"  # All isolate marks, removed again when a user copies from the chat log
 CONFIG_NAMES = [".syncplay", "syncplay.ini"]  # Syncplay searches first to last
 DEFAULT_CONFIG_NAME = "syncplay.ini"
 RECENT_CLIENT_THRESHOLD = "1.7.6"  # This and higher considered 'recent' clients (no warnings)

syncplay/players/mpv.py

@@ -14,6 +14,7 @@
 from syncplay.utils import getRuntimeDir, isURL, findResourcePath
 from syncplay.utils import isMacOS, isWindows, isASCII
 from syncplay.utils import playerPathExists
+from syncplay.utils import isolateBidiText
 from syncplay.vendor.python_mpv_jsonipc.python_mpv_jsonipc import MPV
 
 class MpvPlayer(BasePlayer):
@@ -130,6 +131,14 @@ def mpvErrorCheck(self, line):
 
     def displayMessage(self, message, duration=(constants.OSD_DURATION * 1000), OSDType=constants.OSD_NOTIFICATION,
                        mood=constants.MESSAGE_NEUTRAL):
+        # Isolate the text like the chat log does, so RTL text (e.g. Arabic) keeps its own direction.
+        # Chat messages come here too when the mpv chat is off.
+        chatMessage = re.match(constants.MESSAGE_WITH_USERNAME_REGEX, message, re.UNICODE)
+        if chatMessage:
+            username = isolateBidiText(chatMessage.group("username"))
+            message = "<{}>{}".format(username, isolateBidiText(message[chatMessage.end(1):]))  # all text after "<name>"
+        else:
+            message = isolateBidiText(message, constants.BIDI_ISOLATE_LTR_FORMAT)
         if not self._client._config["chatOutputEnabled"]:
             messageString = self._sanitizeText(message.replace("\\n", "<NEWLINE>")).replace("<NEWLINE>", "\\n")
             self._listener.mpvpipe.show_text(messageString, duration, constants.MPLAYER_OSD_LEVEL)
@@ -138,6 +147,8 @@ def displayMessage(self, message, duration=(constants.OSD_DURATION * 1000), OSDT
         self._listener.sendLine(["script-message-to", "syncplayintf", "{}-osd-{}".format(OSDType, mood), messageString])
 
     def displayChatMessage(self, username, message):
+        username = isolateBidiText(username)
+        message = isolateBidiText(message)
         if not self._client._config["chatOutputEnabled"]:
             messageString = "<{}> {}".format(username, message)
             messageString = self._sanitizeText(messageString.replace("\\n", "<NEWLINE>")).replace("<NEWLINE>", "\\n")

syncplay/resources/syncplayintf.lua

@@ -672,6 +672,28 @@ local function is_regional_indicator(codepoint)
     return codepoint ~= nil and codepoint >= 0x1F1E6 and codepoint <= 0x1F1FF
 end
 
+local function is_rtl_letter(codepoint)
+    -- Hebrew, Arabic, Syriac, Thaana, NKo, Samaritan, Mandaic (+ presentation forms),
+    -- historic RTL scripts, Adlam, Mende Kikakui and Arabic mathematical symbols.
+    -- Direction control characters (LRM, RLM, isolates) are not letters and do not count.
+    return (codepoint >= 0x0590 and codepoint <= 0x08FF)
+        or (codepoint >= 0xFB1D and codepoint <= 0xFDFF)
+        or (codepoint >= 0xFE70 and codepoint <= 0xFEFC)
+        or (codepoint >= 0x10800 and codepoint <= 0x10FFF)
+        or (codepoint >= 0x1E800 and codepoint <= 0x1EFFF)
+end
+
+local function contains_rtl_text(str)
+    local pos = 1
+    while pos <= str:len() do
+        if is_rtl_letter(utf8_codepoint_at(str, pos)) then
+            return true
+        end
+        pos = next_utf8(str, pos)
+    end
+    return false
+end
+
 function wordwrapify_string(line, restoreBackslashSubstitute)
 -- Used to ensure characters wrap on a per-character rather than per-word basis
 -- to avoid issues with long filenames, etc.
@@ -680,6 +702,10 @@ function wordwrapify_string(line, restoreBackslashSubstitute)
     if str == nil or str == "" then
         return ""
     end
+    -- The wrap markers contain spaces, which split RTL words into single letters and reverse them.
+    -- Text with RTL letters gets no markers (whole string, so a multi-line OSD block counts as one).
+    -- libass still wraps it at spaces, and inside a word when the word is wider than the screen.
+    local has_rtl_text = contains_rtl_text(str)
     local newstr = ""
     local currentChar = 1
     local nextChar = 0
@@ -701,6 +727,7 @@ function wordwrapify_string(line, restoreBackslashSubstitute)
             and previousCodepoint ~= 0x200C and previousCodepoint ~= 0x200D
             and previousCodepoint ~= 0x2060 and previousCodepoint ~= 0xFEFF
             and not is_wordwrap_continuation(codepoint)
+            and not has_rtl_text
             and not (is_regional_indicator(previousCodepoint)
                 and is_regional_indicator(codepoint)
                 and regionalIndicatorCount % 2 == 1)

syncplay/ui/gui.py

@@ -37,8 +37,10 @@
         QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_UseHighDpiPixmaps, applyDPIScaling)
 if IsPySide6:
     from PySide6.QtCore import QStandardPaths
+    from PySide6.QtGui import QTextDocumentWriter
 elif IsPySide2:
     from PySide2.QtCore import QStandardPaths
+    from PySide2.QtGui import QTextDocumentWriter
 if isMacOS() and IsPySide:
     from Foundation import NSURL
     from Cocoa import NSString, NSUTF8StringEncoding
@@ -249,6 +251,69 @@ def closeDialog(self):
         self.close()
 
 
+class ChatLogBrowser(QtWidgets.QTextBrowser):
+    # Copy and drag text without the invisible isolate marks that showMessage adds for RTL text.
+    # The copied data is the same as a plain QTextBrowser gives, and the selection is never laid out,
+    # because the chat log is one very long paragraph and laying it out takes seconds.
+    copyFormats = None
+    knownFormats = ("text/plain", "text/html", "text/markdown", "application/vnd.oasis.opendocument.text")
+
+    def createMimeDataFromSelection(self):
+        marks = constants.BIDI_ISOLATE_MARKS
+        selection = self.textCursor()
+        if not any(mark in selection.selectedText() for mark in marks):
+            return super().createMimeDataFromSelection()
+        document = QtGui.QTextDocument()
+        document.setDefaultFont(self.document().defaultFont())
+        cursor = QtGui.QTextCursor(document)
+        cursor.beginEditBlock()  # otherwise Qt lays out the text again after every deletion
+        cursor.insertFragment(selection.selection())
+        # Qt positions count UTF-16 units, so an emoji (or any character above U+FFFF) takes two
+        positions = []
+        position = 0
+        for char in document.toPlainText():
+            if char in marks:
+                positions.append(position)
+            position += 2 if ord(char) > 0xFFFF else 1
+        for position in reversed(positions):
+            cursor.setPosition(position)
+            cursor.setPosition(position + 1, QtGui.QTextCursor.KeepAnchor)
+            cursor.removeSelectedText()
+        cursor.endEditBlock()
+        fragment = QtGui.QTextDocumentFragment(document)
+
+        if ChatLogBrowser.copyFormats is None:
+            probe = QtWidgets.QTextEdit()
+            probe.setPlainText("x")
+            probe.selectAll()
+            probeData = probe.createMimeDataFromSelection()
+            probeData.setParent(probe)  # freed together with the probe
+            ChatLogBrowser.copyFormats = list(probeData.formats())
+        if any(mimeType not in ChatLogBrowser.knownFormats for mimeType in ChatLogBrowser.copyFormats):
+            # A format added by a newer Qt: let Qt build the data (same result, but slower)
+            copySource = QtWidgets.QTextEdit()
+            document.setParent(copySource)  # freed together with copySource
+            copySource.setDocument(document)
+            copySource.selectAll()
+            return copySource.createMimeDataFromSelection()
+        # Same formats, in the same order, as Qt's own copy (QTextEditMimeData)
+        mimeData = QtWidgets.QTextEdit().createMimeDataFromSelection()
+        for mimeType in ChatLogBrowser.copyFormats:
+            if mimeType == "text/plain":
+                mimeData.setText(fragment.toPlainText())
+            elif mimeType == "text/html":
+                html = fragment.toHtml(b"utf-8") if IsPySide2 else fragment.toHtml()
+                mimeData.setData(mimeType, html.encode("utf-8"))
+            elif mimeType == "text/markdown":
+                mimeData.setData(mimeType, fragment.toMarkdown().encode("utf-8"))
+            else:
+                buffer = QtCore.QBuffer()
+                QTextDocumentWriter(buffer, b"ODF").write(fragment)
+                buffer.close()
+                mimeData.setData(mimeType, buffer.data())
+        return mimeData
+
+
 class MainWindow(QtWidgets.QMainWindow):
     insertPosition = None
     playlistState = []
@@ -657,8 +722,12 @@ def showMessage(self, message, noTimestamp=False, isMotd=False):
         if messageWithUsername:
             username = messageWithUsername.group("username")
             message = messageWithUsername.group("message")
+        # Isolate lines and user names with RTL text (e.g. Arabic), so they keep their own direction in the LTR log.
+        # Notification lines are UI text, and all UI languages are LTR. Chat and MOTD lines follow their own words.
+        message = utils.isolateBidiText(message, None if username or isMotd else constants.BIDI_ISOLATE_LTR_FORMAT)
         message = message.replace("&", "&amp;").replace('"', "&quot;").replace("<", "&lt;").replace(">", "&gt;")
         if username:
+            username = utils.isolateBidiText(username)
             message = constants.STYLE_USER_MESSAGE.format(constants.STYLE_USERNAME, username, message)
         # When showing a MOTD, escape spaces and use a monospace font to preserve the look of ASCII art.
         if isMotd:
@@ -1121,6 +1190,7 @@ def showErrorMessage(self, message, criticalerror=False):
         message = str(message)
         if criticalerror:
             QtWidgets.QMessageBox.critical(self, "Syncplay", message)
+        message = utils.isolateBidiText(message, constants.BIDI_ISOLATE_LTR_FORMAT)  # like notification lines in showMessage
         message = message.replace("&", "&amp;").replace('"', "&quot;").replace("<", "&lt;").replace(">", "&gt;")
         message = message.replace("&lt;a href=&quot;https://syncplay.pl/trouble&quot;&gt;", '<a href="https://syncplay.pl/trouble">').replace("&lt;/a&gt;", "</a>")
         message = message.replace("&lt;a href=&quot;https://mpv.io/&quot;&gt;", '<a href="https://mpv.io/">').replace("&lt;/a&gt;", "</a>")
@@ -1645,7 +1715,7 @@ def addTopLayout(self, window):
         window.topSplit = self.topSplitter(Qt.Horizontal, self)
 
         window.outputLayout = QtWidgets.QVBoxLayout()
-        window.outputbox = QtWidgets.QTextBrowser()
+        window.outputbox = ChatLogBrowser()
         if isDarkMode: window.outputbox.document().setDefaultStyleSheet(constants.STYLE_DARK_LINKS_COLOR);
         window.outputbox.setReadOnly(True)
         window.outputbox.setTextInteractionFlags(window.outputbox.textInteractionFlags() | Qt.TextSelectableByKeyboard)

syncplay/utils.py

@@ -13,6 +13,7 @@
 import tempfile
 import time
 import traceback
+import unicodedata
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -406,6 +407,39 @@ def versiontotuple(ver):
     return versiontotuple(version) >= versiontotuple(minVersion)
 
 
+def getBidiIsolateFormat(text):
+    # The Unicode isolate for user text (e.g. a chat message): the direction of most words wins.
+    # If both directions have the same number of words, the first letter decides (first-strong isolate).
+    rtlWords = ltrWords = 0
+    for word in text.split():
+        for char in word:
+            direction = unicodedata.bidirectional(char)
+            if direction in ("R", "AL"):
+                rtlWords += 1
+                break
+            if direction == "L":
+                ltrWords += 1
+                break
+    if rtlWords > ltrWords:
+        return constants.BIDI_ISOLATE_RTL_FORMAT
+    if ltrWords > rtlWords:
+        return constants.BIDI_ISOLATE_LTR_FORMAT
+    return constants.BIDI_ISOLATE_FORMAT
+
+
+def isolateBidiText(text, isolateFormat=None):
+    # Put each line with RTL characters (e.g. Arabic) in Unicode isolate marks, so it keeps its own direction.
+    # Other lines do not change. Without a format, a line follows its own words.
+    # Leading spaces stay outside the isolate, so HTML can still collapse them.
+    lines = []
+    for line in text.split("\n"):
+        content = line.lstrip()
+        if any(unicodedata.bidirectional(char) in ("R", "AL") for char in content):
+            line = line[:len(line) - len(content)] + (isolateFormat or getBidiIsolateFormat(content)).format(content)
+        lines.append(line)
+    return "\n".join(lines)
+
+
 def isURL(path):
     if path is None:
         return False