diff --git a/serialx/platforms/serial_linux.py b/serialx/platforms/serial_linux.py
index 8f08d09..b8ac83f 100644
--- a/serialx/platforms/serial_linux.py
+++ b/serialx/platforms/serial_linux.py
@@ -304,6 +304,8 @@ def _after_configure_port(self) -> None:
             except OSError as exc:
                 if exc.errno in (errno.ENOTTY, errno.EOPNOTSUPP):
                     LOGGER.debug("Device does not support setting low latency")
+                elif exc.errno == errno.EPERM:
+                    LOGGER.debug("User does not have permission to set low latency")
                 else:
                     raise
 
diff --git a/tests/test_serial_linux.py b/tests/test_serial_linux.py
index 0a053e8..fd7428c 100644
--- a/tests/test_serial_linux.py
+++ b/tests/test_serial_linux.py
@@ -101,15 +101,23 @@ def test_set_non_posix_baudrate_zero_speed_raises() -> None:
     assert captured == []
 
 
+@pytest.mark.parametrize(
+    "err",
+    [
+        OSError(errno.EOPNOTSUPP, "Not supported"),
+        OSError(errno.ENOTTY, "Inappropriate ioctl for device"),
+        OSError(errno.EPERM, "Operation not permitted"),
+    ],
+)
 @patch("serialx.platforms.serial_linux.TIOCGSERIAL", TIOCGSERIAL)
 @patch("serialx.platforms.serial_linux.TIOCSSERIAL", TIOCSSERIAL)
-def test_tiocgserial_ioctl_not_supported() -> None:
+def test_tiocgserial_ioctl_not_supported(err: OSError) -> None:
     """Test that TIOCGSERIAL ioctl not supported is handled gracefully."""
     ioctl_orig = fcntl.ioctl
 
     def ioctl(fd: int, request: int, arg: Any = 0, mutate_flag: bool = True) -> Any:
         if request in (TIOCGSERIAL, TIOCSSERIAL):
-            raise OSError(errno.EOPNOTSUPP, "Not supported")
+            raise err
 
         return ioctl_orig(fd, request, arg, mutate_flag)