diff --git a/src/__init__.py b/src/__init__.py index 6a24f29..77c3753 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,2 +1,2 @@ -__version__ = "2.3" +__version__ = "2.4" diff --git a/src/forumparser.py b/src/forumparser.py index 7d6e2aa..54b7abb 100644 --- a/src/forumparser.py +++ b/src/forumparser.py @@ -13,7 +13,7 @@ ####################################################################################################### # PYTHON IMPORTS -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup, Tag from getopt import getopt, GetoptError from json import dump from re import compile @@ -23,32 +23,37 @@ class FParserGlobals: - MODULE_NAME = __name__.split(".")[-1] - BASEURL = "https://www.opena.tv" + MODULE_NAME: str = __name__.split(".")[-1] + BASEURL: str = "https://www.opena.tv" -atvpglobals = FParserGlobals() +fpglobals = FParserGlobals() class FparserHelper: def getHTMLdata(self, url, timeout=(3.05, 6)): - errMsg, htmlData = "", "" try: response = get(url, timeout=timeout) - response.raise_for_status() - errMsg, htmlData = "", response.text - del response - return errMsg, htmlData + return None, response.text except exceptions.RequestException as errMsg: - errMsg = str(errMsg).replace(atvpglobals.BASEURL.replace("https://", ""), "").replace("'", "").replace("/", "") + errMsg = str(errMsg).replace(fpglobals.BASEURL.replace("http://", ""), "").replace("host=,'", "") print(f"[{MODULE_NAME}] ERROR in module 'getHTMLdata': {errMsg}") - return errMsg, htmlData + return errMsg, None + + def getBinaryData(self, url, timeout=(3.05, 6)): + try: + response = get(url, timeout=timeout) + return None, response.content + except exceptions.RequestException as errMsg: + errMsg = str(errMsg).replace(fpglobals.BASEURL.replace("http://", ""), "").replace("host=,'", "") + print(f"[{MODULE_NAME}] ERROR in module 'getBinaryData': {errMsg}") + return errMsg, None def createThreadUrl(self, threadId, startPage=0): - return f"{atvpglobals.BASEURL}/viewtopic.php?t={threadId}&start={startPage}" if threadId else "" + return f"{fpglobals.BASEURL}/viewtopic.php?t={threadId}&start={startPage}" if threadId else "" def createPostUrl(self, postId): - return f"{atvpglobals.BASEURL}/viewtopic.php?p={postId}#p{postId}" if postId else "" + return f"{fpglobals.BASEURL}/viewtopic.php?p={postId}#p{postId}" if postId else "" def parseLatest(self, startPage=0): def setPostKey(key, value, replacements=[]): @@ -60,27 +65,42 @@ def setPostKey(key, value, replacements=[]): latestDict[key] = text latestList, latestUser = [], [] - url = f"{atvpglobals.BASEURL}/index.php?recent_topics_start={startPage}" + url = f"{fpglobals.BASEURL}/index.php?recent_topics_start={startPage}" errMsg, htmlData = fparser.getHTMLdata(url) - if errMsg: + if errMsg or not htmlData: return errMsg, {} pageList, pageUser = [], [] try: xml = BeautifulSoup(htmlData, features="lxml") # .replace('&', '&') # work around BeautifulSoup bug except Exception as errMsg: - print(f"[{MODULE_NAME}] ERROR in module 'parseLatest': {errMsg}") + errText = f"[{MODULE_NAME}] ERROR in module 'parseLatest': {errMsg}" + print(errText) + return errText, {} threadTitle = "aktuelle Themen" currPost = startPage topicList = xml.find("ul", {"class": "topiclist topics collapsible"}) - for post in topicList.find_all("dl"): + posts = topicList.find_all("dl") if isinstance(topicList, Tag) else [] + for post in posts: + if not isinstance(post, Tag): + continue latestDict = {} topicTitle = post.find("a", class_="topictitle") + if not isinstance(topicTitle, Tag): + continue setPostKey("title", topicTitle) - url = topicTitle.get("href", "") - setPostKey("threadId", url[url.find("?t=") + 3:url.rfind("&sid=")]) + href = str(topicTitle.get("href", "") or "") + tThreadPos = href.find("?t=") + if tThreadPos == -1: + continue + sidPos = href.rfind("&sid=") + threadId = href[tThreadPos + 3:sidPos if sidPos != -1 else len(href)] + setPostKey("threadId", threadId) respShow = post.find("div", class_="responsive-show") + if not isinstance(respShow, Tag): + continue userName = respShow.get_text(strip=True)[19:] # "Letzter Beitrag von Testomat « 23 Okt 2025 16:26" - userName = userName[:userName.find("«")] + separator = userName.find("«") + userName = userName[:separator] if separator != -1 else userName setPostKey("userName", userName) setPostKey("latestLine", respShow, replacements=(["\t", "\n"])) setPostKey("sourceLine", post.find('div', class_="responsive-hide"), replacements=(["\t", "\n"])) @@ -110,44 +130,65 @@ def convert2int(valueStr, fallbackInt=0): print(f"[{MODULE_NAME}] ERROR in module 'parseThread': {errMsg}") return errMsg, {} errMsg, htmlData = fparser.getHTMLdata(threadUrl) - if errMsg: + if errMsg or not htmlData: return errMsg, {} + xml = None try: xml = BeautifulSoup(htmlData, features="lxml") # .replace('&', '&') # work around BeautifulSoup bug except Exception as errMsg: print(f"[{MODULE_NAME}] ERROR in module 'parseThread': {errMsg}") - titleLine = xml.title.string.replace(" - openATV Forum", "") # "LCD4linux - Seite 150" + return f"Failed to parse thread page: {errMsg}", {} + if xml is None: + print(f"[{MODULE_NAME}] ERROR in module 'parseThread': {errMsg}") + return f"Failed to parse thread page: {errMsg}", {} + titleLine = xml.title.string if xml.title else "" # "LCD4linux - Seite 150" + titleLine = titleLine.replace(" - openATV Forum", "") if titleLine else "" foundpos = titleLine.rfind("Seite") threadTitle = titleLine[:foundpos - 3] if foundpos != -1 else titleLine - threadId = xml.find("input", {"name": "t", "type": "hidden"}).get("value") + threadInput = xml.find("input", {"name": "t", "type": "hidden"}) + threadId = threadInput.get("value") if isinstance(threadInput, Tag) else None pagination = xml.find("div", {"class": "pagination"}) #