From 2ec55d8216359b84b108d6d25a4e08483ad46d35 Mon Sep 17 00:00:00 2001 From: dave Date: Sun, 03 Mar 2019 23:48:53 +0000 Subject: [PATCH] gfycat breaks me and i dont know why --- dave/modules/title.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/dave/modules/title.py b/dave/modules/title.py index 04c3b97..9fcf8c1 100644 --- a/dave/modules/title.py +++ a/dave/modules/title.py @@ -19,6 +19,9 @@ titles = [] for match in matches: + if "gfycat" in match or ".webm" in match: + continue + if not dave.config.redis.exists("site:{}".format(match)): try: res = get(match, timeout=3, @@ -31,16 +34,20 @@ if res.encoding == 'ISO-8859-1' and not 'ISO-8859-1' in \ res.headers.get('Content-Type', ''): res.encoding = res.apparent_encoding + + try: + soup = BeautifulSoup(res.text, "html.parser") + title = soup.title.string - soup = BeautifulSoup(res.text, "html.parser") - title = soup.title.string - - if title is not None: - title = re.sub(r"(\r?\n|\r| )+", - " ", - title.strip()) - title = title[:140] + (title[140:] and '...') - dave.config.redis.setex("site:{}".format(match), 300, title) + if title is not None: + title = re.sub(r"(\r?\n|\r| )+", + " ", + title.strip()) + title = title[:140] + (title[140:] and '...') + dave.config.redis.setex("site:{}".format(match), 300, title) + except BaseException as e: + log.msg("Failed to grab title", e) + return else: title = str(dave.config.redis.get("site:{}".format(match)), 'utf8') -- rgit 0.1.3