From f0e4aba9708f21d30151b2c4b49fb949d1c91df5 Mon Sep 17 00:00:00 2001 From: Pawky Languish Date: Wed, 11 Dec 2024 16:40:28 +0000 Subject: Try to workaround youtube bot flagging in alternate youtube module (doesn't quite work? might reduce likelyhood ip gets flagged?) --- README | 3 + README.txt | 3 - URLget.py | 40 +++++++++++ requirements.txt | 2 + youtube.alt.py | 205 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ youtube.py | 2 +- youtube.py.old | 179 ------------------------------------------------ 7 files changed, 251 insertions(+), 183 deletions(-) create mode 100644 README delete mode 100644 README.txt create mode 100644 URLget.py create mode 100755 youtube.alt.py delete mode 100755 youtube.py.old diff --git a/README b/README new file mode 100644 index 0000000..ac4778c --- /dev/null +++ b/README @@ -0,0 +1,3 @@ +code is formatted with "black" +#remember to install the libraries, and you probably should use venvs anyway +pip --require-venv install -r requirements.txt diff --git a/README.txt b/README.txt deleted file mode 100644 index ac4778c..0000000 --- a/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -code is formatted with "black" -#remember to install the libraries, and you probably should use venvs anyway -pip --require-venv install -r requirements.txt diff --git a/URLget.py b/URLget.py new file mode 100644 index 0000000..abe27a1 --- /dev/null +++ b/URLget.py @@ -0,0 +1,40 @@ +class URLgetException(Exception): + pass + + +try: + from curl_cffi import requests + + # from curl_cffi.requests.exceptions import HTTPError + print("using curl_cffi") + + def urlget(url): + # probably want to impersonate "chrome", "safari" or "safari_ios" + # could impersonate some more specific versions too I guess + try: + r = requests.get(url, impersonate="safari_ios") + # print(dir(r)) + # print(r.status_code) + except Exception as e: + raise URLgetException(e) + return r.status_code, r.text + +except ModuleNotFoundError: + # fallback to just dumb user-agent spoofing, it will not help, but at least it won't hurt? + from urllib.request import Request, urlopen + + # from urllib.error import HTTPError + print("using urllib.request") + + def urlget(url): + # update as needed I guess + ua = "Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0" + # req=Request(url) + req = Request(url) + req.add_header("User-Agent", ua) + try: + r = urlopen(req) + except Exception as e: + # except HTTPError as e: + raise URLgetException(e) + return r.status, r.read().decode() diff --git a/requirements.txt b/requirements.txt index 8a70ebc..cb7e46d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ ircstates +#OPTIONAL: only for TLS fingerprint spoofing, used by youtube.alt.py +curl_cffi diff --git a/youtube.alt.py b/youtube.alt.py new file mode 100755 index 0000000..ca478d8 --- /dev/null +++ b/youtube.alt.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +from html.parser import HTMLParser +from URLget import urlget, URLgetException + +# from URLget import URLgetException +# urlget=URLget().urlget +# print(urlget("http://ip.envs.net")) +# print(dir(URLget)) + + +class YouTube: + # crude import, lol + # URLget = URLget().URLget + # def __init__(self): + # self.URLget = URLget().URLget + # print(URLget,URLget.URLget) + + video_type = "" + + def mesg(self, msg, t=None): + self.util.mesg(msg, t) + + def match_urls(self, str): + r = [ + i + for i in str.split() + if "https://youtu.be/" in i + or "https://www.youtube.com/watch?v=" in i + or "https://m.youtube.com/watch?v=" in i + or "https://youtube.com/watch?v=" in i + or "https://www.youtube.com/embed/" in i + or "https://www.youtube-nocookie.com/embed/" in i + or "https://music.youtube.com/watch?v=" in i + or "https://youtube.com/shorts/" in i + or "https://www.youtube.com/shorts/" in i + or "https://www.youtube.com/clip/" in i + or "https://youtube.com/clip/" in i + ] + r = list(dict.fromkeys(r)) + n = 0 + for i in r: + if not i.startswith("http"): + r.pop(n) + n += 1 + + return r + + def is_embed(self, str): + return str.startswith("https://www.youtube.com/embed/") or str.startswith( + "https://www.youtube-nocookie.com/embed/" + ) + + def is_ytmusic(self, str): + return str.startswith("https://music.youtube.com/watch?v=") + + def is_ytshorts(self, str): + return str.startswith("https://youtube.com/shorts/") or str.startswith( + "https://www.youtube.com/shorts/" + ) + + def is_clip(self, str): + return str.startswith("https://youtube.com/clip/") or str.startswith( + "https://www.youtube.com/clip/" + ) + + class parseprop(HTMLParser): + def __init__(self): + # print("yt parse init") + HTMLParser.__init__(self) + self.itemprops_list = ["name", "duration", "uploadDate", "interactionCount"] + self.h = {} + if YouTube.video_type == "clip": + self.itemprops_list += ["description"] + print("it is a clip!") + self.title = False + + def handle_data(self, data): + if self.title != False: + # print("title",data) + self.h.update({"html_title": data}) + self.title = False + + def handle_starttag(self, tag, attrs): + if tag == "title": + self.title = True + if (tag != "meta" and tag != "link") or ( + ( + [i for i in attrs if "itemprop" in i] == [] + and ("name", "title") not in attrs + ) + or (tag == "meta" and ("itemprop", "name") in attrs) + ): + return + # print(self,tag,attrs) + for k, v in attrs: + if k == "itemprop": + if v not in self.itemprops_list: + return + x = [v] + if tag == "link" and v == "name": + x = ["channelName"] + elif k == "content": + if attrs[0][1] == "interactionCount": + v = int(v) + x += [v] + elif k == "name" and v == "title": + x = [v] + else: + return + self.h.update({x[0]: x[1]}) + # print(x[0],"=",x[1]) + + def fmt_dur(dur): + h, m, s = 0, 0, 0 + m = dur[2:].split("M") + s = int(m[1][:-1]) + m = int(m[0]) + if m >= 60: + h = m // 60 + m = round((m / 60 - h) * 60) + return f"{h}h {m}m {s}s" + elif h == 0 and m == 0 and s == 0: + return "LIVE" + elif m == 0 and s != 0: + return f"{s}s" + elif s == 0: + return f"{m}m" + else: + return f"{m}m {s}s" + + def yt(self, url): + irc_string = "[\x0304Youtube\x03] \x0307ERROR:\x0308 got no data from server! \x0315(check your URL for typos!)\x03" + ansi_string = "[\x1b[31mYoutube\x1b[0m] \x1b[33;2mERROR:\x1b[33;1m got no data from server! \x1b[37;2m(check your URL for typos!)\x1b[0m" + # self.util.mesg("dbg hello") + url = url.rstrip("\x01") + self.video_type = ( + "clip" + if self.is_clip(url) + else ( + "shorts" + if self.is_ytshorts(url) + else ( + "music" + if self.is_ytmusic(url) + else "embed" if self.is_embed(url) else "video" + ) + ) + ) + video_type = self.video_type + if video_type == "embed": + videoId = url.split("/")[4] + url = f"https://www.youtube.com/watch?v={videoId}" + elif video_type == "music": + for i in url.split("?")[1].split("&"): + if i[0:2] == "v=": + videoId = i[2:] + url = f"https://www.youtube.com/watch?v={videoId}" + elif video_type == "shorts": + videoId = url.split("?")[0].split("/")[-1] + url = f"https://www.youtube.com/watch?v={videoId}" + p = self.parseprop() + # use premature optimization? it should be SLIGHTLY faster, but can sometimes fail + data = b"" + data = data.decode() # bytes to utf-8 + if ( + data.find('meta itemprop="duration"') == -1 + or data.find('meta itemprop="name"') == -1 + ): + try: + status, data = urlget(url) + if status != 200: + irc_string = ( + f"[\x0304Youtube\x03] \x0307ERROR:\x0308 {status} \x0315\x03" + ) + ansi_string = f"[\x1b[31mYoutube\x1b[0m] \x1b[33;2mERROR:\x1b[33;1m {status} \x1b[37;2m\x1b[0m" + except URLgetException as e: + irc_string = f"[\x0304Youtube\x03] \x0307ERROR:\x0308 {e} \x0315\x03" + ansi_string = f"[\x1b[31mYoutube\x1b[0m] \x1b[33;2mERROR:\x1b[33;1m {e} \x1b[37;2m\x1b[0m" + # print(f"\x1b[31m my data is: {data}\x1b[0m") + p.feed(data) + if p.h == {}: + print(ansi_string) + return irc_string, True + elif p.h == {"html_title": "YouTube"}: + irc_string = ( + "[\x0304Youtube\x03] \x0307ERROR:\x0308 flagged as bot \x0315\x03" + ) + ansi_string = "[\x1b[31mYoutube\x1b[0m] \x1b[33;2mERROR:\x1b[33;1m flagged as bot \x1b[37;2m\x1b[0m" + print(ansi_string) + return irc_string, True + y = p.h + print(y) + y.update(duration=self.fmt_dur(y["duration"])) + irc_string = f"[\x0303Youtube\x03] \x02{y['title']}\x02 ({y['duration']}) uploaded by \x1d{y['channelName']}\x1d on {y['uploadDate']}, {y['interactionCount']:,} views" + ansi_string = f"[\x1b[32mYoutube\x1b[0m] \x1b[1m{y['title']}\x1b[0m ({y['duration']}) uploaded by \x1b[03m{y['channelName']}\x1b[0m on {y['uploadDate']}, {y['interactionCount']:,} views" + print(ansi_string) + return irc_string, False + + +if __name__ == "__main__": + import sys + + YouTube.premature_optimization = False + # YouTube.yt(YouTube, sys.argv[1]) + YouTube().yt(sys.argv[1]) diff --git a/youtube.py b/youtube.py index 1e60546..0d51e16 100755 --- a/youtube.py +++ b/youtube.py @@ -96,7 +96,7 @@ class YouTube: video_id = None try: playlist_id = qs["list"][0] - #ignore the "random mix" and "radio" lists + # ignore the "random mix" and "radio" lists if playlist_id.startswith("RD"): playlist_id = None except KeyError: diff --git a/youtube.py.old b/youtube.py.old deleted file mode 100755 index b09b1d1..0000000 --- a/youtube.py.old +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python3 -from html.parser import HTMLParser -from urllib.request import urlopen -from urllib.error import HTTPError - -class YouTube: - video_type = "" - - def mesg(self, msg, t=None): - self.util.mesg(msg, t) - - def match_urls(self, str): - r = [ - i - for i in str.split() - if "https://youtu.be/" in i - or "https://www.youtube.com/watch?v=" in i - or "https://m.youtube.com/watch?v=" in i - or "https://youtube.com/watch?v=" in i - or "https://www.youtube.com/embed/" in i - or "https://www.youtube-nocookie.com/embed/" in i - or "https://music.youtube.com/watch?v=" in i - or "https://youtube.com/shorts/" in i - or "https://www.youtube.com/shorts/" in i - or "https://www.youtube.com/clip/" in i - or "https://youtube.com/clip/" in i - ] - r = list(dict.fromkeys(r)) - n = 0 - for i in r: - if not i.startswith("http"): - r.pop(n) - n += 1 - - return r - - def is_embed(str): - return str.startswith("https://www.youtube.com/embed/") or str.startswith( - "https://www.youtube-nocookie.com/embed/" - ) - - def is_ytmusic(str): - return str.startswith("https://music.youtube.com/watch?v=") - - def is_ytshorts(str): - return str.startswith("https://youtube.com/shorts/") or str.startswith( - "https://www.youtube.com/shorts/" - ) - - def is_clip(str): - return str.startswith("https://youtube.com/clip/") or str.startswith( - "https://www.youtube.com/clip/" - ) - - class parseprop(HTMLParser): - def __init__(self): - #print("yt parse init") - HTMLParser.__init__(self) - self.itemprops_list = ["name", "duration", "uploadDate", "interactionCount"] - self.h = {} - if YouTube.video_type == "clip": - self.itemprops_list += ["description"] - print("it is a clip!") - - def handle_starttag(self, tag, attrs): - if (tag != "meta" and tag != "link") or ( - ( - [i for i in attrs if "itemprop" in i] == [] - and ("name", "title") not in attrs - ) - or (tag == "meta" and ("itemprop", "name") in attrs) - ): - return - # print(self,tag,attrs) - for k, v in attrs: - if k == "itemprop": - if v not in self.itemprops_list: - return - x = [v] - if tag == "link" and v == "name": - x = ["channelName"] - elif k == "content": - if attrs[0][1] == "interactionCount": - v = int(v) - x += [v] - elif k == "name" and v == "title": - x = [v] - else: - return - self.h.update({x[0]: x[1]}) - # print(x[0],"=",x[1]) - - def fmt_dur(dur): - h, m, s = 0, 0, 0 - m = dur[2:].split("M") - s = int(m[1][:-1]) - m = int(m[0]) - if m >= 60: - h = m // 60 - m = round((m / 60 - h) * 60) - return f"{h}h {m}m {s}s" - elif h == 0 and m == 0 and s == 0: - return "LIVE" - elif m == 0 and s != 0: - return f"{s}s" - elif s == 0: - return f"{m}m" - else: - return f"{m}m {s}s" - - def yt(self, url): - irc_string = "[\x0304Youtube\x03] \x0307ERROR:\x0308 got no data from server! \x0315(check your URL for typos!)\x03" - ansi_string = "[\x1b[31mYoutube\x1b[0m] \x1b[33;2mERROR:\x1b[33;1m got no data from server! \x1b[37;2m(check your URL for typos!)\x1b[0m" - # self.util.mesg("dbg hello") - url = url.rstrip("\x01") - self.video_type = ( - "clip" - if self.is_clip(url) - else ( - "shorts" - if self.is_ytshorts(url) - else ( - "music" - if self.is_ytmusic(url) - else "embed" if self.is_embed(url) else "video" - ) - ) - ) - video_type = self.video_type - if video_type == "embed": - videoId = url.split("/")[4] - url = f"https://www.youtube.com/watch?v={videoId}" - elif video_type == "music": - for i in url.split("?")[1].split("&"): - if i[0:2] == "v=": - videoId = i[2:] - url = f"https://www.youtube.com/watch?v={videoId}" - elif video_type == "shorts": - videoId = url.split("?")[0].split("/")[-1] - url = f"https://www.youtube.com/watch?v={videoId}" - p = self.parseprop() - # use premature optimization? it should be SLIGHTLY faster, but can sometimes fail - data = b"" - if self.premature_optimization: - url_h = urlopen(url) - # appears on approximately line 21 or 22, so we read 24 lines to be safe (23-25 should be license comment) - # I tried to read byte amounts but it's hard to make sure no invalid utf8 bytes happen due to partial reads - for i in range(24): - data += url_h.readline() - url_h.close() - data = data.decode() # bytes to utf-8 - if ( - data.find('meta itemprop="duration"') == -1 - or data.find('meta itemprop="name"') == -1 - ): # acts as both fallback for optimization, and in case optimization's turned off - # just read all of the html - try: data = urlopen(url).read().decode() - except HTTPError as e: - irc_string = f"[\x0304Youtube\x03] \x0307ERROR:\x0308 {e} \x0315\x03" - ansi_string = f"[\x1b[31mYoutube\x1b[0m] \x1b[33;2mERROR:\x1b[33;1m {e} \x1b[37;2m\x1b[0m" - # print(f"\x1b[31m my data is: {data}\x1b[0m") - p.feed(data) - if p.h == {}: - print(ansi_string) - return irc_string, True - y = p.h - print(y) - y.update(duration=self.fmt_dur(y["duration"])) - irc_string = f"[\x0303Youtube\x03] \x02{y['title']}\x02 ({y['duration']}) uploaded by \x1d{y['channelName']}\x1d on {y['uploadDate']}, {y['interactionCount']:,} views" - ansi_string = f"[\x1b[32mYoutube\x1b[0m] \x1b[1m{y['title']}\x1b[0m ({y['duration']}) uploaded by \x1b[03m{y['channelName']}\x1b[0m on {y['uploadDate']}, {y['interactionCount']:,} views" - print(ansi_string) - return irc_string, False - - -if __name__ == "__main__": - import sys - - YouTube.premature_optimization = False - YouTube.yt(YouTube, sys.argv[1]) -- cgit 1.4.1-2-gfad0