import re import logging import mimetypes import requests from six.moves.html_parser import HTMLParser _logger = logging.getLogger(__name__) # HTML Parsers class HTMLParsed(Exception): def __init__(self, data): self.data = data # TODO: open temp file, close after 60 seconds with thread.timer() # TODO: switch to bs4 with "html.parser" # TODO: Add media_readme.rst # TODO: Add environment variables to config class ImgurHTMLParser(HTMLParser): """ Scrape the actual image url from an imgur landing page. Imgur intentionally obscures this on most reddit links in order to draw more traffic for their advertisements. There are a couple of tags that supply the relevant info: Note: BeautifulSoup or lxml would be faster here but I wanted to skip adding an extra dependency for something this trivial. """ def handle_starttag(self, tag, attr): if tag == 'meta' and attr[0] == ('name', 'twitter:image'): raise HTMLParsed(attr[1][1]) class ImgurAlbumHTMLParser(HTMLParser): """ Scrape the complete list of images from an imgur album. The HTML parser is very limited, so this assumes the following html structure:
Close up Close up
""" def reset(self): super(ImgurAlbumHTMLParser, self).reset() self.primed = False self.hrefs = [] def handle_starttag(self, tag, attr): if tag == 'div' and ('class', 'post-image') in attr: self.primed = True elif self.primed: self.primed = False if tag == 'a' and attr[0][0] == 'href': self.hrefs.append(attr[0][1]) # MIME Parsers class BaseMIMEParser(object): """ BaseMIMEParser can be sub-classed to define custom handlers for determining the MIME type of external urls. """ pattern = re.compile(r'.*$') @staticmethod def get_mimetype(url): """ Guess based on the file extension. Args: url (text): Web url that was linked to by a reddit submission. Returns: modified_url (text): The url (or filename) that will be used when constructing the command to run. content_type (text): The mime-type that will be used when constructing the command to run. If the mime-type is unknown, return None and the program will fallback to using the web browser. """ filename = url.split('?')[0] content_type, _ = mimetypes.guess_type(filename) return url, content_type class GfycatMIMEParser(BaseMIMEParser): """ Gfycat provides a primitive json api to generate image links. URLs can be downloaded as either gif, webm, or mjpg. Webm was selected because it's fast and works with VLC. https://gfycat.com/api https://gfycat.com/UntidyAcidicIberianemeraldlizard --> https://giant.gfycat.com/UntidyAcidicIberianemeraldlizard.webm """ pattern = re.compile(r'https?://(www\.)?gfycat\.com/[^.]+$') @staticmethod def get_mimetype(url): parts = url.split('/') api_url = '/'.join(parts[:-1] + ['cajax', 'get'] + parts[-1:]) resp = requests.get(api_url) image_url = resp.json()['gfyItem']['webmUrl'] return image_url, 'video/webm' class YoutubeMIMEParser(BaseMIMEParser): """ Youtube videos can be streamed with vlc or downloaded with youtube-dl. Assign a custom mime-type so they can be referenced in mailcap. """ pattern = re.compile( r'(?:https?://)?(m\.)?(?:youtu\.be/|(?:www\.)?youtube\.com/watch' r'(?:\.php)?\'?.*v=)([a-zA-Z0-9\-_]+)') @staticmethod def get_mimetype(url): return url, 'video/x-youtube' class GifvMIMEParser(BaseMIMEParser): """ Special case for .gifv, which is a custom video format for imgur serves as html with a special