From 17a1c27a2ca7353383c20f7bd71f95f84b7d8abc Mon Sep 17 00:00:00 2001 From: Michael Lazar Date: Thu, 12 Mar 2015 19:02:22 -0700 Subject: [PATCH] Implemented unicode sandwich. --- rtv/content.py | 47 +++++++++++++++-------------------------------- rtv/main.py | 4 ++-- rtv/page.py | 7 ++++--- rtv/submission.py | 29 +++++++++++++++-------------- rtv/subreddit.py | 14 +++++++------- rtv/utils.py | 27 +++++++++++++++++++++++++++ 6 files changed, 70 insertions(+), 58 deletions(-) diff --git a/rtv/content.py b/rtv/content.py index 35e7eea..587057c 100644 --- a/rtv/content.py +++ b/rtv/content.py @@ -3,25 +3,10 @@ from datetime import datetime from contextlib import contextmanager import praw -import six import requests from .errors import SubmissionURLError, SubredditNameError -FORCE_ASCII = True - -def clean(string): - - encoding = 'ascii' if FORCE_ASCII else 'utf-8' - - if six.PY2: - out = string.encode(encoding, 'replace') - else: - out = string.encode().decode(encoding, 'replace') - - out = out.replace('\\', '') - return out - def split_text(big_text, width): return [ text @@ -32,14 +17,13 @@ def split_text(big_text, width): for text in (textwrap.wrap(line, width=width) or ['']) ] - def strip_subreddit_url(permalink): """ Grab the subreddit from the permalink because submission.subreddit.url makes a seperate call to the API. """ - subreddit = clean(permalink).split('/')[4] + subreddit = permalink.split('/')[4] return '/r/{}'.format(subreddit) @@ -131,13 +115,12 @@ class BaseContent(object): data['body'] = 'More comments'.format(comment.count) else: data['type'] = 'Comment' - data['body'] = clean(comment.body) + data['body'] = comment.body data['created'] = humanize_timestamp(comment.created_utc) data['score'] = '{} pts'.format(comment.score) - data['author'] = (clean(comment.author.name) if + data['author'] = (comment.author.name if getattr(comment, 'author') else '[deleted]') - - sub_author = (clean(comment.submission.author.name) if + sub_author = (comment.submission.author.name if getattr(comment.submission, 'author') else '[deleted]') data['is_author'] = (data['author'] == sub_author) @@ -155,17 +138,17 @@ class BaseContent(object): data = {} data['object'] = sub data['type'] = 'Submission' - data['title'] = clean(sub.title) - data['text'] = clean(sub.selftext) + data['title'] = sub.title + data['text'] = sub.selftext data['created'] = humanize_timestamp(sub.created_utc) data['comments'] = '{} comments'.format(sub.num_comments) data['score'] = '{} pts'.format(sub.score) - data['author'] = (clean(sub.author.name) if getattr(sub, 'author') + data['author'] = (sub.author.name if getattr(sub, 'author') else '[deleted]') - data['permalink'] = clean(sub.permalink) + data['permalink'] = sub.permalink data['subreddit'] = strip_subreddit_url(sub.permalink) - data['url_full'] = clean(sub.url) - data['url'] = ('selfpost' if is_selfpost(sub.url) else clean(sub.url)) + data['url_full'] = sub.url + data['url'] = ('selfpost' if is_selfpost(sub.url) else sub.url) return data @@ -359,11 +342,11 @@ class SubredditContent(BaseContent): # there is is no other way to check things like multireddits that # don't have a real corresponding subreddit object. content = cls(display_name, submissions, loader) - try: - content.get(0) - except: - # TODO: Trap specific errors - raise SubredditNameError(display_name) + #try: + content.get(0) + #except: + # # TODO: Trap specific errors + # raise SubredditNameError(display_name) return content diff --git a/rtv/main.py b/rtv/main.py index a48e87b..70d9302 100644 --- a/rtv/main.py +++ b/rtv/main.py @@ -5,7 +5,7 @@ import praw from requests.exceptions import ConnectionError, HTTPError from praw.errors import InvalidUserPass -from . import content +from . import utils from .errors import SubmissionURLError, SubredditNameError from .utils import curses_session, load_config, HELP from .subreddit import SubredditPage @@ -64,7 +64,7 @@ def main(): if getattr(args, key) is None: setattr(args, key, val) - content.FORCE_ASCII = args.force_ascii + utils.FORCE_ASCII = args.force_ascii if args.subreddit is None: args.subreddit = 'front' diff --git a/rtv/page.py b/rtv/page.py index 005ad14..0700396 100644 --- a/rtv/page.py +++ b/rtv/page.py @@ -1,6 +1,6 @@ import curses -from .utils import Color +from .utils import Color, clean class Navigator(object): """ @@ -157,14 +157,15 @@ class BasePage(object): self._header_window.bkgd(' ', attr) sub_name = self.content.name.replace('/r/front', 'Front Page ') - self._header_window.addnstr(0, 0, sub_name, n_cols-1) + self._header_window.addnstr(0, 0, clean(sub_name), n_cols-1) if self.reddit.user is not None: username = self.reddit.user.name s_col = (n_cols - len(username) - 1) # Only print the username if it fits in the empty space on the right if (s_col - 1) >= len(sub_name): - self._header_window.addnstr(0, s_col, username, (n_cols-s_col-1)) + n = (n_cols - s_col - 1) + self._header_window.addnstr(0, s_col, clean(username), n) self._header_window.refresh() diff --git a/rtv/submission.py b/rtv/submission.py index d078577..5598d1c 100644 --- a/rtv/submission.py +++ b/rtv/submission.py @@ -6,7 +6,7 @@ import six from .content import SubmissionContent from .page import BasePage -from .utils import LoadScreen, Color, ESCAPE, display_help, open_new_tab +from .utils import LoadScreen, Color, ESCAPE, display_help, open_new_tab, clean class SubmissionPage(BasePage): @@ -99,7 +99,8 @@ class SubmissionPage(BasePage): @staticmethod def draw_comment(win, data, inverted=False): - + import logging + _logger = logging.getLogger(__name__) n_rows, n_cols = win.getmaxyx() n_cols -= 1 @@ -112,14 +113,14 @@ class SubmissionPage(BasePage): text = '{author}'.format(**data) attr = curses.A_BOLD attr |= (Color.BLUE if not data['is_author'] else Color.GREEN) - win.addnstr(row, 1, text, n_cols-1, attr) + win.addnstr(row, 1, clean(text), n_cols-1, attr) text = ' {score} {created}'.format(**data) - win.addnstr(text, n_cols - win.getyx()[1]) + win.addnstr(clean(text), n_cols - win.getyx()[1]) n_body = len(data['split_body']) for row, text in enumerate(data['split_body'], start=offset+1): if row in valid_rows: - win.addnstr(row, 1, text, n_cols-1) + win.addnstr(row, 1, clean(text), n_cols-50) # Vertical line, unfortunately vline() doesn't support custom color so # we have to build it one chr at a time. @@ -145,9 +146,9 @@ class SubmissionPage(BasePage): n_rows, n_cols = win.getmaxyx() n_cols -= 1 text = '{body}'.format(**data) - win.addnstr(0, 1, text, n_cols-1) + win.addnstr(0, 1, clean(text), n_cols-1) text = ' [{count}]'.format(**data) - win.addnstr(text, n_cols - win.getyx()[1], curses.A_BOLD) + win.addnstr(clean(text), n_cols - win.getyx()[1], curses.A_BOLD) attr = Color.get_level(data['level']) for y in range(n_rows): @@ -167,26 +168,26 @@ class SubmissionPage(BasePage): return for row, text in enumerate(data['split_title'], start=1): - win.addnstr(row, 1, text, n_cols, curses.A_BOLD) + win.addnstr(row, 1, clean(text), n_cols, curses.A_BOLD) row = len(data['split_title']) + 1 attr = curses.A_BOLD | Color.GREEN text = '{author}'.format(**data) - win.addnstr(row, 1, text, n_cols, attr) + win.addnstr(row, 1, clean(text), n_cols, attr) text = ' {created} {subreddit}'.format(**data) - win.addnstr(text, n_cols - win.getyx()[1]) + win.addnstr(clean(text), n_cols - win.getyx()[1]) row = len(data['split_title']) + 2 attr = curses.A_UNDERLINE | Color.BLUE text = '{url}'.format(**data) - win.addnstr(row, 1, text, n_cols, attr) + win.addnstr(row, 1, clean(text), n_cols, attr) offset = len(data['split_title']) + 3 for row, text in enumerate(data['split_text'], start=offset): - win.addnstr(row, 1, text, n_cols) + win.addnstr(row, 1, clean(text), n_cols) row = len(data['split_title']) + len(data['split_text']) + 3 text = '{score} {comments}'.format(**data) - win.addnstr(row, 1, text, n_cols, curses.A_BOLD) + win.addnstr(row, 1, clean(text), n_cols, curses.A_BOLD) - win.border() + win.border() \ No newline at end of file diff --git a/rtv/subreddit.py b/rtv/subreddit.py index bab2fe3..b8d6a4a 100644 --- a/rtv/subreddit.py +++ b/rtv/subreddit.py @@ -8,7 +8,7 @@ from .page import BasePage from .submission import SubmissionPage from .content import SubredditContent from .utils import (LoadScreen, Color, text_input, display_message, - display_help, open_new_tab) + display_help, open_new_tab, clean) # Used to keep track of browsing history across the current session @@ -87,7 +87,7 @@ class SubredditPage(BasePage): attr = curses.A_BOLD | Color.CYAN prompt = 'Enter Subreddit: /r/' n_rows, n_cols = self.stdscr.getmaxyx() - self.stdscr.addstr(n_rows-1, 0, prompt, attr) + self.stdscr.addstr(n_rows-1, 0, clean(prompt), attr) self.stdscr.refresh() window = self.stdscr.derwin(1, n_cols-len(prompt),n_rows-1, len(prompt)) window.attrset(attr) @@ -129,7 +129,7 @@ class SubredditPage(BasePage): for row, text in enumerate(data['split_title'], start=offset): if row in valid_rows: attr = curses.A_BOLD - win.addstr(row, 1, text, attr) + win.addstr(row, 1, clean(text), attr) row = n_title + offset if row in valid_rows: @@ -137,16 +137,16 @@ class SubredditPage(BasePage): link_color = Color.MAGENTA if seen else Color.BLUE attr = curses.A_UNDERLINE | link_color text = '{url}'.format(**data) - win.addnstr(row, 1, text, n_cols-1, attr) + win.addnstr(row, 1, clean(text), n_cols-1, attr) row = n_title + offset + 1 if row in valid_rows: text = '{created} {comments} {score}'.format(**data) - win.addnstr(row, 1, text, n_cols-1) + win.addnstr(row, 1, clean(text), n_cols-1) row = n_title + offset + 2 if row in valid_rows: text = '{author}'.format(**data) - win.addnstr(row, 1, text, n_cols-1, curses.A_BOLD) + win.addnstr(row, 1, clean(text), n_cols-1, curses.A_BOLD) text = ' {subreddit}'.format(**data) - win.addnstr(text, n_cols - win.getyx()[1], Color.YELLOW) + win.addnstr(clean(text), n_cols - win.getyx()[1], Color.YELLOW) diff --git a/rtv/utils.py b/rtv/utils.py index 3ea74f4..1986b46 100644 --- a/rtv/utils.py +++ b/rtv/utils.py @@ -7,10 +7,12 @@ import threading from curses import textpad, ascii from contextlib import contextmanager +import six from six.moves import configparser from .errors import EscapePressed +FORCE_ASCII = True ESCAPE = 27 HELP = """ @@ -30,6 +32,31 @@ Submission Mode `RIGHT` or `l` : Fold the selected comment, or load additional comments """ +def clean(string): + """ + Required reading! + http://nedbatchelder.com/text/unipain.html + + Python 2 input string will be a unicode type (unicode code points). Curses + will accept that if all of the points are in the ascii range. However, if + any of the code points are not valid ascii curses will throw a + UnicodeEncodeError: 'ascii' codec can't encode character, ordinal not in + range(128). However, if we encode the unicode to a utf-8 byte string and + pass that to curses, curses will render correctly. + + Python 3 input string will be a string type (unicode code points). Curses + will accept that in all cases. However, the n character count in addnstr + will get screwed up. + + """ + if six.PY2: + string = string.encode('utf-8', 'replace') + else: + string = string.encode('utf-8', 'replace') + pass + + return string + class Color(object): COLORS = {