Implemented unicode sandwich.

rtv/content.py

@@ -3,25 +3,10 @@ from datetime import datetime
 from contextlib import contextmanager
 
 import praw
-import six
 import requests
 
 from .errors import SubmissionURLError, SubredditNameError
 
-FORCE_ASCII = True
-
-def clean(string):
-
-    encoding = 'ascii' if FORCE_ASCII else 'utf-8'
-
-    if six.PY2:
-        out = string.encode(encoding, 'replace')
-    else:
-        out = string.encode().decode(encoding, 'replace')
-    
-    out = out.replace('\\', '')
-    return out
-
 def split_text(big_text, width):
     return [
         text
@@ -32,14 +17,13 @@ def split_text(big_text, width):
         for text in (textwrap.wrap(line, width=width) or [''])
     ]
 
-
 def strip_subreddit_url(permalink):
     """
     Grab the subreddit from the permalink because submission.subreddit.url
     makes a seperate call to the API.
     """
 
-    subreddit = clean(permalink).split('/')[4]
+    subreddit = permalink.split('/')[4]
     return '/r/{}'.format(subreddit)
 
 
@@ -131,13 +115,12 @@ class BaseContent(object):
             data['body'] = 'More comments'.format(comment.count)
         else:
             data['type'] = 'Comment'
-            data['body'] = clean(comment.body)
+            data['body'] = comment.body
             data['created'] = humanize_timestamp(comment.created_utc)
             data['score'] = '{} pts'.format(comment.score)
-            data['author'] = (clean(comment.author.name) if
+            data['author'] = (comment.author.name if
                               getattr(comment, 'author') else '[deleted]')
-
+            sub_author = (comment.submission.author.name if
-            sub_author = (clean(comment.submission.author.name) if
                           getattr(comment.submission, 'author') else '[deleted]')
             data['is_author'] = (data['author'] == sub_author)
 
@@ -155,17 +138,17 @@ class BaseContent(object):
         data = {}
         data['object'] = sub
         data['type'] = 'Submission'
-        data['title'] = clean(sub.title)
+        data['title'] = sub.title
-        data['text'] = clean(sub.selftext)
+        data['text'] = sub.selftext
         data['created'] = humanize_timestamp(sub.created_utc)
         data['comments'] = '{} comments'.format(sub.num_comments)
         data['score'] = '{} pts'.format(sub.score)
-        data['author'] = (clean(sub.author.name) if getattr(sub, 'author')
+        data['author'] = (sub.author.name if getattr(sub, 'author')
                           else '[deleted]')
-        data['permalink'] = clean(sub.permalink)
+        data['permalink'] = sub.permalink
         data['subreddit'] = strip_subreddit_url(sub.permalink)
-        data['url_full'] = clean(sub.url)
+        data['url_full'] = sub.url
-        data['url'] = ('selfpost' if is_selfpost(sub.url) else clean(sub.url))
+        data['url'] = ('selfpost' if is_selfpost(sub.url) else sub.url)
 
         return data
 
@@ -359,11 +342,11 @@ class SubredditContent(BaseContent):
         # there is is no other way to check things like multireddits that
         # don't have a real corresponding subreddit object.
         content = cls(display_name, submissions, loader)
-        try:
+        #try:
-            content.get(0)
+        content.get(0)
-        except:
+        #except:
-            # TODO: Trap specific errors
+        #    # TODO: Trap specific errors
-            raise SubredditNameError(display_name)
+        #    raise SubredditNameError(display_name)
 
         return content
 

rtv/main.py

@@ -5,7 +5,7 @@ import praw
 from requests.exceptions import ConnectionError, HTTPError
 from praw.errors import InvalidUserPass
 
-from . import content
+from . import utils
 from .errors import SubmissionURLError, SubredditNameError
 from .utils import curses_session, load_config, HELP
 from .subreddit import SubredditPage
@@ -64,7 +64,7 @@ def main():
         if getattr(args, key) is None:
             setattr(args, key, val)
 
-    content.FORCE_ASCII = args.force_ascii
+    utils.FORCE_ASCII = args.force_ascii
 
     if args.subreddit is None:
         args.subreddit = 'front'

rtv/page.py

@@ -1,6 +1,6 @@
 import curses
 
-from .utils import Color
+from .utils import Color, clean
 
 class Navigator(object):
     """
@@ -157,14 +157,15 @@ class BasePage(object):
         self._header_window.bkgd(' ', attr)
 
         sub_name = self.content.name.replace('/r/front', 'Front Page ')
-        self._header_window.addnstr(0, 0, sub_name, n_cols-1)
+        self._header_window.addnstr(0, 0, clean(sub_name), n_cols-1)
 
         if self.reddit.user is not None:
             username = self.reddit.user.name
             s_col = (n_cols - len(username) - 1)
             # Only print the username if it fits in the empty space on the right
             if (s_col - 1) >= len(sub_name):
-                self._header_window.addnstr(0, s_col, username, (n_cols-s_col-1))
+                n = (n_cols - s_col - 1)
+                self._header_window.addnstr(0, s_col, clean(username), n)
 
         self._header_window.refresh()
 

rtv/submission.py

@@ -6,7 +6,7 @@ import six
 
 from .content import SubmissionContent
 from .page import BasePage
-from .utils import LoadScreen, Color, ESCAPE, display_help, open_new_tab
+from .utils import LoadScreen, Color, ESCAPE, display_help, open_new_tab, clean
 
 class SubmissionPage(BasePage):
 
@@ -99,7 +99,8 @@ class SubmissionPage(BasePage):
 
     @staticmethod
     def draw_comment(win, data, inverted=False):
-
+        import logging
+        _logger = logging.getLogger(__name__)
         n_rows, n_cols = win.getmaxyx()
         n_cols -= 1
 
@@ -112,14 +113,14 @@ class SubmissionPage(BasePage):
             text = '{author}'.format(**data)
             attr = curses.A_BOLD
             attr |= (Color.BLUE if not data['is_author'] else Color.GREEN)
-            win.addnstr(row, 1, text, n_cols-1, attr)
+            win.addnstr(row, 1, clean(text), n_cols-1, attr)
             text = ' {score} {created}'.format(**data)
-            win.addnstr(text, n_cols - win.getyx()[1])
+            win.addnstr(clean(text), n_cols - win.getyx()[1])
 
         n_body = len(data['split_body'])
         for row, text in enumerate(data['split_body'], start=offset+1):
             if row in valid_rows:
-                win.addnstr(row, 1, text, n_cols-1)
+                win.addnstr(row, 1, clean(text), n_cols-50)
 
         # Vertical line, unfortunately vline() doesn't support custom color so
         # we have to build it one chr at a time.
@@ -145,9 +146,9 @@ class SubmissionPage(BasePage):
         n_rows, n_cols = win.getmaxyx()
         n_cols -= 1
         text = '{body}'.format(**data)
-        win.addnstr(0, 1, text, n_cols-1)
+        win.addnstr(0, 1, clean(text), n_cols-1)
         text = ' [{count}]'.format(**data)
-        win.addnstr(text, n_cols - win.getyx()[1], curses.A_BOLD)
+        win.addnstr(clean(text), n_cols - win.getyx()[1], curses.A_BOLD)
 
         attr = Color.get_level(data['level'])
         for y in range(n_rows):
@@ -167,26 +168,26 @@ class SubmissionPage(BasePage):
             return
 
         for row, text in enumerate(data['split_title'], start=1):
-            win.addnstr(row, 1, text, n_cols, curses.A_BOLD)
+            win.addnstr(row, 1, clean(text), n_cols, curses.A_BOLD)
 
         row = len(data['split_title']) + 1
         attr = curses.A_BOLD | Color.GREEN
         text = '{author}'.format(**data)
-        win.addnstr(row, 1, text, n_cols, attr)
+        win.addnstr(row, 1, clean(text), n_cols, attr)
         text = ' {created} {subreddit}'.format(**data)
-        win.addnstr(text, n_cols - win.getyx()[1])
+        win.addnstr(clean(text), n_cols - win.getyx()[1])
 
         row = len(data['split_title']) + 2
         attr = curses.A_UNDERLINE | Color.BLUE
         text = '{url}'.format(**data)
-        win.addnstr(row, 1, text, n_cols, attr)
+        win.addnstr(row, 1, clean(text), n_cols, attr)
 
         offset = len(data['split_title']) + 3
         for row, text in enumerate(data['split_text'], start=offset):
-            win.addnstr(row, 1, text, n_cols)
+            win.addnstr(row, 1, clean(text), n_cols)
 
         row = len(data['split_title']) + len(data['split_text']) + 3
         text = '{score} {comments}'.format(**data)
-        win.addnstr(row, 1, text, n_cols, curses.A_BOLD)
+        win.addnstr(row, 1, clean(text), n_cols, curses.A_BOLD)
 
         win.border()

rtv/subreddit.py

@@ -8,7 +8,7 @@ from .page import BasePage
 from .submission import SubmissionPage
 from .content import SubredditContent
 from .utils import (LoadScreen, Color, text_input, display_message,
-                    display_help, open_new_tab)
+                    display_help, open_new_tab, clean)
 
 
 # Used to keep track of browsing history across the current session
@@ -87,7 +87,7 @@ class SubredditPage(BasePage):
         attr = curses.A_BOLD | Color.CYAN
         prompt = 'Enter Subreddit: /r/'
         n_rows, n_cols = self.stdscr.getmaxyx()
-        self.stdscr.addstr(n_rows-1, 0, prompt, attr)
+        self.stdscr.addstr(n_rows-1, 0, clean(prompt), attr)
         self.stdscr.refresh()
         window = self.stdscr.derwin(1, n_cols-len(prompt),n_rows-1, len(prompt))
         window.attrset(attr)
@@ -129,7 +129,7 @@ class SubredditPage(BasePage):
         for row, text in enumerate(data['split_title'], start=offset):
             if row in valid_rows:
                 attr = curses.A_BOLD
-                win.addstr(row, 1, text, attr)
+                win.addstr(row, 1, clean(text), attr)
 
         row = n_title + offset
         if row in valid_rows:
@@ -137,16 +137,16 @@ class SubredditPage(BasePage):
             link_color = Color.MAGENTA if seen else Color.BLUE
             attr = curses.A_UNDERLINE | link_color
             text = '{url}'.format(**data)
-            win.addnstr(row, 1, text, n_cols-1, attr)
+            win.addnstr(row, 1, clean(text), n_cols-1, attr)
 
         row = n_title + offset + 1
         if row in valid_rows:
             text = '{created} {comments} {score}'.format(**data)
-            win.addnstr(row, 1, text, n_cols-1)
+            win.addnstr(row, 1, clean(text), n_cols-1)
 
         row = n_title + offset + 2
         if row in valid_rows:
             text = '{author}'.format(**data)
-            win.addnstr(row, 1, text, n_cols-1, curses.A_BOLD)
+            win.addnstr(row, 1, clean(text), n_cols-1, curses.A_BOLD)
             text = ' {subreddit}'.format(**data)
-            win.addnstr(text, n_cols - win.getyx()[1], Color.YELLOW)
+            win.addnstr(clean(text), n_cols - win.getyx()[1], Color.YELLOW)

rtv/utils.py

@@ -7,10 +7,12 @@ import threading
 from curses import textpad, ascii
 from contextlib import contextmanager
 
+import six
 from six.moves import configparser
 
 from .errors import EscapePressed
 
+FORCE_ASCII = True
 ESCAPE = 27
 
 HELP = """
@@ -30,6 +32,31 @@ Submission Mode
   `RIGHT` or `l`      : Fold the selected comment, or load additional comments
 """
 
+def clean(string):
+    """
+    Required reading!
+        http://nedbatchelder.com/text/unipain.html
+
+    Python 2 input string will be a unicode type (unicode code points). Curses
+    will accept that if all of the points are in the ascii range. However, if
+    any of the code points are not valid ascii curses will throw a
+    UnicodeEncodeError: 'ascii' codec can't encode character, ordinal not in
+    range(128). However, if we encode the unicode to a utf-8 byte string and
+    pass that to curses, curses will render correctly.
+
+    Python 3 input string will be a string type (unicode code points). Curses
+    will accept that in all cases. However, the n character count in addnstr
+    will get screwed up.
+
+    """
+    if six.PY2:
+        string = string.encode('utf-8', 'replace')
+    else:
+        string = string.encode('utf-8', 'replace')
+        pass
+
+    return string
+
 class Color(object):
 
     COLORS = {