Fix bad filenames for Python3.

In Python3, if there are filenames encoded with 8-bit encodings, there
might be an issues with converting them into unicode objects. This is a
workaround on this subject. Python2 is not affected.

Other than that, there was tests added to cover this case, appropriate
Makefile which automate creating venvs for both: Python 2 and 3, and
also there is a check against pep8 rules using flake8.

.gitignore

@@ -0,0 +1,3 @@
+.test
+__pycache__
+adbfsc

Makefile

@@ -0,0 +1,55 @@
+# simple makefile for running tests for the adbfs plugin
+
+all: test_dir py2 py3 flake8
+
+TEST_DIR='.test'
+PY2_VENV=$(TEST_DIR)/py2
+PY3_VENV=$(TEST_DIR)/py3
+FL8_VENV=$(TEST_DIR)/flake8
+TST_EXISTS=$(shell [ -e $(TEST_DIR) ] && echo 1 || echo 0)
+PY2_EXISTS=$(shell [ -e $(PY2_VENV) ] && echo 1 || echo 0)
+PY3_EXISTS=$(shell [ -e $(PY3_VENV) ] && echo 1 || echo 0)
+FL8_EXISTS=$(shell [ -e $(FL8_VENV) ] && echo 1 || echo 0)
+
+py3: test_dir virtualenv3
+	.test/py3/bin/python test_adbfs.py
+
+py2: test_dir virtualenv2
+	.test/py2/bin/python test_adbfs.py
+
+flake8: test_dir virtualenv_flake8
+	.test/flake8/bin/flake8 adbfs test_adbfs.py
+
+ifeq ($(TST_EXISTS), 0)
+test_dir:
+	mkdir -p .test
+else
+test_dir:
+endif
+
+ifeq ($(PY3_EXISTS), 0)
+virtualenv3:
+	virtualenv -p python3 $(PY3_VENV)
+	$(PY3_VENV)/bin/pip install six
+else
+virtualenv3:
+endif
+
+ifeq ($(PY2_EXISTS), 0)
+virtualenv2:
+	virtualenv -p python2 $(PY2_VENV)
+	$(PY2_VENV)/bin/pip install mock
+else
+virtualenv2:
+endif
+
+ifeq ($(FL8_EXISTS), 0)
+virtualenv_flake8:
+	virtualenv -p python2 $(FL8_VENV)
+	$(FL8_VENV)/bin/pip install flake8
+else
+virtualenv_flake8:
+endif
+
+clean:
+	rm -fr $(TEST_DIR) __pycache__ adbfsc

README.rst

@@ -5,13 +5,15 @@ Midnight Commander adbfs external fs plugin
 This is Midnight Commander extfs plugin for browsing Android device through
 ``adb`` interface written in Python.
 
+
 Rquirements
 ===========
 
-* Python 2.7
+* Python 2.7 or 3.x (tested on 3.5.4)
 * ``adb`` installed and in ``$PATH`` or provided via the config file
 * An Android device or emulator preferably rooted
-* Busybox installed and available in the path on the device
+* ``busybox`` (``toolbox``, ``toybox``) installed and available in the path on
+  the device
 
 Make sure, that issuing from command line:
 
@@ -23,6 +25,7 @@ Make sure, that issuing from command line:
 
 it should display files from root directory on the device.
 
+
 Features
 ========
 
@@ -34,12 +37,14 @@ Features
 * Symbolic links in lists are corrected to be relative to the file system
 * Symbolic links also point to the right target, skipping intermediate links
 
+
 Installation
 ============
 
 Copy adbfs into ``~/.local/share/mc/extfs.d/`` directory and make it executable
 if needed.
 
+
 Usage
 =====
 
@@ -47,13 +52,14 @@ To use it, just issue:
 
 .. code:: shell-session
 
-   cd adbfs://
+   $ cd adbfs://
 
 under MC - after some time you should see the files and directories on your
-device. For convenience you can add a bookmark (accessible under CTRL+\) for
+device. For convenience you can add a bookmark (accessible under CTRL+\\) for
 fast access. The time is depended on how many files and directories you have on
 your device and how fast it is :)
 
+
 Configuration
 =============
 
@@ -65,11 +71,11 @@ You can configure behaviour of this plugin using ``.ini`` file located under
 
    [adbfs]
    debug = false
-   skip_dirs = true
    dirs_to_skip = ["acct", "charger", "d", "dev", "proc", "sys"]
    suppress_colors = false
    root =
    adb_command = adb
+   adb_connect =
 
 where:
 
@@ -80,11 +86,53 @@ where:
   everything (slow!)
 * ``suppress_colors`` this option will make ``busybox`` not to display colors,
   helpful, if ``busybox ls`` is configured to display colors by default. Does
-  not affect ``toolbox``.
+  not affect ``toolbox`` or ``toybox``.
 * ``root`` root directory to read. Everything outside of that directory will be
   omitted. That would be the fastest way to access certain location on the
   device. Note, that ``dirs_to_skip`` still apply inside this directory.
-* ``adb_command`` = path to ``adb`` command.
+* ``adb_command`` absolute or relative path to ``adb`` command. `~/` or
+  environment variables are allowed.
+* ``adb_connect`` specifies if connection to specific device needs to be
+  performed before accessing shell. It is useful for *adb over network*
+  feature. Typical value here is a device IP address with optional port, which
+  defaults to 5555.
+
+
+Contribution
+============
+
+There is a ``Makefile`` in the top directory, which is basic helper for running
+the tests. Please use it, and adapt/add tests for provided fixes/functionality.
+
+It requires GNU ``make`` program, and also ``virtualenv`` besides Python in
+version 2 and 3. Using it is simple as running following command:
+
+.. code:: shell-session
+
+   $ make
+
+it will run `py2`, `py3` and `flake8` jobs to check it against the code. For
+running tests against Python 3:
+
+.. code:: shell-session
+
+   $ make py3
+
+or Python 2:
+
+.. code:: shell-session
+
+   $ make py2
+
+or flake 8:
+
+.. code:: shell-session
+
+   $ make flake8
+
+Exit status on any of those means that test fail. Appropriate message/traceback
+will also be visible.
+
 
 Limitations
 ===========
@@ -96,6 +144,7 @@ Limitations
 * The implementation is experimental and it's by now working with mine device;
   while it might not work with yours
 
+
 License
 =======
 

adbfs

@@ -5,28 +5,41 @@ adbfs Virtual filesystem for Midnight Commander
 * Copyright (c) 2016, Roman Dobosz,
 * Published under 3-clause BSD-style license (see LICENSE file)
 """
-
+try:
-import ConfigParser
+    import ConfigParser as configparser
+except ImportError:
+    import configparser
 import argparse
 from datetime import datetime
-import errno
 import json
 import os
 import re
 import subprocess
 import sys
 
-__version__ = 0.10
+__version__ = 0.12
 
 XDG_CONFIG_HOME = os.getenv('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
 
 
-class NoBoxFoundException(OSError):
+def check_output(command_list, stderr=None):
     """
-    Exception raised in case of not found either toolbox or busybox on remote
+    For some reason, in py3 it was decided that command output should be bytes
-    filesystem accessed via adb
+    instead of string. This little function will check if we have string or
+    bytes and in case of bytes it will convert it to string.
     """
-    pass
+    result = subprocess.check_output(command_list, stderr=stderr)
+    if not isinstance(result, str):
+        _result = []
+        for t in result.split(b'\n'):
+            if not t:
+                continue
+            try:
+                _result.append(t.decode('utf-8'))
+            except UnicodeDecodeError:
+                _result.append(t.decode('iso-8859-1'))
+        result = '\n'.join(_result) + '\n'
+    return result
 
 
 class Conf(object):
@@ -44,6 +57,16 @@ class Conf(object):
              'toolbox': {'ls': 'toolbox ls -anl',
                          'rls':  'toolbox ls -Ranl {}',
                          'file_re': r'^(?P<perms>[-bcdlps][-rwxsStT]{9})\s+'
+                                    r'(?P<uid>\d+)\s+'
+                                    r'(?P<gid>\d+)\s+'
+                                    r'(?P<size>\d+)?\s'
+                                    r'(?P<date>\d{4}-\d{2}-\d{2}\s'
+                                    r'\d{2}:\d{2})\s'
+                                    r'(?P<name>.*)'},
+             'toybox': {'ls': 'toybox ls -anl',
+                        'rls':  'toybox ls -Ranl {}',
+                        'file_re': r'^(?P<perms>[-bcdlps][-rwxsStT]{9})\s+'
+                                   r'(?P<links>\d+)\s+'
                                    r'(?P<uid>\d+)\s+'
                                    r'(?P<gid>\d+)\s+'
                                    r'(?P<size>\d+)?\s'
@@ -58,8 +81,10 @@ class Conf(object):
         self.root = None
         self.suppress_colors = False
         self.adb_command = 'adb'
+        self.adb_connect = ''
 
         self.read()
+        self.connect()
         self.get_the_box()
 
     def get_the_box(self):
@@ -67,8 +92,7 @@ class Conf(object):
         cmd = [self.adb_command] + 'shell which'.split()
         try:
             with open(os.devnull, 'w') as fnull:
-                result = subprocess.check_output(cmd + ['busybox'],
+                result = check_output(cmd + ['busybox'], stderr=fnull)
-                                                 stderr=fnull)
             if 'busybox' in result:
                 self.box = Conf.boxes['busybox']
                 if self.suppress_colors:
@@ -82,8 +106,18 @@ class Conf(object):
 
         try:
             with open(os.devnull, 'w') as fnull:
-                result = subprocess.check_output(cmd + ['toolbox'],
+                result = check_output(cmd + ['toybox'], stderr=fnull)
-                                                 stderr=fnull)
+
+            if 'toybox' in result:
+                self.box = Conf.boxes['toybox']
+                Adb.file_re = re.compile(self.box['file_re'])
+                return
+        except subprocess.CalledProcessError:
+            pass
+
+        try:
+            with open(os.devnull, 'w') as fnull:
+                result = check_output(cmd + ['toolbox'], stderr=fnull)
 
             if 'toolbox' in result:
                 self.box = Conf.boxes['toolbox']
@@ -92,8 +126,56 @@ class Conf(object):
         except subprocess.CalledProcessError:
             pass
 
-        raise NoBoxFoundException(errno.ENOENT,
+        sys.stderr.write('There is no toolbox or busybox available.\n')
-                                  'There is no toolbox or busybox available')
+        sys.exit(1)
+
+    def get_attached_devices(self):
+        """Return a list of attached devices"""
+        cmd = [self.adb_command, 'devices']
+        devices = []
+
+        try:
+            with open(os.devnull, 'w') as fnull:
+                result = check_output(cmd, stderr=fnull)
+        except subprocess.CalledProcessError:
+            result = ''
+
+        for line in result.split('\n'):
+            if line.startswith('*'):
+                continue
+            if line.strip() == 'List of devices attached':
+                continue
+            if line.strip() == '':
+                continue
+            identifier, _ = line.split()
+            devices.append(identifier)
+
+        return devices
+
+    def connect(self):
+        """
+        If adb_connect is non empty string, perform connecting to specified
+        device over network using an address (or hostname).
+        """
+        if not self.adb_connect:
+            return
+
+        devices = self.get_attached_devices()
+
+        for device in devices:
+            if self.adb_connect in device:
+                return  # already connected, no need to reconnect
+
+        cmd = [self.adb_command, 'connect', self.adb_connect]
+        with open(os.devnull, 'w') as fnull:
+            result = check_output(cmd, stderr=fnull)
+        if result.split()[0] == 'connected':
+            subprocess.call([self.adb_command, 'wait-for-device'])
+            return
+
+        sys.stderr.write('Unable to connect to `%s\'. Is adb over network '
+                         'enabled on device?\n' % self.adb_connect)
+        sys.exit(2)
 
     def read(self):
         """
@@ -107,18 +189,19 @@ class Conf(object):
         if not os.path.exists(conf_fname):
             return
 
-        cfg = ConfigParser.SafeConfigParser()
+        cfg = configparser.SafeConfigParser()
         cfg_map = {'debug': (cfg.getboolean, 'debug'),
                    'dirs_to_skip': (cfg.get, 'dirs_to_skip'),
                    'suppress_colors': (cfg.get, 'suppress_colors'),
                    'root': (cfg.get, 'root'),
-                   'adb_command': (cfg.get, 'adb')}
+                   'adb_command': (cfg.get, 'adb_command'),
+                   'adb_connect': (cfg.get, 'adb_connect')}
         cfg.read(conf_fname)
 
         for key, (function, attr) in cfg_map.items():
             try:
                 setattr(self, attr, function('adbfs', key))
-            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            except (configparser.NoSectionError, configparser.NoOptionError):
                 pass
 
         if self.dirs_to_skip and isinstance(self.dirs_to_skip, str):
@@ -259,7 +342,7 @@ class Adb(object):
         cmd = [self.conf.adb_command] + 'shell su -c whoami'.split()
         try:
             with open(os.devnull, 'w') as fnull:
-                result = subprocess.check_output(cmd, stderr=fnull)
+                result = check_output(cmd, stderr=fnull)
 
         except subprocess.CalledProcessError:
             return
@@ -313,9 +396,9 @@ class Adb(object):
 
         try:
             if self.conf.debug:
-                print 'executing', ' '.join(command)
+                print('executing', ' '.join(command))
 
-            lines = subprocess.check_output(command)
+            lines = check_output(command)
         except subprocess.CalledProcessError:
             sys.stderr.write('Cannot read directory. Is device connected?\n')
             return 1
@@ -375,9 +458,9 @@ class Adb(object):
 
         try:
             if self.conf.debug:
-                print 'executing', ' '.join(command)
+                print('executing', ' '.join(command))
 
-            lines = subprocess.check_output(command)
+            lines = check_output(command)
         except subprocess.CalledProcessError:
             sys.stderr.write('Cannot read directory. Is device connected?\n')
             return 1
@@ -485,7 +568,7 @@ class Adb(object):
 
         cmd = [self.conf.adb_command, 'shell', 'rm', dst]
         try:
-            err = subprocess.check_output(cmd)
+            err = check_output(cmd)
         except subprocess.CalledProcessError:
             sys.stderr.write('Error executing adb shell')
             return 1
@@ -503,7 +586,7 @@ class Adb(object):
 
         cmd = [self.conf.adb_command, 'shell', 'rm', '-r', dst]
         try:
-            err = subprocess.check_output(cmd)
+            err = check_output(cmd)
         except subprocess.CalledProcessError:
             sys.stderr.write('Error executing adb shell')
             return 1
@@ -524,7 +607,7 @@ class Adb(object):
 
         cmd = [self.conf.adb_command, 'shell', 'mkdir', dst]
         try:
-            err = subprocess.check_output(cmd)
+            err = check_output(cmd)
         except subprocess.CalledProcessError:
             sys.stderr.write('Error executing adb shell')
             return 1

test_adbfs.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+import os
+import six
+import unittest
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+FILE = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'adbfs')
+
+try:
+    from importlib.util import spec_from_loader, module_from_spec
+    from importlib.machinery import SourceFileLoader
+    spec = spec_from_loader("adbfs", SourceFileLoader("adbfs", FILE))
+    adbfs = module_from_spec(spec)
+    spec.loader.exec_module(adbfs)
+except ImportError:
+    # py27
+    import imp
+    adbfs = imp.load_source('adbfs', FILE)
+
+LISTING = '''\
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/Grüß Gott
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/\x80
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/Γεια σας
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/Здравствуйте
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/שָׁלוֹם
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/السَّلامُ عَلَيْكُمْ
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/გამარჯობა
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/こんにちは。
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/안녕하십니까
+'''  # noqa
+
+
+class TestCheckOutput(unittest.TestCase):
+
+    @mock.patch('subprocess.check_output')
+    def test_check_output(self, out):
+        """
+        As for Python2 (and its last version: 2.7), subprocess.check_output
+        always return string like objects, contrary to bytes - no conversion
+        to string is needed.
+        Python3 treats string as unicode objects, but subprocess.check_output
+        returns bytes object, which is equvalend for py2 string… annoying.
+        """
+        if six.PY3:
+            out.return_value = bytes(LISTING, 'utf-8')
+        else:
+            out.return_value = LISTING
+        result = adbfs.check_output(None)
+        self.assertEqual(result, LISTING)
+
+    @mock.patch('subprocess.check_output')
+    def test_check_output_py3_invalid_char(self, out):
+        """
+        Special case for py3. We have bytes with some weird character - like
+        some system write something with codepage, instead of utf8.
+        """
+        if six.PY2:
+            # doesn't affect Python 2
+            return
+        line = (b'-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 '
+                b'/storage/emulated/0/\xe2\n')  # Latin 1 char â
+        out.return_value = bytes(line)
+        result = adbfs.check_output(None)
+        self.assertEqual(result, line.decode('iso-8859-1'))
+
+
+if __name__ == "__main__":
+    unittest.main()